From fc4fe337035fe5c1a221e2ca3255c8294db9e2ce Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 8 Jun 2022 11:58:16 +0100 Subject: [PATCH 001/560] Add Nick to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 8a88a96bfee5a..ae346ac956637 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -602,3 +602,4 @@ put them back in again.` >}} * Rob Pickerill * Andrey * Eric Wolf <19wolf@gmail.com> + * Nick From a449dd7d1cf9902e0bd5c3f502364a9ac1157790 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 8 Jun 2022 11:58:16 +0100 Subject: [PATCH 002/560] Add Jason Zheng to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index ae346ac956637..25fb30f5ae5a2 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -603,3 +603,4 @@ put them back in again.` >}} * Andrey * Eric Wolf <19wolf@gmail.com> * Nick + * Jason Zheng From 9eb3470c9c6e8ce3387bb845cb637713dbf2c6a5 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 8 Jun 2022 11:58:16 +0100 Subject: [PATCH 003/560] Add Matthew Vernon to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 25fb30f5ae5a2..23f53103628a0 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -604,3 +604,4 @@ put them back in again.` >}} * Eric Wolf <19wolf@gmail.com> * Nick * Jason Zheng + * Matthew Vernon From 26db80c2709351478ce570455b22f64b6161519a Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 8 Jun 2022 11:28:33 +0100 Subject: [PATCH 004/560] ftp: revert to upstream github.com/jlaffaye/ftp from our fork ...now all of our patches have been merged #5810 --- backend/ftp/ftp.go | 2 +- go.mod | 2 +- go.sum | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index c5b39b24c5771..92b84fe9c014c 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -15,7 +15,7 @@ import ( "sync" "time" - "github.com/rclone/ftp" + "github.com/jlaffaye/ftp" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/config" diff --git a/go.mod b/go.mod index 5051c7c64414a..f9364e3219928 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.12.1 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 - github.com/rclone/ftp v1.0.0-210902h github.com/rfjakob/eme v1.1.2 github.com/shirou/gopsutil/v3 v3.22.3 github.com/sirupsen/logrus v1.8.1 @@ -76,6 +75,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af golang.org/x/mobile v0.0.0-20220414153400-ce6a79cf6a13 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect diff --git a/go.sum b/go.sum index 16b4000956b18..ff01c201275dc 100644 --- a/go.sum +++ b/go.sum @@ -320,6 +320,10 @@ github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc= github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= github.com/hanwen/go-fuse/v2 v2.1.0 h1:+32ffteETaLYClUj0a3aHjZ1hOPxxaNEHiZiujuDaek= github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -346,8 +350,9 @@ github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aW github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= -github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf h1:2IYBd5TD/maMqTU2YUzp2tJL4cNaOYQ9EBullN9t9pk= github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= +github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af h1:sh8vAWJ+vr9izhkDAMS3JRGDIjj0tNVwxfwd+2U2xMo= +github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af/go.mod h1:oZaomI+9/et52UBjvNU9LCIqmgt816+7ljXCx0EIPzo= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -494,8 +499,6 @@ github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU= -github.com/rclone/ftp v1.0.0-210902h h1:e9rbDiTdorXRsRtUOdbr6asesJkYZQ9efy1ts5OEBb8= -github.com/rclone/ftp v1.0.0-210902h/go.mod h1:GtHgnfXJAx17bmdVU8kiItiUNFkMbFt+sIg0SwAfyx0= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= From fb587371426addf6684ab14592c294df7d15da59 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 20 Apr 2022 17:56:15 +0100 Subject: [PATCH 005/560] fstests: check for wrapped errors in ListR test --- fstest/fstests/fstests.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 9b3031acdefc3..11cf4dc8bb994 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -1558,12 +1558,12 @@ func Run(t *testing.T, opt *Opt) { } return nil }) - if err != errFound && err != errTooMany { + if !errors.Is(err, errFound) && !errors.Is(err, errTooMany) { assert.NoError(t, err) } - if err != errTooMany { - assert.True(t, file1Found, "file1Root not found") - assert.True(t, file2Found, "file2Root not found") + if !errors.Is(err, errTooMany) { + assert.True(t, file1Found, "file1Root %q not found", file1Root.Path) + assert.True(t, file2Found, "file2Root %q not found", file2Root.Path) } else { t.Logf("Too many files to list - giving up") } From e58d75e4d7aa6969c202a962a06635d83a52d1af Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 21 Apr 2022 09:58:59 +0100 Subject: [PATCH 006/560] drive: make backend config -o config add a combined AllDrives remote This adjusts rclone backend drives -o config drive: So that it also emits a config section called `AllDrives` which uses the combine backend to make a backend which combines all the shared drives into one. It also makes sure that all the shared drive names are valid rclone config names, deduplicating if necessary. Fixes #4506 --- backend/drive/drive.go | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index dc26a39d70790..9aeeec5a04153 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -18,6 +18,7 @@ import ( "mime" "net/http" "path" + "regexp" "sort" "strconv" "strings" @@ -3241,7 +3242,7 @@ This will return a JSON list of objects like this With the -o config parameter it will output the list in a format suitable for adding to a config file to make aliases for all the -drives found. +drives found and a combined drive. [My Drive] type = alias @@ -3251,10 +3252,15 @@ drives found. type = alias remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=: -Adding this to the rclone config file will cause those team drives to -be accessible with the aliases shown. This may require manual editing -of the names. + [AllDrives] + type = combine + remote = "My Drive=My Drive:" "Test Drive=Test Drive:" +Adding this to the rclone config file will cause those team drives to +be accessible with the aliases shown. Any illegal charactes will be +substituted with "_" and duplicate names will have numbers suffixed. +It will also add a remote called AllDrives which shows all the shared +drives combined into one directory tree. `, }, { Name: "untrash", @@ -3370,14 +3376,30 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str if err != nil { return nil, err } + re := regexp.MustCompile(`[^\w_. -]+`) if _, ok := opt["config"]; ok { lines := []string{} - for _, drive := range drives { + upstreams := []string{} + names := make(map[string]struct{}, len(drives)) + for i, drive := range drives { + name := re.ReplaceAllString(drive.Name, "_") + for { + if _, found := names[name]; !found { + break + } + name += fmt.Sprintf("-%d", i) + } + names[name] = struct{}{} lines = append(lines, "") - lines = append(lines, fmt.Sprintf("[%s]", drive.Name)) + lines = append(lines, fmt.Sprintf("[%s]", name)) lines = append(lines, fmt.Sprintf("type = alias")) lines = append(lines, fmt.Sprintf("remote = %s,team_drive=%s,root_folder_id=:", f.name, drive.Id)) + upstreams = append(upstreams, fmt.Sprintf(`"%s=%s:"`, name, name)) } + lines = append(lines, "") + lines = append(lines, fmt.Sprintf("[AllDrives]")) + lines = append(lines, fmt.Sprintf("type = combine")) + lines = append(lines, fmt.Sprintf("upstreams = %s", strings.Join(upstreams, " "))) return lines, nil } return drives, nil From 4b358ff43ba941e34b1c66134b756b4e8a484822 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 20 Apr 2022 17:57:43 +0100 Subject: [PATCH 007/560] combine: backend to combine multiple remotes in one directory tree Fixes #5600 --- README.md | 13 + backend/all/all.go | 1 + backend/combine/combine.go | 941 +++++++++++++++++++++++ backend/combine/combine_internal_test.go | 94 +++ backend/combine/combine_test.go | 79 ++ bin/make_manual.py | 1 + docs/content/_index.md | 14 + docs/content/combine.md | 156 ++++ docs/content/docs.md | 1 + docs/layouts/chrome/navbar.html | 1 + fstest/test_all/config.yaml | 3 + 11 files changed, 1304 insertions(+) create mode 100644 backend/combine/combine.go create mode 100644 backend/combine/combine_internal_test.go create mode 100644 backend/combine/combine_test.go create mode 100644 docs/content/combine.md diff --git a/README.md b/README.md index 0235323b5ada6..206517733e77a 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,19 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and Please see [the full list of all storage providers and their features](https://rclone.org/overview/) +### Virtual storage providers + +These backends adapt or modify other storage providers + + * Alias: rename existing remotes [:page_facing_up:](https://rclone.org/alias/) + * Cache: cache remotes (DEPRECATED) [:page_facing_up:](https://rclone.org/cache/) + * Chunker: split large files [:page_facing_up:](https://rclone.org/chunker/) + * Combine: combine multiple remotes into a directory tree [:page_facing_up:](https://rclone.org/combine/) + * Compress: compress files [:page_facing_up:](https://rclone.org/compress/) + * Crypt: encrypt files [:page_facing_up:](https://rclone.org/crypt/) + * Hasher: hash files [:page_facing_up:](https://rclone.org/hasher/) + * Union: join multiple remotes to work together [:page_facing_up:](https://rclone.org/union/) + ## Features * MD5/SHA-1 hashes checked at all times for file integrity diff --git a/backend/all/all.go b/backend/all/all.go index a93909eca9053..44414b2879dd7 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -9,6 +9,7 @@ import ( _ "github.com/rclone/rclone/backend/box" _ "github.com/rclone/rclone/backend/cache" _ "github.com/rclone/rclone/backend/chunker" + _ "github.com/rclone/rclone/backend/combine" _ "github.com/rclone/rclone/backend/compress" _ "github.com/rclone/rclone/backend/crypt" _ "github.com/rclone/rclone/backend/drive" diff --git a/backend/combine/combine.go b/backend/combine/combine.go new file mode 100644 index 0000000000000..c3c5ff38a792d --- /dev/null +++ b/backend/combine/combine.go @@ -0,0 +1,941 @@ +// Package combine implents a backend to combine multipe remotes in a directory tree +package combine + +/* + Have API to add/remove branches in the combine +*/ + +import ( + "context" + "errors" + "fmt" + "io" + "path" + "strings" + "sync" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/walk" + "golang.org/x/sync/errgroup" +) + +// Register with Fs +func init() { + fsi := &fs.RegInfo{ + Name: "combine", + Description: "Combine several remotes into one", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "upstreams", + Help: `Upstreams for combining + +These should be in the form + + dir=remote:path dir2=remote2:path + +Where before the = is specified the root directory and after is the remote to +put there. + +Embedded spaces can be added using quotes + + "dir=remote:path with space" "dir2=remote2:path with space" + +`, + Required: true, + Default: fs.SpaceSepList(nil), + }}, + } + fs.Register(fsi) +} + +// Options defines the configuration for this backend +type Options struct { + Upstreams fs.SpaceSepList `config:"upstreams"` +} + +// Fs represents a combine of upstreams +type Fs struct { + name string // name of this remote + features *fs.Features // optional features + opt Options // options for this Fs + root string // the path we are working on + hashSet hash.Set // common hashes + when time.Time // directory times + upstreams map[string]*upstream // map of upstreams +} + +// adjustment stores the info to add a prefix to a path or chop characters off +type adjustment struct { + root string + rootSlash string + mountpoint string + mountpointSlash string +} + +// newAdjustment makes a new path adjustment adjusting between mountpoint and root +// +// mountpoint is the point the upstream is mounted and root is the combine root +func newAdjustment(root, mountpoint string) (a adjustment) { + return adjustment{ + root: root, + rootSlash: root + "/", + mountpoint: mountpoint, + mountpointSlash: mountpoint + "/", + } +} + +var errNotUnderRoot = errors.New("file not under root") + +// do makes the adjustment on s, mapping an upstream path into a combine path +func (a *adjustment) do(s string) (string, error) { + absPath := join(a.mountpoint, s) + if a.root == "" { + return absPath, nil + } + if absPath == a.root { + return "", nil + } + if !strings.HasPrefix(absPath, a.rootSlash) { + return "", errNotUnderRoot + } + return absPath[len(a.rootSlash):], nil +} + +// undo makes the adjustment on s, mapping a combine path into an upstream path +func (a *adjustment) undo(s string) (string, error) { + absPath := join(a.root, s) + if absPath == a.mountpoint { + return "", nil + } + if !strings.HasPrefix(absPath, a.mountpointSlash) { + return "", errNotUnderRoot + } + return absPath[len(a.mountpointSlash):], nil +} + +// upstream represents an upstream Fs +type upstream struct { + f fs.Fs + parent *Fs + dir string // directory the upstream is mounted + pathAdjustment adjustment // how to fiddle with the path +} + +// Create an upstream from the directory it is mounted on and the remote +func (f *Fs) newUpstream(ctx context.Context, dir, remote string) (*upstream, error) { + uFs, err := cache.Get(ctx, remote) + if err == fs.ErrorIsFile { + return nil, fmt.Errorf("can't combine files yet, only directories %q: %w", remote, err) + } + if err != nil { + return nil, fmt.Errorf("failed to create upstream %q: %w", remote, err) + } + u := &upstream{ + f: uFs, + parent: f, + dir: dir, + pathAdjustment: newAdjustment(f.root, dir), + } + return u, nil +} + +// NewFs constructs an Fs from the path. +// +// The returned Fs is the actual Fs, referenced by remote in the config +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs.Fs, err error) { + // defer log.Trace(nil, "name=%q, root=%q, m=%v", name, root, m)("f=%+v, err=%v", &outFs, &err) + // Parse config into Options struct + opt := new(Options) + err = configstruct.Set(m, opt) + if err != nil { + return nil, err + } + // Backward compatible to old config + if len(opt.Upstreams) == 0 { + return nil, errors.New("combine can't point to an empty upstream - check the value of the upstreams setting") + } + for _, u := range opt.Upstreams { + if strings.HasPrefix(u, name+":") { + return nil, errors.New("can't point combine remote at itself - check the value of the upstreams setting") + } + } + isDir := false + for strings.HasSuffix(root, "/") { + root = root[:len(root)-1] + isDir = true + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + upstreams: make(map[string]*upstream, len(opt.Upstreams)), + when: time.Now(), + } + + g, gCtx := errgroup.WithContext(ctx) + var mu sync.Mutex + for _, upstream := range opt.Upstreams { + upstream := upstream + g.Go(func() (err error) { + equal := strings.IndexRune(upstream, '=') + if equal < 0 { + return fmt.Errorf("no \"=\" in upstream definition %q", upstream) + } + dir, remote := upstream[:equal], upstream[equal+1:] + if dir == "" { + return fmt.Errorf("empty dir in upstream definition %q", upstream) + } + if remote == "" { + return fmt.Errorf("empty remote in upstream definition %q", upstream) + } + if strings.IndexRune(dir, '/') >= 0 { + return fmt.Errorf("dirs can't contain / (yet): %q", dir) + } + u, err := f.newUpstream(gCtx, dir, remote) + if err != nil { + return err + } + mu.Lock() + f.upstreams[dir] = u + mu.Unlock() + return nil + }) + } + err = g.Wait() + if err != nil { + return nil, err + } + // check features + var features = (&fs.Features{ + CaseInsensitive: true, + DuplicateFiles: false, + ReadMimeType: true, + WriteMimeType: true, + CanHaveEmptyDirectories: true, + BucketBased: true, + SetTier: true, + GetTier: true, + }).Fill(ctx, f) + canMove := true + for _, u := range f.upstreams { + features = features.Mask(ctx, u.f) // Mask all upstream fs + if !operations.CanServerSideMove(u.f) { + canMove = false + } + } + // We can move if all remotes support Move or Copy + if canMove { + features.Move = f.Move + } + + // Enable ListR when upstreams either support ListR or is local + // But not when all upstreams are local + if features.ListR == nil { + for _, u := range f.upstreams { + if u.f.Features().ListR != nil { + features.ListR = f.ListR + } else if !u.f.Features().IsLocal { + features.ListR = nil + break + } + } + } + + // Enable Purge when any upstreams support it + if features.Purge == nil { + for _, u := range f.upstreams { + if u.f.Features().Purge != nil { + features.Purge = f.Purge + break + } + } + } + + // Enable Shutdown when any upstreams support it + if features.Shutdown == nil { + for _, u := range f.upstreams { + if u.f.Features().Shutdown != nil { + features.Shutdown = f.Shutdown + break + } + } + } + + // Enable DirCacheFlush when any upstreams support it + if features.DirCacheFlush == nil { + for _, u := range f.upstreams { + if u.f.Features().DirCacheFlush != nil { + features.DirCacheFlush = f.DirCacheFlush + break + } + } + } + + // Enable ChangeNotify when any upstreams support it + if features.ChangeNotify == nil { + for _, u := range f.upstreams { + if u.f.Features().ChangeNotify != nil { + features.ChangeNotify = f.ChangeNotify + break + } + } + } + + f.features = features + + // Get common intersection of hashes + var hashSet hash.Set + var first = true + for _, u := range f.upstreams { + if first { + hashSet = u.f.Hashes() + first = false + } else { + hashSet = hashSet.Overlap(u.f.Hashes()) + } + } + f.hashSet = hashSet + + // Check to see if the root is actually a file + if f.root != "" && !isDir { + _, err := f.NewObject(ctx, "") + if err != nil { + if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile || err == fs.ErrorIsDir { + // File doesn't exist or is a directory so return old f + return f, nil + } + return nil, err + } + + // Check to see if the root path is actually an existing file + f.root = path.Dir(f.root) + if f.root == "." { + f.root = "" + } + // Adjust path adjustment to remove leaf + for _, u := range f.upstreams { + u.pathAdjustment = newAdjustment(f.root, u.dir) + } + return f, fs.ErrorIsFile + } + return f, nil +} + +// Run a function over all the upstreams in parallel +func (f *Fs) multithread(ctx context.Context, fn func(context.Context, *upstream) error) error { + g, gCtx := errgroup.WithContext(ctx) + for _, u := range f.upstreams { + u := u + g.Go(func() (err error) { + return fn(gCtx, u) + }) + } + return g.Wait() +} + +// join the elements together but unline path.Join return empty string +func join(elem ...string) string { + result := path.Join(elem...) + if result == "." { + return "" + } + if len(result) > 0 && result[0] == '/' { + result = result[1:] + } + return result +} + +// find the upstream for the remote passed in, returning the upstream and the adjusted path +func (f *Fs) findUpstream(remote string) (u *upstream, uRemote string, err error) { + // defer log.Trace(remote, "")("f=%v, uRemote=%q, err=%v", &u, &uRemote, &err) + for _, u := range f.upstreams { + uRemote, err = u.pathAdjustment.undo(remote) + if err == nil { + return u, uRemote, nil + } + } + return nil, "", fmt.Errorf("combine for remote %q: %w", remote, fs.ErrorDirNotFound) +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("combine root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Rmdir removes the root directory of the Fs object +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + // The root always exists + if f.root == "" && dir == "" { + return nil + } + u, uRemote, err := f.findUpstream(dir) + if err != nil { + return err + } + return u.f.Rmdir(ctx, uRemote) +} + +// Hashes returns hash.HashNone to indicate remote hashing is unavailable +func (f *Fs) Hashes() hash.Set { + return f.hashSet +} + +// Mkdir makes the root directory of the Fs object +func (f *Fs) Mkdir(ctx context.Context, dir string) error { + // The root always exists + if f.root == "" && dir == "" { + return nil + } + u, uRemote, err := f.findUpstream(dir) + if err != nil { + return err + } + return u.f.Mkdir(ctx, uRemote) +} + +// purge the upstream or fallback to a slow way +func (u *upstream) purge(ctx context.Context, dir string) (err error) { + if do := u.f.Features().Purge; do != nil { + err = do(ctx, dir) + } else { + err = operations.Purge(ctx, u.f, dir) + } + return err +} + +// Purge all files in the directory +// +// Implement this if you have a way of deleting all the files +// quicker than just running Remove() on the result of List() +// +// Return an error if it doesn't exist +func (f *Fs) Purge(ctx context.Context, dir string) error { + if f.root == "" && dir == "" { + return f.multithread(ctx, func(ctx context.Context, u *upstream) error { + return u.purge(ctx, "") + }) + } + u, uRemote, err := f.findUpstream(dir) + if err != nil { + return err + } + return u.purge(ctx, uRemote) +} + +// Copy src to this remote using server-side copy operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + + dstU, dstRemote, err := f.findUpstream(remote) + if err != nil { + return nil, err + } + + do := dstU.f.Features().Copy + if do == nil { + return nil, fs.ErrorCantCopy + } + + o, err := do(ctx, srcObj.Object, dstRemote) + if err != nil { + return nil, err + } + + return dstU.newObject(o), nil +} + +// Move src to this remote using server-side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + dstU, dstRemote, err := f.findUpstream(remote) + if err != nil { + return nil, err + } + + do := dstU.f.Features().Move + useCopy := false + if do == nil { + do = dstU.f.Features().Copy + if do == nil { + return nil, fs.ErrorCantMove + } + useCopy = true + } + + o, err := do(ctx, srcObj.Object, dstRemote) + if err != nil { + return nil, err + } + + // If did Copy then remove the source object + if useCopy { + err = srcObj.Remove(ctx) + if err != nil { + return nil, err + } + } + + return dstU.newObject(o), nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server-side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) { + // defer log.Trace(f, "src=%v, srcRemote=%q, dstRemote=%q", src, srcRemote, dstRemote)("err=%v", &err) + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(src, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + + dstU, dstURemote, err := f.findUpstream(dstRemote) + if err != nil { + return err + } + + srcU, srcURemote, err := srcFs.findUpstream(srcRemote) + if err != nil { + return err + } + + do := dstU.f.Features().DirMove + if do == nil { + return fs.ErrorCantDirMove + } + + fs.Logf(dstU.f, "srcU.f=%v, srcURemote=%q, dstURemote=%q", srcU.f, srcURemote, dstURemote) + return do(ctx, srcU.f, srcURemote, dstURemote) +} + +// ChangeNotify calls the passed function with a path +// that has had changes. If the implementation +// uses polling, it should adhere to the given interval. +// At least one value will be written to the channel, +// specifying the initial value and updated values might +// follow. A 0 Duration should pause the polling. +// The ChangeNotify implementation must empty the channel +// regularly. When the channel gets closed, the implementation +// should stop polling and release resources. +func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), ch <-chan time.Duration) { + var uChans []chan time.Duration + + for _, u := range f.upstreams { + u := u + if do := u.f.Features().ChangeNotify; do != nil { + ch := make(chan time.Duration) + uChans = append(uChans, ch) + wrappedNotifyFunc := func(path string, entryType fs.EntryType) { + newPath, err := u.pathAdjustment.do(path) + if err != nil { + fs.Logf(f, "ChangeNotify: unable to process %q: %s", path, err) + return + } + fs.Debugf(f, "ChangeNotify: path %q entryType %d", newPath, entryType) + notifyFunc(newPath, entryType) + } + do(ctx, wrappedNotifyFunc, ch) + } + } + + go func() { + for i := range ch { + for _, c := range uChans { + c <- i + } + } + for _, c := range uChans { + close(c) + } + }() +} + +// DirCacheFlush resets the directory cache - used in testing +// as an optional interface +func (f *Fs) DirCacheFlush() { + ctx := context.Background() + _ = f.multithread(ctx, func(ctx context.Context, u *upstream) error { + if do := u.f.Features().DirCacheFlush; do != nil { + do() + } + return nil + }) +} + +func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) { + srcPath := src.Remote() + u, uRemote, err := f.findUpstream(srcPath) + if err != nil { + return nil, err + } + uSrc := operations.NewOverrideRemote(src, uRemote) + var o fs.Object + if stream { + o, err = u.f.Features().PutStream(ctx, in, uSrc, options...) + } else { + o, err = u.f.Put(ctx, in, uSrc, options...) + } + if err != nil { + return nil, err + } + return u.newObject(o), nil +} + +// Put in to the remote path with the modTime given of the given size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + o, err := f.NewObject(ctx, src.Remote()) + switch err { + case nil: + return o, o.Update(ctx, in, src, options...) + case fs.ErrorObjectNotFound: + return f.put(ctx, in, src, false, options...) + default: + return nil, err + } +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + o, err := f.NewObject(ctx, src.Remote()) + switch err { + case nil: + return o, o.Update(ctx, in, src, options...) + case fs.ErrorObjectNotFound: + return f.put(ctx, in, src, true, options...) + default: + return nil, err + } +} + +// About gets quota information from the Fs +func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { + usage := &fs.Usage{ + Total: new(int64), + Used: new(int64), + Trashed: new(int64), + Other: new(int64), + Free: new(int64), + Objects: new(int64), + } + for _, u := range f.upstreams { + doAbout := u.f.Features().About + if doAbout == nil { + continue + } + usg, err := doAbout(ctx) + if errors.Is(err, fs.ErrorDirNotFound) { + continue + } + if err != nil { + return nil, err + } + if usg.Total != nil && usage.Total != nil { + *usage.Total += *usg.Total + } else { + usage.Total = nil + } + if usg.Used != nil && usage.Used != nil { + *usage.Used += *usg.Used + } else { + usage.Used = nil + } + if usg.Trashed != nil && usage.Trashed != nil { + *usage.Trashed += *usg.Trashed + } else { + usage.Trashed = nil + } + if usg.Other != nil && usage.Other != nil { + *usage.Other += *usg.Other + } else { + usage.Other = nil + } + if usg.Free != nil && usage.Free != nil { + *usage.Free += *usg.Free + } else { + usage.Free = nil + } + if usg.Objects != nil && usage.Objects != nil { + *usage.Objects += *usg.Objects + } else { + usage.Objects = nil + } + } + return usage, nil +} + +// Wraps entries for this upstream +func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.DirEntries, error) { + for i, entry := range entries { + switch x := entry.(type) { + case fs.Object: + entries[i] = u.newObject(x) + case fs.Directory: + newDir := fs.NewDirCopy(ctx, x) + newPath, err := u.pathAdjustment.do(newDir.Remote()) + if err != nil { + return nil, err + } + newDir.SetRemote(newPath) + entries[i] = newDir + default: + return nil, fmt.Errorf("unknown entry type %T", entry) + } + } + return entries, nil +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + // defer log.Trace(f, "dir=%q", dir)("entries = %v, err=%v", &entries, &err) + if f.root == "" && dir == "" { + entries = make(fs.DirEntries, 0, len(f.upstreams)) + for combineDir := range f.upstreams { + d := fs.NewDir(combineDir, f.when) + entries = append(entries, d) + } + return entries, nil + } + u, uRemote, err := f.findUpstream(dir) + if err != nil { + return nil, err + } + entries, err = u.f.List(ctx, uRemote) + if err != nil { + return nil, err + } + return u.wrapEntries(ctx, entries) +} + +// ListR lists the objects and directories of the Fs starting +// from dir recursively into out. +// +// dir should be "" to start from the root, and should not +// have trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +// +// It should call callback for each tranche of entries read. +// These need not be returned in any particular order. If +// callback returns an error then the listing will stop +// immediately. +// +// Don't implement this unless you have a more efficient way +// of listing recursively that doing a directory traversal. +func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { + // defer log.Trace(f, "dir=%q, callback=%v", dir, callback)("err=%v", &err) + if f.root == "" && dir == "" { + rootEntries, err := f.List(ctx, "") + if err != nil { + return err + } + err = callback(rootEntries) + if err != nil { + return err + } + var mu sync.Mutex + syncCallback := func(entries fs.DirEntries) error { + mu.Lock() + defer mu.Unlock() + return callback(entries) + } + err = f.multithread(ctx, func(ctx context.Context, u *upstream) error { + return f.ListR(ctx, u.dir, syncCallback) + }) + if err != nil { + return err + } + return nil + } + u, uRemote, err := f.findUpstream(dir) + if err != nil { + return err + } + wrapCallback := func(entries fs.DirEntries) error { + entries, err := u.wrapEntries(ctx, entries) + if err != nil { + return err + } + return callback(entries) + } + if do := u.f.Features().ListR; do != nil { + err = do(ctx, uRemote, wrapCallback) + } else { + err = walk.ListR(ctx, u.f, uRemote, true, -1, walk.ListAll, wrapCallback) + } + if err == fs.ErrorDirNotFound { + err = nil + } + return err +} + +// NewObject creates a new remote combine file object +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + u, uRemote, err := f.findUpstream(remote) + if err != nil { + return nil, err + } + if uRemote == "" || strings.HasSuffix(uRemote, "/") { + return nil, fs.ErrorIsDir + } + o, err := u.f.NewObject(ctx, uRemote) + if err != nil { + return nil, err + } + return u.newObject(o), nil +} + +// Precision is the greatest Precision of all upstreams +func (f *Fs) Precision() time.Duration { + var greatestPrecision time.Duration + for _, u := range f.upstreams { + uPrecision := u.f.Precision() + if uPrecision > greatestPrecision { + greatestPrecision = uPrecision + } + } + return greatestPrecision +} + +// Shutdown the backend, closing any background tasks and any +// cached connections. +func (f *Fs) Shutdown(ctx context.Context) error { + return f.multithread(ctx, func(ctx context.Context, u *upstream) error { + if do := u.f.Features().Shutdown; do != nil { + return do(ctx) + } + return nil + }) +} + +// Object describes a wrapped Object +// +// This is a wrapped Object which knows its path prefix +type Object struct { + fs.Object + u *upstream +} + +func (u *upstream) newObject(o fs.Object) *Object { + return &Object{ + Object: o, + u: u, + } +} + +// Fs returns read only access to the Fs that this object is part of +func (o *Object) Fs() fs.Info { + return o.u.parent +} + +// String returns the remote path +func (o *Object) String() string { + return o.Remote() +} + +// Remote returns the remote path +func (o *Object) Remote() string { + newPath, err := o.u.pathAdjustment.do(o.Object.String()) + if err != nil { + fs.Errorf(o, "Bad object: %v", err) + return err.Error() + } + return newPath +} + +// MimeType returns the content type of the Object if known +func (o *Object) MimeType(ctx context.Context) (mimeType string) { + if do, ok := o.Object.(fs.MimeTyper); ok { + mimeType = do.MimeType(ctx) + } + return mimeType +} + +// UnWrap returns the Object that this Object is wrapping or +// nil if it isn't wrapping anything +func (o *Object) UnWrap() fs.Object { + return o.Object +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.PutStreamer = (*Fs)(nil) + _ fs.Copier = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.ChangeNotifier = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) + _ fs.ListRer = (*Fs)(nil) + _ fs.Shutdowner = (*Fs)(nil) +) diff --git a/backend/combine/combine_internal_test.go b/backend/combine/combine_internal_test.go new file mode 100644 index 0000000000000..5c889bb8c551b --- /dev/null +++ b/backend/combine/combine_internal_test.go @@ -0,0 +1,94 @@ +package combine + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAdjustmentDo(t *testing.T) { + for _, test := range []struct { + root string + mountpoint string + in string + want string + wantErr error + }{ + { + root: "", + mountpoint: "mountpoint", + in: "path/to/file.txt", + want: "mountpoint/path/to/file.txt", + }, + { + root: "mountpoint", + mountpoint: "mountpoint", + in: "path/to/file.txt", + want: "path/to/file.txt", + }, + { + root: "mountpoint/path", + mountpoint: "mountpoint", + in: "path/to/file.txt", + want: "to/file.txt", + }, + { + root: "mountpoint/path", + mountpoint: "mountpoint", + in: "wrongpath/to/file.txt", + want: "", + wantErr: errNotUnderRoot, + }, + } { + what := fmt.Sprintf("%+v", test) + a := newAdjustment(test.root, test.mountpoint) + got, gotErr := a.do(test.in) + assert.Equal(t, test.wantErr, gotErr) + assert.Equal(t, test.want, got, what) + } + +} + +func TestAdjustmentUndo(t *testing.T) { + for _, test := range []struct { + root string + mountpoint string + in string + want string + wantErr error + }{ + { + root: "", + mountpoint: "mountpoint", + in: "mountpoint/path/to/file.txt", + want: "path/to/file.txt", + }, + { + root: "mountpoint", + mountpoint: "mountpoint", + in: "path/to/file.txt", + want: "path/to/file.txt", + }, + { + root: "mountpoint/path", + mountpoint: "mountpoint", + in: "to/file.txt", + want: "path/to/file.txt", + }, + { + root: "wrongmountpoint/path", + mountpoint: "mountpoint", + in: "to/file.txt", + want: "", + wantErr: errNotUnderRoot, + }, + } { + what := fmt.Sprintf("%+v", test) + a := newAdjustment(test.root, test.mountpoint) + got, gotErr := a.undo(test.in) + assert.Equal(t, test.wantErr, gotErr) + assert.Equal(t, test.want, got, what) + } + +} diff --git a/backend/combine/combine_test.go b/backend/combine/combine_test.go new file mode 100644 index 0000000000000..c6274c562097a --- /dev/null +++ b/backend/combine/combine_test.go @@ -0,0 +1,79 @@ +// Test Combine filesystem interface +package combine_test + +import ( + "testing" + + _ "github.com/rclone/rclone/backend/local" + _ "github.com/rclone/rclone/backend/memory" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + if *fstest.RemoteName == "" { + t.Skip("Skipping as -remote not set") + } + fstests.Run(t, &fstests.Opt{ + RemoteName: *fstest.RemoteName, + UnimplementableFsMethods: []string{"OpenWriterAt", "DuplicateFiles"}, + UnimplementableObjectMethods: []string{"MimeType"}, + }) +} + +func TestLocal(t *testing.T) { + if *fstest.RemoteName != "" { + t.Skip("Skipping as -remote set") + } + dirs := MakeTestDirs(t, 3) + upstreams := "dir1=" + dirs[0] + " dir2=" + dirs[1] + " dir3=" + dirs[2] + name := "TestCombineLocal" + fstests.Run(t, &fstests.Opt{ + RemoteName: name + ":dir1", + ExtraConfig: []fstests.ExtraConfigItem{ + {Name: name, Key: "type", Value: "combine"}, + {Name: name, Key: "upstreams", Value: upstreams}, + }, + }) +} + +func TestMemory(t *testing.T) { + if *fstest.RemoteName != "" { + t.Skip("Skipping as -remote set") + } + upstreams := "dir1=:memory:dir1 dir2=:memory:dir2 dir3=:memory:dir3" + name := "TestCombineMemory" + fstests.Run(t, &fstests.Opt{ + RemoteName: name + ":dir1", + ExtraConfig: []fstests.ExtraConfigItem{ + {Name: name, Key: "type", Value: "combine"}, + {Name: name, Key: "upstreams", Value: upstreams}, + }, + }) +} + +func TestMixed(t *testing.T) { + if *fstest.RemoteName != "" { + t.Skip("Skipping as -remote set") + } + dirs := MakeTestDirs(t, 2) + upstreams := "dir1=" + dirs[0] + " dir2=" + dirs[1] + " dir3=:memory:dir3" + name := "TestCombineMixed" + fstests.Run(t, &fstests.Opt{ + RemoteName: name + ":dir1", + ExtraConfig: []fstests.ExtraConfigItem{ + {Name: name, Key: "type", Value: "combine"}, + {Name: name, Key: "upstreams", Value: upstreams}, + }, + }) +} + +// MakeTestDirs makes directories in /tmp for testing +func MakeTestDirs(t *testing.T, n int) (dirs []string) { + for i := 1; i <= n; i++ { + dir := t.TempDir() + dirs = append(dirs, dir) + } + return dirs +} diff --git a/bin/make_manual.py b/bin/make_manual.py index 0afa5f2b577c5..335f4c8148111 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -38,6 +38,7 @@ "sharefile.md", "crypt.md", "compress.md", + "combine.md", "dropbox.md", "filefabric.md", "ftp.md", diff --git a/docs/content/_index.md b/docs/content/_index.md index 653ce53dd3d8a..c7a213bfa3970 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -170,6 +170,20 @@ WebDAV or S3, that work out of the box.) {{< provider name="The local filesystem" home="/local/" config="/local/" end="true">}} {{< /provider_list >}} +## Virtual providers + +These backends adapt or modify other storage providers: + +{{< provider name="Alias: rename existing remotes" home="/alias/" config="/alias/" >}} +{{< provider name="Cache: cache remotes (DEPRECATED)" home="/cache/" config="/cache/" >}} +{{< provider name="Chunker: split large files" home="/chunker/" config="/chunker/" >}} +{{< provider name="Combine: combine multiple remotes into a directory tree" home="/combine/" config="/combine/" >}} +{{< provider name="Compress: compress files" home="/compress/" config="/compress/" >}} +{{< provider name="Crypt: encrypt files" home="/crypt/" config="/crypt/" >}} +{{< provider name="Hasher: hash files" home="/hasher/" config="/hasher/" >}} +{{< provider name="Union: join multiple remotes to work together" home="/union/" config="/union/" >}} + + ## Links * {{< icon "fa fa-home" >}} [Home page](https://rclone.org/) diff --git a/docs/content/combine.md b/docs/content/combine.md new file mode 100644 index 0000000000000..98e174a04de3e --- /dev/null +++ b/docs/content/combine.md @@ -0,0 +1,156 @@ +--- +title: "Combine" +description: "Combine several remotes into one" +--- + +# {{< icon "fa fa-folder-plus" >}} Combine + +The `combine` backend joins remotes together into a single directory +tree. + +For example you might have a remote for images on one provider: + +``` +$ rclone tree s3:imagesbucket +/ +├── image1.jpg +└── image2.jpg +``` + +And a remote for files on another: + +``` +$ rclone tree drive:important/files +/ +├── file1.txt +└── file2.txt +``` + +The `combine` backend can join these together into a synthetic +directory structure like this: + +``` +$ rclone tree combined: +/ +├── files +│ ├── file1.txt +│ └── file2.txt +└── images + ├── image1.jpg + └── image2.jpg +``` + +You'd do this by specifying an `upstreams` parameter in the config +like this + + upstreams = images=s3:imagesbucket files=drive:important/files + +During the initial setup with `rclone config` you will specify the +upstreams remotes as a space separated list. The upstream remotes can +either be a local paths or other remotes. + +## Configuration + +Here is an example of how to make a combine called `remote` for the +example above. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +... +XX / Combine several remotes into one + \ (combine) +... +Storage> combine +Option upstreams. +Upstreams for combining +These should be in the form + dir=remote:path dir2=remote2:path +Where before the = is specified the root directory and after is the remote to +put there. +Embedded spaces can be added using quotes + "dir=remote:path with space" "dir2=remote2:path with space" +Enter a fs.SpaceSepList value. +upstreams> images=s3:imagesbucket files=drive:important/files +-------------------- +[remote] +type = combine +upstreams = images=s3:imagesbucket files=drive:important/files +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +### Configuring for Google Drive Shared Drives + +Rclone has a convenience feature for making a combine backend for all +the shared drives you have access to. + +Assuming your main (non shared drive) Google drive remote is called +`drive:` you would run + + rclone backend -o config drives drive: + +This would produce something like this: + + [My Drive] + type = alias + remote = drive,team_drive=0ABCDEF-01234567890,root_folder_id=: + + [Test Drive] + type = alias + remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=: + + [AllDrives] + type = combine + remote = "My Drive=My Drive:" "Test Drive=Test Drive:" + +If you then add that config to your config file (find it with `rclone +config file`) then you can access all the shared drives in one place +with the `AllDrives:` remote. + +See [the Google Drive docs](/drive/#drives) for full info. + +{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/combine/combine.go then run make backenddocs" >}} +### Standard options + +Here are the standard options specific to combine (Combine several remotes into one). + +#### --combine-upstreams + +Upstreams for combining + +These should be in the form + + dir=remote:path dir2=remote2:path + +Where before the = is specified the root directory and after is the remote to +put there. + +Embedded spaces can be added using quotes + + "dir=remote:path with space" "dir2=remote2:path with space" + + + +Properties: + +- Config: upstreams +- Env Var: RCLONE_COMBINE_UPSTREAMS +- Type: SpaceSepList +- Default: + +{{< rem autogenerated options stop >}} diff --git a/docs/content/docs.md b/docs/content/docs.md index ac714c0f7fd4d..9be5038b8f42f 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -37,6 +37,7 @@ See the following for detailed instructions for * [Chunker](/chunker/) - transparently splits large files for other remotes * [Citrix ShareFile](/sharefile/) * [Compress](/compress/) + * [Combine](/combine/) * [Crypt](/crypt/) - to encrypt other remotes * [DigitalOcean Spaces](/s3/#digitalocean-spaces) * [Digi Storage](/koofr/#digi-storage) diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 552b976cac986..32c05d3e0c346 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -60,6 +60,7 @@ Box Chunker (splits large files) Compress (transparent gzip compression) + Combine (remotes into a directory tree) Citrix ShareFile Crypt (encrypts the others) Digi Storage diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 1b6bb9e1dc7bf..f0abff00739e9 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -91,6 +91,9 @@ backends: fastlist: true maxfile: 1k ## end chunker + - backend: "combine" + remote: "TestCombine:dir1" + fastlist: false ## begin compress - backend: "compress" remote: "TestCompress:" From 20aaeba547d802199f179ca7f704efef2a665450 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 8 Jun 2022 16:29:35 +0200 Subject: [PATCH 008/560] docs: clarify backend support for setting modtime only (#5638) --- docs/content/overview.md | 130 ++++++++++++++++++++++++--------------- 1 file changed, 81 insertions(+), 49 deletions(-) diff --git a/docs/content/overview.md b/docs/content/overview.md index 73974b65b2870..b652ac7558a7b 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -14,48 +14,48 @@ show through. Here is an overview of the major features of each cloud storage system. -| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | -| ---------------------------- |:-----------:|:-------:|:----------------:|:---------------:|:---------:| -| 1Fichier | Whirlpool | No | No | Yes | R | -| Akamai Netstorage | MD5, SHA256 | Yes | No | No | R | -| Amazon Drive | MD5 | No | Yes | No | R | -| Amazon S3 (or S3 compatible) | MD5 | Yes | No | No | R/W | -| Backblaze B2 | SHA1 | Yes | No | No | R/W | -| Box | SHA1 | Yes | Yes | No | - | -| Citrix ShareFile | MD5 | Yes | Yes | No | - | -| Dropbox | DBHASH ¹ | Yes | Yes | No | - | -| Enterprise File Fabric | - | Yes | Yes | No | R/W | -| FTP | - | No | No | No | - | -| Google Cloud Storage | MD5 | Yes | No | No | R/W | -| Google Drive | MD5 | Yes | No | Yes | R/W | -| Google Photos | - | No | No | Yes | R | -| HDFS | - | Yes | No | No | - | -| HTTP | - | No | No | No | R | -| Hubic | MD5 | Yes | No | No | R/W | -| Internet Archive | MD5, SHA1, CRC32 | Yes | No | No | - | -| Jottacloud | MD5 | Yes | Yes | No | R | -| Koofr | MD5 | No | Yes | No | - | -| Mail.ru Cloud | Mailru ⁶ | Yes | Yes | No | - | -| Mega | - | No | No | Yes | - | -| Memory | MD5 | Yes | No | No | - | -| Microsoft Azure Blob Storage | MD5 | Yes | No | No | R/W | -| Microsoft OneDrive | SHA1 ⁵ | Yes | Yes | No | R | -| OpenDrive | MD5 | Yes | Yes | Partial ⁸ | - | -| OpenStack Swift | MD5 | Yes | No | No | R/W | -| pCloud | MD5, SHA1 ⁷ | Yes | No | No | W | -| premiumize.me | - | No | Yes | No | R | -| put.io | CRC-32 | Yes | No | Yes | R | -| QingStor | MD5 | No | No | No | R/W | -| Seafile | - | No | No | No | - | -| SFTP | MD5, SHA1 ² | Yes | Depends | No | - | -| Sia | - | No | No | No | - | -| SugarSync | - | No | No | No | - | -| Storj | - | Yes | No | No | - | -| Uptobox | - | No | No | Yes | - | -| WebDAV | MD5, SHA1 ³ | Yes ⁴ | Depends | No | - | -| Yandex Disk | MD5 | Yes | No | No | R | -| Zoho WorkDrive | - | No | No | No | - | -| The local filesystem | All | Yes | Depends | No | - | +| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | +| ---------------------------- |:----------------:|:-------:|:----------------:|:---------------:|:---------:| +| 1Fichier | Whirlpool | - | No | Yes | R | +| Akamai Netstorage | MD5, SHA256 | R/W | No | No | R | +| Amazon Drive | MD5 | - | Yes | No | R | +| Amazon S3 (or S3 compatible) | MD5 | R/W | No | No | R/W | +| Backblaze B2 | SHA1 | R/W | No | No | R/W | +| Box | SHA1 | R/W | Yes | No | - | +| Citrix ShareFile | MD5 | R/W | Yes | No | - | +| Dropbox | DBHASH ¹ | R | Yes | No | - | +| Enterprise File Fabric | - | R/W | Yes | No | R/W | +| FTP | - | R/W ¹⁰ | No | No | - | +| Google Cloud Storage | MD5 | R/W | No | No | R/W | +| Google Drive | MD5 | R/W | No | Yes | R/W | +| Google Photos | - | - | No | Yes | R | +| HDFS | - | R/W | No | No | - | +| HTTP | - | R | No | No | R | +| Hubic | MD5 | R/W | No | No | R/W | +| Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | +| Jottacloud | MD5 | R/W | Yes | No | R | +| Koofr | MD5 | - | Yes | No | - | +| Mail.ru Cloud | Mailru ⁶ | R/W | Yes | No | - | +| Mega | - | - | No | Yes | - | +| Memory | MD5 | R/W | No | No | - | +| Microsoft Azure Blob Storage | MD5 | R/W | No | No | R/W | +| Microsoft OneDrive | SHA1 ⁵ | R/W | Yes | No | R | +| OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | +| OpenStack Swift | MD5 | R/W | No | No | R/W | +| pCloud | MD5, SHA1 ⁷ | R | No | No | W | +| premiumize.me | - | - | Yes | No | R | +| put.io | CRC-32 | R/W | No | Yes | R | +| QingStor | MD5 | - ⁹ | No | No | R/W | +| Seafile | - | - | No | No | - | +| SFTP | MD5, SHA1 ² | R/W | Depends | No | - | +| Sia | - | - | No | No | - | +| SugarSync | - | - | No | No | - | +| Storj | - | R | No | No | - | +| Uptobox | - | - | No | Yes | - | +| WebDAV | MD5, SHA1 ³ | R ⁴ | Depends | No | - | +| Yandex Disk | MD5 | R/W | No | No | R | +| Zoho WorkDrive | - | - | No | No | - | +| The local filesystem | All | R/W | Depends | No | - | ### Notes @@ -84,6 +84,15 @@ storage platform has been determined to allow duplicate files, and it is possible to create them with `rclone`. It may be that this is a mistake or an unsupported feature. +⁹ QingStor does not support SetModTime for objects bigger than 5 GiB. + +¹⁰ FTP supports modtimes for the major FTP servers, and also others +if they advertised required protocol extensions. See [this](/ftp/#modified-time) +for more details. + +¹¹ Internet Archive requires option `wait_archive` to be set to a non-zero value +for full modtime support. + ### Hash ### The cloud storage system supports various hash types of the objects. @@ -96,13 +105,36 @@ systems they must support a common hash type. ### ModTime ### -The cloud storage system supports setting modification times on -objects. If it does then this enables a using the modification times -as part of the sync. If not then only the size will be checked by -default, though the MD5SUM can be checked with the `--checksum` flag. - -All cloud storage systems support some kind of date on the object and -these will be set when transferring from the cloud storage system. +Allmost all cloud storage systems store some sort of timestamp +on objects, but several of them not something that is appropriate +to use for syncing. E.g. some backends will only write a timestamp +that represent the time of the upload. To be relevant for syncing +it should be able to store the modification time of the source +object. If this is not the case, rclone will only check the file +size by default, though can be configured to check the file hash +(with the `--checksum` flag). Ideally it should also be possible to +change the timestamp of an existing file without having to re-upload it. + +Storage systems with a `-` in the ModTime column, means the +modification read on objects is not the modification time of the +file when uploaded. It is most likely the time the file was uploaded, +or possibly something else (like the time the picture was taken in +Google Photos). + +Storage systems with a `R` (for read-only) in the ModTime column, +means the it keeps modification times on objects, and updates them +when uploading objects, but it does not support changing only the +modification time (`SetModTime` operation) without re-uploading, +possibly not even without deleting existing first. Some operations +in rclone, such as `copy` and `sync` commands, will automatically +check for `SetModTime` support and re-upload if necessary to keep +the modification times in sync. Other commands will not work +without `SetModTime` support, e.g. `touch` command on an existing +file will fail, and changes to modification time only on a files +in a `mount` will be silently ignored. + +Storage systems with `R/W` (for read/write) in the ModTime column, +means they do also support modtime-only operations. ### Case Insensitive ### From f4f0e444bf60b7bf6ebac9fa1e8a94d6821bdd40 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 8 Jun 2022 09:29:01 +0200 Subject: [PATCH 009/560] filter: allow multiple --exclude-if-present flags - fixes #6219 --- docs/content/docs.md | 1 + docs/content/filtering.md | 6 +++--- docs/content/flags.md | 2 +- fs/filter/filter.go | 22 +++++++++++++--------- fs/filter/filterflags/filterflags.go | 2 +- fs/operations/listdirsorted_test.go | 4 ++-- fs/operations/operations_test.go | 2 +- fs/sync/sync_test.go | 2 +- fs/walk/walk.go | 10 ++++++---- fs/walk/walk_test.go | 4 ++-- 10 files changed, 31 insertions(+), 24 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index 9be5038b8f42f..50bc83cfae7a0 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -2136,6 +2136,7 @@ For the filtering options * `--filter-from` * `--exclude` * `--exclude-from` + * `--exclude-if-present` * `--include` * `--include-from` * `--files-from` diff --git a/docs/content/filtering.md b/docs/content/filtering.md index f306accf51d2d..de191ddc22ca4 100644 --- a/docs/content/filtering.md +++ b/docs/content/filtering.md @@ -750,7 +750,9 @@ Useful for debugging. The `--exclude-if-present` flag controls whether a directory is within the scope of an rclone command based on the presence of a -named file within it. +named file within it. The flag can be repeated to check for +multiple file names, presence of any of them will exclude the +directory. This flag has a priority over other filter flags. @@ -764,8 +766,6 @@ E.g. for the following directory structure: The command `rclone ls --exclude-if-present .ignore dir1` does not list `dir3`, `file3` or `.ignore`. -`--exclude-if-present` can only be used once in an rclone command. - ## Common pitfalls The most frequent filter support issues on diff --git a/docs/content/flags.md b/docs/content/flags.md index 69a0ea380ff1b..bfc367e63afdc 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -47,7 +47,7 @@ These flags are available for every command. --error-on-no-transfer Sets exit code 9 if no files are transferred, useful in scripts --exclude stringArray Exclude files matching pattern --exclude-from stringArray Read exclude patterns from file (use - to read from stdin) - --exclude-if-present string Exclude directories if filename is present + --exclude-if-present stringArray Exclude directories if filename is present --expect-continue-timeout duration Timeout when using expect / 100-continue in HTTP (default 1s) --fast-list Use recursive list if available; uses more memory but fewer transactions --files-from stringArray Read list of source-file names from file (use - to read from stdin) diff --git a/fs/filter/filter.go b/fs/filter/filter.go index 72a25462f1b46..37071f9d26646 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -86,7 +86,7 @@ type Opt struct { FilterFrom []string ExcludeRule []string ExcludeFrom []string - ExcludeFile string + ExcludeFile []string IncludeRule []string IncludeFrom []string FilesFrom []string @@ -392,8 +392,10 @@ func (f *Filter) ListContainsExcludeFile(entries fs.DirEntries) bool { obj, ok := entry.(fs.Object) if ok { basename := path.Base(obj.Remote()) - if basename == f.Opt.ExcludeFile { - return true + for _, excludeFile := range f.Opt.ExcludeFile { + if basename == excludeFile { + return true + } } } } @@ -436,12 +438,14 @@ func (f *Filter) IncludeDirectory(ctx context.Context, fs fs.Fs) func(string) (b // empty string (for testing). func (f *Filter) DirContainsExcludeFile(ctx context.Context, fremote fs.Fs, remote string) (bool, error) { if len(f.Opt.ExcludeFile) > 0 { - exists, err := fs.FileExists(ctx, fremote, path.Join(remote, f.Opt.ExcludeFile)) - if err != nil { - return false, err - } - if exists { - return true, nil + for _, excludeFile := range f.Opt.ExcludeFile { + exists, err := fs.FileExists(ctx, fremote, path.Join(remote, excludeFile)) + if err != nil { + return false, err + } + if exists { + return true, nil + } } } return false, nil diff --git a/fs/filter/filterflags/filterflags.go b/fs/filter/filterflags/filterflags.go index e926fd8946a28..3963e0c04d476 100644 --- a/fs/filter/filterflags/filterflags.go +++ b/fs/filter/filterflags/filterflags.go @@ -34,7 +34,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.StringArrayVarP(flagSet, &Opt.FilterFrom, "filter-from", "", nil, "Read filtering patterns from a file (use - to read from stdin)") flags.StringArrayVarP(flagSet, &Opt.ExcludeRule, "exclude", "", nil, "Exclude files matching pattern") flags.StringArrayVarP(flagSet, &Opt.ExcludeFrom, "exclude-from", "", nil, "Read exclude patterns from file (use - to read from stdin)") - flags.StringVarP(flagSet, &Opt.ExcludeFile, "exclude-if-present", "", "", "Exclude directories if filename is present") + flags.StringArrayVarP(flagSet, &Opt.ExcludeFile, "exclude-if-present", "", nil, "Exclude directories if filename is present") flags.StringArrayVarP(flagSet, &Opt.IncludeRule, "include", "", nil, "Include files matching pattern") flags.StringArrayVarP(flagSet, &Opt.IncludeFrom, "include-from", "", nil, "Read include patterns from file (use - to read from stdin)") flags.StringArrayVarP(flagSet, &Opt.FilesFrom, "files-from", "", nil, "Read list of source-file names from file (use - to read from stdin)") diff --git a/fs/operations/listdirsorted_test.go b/fs/operations/listdirsorted_test.go index ae09ab4c9534a..5a4e3eeeade1a 100644 --- a/fs/operations/listdirsorted_test.go +++ b/fs/operations/listdirsorted_test.go @@ -81,7 +81,7 @@ func TestListDirSorted(t *testing.T) { assert.Equal(t, "sub dir/sub sub dir/", str(1)) // testing ignore file - fi.Opt.ExcludeFile = ".ignore" + fi.Opt.ExcludeFile = []string{".ignore"} items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir") require.NoError(t, err) @@ -98,7 +98,7 @@ func TestListDirSorted(t *testing.T) { assert.Equal(t, "sub dir/ignore dir/.ignore", str(0)) assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1)) - fi.Opt.ExcludeFile = "" + fi.Opt.ExcludeFile = nil items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir") require.NoError(t, err) require.Len(t, items, 2) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 363d88f59d1b7..83df26398a5e6 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1309,7 +1309,7 @@ func TestOverlappingFilterCheckWithFilter(t *testing.T) { fi, err := filter.NewFilter(nil) require.NoError(t, err) require.NoError(t, fi.Add(false, "*/exclude/")) - fi.Opt.ExcludeFile = ".ignore" + fi.Opt.ExcludeFile = []string{".ignore"} ctx = filter.ReplaceConfig(ctx, fi) src := &testFs{testFsInfo{name: "name", root: "root"}} diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index bd0772bd64bfd..19e225fd92a76 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -1452,7 +1452,7 @@ func TestSyncOverlapWithFilter(t *testing.T) { require.NoError(t, err) require.NoError(t, fi.Add(false, "/rclone-sync-test/")) require.NoError(t, fi.Add(false, "*/layer2/")) - fi.Opt.ExcludeFile = ".ignore" + fi.Opt.ExcludeFile = []string{".ignore"} ctx = filter.ReplaceConfig(ctx, fi) subRemoteName := r.FremoteName + "/rclone-sync-test" diff --git a/fs/walk/walk.go b/fs/walk/walk.go index 5f93cca493f98..b6ca99c325e52 100644 --- a/fs/walk/walk.go +++ b/fs/walk/walk.go @@ -507,10 +507,12 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo // Check if we need to prune a directory later. if !includeAll && len(fi.Opt.ExcludeFile) > 0 { basename := path.Base(x.Remote()) - if basename == fi.Opt.ExcludeFile { - excludeDir := parentDir(x.Remote()) - toPrune[excludeDir] = true - fs.Debugf(basename, "Excluded from sync (and deletion) based on exclude file") + for _, excludeFile := range fi.Opt.ExcludeFile { + if basename == excludeFile { + excludeDir := parentDir(x.Remote()) + toPrune[excludeDir] = true + fs.Debugf(basename, "Excluded from sync (and deletion) based on exclude file") + } } } case fs.Directory: diff --git a/fs/walk/walk_test.go b/fs/walk/walk_test.go index ecfa3ed7bc635..cbfa82ce830b2 100644 --- a/fs/walk/walk_test.go +++ b/fs/walk/walk_test.go @@ -736,13 +736,13 @@ b/c/d/ e `, nil, "", -1, "ign", true}, } { - fi.Opt.ExcludeFile = test.excludeFile + fi.Opt.ExcludeFile = []string{test.excludeFile} r, err := walkRDirTree(context.Background(), nil, test.root, test.includeAll, test.level, makeListRCallback(test.entries, test.err)) assert.Equal(t, test.err, err, fmt.Sprintf("%+v", test)) assert.Equal(t, test.want, r.String(), fmt.Sprintf("%+v", test)) } // Set to default value, to avoid side effects - fi.Opt.ExcludeFile = "" + fi.Opt.ExcludeFile = nil } func TestListType(t *testing.T) { From 700ca23a718adeb9b8cc6671a065ebe2f347cb3d Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Tue, 18 Jan 2022 22:38:24 +0100 Subject: [PATCH 010/560] config: add utility function for backend config with list and custom input --- backend/jottacloud/jottacloud.go | 6 +-- backend/onedrive/onedrive.go | 2 +- fs/backend_config.go | 75 +++++++++++++++++++++++++++++--- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index c75774d93f643..3322d8d8c3a6b 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -127,7 +127,7 @@ func init() { func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { switch config.State { case "": - return fs.ConfigChooseFixed("auth_type_done", "config_type", `Authentication type.`, []fs.OptionExample{{ + return fs.ConfigChooseExclusiveFixed("auth_type_done", "config_type", `Authentication type.`, []fs.OptionExample{{ Value: "standard", Help: "Standard authentication.\nUse this if you're a normal Jottacloud user.", }, { @@ -286,7 +286,7 @@ machines.`) if err != nil { return nil, err } - return fs.ConfigChoose("choose_device_result", "config_device", `Please select the device to use. Normally this will be Jotta`, len(acc.Devices), func(i int) (string, string) { + return fs.ConfigChooseExclusive("choose_device_result", "config_device", `Please select the device to use. Normally this will be Jotta`, len(acc.Devices), func(i int) (string, string) { return acc.Devices[i].Name, "" }) case "choose_device_result": @@ -304,7 +304,7 @@ machines.`) if err != nil { return nil, err } - return fs.ConfigChoose("choose_device_mountpoint", "config_mountpoint", `Please select the mountpoint to use. Normally this will be Archive.`, len(dev.MountPoints), func(i int) (string, string) { + return fs.ConfigChooseExclusive("choose_device_mountpoint", "config_mountpoint", `Please select the mountpoint to use. Normally this will be Archive.`, len(dev.MountPoints), func(i int) (string, string) { return dev.MountPoints[i].Name, "" }) case "choose_device_mountpoint": diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 80bf340c607e3..132ed77e66a9c 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -424,7 +424,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf switch config.State { case "choose_type": - return fs.ConfigChooseFixed("choose_type_done", "config_type", "Type of connection", []fs.OptionExample{{ + return fs.ConfigChooseExclusiveFixed("choose_type_done", "config_type", "Type of connection", []fs.OptionExample{{ Value: "onedrive", Help: "OneDrive Personal or Business", }, { diff --git a/fs/backend_config.go b/fs/backend_config.go index cef15bf56a456..1e188a0ac1300 100644 --- a/fs/backend_config.go +++ b/fs/backend_config.go @@ -176,7 +176,13 @@ func ConfigConfirm(state string, Default bool, name string, help string) (*Confi }, nil } -// ConfigChooseFixed returns a ConfigOut structure which has a list of items to choose from. +// ConfigChooseExclusiveFixed returns a ConfigOut structure which has a list of +// items to choose from. +// +// Possible items must be supplied as a fixed list. +// +// User is required to supply a value, and is restricted to the specified list, +// i.e. free text input is not allowed. // // state should be the next state required // name is the config name for this item @@ -185,8 +191,8 @@ func ConfigConfirm(state string, Default bool, name string, help string) (*Confi // // It chooses the first item to be the default. // If there are no items then it will return an error. -// If there is only one item it will short cut to the next state -func ConfigChooseFixed(state string, name string, help string, items []OptionExample) (*ConfigOut, error) { +// If there is only one item it will short cut to the next state. +func ConfigChooseExclusiveFixed(state string, name string, help string, items []OptionExample) (*ConfigOut, error) { if len(items) == 0 { return nil, fmt.Errorf("no items found in: %s", help) } @@ -208,7 +214,13 @@ func ConfigChooseFixed(state string, name string, help string, items []OptionExa return choose, nil } -// ConfigChoose returns a ConfigOut structure which has a list of items to choose from. +// ConfigChooseExclusive returns a ConfigOut structure which has a list of +// items to choose from. +// +// Possible items are retrieved from a supplied function. +// +// User is required to supply a value, and is restricted to the specified list, +// i.e. free text input is not allowed. // // state should be the next state required // name is the config name for this item @@ -218,7 +230,60 @@ func ConfigChooseFixed(state string, name string, help string, items []OptionExa // // It chooses the first item to be the default. // If there are no items then it will return an error. -// If there is only one item it will short cut to the next state +// If there is only one item it will short cut to the next state. +func ConfigChooseExclusive(state string, name string, help string, n int, getItem func(i int) (itemValue string, itemHelp string)) (*ConfigOut, error) { + items := make(OptionExamples, n) + for i := range items { + items[i].Value, items[i].Help = getItem(i) + } + return ConfigChooseExclusiveFixed(state, name, help, items) +} + +// ConfigChooseFixed returns a ConfigOut structure which has a list of +// suggested items. +// +// Suggested items must be supplied as a fixed list. +// +// User is required to supply a value, but is not restricted to the specified +// list, i.e. free text input is accepted. +// +// state should be the next state required +// name is the config name for this item +// help should be the help shown to the user +// items should be the items in the list +// +// It chooses the first item to be the default. +func ConfigChooseFixed(state string, name string, help string, items []OptionExample) (*ConfigOut, error) { + choose := &ConfigOut{ + State: state, + Option: &Option{ + Name: name, + Help: help, + Examples: items, + Required: true, + }, + } + if len(choose.Option.Examples) > 0 { + choose.Option.Default = choose.Option.Examples[0].Value + } + return choose, nil +} + +// ConfigChoose returns a ConfigOut structure which has a list of suggested +// items. +// +// Suggested items are retrieved from a supplied function. +// +// User is required to supply a value, but is not restricted to the specified +// list, i.e. free text input is accepted. +// +// state should be the next state required +// name is the config name for this item +// help should be the help shown to the user +// n should be the number of items in the list +// getItem should return the items (value, help) +// +// It chooses the first item to be the default. func ConfigChoose(state string, name string, help string, n int, getItem func(i int) (itemValue string, itemHelp string)) (*ConfigOut, error) { items := make(OptionExamples, n) for i := range items { From 01340acad24bc07f8716c0924d6ecfdfca20f1be Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Tue, 18 Jan 2022 20:59:50 +0100 Subject: [PATCH 011/560] jottacloud: add support for upload to custom device and mountpoint See #5926 --- backend/jottacloud/jottacloud.go | 151 ++++++++++++++++++++++++++++--- docs/content/jottacloud.md | 115 ++++++++++++++--------- 2 files changed, 212 insertions(+), 54 deletions(-) diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 3322d8d8c3a6b..65df99f585915 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -127,7 +127,7 @@ func init() { func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { switch config.State { case "": - return fs.ConfigChooseExclusiveFixed("auth_type_done", "config_type", `Authentication type.`, []fs.OptionExample{{ + return fs.ConfigChooseExclusiveFixed("auth_type_done", "config_type", `Select authentication type.`, []fs.OptionExample{{ Value: "standard", Help: "Standard authentication.\nUse this if you're a normal Jottacloud user.", }, { @@ -145,7 +145,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf return fs.ConfigGoto(config.Result) case "standard": // configure a jottacloud backend using the modern JottaCli token based authentication m.Set("configVersion", fmt.Sprint(configVersion)) - return fs.ConfigInput("standard_token", "config_login_token", "Personal login token.\n\nGenerate here: https://www.jottacloud.com/web/secure") + return fs.ConfigInput("standard_token", "config_login_token", "Personal login token.\nGenerate here: https://www.jottacloud.com/web/secure") case "standard_token": loginToken := config.Result m.Set(configClientID, defaultClientID) @@ -262,7 +262,11 @@ machines.`) }, }) case "choose_device": - return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", "Use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?") + return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", `Use a non-standard device/mountpoint? +Choosing no, the default, will let you access the storage used for the archive +section of the official Jottacloud client. If you instead want to access the +sync or the backup section, for example, you must choose yes.`) + case "choose_device_query": if config.Result != "true" { m.Set(configDevice, "") @@ -286,12 +290,27 @@ machines.`) if err != nil { return nil, err } - return fs.ConfigChooseExclusive("choose_device_result", "config_device", `Please select the device to use. Normally this will be Jotta`, len(acc.Devices), func(i int) (string, string) { - return acc.Devices[i].Name, "" + + deviceNames := make([]string, len(acc.Devices)) + for i, dev := range acc.Devices { + if i > 0 && dev.Name == defaultDevice { + // Insert the special Jotta device as first entry, making it the default choice. + copy(deviceNames[1:i+1], deviceNames[0:i]) + deviceNames[0] = dev.Name + } else { + deviceNames[i] = dev.Name + } + } + + help := fmt.Sprintf(`The device to use. In standard setup the built-in %s device is used, +which contains predefined mountpoints for archive, sync etc. All other devices +are treated as backup devices by the official Jottacloud client. You may create +a new by entering a unique name.`, defaultDevice) + return fs.ConfigChoose("choose_device_result", "config_device", help, len(deviceNames), func(i int) (string, string) { + return deviceNames[i], "" }) case "choose_device_result": device := config.Result - m.Set(configDevice, device) oAuthClient, _, err := getOAuthClient(ctx, name, m) if err != nil { @@ -300,16 +319,89 @@ machines.`) srv := rest.NewClient(oAuthClient).SetRoot(rootURL) username, _ := m.Get(configUsername) - dev, err := getDeviceInfo(ctx, srv, path.Join(username, device)) + + acc, err := getDriveInfo(ctx, srv, username) if err != nil { return nil, err } - return fs.ConfigChooseExclusive("choose_device_mountpoint", "config_mountpoint", `Please select the mountpoint to use. Normally this will be Archive.`, len(dev.MountPoints), func(i int) (string, string) { + isNew := true + for _, dev := range acc.Devices { + if strings.EqualFold(dev.Name, device) { // If device name exists with different casing we prefer the existing (not sure if and how the api handles the opposite) + device = dev.Name // Prefer same casing as existing, e.g. if user entered "jotta" we use the standard casing "Jotta" instead + isNew = false + break + } + } + var dev *api.JottaDevice + if isNew { + fs.Debugf(nil, "Creating new device: %s", device) + dev, err = createDevice(ctx, srv, path.Join(username, device)) + if err != nil { + return nil, err + } + } + m.Set(configDevice, device) + + if !isNew { + dev, err = getDeviceInfo(ctx, srv, path.Join(username, device)) + if err != nil { + return nil, err + } + } + + var help string + if device == defaultDevice { + // With built-in Jotta device the mountpoint choice is exclusive, + // we do not want to risk any problems by creating new mountpoints on it. + help = fmt.Sprintf(`The mountpoint to use on the built-in device %s. +The standard setup is to use the %s mountpoint. Most other mountpoints +have very limited support in rclone and should generally be avoided.`, defaultDevice, defaultMountpoint) + return fs.ConfigChooseExclusive("choose_device_mountpoint", "config_mountpoint", help, len(dev.MountPoints), func(i int) (string, string) { + return dev.MountPoints[i].Name, "" + }) + } + help = fmt.Sprintf(`The mountpoint to use on the non-standard device %s. +You may create a new by entering a unique name.`, device) + return fs.ConfigChoose("choose_device_mountpoint", "config_mountpoint", help, len(dev.MountPoints), func(i int) (string, string) { return dev.MountPoints[i].Name, "" }) case "choose_device_mountpoint": mountpoint := config.Result + + oAuthClient, _, err := getOAuthClient(ctx, name, m) + if err != nil { + return nil, err + } + srv := rest.NewClient(oAuthClient).SetRoot(rootURL) + + username, _ := m.Get(configUsername) + device, _ := m.Get(configDevice) + + dev, err := getDeviceInfo(ctx, srv, path.Join(username, device)) + if err != nil { + return nil, err + } + isNew := true + for _, mnt := range dev.MountPoints { + if strings.EqualFold(mnt.Name, mountpoint) { + mountpoint = mnt.Name + isNew = false + break + } + } + + if isNew { + if device == defaultDevice { + return nil, fmt.Errorf("custom mountpoints not supported on built-in %s device: %w", defaultDevice, err) + } + fs.Debugf(nil, "Creating new mountpoint: %s", mountpoint) + _, err := createMountPoint(ctx, srv, path.Join(username, device, mountpoint)) + if err != nil { + return nil, err + } + } m.Set(configMountpoint, mountpoint) + return fs.ConfigGoto("end") case "end": // All the config flows end up here in case we need to carry on with something @@ -338,6 +430,7 @@ type Fs struct { opt Options features *fs.Features endpointURL string + allocateURL string srv *rest.Client apiSrv *rest.Client pacer *fs.Pacer @@ -588,6 +681,37 @@ func getDeviceInfo(ctx context.Context, srv *rest.Client, path string) (info *ap return info, nil } +// createDevice makes a device +func createDevice(ctx context.Context, srv *rest.Client, path string) (info *api.JottaDevice, err error) { + opts := rest.Opts{ + Method: "POST", + Path: urlPathEscape(path), + Parameters: url.Values{}, + } + + opts.Parameters.Set("type", "WORKSTATION") + + _, err = srv.CallXML(ctx, &opts, nil, &info) + if err != nil { + return nil, fmt.Errorf("couldn't create device: %w", err) + } + return info, nil +} + +// createMountPoint makes a mount point +func createMountPoint(ctx context.Context, srv *rest.Client, path string) (info *api.JottaMountPoint, err error) { + opts := rest.Opts{ + Method: "POST", + Path: urlPathEscape(path), + } + + _, err = srv.CallXML(ctx, &opts, nil, &info) + if err != nil { + return nil, fmt.Errorf("couldn't create mountpoint: %w", err) + } + return info, nil +} + // setEndpointURL generates the API endpoint URL func (f *Fs) setEndpointURL() { if f.opt.Device == "" { @@ -597,6 +721,7 @@ func (f *Fs) setEndpointURL() { f.opt.Mountpoint = defaultMountpoint } f.endpointURL = path.Join(f.user, f.opt.Device, f.opt.Mountpoint) + f.allocateURL = path.Join("/jfs", f.opt.Device, f.opt.Mountpoint) } // readMetaDataForPath reads the metadata from the path @@ -662,6 +787,11 @@ func (f *Fs) filePath(file string) string { return urlPathEscape(f.filePathRaw(file)) } +// allocatePathRaw returns an unescaped file path (f.root, file) +func (f *Fs) allocatePathRaw(file string) string { + return path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, file))) +} + // Jottacloud requires the grant_type 'refresh_token' string // to be uppercase and throws a 400 Bad Request if we use the // lower case used by the oauth2 module @@ -1101,9 +1231,6 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { - if f.opt.Device != "Jotta" { - return nil, errors.New("upload not supported for devices other than Jotta") - } o := f.createObject(src.Remote(), src.ModTime(ctx), src.Size()) return o, o.Update(ctx, in, src, options...) } @@ -1738,7 +1865,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op Created: fileDate, Modified: fileDate, Md5: md5String, - Path: path.Join(o.fs.opt.Mountpoint, o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))), + Path: path.Join(o.fs.allocateURL, o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))), } // send it diff --git a/docs/content/jottacloud.md b/docs/content/jottacloud.md index 77afba24ac8f4..ef7ea558627e7 100644 --- a/docs/content/jottacloud.md +++ b/docs/content/jottacloud.md @@ -75,60 +75,83 @@ s) Set configuration password q) Quit config n/s/q> n name> remote +Option Storage. Type of storage to configure. -Enter a string value. Press Enter for the default (""). -Choose a number from below, or type in your own value +Choose a number from below, or type in your own value. [snip] XX / Jottacloud - \ "jottacloud" + \ (jottacloud) [snip] Storage> jottacloud -** See help for jottacloud backend at: https://rclone.org/jottacloud/ ** - -Edit advanced config? (y/n) -y) Yes -n) No -y/n> n -Remote config -Use legacy authentication?. -This is only required for certain whitelabel versions of Jottacloud and not recommended for normal users. +Edit advanced config? y) Yes n) No (default) y/n> n - -Generate a personal login token here: https://www.jottacloud.com/web/secure +Option config_type. +Select authentication type. +Choose a number from below, or type in an existing string value. +Press Enter for the default (standard). + / Standard authentication. + 1 | Use this if you're a normal Jottacloud user. + \ (standard) + / Legacy authentication. + 2 | This is only required for certain whitelabel versions of Jottacloud and not recommended for normal users. + \ (legacy) + / Telia Cloud authentication. + 3 | Use this if you are using Telia Cloud. + \ (telia) + / Tele2 Cloud authentication. + 4 | Use this if you are using Tele2 Cloud. + \ (tele2) +config_type> 1 +Personal login token. +Generate here: https://www.jottacloud.com/web/secure Login Token> - -Do you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client? - +Use a non-standard device/mountpoint? +Choosing no, the default, will let you access the storage used for the archive +section of the official Jottacloud client. If you instead want to access the +sync or the backup section, for example, you must choose yes. y) Yes -n) No +n) No (default) y/n> y -Please select the device to use. Normally this will be Jotta -Choose a number from below, or type in an existing value +Option config_device. +The device to use. In standard setup the built-in Jotta device is used, +which contains predefined mountpoints for archive, sync etc. All other devices +are treated as backup devices by the official Jottacloud client. You may create +a new by entering a unique name. +Choose a number from below, or type in your own string value. +Press Enter for the default (DESKTOP-3H31129). 1 > DESKTOP-3H31129 2 > Jotta -Devices> 2 -Please select the mountpoint to user. Normally this will be Archive -Choose a number from below, or type in an existing value +config_device> 2 +Option config_mountpoint. +The mountpoint to use for the built-in device Jotta. +The standard setup is to use the Archive mountpoint. Most other mountpoints +have very limited support in rclone and should generally be avoided. +Choose a number from below, or type in an existing string value. +Press Enter for the default (Archive). 1 > Archive - 2 > Links + 2 > Shared 3 > Sync - -Mountpoints> 1 +config_mountpoint> 1 -------------------- -[jotta] +[remote] type = jottacloud +configVersion = 1 +client_id = jottacli +client_secret = +tokenURL = https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token token = {........} +username = 2940e57271a93d987d6f8a21 device = Jotta mountpoint = Archive -configVersion = 1 -------------------- -y) Yes this is OK +y) Yes this is OK (default) e) Edit this remote d) Delete this remote y/e/d> y ``` + Once configured you can then use `rclone` like this, List directories in top level of your Jottacloud @@ -145,19 +168,27 @@ To copy a local directory to an Jottacloud directory called backup ### Devices and Mountpoints -The official Jottacloud client registers a device for each computer you install it on, -and then creates a mountpoint for each folder you select for Backup. -The web interface uses a special device called Jotta for the Archive and Sync mountpoints. - -With rclone you'll want to use the Jotta/Archive device/mountpoint in most cases, however if you -want to access files uploaded by any of the official clients rclone provides the option to select -other devices and mountpoints during config. Note that uploading files is currently not supported -to other devices than Jotta. - -The built-in Jotta device may also contain several other mountpoints, such as: Latest, Links, Shared and Trash. -These are special mountpoints with a different internal representation than the "regular" mountpoints. -Rclone will only to a very limited degree support them. Generally you should avoid these, unless you know what you -are doing. +The official Jottacloud client registers a device for each computer you install +it on, and shows them in the backup section of the user interface. For each +folder you select for backup it will create a mountpoint within this device. +A built-in device called Jotta is special, and contains mountpoints Archive, +Sync and some others, used for corresponding features in official clients. + +With rclone you'll want to use the standard Jotta/Archive device/mountpoint in +most cases. However, you may for example want to access files from the sync or +backup functionality provided by the official clients, and rclone therefore +provides the option to select other devices and mountpoints during config. + +You are allowed to create new devices and mountpoints. All devices except the +built-in Jotta device are treated as backup devices by official Jottacloud +clients, and the mountpoints on them are individual backup sets. + +With the built-in Jotta device, only existing, built-in, mountpoints can be +selected. In addition to the mentioned Archive and Sync, it may contain +several other mountpoints such as: Latest, Links, Shared and Trash. All of +these are special mountpoints with a different internal representation than +the "regular" mountpoints. Rclone will only to a very limited degree support +them. Generally you should avoid these, unless you know what you are doing. ### --fast-list From a571c1fb46f531fb88409129f31c245c5f9274d3 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Tue, 18 Jan 2022 21:06:22 +0100 Subject: [PATCH 012/560] jottacloud: refactor naming of different api urls --- backend/jottacloud/jottacloud.go | 54 ++++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 65df99f585915..cde7fb0760287 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -46,9 +46,9 @@ const ( decayConstant = 2 // bigger for slower decay, exponential defaultDevice = "Jotta" defaultMountpoint = "Archive" - rootURL = "https://jfs.jottacloud.com/jfs/" + jfsURL = "https://jfs.jottacloud.com/jfs/" apiURL = "https://api.jottacloud.com/" - baseURL = "https://www.jottacloud.com/" + wwwURL = "https://www.jottacloud.com/" cachePrefix = "rclone-jcmd5-" configDevice = "device" configMountpoint = "mountpoint" @@ -277,7 +277,7 @@ sync or the backup section, for example, you must choose yes.`) if err != nil { return nil, err } - srv := rest.NewClient(oAuthClient).SetRoot(rootURL) + jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL) apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL) cust, err := getCustomerInfo(ctx, apiSrv) @@ -286,7 +286,7 @@ sync or the backup section, for example, you must choose yes.`) } m.Set(configUsername, cust.Username) - acc, err := getDriveInfo(ctx, srv, cust.Username) + acc, err := getDriveInfo(ctx, jfsSrv, cust.Username) if err != nil { return nil, err } @@ -316,11 +316,11 @@ a new by entering a unique name.`, defaultDevice) if err != nil { return nil, err } - srv := rest.NewClient(oAuthClient).SetRoot(rootURL) + jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL) username, _ := m.Get(configUsername) - acc, err := getDriveInfo(ctx, srv, username) + acc, err := getDriveInfo(ctx, jfsSrv, username) if err != nil { return nil, err } @@ -335,7 +335,7 @@ a new by entering a unique name.`, defaultDevice) var dev *api.JottaDevice if isNew { fs.Debugf(nil, "Creating new device: %s", device) - dev, err = createDevice(ctx, srv, path.Join(username, device)) + dev, err = createDevice(ctx, jfsSrv, path.Join(username, device)) if err != nil { return nil, err } @@ -343,7 +343,7 @@ a new by entering a unique name.`, defaultDevice) m.Set(configDevice, device) if !isNew { - dev, err = getDeviceInfo(ctx, srv, path.Join(username, device)) + dev, err = getDeviceInfo(ctx, jfsSrv, path.Join(username, device)) if err != nil { return nil, err } @@ -372,12 +372,12 @@ You may create a new by entering a unique name.`, device) if err != nil { return nil, err } - srv := rest.NewClient(oAuthClient).SetRoot(rootURL) + jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL) username, _ := m.Get(configUsername) device, _ := m.Get(configDevice) - dev, err := getDeviceInfo(ctx, srv, path.Join(username, device)) + dev, err := getDeviceInfo(ctx, jfsSrv, path.Join(username, device)) if err != nil { return nil, err } @@ -395,7 +395,7 @@ You may create a new by entering a unique name.`, device) return nil, fmt.Errorf("custom mountpoints not supported on built-in %s device: %w", defaultDevice, err) } fs.Debugf(nil, "Creating new mountpoint: %s", mountpoint) - _, err := createMountPoint(ctx, srv, path.Join(username, device, mountpoint)) + _, err := createMountPoint(ctx, jfsSrv, path.Join(username, device, mountpoint)) if err != nil { return nil, err } @@ -431,7 +431,7 @@ type Fs struct { features *fs.Features endpointURL string allocateURL string - srv *rest.Client + jfsSrv *rest.Client apiSrv *rest.Client pacer *fs.Pacer tokenRenewer *oauthutil.Renew // renew the token on expiry @@ -733,7 +733,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Jo var result api.JottaFile var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(ctx, &opts, nil, &result) + resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result) return shouldRetry(ctx, resp, err) }) @@ -899,7 +899,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e name: name, root: root, opt: *opt, - srv: rest.NewClient(oAuthClient).SetRoot(rootURL), + jfsSrv: rest.NewClient(oAuthClient).SetRoot(jfsURL), apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL), pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), } @@ -909,7 +909,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e ReadMimeType: true, WriteMimeType: false, }).Fill(ctx, f) - f.srv.SetErrorHandler(errorHandler) + f.jfsSrv.SetErrorHandler(errorHandler) if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now f.features.ListR = nil } @@ -994,7 +994,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (jf *api.JottaFolder, e opts.Parameters.Set("mkDir", "true") err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(ctx, &opts, nil, &jf) + resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &jf) return shouldRetry(ctx, resp, err) }) if err != nil { @@ -1023,7 +1023,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e var resp *http.Response var result api.JottaFolder err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(ctx, &opts, nil, &result) + resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result) return shouldRetry(ctx, resp, err) }) @@ -1181,7 +1181,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(ctx, &opts) + resp, err = f.jfsSrv.Call(ctx, &opts) if err != nil { return shouldRetry(ctx, resp, err) } @@ -1291,7 +1291,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.Call(ctx, &opts) + resp, err = f.jfsSrv.Call(ctx, &opts) return shouldRetry(ctx, resp, err) }) if err != nil { @@ -1344,7 +1344,7 @@ func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(ctx, &opts, nil, &info) + resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &info) return shouldRetry(ctx, resp, err) }) @@ -1369,7 +1369,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap var resp *http.Response err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(ctx, &opts, nil, &info) + resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &info) return shouldRetry(ctx, resp, err) }) if err != nil { @@ -1503,7 +1503,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, var resp *http.Response var result api.JottaFile err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallXML(ctx, &opts, nil, &result) + resp, err = f.jfsSrv.CallXML(ctx, &opts, nil, &result) return shouldRetry(ctx, resp, err) }) @@ -1529,19 +1529,19 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, return "", errors.New("couldn't create public link - no uri received") } if result.PublicSharePath != "" { - webLink := joinPath(baseURL, result.PublicSharePath) + webLink := joinPath(wwwURL, result.PublicSharePath) fs.Debugf(nil, "Web link: %s", webLink) } else { fs.Debugf(nil, "No web link received") } - directLink := joinPath(baseURL, fmt.Sprintf("opin/io/downloadPublic/%s/%s", f.user, result.PublicURI)) + directLink := joinPath(wwwURL, fmt.Sprintf("opin/io/downloadPublic/%s/%s", f.user, result.PublicURI)) fs.Debugf(nil, "Direct link: %s", directLink) return directLink, nil } // About gets quota information func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { - info, err := getDriveInfo(ctx, f.srv, f.user) + info, err := getDriveInfo(ctx, f.jfsSrv, f.user) if err != nil { return nil, err } @@ -1744,7 +1744,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read opts.Parameters.Set("mode", "bin") err = o.fs.pacer.Call(func() (bool, error) { - resp, err = o.fs.srv.Call(ctx, &opts) + resp, err = o.fs.jfsSrv.Call(ctx, &opts) return shouldRetry(ctx, resp, err) }) if err != nil { @@ -1935,7 +1935,7 @@ func (o *Object) remove(ctx context.Context, hard bool) error { } return o.fs.pacer.Call(func() (bool, error) { - resp, err := o.fs.srv.CallXML(ctx, &opts, nil, nil) + resp, err := o.fs.jfsSrv.CallXML(ctx, &opts, nil, nil) return shouldRetry(ctx, resp, err) }) } From b2388f129442f4e610d0508e01f8e66a3b337a60 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Tue, 18 Jan 2022 21:48:44 +0100 Subject: [PATCH 013/560] jottacloud: refactor endpoint paths --- backend/jottacloud/jottacloud.go | 62 +++++++++++++++++++------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index cde7fb0760287..1271eda1af474 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -424,17 +424,17 @@ type Options struct { // Fs represents a remote jottacloud type Fs struct { - name string - root string - user string - opt Options - features *fs.Features - endpointURL string - allocateURL string - jfsSrv *rest.Client - apiSrv *rest.Client - pacer *fs.Pacer - tokenRenewer *oauthutil.Renew // renew the token on expiry + name string + root string + user string + opt Options + features *fs.Features + fileEndpoint string + allocateEndpoint string + jfsSrv *rest.Client + apiSrv *rest.Client + pacer *fs.Pacer + tokenRenewer *oauthutil.Renew // renew the token on expiry } // Object describes a jottacloud object @@ -712,16 +712,16 @@ func createMountPoint(ctx context.Context, srv *rest.Client, path string) (info return info, nil } -// setEndpointURL generates the API endpoint URL -func (f *Fs) setEndpointURL() { +// setEndpoints generates the API endpoints +func (f *Fs) setEndpoints() { if f.opt.Device == "" { f.opt.Device = defaultDevice } if f.opt.Mountpoint == "" { f.opt.Mountpoint = defaultMountpoint } - f.endpointURL = path.Join(f.user, f.opt.Device, f.opt.Mountpoint) - f.allocateURL = path.Join("/jfs", f.opt.Device, f.opt.Mountpoint) + f.fileEndpoint = path.Join(f.user, f.opt.Device, f.opt.Mountpoint) + f.allocateEndpoint = path.Join("/jfs", f.opt.Device, f.opt.Mountpoint) } // readMetaDataForPath reads the metadata from the path @@ -778,18 +778,30 @@ func urlPathEscape(in string) string { } // filePathRaw returns an unescaped file path (f.root, file) -func (f *Fs) filePathRaw(file string) string { - return path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, file))) +// Optionally made absolute by prefixing with "/", typically required when used +// as request parameter instead of the path (which is relative to some root url). +func (f *Fs) filePathRaw(file string, absolute bool) string { + prefix := "" + if absolute { + prefix = "/" + } + return path.Join(prefix, f.fileEndpoint, f.opt.Enc.FromStandardPath(path.Join(f.root, file))) } // filePath returns an escaped file path (f.root, file) func (f *Fs) filePath(file string) string { - return urlPathEscape(f.filePathRaw(file)) + return urlPathEscape(f.filePathRaw(file, false)) } -// allocatePathRaw returns an unescaped file path (f.root, file) -func (f *Fs) allocatePathRaw(file string) string { - return path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, file))) +// allocatePathRaw returns an unescaped allocate file path (f.root, file) +// Optionally made absolute by prefixing with "/", typically required when used +// as request parameter instead of the path (which is relative to some root url). +func (f *Fs) allocatePathRaw(file string, absolute bool) string { + prefix := "" + if absolute { + prefix = "/" + } + return path.Join(prefix, f.allocateEndpoint, f.opt.Enc.FromStandardPath(path.Join(f.root, file))) } // Jottacloud requires the grant_type 'refresh_token' string @@ -928,7 +940,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e return nil, err } f.user = cust.Username - f.setEndpointURL() + f.setEndpoints() if root != "" && !rootIsDir { // Check to see if the root actually an existing file @@ -1365,7 +1377,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap Parameters: url.Values{}, } - opts.Parameters.Set(method, "/"+path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, dest)))) + opts.Parameters.Set(method, f.filePathRaw(dest, true)) var resp *http.Response err = f.pacer.Call(func() (bool, error) { @@ -1478,7 +1490,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string return fs.ErrorDirExists } - _, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote) + _, err = f.copyOrMove(ctx, "mvDir", path.Join(f.fileEndpoint, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote) if err != nil { return fmt.Errorf("couldn't move directory: %w", err) @@ -1865,7 +1877,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op Created: fileDate, Modified: fileDate, Md5: md5String, - Path: path.Join(o.fs.allocateURL, o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))), + Path: o.fs.allocatePathRaw(o.remote, true), } // send it From e3d44612c1cf68d920cb090cd8d2f2380bee12f2 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 19 Jan 2022 10:26:05 +0100 Subject: [PATCH 014/560] jottacloud: error strings should not be capitalized --- backend/jottacloud/jottacloud.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 1271eda1af474..12fe1d255cb6d 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -834,12 +834,12 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth if ok { ver, err = strconv.Atoi(version) if err != nil { - return nil, nil, errors.New("Failed to parse config version") + return nil, nil, errors.New("failed to parse config version") } ok = (ver == configVersion) || (ver == legacyConfigVersion) } if !ok { - return nil, nil, errors.New("Outdated config - please reconfigure this backend") + return nil, nil, errors.New("outdated config - please reconfigure this backend") } baseClient := fshttp.NewClient(ctx) @@ -885,7 +885,7 @@ func getOAuthClient(ctx context.Context, name string, m configmap.Mapper) (oAuth // Create OAuth Client oAuthClient, ts, err = oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient) if err != nil { - return nil, nil, fmt.Errorf("Failed to configure Jottacloud oauth client: %w", err) + return nil, nil, fmt.Errorf("failed to configure Jottacloud oauth client: %w", err) } return oAuthClient, ts, nil } @@ -1172,7 +1172,7 @@ func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback if expected.Folders != actual.Folders || expected.Files != actual.Files { - return fmt.Errorf("Invalid result from listStream: expected[%#v] != actual[%#v]", expected, actual) + return fmt.Errorf("invalid result from listStream: expected[%#v] != actual[%#v]", expected, actual) } return nil } From ef089dd867e15e257efa7d635421c615a4dd15ea Mon Sep 17 00:00:00 2001 From: Noah Hsu Date: Thu, 9 Jun 2022 00:24:35 +0800 Subject: [PATCH 015/560] webdav: add SharePoint in other specific regions support --- backend/webdav/odrvcookie/fetch.go | 36 +++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/backend/webdav/odrvcookie/fetch.go b/backend/webdav/odrvcookie/fetch.go index f6b25d29eeedc..c3ade04e5d5a3 100644 --- a/backend/webdav/odrvcookie/fetch.go +++ b/backend/webdav/odrvcookie/fetch.go @@ -74,7 +74,7 @@ xmlns:u="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-util http://www.w3.org/2005/08/addressing/anonymous -https://login.microsoftonline.com/extSTS.srf +{{ .SPTokenURL }} @@ -159,11 +159,37 @@ func (ca *CookieAuth) getSPCookie(conf *SharepointSuccessResponse) (*CookieRespo return &cookieResponse, nil } +var spTokenURLMap = map[string]string{ + "com": "https://login.microsoftonline.com", + "cn": "https://login.chinacloudapi.cn", + "us": "https://login.microsoftonline.us", + "de": "https://login.microsoftonline.de", +} + +func getSPTokenURL(endpoint string) (string, error) { + spRoot, err := url.Parse(endpoint) + if err != nil { + return "", fmt.Errorf("error while parse endpoint: %w", err) + } + domains := strings.Split(spRoot.Host, ".") + tld := domains[len(domains)-1] + spTokenURL, ok := spTokenURLMap[tld] + if !ok { + return "", fmt.Errorf("error while get SPToken url, unsupported tld: %s", tld) + } + return spTokenURL + "/extSTS.srf", nil +} + func (ca *CookieAuth) getSPToken(ctx context.Context) (conf *SharepointSuccessResponse, err error) { + spTokenURL, err := getSPTokenURL(ca.endpoint) + if err != nil { + return nil, err + } reqData := map[string]interface{}{ - "Username": ca.user, - "Password": ca.pass, - "Address": ca.endpoint, + "Username": ca.user, + "Password": ca.pass, + "Address": ca.endpoint, + "SPTokenURL": spTokenURL, } t := template.Must(template.New("authXML").Parse(reqString)) @@ -175,7 +201,7 @@ func (ca *CookieAuth) getSPToken(ctx context.Context) (conf *SharepointSuccessRe // Create and execute the first request which returns an auth token for the sharepoint service // With this token we can authenticate on the login page and save the returned cookies - req, err := http.NewRequestWithContext(ctx, "POST", "https://login.microsoftonline.com/extSTS.srf", buf) + req, err := http.NewRequestWithContext(ctx, "POST", spTokenURL, buf) if err != nil { return nil, err } From 61a75bfe07c565fbbaa4f68f33a014cb6babae0b Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sat, 28 May 2022 13:52:48 +0200 Subject: [PATCH 016/560] config: add empty line between sections to improve readability See #6211 --- fs/config/ui.go | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/fs/config/ui.go b/fs/config/ui.go index 0e54394b13f1e..6dbe0c5d3fad0 100644 --- a/fs/config/ui.go +++ b/fs/config/ui.go @@ -338,6 +338,11 @@ func OkRemote(name string) bool { return false } +// newSection prints an empty line to separate sections +func newSection() { + fmt.Println() +} + // backendConfig configures the backend starting from the state passed in // // The is the user interface loop that drives the post configuration backend config. @@ -387,6 +392,7 @@ func backendConfig(ctx context.Context, name string, m configmap.Mapper, ri *fs. if out.State == "" { break } + newSection() } return nil } @@ -516,7 +522,7 @@ func NewRemote(ctx context.Context, name string) error { break } LoadedData().SetValue(name, "type", newType) - + newSection() _, err = CreateRemote(ctx, name, newType, nil, UpdateRemoteOpt{ All: true, }) @@ -527,13 +533,15 @@ func NewRemote(ctx context.Context, name string) error { SaveConfig() return nil } + newSection() return EditRemote(ctx, ri, name) } // EditRemote gets the user to edit a remote func EditRemote(ctx context.Context, ri *fs.RegInfo, name string) error { - ShowRemote(name) fmt.Printf("Edit remote\n") + ShowRemote(name) + newSection() for { _, err := UpdateRemote(ctx, name, nil, UpdateRemoteOpt{ All: true, @@ -626,29 +634,44 @@ func EditConfig(ctx context.Context) (err error) { } switch i := Command(what); i { case 'e': + newSection() name := ChooseRemote() + newSection() fs := mustFindByName(name) err = EditRemote(ctx, fs, name) if err != nil { return err } case 'n': - err = NewRemote(ctx, NewRemoteName()) + newSection() + name := NewRemoteName() + newSection() + err = NewRemote(ctx, name) if err != nil { return err } case 'd': + newSection() name := ChooseRemote() + newSection() DeleteRemote(name) case 'r': - RenameRemote(ChooseRemote()) + newSection() + name := ChooseRemote() + newSection() + RenameRemote(name) case 'c': - CopyRemote(ChooseRemote()) + newSection() + name := ChooseRemote() + newSection() + CopyRemote(name) case 's': + newSection() SetPassword() case 'q': return nil } + newSection() } } From b246584a02df46afece9b7f5d55df83fc9a31a86 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sat, 28 May 2022 15:53:47 +0200 Subject: [PATCH 017/560] config: more readable listing of remote options Differentiate output of 'config show remote' command from listing options as part of interactive config process for consistency: 'config show remote' consistent with 'config show', while listing in interactive config consistent with other output. See #6211 --- fs/config/ui.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/fs/config/ui.go b/fs/config/ui.go index 6dbe0c5d3fad0..73b2925b26a17 100644 --- a/fs/config/ui.go +++ b/fs/config/ui.go @@ -298,10 +298,8 @@ func mustFindByName(name string) *fs.RegInfo { return fs.MustFind(fsType) } -// ShowRemote shows the contents of the remote -func ShowRemote(name string) { - fmt.Printf("--------------------\n") - fmt.Printf("[%s]\n", name) +// printRemoteOptions prints the options of the remote +func printRemoteOptions(name string, prefix string, sep string) { fs := mustFindByName(name) for _, key := range LoadedData().GetKeyList(name) { isPassword := false @@ -313,17 +311,28 @@ func ShowRemote(name string) { } value := FileGet(name, key) if isPassword && value != "" { - fmt.Printf("%s = *** ENCRYPTED ***\n", key) + fmt.Printf("%s%s%s*** ENCRYPTED ***\n", prefix, key, sep) } else { - fmt.Printf("%s = %s\n", key, value) + fmt.Printf("%s%s%s%s\n", prefix, key, sep, value) } } - fmt.Printf("--------------------\n") +} + +// listRemoteOptions lists the options of the remote +func listRemoteOptions(name string) { + printRemoteOptions(name, "- ", ": ") +} + +// ShowRemote shows the contents of the remote in config file format +func ShowRemote(name string) { + fmt.Printf("[%s]\n", name) + printRemoteOptions(name, "", " = ") } // OkRemote prints the contents of the remote and ask if it is OK func OkRemote(name string) bool { - ShowRemote(name) + fmt.Printf("Configuration of %q remote:\n", name) + listRemoteOptions(name) switch i := CommandDefault([]string{"yYes this is OK", "eEdit this remote", "dDelete this remote"}, 0); i { case 'y': return true @@ -539,8 +548,8 @@ func NewRemote(ctx context.Context, name string) error { // EditRemote gets the user to edit a remote func EditRemote(ctx context.Context, ri *fs.RegInfo, name string) error { - fmt.Printf("Edit remote\n") - ShowRemote(name) + fmt.Printf("Edit existing %q remote with options:\n", name) + listRemoteOptions(name) newSection() for { _, err := UpdateRemote(ctx, name, nil, UpdateRemoteOpt{ From acd7ad9190cc11792dc7beeeb56d9f1aa31385ab Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 1 Jun 2022 21:50:20 +0200 Subject: [PATCH 018/560] config: ajust section headers to improve readability See #6211 --- fs/config/ui.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fs/config/ui.go b/fs/config/ui.go index 73b2925b26a17..60b5b3bc61e1e 100644 --- a/fs/config/ui.go +++ b/fs/config/ui.go @@ -285,6 +285,7 @@ func ShowRemotes() { func ChooseRemote() string { remotes := LoadedData().GetSectionList() sort.Strings(remotes) + fmt.Println("Select remote.") return Choose("remote", "value", remotes, nil, "", true, false) } @@ -331,8 +332,10 @@ func ShowRemote(name string) { // OkRemote prints the contents of the remote and ask if it is OK func OkRemote(name string) bool { - fmt.Printf("Configuration of %q remote:\n", name) + fmt.Println("Configuration complete.") + fmt.Println("Options:") listRemoteOptions(name) + fmt.Printf("Keep this %q remote?\n", name) switch i := CommandDefault([]string{"yYes this is OK", "eEdit this remote", "dDelete this remote"}, 0); i { case 'y': return true @@ -492,6 +495,7 @@ func ChooseOption(o *fs.Option, name string) string { // NewRemoteName asks the user for a name for a new remote func NewRemoteName() (name string) { for { + fmt.Println("Enter name for new remote.") fmt.Printf("name> ") name = ReadLine() if LoadedData().HasSection(name) { @@ -548,7 +552,7 @@ func NewRemote(ctx context.Context, name string) error { // EditRemote gets the user to edit a remote func EditRemote(ctx context.Context, ri *fs.RegInfo, name string) error { - fmt.Printf("Edit existing %q remote with options:\n", name) + fmt.Printf("Editing existing %q remote with options:\n", name) listRemoteOptions(name) newSection() for { From 08a897424bc8c2fe27a5d8c9477b3c2043708c5a Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 8 Jun 2022 18:29:28 +0100 Subject: [PATCH 019/560] Add Noah Hsu to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 23f53103628a0..8d5cbbac44fbe 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -605,3 +605,4 @@ put them back in again.` >}} * Nick * Jason Zheng * Matthew Vernon + * Noah Hsu From 6c2331ffd73096c3957844bd054fc94cdf2654e8 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 8 Jun 2022 18:01:06 +0100 Subject: [PATCH 020/560] mount: skip tests on CI even if >= 2 processors --- cmd/mount/mount_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/mount/mount_test.go b/cmd/mount/mount_test.go index 2c1e027f1ce06..64f53eadcb841 100644 --- a/cmd/mount/mount_test.go +++ b/cmd/mount/mount_test.go @@ -7,6 +7,7 @@ import ( "runtime" "testing" + "github.com/rclone/rclone/fstest/testy" "github.com/rclone/rclone/vfs/vfstest" ) @@ -14,5 +15,6 @@ func TestMount(t *testing.T) { if runtime.NumCPU() <= 2 { t.Skip("FIXME skipping mount tests as they lock up on <= 2 CPUs - See: https://github.com/rclone/rclone/issues/3154") } + testy.SkipUnreliable(t) vfstest.RunTests(t, false, mount) } From bb6edb3c39c31e194dfc4511057ebf6741fef68e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 8 Jun 2022 17:44:55 +0100 Subject: [PATCH 021/560] build: update dependencies Also: - azureblob: fix compile after API change in upstream library --- backend/azureblob/azureblob.go | 2 +- go.mod | 35 +++++----- go.sum | 122 +++++++++++++++++++++------------ 3 files changed, 97 insertions(+), 62 deletions(-) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index f94c1ea00f1bf..58f4552b9bd4f 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -1763,7 +1763,7 @@ func (o *Object) SetTier(tier string) error { blob := o.getBlobReference() ctx := context.Background() err := o.fs.pacer.Call(func() (bool, error) { - _, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{}) + _, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{}, azblob.RehydratePriorityStandard) return o.fs.shouldRetry(ctx, err) }) diff --git a/go.mod b/go.mod index f9364e3219928..75a242ff7a701 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.16 require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 github.com/Azure/azure-pipeline-go v0.2.3 - github.com/Azure/azure-storage-blob-go v0.14.0 - github.com/Azure/go-autorest/autorest/adal v0.9.18 + github.com/Azure/azure-storage-blob-go v0.15.0 + github.com/Azure/go-autorest/autorest/adal v0.9.20 github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5 github.com/Unknwon/goconfig v1.0.0 @@ -16,7 +16,7 @@ require ( github.com/anacrolix/dms v1.4.0 github.com/artyom/mtab v1.0.0 github.com/atotto/clipboard v0.1.4 - github.com/aws/aws-sdk-go v1.43.30 + github.com/aws/aws-sdk-go v1.44.29 github.com/buengese/sgzip v0.1.1 github.com/colinmarc/hdfs/v2 v2.3.0 github.com/coreos/go-semver v0.3.0 @@ -30,7 +30,7 @@ require ( github.com/iguanesolutions/go-systemd/v5 v5.1.0 github.com/jcmturner/gokrb5/v8 v8.4.2 github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 - github.com/klauspost/compress v1.15.1 + github.com/klauspost/compress v1.15.6 github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348 github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a github.com/mattn/go-colorable v0.1.12 @@ -42,15 +42,15 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/sftp v1.13.5-0.20211228200725-31aac3e1878d github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v1.12.1 + github.com/prometheus/client_golang v1.12.2 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 github.com/rfjakob/eme v1.1.2 - github.com/shirou/gopsutil/v3 v3.22.3 + github.com/shirou/gopsutil/v3 v3.22.5 github.com/sirupsen/logrus v1.8.1 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.7.2 github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8 github.com/winfsp/cgofuse v1.5.1-0.20220421173602-ce7e5a65cac7 github.com/xanzy/ssh-agent v0.3.1 @@ -58,16 +58,16 @@ require ( github.com/yunify/qingstor-sdk-go/v3 v3.2.0 go.etcd.io/bbolt v1.3.6 goftp.io/server v0.4.1 - golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 - golang.org/x/net v0.0.0-20220325170049-de3da57026de - golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f + golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e + golang.org/x/net v0.0.0-20220607020251-c690dde0001d + golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb + golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a golang.org/x/text v0.3.7 - golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 - google.golang.org/api v0.74.0 + golang.org/x/time v0.0.0-20220411224347-583f2d630306 + google.golang.org/api v0.83.0 gopkg.in/yaml.v2 v2.4.0 - storj.io/uplink v1.8.1 + storj.io/uplink v1.9.0 ) require ( @@ -76,7 +76,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af - golang.org/x/mobile v0.0.0-20220414153400-ce6a79cf6a13 - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd + golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 ) diff --git a/go.sum b/go.sum index ff01c201275dc..8d64ccdfae170 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,10 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0 h1:b1zWmYuuHz7gO9kDcM/EpHGr06UgsYNRpNJzI2kFiLM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -61,13 +63,13 @@ dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= -github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= +github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= +github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= -github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg= +github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= @@ -116,8 +118,8 @@ github.com/artyom/mtab v1.0.0 h1:r7OSVo5Jeqi8+LotZ0rT2kzfPIBp9KCpEJP8RQqGmSE= github.com/artyom/mtab v1.0.0/go.mod h1:EHpkp5OmPfS1yZX+/DFTztlJ9di5UzdDLX1/XzWPXw8= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/aws/aws-sdk-go v1.43.30 h1:Q3lgrX/tz/MkEiPVVQnOQThBAK2QC2SCTCKTD1mwGFA= -github.com/aws/aws-sdk-go v1.43.30/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.29 h1:53YWlelsMiYmGxuTRpAq7Xp+pE+0esAVqNFiNyekU+A= +github.com/aws/aws-sdk-go v1.44.29/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -147,6 +149,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/colinmarc/hdfs/v2 v2.3.0 h1:tMxOjXn6+7iPUlxAyup9Ha2hnmLe3Sv5DM2qqbSQ2VY= github.com/colinmarc/hdfs/v2 v2.3.0/go.mod h1:nsyY1uyQOomU34KVQk9Qb/lDJobN1MQ/9WS6IqcVZno= @@ -178,6 +181,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -265,8 +269,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -304,8 +309,10 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0 h1:s7jOdKSaksJVOxE0Y/S32otcfiP+UQ0cL8/GTKaONwE= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -373,8 +380,8 @@ github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= +github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348 h1:Lrn8srO9JDBCf2iPjqy62stl49UDwoOxZ9/NGVi+fnk= @@ -477,8 +484,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -515,8 +522,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil/v3 v3.22.3 h1:UebRzEomgMpv61e3hgD1tGooqX5trFbdU/ehphbHd00= -github.com/shirou/gopsutil/v3 v3.22.3/go.mod h1:D01hZJ4pVHPpCTZ3m3T2+wDF2YAGfd+H4ifUguaQzHM= +github.com/shirou/gopsutil/v3 v3.22.5 h1:atX36I/IXgFiB81687vSiBI5zrMsxcIBkP9cQMJQoJA= +github.com/shirou/gopsutil/v3 v3.22.5/go.mod h1:so9G9VzeHt/hsd0YwqprnjHnfARAUktauykSbr+y2gA= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -574,8 +581,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8 h1:IGJQmLBLYBdAknj21W3JsVof0yjEXfy1Q0K3YZebDOg= github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= @@ -610,8 +618,9 @@ github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ github.com/zeebo/admission/v3 v3.0.3/go.mod h1:2OWyAS5yo0Xvj2AEUosOjTUHxaY0oIIiCrXGKCYzWpo= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/errs v1.2.2 h1:5NFypMTuSdoySVTqlNs1dEoU21QVamMQJxW/Fii5O7g= github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs= +github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= github.com/zeebo/float16 v0.1.0/go.mod h1:fssGvvXu+XS8MH57cKmyrLB/cqioYeYX/2mXCN3a5wo= github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54/go.mod h1:EI8LcOBDlSL3POyqwC1eJhOYlMBMidES+613EtmmT5w= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= @@ -644,13 +653,14 @@ golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o= -golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -679,8 +689,8 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20220414153400-ce6a79cf6a13 h1:C4eR3yV9J3RkP8WKkcQzpcPyr/foOcUtxW72DvJEOlI= -golang.org/x/mobile v0.0.0-20220414153400-ce6a79cf6a13/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= +golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd h1:x1GptNaTtxPAlTVIAJk61fuXg0y17h09DTxyb+VNC/k= +golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -738,13 +748,17 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -764,8 +778,11 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a h1:qfl7ob3DIEs3Ml9oLuPwY2N04gymzAW04WsUQHIClgM= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -777,8 +794,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -821,7 +839,6 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -861,11 +878,14 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f h1:rlezHXNlxYWvBCzNses9Dlc7nGFaNMJeqLolcmQSSZY= -golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -880,8 +900,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydthsj6oOd6l9bpuFcNoUvTs= -golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= +golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -944,8 +964,10 @@ golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -983,8 +1005,11 @@ google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tD google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.83.0 h1:pMvST+6v+46Gabac4zlJlalxZjCeRcepwg2EdBU+nCc= +google.golang.org/api v0.83.0/go.mod h1:CNywQoj/AfhTw26ZWAa6LwOv+6WFxHmeLPZq2uncLZk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1065,8 +1090,15 @@ google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb h1:0m9wktIpOxGw+SSKmydXWB3Z3GTfcPP6+q75HCQa6HI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 h1:qRu95HZ148xXw+XeZ3dvqe85PxH4X8+jIo0iRPKcEnM= +google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1097,8 +1129,11 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1112,8 +1147,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1136,8 +1172,8 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1152,9 +1188,9 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= -storj.io/common v0.0.0-20220317162831-b0b4a044a95f h1:3Bl5reCTJF8Py4DLIj/drvLc91pCiTHOSMIfGAVtOXs= -storj.io/common v0.0.0-20220317162831-b0b4a044a95f/go.mod h1:xW3PPPGBo4bdMtEP9GREnmxQptmJNuDg1tEHcA4zqog= -storj.io/drpc v0.0.29 h1:Ihd4ls/JQFr0lctefie3iu+3QM4duccCKr9uMzf4sKY= -storj.io/drpc v0.0.29/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= -storj.io/uplink v1.8.1 h1:eu3lpnsgIyE7I9QrwqP1rsLUU8M+qSvUxSj5xjnKgOE= -storj.io/uplink v1.8.1/go.mod h1:2lc2LprGbajoDCV5/QIOzAJiFwK1Lrw8m60kx6UrtT8= +storj.io/common v0.0.0-20220414110316-a5cb7172d6bf h1:D5xZTDOlTTQWdAWeKKm2pFLcz1sceH+f/pVAcYB9jL8= +storj.io/common v0.0.0-20220414110316-a5cb7172d6bf/go.mod h1:LBJrpAqL4MNSrhGEwc8SJ+tIVtgfCtFEZqDy6/0j67A= +storj.io/drpc v0.0.30 h1:jqPe4T9KEu3CDBI05A2hCMgMSHLtd/E0N0yTF9QreIE= +storj.io/drpc v0.0.30/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= +storj.io/uplink v1.9.0 h1:Zg1kX1VqOQIKm0yAukteKpLuT68Be3euyNRML612ERM= +storj.io/uplink v1.9.0/go.mod h1:f6D8306j5mnRHnPDKWCiwtPM6ukyGg77to9LaAY9l6k= From 218bf2183de32d838bf20e987f2b24064625d2be Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 8 Jun 2022 20:47:58 +0200 Subject: [PATCH 022/560] docs: add missing backends from listing of optional features (Akamai, Koofr and Sia) --- docs/content/overview.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/content/overview.md b/docs/content/overview.md index b652ac7558a7b..c107723eab65c 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -446,8 +446,9 @@ upon backend-specific capabilities. | Name | Purge | Copy | Move | DirMove | CleanUp | ListR | StreamUpload | LinkSharing | About | EmptyDir | | ---------------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:|:------------:|:------------:|:-----:|:--------:| | 1Fichier | No | Yes | Yes | No | No | No | No | Yes | No | Yes | +| Akamai Netstorage | Yes | No | No | No | No | Yes | Yes | No | No | Yes | | Amazon Drive | Yes | No | Yes | Yes | No | No | No | No | No | Yes | -| Amazon S3 | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | +| Amazon S3 (or S3 compatible) | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | | Backblaze B2 | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | | Box | Yes | Yes | Yes | Yes | Yes ‡‡ | No | Yes | Yes | Yes | Yes | | Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | Yes | No | No | Yes | @@ -462,6 +463,7 @@ upon backend-specific capabilities. | Hubic | Yes † | Yes | No | No | No | Yes | Yes | No | Yes | No | | Internet Archive | No | Yes | No | No | Yes | Yes | No | Yes | Yes | No | | Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | +| Koofr | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | Yes | | Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | Mega | Yes | No | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | @@ -475,6 +477,7 @@ upon backend-specific capabilities. | QingStor | No | Yes | No | No | Yes | Yes | No | No | No | No | | Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | | SFTP | No | No | Yes | Yes | No | No | Yes | No | Yes | Yes | +| Sia | No | No | No | No | No | No | Yes | No | No | Yes | | SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes | | Storj | Yes † | No | Yes | No | No | Yes | Yes | No | No | No | | Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | From b4091f282aeeeeb96195cdd4e52b64ac738cdb15 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 27 Oct 2021 17:01:54 +0200 Subject: [PATCH 023/560] sftp: add support for about and hashsum on windows server Windows shells like cmd and powershell needs to use different quoting/escaping of strings and paths than the unix shell, and also absolute paths must be fixed by removing leading slash that the POSIX formatted paths have (e.g. /C:/Users does not work in shell, it must be converted to C:/Users). Tries to autodetect shell type (cmd, powershell, unix) on first use. Implemented default builtin powershell functions for hashsum and about when remote shell is powershell. See #5763 Fixes #5758 --- backend/sftp/sftp.go | 326 +++++++++++++++++++++++------ backend/sftp/sftp_internal_test.go | 41 +++- docs/content/sftp.md | 152 ++++++++++++-- 3 files changed, 433 insertions(+), 86 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 9f7de872526d4..a75d91902d796 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -39,6 +39,8 @@ import ( ) const ( + defaultShellType = "unix" + shellTypeNotSupported = "none" hashCommandNotSupported = "none" minSleep = 100 * time.Millisecond maxSleep = 2 * time.Second @@ -47,7 +49,9 @@ const ( ) var ( - currentUser = env.CurrentUser() + currentUser = env.CurrentUser() + posixWinAbsPathRegex = regexp.MustCompile(`^/[a-zA-Z]\:($|/)`) // E.g. "/C:" or anything starting with "/C:/" + unixShellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\n-]") ) func init() { @@ -148,16 +152,16 @@ If this is set and no password is supplied then rclone will: }, { Name: "path_override", Default: "", - Help: `Override path used by SSH connection. + Help: `Override path used by SSH shell commands. This allows checksum calculation when SFTP and SSH paths are different. This issue affects among others Synology NAS boxes. -Shared folders can be found in directories representing volumes +E.g. if shared folders can be found in directories representing volumes: rclone sync /home/local/directory remote:/directory --sftp-path-override /volume2/directory -Home directory can be found in a shared folder called "home" +E.g. if home directory can be found in a shared folder called "home": rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory`, Advanced: true, @@ -166,6 +170,26 @@ Home directory can be found in a shared folder called "home" Default: true, Help: "Set the modified time on the remote if set.", Advanced: true, + }, { + Name: "shell_type", + Default: "", + Help: "The type of SSH shell on remote server, if any.\n\nLeave blank for autodetect.", + Advanced: true, + Examples: []fs.OptionExample{ + { + Value: shellTypeNotSupported, + Help: "No shell access", + }, { + Value: "unix", + Help: "Unix shell", + }, { + Value: "powershell", + Help: "PowerShell", + }, { + Value: "cmd", + Help: "Windows Command Prompt", + }, + }, }, { Name: "md5sum_command", Default: "", @@ -270,6 +294,7 @@ type Options struct { AskPassword bool `config:"ask_password"` PathOverride string `config:"path_override"` SetModTime bool `config:"set_modtime"` + ShellType string `config:"shell_type"` Md5sumCommand string `config:"md5sum_command"` Sha1sumCommand string `config:"sha1sum_command"` SkipLinks bool `config:"skip_links"` @@ -286,6 +311,8 @@ type Fs struct { name string root string absRoot string + shellRoot string + shellType string opt Options // parsed options ci *fs.ConfigInfo // global config m configmap.Mapper // config @@ -542,7 +569,7 @@ func (f *Fs) drainPool(ctx context.Context) (err error) { f.drain.Stop() } if len(f.pool) != 0 { - fs.Debugf(f, "closing %d unused connections", len(f.pool)) + fs.Debugf(f, "Closing %d unused connections", len(f.pool)) } for i, c := range f.pool { if cErr := c.closed(); cErr == nil { @@ -739,7 +766,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // // Just send the password back for all questions func (f *Fs) keyboardInteractiveReponse(user, instruction string, questions []string, echos []bool, pass string) ([]string, error) { - fs.Debugf(f, "keyboard interactive auth requested") + fs.Debugf(f, "Keyboard interactive auth requested") answers := make([]string, len(questions)) for i := range answers { answers[i] = pass @@ -769,6 +796,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m f.name = name f.root = root f.absRoot = root + f.shellRoot = root f.opt = *opt f.m = m f.config = sshConfig @@ -778,7 +806,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m f.savedpswd = "" // set the pool drainer timer going if f.opt.IdleTimeout > 0 { - f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) }) + f.drain = time.AfterFunc(time.Duration(f.opt.IdleTimeout), func() { _ = f.drainPool(ctx) }) } f.features = (&fs.Features{ @@ -790,16 +818,59 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m if err != nil { return nil, fmt.Errorf("NewFs: %w", err) } - cwd, err := c.sftpClient.Getwd() - f.putSftpConnection(&c, nil) - if err != nil { - fs.Debugf(f, "Failed to read current directory - using relative paths: %v", err) - } else if !path.IsAbs(f.root) { - f.absRoot = path.Join(cwd, f.root) - fs.Debugf(f, "Using absolute root directory %q", f.absRoot) + // Check remote shell type, try to auto-detect if not configured and save to config for later + if f.opt.ShellType != "" { + f.shellType = f.opt.ShellType + fs.Debugf(f, "Shell type %q from config", f.shellType) + } else { + session, err := c.sshClient.NewSession() + if err != nil { + f.shellType = shellTypeNotSupported + fs.Debugf(f, "Failed to get shell session for shell type detection command: %v", err) + } else { + var stdout, stderr bytes.Buffer + session.Stdout = &stdout + session.Stderr = &stderr + shellCmd := "echo ${ShellId}%ComSpec%" + fs.Debugf(f, "Running shell type detection remote command: %s", shellCmd) + err = session.Run(shellCmd) + _ = session.Close() + if err != nil { + f.shellType = defaultShellType + fs.Debugf(f, "Remote command failed: %v (stdout=%v) (stderr=%v)", err, bytes.TrimSpace(stdout.Bytes()), bytes.TrimSpace(stderr.Bytes())) + } else { + outBytes := stdout.Bytes() + fs.Debugf(f, "Remote command result: %s", outBytes) + outString := string(bytes.TrimSpace(stdout.Bytes())) + if strings.HasPrefix(outString, "Microsoft.PowerShell") { // If PowerShell: "Microsoft.PowerShell%ComSpec%" + f.shellType = "powershell" + } else if !strings.HasSuffix(outString, "%ComSpec%") { // If Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe" + f.shellType = "cmd" + } else { // If Unix: "%ComSpec%" + f.shellType = "unix" + } + } + } + // Save permanently in config to avoid the extra work next time + fs.Debugf(f, "Shell type %q detected (set option shell_type to override)", f.shellType) + f.m.Set("shell_type", f.shellType) + } + // Ensure we have absolute path to root + // It appears that WS FTP doesn't like relative paths, + // and the openssh sftp tool also uses absolute paths. + if !path.IsAbs(f.root) { + path, err := c.sftpClient.RealPath(f.root) + if err != nil { + fs.Debugf(f, "Failed to resolve path - using relative paths: %v", err) + } else { + f.absRoot = path + fs.Debugf(f, "Relative path resolved to %q", f.absRoot) + } } + f.putSftpConnection(&c, err) if root != "" { - // Check to see if the root actually an existing file + // Check to see if the root is actually an existing file, + // and if so change the filesystem root to its parent directory. oldAbsRoot := f.absRoot remote := path.Base(root) f.root = path.Dir(root) @@ -807,20 +878,24 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m if f.root == "." { f.root = "" } - _, err := f.NewObject(ctx, remote) + _, err = f.NewObject(ctx, remote) if err != nil { - if err == fs.ErrorObjectNotFound || err == fs.ErrorIsDir { - // File doesn't exist so return old f - f.root = root - f.absRoot = oldAbsRoot - return f, nil + if err != fs.ErrorObjectNotFound && err != fs.ErrorIsDir { + return nil, err } - return nil, err + // File doesn't exist so keep the old f + f.root = root + f.absRoot = oldAbsRoot + err = nil + } else { + // File exists so change fs to point to the parent and return it with an error + err = fs.ErrorIsFile } - // return an error with an fs which points to the parent - return f, fs.ErrorIsFile + } else { + err = nil } - return f, nil + fs.Debugf(f, "Using root directory %q", f.absRoot) + return f, err } // Name returns the configured name of the file system @@ -1155,74 +1230,147 @@ func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) { // Hashes returns the supported hash types of the filesystem func (f *Fs) Hashes() hash.Set { ctx := context.TODO() - if f.opt.DisableHashCheck { - return hash.Set(hash.None) - } if f.cachedHashes != nil { return *f.cachedHashes } + hashSet := hash.NewHashSet() + f.cachedHashes = &hashSet + + if f.opt.DisableHashCheck || f.shellType == shellTypeNotSupported { + return hashSet + } + // look for a hash command which works - checkHash := func(commands []string, expected string, hashCommand *string, changed *bool) bool { + checkHash := func(hashType hash.Type, commands []struct{ hashFile, hashEmpty string }, expected string, hashCommand *string, changed *bool) bool { if *hashCommand == hashCommandNotSupported { return false } if *hashCommand != "" { return true } + fs.Debugf(f, "Checking default %v hash commands", hashType) *changed = true for _, command := range commands { - output, err := f.run(ctx, command) + output, err := f.run(ctx, command.hashEmpty) if err != nil { + fs.Debugf(f, "Hash command skipped: %v", err) continue } output = bytes.TrimSpace(output) - fs.Debugf(f, "checking %q command: %q", command, output) if parseHash(output) == expected { - *hashCommand = command + *hashCommand = command.hashFile + fs.Debugf(f, "Hash command accepted") return true } + fs.Debugf(f, "Hash command skipped: Wrong output") } *hashCommand = hashCommandNotSupported return false } changed := false - md5Works := checkHash([]string{"md5sum", "md5 -r", "rclone md5sum"}, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed) - sha1Works := checkHash([]string{"sha1sum", "sha1 -r", "rclone sha1sum"}, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed) + md5Commands := []struct { + hashFile, hashEmpty string + }{ + {"md5sum", "md5sum"}, + {"md5 -r", "md5 -r"}, + {"rclone md5sum", "rclone md5sum"}, + } + sha1Commands := []struct { + hashFile, hashEmpty string + }{ + {"sha1sum", "sha1sum"}, + {"sha1 -r", "sha1 -r"}, + {"rclone sha1sum", "rclone sha1sum"}, + } + if f.shellType == "powershell" { + md5Commands = append(md5Commands, struct { + hashFile, hashEmpty string + }{ + "&{param($Path);Get-FileHash -Algorithm MD5 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}", + "Get-FileHash -Algorithm MD5 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}", + }) + + sha1Commands = append(sha1Commands, struct { + hashFile, hashEmpty string + }{ + "&{param($Path);Get-FileHash -Algorithm SHA1 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower()) ${Path}\"}}", + "Get-FileHash -Algorithm SHA1 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}", + }) + } + + md5Works := checkHash(hash.MD5, md5Commands, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed) + sha1Works := checkHash(hash.SHA1, sha1Commands, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed) if changed { + // Save permanently in config to avoid the extra work next time + fs.Debugf(f, "Setting hash command for %v to %q (set sha1sum_command to override)", hash.MD5, f.opt.Md5sumCommand) f.m.Set("md5sum_command", f.opt.Md5sumCommand) + fs.Debugf(f, "Setting hash command for %v to %q (set md5sum_command to override)", hash.SHA1, f.opt.Sha1sumCommand) f.m.Set("sha1sum_command", f.opt.Sha1sumCommand) } - set := hash.NewHashSet() if sha1Works { - set.Add(hash.SHA1) + hashSet.Add(hash.SHA1) } if md5Works { - set.Add(hash.MD5) + hashSet.Add(hash.MD5) } - f.cachedHashes = &set - return set + return hashSet } // About gets usage stats func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { - escapedPath := shellEscape(f.root) - if f.opt.PathOverride != "" { - escapedPath = shellEscape(path.Join(f.opt.PathOverride, f.root)) + if f.shellType == shellTypeNotSupported || f.shellType == "cmd" { + fs.Debugf(f, "About shell command is not available for shell type %q (set option shell_type to override)", f.shellType) + return nil, fmt.Errorf("not supported with shell type %q", f.shellType) } - if len(escapedPath) == 0 { - escapedPath = "/" + aboutShellPath := f.remoteShellPath("") + if aboutShellPath == "" { + aboutShellPath = "/" } - stdout, err := f.run(ctx, "df -k "+escapedPath) + fs.Debugf(f, "About path %q", aboutShellPath) + aboutShellPathArg, err := f.quoteOrEscapeShellPath(aboutShellPath) if err != nil { + return nil, err + } + // PowerShell + if f.shellType == "powershell" { + shellCmd := "Get-Item " + aboutShellPathArg + " -ErrorAction Stop|Select-Object -First 1 -ExpandProperty PSDrive|ForEach-Object{\"$($_.Used) $($_.Free)\"}" + fs.Debugf(f, "About using shell command for shell type %q", f.shellType) + stdout, err := f.run(ctx, shellCmd) + if err != nil { + fs.Debugf(f, "About shell command for shell type %q failed (set option shell_type to override): %v", f.shellType, err) + return nil, fmt.Errorf("powershell command failed: %w", err) + } + split := strings.Fields(string(stdout)) + usage := &fs.Usage{} + if len(split) == 2 { + usedValue, usedErr := strconv.ParseInt(split[0], 10, 64) + if usedErr == nil { + usage.Used = fs.NewUsageValue(usedValue) + } + freeValue, freeErr := strconv.ParseInt(split[1], 10, 64) + if freeErr == nil { + usage.Free = fs.NewUsageValue(freeValue) + if usedErr == nil { + usage.Total = fs.NewUsageValue(usedValue + freeValue) + } + } + } + return usage, nil + } + // Unix/default shell + shellCmd := "df -k " + aboutShellPathArg + fs.Debugf(f, "About using shell command for shell type %q", f.shellType) + stdout, err := f.run(ctx, shellCmd) + if err != nil { + fs.Debugf(f, "About shell command for shell type %q failed (set option shell_type to override): %v", f.shellType, err) return nil, fmt.Errorf("your remote may not have the required df utility: %w", err) } - usageTotal, usageUsed, usageAvail := parseUsage(stdout) usage := &fs.Usage{} if usageTotal >= 0 { @@ -1287,31 +1435,78 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) { return "", hash.ErrUnsupported } - escapedPath := shellEscape(o.path()) - if o.fs.opt.PathOverride != "" { - escapedPath = shellEscape(path.Join(o.fs.opt.PathOverride, o.remote)) + shellPathArg, err := o.fs.quoteOrEscapeShellPath(o.shellPath()) + if err != nil { + return "", fmt.Errorf("failed to calculate %v hash: %w", r, err) } - b, err := o.fs.run(ctx, hashCmd+" "+escapedPath) + outBytes, err := o.fs.run(ctx, hashCmd+" "+shellPathArg) if err != nil { return "", fmt.Errorf("failed to calculate %v hash: %w", r, err) } - - str := parseHash(b) + hashString := parseHash(outBytes) + fs.Debugf(o, "Parsed hash: %s", hashString) if r == hash.MD5 { - o.md5sum = &str + o.md5sum = &hashString } else if r == hash.SHA1 { - o.sha1sum = &str + o.sha1sum = &hashString + } + return hashString, nil +} + +// quoteOrEscapeShellPath makes path a valid string argument in configured shell +// and also ensures it cannot cause unintended behavior. +func quoteOrEscapeShellPath(shellType string, shellPath string) (string, error) { + // PowerShell + if shellType == "powershell" { + return "'" + strings.ReplaceAll(shellPath, "'", "''") + "'", nil } - return str, nil + // Windows Command Prompt + if shellType == "cmd" { + if strings.Contains(shellPath, "\"") { + return "", fmt.Errorf("path is not valid in shell type %s: %s", shellType, shellPath) + } + return "\"" + shellPath + "\"", nil + } + // Unix shell + safe := unixShellEscapeRegex.ReplaceAllString(shellPath, `\$0`) + return strings.ReplaceAll(safe, "\n", "'\n'"), nil +} + +// quoteOrEscapeShellPath makes path a valid string argument in configured shell +func (f *Fs) quoteOrEscapeShellPath(shellPath string) (string, error) { + return quoteOrEscapeShellPath(f.shellType, shellPath) } -var shellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\n-]") +// remotePath returns the native SFTP path of the file or directory at the remote given +func (f *Fs) remotePath(remote string) string { + return path.Join(f.absRoot, remote) +} -// Escape a string s.t. it cannot cause unintended behavior -// when sending it to a shell. -func shellEscape(str string) string { - safe := shellEscapeRegex.ReplaceAllString(str, `\$0`) - return strings.ReplaceAll(safe, "\n", "'\n'") +// remoteShellPath returns the SSH shell path of the file or directory at the remote given +func (f *Fs) remoteShellPath(remote string) string { + if f.opt.PathOverride != "" { + shellPath := path.Join(f.opt.PathOverride, remote) + fs.Debugf(f, "Shell path redirected to %q with option path_override", shellPath) + return shellPath + } + shellPath := path.Join(f.absRoot, remote) + if f.shellType == "powershell" || f.shellType == "cmd" { + // If remote shell is powershell or cmd, then server is probably Windows. + // The sftp package converts everything to POSIX paths: Forward slashes, and + // absolute paths starts with a slash. An absolute path on a Windows server will + // then look like this "/C:/Windows/System32". We must remove the "/" prefix + // to make this a valid path for shell commands. In case of PowerShell there is a + // possibility that it is a Unix server, with PowerShell Core shell, but assuming + // root folders with names such as "C:" are rare, we just take this risk, + // and option path_override can always be used to work around corner cases. + if posixWinAbsPathRegex.MatchString(shellPath) { + shellPath = strings.TrimPrefix(shellPath, "/") + fs.Debugf(f, "Shell path adjusted to %q (set option path_override to override)", shellPath) + return shellPath + } + } + fs.Debugf(f, "Shell path %q", shellPath) + return shellPath } // Converts a byte array from the SSH session returned by @@ -1362,9 +1557,14 @@ func (o *Object) ModTime(ctx context.Context) time.Time { return o.modTime } -// path returns the native path of the object +// path returns the native SFTP path of the object func (o *Object) path() string { - return path.Join(o.fs.absRoot, o.remote) + return o.fs.remotePath(o.remote) +} + +// shellPath returns the SSH shell path of the object +func (o *Object) shellPath() string { + return o.fs.remoteShellPath(o.remote) } // setMetadata updates the info in the object from the stat result passed in diff --git a/backend/sftp/sftp_internal_test.go b/backend/sftp/sftp_internal_test.go index e73cc417efe18..0803321c8b947 100644 --- a/backend/sftp/sftp_internal_test.go +++ b/backend/sftp/sftp_internal_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestShellEscape(t *testing.T) { +func TestShellEscapeUnix(t *testing.T) { for i, test := range []struct { unescaped, escaped string }{ @@ -20,7 +20,44 @@ func TestShellEscape(t *testing.T) { {"/test/\n", "/test/'\n'"}, {":\"'", ":\\\"\\'"}, } { - got := shellEscape(test.unescaped) + got, err := quoteOrEscapeShellPath("unix", test.unescaped) + assert.NoError(t, err) + assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped)) + } +} + +func TestShellEscapeCmd(t *testing.T) { + for i, test := range []struct { + unescaped, escaped string + ok bool + }{ + {"", "\"\"", true}, + {"c:/this/is/harmless", "\"c:/this/is/harmless\"", true}, + {"c:/test¬epad", "\"c:/test¬epad\"", true}, + {"c:/test\"&\"notepad", "", false}, + } { + got, err := quoteOrEscapeShellPath("cmd", test.unescaped) + if test.ok { + assert.NoError(t, err) + assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped)) + } else { + assert.Error(t, err) + } + } +} + +func TestShellEscapePowerShell(t *testing.T) { + for i, test := range []struct { + unescaped, escaped string + }{ + {"", "''"}, + {"c:/this/is/harmless", "'c:/this/is/harmless'"}, + {"c:/test¬epad", "'c:/test¬epad'"}, + {"c:/test\"&\"notepad", "'c:/test\"&\"notepad'"}, + {"c:/test'&'notepad", "'c:/test''&''notepad'"}, + } { + got, err := quoteOrEscapeShellPath("powershell", test.unescaped) + assert.NoError(t, err) assert.Equal(t, test.escaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped)) } } diff --git a/docs/content/sftp.md b/docs/content/sftp.md index 5bc27dcd88fc7..ecd21899087e8 100644 --- a/docs/content/sftp.md +++ b/docs/content/sftp.md @@ -28,6 +28,9 @@ Note that some SFTP servers will need the leading / - Synology is a good example of this. rsync.net, on the other hand, requires users to OMIT the leading /. +Note that by default rclone will try to execute shell commands on +the server, see [shell access considerations](#shell-access-considerations). + ## Configuration Here is an example of making an SFTP configuration. First run @@ -244,6 +247,116 @@ And then at the end of the session These commands can be used in scripts of course. +### Shell access + +Some functionality of the SFTP backend relies on remote shell access, +and the possibility to execute commands. This includes [checksum](#checksum), +and in some cases also [about](#about-command). The shell commands that +must be executed may be different on different type of shells, and also +quoting/escaping of file path arguments containing special characters may +be different. Rclone therefore needs to know what type of shell it is, +and if shell access is available at all. + +Most servers run on some version of Unix, and then a basic Unix shell can +be assumed, without further distinction. Windows 10, Server 2019, and later +can also run a SSH server, which is a port of OpenSSH (see official +[installation guide](https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse)). On a Windows server the shell handling is different: Although it can also +be set up to use a Unix type shell, e.g. Cygwin bash, the default is to +use Windows Command Prompt (cmd.exe), and PowerShell is a recommended +alternative. All of these have bahave differently, which rclone must handle. + +Rclone tries to auto-detect what type of shell is used on the server, +first time you access the SFTP remote. If a remote shell session is +successfully created, it will look for indications that it is CMD or +PowerShell, with fall-back to Unix if not something else is detected. +If unable to even create a remote shell session, then shell command +execution will be disabled entirely. The result is stored in the SFTP +remote configuration, in option `shell_type`, so that the auto-detection +only have to be performed once. If you manually set a value for this +option before first run, the auto-detection will be skipped, and if +you set a different value later this will override any existing. +Value `none` can be set to avoid any attempts at executing shell +commands, e.g. if this is not allowed on the server. + +When the server is [rclone serve sftp](/commands/rclone_serve_sftp/), +the rclone SFTP remote will detect this as a Unix type shell - even +if it is running on Windows. This server does not actually have a shell, +but it accepts input commands matching the specific ones that the +SFTP backend relies on for Unix shells, e.g. `md5sum` and `df`. Also +it handles the string escape rules used for Unix shell. Treating it +as a Unix type shell from a SFTP remote will therefore always be +correct, and support all features. + +#### Shell access considerations + +The shell type auto-detection logic, described above, means that +by default rclone will try to run a shell command the first time +a new sftp remote is accessed. If you configure a sftp remote +without a config file, e.g. an [on the fly](/docs/#backend-path-to-dir]) +remote, rclone will have nowhere to store the result, and it +will re-run the command on every access. To avoid this you should +explicitely set the `shell_type` option to the correct value, +or to `none` if you want to prevent rclone from executing any +remote shell commands. + +It is also important to note that, since the shell type decides +how quoting and escaping of file paths used as command-line arguments +are performed, configuring the wrong shell type may leave you exposed +to command injection exploits. Make sure to confirm the auto-detected +shell type, or explicitely set the shell type you know is correct, +or disable shell access until you know. + +### Checksum + +SFTP does not natively support checksums (file hash), but rclone +is able to use checksumming if the same login has shell access, +and can execute remote commands. If there is a command that can +calculate compatible checksums on the remote system, Rclone can +then be configured to execute this whenever a checksum is needed, +and read back the results. Currently MD5 and SHA-1 are supported. + +Normally this requires an external utility being available on +the server. By default rclone will try commands `md5sum`, `md5` +and `rclone md5sum` for MD5 checksums, and the first one found usable +will be picked. Same with `sha1sum`, `sha1` and `rclone sha1sum` +commands for SHA-1 checksums. These utilities normally need to +be in the remote's PATH to be found. + +In some cases the shell itself is capable of calculating checksums. +PowerShell is an example of such a shell. If rclone detects that the +remote shell is PowerShell, which means it most probably is a +Windows OpenSSH server, rclone will use a predefined script block +to produce the checksums when no external checksum commands are found +(see [shell access](#shell-access)). This assumes PowerShell version +4.0 or newer. + +The options `md5sum_command` and `sha1_command` can be used to customize +the command to be executed for calculation of checksums. You can for +example set a specific path to where md5sum and sha1sum executables +are located, or use them to specify some other tools that print checksums +in compatible format. The value can include command-line arguments, +or even shell script blocks as with PowerShell. Rclone has subcommands +[md5sum](/commands/rclone_md5sum/) and [sha1sum](/commands/rclone_sha1sum/) +that use compatible format, which means if you have an rclone executable +on the server it can be used. As mentioned above, they will be automatically +picked up if found in PATH, but if not you can set something like +`/path/to/rclone md5sum` as the value of option `md5sum_command` to +make sure a specific executable is used. + +Remote checksumming is recommended and enabled by default. First time +rclone is using a SFTP remote, if options `md5sum_command` or `sha1_command` +are not set, it will check if any of the default commands for each of them, +as described above, can be used. The result will be saved in the remote +configuration, so next time it will use the same. Value `none` +will be set if none of the default commands could be used for a specific +algorithm, and this algorithm will not be supported by the remote. + +Disabling the checksumming may be required if you are connecting to SFTP servers +which are not under your control, and to which the execution of remote shell +commands is prohibited. Set the configuration option `disable_hashcheck` +to `true` to disable checksumming entirely, or set `shell_type` to `none` +to disable all functionality based on remote shell command execution. + ### Modified time Modified times are stored on the server to 1 second precision. @@ -255,6 +368,20 @@ upload (for example, certain configurations of ProFTPd with mod_sftp). If you are using one of these servers, you can set the option `set_modtime = false` in your RClone backend configuration to disable this behaviour. +### About command + +SFTP supports the [about](/commands/rclone_about/) command if the +same login has access to a Unix shell, where the `df` command is +available (e.g. in the remote's PATH). `about` will return the +total space, free space, and used space on the remote for the disk +of the specified path on the remote or, if not set, the disk of +the root on the remote. `about` will fail if it does not have +shell access or if `df` is not found. + +If the server shell is PowerShell, probably with a Windows OpenSSH +server, rclone supports `about` using a built-in shell command +(see [shell access](#shell-access)). + {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/sftp/sftp.go then run make backenddocs" >}} ### Standard options @@ -637,25 +764,9 @@ Properties: ## Limitations -SFTP supports checksums if the same login has shell access and `md5sum` -or `sha1sum` as well as `echo` are in the remote's PATH. -This remote checksumming (file hashing) is recommended and enabled by default. -Disabling the checksumming may be required if you are connecting to SFTP servers -which are not under your control, and to which the execution of remote commands -is prohibited. Set the configuration option `disable_hashcheck` to `true` to -disable checksumming. - -SFTP also supports `about` if the same login has shell -access and `df` are in the remote's PATH. `about` will -return the total space, free space, and used space on the remote -for the disk of the specified path on the remote or, if not set, -the disk of the root on the remote. -`about` will fail if it does not have shell -access or if `df` is not in the remote's PATH. - -Note that some SFTP servers (e.g. Synology) the paths are different for -SSH and SFTP so the hashes can't be calculated properly. For them -using `disable_hashcheck` is a good idea. +On some SFTP servers (e.g. Synology) the paths are different +for SSH and SFTP so the hashes can't be calculated properly. +For them using `disable_hashcheck` is a good idea. The only ssh agent supported under Windows is Putty's pageant. @@ -670,11 +781,10 @@ SFTP isn't supported under plan9 until [this issue](https://github.com/pkg/sftp/issues/156) is fixed. Note that since SFTP isn't HTTP based the following flags don't work -with it: `--dump-headers`, `--dump-bodies`, `--dump-auth` +with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`. Note that `--timeout` and `--contimeout` are both supported. - ## rsync.net {#rsync-net} rsync.net is supported through the SFTP backend. From 5db9a2f8313ce240daf0a1b2fa8c5337bc291c9d Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 31 Oct 2021 14:04:36 +0100 Subject: [PATCH 024/560] sftp: use vendor-specific VFS statistics extension for about if available See #5763 --- backend/sftp/sftp.go | 36 ++++++++++++++++++++++++++++++++++++ docs/content/sftp.md | 24 +++++++++++++----------- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index a75d91902d796..305fbde983925 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -1324,6 +1324,42 @@ func (f *Fs) Hashes() hash.Set { // About gets usage stats func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { + // If server implements the vendor-specific VFS statistics extension prefer that + // (OpenSSH implements it on using syscall.Statfs on Linux and API function GetDiskFreeSpace on Windows) + c, err := f.getSftpConnection(ctx) + if err != nil { + return nil, err + } + var vfsStats *sftp.StatVFS + if _, found := c.sftpClient.HasExtension("statvfs@openssh.com"); found { + fs.Debugf(f, "Server has VFS statistics extension") + aboutPath := f.absRoot + if aboutPath == "" { + aboutPath = "/" + } + fs.Debugf(f, "About path %q", aboutPath) + vfsStats, err = c.sftpClient.StatVFS(aboutPath) + } + f.putSftpConnection(&c, err) // Return to pool asap, if running shell command below it will be re-used + if vfsStats != nil { + total := vfsStats.TotalSpace() + free := vfsStats.FreeSpace() + used := total - free + return &fs.Usage{ + Total: fs.NewUsageValue(int64(total)), + Used: fs.NewUsageValue(int64(used)), + Free: fs.NewUsageValue(int64(free)), + }, nil + } else if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, err + } + fs.Debugf(f, "Failed to retrieve VFS statistics, trying shell command instead: %v", err) + } else { + fs.Debugf(f, "Server does not have the VFS statistics extension, trying shell command instead") + } + + // Fall back to shell command method if possible if f.shellType == shellTypeNotSupported || f.shellType == "cmd" { fs.Debugf(f, "About shell command is not available for shell type %q (set option shell_type to override)", f.shellType) return nil, fmt.Errorf("not supported with shell type %q", f.shellType) diff --git a/docs/content/sftp.md b/docs/content/sftp.md index ecd21899087e8..a0fc7271cb92e 100644 --- a/docs/content/sftp.md +++ b/docs/content/sftp.md @@ -370,17 +370,19 @@ your RClone backend configuration to disable this behaviour. ### About command -SFTP supports the [about](/commands/rclone_about/) command if the -same login has access to a Unix shell, where the `df` command is -available (e.g. in the remote's PATH). `about` will return the -total space, free space, and used space on the remote for the disk -of the specified path on the remote or, if not set, the disk of -the root on the remote. `about` will fail if it does not have -shell access or if `df` is not found. - -If the server shell is PowerShell, probably with a Windows OpenSSH -server, rclone supports `about` using a built-in shell command -(see [shell access](#shell-access)). +The `about` command returns the total space, free space, and used +space on the remote for the disk of the specified path on the remote or, +if not set, the disk of the root on the remote. + +SFTP usually supports the [about](/commands/rclone_about/) command, but +it depends on the server. If the server implements the vendor-specific +VFS statistics extension, which is normally the case with OpenSSH instances, +it will be used. If not, but the same login has access to a Unix shell, +where the `df` command is available (e.g. in the remote's PATH), then +this will be used instead. If the server shell is PowerShell, probably +with a Windows OpenSSH server, rclone will use a built-in shell command +(see [shell access](#shell-access)). If none of the above is applicable, +`about` will fail. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/sftp/sftp.go then run make backenddocs" >}} ### Standard options From 7c1f2d7c841b56136987a44ca58f5d0505ccb00d Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 9 Jun 2022 12:45:24 +0100 Subject: [PATCH 025/560] dirtree: fix tests with -fast-list In commit da404dc0f28e919c sync,copy: Fix --fast-list --create-empty-src-dirs and --exclude The fix caused DirTree.AddDir to be called with the root directory. This in turn caused a spurious directory entry in the DirTree which caused tests with the -fast-list flag to fail with directory not found errors. --- fs/dirtree/dirtree.go | 5 ++++- fs/dirtree/dirtree_test.go | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/fs/dirtree/dirtree.go b/fs/dirtree/dirtree.go index cb78ab9df6b34..2785dcbed2ec8 100644 --- a/fs/dirtree/dirtree.go +++ b/fs/dirtree/dirtree.go @@ -40,9 +40,12 @@ func (dt DirTree) Add(entry fs.DirEntry) { // this creates the directory itself if required // it doesn't create parents func (dt DirTree) AddDir(entry fs.DirEntry) { + dirPath := entry.Remote() + if dirPath == "" { + return + } dt.Add(entry) // create the directory itself if it doesn't exist already - dirPath := entry.Remote() if _, ok := dt[dirPath]; !ok { dt[dirPath] = nil } diff --git a/fs/dirtree/dirtree_test.go b/fs/dirtree/dirtree_test.go index b292ade0aa67e..d548d8c6f4728 100644 --- a/fs/dirtree/dirtree_test.go +++ b/fs/dirtree/dirtree_test.go @@ -51,6 +51,14 @@ func TestDirTreeAddDir(t *testing.T) { dir/subdir/ sausage/ dir/subdir/sausage/ +`, dt.String()) + d = mockdir.New("") + dt.AddDir(d) + assert.Equal(t, `/ + potato/ +dir/subdir/ + sausage/ +dir/subdir/sausage/ `, dt.String()) } From cfe0911e0d9fbfc8a931c981216021689e53595f Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 9 Jun 2022 13:19:11 +0100 Subject: [PATCH 026/560] sync: fix tests for overlapping with filter In commit 3ccf222acb9697c3 sync: overlap check is now filter-sensitive The tests were attempting to write invalid objects on some backends due to a leading / on the object name. This fix also adds a few more test cases and makes sure the tests can be run individually. --- fs/operations/operations_test.go | 3 +++ fs/sync/sync_test.go | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 83df26398a5e6..10475562aef2d 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1284,6 +1284,7 @@ func TestOverlappingFilterCheckWithoutFilter(t *testing.T) { expected bool }{ {"name", "root", true}, + {"name", "/root", true}, {"namey", "root", false}, {"name", "rooty", false}, {"namey", "rooty", false}, @@ -1320,10 +1321,12 @@ func TestOverlappingFilterCheckWithFilter(t *testing.T) { expected bool }{ {"name", "root", true}, + {"name", "/root", true}, {"name", "root/", true}, {"name", "root" + slash, true}, {"name", "root/exclude", false}, {"name", "root/exclude/", false}, + {"name", "/root/exclude/", false}, {"name", "root" + slash + "exclude", false}, {"name", "root" + slash + "exclude" + slash, false}, {"name", "root/.ignore", false}, diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 19e225fd92a76..4bf60c47121da 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -1469,18 +1469,20 @@ func TestSyncOverlapWithFilter(t *testing.T) { FremoteSync3, err := fs.NewFs(ctx, subRemoteName3) require.NoError(t, FremoteSync3.Mkdir(ctx, "")) require.NoError(t, err) - r.WriteObject(context.Background(), "/rclone-sync-test-ignore-file/.ignore", "-", t1) + r.WriteObject(context.Background(), "rclone-sync-test-ignore-file/.ignore", "-", t1) checkErr := func(err error) { require.Error(t, err) assert.True(t, fserrors.IsFatalError(err)) assert.Equal(t, fs.ErrorOverlapping.Error(), err.Error()) + accounting.GlobalStats().ResetCounters() } checkNoErr := func(err error) { require.NoError(t, err) } + accounting.GlobalStats().ResetCounters() checkNoErr(Sync(ctx, FremoteSync, r.Fremote, false)) checkErr(Sync(ctx, r.Fremote, FremoteSync, false)) checkErr(Sync(ctx, r.Fremote, r.Fremote, false)) From aeb5dc28926c5fedffa4b68c54eac8d8d26afcb7 Mon Sep 17 00:00:00 2001 From: buengese Date: Sun, 12 Jun 2022 15:28:36 +0200 Subject: [PATCH 027/560] docs/zoho: add a section explaining client_id setup --- docs/content/zoho.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/content/zoho.md b/docs/content/zoho.md index 27199b6a08a21..ba155dfed10f3 100644 --- a/docs/content/zoho.md +++ b/docs/content/zoho.md @@ -234,3 +234,15 @@ Properties: - Default: Del,Ctl,InvalidUtf8 {{< rem autogenerated options stop >}} + +## Setting up your own client_id + +For Zoho we advise you to set up your own client_id. To do so you have to complete the following steps. + +1. Log in to the [Zoho API Console](https://api-console.zoho.com) + +2. Create a new client of type "Server-based Application". The name and website don't matter, but you must add the redirect URL `http://localhost:53682/`. + +3. Once the client is created, you can go to the settings tab and enable it in other regions. + +The client id and client secret can now be used with rclone. From cee79f27eef840d5efa005be68b632c5de16b43b Mon Sep 17 00:00:00 2001 From: buengese Date: Sun, 12 Jun 2022 15:37:30 +0200 Subject: [PATCH 028/560] zoho: add Japan and China regions --- backend/zoho/zoho.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.go index 2132037c6f2e2..e0082cacd44d8 100644 --- a/backend/zoho/zoho.go +++ b/backend/zoho/zoho.go @@ -172,6 +172,12 @@ browser.`, }, { Value: "in", Help: "India", + }, { + Value: "jp", + Help: "Japan", + }, { + Value: "com.cn", + Help: "China", }, { Value: "com.au", Help: "Australia", From 3a20929db40508204ec88bef516fd2c397a51ecc Mon Sep 17 00:00:00 2001 From: buengese Date: Sun, 12 Jun 2022 21:35:26 +0200 Subject: [PATCH 029/560] uptobox: fix root path handling - fixes #5903 --- backend/uptobox/uptobox.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/uptobox/uptobox.go b/backend/uptobox/uptobox.go index 94ef96fd90dc7..5ffd4baf16905 100644 --- a/backend/uptobox/uptobox.go +++ b/backend/uptobox/uptobox.go @@ -163,7 +163,7 @@ func (f *Fs) splitPathFull(pth string) (string, string) { } // splitPath is modified splitPath version that doesn't include the seperator -// in the base part +// in the base path func (f *Fs) splitPath(pth string) (string, string) { // chop of any leading or trailing '/' pth = strings.Trim(pth, "/") @@ -201,7 +201,11 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe opt: *opt, pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))), } - f.root = root + if root == "/" || root == "." { + f.root = "" + } else { + f.root = root + } f.features = (&fs.Features{ DuplicateFiles: true, CanHaveEmptyDirectories: true, From 0f41e91d411fdfbd517a2c2a8f76bdaeb9b0ae47 Mon Sep 17 00:00:00 2001 From: Roberto Ricci Date: Tue, 9 Nov 2021 16:09:12 +0100 Subject: [PATCH 030/560] cmd/ncdu: display correct path in delete confirmation dialog If the remote on the command line is "remote:subdir", when deleting "filename", the confirmation message shows the path "remote:subdirfilename". Using fspath.JoinRootPath() fixes this. Also use this function and fs.ConfigString() in other parts of the file, since they are more appropriate. --- cmd/ncdu/ncdu.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go index 6359184b58130..4ef08f3468a3d 100644 --- a/cmd/ncdu/ncdu.go +++ b/cmd/ncdu/ncdu.go @@ -19,6 +19,7 @@ import ( "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/cmd/ncdu/scan" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/operations" "github.com/spf13/cobra" ) @@ -529,7 +530,7 @@ func (u *UI) delete() { } u.popupBox([]string{ "Delete this file?", - u.fsName + dirEntry.String()}) + fspath.JoinRootPath(u.fsName, dirEntry.String())}) } else { u.boxMenuHandler = func(f fs.Fs, p string, o int) (string, error) { if o != 1 { @@ -548,7 +549,7 @@ func (u *UI) delete() { u.popupBox([]string{ "Purge this directory?", "ALL files in it will be deleted", - u.fsName + dirEntry.String()}) + fspath.JoinRootPath(u.fsName, dirEntry.String())}) } } @@ -659,7 +660,7 @@ func (u *UI) sortCurrentDir() { func (u *UI) setCurrentDir(d *scan.Dir) { u.d = d u.entries = d.Entries() - u.path = path.Join(u.fsName, d.Path()) + u.path = fspath.JoinRootPath(u.fsName, d.Path()) u.sortCurrentDir() } @@ -742,7 +743,7 @@ func NewUI(f fs.Fs) *UI { f: f, path: "Waiting for root...", dirListHeight: 20, // updated in Draw - fsName: f.Name() + ":" + f.Root(), + fsName: fs.ConfigString(f), showGraph: true, showCounts: false, showDirAverageSize: false, From 5006ede266074f223243ab3f6f6adf68d817ab7c Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 8 Jun 2022 23:02:01 +0200 Subject: [PATCH 031/560] mailru: use variable type int instead of time.Duration for keeping number of seconds Fixes problem reported by staticcheck lint tool (v0.3.2): Poorly chosen name for variable of type time.Duration (ST1011). --- backend/mailru/mailru.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go index c1a51cf4af1f7..033f1d16d8a61 100644 --- a/backend/mailru/mailru.go +++ b/backend/mailru/mailru.go @@ -2277,7 +2277,7 @@ type serverPool struct { pool pendingServerMap mu sync.Mutex path string - expirySec time.Duration + expirySec int fs *Fs } @@ -2384,7 +2384,7 @@ func (p *serverPool) addServer(url string, now time.Time) { p.mu.Lock() defer p.mu.Unlock() - expiry := now.Add(p.expirySec * time.Second) + expiry := now.Add(time.Duration(p.expirySec) * time.Second) expiryStr := []byte("-") if p.fs.ci.LogLevel >= fs.LogLevelInfo { From 70d1d8d760f93e3fe08e78e9e2441335cef2eade Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 12 Jun 2022 12:37:00 +0200 Subject: [PATCH 032/560] mount: replace deprecated fuse.ENOSYS with syscall.ENOSYS --- cmd/mount/dir.go | 3 ++- cmd/mount/file.go | 9 +++++---- cmd/mount/fs.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/mount/dir.go b/cmd/mount/dir.go index 2acb0c48e5cba..c49373d59892a 100644 --- a/cmd/mount/dir.go +++ b/cmd/mount/dir.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "os" + "syscall" "time" "bazil.org/fuse" @@ -237,7 +238,7 @@ var _ fusefs.NodeLinker = (*Dir)(nil) // existing Node. Receiver must be a directory. func (d *Dir) Link(ctx context.Context, req *fuse.LinkRequest, old fusefs.Node) (newNode fusefs.Node, err error) { defer log.Trace(d, "req=%v, old=%v", req, old)("new=%v, err=%v", &newNode, &err) - return nil, fuse.ENOSYS + return nil, syscall.ENOSYS } // Check interface satisfied diff --git a/cmd/mount/file.go b/cmd/mount/file.go index 9e8b977db07d5..434ed753fe12f 100644 --- a/cmd/mount/file.go +++ b/cmd/mount/file.go @@ -5,6 +5,7 @@ package mount import ( "context" + "syscall" "time" "bazil.org/fuse" @@ -98,14 +99,14 @@ func (f *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) (err error) { // // If there is no xattr by that name, returns fuse.ErrNoXattr. func (f *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error { - return fuse.ENOSYS // we never implement this + return syscall.ENOSYS // we never implement this } var _ fusefs.NodeGetxattrer = (*File)(nil) // Listxattr lists the extended attributes recorded for the node. func (f *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error { - return fuse.ENOSYS // we never implement this + return syscall.ENOSYS // we never implement this } var _ fusefs.NodeListxattrer = (*File)(nil) @@ -113,7 +114,7 @@ var _ fusefs.NodeListxattrer = (*File)(nil) // Setxattr sets an extended attribute with the given name and // value for the node. func (f *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error { - return fuse.ENOSYS // we never implement this + return syscall.ENOSYS // we never implement this } var _ fusefs.NodeSetxattrer = (*File)(nil) @@ -122,7 +123,7 @@ var _ fusefs.NodeSetxattrer = (*File)(nil) // // If there is no xattr by that name, returns fuse.ErrNoXattr. func (f *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error { - return fuse.ENOSYS // we never implement this + return syscall.ENOSYS // we never implement this } var _ fusefs.NodeRemovexattrer = (*File)(nil) diff --git a/cmd/mount/fs.go b/cmd/mount/fs.go index 8e9dad7513a8a..43d37802765c3 100644 --- a/cmd/mount/fs.go +++ b/cmd/mount/fs.go @@ -98,7 +98,7 @@ func translateError(err error) error { case vfs.EROFS: return fuse.Errno(syscall.EROFS) case vfs.ENOSYS, fs.ErrorNotImplemented: - return fuse.ENOSYS + return syscall.ENOSYS case vfs.EINVAL: return fuse.Errno(syscall.EINVAL) } From afa30abd33d031076ec8ac6354d6d2dee7ee730a Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 12 Jun 2022 12:24:45 +0200 Subject: [PATCH 033/560] mount: remove legacy OS X remnants --- cmd/mount/dir.go | 1 - cmd/mount/file.go | 1 - cmd/mount/mount.go | 7 ------- 3 files changed, 9 deletions(-) diff --git a/cmd/mount/dir.go b/cmd/mount/dir.go index c49373d59892a..47ec4bf2da5d2 100644 --- a/cmd/mount/dir.go +++ b/cmd/mount/dir.go @@ -39,7 +39,6 @@ func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) (err error) { a.Atime = modTime a.Mtime = modTime a.Ctime = modTime - a.Crtime = modTime // FIXME include Valid so get some caching? // FIXME fs.Debugf(d.path, "Dir.Attr %+v", a) return nil diff --git a/cmd/mount/file.go b/cmd/mount/file.go index 434ed753fe12f..1a0ad6c2860f5 100644 --- a/cmd/mount/file.go +++ b/cmd/mount/file.go @@ -37,7 +37,6 @@ func (f *File) Attr(ctx context.Context, a *fuse.Attr) (err error) { a.Atime = modTime a.Mtime = modTime a.Ctime = modTime - a.Crtime = modTime a.Blocks = Blocks return nil } diff --git a/cmd/mount/mount.go b/cmd/mount/mount.go index d6fbf1a92e7a8..db0f23bd38245 100644 --- a/cmd/mount/mount.go +++ b/cmd/mount/mount.go @@ -27,7 +27,6 @@ func mountOptions(VFS *vfs.VFS, device string, opt *mountlib.Options) (options [ fuse.MaxReadahead(uint32(opt.MaxReadAhead)), fuse.Subtype("rclone"), fuse.FSName(device), - fuse.VolumeName(opt.VolumeName), // Options from benchmarking in the fuse module //fuse.MaxReadahead(64 * 1024 * 1024), @@ -105,12 +104,6 @@ func mount(VFS *vfs.VFS, mountpoint string, opt *mountlib.Options) (<-chan error errChan <- err }() - // check if the mount process has an error to report - <-c.Ready - if err := c.MountError; err != nil { - return nil, nil, err - } - unmount := func() error { // Shutdown the VFS filesys.VFS.Shutdown() From 74bd7f3381ed4322750a88e7e26e413b69e307da Mon Sep 17 00:00:00 2001 From: buengese Date: Mon, 13 Jun 2022 20:11:31 +0200 Subject: [PATCH 034/560] pcloud: fix about with no free space left --- backend/pcloud/pcloud.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index 82f9f42f53dbf..cb470599c9b9b 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -908,10 +908,14 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { if err != nil { return nil, err } + free := q.Quota - q.UsedQuota + if free < 0 { + free = 0 + } usage = &fs.Usage{ - Total: fs.NewUsageValue(q.Quota), // quota of bytes that can be used - Used: fs.NewUsageValue(q.UsedQuota), // bytes in use - Free: fs.NewUsageValue(q.Quota - q.UsedQuota), // bytes which can be uploaded before reaching the quota + Total: fs.NewUsageValue(q.Quota), // quota of bytes that can be used + Used: fs.NewUsageValue(q.UsedQuota), // bytes in use + Free: fs.NewUsageValue(free), // bytes which can be uploaded before reaching the quota } return usage, nil } From ec117593f18c8abbf00a6a9a067f576726bc7224 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 8 Jun 2022 22:25:17 +0200 Subject: [PATCH 035/560] Fix lint issues reported by staticcheck Used staticcheck 2022.1.2 (v0.3.2) See: staticcheck.io --- backend/amazonclouddrive/amazonclouddrive.go | 2 +- backend/b2/b2.go | 15 +++------- backend/box/box.go | 2 +- backend/cache/storage_memory.go | 5 +--- backend/cache/storage_persistent.go | 5 +--- backend/combine/combine.go | 2 +- backend/compress/compress.go | 2 +- backend/drive/drive.go | 14 ++++----- backend/filefabric/filefabric.go | 2 +- .../googlecloudstorage/googlecloudstorage.go | 2 +- backend/googlephotos/googlephotos_test.go | 2 +- backend/googlephotos/pattern_test.go | 2 +- backend/hdfs/object.go | 4 +-- backend/hubic/hubic.go | 2 +- backend/jottacloud/jottacloud.go | 5 +--- backend/local/about_windows.go | 6 +++- backend/mailru/mailru.go | 4 +-- backend/netstorage/netstorage.go | 30 +++++++++---------- backend/onedrive/api/types.go | 2 +- backend/onedrive/onedrive.go | 15 ++++++---- backend/opendrive/opendrive.go | 9 ++---- backend/pcloud/api/types.go | 2 +- backend/pcloud/pcloud.go | 2 +- backend/qingstor/qingstor.go | 7 +---- backend/s3/s3.go | 6 ++-- backend/seafile/seafile.go | 10 +++---- backend/sharefile/sharefile.go | 4 +-- backend/sia/sia.go | 4 +-- backend/sugarsync/sugarsync.go | 2 +- backend/swift/swift.go | 8 ++--- backend/uptobox/uptobox.go | 7 ++--- backend/webdav/webdav.go | 9 ++---- backend/yandex/api/types.go | 2 +- backend/yandex/yandex.go | 13 +++----- backend/zoho/zoho.go | 2 +- cmd/cmd.go | 4 +-- cmd/cryptdecode/cryptdecode.go | 4 +-- cmd/selfupdate/selfupdate.go | 8 ++--- cmd/serve/docker/docker.go | 2 +- cmd/serve/proxy/proxy.go | 4 +-- cmd/test/makefiles/makefiles.go | 1 - fs/accounting/accounting.go | 2 +- fs/accounting/stats_groups_test.go | 4 +-- fs/asyncreader/asyncreader.go | 4 +-- fs/config/configfile/configfile.go | 5 +--- fs/config/rc.go | 5 +--- fs/filter/filter.go | 2 +- fs/fserrors/error.go | 2 +- fs/fserrors/error_test.go | 2 +- fs/fspath/path.go | 2 +- fs/log/caller_hook.go | 2 +- fs/open_options.go | 4 +-- fs/rc/rcserver/rcserver.go | 2 +- fs/rc/webgui/plugins.go | 6 ++-- fstest/fstests/fstests.go | 5 ++-- lib/encoder/internal/gen/main.go | 8 ++--- lib/kv/internal_test.go | 2 -- lib/oauthutil/oauthutil.go | 3 +- lib/pacer/tokens.go | 1 - lib/version/version.go | 2 +- vfs/dir.go | 2 +- vfs/read.go | 2 +- vfs/vfscache/downloaders/downloaders.go | 2 +- vfs/vfscache/item.go | 4 +-- vfs/vfstest/write_non_unix.go | 5 +--- vfs/write.go | 4 +-- 66 files changed, 130 insertions(+), 185 deletions(-) diff --git a/backend/amazonclouddrive/amazonclouddrive.go b/backend/amazonclouddrive/amazonclouddrive.go index 58a14427bc4f4..e5f5512ed2794 100644 --- a/backend/amazonclouddrive/amazonclouddrive.go +++ b/backend/amazonclouddrive/amazonclouddrive.go @@ -435,7 +435,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, title string, directorie query += " AND kind:" + folderKind } else if filesOnly { query += " AND kind:" + fileKind - } else { + //} else { // FIXME none of these work //query += " AND kind:(" + fileKind + " OR " + folderKind + ")" //query += " AND (kind:" + fileKind + " OR kind:" + folderKind + ")" diff --git a/backend/b2/b2.go b/backend/b2/b2.go index ba3e03eb75895..322aa50a56efd 100644 --- a/backend/b2/b2.go +++ b/backend/b2/b2.go @@ -280,7 +280,7 @@ func (f *Fs) Root() string { // String converts this Fs to a string func (f *Fs) String() string { if f.rootBucket == "" { - return fmt.Sprintf("B2 root") + return "B2 root" } if f.rootDirectory == "" { return fmt.Sprintf("B2 bucket %s", f.rootBucket) @@ -1205,10 +1205,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error { } } var isUnfinishedUploadStale = func(timestamp api.Timestamp) bool { - if time.Since(time.Time(timestamp)).Hours() > 24 { - return true - } - return false + return time.Since(time.Time(timestamp)).Hours() > 24 } // Delete Config.Transfers in parallel @@ -1485,13 +1482,9 @@ func (o *Object) Size() int64 { // // Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html // Some tools (e.g. Cyberduck) use this -func cleanSHA1(sha1 string) (out string) { - out = strings.ToLower(sha1) +func cleanSHA1(sha1 string) string { const unverified = "unverified:" - if strings.HasPrefix(out, unverified) { - out = out[len(unverified):] - } - return out + return strings.TrimPrefix(strings.ToLower(sha1), unverified) } // decodeMetaDataRaw sets the metadata from the data passed in diff --git a/backend/box/box.go b/backend/box/box.go index bde259cead077..55d8a40303c08 100644 --- a/backend/box/box.go +++ b/backend/box/box.go @@ -897,7 +897,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, srcPath := srcObj.fs.rootSlash() + srcObj.remote dstPath := f.rootSlash() + remote - if strings.ToLower(srcPath) == strings.ToLower(dstPath) { + if strings.EqualFold(srcPath, dstPath) { return nil, fmt.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) } diff --git a/backend/cache/storage_memory.go b/backend/cache/storage_memory.go index 8e8a360fce5ef..25ac3bb6258f2 100644 --- a/backend/cache/storage_memory.go +++ b/backend/cache/storage_memory.go @@ -76,10 +76,7 @@ func (m *Memory) CleanChunksByAge(chunkAge time.Duration) { // CleanChunksByNeed will cleanup chunks after the FS passes a specific chunk func (m *Memory) CleanChunksByNeed(offset int64) { - var items map[string]cache.Item - - items = m.db.Items() - for key := range items { + for key := range m.db.Items() { sepIdx := strings.LastIndex(key, "-") keyOffset, err := strconv.ParseInt(key[sepIdx+1:], 10, 64) if err != nil { diff --git a/backend/cache/storage_persistent.go b/backend/cache/storage_persistent.go index 89af35f87342c..0de9a39633396 100644 --- a/backend/cache/storage_persistent.go +++ b/backend/cache/storage_persistent.go @@ -456,10 +456,7 @@ func (b *Persistent) HasEntry(remote string) bool { return fmt.Errorf("couldn't find object (%v)", remote) }) - if err == nil { - return true - } - return false + return err == nil } // HasChunk confirms the existence of a single chunk of an object diff --git a/backend/combine/combine.go b/backend/combine/combine.go index c3c5ff38a792d..934ed76f44ada 100644 --- a/backend/combine/combine.go +++ b/backend/combine/combine.go @@ -195,7 +195,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs if remote == "" { return fmt.Errorf("empty remote in upstream definition %q", upstream) } - if strings.IndexRune(dir, '/') >= 0 { + if strings.ContainsRune(dir, '/') { return fmt.Errorf("dirs can't contain / (yet): %q", dir) } u, err := f.newUpstream(gCtx, dir, remote) diff --git a/backend/compress/compress.go b/backend/compress/compress.go index 9656fb57308be..6d3397dfb0019 100644 --- a/backend/compress/compress.go +++ b/backend/compress/compress.go @@ -53,7 +53,7 @@ const ( Gzip = 2 ) -var nameRegexp = regexp.MustCompile("^(.+?)\\.([A-Za-z0-9-_]{11})$") +var nameRegexp = regexp.MustCompile(`^(.+?)\.([A-Za-z0-9-_]{11})$`) // Register with Fs func init() { diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 9aeeec5a04153..c5f5827f7b65b 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -2020,7 +2020,7 @@ func splitID(compositeID string) (actualID, shortcutID string) { // isShortcutID returns true if compositeID refers to a shortcut func isShortcutID(compositeID string) bool { - return strings.IndexRune(compositeID, shortcutSeparator) >= 0 + return strings.ContainsRune(compositeID, shortcutSeparator) } // actualID returns an actual ID from a composite ID @@ -3327,7 +3327,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str out["service_account_file"] = f.opt.ServiceAccountFile } if _, ok := opt["chunk_size"]; ok { - out["chunk_size"] = fmt.Sprintf("%s", f.opt.ChunkSize) + out["chunk_size"] = f.opt.ChunkSize.String() } return out, nil case "set": @@ -3344,11 +3344,11 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str } if chunkSize, ok := opt["chunk_size"]; ok { chunkSizeMap := make(map[string]string) - chunkSizeMap["previous"] = fmt.Sprintf("%s", f.opt.ChunkSize) + chunkSizeMap["previous"] = f.opt.ChunkSize.String() if err = f.changeChunkSize(chunkSize); err != nil { return out, err } - chunkSizeString := fmt.Sprintf("%s", f.opt.ChunkSize) + chunkSizeString := f.opt.ChunkSize.String() f.m.Set("chunk_size", chunkSizeString) chunkSizeMap["current"] = chunkSizeString out["chunk_size"] = chunkSizeMap @@ -3392,13 +3392,13 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str names[name] = struct{}{} lines = append(lines, "") lines = append(lines, fmt.Sprintf("[%s]", name)) - lines = append(lines, fmt.Sprintf("type = alias")) + lines = append(lines, "type = alias") lines = append(lines, fmt.Sprintf("remote = %s,team_drive=%s,root_folder_id=:", f.name, drive.Id)) upstreams = append(upstreams, fmt.Sprintf(`"%s=%s:"`, name, name)) } lines = append(lines, "") - lines = append(lines, fmt.Sprintf("[AllDrives]")) - lines = append(lines, fmt.Sprintf("type = combine")) + lines = append(lines, "[AllDrives]") + lines = append(lines, "type = combine") lines = append(lines, fmt.Sprintf("upstreams = %s", strings.Join(upstreams, " "))) return lines, nil } diff --git a/backend/filefabric/filefabric.go b/backend/filefabric/filefabric.go index 7f49579b479db..a70e1f7aea2c3 100644 --- a/backend/filefabric/filefabric.go +++ b/backend/filefabric/filefabric.go @@ -490,7 +490,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // Root is a dir - cache its ID f.dirCache.Put(f.root, info.ID) } - } else { + //} else { // Root is not found so a directory } } diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index 2f8a8856278e5..e010dfd88bdd7 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -391,7 +391,7 @@ func (f *Fs) Root() string { // String converts this Fs to a string func (f *Fs) String() string { if f.rootBucket == "" { - return fmt.Sprintf("GCS root") + return "GCS root" } if f.rootDirectory == "" { return fmt.Sprintf("GCS bucket %s", f.rootBucket) diff --git a/backend/googlephotos/googlephotos_test.go b/backend/googlephotos/googlephotos_test.go index c65a02700e2d5..11cb28e1bf271 100644 --- a/backend/googlephotos/googlephotos_test.go +++ b/backend/googlephotos/googlephotos_test.go @@ -37,7 +37,7 @@ func TestIntegration(t *testing.T) { } f, err := fs.NewFs(ctx, *fstest.RemoteName) if err == fs.ErrorNotFoundInConfigFile { - t.Skip(fmt.Sprintf("Couldn't create google photos backend - skipping tests: %v", err)) + t.Skipf("Couldn't create google photos backend - skipping tests: %v", err) } require.NoError(t, err) diff --git a/backend/googlephotos/pattern_test.go b/backend/googlephotos/pattern_test.go index 6d58e5475e036..b446615c22120 100644 --- a/backend/googlephotos/pattern_test.go +++ b/backend/googlephotos/pattern_test.go @@ -50,7 +50,7 @@ func (f *testLister) listAlbums(ctx context.Context, shared bool) (all *albums, // mock listUploads for testing func (f *testLister) listUploads(ctx context.Context, dir string) (entries fs.DirEntries, err error) { - entries, _ = f.uploaded[dir] + entries = f.uploaded[dir] return entries, nil } diff --git a/backend/hdfs/object.go b/backend/hdfs/object.go index d7b94afbd3bae..976bc7f02ea9c 100644 --- a/backend/hdfs/object.go +++ b/backend/hdfs/object.go @@ -115,7 +115,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return err } - info, err := o.fs.client.Stat(realpath) + _, err = o.fs.client.Stat(realpath) if err == nil { err = o.fs.client.Remove(realpath) if err != nil { @@ -147,7 +147,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return err } - info, err = o.fs.client.Stat(realpath) + info, err := o.fs.client.Stat(realpath) if err != nil { return err } diff --git a/backend/hubic/hubic.go b/backend/hubic/hubic.go index c655c038efa4d..5c284194eff0d 100644 --- a/backend/hubic/hubic.go +++ b/backend/hubic/hubic.go @@ -138,7 +138,7 @@ func (f *Fs) getCredentials(ctx context.Context) (err error) { return err } f.expires = expires - fs.Debugf(f, "Got swift credentials (expiry %v in %v)", f.expires, f.expires.Sub(time.Now())) + fs.Debugf(f, "Got swift credentials (expiry %v in %v)", f.expires, time.Until(f.expires)) return nil } diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 12fe1d255cb6d..735b5181fb080 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -1252,10 +1252,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . func (f *Fs) mkParentDir(ctx context.Context, dirPath string) error { // defer log.Trace(dirPath, "")("") // chop off trailing / if it exists - if strings.HasSuffix(dirPath, "/") { - dirPath = dirPath[:len(dirPath)-1] - } - parent := path.Dir(dirPath) + parent := path.Dir(strings.TrimSuffix(dirPath, "/")) if parent == "." { parent = "" } diff --git a/backend/local/about_windows.go b/backend/local/about_windows.go index d8daed7644fa2..f4fba6195a1ad 100644 --- a/backend/local/about_windows.go +++ b/backend/local/about_windows.go @@ -17,8 +17,12 @@ var getFreeDiskSpace = syscall.NewLazyDLL("kernel32.dll").NewProc("GetDiskFreeSp // About gets quota information func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { var available, total, free int64 + root, e := syscall.UTF16PtrFromString(f.root) + if e != nil { + return nil, fmt.Errorf("failed to read disk usage: %w", e) + } _, _, e1 := getFreeDiskSpace.Call( - uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(f.root))), + uintptr(unsafe.Pointer(root)), uintptr(unsafe.Pointer(&available)), // lpFreeBytesAvailable - for this user uintptr(unsafe.Pointer(&total)), // lpTotalNumberOfBytes uintptr(unsafe.Pointer(&free)), // lpTotalNumberOfFreeBytes diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go index 033f1d16d8a61..24693c67cfcb8 100644 --- a/backend/mailru/mailru.go +++ b/backend/mailru/mailru.go @@ -1723,7 +1723,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return err } - if bytes.Compare(fileHash, newHash) != 0 { + if !bytes.Equal(fileHash, newHash) { if o.fs.opt.CheckHash { return mrhash.ErrorInvalidHash } @@ -2262,7 +2262,7 @@ func (e *endHandler) handle(err error) error { } newHash := e.hasher.Sum(nil) - if bytes.Compare(o.mrHash, newHash) == 0 { + if bytes.Equal(o.mrHash, newHash) { return io.EOF } if o.fs.opt.CheckHash { diff --git a/backend/netstorage/netstorage.go b/backend/netstorage/netstorage.go index cbb2b04888b77..313679e272f8d 100755 --- a/backend/netstorage/netstorage.go +++ b/backend/netstorage/netstorage.go @@ -903,22 +903,20 @@ func (f *Fs) netStorageStatRequest(ctx context.Context, URL string, directory bo files = statResp.Files f.setStatCache(URL, files) } - if files != nil { - // Multiple objects can be returned with the "slash=both" option, - // when file/symlink/directory has the same name - for i := range files { - if files[i].Type == "symlink" { - // Add .rclonelink suffix to allow local backend code to convert to a symlink. - files[i].Name += ".rclonelink" - fs.Infof(nil, "Converting a symlink to the rclonelink on the stat request %s", files[i].Name) - } - entrywanted := (directory && files[i].Type == "dir") || - (!directory && files[i].Type != "dir") - if entrywanted { - filestamp := files[0] - files[0] = files[i] - files[i] = filestamp - } + // Multiple objects can be returned with the "slash=both" option, + // when file/symlink/directory has the same name + for i := range files { + if files[i].Type == "symlink" { + // Add .rclonelink suffix to allow local backend code to convert to a symlink. + files[i].Name += ".rclonelink" + fs.Infof(nil, "Converting a symlink to the rclonelink on the stat request %s", files[i].Name) + } + entrywanted := (directory && files[i].Type == "dir") || + (!directory && files[i].Type != "dir") + if entrywanted { + filestamp := files[0] + files[0] = files[i] + files[i] = filestamp } } return files, nil diff --git a/backend/onedrive/api/types.go b/backend/onedrive/api/types.go index 394403b0e58ac..236c1c68218b4 100644 --- a/backend/onedrive/api/types.go +++ b/backend/onedrive/api/types.go @@ -292,7 +292,7 @@ type AsyncOperationStatus struct { func (i *Item) GetID() string { if i.IsRemote() && i.RemoteItem.ID != "" { return i.RemoteItem.ParentReference.DriveID + "#" + i.RemoteItem.ID - } else if i.ParentReference != nil && strings.Index(i.ID, "#") == -1 { + } else if i.ParentReference != nil && !strings.Contains(i.ID, "#") { return i.ParentReference.DriveID + "#" + i.ID } return i.ID diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 132ed77e66a9c..a077843ed3f18 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -660,7 +660,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err } } case 401: - if len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { + if len(resp.Header["Www-Authenticate"]) == 1 && strings.Contains(resp.Header["Www-Authenticate"][0], "expired_token") { retry = true fs.Debugf(nil, "Should retry: %v", err) } else if err != nil && strings.Contains(err.Error(), "Unable to initialize RPS") { @@ -716,8 +716,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It firstSlashIndex := strings.IndexRune(path, '/') if f.driveType != driveTypePersonal || firstSlashIndex == -1 { - var opts rest.Opts - opts = f.newOptsCallWithPath(ctx, path, "GET", "") + opts := f.newOptsCallWithPath(ctx, path, "GET", "") opts.Path = strings.TrimSuffix(opts.Path, ":") err = f.pacer.Call(func() (bool, error) { resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) @@ -1287,7 +1286,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, if srcObj.fs == f { srcPath := srcObj.rootPath() dstPath := f.rootPath(remote) - if strings.ToLower(srcPath) == strings.ToLower(dstPath) { + if strings.EqualFold(srcPath, dstPath) { return nil, fmt.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) } } @@ -2185,7 +2184,7 @@ func (o *Object) ID() string { // Such a normalized ID can come from (*Item).GetID() func (f *Fs) parseNormalizedID(ID string) (string, string, string) { rootURL := graphAPIEndpoint[f.opt.Region] + "/v1.0/drives" - if strings.Index(ID, "#") >= 0 { + if strings.Contains(ID, "#") { s := strings.Split(ID, "#") return s[1], s[0], rootURL } @@ -2359,6 +2358,9 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken string, err error) { delta, err := f.changeNotifyNextChange(ctx, "latest") + if err != nil { + return + } parsedURL, err := url.Parse(delta.DeltaLink) if err != nil { return @@ -2388,6 +2390,9 @@ func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts { func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (nextDeltaToken string, err error) { delta, err := f.changeNotifyNextChange(ctx, deltaToken) + if err != nil { + return + } parsedURL, err := url.Parse(delta.DeltaLink) if err != nil { return diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.go index 96f29aa33de43..6e5967e626c1c 100644 --- a/backend/opendrive/opendrive.go +++ b/backend/opendrive/opendrive.go @@ -361,7 +361,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, srcPath := srcObj.fs.rootSlash() + srcObj.remote dstPath := f.rootSlash() + remote - if strings.ToLower(srcPath) == strings.ToLower(dstPath) { + if strings.EqualFold(srcPath, dstPath) { return nil, fmt.Errorf("Can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) } @@ -588,9 +588,6 @@ func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *Fold if err != nil { return nil, err } - if resp != nil { - } - return info, err } @@ -611,13 +608,13 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . return nil, err } - if "" == o.id { + if o.id == "" { // Attempt to read ID, ignore error // FIXME is this correct? _ = o.readMetaData(ctx) } - if "" == o.id { + if o.id == "" { // We need to create an ID for this file var resp *http.Response response := createFileResponse{} diff --git a/backend/pcloud/api/types.go b/backend/pcloud/api/types.go index 974d9f5f36fbd..5ae2b6b6bb494 100644 --- a/backend/pcloud/api/types.go +++ b/backend/pcloud/api/types.go @@ -136,7 +136,7 @@ func (g *GetFileLinkResult) IsValid() bool { if len(g.Hosts) == 0 { return false } - return time.Time(g.Expires).Sub(time.Now()) > 30*time.Second + return time.Until(time.Time(g.Expires)) > 30*time.Second } // URL returns a URL from the Path and Hosts. Check with IsValid diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index cb470599c9b9b..a73593b4059da 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -227,7 +227,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err } } - if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { + if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Contains(resp.Header["Www-Authenticate"][0], "expired_token") { doRetry = true fs.Debugf(nil, "Should retry: %v", err) } diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go index d4053e250ffca..5f5134099be09 100644 --- a/backend/qingstor/qingstor.go +++ b/backend/qingstor/qingstor.go @@ -573,9 +573,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck if addBucket { remote = path.Join(bucket, remote) } - if strings.HasSuffix(remote, "/") { - remote = remote[:len(remote)-1] - } + remote = strings.TrimSuffix(remote, "/") err = fn(remote, &qs.KeyType{Key: &remote}, true) if err != nil { return err @@ -775,8 +773,6 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error { retries++ wasDeleted = true continue - default: - break } break } @@ -854,7 +850,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error { continue default: err = e - break } } } else { diff --git a/backend/s3/s3.go b/backend/s3/s3.go index ea04ed653a2a4..0a7b216f488ea 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1976,7 +1976,7 @@ func (f *Fs) Root() string { // String converts this Fs to a string func (f *Fs) String() string { if f.rootBucket == "" { - return fmt.Sprintf("S3 root") + return "S3 root" } if f.rootDirectory == "" { return fmt.Sprintf("S3 bucket %s", f.rootBucket) @@ -2669,9 +2669,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck if addBucket { remote = path.Join(bucket, remote) } - if strings.HasSuffix(remote, "/") { - remote = remote[:len(remote)-1] - } + remote = strings.TrimSuffix(remote, "/") err = fn(remote, &s3.Object{Key: &remote}, true) if err != nil { return err diff --git a/backend/seafile/seafile.go b/backend/seafile/seafile.go index 76f9843abca50..15a7086411484 100644 --- a/backend/seafile/seafile.go +++ b/backend/seafile/seafile.go @@ -453,7 +453,7 @@ func (f *Fs) Root() string { // String converts this Fs to a string func (f *Fs) String() string { if f.libraryName == "" { - return fmt.Sprintf("seafile root") + return "seafile root" } library := "library" if f.encrypted { @@ -984,9 +984,9 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, if err != nil { return "", err } - if shareLinks != nil && len(shareLinks) > 0 { + if len(shareLinks) > 0 { for _, shareLink := range shareLinks { - if shareLink.IsExpired == false { + if !shareLink.IsExpired { return shareLink.Link, nil } } @@ -1053,7 +1053,7 @@ func (f *Fs) isLibraryInCache(libraryName string) bool { return false } value, found := f.libraries.GetMaybe(librariesCacheKey) - if found == false { + if !found { return false } libraries := value.([]api.Library) @@ -1130,7 +1130,7 @@ func (f *Fs) mkLibrary(ctx context.Context, libraryName, password string) error } // Stores the library details into the cache value, found := f.libraries.GetMaybe(librariesCacheKey) - if found == false { + if !found { // Don't update the cache at that point return nil } diff --git a/backend/sharefile/sharefile.go b/backend/sharefile/sharefile.go index 69da61cb9cef3..e6add47c2e6ac 100644 --- a/backend/sharefile/sharefile.go +++ b/backend/sharefile/sharefile.go @@ -1077,7 +1077,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj } dstLeaf = f.opt.Enc.FromStandardName(dstLeaf) - sameName := strings.ToLower(srcLeaf) == strings.ToLower(dstLeaf) + sameName := strings.EqualFold(srcLeaf, dstLeaf) if sameName && srcParentID == dstParentID { return nil, fmt.Errorf("copy: can't copy to a file in the same directory whose name only differs in case: %q vs %q", srcLeaf, dstLeaf) } @@ -1096,7 +1096,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj directCopy = true } else if err != nil { return nil, fmt.Errorf("copy: failed to examine destination dir: %w", err) - } else { + //} else { // otherwise need to copy via a temporary directory } } diff --git a/backend/sia/sia.go b/backend/sia/sia.go index 2017332ce1b7a..2a8235638f282 100644 --- a/backend/sia/sia.go +++ b/backend/sia/sia.go @@ -423,9 +423,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e return nil, err } - if strings.HasSuffix(opt.APIURL, "/") { - opt.APIURL = strings.TrimSuffix(opt.APIURL, "/") - } + opt.APIURL = strings.TrimSuffix(opt.APIURL, "/") // Parse the endpoint u, err := url.Parse(opt.APIURL) diff --git a/backend/sugarsync/sugarsync.go b/backend/sugarsync/sugarsync.go index 4b57f4184a8a8..2041bdc05c9f8 100644 --- a/backend/sugarsync/sugarsync.go +++ b/backend/sugarsync/sugarsync.go @@ -872,7 +872,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, srcPath := srcObj.fs.rootSlash() + srcObj.remote dstPath := f.rootSlash() + remote - if strings.ToLower(srcPath) == strings.ToLower(dstPath) { + if strings.EqualFold(srcPath, dstPath) { return nil, fmt.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) } diff --git a/backend/swift/swift.go b/backend/swift/swift.go index 5e3d96933d8f9..074e2daf7f9c6 100644 --- a/backend/swift/swift.go +++ b/backend/swift/swift.go @@ -268,7 +268,7 @@ func (f *Fs) Root() string { // String converts this Fs to a string func (f *Fs) String() string { if f.rootContainer == "" { - return fmt.Sprintf("Swift root") + return "Swift root" } if f.rootDirectory == "" { return fmt.Sprintf("Swift container %s", f.rootContainer) @@ -1271,7 +1271,7 @@ func (o *Object) getSegmentsLargeObject(ctx context.Context) (map[string][]strin if _, ok := containerSegments[segmentContainer]; !ok { containerSegments[segmentContainer] = make([]string, 0, len(segmentObjects)) } - segments, _ := containerSegments[segmentContainer] + segments := containerSegments[segmentContainer] segments = append(segments, segment.Name) containerSegments[segmentContainer] = segments } @@ -1363,7 +1363,7 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift. return } fs.Debugf(o, "Delete segments when err raise %v", err) - if segmentInfos == nil || len(segmentInfos) == 0 { + if len(segmentInfos) == 0 { return } _ctx := context.Background() @@ -1418,7 +1418,7 @@ func (o *Object) updateChunks(ctx context.Context, in0 io.Reader, headers swift. } func deleteChunks(ctx context.Context, o *Object, segmentsContainer string, segmentInfos []string) { - if segmentInfos == nil || len(segmentInfos) == 0 { + if len(segmentInfos) == 0 { return } for _, v := range segmentInfos { diff --git a/backend/uptobox/uptobox.go b/backend/uptobox/uptobox.go index 5ffd4baf16905..56c4f80a1ec4d 100644 --- a/backend/uptobox/uptobox.go +++ b/backend/uptobox/uptobox.go @@ -703,8 +703,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, } // copy the old object and apply the changes - var newObj Object - newObj = *srcObj + newObj := *srcObj newObj.remote = remote newObj.fs = f return &newObj, nil @@ -759,7 +758,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string } // check if the destination allready exists dstPath := f.dirPath(dstRemote) - dstInfo, err := f.readMetaDataForPath(ctx, dstPath, &api.MetadataRequestOptions{Limit: 1}) + _, err = f.readMetaDataForPath(ctx, dstPath, &api.MetadataRequestOptions{Limit: 1}) if err == nil { return fs.ErrorDirExists } @@ -772,7 +771,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string } // find the destination parent dir - dstInfo, err = f.readMetaDataForPath(ctx, dstBase, &api.MetadataRequestOptions{Limit: 1}) + dstInfo, err := f.readMetaDataForPath(ctx, dstBase, &api.MetadataRequestOptions{Limit: 1}) if err != nil { return fmt.Errorf("dirmove: failed to read destination: %w", err) } diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go index 64c3e983a7fb0..7016b289cd042 100644 --- a/backend/webdav/webdav.go +++ b/backend/webdav/webdav.go @@ -716,9 +716,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file subPath = f.opt.Enc.ToStandardPath(subPath) } remote := path.Join(dir, subPath) - if strings.HasSuffix(remote, "/") { - remote = remote[:len(remote)-1] - } + remote = strings.TrimSuffix(remote, "/") // the listing contains info about itself which we ignore if remote == dir { @@ -820,10 +818,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt func (f *Fs) mkParentDir(ctx context.Context, dirPath string) (err error) { // defer log.Trace(dirPath, "")("err=%v", &err) // chop off trailing / if it exists - if strings.HasSuffix(dirPath, "/") { - dirPath = dirPath[:len(dirPath)-1] - } - parent := path.Dir(dirPath) + parent := path.Dir(strings.TrimSuffix(dirPath, "/")) if parent == "." { parent = "" } diff --git a/backend/yandex/api/types.go b/backend/yandex/api/types.go index 978b1d00ce1d8..7e346b1ffe153 100644 --- a/backend/yandex/api/types.go +++ b/backend/yandex/api/types.go @@ -131,7 +131,7 @@ func (m *SortMode) String() string { // UnmarshalJSON sort mode func (m *SortMode) UnmarshalJSON(value []byte) error { - if value == nil || len(value) == 0 { + if len(value) == 0 { m.mode = "" return nil } diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.go index 265802cb5dda0..7eea7187f2223 100644 --- a/backend/yandex/yandex.go +++ b/backend/yandex/yandex.go @@ -468,7 +468,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) { } // If creating a directory with a : use (undocumented) disk: prefix - if strings.IndexRune(path, ':') >= 0 { + if strings.ContainsRune(path, ':') { path = "disk:" + path } opts.Parameters.Set("path", f.opt.Enc.FromStandardPath(path)) @@ -506,10 +506,8 @@ func (f *Fs) mkDirs(ctx context.Context, path string) (err error) { var mkdirpath = "/" //path separator / for _, element := range dirs { if element != "" { - mkdirpath += element + "/" //path separator / - if err = f.CreateDir(ctx, mkdirpath); err != nil { - // ignore errors while creating dirs - } + mkdirpath += element + "/" //path separator / + _ = f.CreateDir(ctx, mkdirpath) // ignore errors while creating dirs } } } @@ -522,10 +520,7 @@ func (f *Fs) mkDirs(ctx context.Context, path string) (err error) { func (f *Fs) mkParentDirs(ctx context.Context, resPath string) error { // defer log.Trace(dirPath, "")("") // chop off trailing / if it exists - if strings.HasSuffix(resPath, "/") { - resPath = resPath[:len(resPath)-1] - } - parent := path.Dir(resPath) + parent := path.Dir(strings.TrimSuffix(resPath, "/")) if parent == "." { parent = "" } diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.go index e0082cacd44d8..6175be3c5f327 100644 --- a/backend/zoho/zoho.go +++ b/backend/zoho/zoho.go @@ -287,7 +287,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err } authRetry := false - if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 { + if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Contains(resp.Header["Www-Authenticate"][0], "expired_token") { authRetry = true fs.Debugf(nil, "Should retry: %v", err) } diff --git a/cmd/cmd.go b/cmd/cmd.go index f2aea1fdf4714..82ae1ad838712 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -273,7 +273,7 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) { break } if retryAfter := accounting.GlobalStats().RetryAfter(); !retryAfter.IsZero() { - d := retryAfter.Sub(time.Now()) + d := time.Until(retryAfter) if d > 0 { fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d) time.Sleep(d) @@ -458,7 +458,7 @@ func initConfig() { }) } - if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); m == false { + if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); !m { fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.") ci.DataRateUnit = "bytes" } else { diff --git a/cmd/cryptdecode/cryptdecode.go b/cmd/cryptdecode/cryptdecode.go index c8859ebce7277..bdfd3e22127bd 100644 --- a/cmd/cryptdecode/cryptdecode.go +++ b/cmd/cryptdecode/cryptdecode.go @@ -75,7 +75,7 @@ func cryptDecode(cipher *crypt.Cipher, args []string) error { } } - fmt.Printf(output) + fmt.Print(output) return nil } @@ -89,7 +89,7 @@ func cryptEncode(cipher *crypt.Cipher, args []string) error { output += fmt.Sprintln(fileName, "\t", encryptedFileName) } - fmt.Printf(output) + fmt.Print(output) return nil } diff --git a/cmd/selfupdate/selfupdate.go b/cmd/selfupdate/selfupdate.go index f15bce83ce7dd..5bfd1cb667067 100644 --- a/cmd/selfupdate/selfupdate.go +++ b/cmd/selfupdate/selfupdate.go @@ -200,9 +200,7 @@ func InstallUpdate(ctx context.Context, opt *Options) error { savedFile := "" if runtime.GOOS == "windows" { savedFile = targetFile - if strings.HasSuffix(savedFile, ".exe") { - savedFile = savedFile[:len(savedFile)-4] - } + savedFile = strings.TrimSuffix(savedFile, ".exe") savedFile += ".old.exe" } @@ -319,9 +317,7 @@ func makeRandomExeName(baseName, extension string) (string, error) { const maxAttempts = 5 if runtime.GOOS == "windows" { - if strings.HasSuffix(baseName, ".exe") { - baseName = baseName[:len(baseName)-4] - } + baseName = strings.TrimSuffix(baseName, ".exe") extension += ".exe" } diff --git a/cmd/serve/docker/docker.go b/cmd/serve/docker/docker.go index f9f6fcd939be4..b9702d87f3203 100644 --- a/cmd/serve/docker/docker.go +++ b/cmd/serve/docker/docker.go @@ -20,7 +20,7 @@ var ( pluginName = "rclone" pluginScope = "local" baseDir = "/var/lib/docker-volumes/rclone" - sockDir = "/run/docker/plugins" + sockDir = "/run/docker/plugins" // location of unix sockets (only relevant on Linux and FreeBSD) defSpecDir = "/etc/docker/plugins" stateFile = "docker-plugin.state" socketAddr = "" // TCP listening address or empty string for Unix socket diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.go index 861861ee99d85..9e1d15878ba2f 100644 --- a/cmd/serve/proxy/proxy.go +++ b/cmd/serve/proxy/proxy.go @@ -156,11 +156,11 @@ func (p *Proxy) run(in map[string]string) (config configmap.Simple, err error) { fs.Debugf(nil, "Calling proxy %v", p.cmdLine) duration := time.Since(start) if err != nil { - return nil, fmt.Errorf("proxy: failed on %v: %q: %w", p.cmdLine, strings.TrimSpace(string(stderr.Bytes())), err) + return nil, fmt.Errorf("proxy: failed on %v: %q: %w", p.cmdLine, strings.TrimSpace(stderr.String()), err) } err = json.Unmarshal(stdout.Bytes(), &config) if err != nil { - return nil, fmt.Errorf("proxy: failed to read output: %q: %w", string(stdout.Bytes()), err) + return nil, fmt.Errorf("proxy: failed to read output: %q: %w", stdout.String(), err) } fs.Debugf(nil, "Proxy returned in %v", duration) diff --git a/cmd/test/makefiles/makefiles.go b/cmd/test/makefiles/makefiles.go index 83a1f5842099f..b778accd0785d 100644 --- a/cmd/test/makefiles/makefiles.go +++ b/cmd/test/makefiles/makefiles.go @@ -254,7 +254,6 @@ func (d *dir) createDirectories() { return } } - return } // list the directory hierarchy diff --git a/fs/accounting/accounting.go b/fs/accounting/accounting.go index e3ed4972c83da..53d6549fb65d0 100644 --- a/fs/accounting/accounting.go +++ b/fs/accounting/accounting.go @@ -444,7 +444,7 @@ func (acc *Account) speed() (bps, current float64) { return 0, 0 } // Calculate speed from first read. - total := float64(time.Now().Sub(acc.values.start)) / float64(time.Second) + total := float64(time.Since(acc.values.start)) / float64(time.Second) if total > 0 { bps = float64(acc.values.bytes) / total } else { diff --git a/fs/accounting/stats_groups_test.go b/fs/accounting/stats_groups_test.go index 2fd08c39df9a6..c111a08d80dc3 100644 --- a/fs/accounting/stats_groups_test.go +++ b/fs/accounting/stats_groups_test.go @@ -111,9 +111,9 @@ func TestStatsGroupOperations(t *testing.T) { runtime.GC() runtime.ReadMemStats(&end) - t.Log(fmt.Sprintf("%+v\n%+v", start, end)) + t.Logf("%+v\n%+v", start, end) diff := percentDiff(start.HeapObjects, end.HeapObjects) - if diff > 1 || diff < 0 { + if diff > 1 { t.Errorf("HeapObjects = %d, expected %d", end.HeapObjects, start.HeapObjects) } }) diff --git a/fs/asyncreader/asyncreader.go b/fs/asyncreader/asyncreader.go index 44f8fb24528c3..a39ae880724b4 100644 --- a/fs/asyncreader/asyncreader.go +++ b/fs/asyncreader/asyncreader.go @@ -68,8 +68,8 @@ func (a *AsyncReader) init(rd io.ReadCloser, buffers int) { a.in = rd a.ready = make(chan *buffer, buffers) a.token = make(chan struct{}, buffers) - a.exit = make(chan struct{}, 0) - a.exited = make(chan struct{}, 0) + a.exit = make(chan struct{}) + a.exited = make(chan struct{}) a.buffers = buffers a.cur = nil a.size = softStartInitial diff --git a/fs/config/configfile/configfile.go b/fs/config/configfile/configfile.go index 65971492d6cff..df01f25a5499e 100644 --- a/fs/config/configfile/configfile.go +++ b/fs/config/configfile/configfile.go @@ -187,10 +187,7 @@ func (s *Storage) Serialize() (string, error) { func (s *Storage) HasSection(section string) bool { s.check() _, err := s.gc.GetSection(section) - if err != nil { - return false - } - return true + return err == nil } // DeleteSection removes the named section and all config from the diff --git a/fs/config/rc.go b/fs/config/rc.go index fbbc7bdc9b2d3..db5d48ccaea32 100644 --- a/fs/config/rc.go +++ b/fs/config/rc.go @@ -72,10 +72,7 @@ See the [listremotes command](/commands/rclone_listremotes/) command for more in // Return the a list of remotes in the config file func rcListRemotes(ctx context.Context, in rc.Params) (out rc.Params, err error) { - var remotes = []string{} - for _, remote := range LoadedData().GetSectionList() { - remotes = append(remotes, remote) - } + remotes := LoadedData().GetSectionList() out = rc.Params{ "remotes": remotes, } diff --git a/fs/filter/filter.go b/fs/filter/filter.go index 37071f9d26646..e35018fa50112 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -596,7 +596,7 @@ func (f *Filter) UsesDirectoryFilters() bool { } rule := f.dirRules.rules[0] re := rule.Regexp.String() - if rule.Include == true && re == "^.*$" { + if rule.Include && re == "^.*$" { return false } return true diff --git a/fs/fserrors/error.go b/fs/fserrors/error.go index 4cdf7ac9d1ad9..12c971cd2c5ec 100644 --- a/fs/fserrors/error.go +++ b/fs/fserrors/error.go @@ -258,7 +258,7 @@ func NewErrorRetryAfter(d time.Duration) ErrorRetryAfter { // Error returns the textual version of the error func (e ErrorRetryAfter) Error() string { - return fmt.Sprintf("try again after %v (%v)", time.Time(e).Format(time.RFC3339Nano), time.Time(e).Sub(time.Now())) + return fmt.Sprintf("try again after %v (%v)", time.Time(e).Format(time.RFC3339Nano), time.Until(time.Time(e))) } // RetryAfter returns the time the operation should be retried at or diff --git a/fs/fserrors/error_test.go b/fs/fserrors/error_test.go index a0cf4e2e668f7..ec8426660af37 100644 --- a/fs/fserrors/error_test.go +++ b/fs/fserrors/error_test.go @@ -181,7 +181,7 @@ func TestShouldRetry(t *testing.T) { func TestRetryAfter(t *testing.T) { e := NewErrorRetryAfter(time.Second) after := e.RetryAfter() - dt := after.Sub(time.Now()) + dt := time.Until(after) assert.True(t, dt >= 900*time.Millisecond && dt <= 1100*time.Millisecond) assert.True(t, IsRetryAfterError(e)) assert.False(t, IsRetryAfterError(io.EOF)) diff --git a/fs/fspath/path.go b/fs/fspath/path.go index 99bdbe28b6315..285d843fff760 100644 --- a/fs/fspath/path.go +++ b/fs/fspath/path.go @@ -99,7 +99,7 @@ func Parse(path string) (parsed Parsed, err error) { return parsed, errCantBeEmpty } // If path has no `:` in, it must be a local path - if strings.IndexRune(path, ':') < 0 { + if !strings.ContainsRune(path, ':') { return parsed, nil } // States for parser diff --git a/fs/log/caller_hook.go b/fs/log/caller_hook.go index 7a72999e4fedf..13941bc02308e 100644 --- a/fs/log/caller_hook.go +++ b/fs/log/caller_hook.go @@ -45,7 +45,7 @@ func findCaller(skip int) string { line := 0 for i := 0; i < 10; i++ { file, line = getCaller(skip + i) - if !strings.HasPrefix(file, "logrus") && strings.Index(file, "log.go") < 0 { + if !strings.HasPrefix(file, "logrus") && !strings.Contains(file, "log.go") { break } } diff --git a/fs/open_options.go b/fs/open_options.go index 84e43045c20d2..938e38d04fd6f 100644 --- a/fs/open_options.go +++ b/fs/open_options.go @@ -75,7 +75,7 @@ func ParseRangeOption(s string) (po *RangeOption, err error) { return nil, errors.New("Range: header invalid: doesn't start with " + preamble) } s = s[len(preamble):] - if strings.IndexRune(s, ',') >= 0 { + if strings.ContainsRune(s, ',') { return nil, errors.New("Range: header invalid: contains multiple ranges which isn't supported") } dash := strings.IndexRune(s, '-') @@ -250,7 +250,7 @@ func (o NullOption) Header() (key string, value string) { // String formats the option into human-readable form func (o NullOption) String() string { - return fmt.Sprintf("NullOption()") + return "NullOption()" } // Mandatory returns whether the option must be parsed or can be ignored diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.go index e2f8bd22ee80f..0d246a2f60cdf 100644 --- a/fs/rc/rcserver/rcserver.go +++ b/fs/rc/rcserver/rcserver.go @@ -378,7 +378,7 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string) if s.opt.WebUI { pluginsMatchResult := webgui.PluginsMatch.FindStringSubmatch(path) - if pluginsMatchResult != nil && len(pluginsMatchResult) > 2 { + if len(pluginsMatchResult) > 2 { ok := webgui.ServePluginOK(w, r, pluginsMatchResult) if !ok { r.URL.Path = fmt.Sprintf("/%s/%s/app/build/%s", pluginsMatchResult[1], pluginsMatchResult[2], pluginsMatchResult[3]) diff --git a/fs/rc/webgui/plugins.go b/fs/rc/webgui/plugins.go index 3a97fd6795a76..8f212be582e5e 100644 --- a/fs/rc/webgui/plugins.go +++ b/fs/rc/webgui/plugins.go @@ -293,7 +293,7 @@ func ServePluginOK(w http.ResponseWriter, r *http.Request, pluginsMatchResult [] return true } -var referrerPathReg = regexp.MustCompile("^(https?):\\/\\/(.+):([0-9]+)?\\/(.*)\\/?\\?(.*)$") +var referrerPathReg = regexp.MustCompile(`^(https?):\/\/(.+):([0-9]+)?\/(.*)\/?\?(.*)$`) // ServePluginWithReferrerOK check if redirectReferrer is set for the referred a plugin, if yes, // sends a redirect to actual url. This function is useful for plugins to refer to absolute paths when @@ -306,9 +306,9 @@ func ServePluginWithReferrerOK(w http.ResponseWriter, r *http.Request, path stri referrer := r.Referer() referrerPathMatch := referrerPathReg.FindStringSubmatch(referrer) - if referrerPathMatch != nil && len(referrerPathMatch) > 3 { + if len(referrerPathMatch) > 3 { referrerPluginMatch := PluginsMatch.FindStringSubmatch(referrerPathMatch[4]) - if referrerPluginMatch != nil && len(referrerPluginMatch) > 2 { + if len(referrerPluginMatch) > 2 { pluginKey := fmt.Sprintf("%s/%s", referrerPluginMatch[1], referrerPluginMatch[2]) currentPlugin, err := loadedPlugins.GetPluginByName(pluginKey) if err != nil { diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 11cf4dc8bb994..58481416a56b9 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -387,8 +387,7 @@ func Run(t *testing.T, opt *Opt) { // Skip if remote is not SetTier and GetTier capable skipIfNotSetTier := func(t *testing.T) { skipIfNotOk(t) - if f.Features().SetTier == false || - f.Features().GetTier == false { + if !f.Features().SetTier || !f.Features().GetTier { t.Skip("FS has no SetTier & GetTier interfaces") } } @@ -735,7 +734,7 @@ func Run(t *testing.T, opt *Opt) { TestPutLarge(ctx, t, f, &fstest.Item{ ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"), - Path: fmt.Sprintf("zero-length-file"), + Path: "zero-length-file", Size: int64(0), }) }) diff --git a/lib/encoder/internal/gen/main.go b/lib/encoder/internal/gen/main.go index 38d33e1caf494..a94c3793580df 100644 --- a/lib/encoder/internal/gen/main.go +++ b/lib/encoder/internal/gen/main.go @@ -465,17 +465,13 @@ func getMapping(mask encoder.MultiEncoder) mapping { } func collectEncodables(m []mapping) (out []rune) { for _, s := range m { - for _, r := range s.src { - out = append(out, r) - } + out = append(out, s.src...) } return } func collectEncoded(m []mapping) (out []rune) { for _, s := range m { - for _, r := range s.dst { - out = append(out, r) - } + out = append(out, s.dst...) } return } diff --git a/lib/kv/internal_test.go b/lib/kv/internal_test.go index ae40c6ed4a11a..1c0834fbba5c9 100644 --- a/lib/kv/internal_test.go +++ b/lib/kv/internal_test.go @@ -17,7 +17,6 @@ func TestKvConcurrency(t *testing.T) { require.Equal(t, 0, len(dbMap), "no databases can be started initially") const threadNum = 5 - const facility = "test" var wg sync.WaitGroup ctx := context.Background() results := make([]*DB, threadNum) @@ -55,7 +54,6 @@ func TestKvConcurrency(t *testing.T) { func TestKvExit(t *testing.T) { require.Equal(t, 0, len(dbMap), "no databases can be started initially") const dbNum = 5 - const openNum = 2 ctx := context.Background() for i := 0; i < dbNum; i++ { facility := fmt.Sprintf("test-%d", i) diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go index 56c936d712c9d..53a08900c0bcc 100644 --- a/lib/oauthutil/oauthutil.go +++ b/lib/oauthutil/oauthutil.go @@ -280,7 +280,7 @@ func (ts *TokenSource) timeToExpiry() time.Duration { if t.Expiry.IsZero() { return 3e9 * time.Second // ~95 years } - return t.Expiry.Sub(time.Now()) + return time.Until(t.Expiry) } // OnExpiry returns a channel which has the time written to it when @@ -766,7 +766,6 @@ func (s *authServer) Init() error { } fs.Debugf(nil, "Redirecting browser to: %s", s.authURL) http.Redirect(w, req, s.authURL, http.StatusTemporaryRedirect) - return }) mux.HandleFunc("/", s.handleAuth) diff --git a/lib/pacer/tokens.go b/lib/pacer/tokens.go index b4f905ba9be85..d1e1e7cea7559 100644 --- a/lib/pacer/tokens.go +++ b/lib/pacer/tokens.go @@ -22,7 +22,6 @@ func NewTokenDispenser(n int) *TokenDispenser { // Get gets a token from the pool - don't forget to return it with Put func (td *TokenDispenser) Get() { <-td.tokens - return } // Put returns a token diff --git a/lib/version/version.go b/lib/version/version.go index 0aebe3d61dbfa..e8a4c17f6bdbd 100644 --- a/lib/version/version.go +++ b/lib/version/version.go @@ -11,7 +11,7 @@ import ( const versionFormat = "-v2006-01-02-150405.000" -var versionRegexp = regexp.MustCompile("-v\\d{4}-\\d{2}-\\d{2}-\\d{6}-\\d{3}") +var versionRegexp = regexp.MustCompile(`-v\d{4}-\d{2}-\d{2}-\d{6}-\d{3}`) // Split fileName into base and extension so that base + ext == fileName func splitExt(fileName string) (base, ext string) { diff --git a/vfs/dir.go b/vfs/dir.go index 2b2eda5a29e58..004595b77435b 100644 --- a/vfs/dir.go +++ b/vfs/dir.go @@ -506,7 +506,7 @@ func (d *Dir) _purgeVirtual() { // if remote can have empty directories then a // new dir will be read in the listing d._deleteVirtual(name) - } else { + //} else { // leave the empty directory marker } case vAddFile: diff --git a/vfs/read.go b/vfs/read.go index a128b03133610..88bcabfce2109 100644 --- a/vfs/read.go +++ b/vfs/read.go @@ -477,7 +477,7 @@ func (fh *ReadFileHandle) Release() error { err := fh.close() if err != nil { fs.Errorf(fh.remote, "ReadFileHandle.Release error: %v", err) - } else { + //} else { // fs.Debugf(fh.remote, "ReadFileHandle.Release OK") } return err diff --git a/vfs/vfscache/downloaders/downloaders.go b/vfs/vfscache/downloaders/downloaders.go index c6d769a4fbfc3..42ca677fd8931 100644 --- a/vfs/vfscache/downloaders/downloaders.go +++ b/vfs/vfscache/downloaders/downloaders.go @@ -359,7 +359,7 @@ func (dls *Downloaders) _ensureDownloader(r ranges.Range) (err error) { return nil } // Downloader not found so start a new one - dl, err = dls._newDownloader(r) + _, err = dls._newDownloader(r) if err != nil { dls._countErrors(0, err) return fmt.Errorf("failed to start downloader: %w", err) diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go index d61d6062adf25..d6a6e20b8d215 100644 --- a/vfs/vfscache/item.go +++ b/vfs/vfscache/item.go @@ -779,7 +779,7 @@ func (item *Item) _checkObject(o fs.Object) error { } else { fs.Debugf(item.name, "vfs cache: remote object has gone but local object modified - keeping it") } - } else { + //} else { // no remote object && no local object // OK } @@ -946,7 +946,7 @@ func (item *Item) Reset() (rr ResetResult, spaceFreed int64, err error) { /* Do not need to reset an empty cache file unless it was being reset and the reset failed. Some thread(s) may be waiting on the reset's succesful completion in that case. */ - if item.info.Rs.Size() == 0 && item.beingReset == false { + if item.info.Rs.Size() == 0 && !item.beingReset { return SkippedEmpty, 0, nil } diff --git a/vfs/vfstest/write_non_unix.go b/vfs/vfstest/write_non_unix.go index 84b5bf00a4421..637d5b5bd43a6 100644 --- a/vfs/vfstest/write_non_unix.go +++ b/vfs/vfstest/write_non_unix.go @@ -17,10 +17,7 @@ func TestWriteFileDoubleClose(t *testing.T) { // writeTestDup performs the platform-specific implementation of the dup() syscall func writeTestDup(oldfd uintptr) (uintptr, error) { - p, err := windows.GetCurrentProcess() - if err != nil { - return 0, err - } + p := windows.CurrentProcess() var h windows.Handle return uintptr(h), windows.DuplicateHandle(p, windows.Handle(oldfd), p, &h, 0, true, windows.DUPLICATE_SAME_ACCESS) } diff --git a/vfs/write.go b/vfs/write.go index 4f05903123d53..24595e0419ccb 100644 --- a/vfs/write.go +++ b/vfs/write.go @@ -247,7 +247,7 @@ func (fh *WriteFileHandle) Flush() error { err := fh.close() if err != nil { fs.Errorf(fh.remote, "WriteFileHandle.Flush error: %v", err) - } else { + //} else { // fs.Debugf(fh.remote, "WriteFileHandle.Flush OK") } return err @@ -268,7 +268,7 @@ func (fh *WriteFileHandle) Release() error { err := fh.close() if err != nil { fs.Errorf(fh.remote, "WriteFileHandle.Release error: %v", err) - } else { + //} else { // fs.Debugf(fh.remote, "WriteFileHandle.Release OK") } return err From 02b4638a2212b903c1730d82acabc49816acee66 Mon Sep 17 00:00:00 2001 From: m00594701 Date: Tue, 7 Jun 2022 15:41:46 +0800 Subject: [PATCH 036/560] backend: add Huawei OBS to s3 provider list --- README.md | 1 + backend/s3/s3.go | 116 ++++++++++++++++++++++++++- docs/content/s3.md | 192 ++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 295 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 206517733e77a..7a955d31f5162 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/) * HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/) * HTTP [:page_facing_up:](https://rclone.org/http/) + * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs) * Hubic [:page_facing_up:](https://rclone.org/hubic/) * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/) * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 0a7b216f488ea..cd4f3e06b9ca3 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -60,7 +60,7 @@ import ( func init() { fs.Register(&fs.RegInfo{ Name: "s3", - Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi", + Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi", NewFs: NewFs, CommandHelp: commandHelp, Options: []fs.Option{{ @@ -92,6 +92,9 @@ func init() { }, { Value: "Dreamhost", Help: "Dreamhost DreamObjects", + }, { + Value: "HuaweiOBS", + Help: "Huawei Object Storage Service", }, { Value: "IBMCOS", Help: "IBM COS S3", @@ -305,6 +308,56 @@ func init() { Value: "pl-waw", Help: "Warsaw, Poland", }}, + }, { + Name: "region", + Help: "Region to connect to. - the location where your bucket will be created and your data stored. Need bo be same with your endpoint.\n", + Provider: "HuaweiOBS", + Examples: []fs.OptionExample{{ + Value: "af-south-1", + Help: "AF-Johannesburg", + }, { + Value: "ap-southeast-2", + Help: "AP-Bangkok", + }, { + Value: "ap-southeast-3", + Help: "AP-Singapore", + }, { + Value: "cn-east-3", + Help: "CN East-Shanghai1", + }, { + Value: "cn-east-2", + Help: "CN East-Shanghai2", + }, { + Value: "cn-north-1", + Help: "CN North-Beijing1", + }, { + Value: "cn-north-4", + Help: "CN North-Beijing4", + }, { + Value: "cn-south-1", + Help: "CN South-Guangzhou", + }, { + Value: "ap-southeast-1", + Help: "CN-Hong Kong", + }, { + Value: "sa-argentina-1", + Help: "LA-Buenos Aires1", + }, { + Value: "sa-peru-1", + Help: "LA-Lima1", + }, { + Value: "na-mexico-1", + Help: "LA-Mexico City1", + }, { + Value: "sa-chile-1", + Help: "LA-Santiago2", + }, { + Value: "sa-brazil-1", + Help: "LA-Sao Paulo1", + }, { + Value: "ru-northwest-2", + Help: "RU-Moscow2", + }}, }, { Name: "region", Help: "Region to connect to.", @@ -316,7 +369,7 @@ func init() { }, { Name: "region", Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.", - Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS", + Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS", Examples: []fs.OptionExample{{ Value: "", Help: "Use this if unsure.\nWill use v4 signatures and an empty region.", @@ -708,6 +761,57 @@ func init() { Value: "oss-me-east-1.aliyuncs.com", Help: "Middle East 1 (Dubai)", }}, + }, { + // obs endpoints: https://developer.huaweicloud.com/intl/en-us/endpoint?OBS + Name: "endpoint", + Help: "Endpoint for OBS API.", + Provider: "HuaweiOBS", + Examples: []fs.OptionExample{{ + Value: "obs.af-south-1.myhuaweicloud.com", + Help: "AF-Johannesburg", + }, { + Value: "obs.ap-southeast-2.myhuaweicloud.com", + Help: "AP-Bangkok", + }, { + Value: "obs.ap-southeast-3.myhuaweicloud.com", + Help: "AP-Singapore", + }, { + Value: "obs.cn-east-3.myhuaweicloud.com", + Help: "CN East-Shanghai1", + }, { + Value: "obs.cn-east-2.myhuaweicloud.com", + Help: "CN East-Shanghai2", + }, { + Value: "obs.cn-north-1.myhuaweicloud.com", + Help: "CN North-Beijing1", + }, { + Value: "obs.cn-north-4.myhuaweicloud.com", + Help: "CN North-Beijing4", + }, { + Value: "obs.cn-south-1.myhuaweicloud.com", + Help: "CN South-Guangzhou", + }, { + Value: "obs.ap-southeast-1.myhuaweicloud.com", + Help: "CN-Hong Kong", + }, { + Value: "obs.sa-argentina-1.myhuaweicloud.com", + Help: "LA-Buenos Aires1", + }, { + Value: "obs.sa-peru-1.myhuaweicloud.com", + Help: "LA-Lima1", + }, { + Value: "obs.na-mexico-1.myhuaweicloud.com", + Help: "LA-Mexico City1", + }, { + Value: "obs.sa-chile-1.myhuaweicloud.com", + Help: "LA-Santiago2", + }, { + Value: "obs.sa-brazil-1.myhuaweicloud.com", + Help: "LA-Sao Paulo1", + }, { + Value: "obs.ru-northwest-2.myhuaweicloud.com", + Help: "RU-Moscow2", + }}, }, { Name: "endpoint", Help: "Endpoint for Scaleway Object Storage.", @@ -879,7 +983,7 @@ func init() { }, { Name: "endpoint", Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.", - Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp", + Provider: "!AWS,IBMCOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp", Examples: []fs.OptionExample{{ Value: "objects-us-east-1.dream.io", Help: "Dream Objects endpoint", @@ -1289,7 +1393,7 @@ func init() { }, { Name: "location_constraint", Help: "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.", - Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS", + Provider: "!AWS,IBMCOS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS", }, { Name: "acl", Help: `Canned ACL used when creating buckets and storing or copying objects. @@ -2248,6 +2352,10 @@ func setQuirks(opt *Options) { // No quirks case "Alibaba": useMultipartEtag = false // Alibaba seems to calculate multipart Etags differently from AWS + case "HuaweiOBS": + // Huawei OBS PFS is not support listObjectV2, and if turn on the urlEncodeListing, marker will not work and keep list same page forever. + urlEncodeListings = false + listObjectsV2 = false case "Ceph": listObjectsV2 = false virtualHostStyle = false diff --git a/docs/content/s3.md b/docs/content/s3.md index 8c01c8c13b2ee..15b26e7e2944c 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -16,6 +16,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}} {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}} {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} +{{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}} {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}} {{< provider name="RackCorp Object Storage" home="https://www.rackcorp.com/" config="/s3/#RackCorp" >}} @@ -569,7 +570,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS). +Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS). #### --s3-provider @@ -596,6 +597,8 @@ Properties: - Digital Ocean Spaces - "Dreamhost" - Dreamhost DreamObjects + - "HuaweiOBS" + - Huawei Object Storage Service - "IBMCOS" - IBM COS S3 - "LyveCloud" @@ -833,7 +836,7 @@ Properties: - Config: region - Env Var: RCLONE_S3_REGION -- Provider: !AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS +- Provider: !AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS - Type: string - Required: false - Examples: @@ -946,6 +949,48 @@ Endpoint for Arvan Cloud Object Storage (AOS) API. - "s3.ir-tbz-sh1.arvanstorage.com" - Tabriz Iran (Shahriar) +#### --s3-endpoint + +Endpoint for Huawei Cloud Object Storage Service (OBS) API. + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Provider: HuaweiOBS +- Type: string +- Required: false +- Examples: + - "obs.af-south-1.myhuaweicloud.com" + - AF-Johannesburg Endpoint + - "obs.ap-southeast-2.myhuaweicloud.com" + - AP-Bangkok Endpoint + - "obs.ap-southeast-3.myhuaweicloud.com" + - AP-Singapore Endpoint + - "obs.cn-east-3.myhuaweicloud.com" + - CN East-Shanghai1 Endpoint + - "obs.cn-east-2.myhuaweicloud.com" + - CN East-Shanghai2 Endpoint + - "obs.cn-north-1.myhuaweicloud.com" + - CN North-Beijing1 Endpoint + - "obs.cn-north-4.myhuaweicloud.com" + - CN North-Beijing4 Endpoint + - "obs.cn-south-1.myhuaweicloud.com" + - CN South-Guangzhou Endpoint + - "obs.ap-southeast-1.myhuaweicloud.com" + - CN-Hong Kong Endpoint + - "obs.sa-argentina-1.myhuaweicloud.com" + - LA-Buenos Aires1 Endpoint + - "obs.sa-peru-1.myhuaweicloud.com" + - LA-Lima1 Endpoint + - "obs.na-mexico-1.myhuaweicloud.com" + - LA-Mexico City1 Endpoint + - "obs.sa-chile-1.myhuaweicloud.com" + - LA-Santiago2 Endpoint + - "obs.sa-brazil-1.myhuaweicloud.com" + - LA-Sao Paulo1 Endpoint + - "obs.ru-northwest-2.myhuaweicloud.com" + - RU-Moscow2 Endpoint + + #### --s3-endpoint @@ -1317,7 +1362,7 @@ Properties: - Config: endpoint - Env Var: RCLONE_S3_ENDPOINT -- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp +- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,HuaweiOBS - Type: string - Required: false - Examples: @@ -1556,7 +1601,7 @@ Properties: - Config: location_constraint - Env Var: RCLONE_S3_LOCATION_CONSTRAINT -- Provider: !AWS,IBMCOS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS +- Provider: !AWS,IBMCOS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS,HuaweiOBS - Type: string - Required: false @@ -1784,7 +1829,7 @@ Properties: ### Advanced options -Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS). +Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS). #### --s3-bucket-acl @@ -2590,7 +2635,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. ... -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) ... Storage> s3 @@ -2731,6 +2776,133 @@ Once configured, you can create a new Space and begin copying files. For example rclone mkdir spaces:my-new-space rclone copy /path/to/files spaces:my-new-space ``` +### Huawei OBS {#huawei-obs} + +Object Storage Service (OBS) provides stable, secure, efficient, and easy-to-use cloud storage that lets you store virtually any volume of unstructured data in any format and access it from anywhere. + +OBS provides an S3 interface, you can copy and modify the following configuration and add it to your rclone configuration file. +``` +[obs] +type = s3 +provider = HuaweiOBS +access_key_id = your-access-key-id +secret_access_key = your-secret-access-key +region = af-south-1 +endpoint = obs.af-south-1.myhuaweicloud.com +acl = private +``` + +Or you can also configure via the interactive command line: +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> obs +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +[snip] + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \ (s3) +[snip] +Storage> 5 +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. +[snip] + 9 / Huawei Object Storage Service + \ (HuaweiOBS) +[snip] +provider> 9 +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) +env_auth> 1 +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> your-access-key-id +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> your-secret-access-key +Option region. +Region to connect to. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / AF-Johannesburg + \ (af-south-1) + 2 / AP-Bangkok + \ (ap-southeast-2) +[snip] +region> 1 +Option endpoint. +Endpoint for OBS API. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / AF-Johannesburg + \ (obs.af-south-1.myhuaweicloud.com) + 2 / AP-Bangkok + \ (obs.ap-southeast-2.myhuaweicloud.com) +[snip] +endpoint> 1 +Option acl. +Canned ACL used when creating buckets and storing or copying objects. +This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Note that this ACL is applied when server-side copying objects as S3 +doesn't copy the ACL from the source but rather writes a fresh one. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \ (private) +[snip] +acl> 1 +Edit advanced config? +y) Yes +n) No (default) +y/n> +-------------------- +[obs] +type = s3 +provider = HuaweiOBS +access_key_id = your-access-key-id +secret_access_key = your-secret-access-key +region = af-south-1 +endpoint = obs.af-south-1.myhuaweicloud.com +acl = private +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +Current remotes: + +Name Type +==== ==== +obs s3 + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> q +``` ### IBM COS (S3) @@ -3045,7 +3217,7 @@ Choose `s3` backend Type of storage to configure. Choose a number from below, or type in your own value. [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \ (s3) [snip] Storage> s3 @@ -3346,7 +3518,7 @@ Type of storage to configure. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [snip] - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -3456,7 +3628,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. ... - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \ (s3) ... Storage> s3 @@ -3827,7 +3999,7 @@ Choose a number from below, or type in your own value \ "alias" 3 / Amazon Drive \ "amazon cloud drive" - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS \ "s3" [snip] Storage> s3 From 6602e1a851c172248f379218c1ae762829c013eb Mon Sep 17 00:00:00 2001 From: "Art M. Gallagher" Date: Tue, 14 Jun 2022 09:55:40 +0100 Subject: [PATCH 037/560] mega: document using MEGAcmd to help with login failures Added extra subsection under Failure to log-in See: https://forum.rclone.org/t/30935 --- docs/content/mega.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/content/mega.md b/docs/content/mega.md index f3abf95fc9d7b..27882e34232fd 100644 --- a/docs/content/mega.md +++ b/docs/content/mega.md @@ -107,6 +107,44 @@ Use `rclone dedupe` to fix duplicated files. ### Failure to log-in +#### Object not found + +If you are connecting to your Mega remote for the first time, +to test access and syncronisation, you may receive an error such as + +``` +Failed to create file system for "my-mega-remote:": +couldn't login: Object (typically, node or user) not found +``` + +The diagnostic steps often recommended in the [rclone forum](https://forum.rclone.org/search?q=mega) +start with the **MEGAcmd** utility. Note that this refers to +the official C++ command from https://github.com/meganz/MEGAcmd +and not the go language built command from t3rm1n4l/megacmd +that is no longer maintained. + +Follow the instructions for installing MEGAcmd and try accessing +your remote as they recommend. You can establish whether or not +you can log in using MEGAcmd, and obtain diagnostic information +to help you, and search or work with others in the forum. + +``` +MEGA CMD> login me@example.com +Password: +Fetching nodes ... +Loading transfers from local cache +Login complete as me@example.com +me@example.com:/$ +``` + +Note that some have found issues with passwords containing special +characters. If you can not log on with rclone, but MEGAcmd logs on +just fine, then consider changing your password temporarily to +pure alphanumeric characters, in case that helps. + + +#### Repeated commands blocks access + Mega remotes seem to get blocked (reject logins) under "heavy use". We haven't worked out the exact blocking rules but it seems to be related to fast paced, successive rclone commands. From 50c2e37aac746466ed4159ada1d40316ea302481 Mon Sep 17 00:00:00 2001 From: Sven Gerber <49589423+svengerber@users.noreply.github.com> Date: Tue, 14 Jun 2022 11:21:23 +0200 Subject: [PATCH 038/560] onedrive: add access scopes option By default, rclone always requests read and write permissions. No matter what settings you configure in the AAD application. This option allows to explicitly request readonly permissions Migrated read only option to access scope option and set disable_site_permission option to hidden. --- backend/onedrive/onedrive.go | 45 ++++++++++++++++++++++++++++-------- docs/content/onedrive.md | 7 ++++-- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index a077843ed3f18..f8ab22a27480d 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -65,12 +65,12 @@ var ( authPath = "/common/oauth2/v2.0/authorize" tokenPath = "/common/oauth2/v2.0/token" - scopesWithSitePermission = []string{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access", "Sites.Read.All"} - scopesWithoutSitePermission = []string{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access"} + scopeAccess = fs.SpaceSepList{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "Sites.Read.All", "offline_access"} + scopeAccessWithoutSites = fs.SpaceSepList{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access"} // Description of how to auth for this app for a business account oauthConfig = &oauth2.Config{ - Scopes: scopesWithSitePermission, + Scopes: scopeAccess, ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectLocalhostURL, @@ -150,6 +150,27 @@ there through a path traversal. `, Advanced: true, }, { + Name: "access_scopes", + Help: `Set scopes to be requested by rclone. + +Choose or manually enter a custom space separated list with all scopes, that rclone should request. +`, + Default: scopeAccess, + Advanced: true, + Examples: []fs.OptionExample{ + { + Value: "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access", + Help: "Read and write access to all resources", + }, + { + Value: "Files.Read Files.Read.All Sites.Read.All offline_access", + Help: "Read only access to all resources", + }, + { + Value: "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All offline_access", + Help: "Read and write access to all resources, without the ability to browse SharePoint sites. \nSame as if disable_site_permission was set to true", + }, + }}, { Name: "disable_site_permission", Help: `Disable the request for Sites.Read.All permission. @@ -160,6 +181,7 @@ application, and your organization disallows users to consent app permission request on their own.`, Default: false, Advanced: true, + Hide: fs.OptionHideBoth, }, { Name: "expose_onenote_files", Help: `Set to make OneNote files show up in directory listings. @@ -401,11 +423,16 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf region, graphURL := getRegionURL(m) if config.State == "" { + var accessScopes fs.SpaceSepList + accessScopesString, _ := m.Get("access_scopes") + err := accessScopes.Set(accessScopesString) + if err != nil { + return nil, fmt.Errorf("failed to parse access_scopes: %w", err) + } + oauthConfig.Scopes = []string(accessScopes) disableSitePermission, _ := m.Get("disable_site_permission") if disableSitePermission == "true" { - oauthConfig.Scopes = scopesWithoutSitePermission - } else { - oauthConfig.Scopes = scopesWithSitePermission + oauthConfig.Scopes = scopeAccessWithoutSites } oauthConfig.Endpoint = oauth2.Endpoint{ AuthURL: authEndpoint[region] + authPath, @@ -562,6 +589,7 @@ type Options struct { DriveType string `config:"drive_type"` RootFolderID string `config:"root_folder_id"` DisableSitePermission bool `config:"disable_site_permission"` + AccessScopes fs.SpaceSepList `config:"access_scopes"` ExposeOneNoteFiles bool `config:"expose_onenote_files"` ServerSideAcrossConfigs bool `config:"server_side_across_configs"` ListChunk int64 `config:"list_chunk"` @@ -829,10 +857,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e } rootURL := graphAPIEndpoint[opt.Region] + "/v1.0" + "/drives/" + opt.DriveID + oauthConfig.Scopes = opt.AccessScopes if opt.DisableSitePermission { - oauthConfig.Scopes = scopesWithoutSitePermission - } else { - oauthConfig.Scopes = scopesWithSitePermission + oauthConfig.Scopes = scopeAccessWithoutSites } oauthConfig.Endpoint = oauth2.Endpoint{ AuthURL: authEndpoint[opt.Region] + authPath, diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index 7bb4558755934..a847d2a3879aa 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -132,12 +132,15 @@ Client ID and Key by following the steps below: 2. Enter a name for your app, choose account type `Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)`, select `Web` in `Redirect URI`, then type (do not copy and paste) `http://localhost:53682/` and click Register. Copy and keep the `Application (client) ID` under the app name for later use. 3. Under `manage` select `Certificates & secrets`, click `New client secret`. Enter a description (can be anything) and set `Expires` to 24 months. Copy and keep that secret _Value_ for later use (you _won't_ be able to see this value afterwards). 4. Under `manage` select `API permissions`, click `Add a permission` and select `Microsoft Graph` then select `delegated permissions`. -5. Search and select the following permissions: `Files.Read`, `Files.ReadWrite`, `Files.Read.All`, `Files.ReadWrite.All`, `offline_access`, `User.Read`, and optionally `Sites.Read.All` (see below). Once selected click `Add permissions` at the bottom. +5. Search and select the following permissions: `Files.Read`, `Files.ReadWrite`, `Files.Read.All`, `Files.ReadWrite.All`, `offline_access`, `User.Read` and `Sites.Read.All` (if custom access scopes are configured, select the permissions accordingly). Once selected click `Add permissions` at the bottom. Now the application is complete. Run `rclone config` to create or edit a OneDrive remote. Supply the app ID and password as Client ID and Secret, respectively. rclone will walk you through the remaining steps. -The `Sites.Read.All` permission is required if you need to [search SharePoint sites when configuring the remote](https://github.com/rclone/rclone/pull/5883). However, if that permission is not assigned, you need to set `disable_site_permission` option to true in the advanced options. +The access_scopes option allows you to configure the permissions requested by rclone. +See [Microsoft Docs](https://docs.microsoft.com/en-us/graph/permissions-reference#files-permissions) for more information about the different scopes. + +The `Sites.Read.All` permission is required if you need to [search SharePoint sites when configuring the remote](https://github.com/rclone/rclone/pull/5883). However, if that permission is not assigned, you need to exclude `Sites.Read.All` from your access scopes or set `disable_site_permission` option to true in the advanced options. ### Modification time and hashes From 0279bf3abb93e86e5fda78ffea014adaac72ff81 Mon Sep 17 00:00:00 2001 From: CrossR Date: Thu, 20 May 2021 20:39:04 +0100 Subject: [PATCH 039/560] ncdu: implement multi selection Co-authored-by: buengese --- cmd/ncdu/ncdu.go | 103 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go index 4ef08f3468a3d..963ddb628df15 100644 --- a/cmd/ncdu/ncdu.go +++ b/cmd/ncdu/ncdu.go @@ -92,6 +92,9 @@ func helpText() (tr []string) { " u toggle human-readable format", " n,s,C,A sort by name,size,count,average size", " d delete file/directory", + " v select file/directory", + " V enter visual select mode", + " D delete selected files/directories", } if !clipboard.Unsupported { tr = append(tr, " y copy current path to clipboard") @@ -126,11 +129,13 @@ type UI struct { showCounts bool // toggle showing counts showDirAverageSize bool // toggle average size humanReadable bool // toggle human-readable format + visualSelectMode bool // toggle visual selection mode sortByName int8 // +1 for normal, 0 for off, -1 for reverse sortBySize int8 sortByCount int8 sortByAverageSize int8 dirPosMap map[string]dirPos // store for directory positions + selectedEntries map[string]dirPos // selected entries of current directory } // Where we have got to in the directory listing @@ -361,6 +366,7 @@ func (u *UI) Draw() error { break } attrs, err := u.d.AttrI(u.sortPerm[n]) + _, isSelected := u.selectedEntries[entry.String()] fg := termbox.ColorWhite if attrs.EntriesHaveErrors { fg = termbox.ColorYellow @@ -368,6 +374,9 @@ func (u *UI) Draw() error { if err != nil { fg = termbox.ColorRed } + if isSelected { + fg = termbox.ColorLightYellow + } bg := termbox.ColorBlack if n == dirPos.entry { fg, bg = bg, fg @@ -494,6 +503,11 @@ func (u *UI) move(d int) { dirPos.offset = entries - 1 } + // toggle the current file for selection in selection mode + if u.visualSelectMode { + u.toggleSelectForCursor() + } + // write dirPos back for later u.dirPosMap[u.path] = dirPos } @@ -503,11 +517,19 @@ func (u *UI) removeEntry(pos int) { u.setCurrentDir(u.d) } -// delete the entry at the current position func (u *UI) delete() { if u.d == nil || len(u.entries) == 0 { return } + if len(u.selectedEntries) > 0 { + u.deleteSelected() + } else { + u.deleteSingle() + } +} + +// delete the entry at the current position +func (u *UI) deleteSingle() { ctx := context.Background() cursorPos := u.dirPosMap[u.path] dirPos := u.sortPerm[cursorPos.entry] @@ -553,6 +575,62 @@ func (u *UI) delete() { } } +func (u *UI) deleteSelected() { + ctx := context.Background() + + u.boxMenu = []string{"cancel", "confirm"} + + u.boxMenuHandler = func(f fs.Fs, p string, o int) (string, error) { + if o != 1 { + return "Aborted!", nil + } + + positionsToDelete := make([]int, len(u.selectedEntries)) + i := 0 + + for key, cursorPos := range u.selectedEntries { + + dirPos := u.sortPerm[cursorPos.entry] + dirEntry := u.entries[dirPos] + var err error + + if obj, isFile := dirEntry.(fs.Object); isFile { + err = operations.DeleteFile(ctx, obj) + } else { + err = operations.Purge(ctx, f, dirEntry.String()) + } + + if err != nil { + return "", err + } + + delete(u.selectedEntries, key) + positionsToDelete[i] = dirPos + i++ + } + + // deleting all entries at once, as doing it during the deletions + // could cause issues. + sort.Slice(positionsToDelete, func(i, j int) bool { + return positionsToDelete[i] > positionsToDelete[j] + }) + for _, dirPos := range positionsToDelete { + u.removeEntry(dirPos) + } + + // move cursor at end if needed + cursorPos := u.dirPosMap[u.path] + if cursorPos.entry >= len(u.entries) { + u.move(-1) + } + + return "Successfully deleted all items!", nil + } + u.popupBox([]string{ + "Delete selected items?", + fmt.Sprintf("ALL %d items will be deleted", len(u.selectedEntries))}) +} + func (u *UI) displayPath() { u.togglePopupBox([]string{ "Current Path", @@ -661,6 +739,8 @@ func (u *UI) setCurrentDir(d *scan.Dir) { u.d = d u.entries = d.Entries() u.path = fspath.JoinRootPath(u.fsName, d.Path()) + u.selectedEntries = make(map[string]dirPos) + u.visualSelectMode = false u.sortCurrentDir() } @@ -737,6 +817,20 @@ func (u *UI) toggleSort(sortType *int8) { u.sortCurrentDir() } +func (u *UI) toggleSelectForCursor() { + cursorPos := u.dirPosMap[u.path] + dirPos := u.sortPerm[cursorPos.entry] + dirEntry := u.entries[dirPos] + + _, present := u.selectedEntries[dirEntry.String()] + + if present { + delete(u.selectedEntries, dirEntry.String()) + } else { + u.selectedEntries[dirEntry.String()] = cursorPos + } +} + // NewUI creates a new user interface for ncdu on f func NewUI(f fs.Fs) *UI { return &UI{ @@ -752,6 +846,7 @@ func NewUI(f fs.Fs) *UI { sortBySize: 1, sortByCount: 0, dirPosMap: make(map[string]dirPos), + selectedEntries: make(map[string]dirPos), } } @@ -845,6 +940,10 @@ outer: u.toggleSort(&u.sortByName) case 's': u.toggleSort(&u.sortBySize) + case 'v': + u.toggleSelectForCursor() + case 'V': + u.visualSelectMode = !u.visualSelectMode case 'C': u.toggleSort(&u.sortByCount) case 'A': @@ -857,6 +956,8 @@ outer: u.delete() case 'u': u.humanReadable = !u.humanReadable + case 'D': + u.deleteSelected() case '?': u.togglePopupBox(helpText()) From 621c4ebe15600635a2d085a55b521fe39cea2511 Mon Sep 17 00:00:00 2001 From: buengese Date: Wed, 15 Jun 2022 15:01:16 +0200 Subject: [PATCH 040/560] bin/make_backend_docs: allow generation of docs for just one backend --- bin/make_backend_docs.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/bin/make_backend_docs.py b/bin/make_backend_docs.py index 8753061f94bb6..1871cc68bd044 100755 --- a/bin/make_backend_docs.py +++ b/bin/make_backend_docs.py @@ -3,9 +3,11 @@ Make backend documentation """ +import sys import os import io import subprocess +from pathlib import Path marker = "{{< rem autogenerated options" start = marker + " start" @@ -16,18 +18,19 @@ def find_backends(): """Return a list of all backends""" return [ x for x in os.listdir("backend") if x not in ("all",) ] -def output_docs(backend, out): +def output_docs(backend, out, cwd): """Output documentation for backend options to out""" out.flush() - subprocess.check_call(["rclone", "help", "backend", backend], stdout=out) + subprocess.check_call(["./rclone", "help", "backend", backend], stdout=out) -def output_backend_tool_docs(backend, out): +def output_backend_tool_docs(backend, out, cwd): """Output documentation for backend tool to out""" out.flush() - subprocess.call(["rclone", "backend", "help", backend], stdout=out, stderr=subprocess.DEVNULL) + subprocess.call(["./rclone", "backend", "help", backend], stdout=out, stderr=subprocess.DEVNULL) def alter_doc(backend): """Alter the documentation for backend""" + rclone_bin_dir = Path(sys.path[0]).parent.absolute() doc_file = "docs/content/"+backend+".md" if not os.path.exists(doc_file): raise ValueError("Didn't find doc file %s" % (doc_file,)) @@ -41,8 +44,8 @@ def alter_doc(backend): in_docs = True start_full = (start + "\" - DO NOT EDIT - instead edit fs.RegInfo in backend/%s/%s.go then run make backenddocs\" " + end + "\n") % (backend, backend) out_file.write(start_full) - output_docs(backend, out_file) - output_backend_tool_docs(backend, out_file) + output_docs(backend, out_file, rclone_bin_dir) + output_backend_tool_docs(backend, out_file, rclone_bin_dir) out_file.write(stop+" "+end+"\n") altered = True if not in_docs: @@ -55,7 +58,18 @@ def alter_doc(backend): if not altered: raise ValueError("Didn't find '%s' markers for in %s" % (start, doc_file)) -if __name__ == "__main__": + +def main(args): + # single backend + if (len(args) == 2): + try: + alter_doc(args[1]) + print("Added docs for %s backend" % args[1]) + except Exception as e: + print("Failed adding docs for %s backend: %s" % (args[1], e)) + return + + # all backends failed, success = 0, 0 for backend in find_backends(): try: @@ -66,3 +80,6 @@ def alter_doc(backend): else: success += 1 print("Added docs for %d backends with %d failures" % (success, failed)) + +if __name__ == "__main__": + main(sys.argv) From 32f913ffbdd6c6f447584eed68a5a9bf09d220e0 Mon Sep 17 00:00:00 2001 From: buengese Date: Tue, 14 Jun 2022 14:34:37 +0200 Subject: [PATCH 041/560] pcloud: fix cleanup - fixes #3853 --- backend/pcloud/pcloud.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index a73593b4059da..c558e4495a066 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -24,6 +24,7 @@ import ( "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/lib/dircache" @@ -130,6 +131,19 @@ with rclone authorize. Value: "eapi.pcloud.com", Help: "EU region", }}, + }, { + Name: "username", + Help: `Your pcloud username. + +This is only required when you want to use the cleanup command. Due to a bug +in the pcloud API the required API does not support OAuth authentication so +we have to rely on user password authentication for it.`, + Advanced: true, + }, { + Name: "password", + Help: "Your pcloud password.", + IsPassword: true, + Advanced: true, }}...), }) } @@ -139,6 +153,8 @@ type Options struct { Enc encoder.MultiEncoder `config:"encoding"` RootFolderID string `config:"root_folder_id"` Hostname string `config:"hostname"` + Username string `config:"username"` + Password string `config:"password"` } // Fs represents a remote pcloud @@ -148,6 +164,7 @@ type Fs struct { opt Options // parsed options features *fs.Features // optional features srv *rest.Client // the connection to the server + cleanupSrv *rest.Client // the connection used for the cleanup method dirCache *dircache.DirCache // Map of directory path to directory id pacer *fs.Pacer // pacer for API calls tokenRenewer *oauthutil.Renew // renew the token on expiry @@ -293,6 +310,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e } updateTokenURL(oauthConfig, opt.Hostname) + canCleanup := opt.Username != "" && opt.Password != "" f := &Fs{ name: name, root: root, @@ -300,10 +318,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e srv: rest.NewClient(oAuthClient).SetRoot("https://" + opt.Hostname), pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), } + if canCleanup { + f.cleanupSrv = rest.NewClient(fshttp.NewClient(ctx)).SetRoot("https://" + opt.Hostname) + } f.features = (&fs.Features{ CaseInsensitive: false, CanHaveEmptyDirectories: true, }).Fill(ctx, f) + if !canCleanup { + f.features.CleanUp = nil + } f.srv.SetErrorHandler(errorHandler) // Renew the token in the background @@ -729,10 +753,12 @@ func (f *Fs) CleanUp(ctx context.Context) error { Parameters: url.Values{}, } opts.Parameters.Set("folderid", dirIDtoNumber(rootID)) + opts.Parameters.Set("username", f.opt.Username) + opts.Parameters.Set("password", obscure.MustReveal(f.opt.Password)) var resp *http.Response var result api.Error return f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) + resp, err = f.cleanupSrv.CallJSON(ctx, &opts, nil, &result) err = result.Update(err) return shouldRetry(ctx, resp, err) }) From 93a25498cfed086f9df7970d987abd24762d49b8 Mon Sep 17 00:00:00 2001 From: buengese Date: Wed, 15 Jun 2022 16:44:13 +0200 Subject: [PATCH 042/560] docs/pcloud: document the cleanup issues --- docs/content/pcloud.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/content/pcloud.md b/docs/content/pcloud.md index 4af0c968b26da..eb11905dd5987 100644 --- a/docs/content/pcloud.md +++ b/docs/content/pcloud.md @@ -112,6 +112,13 @@ Deleted files will be moved to the trash. Your subscription level will determine how long items stay in the trash. `rclone cleanup` can be used to empty the trash. +### Emptying the trash + +Due to an API limitation, the `rclone cleanup` command will only work if you +set your username and password in the advanced options for this backend. +Since we generally want to avoid storing user passwords in the rclone config +file, we advise you to only set this up if you need the `rclone cleanup` command to work. + ### Root folder ID You can set the `root_folder_id` for rclone. This is the directory @@ -251,4 +258,32 @@ Properties: - "eapi.pcloud.com" - EU region +#### --pcloud-username + +Your pcloud username. + +This is only required when you want to use the cleanup command. Due to a bug +in the pcloud API the required API does not support OAuth authentication so +we have to rely on user password authentication for it. + +Properties: + +- Config: username +- Env Var: RCLONE_PCLOUD_USERNAME +- Type: string +- Required: false + +#### --pcloud-password + +Your pcloud password. + +**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/). + +Properties: + +- Config: password +- Env Var: RCLONE_PCLOUD_PASSWORD +- Type: string +- Required: false + {{< rem autogenerated options stop >}} From 592358148da684c087cb79d6c0f135c667b88d30 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 16 Jun 2022 16:47:47 +0100 Subject: [PATCH 043/560] Add m00594701 to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 8d5cbbac44fbe..a34d9b2f605d5 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -606,3 +606,4 @@ put them back in again.` >}} * Jason Zheng * Matthew Vernon * Noah Hsu + * m00594701 From 41f3ceb67df596f5af607c9c5e3dfb56d1c85b25 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 16 Jun 2022 16:47:47 +0100 Subject: [PATCH 044/560] Add Art M. Gallagher to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index a34d9b2f605d5..1316c13ec09bd 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -607,3 +607,4 @@ put them back in again.` >}} * Matthew Vernon * Noah Hsu * m00594701 + * Art M. Gallagher From c390098262bd4da9c50967e83514203ed7023845 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 16 Jun 2022 16:47:47 +0100 Subject: [PATCH 045/560] Add Sven Gerber to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 1316c13ec09bd..23316188995ec 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -608,3 +608,4 @@ put them back in again.` >}} * Noah Hsu * m00594701 * Art M. Gallagher + * Sven Gerber <49589423+svengerber@users.noreply.github.com> From 6c832a72ee9bd3500d01f8232da0241cb24dc4f5 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 16 Jun 2022 16:47:47 +0100 Subject: [PATCH 046/560] Add CrossR to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 23316188995ec..91fe78693606b 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -609,3 +609,4 @@ put them back in again.` >}} * m00594701 * Art M. Gallagher * Sven Gerber <49589423+svengerber@users.noreply.github.com> + * CrossR From 626a416ff8946b4fcc86e34780c90d64d1c481b8 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 16 Jun 2022 11:11:14 +0100 Subject: [PATCH 047/560] vfs: factor out the VFS option initialization for re-use #3259 --- vfs/vfs.go | 8 ++------ vfs/vfscommon/options.go | 11 +++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/vfs/vfs.go b/vfs/vfs.go index d220d015e2091..fc4eb126442a1 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -193,12 +193,8 @@ func New(f fs.Fs, opt *vfscommon.Options) *VFS { vfs.Opt = vfscommon.DefaultOpt } - // Mask the permissions with the umask - vfs.Opt.DirPerms &= ^os.FileMode(vfs.Opt.Umask) - vfs.Opt.FilePerms &= ^os.FileMode(vfs.Opt.Umask) - - // Make sure directories are returned as directories - vfs.Opt.DirPerms |= os.ModeDir + // Fill out anything else + vfs.Opt.Init() // Find a VFS with the same name and options and return it if possible activeMu.Lock() diff --git a/vfs/vfscommon/options.go b/vfs/vfscommon/options.go index 90cbe5bb7f33d..5c1ae7900e669 100644 --- a/vfs/vfscommon/options.go +++ b/vfs/vfscommon/options.go @@ -62,3 +62,14 @@ var DefaultOpt = Options{ ReadAhead: 0 * fs.Mebi, UsedIsSize: false, } + +// Init the options, making sure everything is withing range +func (opt *Options) Init() { + // Mask the permissions with the umask + opt.DirPerms &= ^os.FileMode(opt.Umask) + opt.FilePerms &= ^os.FileMode(opt.Umask) + + // Make sure directories are returned as directories + opt.DirPerms |= os.ModeDir + +} From 4a382c09ecab5cee85726d7b4215bb502b7c65b9 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 16 Jun 2022 12:08:11 +0100 Subject: [PATCH 048/560] mount: run tests in a subprocess to fix deadlock - fixes #3259 Before this change we ran the tests and the mount in the same process. This could cause deadlocks and often did, and made the mount tests very unreliable. This fixes the problem by running the mount in a seperate process and commanding it via a pipe over stdin/stdout. --- vfs/vfstest/dir.go | 8 +- vfs/vfstest/fs.go | 190 +++++++------------------- vfs/vfstest/submount.go | 276 ++++++++++++++++++++++++++++++++++++++ vfs/vfstest/write.go | 4 +- vfs/vfstest/write_unix.go | 2 +- 5 files changed, 327 insertions(+), 153 deletions(-) create mode 100644 vfs/vfstest/submount.go diff --git a/vfs/vfstest/dir.go b/vfs/vfstest/dir.go index 8c6273a1c5a11..0cf02ed0f8814 100644 --- a/vfs/vfstest/dir.go +++ b/vfs/vfstest/dir.go @@ -5,7 +5,6 @@ import ( "testing" "time" - "github.com/rclone/rclone/fs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -175,15 +174,12 @@ func TestDirCacheFlush(t *testing.T) { err := run.fremote.Mkdir(context.Background(), "dir/subdir") require.NoError(t, err) - root, err := run.vfs.Root() - require.NoError(t, err) - // expect newly created "subdir" on remote to not show up - root.ForgetPath("otherdir", fs.EntryDirectory) + run.forget("otherdir") run.readLocal(t, localDm, "") assert.Equal(t, dm, localDm, "expected vs fuse mount") - root.ForgetPath("dir", fs.EntryDirectory) + run.forget("dir") dm = newDirMap("otherdir/|otherdir/file 1|dir/|dir/file 1|dir/subdir/") run.readLocal(t, localDm, "") assert.Equal(t, dm, localDm, "expected vs fuse mount") diff --git a/vfs/vfstest/fs.go b/vfs/vfstest/fs.go index bb9a61fc50ed4..6781fbe312596 100644 --- a/vfs/vfstest/fs.go +++ b/vfs/vfstest/fs.go @@ -3,11 +3,11 @@ package vfstest import ( + "bufio" "context" "flag" "fmt" "io" - "io/ioutil" "log" "os" "os/exec" @@ -16,6 +16,7 @@ import ( "reflect" "runtime" "strings" + "sync" "testing" "time" @@ -24,8 +25,6 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/fstest" - "github.com/rclone/rclone/lib/file" - "github.com/rclone/rclone/vfs" "github.com/rclone/rclone/vfs/vfscommon" "github.com/rclone/rclone/vfs/vfsflags" "github.com/stretchr/testify/assert" @@ -36,16 +35,19 @@ const ( waitForWritersDelay = 30 * time.Second // time to wait for existing writers ) -var ( - mountFn mountlib.MountFn -) - // RunTests runs all the tests against all the VFS cache modes // -// If useVFS is set then it runs the tests against a VFS rather than amount -func RunTests(t *testing.T, useVFS bool, fn mountlib.MountFn) { - mountFn = fn +// If useVFS is set then it runs the tests against a VFS rather than a +// mount +// +// If useVFS is not set then it runs the mount in a subprocess in +// order to avoid kernel deadlocks. +func RunTests(t *testing.T, useVFS bool, mountFn mountlib.MountFn) { flag.Parse() + if isSubProcess() { + startMount(mountFn, useVFS, *runMount) + return + } tests := []struct { cacheMode vfscommon.CacheMode writeBack time.Duration @@ -56,9 +58,11 @@ func RunTests(t *testing.T, useVFS bool, fn mountlib.MountFn) { {cacheMode: vfscommon.CacheModeFull}, {cacheMode: vfscommon.CacheModeFull, writeBack: 100 * time.Millisecond}, } - run = newRun(useVFS) for _, test := range tests { - run.cacheMode(test.cacheMode, test.writeBack) + vfsOpt := vfsflags.Opt + vfsOpt.CacheMode = test.cacheMode + vfsOpt.WriteBack = test.writeBack + run = newRun(useVFS, &vfsOpt, mountFn) what := fmt.Sprintf("CacheMode=%v", test.cacheMode) if test.writeBack > 0 { what += fmt.Sprintf(",WriteBack=%v", test.writeBack) @@ -93,24 +97,29 @@ func RunTests(t *testing.T, useVFS bool, fn mountlib.MountFn) { t.Run("TestWriteFileAppend", TestWriteFileAppend) }) log.Printf("Finished test run with %s (ok=%v)", what, ok) + run.Finalise() if !ok { break } } - run.Finalise() } // Run holds the remotes for a test run type Run struct { os Oser - vfs *vfs.VFS + vfsOpt *vfscommon.Options useVFS bool // set if we are testing a VFS not a mount - mnt *mountlib.MountPoint mountPath string fremote fs.Fs fremoteName string cleanRemote func() skip bool + // For controlling the subprocess running the mount + cmdMu sync.Mutex + cmd *exec.Cmd + in io.ReadCloser + out io.WriteCloser + scanner *bufio.Scanner } // run holds the master Run data @@ -122,10 +131,12 @@ var run *Run // r.fremote is an empty remote Fs // // Finalise() will tidy them away when done. -func newRun(useVFS bool) *Run { +func newRun(useVFS bool, vfsOpt *vfscommon.Options, mountFn mountlib.MountFn) *Run { r := &Run{ useVFS: useVFS, + vfsOpt: vfsOpt, } + r.vfsOpt.Init() fstest.Initialise() var err error @@ -139,118 +150,10 @@ func newRun(useVFS bool) *Run { log.Fatalf("Failed to open mkdir %q: %v", *fstest.RemoteName, err) } - if !r.useVFS { - r.mountPath = findMountPath() - } - // Mount it up - r.mount() - + r.startMountSubProcess() return r } -func findMountPath() string { - if runtime.GOOS != "windows" { - mountPath, err := ioutil.TempDir("", "rclonefs-mount") - if err != nil { - log.Fatalf("Failed to create mount dir: %v", err) - } - return mountPath - } - - // Find a free drive letter - letter := file.FindUnusedDriveLetter() - drive := "" - if letter == 0 { - log.Fatalf("Couldn't find free drive letter for test") - } else { - drive = string(letter) + ":" - } - return drive -} - -func (r *Run) mount() { - log.Printf("mount %q %q", r.fremote, r.mountPath) - var err error - r.mnt = mountlib.NewMountPoint(mountFn, r.mountPath, r.fremote, &mountlib.Opt, &vfsflags.Opt) - - _, err = r.mnt.Mount() - if err != nil { - log.Printf("mount FAILED: %v", err) - r.skip = true - } else { - log.Printf("mount OK") - } - r.vfs = r.mnt.VFS - if r.useVFS { - r.os = vfsOs{r.vfs} - } else { - r.os = realOs{} - } - -} - -func (r *Run) umount() { - if r.skip { - log.Printf("FUSE not found so skipping umount") - return - } - /* - log.Printf("Calling fusermount -u %q", r.mountPath) - err := exec.Command("fusermount", "-u", r.mountPath).Run() - if err != nil { - log.Printf("fusermount failed: %v", err) - } - */ - log.Printf("Unmounting %q", r.mountPath) - err := r.mnt.Unmount() - if err != nil { - log.Printf("signal to umount failed - retrying: %v", err) - time.Sleep(3 * time.Second) - err = r.mnt.Unmount() - } - if err != nil { - log.Fatalf("signal to umount failed: %v", err) - } - log.Printf("Waiting for umount") - err = <-r.mnt.ErrChan - if err != nil { - log.Fatalf("umount failed: %v", err) - } - - // Cleanup the VFS cache - umount has called Shutdown - err = r.vfs.CleanUp() - if err != nil { - log.Printf("Failed to cleanup the VFS cache: %v", err) - } -} - -// cacheMode flushes the VFS and changes the CacheMode and the writeBack time -func (r *Run) cacheMode(cacheMode vfscommon.CacheMode, writeBack time.Duration) { - if r.skip { - log.Printf("FUSE not found so skipping cacheMode") - return - } - // Wait for writers to finish - r.vfs.WaitForWriters(waitForWritersDelay) - // Empty and remake the remote - r.cleanRemote() - err := r.fremote.Mkdir(context.Background(), "") - if err != nil { - log.Fatalf("Failed to open mkdir %q: %v", *fstest.RemoteName, err) - } - // Empty the cache - err = r.vfs.CleanUp() - if err != nil { - log.Printf("Failed to cleanup the VFS cache: %v", err) - } - // Reset the cache mode - r.vfs.SetCacheMode(cacheMode) - r.vfs.Opt.WriteBack = writeBack - // Flush the directory cache - r.vfs.FlushDirCache() - -} - func (r *Run) skipIfNoFUSE(t *testing.T) { if r.skip { t.Skip("FUSE not found so skipping test") @@ -265,11 +168,15 @@ func (r *Run) skipIfVFS(t *testing.T) { // Finalise cleans the remote and unmounts func (r *Run) Finalise() { - r.umount() + if !r.useVFS { + r.sendMountCommand("exit") + _, err := r.cmd.Process.Wait() + if err != nil { + log.Fatalf("mount sub process failed: %v", err) + } + } r.cleanRemote() - if r.useVFS { - // FIXME - } else { + if !r.useVFS { err := os.RemoveAll(r.mountPath) if err != nil { log.Printf("Failed to clean mountPath %q: %v", r.mountPath, err) @@ -284,9 +191,9 @@ func (r *Run) path(filePath string) string { } // return windows drive letter root as E:\ if filePath == "" && runtime.GOOS == "windows" { - return run.mountPath + `\` + return r.mountPath + `\` } - return filepath.Join(run.mountPath, filepath.FromSlash(filePath)) + return filepath.Join(r.mountPath, filepath.FromSlash(filePath)) } type dirMap map[string]struct{} @@ -323,10 +230,10 @@ func (r *Run) readLocal(t *testing.T, dir dirMap, filePath string) { if fi.IsDir() { dir[name+"/"] = struct{}{} r.readLocal(t, dir, name) - assert.Equal(t, run.vfs.Opt.DirPerms&os.ModePerm, fi.Mode().Perm()) + assert.Equal(t, r.vfsOpt.DirPerms&os.ModePerm, fi.Mode().Perm()) } else { dir[fmt.Sprintf("%s %d", name, fi.Size())] = struct{}{} - assert.Equal(t, run.vfs.Opt.FilePerms&os.ModePerm, fi.Mode().Perm()) + assert.Equal(t, r.vfsOpt.FilePerms&os.ModePerm, fi.Mode().Perm()) } } } @@ -374,11 +281,6 @@ func (r *Run) checkDir(t *testing.T, dirString string) { assert.Equal(t, dm, localDm, "expected vs fuse mount") } -// wait for any files being written to be released by fuse -func (r *Run) waitForWriters() { - run.vfs.WaitForWriters(waitForWritersDelay) -} - // writeFile writes data to a file named by filename. // If the file does not exist, WriteFile creates it with permissions perm; // otherwise writeFile truncates it before writing. @@ -415,25 +317,25 @@ func (r *Run) createFile(t *testing.T, filepath string, contents string) { func (r *Run) readFile(t *testing.T, filepath string) string { filepath = r.path(filepath) - result, err := run.os.ReadFile(filepath) + result, err := r.os.ReadFile(filepath) require.NoError(t, err) return string(result) } func (r *Run) mkdir(t *testing.T, filepath string) { filepath = r.path(filepath) - err := run.os.Mkdir(filepath, 0700) + err := r.os.Mkdir(filepath, 0700) require.NoError(t, err) } func (r *Run) rm(t *testing.T, filepath string) { filepath = r.path(filepath) - err := run.os.Remove(filepath) + err := r.os.Remove(filepath) require.NoError(t, err) // Wait for file to disappear from listing for i := 0; i < 100; i++ { - _, err := run.os.Stat(filepath) + _, err := r.os.Stat(filepath) if os.IsNotExist(err) { return } @@ -444,7 +346,7 @@ func (r *Run) rm(t *testing.T, filepath string) { func (r *Run) rmdir(t *testing.T, filepath string) { filepath = r.path(filepath) - err := run.os.Remove(filepath) + err := r.os.Remove(filepath) require.NoError(t, err) } @@ -470,5 +372,5 @@ func TestRoot(t *testing.T) { fi, err := os.Lstat(run.mountPath) require.NoError(t, err) assert.True(t, fi.IsDir()) - assert.Equal(t, run.vfs.Opt.DirPerms&os.ModePerm, fi.Mode().Perm()) + assert.Equal(t, run.vfsOpt.DirPerms&os.ModePerm, fi.Mode().Perm()) } diff --git a/vfs/vfstest/submount.go b/vfs/vfstest/submount.go new file mode 100644 index 0000000000000..e405dd847658b --- /dev/null +++ b/vfs/vfstest/submount.go @@ -0,0 +1,276 @@ +package vfstest + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "runtime" + "strings" + "time" + + "github.com/rclone/rclone/cmd/mountlib" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/cache" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/lib/file" + "github.com/rclone/rclone/vfs" + "github.com/rclone/rclone/vfs/vfscommon" +) + +// Functions to run and control the mount subprocess + +var ( + runMount = flag.String("run-mount", "", "If set, run the mount subprocess with the options (internal use only)") +) + +// Options for the mount sub processes passed with the -run-mount flag +type runMountOpt struct { + MountPoint string + MountOpt mountlib.Options + VFSOpt vfscommon.Options + Remote string +} + +// Start the mount subprocess and wait for it to start +func (r *Run) startMountSubProcess() { + // If testing the VFS we don't start a subprocess, we just use + // the VFS directly + if r.useVFS { + vfs := vfs.New(r.fremote, r.vfsOpt) + r.os = vfsOs{vfs} + return + } + r.os = realOs{} + r.mountPath = findMountPath() + log.Printf("startMountSubProcess %q (%q) %q", r.fremote, r.fremoteName, r.mountPath) + + opt := runMountOpt{ + MountPoint: r.mountPath, + MountOpt: mountlib.Opt, + VFSOpt: *r.vfsOpt, + Remote: r.fremoteName, + } + + opts, err := json.Marshal(&opt) + if err != nil { + log.Fatal(err) + } + + // Re-run this executable with a new option -run-mount + args := append(os.Args, "-run-mount", string(opts)) + r.cmd = exec.Command(args[0], args[1:]...) + r.cmd.Stderr = os.Stderr + r.out, err = r.cmd.StdinPipe() + if err != nil { + log.Fatal(err) + } + r.in, err = r.cmd.StdoutPipe() + if err != nil { + log.Fatal(err) + } + err = r.cmd.Start() + if err != nil { + log.Fatal("startMountSubProcess failed", err) + } + r.scanner = bufio.NewScanner(r.in) + + // Wait it for startup + log.Print("Waiting for mount to start") + for r.scanner.Scan() { + rx := strings.TrimSpace(r.scanner.Text()) + if rx == "STARTED" { + break + } + log.Printf("..Mount said: %s", rx) + } + if r.scanner.Err() != nil { + log.Printf("scanner err %v", r.scanner.Err()) + } + + log.Printf("startMountSubProcess: end") +} + +// Find a free path to run the mount on +func findMountPath() string { + if runtime.GOOS != "windows" { + mountPath, err := ioutil.TempDir("", "rclonefs-mount") + if err != nil { + log.Fatalf("Failed to create mount dir: %v", err) + } + return mountPath + } + + // Find a free drive letter + letter := file.FindUnusedDriveLetter() + drive := "" + if letter == 0 { + log.Fatalf("Couldn't find free drive letter for test") + } else { + drive = string(letter) + ":" + } + return drive +} + +// Return true if we are running as a subprocess to run the mount +func isSubProcess() bool { + return *runMount != "" +} + +// Run the mount - this is running in a subprocesses and the config +// is passed JSON encoded as the -run-mount parameter +// +// It reads commands from standard input and writes results to +// standard output. +func startMount(mountFn mountlib.MountFn, useVFS bool, opts string) { + log.Print("startMount") + ctx := context.Background() + + var opt runMountOpt + err := json.Unmarshal([]byte(opts), &opt) + if err != nil { + log.Fatalf("Unmarshal failed: %v", err) + } + + fstest.Initialise() + + f, err := cache.Get(ctx, opt.Remote) + if err != nil { + log.Fatalf("Failed to open remote %q: %v", opt.Remote, err) + } + + err = f.Mkdir(ctx, "") + if err != nil { + log.Fatalf("Failed to mkdir %q: %v", opt.Remote, err) + } + + log.Printf("startMount: Mounting %q on %q with %q", opt.Remote, opt.MountPoint, opt.VFSOpt.CacheMode) + mnt := mountlib.NewMountPoint(mountFn, opt.MountPoint, f, &opt.MountOpt, &opt.VFSOpt) + + _, err = mnt.Mount() + if err != nil { + log.Fatalf("mount FAILED %q: %v", opt.Remote, err) + } + defer umount(mnt) + log.Printf("startMount: mount OK") + fmt.Println("STARTED") // signal to parent all is good + + // Read commands from stdin + scanner := bufio.NewScanner(os.Stdin) + exit := false + for !exit && scanner.Scan() { + rx := strings.Trim(scanner.Text(), "\r\n") + var tx string + tx, exit = doMountCommand(mnt.VFS, rx) + fmt.Println(tx) + } + + err = scanner.Err() + if err != nil { + log.Fatalf("scanner failed %q: %v", opt.Remote, err) + } +} + +// Do a mount command which is a line read from stdin and return a +// line to send to stdout with an exit flag. +// +// The format of the lines is +// command \t parameter (optional) +// The response should be +// OK|ERR \t result (optional) +func doMountCommand(vfs *vfs.VFS, rx string) (tx string, exit bool) { + command := strings.Split(rx, "\t") + // log.Printf("doMountCommand: %q received", command) + var out = []string{"OK", ""} + switch command[0] { + case "waitForWriters": + vfs.WaitForWriters(waitForWritersDelay) + case "forget": + root, err := vfs.Root() + if err != nil { + out = []string{"ERR", err.Error()} + } else { + root.ForgetPath(command[1], fs.EntryDirectory) + } + case "exit": + exit = true + default: + out = []string{"ERR", "command not found"} + } + return strings.Join(out, "\t"), exit +} + +// Send a command to the mount subprocess and await a response +func (r *Run) sendMountCommand(args ...string) { + r.cmdMu.Lock() + defer r.cmdMu.Unlock() + tx := strings.Join(args, "\t") + // log.Printf("Send mount command: %q", tx) + var rx string + if r.useVFS { + // if using VFS do the VFS command directly + rx, _ = doMountCommand(r.os.(vfsOs).VFS, tx) + } else { + _, err := io.WriteString(r.out, tx+"\n") + if err != nil { + log.Fatalf("WriteString err %v", err) + } + if !r.scanner.Scan() { + log.Fatalf("Mount has gone away") + } + rx = strings.Trim(r.scanner.Text(), "\r\n") + } + in := strings.Split(rx, "\t") + // log.Printf("Answer is %q", in) + if in[0] != "OK" { + log.Fatalf("Error from mount: %q", in[1:]) + } +} + +// wait for any files being written to be released by fuse +func (r *Run) waitForWriters() { + r.sendMountCommand("waitForWriters") +} + +// forget the directory passed in +func (r *Run) forget(dir string) { + r.sendMountCommand("forget", dir) +} + +// Unmount the mount +func umount(mnt *mountlib.MountPoint) { + /* + log.Printf("Calling fusermount -u %q", mountPath) + err := exec.Command("fusermount", "-u", mountPath).Run() + if err != nil { + log.Printf("fusermount failed: %v", err) + } + */ + log.Printf("Unmounting %q", mnt.MountPoint) + err := mnt.Unmount() + if err != nil { + log.Printf("signal to umount failed - retrying: %v", err) + time.Sleep(3 * time.Second) + err = mnt.Unmount() + } + if err != nil { + log.Fatalf("signal to umount failed: %v", err) + } + log.Printf("Waiting for umount") + err = <-mnt.ErrChan + if err != nil { + log.Fatalf("umount failed: %v", err) + } + + // Cleanup the VFS cache - umount has called Shutdown + err = mnt.VFS.CleanUp() + if err != nil { + log.Printf("Failed to cleanup the VFS cache: %v", err) + } +} diff --git a/vfs/vfstest/write.go b/vfs/vfstest/write.go index 03ad7bfa1ae68..d4c2e0d92a5c6 100644 --- a/vfs/vfstest/write.go +++ b/vfs/vfstest/write.go @@ -91,7 +91,7 @@ func TestWriteFileDup(t *testing.T) { run.skipIfVFS(t) run.skipIfNoFUSE(t) - if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites { + if run.vfsOpt.CacheMode < vfscommon.CacheModeWrites { t.Skip("not supported on vfs-cache-mode < writes") return } @@ -136,7 +136,7 @@ func TestWriteFileDup(t *testing.T) { func TestWriteFileAppend(t *testing.T) { run.skipIfNoFUSE(t) - if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites { + if run.vfsOpt.CacheMode < vfscommon.CacheModeWrites { t.Skip("not supported on vfs-cache-mode < writes") return } diff --git a/vfs/vfstest/write_unix.go b/vfs/vfstest/write_unix.go index 0d412be07469b..4ce40c970c70d 100644 --- a/vfs/vfstest/write_unix.go +++ b/vfs/vfstest/write_unix.go @@ -46,7 +46,7 @@ func TestWriteFileDoubleClose(t *testing.T) { // write to the other dup _, err = unix.Write(fd2, buf) - if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites { + if run.vfsOpt.CacheMode < vfscommon.CacheModeWrites { // produces an error if cache mode < writes assert.Error(t, err, "input/output error") } else { From a0cb3bbd02e9d3a2843adac596c79732adfd144c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 16 Jun 2022 12:33:58 +0100 Subject: [PATCH 049/560] mount: allow tests to run on CI --- cmd/mount/mount_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cmd/mount/mount_test.go b/cmd/mount/mount_test.go index 64f53eadcb841..ec5422cc9189b 100644 --- a/cmd/mount/mount_test.go +++ b/cmd/mount/mount_test.go @@ -4,17 +4,11 @@ package mount import ( - "runtime" "testing" - "github.com/rclone/rclone/fstest/testy" "github.com/rclone/rclone/vfs/vfstest" ) func TestMount(t *testing.T) { - if runtime.NumCPU() <= 2 { - t.Skip("FIXME skipping mount tests as they lock up on <= 2 CPUs - See: https://github.com/rclone/rclone/issues/3154") - } - testy.SkipUnreliable(t) vfstest.RunTests(t, false, mount) } From 2e91287b2eec71d8a57bd2b8e1958a9d9bf19d43 Mon Sep 17 00:00:00 2001 From: Maciej Radzikowski Date: Thu, 16 Jun 2022 22:29:36 +0200 Subject: [PATCH 050/560] docs/s3: add note about chunk size decreasing progress accuracy --- backend/s3/s3.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index cd4f3e06b9ca3..ac5e112cb2c66 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1677,7 +1677,14 @@ Files of unknown size are uploaded with the configured chunk_size. Since the default chunk size is 5 MiB and there can be at most 10,000 chunks, this means that by default the maximum size of a file you can stream upload is 48 GiB. If you wish to stream upload -larger files then you will need to increase chunk_size.`, +larger files then you will need to increase chunk_size. + +Increasing the chunk size decreases the accuracy of the progress +statistics displayed with "-P" flag. Rclone treats chunk as sent when +it's buffered by the AWS SDK, when in fact it may still be uploading. +A bigger chunk size means a bigger AWS SDK buffer and progress +reporting more deviating from the truth. +`, Default: minChunkSize, Advanced: true, }, { From e87e331f4c566ba96752a6dadbc7c99f377ab1ff Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 14 Jun 2022 09:44:37 +0100 Subject: [PATCH 051/560] drive: make --drive-shared-with-me work with shared drives Fixes #6247 --- backend/drive/drive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index c5f5827f7b65b..176b94e6e890c 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -886,7 +886,7 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie } list.SupportsAllDrives(true) list.IncludeItemsFromAllDrives(true) - if f.isTeamDrive { + if f.isTeamDrive && !f.opt.SharedWithMe { list.DriveId(f.opt.TeamDriveID) list.Corpora("drive") } From 411013dbdceb6168219639744dbcbbb24fc495ae Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 23 May 2022 17:24:53 +0100 Subject: [PATCH 052/560] drive: add --drive-resource-key for accessing link-shared files --- backend/drive/drive.go | 61 +++++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 176b94e6e890c..2a4d0a64b61b4 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -565,6 +565,27 @@ If this is set then rclone will not show any dangling shortcuts in listings. `, Advanced: true, Default: false, + }, { + Name: "resource_key", + Help: `Resource key for accessing a link-shared file. + +If you need to access files shared with a link like this + + https://drive.google.com/drive/folders/XXX?resourcekey=YYY&usp=sharing + +Then you will need to use the first part "XXX" as the "root_folder_id" +and the second part "YYY" as the "resource_key" otherwise you will get +404 not found errors when trying to access the directory. + +See: https://developers.google.com/drive/api/guides/resource-keys + +This resource key requirement only applies to a subset of old files. + +Note also that opening the folder once in the web interface (with the +user you've authenticated rclone with) seems to be enough so that the +resource key is no needed. +`, + Advanced: true, }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, @@ -626,6 +647,7 @@ type Options struct { StopOnDownloadLimit bool `config:"stop_on_download_limit"` SkipShortcuts bool `config:"skip_shortcuts"` SkipDanglingShortcuts bool `config:"skip_dangling_shortcuts"` + ResourceKey string `config:"resource_key"` Enc encoder.MultiEncoder `config:"encoding"` } @@ -651,6 +673,7 @@ type Fs struct { grouping int32 // number of IDs to search at once in ListR - read with atomic listRmu *sync.Mutex // protects listRempties listRempties map[string]struct{} // IDs of supposedly empty directories which triggered grouping disable + dirResourceKeys *sync.Map // map directory ID to resource key } type baseObject struct { @@ -802,6 +825,7 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie // We must not filter with parent when we try list "ROOT" with drive-shared-with-me // If we need to list file inside those shared folders, we must search it without sharedWithMe parentsQuery := bytes.NewBufferString("(") + var resourceKeys []string for _, dirID := range dirIDs { if dirID == "" { continue @@ -822,7 +846,12 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie } else { _, _ = fmt.Fprintf(parentsQuery, "'%s' in parents", dirID) } + resourceKey, hasResourceKey := f.dirResourceKeys.Load(dirID) + if hasResourceKey { + resourceKeys = append(resourceKeys, fmt.Sprintf("%s/%s", dirID, resourceKey)) + } } + resourceKeysHeader := strings.Join(resourceKeys, ",") if parentsQuery.Len() > 1 { _ = parentsQuery.WriteByte(')') query = append(query, parentsQuery.String()) @@ -894,6 +923,10 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie if f.rootFolderID == "appDataFolder" { list.Spaces("appDataFolder") } + // Add resource Keys if necessary + if resourceKeysHeader != "" { + list.Header().Add("X-Goog-Drive-Resource-Keys", resourceKeysHeader) + } fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.fileFields) @@ -1153,15 +1186,16 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err ci := fs.GetConfig(ctx) f := &Fs{ - name: name, - root: root, - opt: *opt, - ci: ci, - pacer: fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst))), - m: m, - grouping: listRGrouping, - listRmu: new(sync.Mutex), - listRempties: make(map[string]struct{}), + name: name, + root: root, + opt: *opt, + ci: ci, + pacer: fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst))), + m: m, + grouping: listRGrouping, + listRmu: new(sync.Mutex), + listRempties: make(map[string]struct{}), + dirResourceKeys: new(sync.Map), } f.isTeamDrive = opt.TeamDriveID != "" f.fileFields = f.getFileFields() @@ -1223,6 +1257,11 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e f.dirCache = dircache.New(f.root, f.rootFolderID, f) + // If resource key is set then cache it for the root folder id + if f.opt.ResourceKey != "" { + f.dirResourceKeys.Store(f.rootFolderID, f.opt.ResourceKey) + } + // Parse extensions if f.opt.Extensions != "" { if f.opt.ExportExtensions != defaultExportExtensions { @@ -2091,6 +2130,10 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File case item.MimeType == driveFolderType: // cache the directory ID for later lookups f.dirCache.Put(remote, item.Id) + // cache the resource key for later lookups + if item.ResourceKey != "" { + f.dirResourceKeys.Store(item.Id, item.ResourceKey) + } when, _ := time.Parse(timeFormatIn, item.ModifiedTime) d := fs.NewDir(remote, when).SetID(item.Id) if len(item.Parents) > 0 { From 4d72abf3899becdc6e8183057e326e281694c920 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 5 May 2022 17:12:57 +0100 Subject: [PATCH 053/560] dropbox: fix nil pointer exception on dropbox impersonate user not found Fixes #6139 --- backend/dropbox/dropbox.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 000bfd5e42eeb..69d0b4e7d326c 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -472,10 +472,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e args := team.NewMembersGetInfoArgs(members) memberIds, err := f.team.MembersGetInfo(args) - if err != nil { return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err) } + if len(memberIds) == 0 || memberIds[0].MemberInfo == nil || memberIds[0].MemberInfo.Profile == nil { + return nil, fmt.Errorf("dropbox team member not found: %q", opt.Impersonate) + } cfg.AsMemberID = memberIds[0].MemberInfo.Profile.MemberProfile.TeamMemberId } From 4f94b2780035114b8aa963a00c5725c5b84f9a3c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 26 Apr 2022 09:22:30 +0100 Subject: [PATCH 054/560] check: implement --no-traverse and --no-unicode-normalization See: https://forum.rclone.org/t/rclone-check-head-or-list-object-from-source/30400 --- fs/operations/check.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fs/operations/check.go b/fs/operations/check.go index 12c63e70182d2..a7a52e299b28d 100644 --- a/fs/operations/check.go +++ b/fs/operations/check.go @@ -219,11 +219,13 @@ func CheckFn(ctx context.Context, opt *CheckOpt) error { // set up a march over fdst and fsrc m := &march.March{ - Ctx: ctx, - Fdst: c.opt.Fdst, - Fsrc: c.opt.Fsrc, - Dir: "", - Callback: c, + Ctx: ctx, + Fdst: c.opt.Fdst, + Fsrc: c.opt.Fsrc, + Dir: "", + Callback: c, + NoTraverse: ci.NoTraverse, + NoUnicodeNormalization: ci.NoUnicodeNormalization, } fs.Debugf(c.opt.Fdst, "Waiting for checks to finish") err := m.Run(ctx) From 1d2fe0d8564bc679ece166c24b24e6fe7dc1455c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 3 Apr 2022 13:39:42 +0100 Subject: [PATCH 055/560] union: enable passing of options to upstreams and policies #6071 This factors out the options into a sub package so they can be passed to upstreams and used in policies. --- backend/union/common/options.go | 16 ++++++++++++++++ backend/union/union.go | 17 ++++------------- backend/union/upstream/upstream.go | 7 +++++-- 3 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 backend/union/common/options.go diff --git a/backend/union/common/options.go b/backend/union/common/options.go new file mode 100644 index 0000000000000..cc275cc3722b1 --- /dev/null +++ b/backend/union/common/options.go @@ -0,0 +1,16 @@ +// Package common defines code common to the union and the policies +// +// These need to be defined in a separate package to avoid import loops +package common + +import "github.com/rclone/rclone/fs" + +// Options defines the configuration for this backend +type Options struct { + Upstreams fs.SpaceSepList `config:"upstreams"` + Remotes fs.SpaceSepList `config:"remotes"` // Deprecated + ActionPolicy string `config:"action_policy"` + CreatePolicy string `config:"create_policy"` + SearchPolicy string `config:"search_policy"` + CacheTime int `config:"cache_time"` +} diff --git a/backend/union/union.go b/backend/union/union.go index ed17cd02099e5..dc4e2c5896e9d 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -13,6 +13,7 @@ import ( "sync" "time" + "github.com/rclone/rclone/backend/union/common" "github.com/rclone/rclone/backend/union/policy" "github.com/rclone/rclone/backend/union/upstream" "github.com/rclone/rclone/fs" @@ -54,21 +55,11 @@ func init() { fs.Register(fsi) } -// Options defines the configuration for this backend -type Options struct { - Upstreams fs.SpaceSepList `config:"upstreams"` - Remotes fs.SpaceSepList `config:"remotes"` // Deprecated - ActionPolicy string `config:"action_policy"` - CreatePolicy string `config:"create_policy"` - SearchPolicy string `config:"search_policy"` - CacheTime int `config:"cache_time"` -} - // Fs represents a union of upstreams type Fs struct { name string // name of this remote features *fs.Features // optional features - opt Options // options for this Fs + opt common.Options // options for this Fs root string // the path we are working on upstreams []*upstream.Fs // slice of upstreams hashSet hash.Set // intersection of hash types @@ -808,7 +799,7 @@ func (f *Fs) Shutdown(ctx context.Context) error { // The returned Fs is the actual Fs, referenced by remote in the config func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct - opt := new(Options) + opt := new(common.Options) err := configstruct.Set(m, opt) if err != nil { return nil, err @@ -836,7 +827,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e errs := Errors(make([]error, len(opt.Upstreams))) multithread(len(opt.Upstreams), func(i int) { u := opt.Upstreams[i] - upstreams[i], errs[i] = upstream.New(ctx, u, root, time.Duration(opt.CacheTime)*time.Second) + upstreams[i], errs[i] = upstream.New(ctx, u, root, opt) }) var usedUpstreams []*upstream.Fs var fserr error diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go index 86575c512c31f..41f30f4009ce5 100644 --- a/backend/union/upstream/upstream.go +++ b/backend/union/upstream/upstream.go @@ -11,6 +11,7 @@ import ( "sync/atomic" "time" + "github.com/rclone/rclone/backend/union/common" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/fspath" @@ -26,6 +27,7 @@ type Fs struct { fs.Fs RootFs fs.Fs RootPath string + Opt *common.Options writable bool creatable bool usage *fs.Usage // Cache the usage @@ -61,17 +63,18 @@ type Entry interface { // New creates a new Fs based on the // string formatted `type:root_path(:ro/:nc)` -func New(ctx context.Context, remote, root string, cacheTime time.Duration) (*Fs, error) { +func New(ctx context.Context, remote, root string, opt *common.Options) (*Fs, error) { configName, fsPath, err := fspath.SplitFs(remote) if err != nil { return nil, err } f := &Fs{ RootPath: strings.TrimRight(root, "/"), + Opt: opt, writable: true, creatable: true, cacheExpiry: time.Now().Unix(), - cacheTime: cacheTime, + cacheTime: time.Duration(opt.CacheTime) * time.Second, usage: &fs.Usage{}, } if strings.HasSuffix(fsPath, ":ro") { From 1e1af46a12afcd88d92fc22cffd62ca8b4e9396d Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 4 Apr 2022 09:32:50 +0100 Subject: [PATCH 056/560] union: fix get free space for remotes which don't support it #6071 Before this fix GetFreeSpace returned math.MaxInt64 for remotes which don't support reading free space, however this is used in various comparison routines as a too large value, meaning that remotes of size math.MaxInt64 were never being selected. This fixes GetFreeSpace to return math.MaxInt64 - 1 so then can be selected. It also fixes GetUsedSpace the same way however as the default for not supported was 0 this was very unlikely to have ever caused a problem. --- backend/union/upstream/upstream.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go index 41f30f4009ce5..1a85d783fe44b 100644 --- a/backend/union/upstream/upstream.go +++ b/backend/union/upstream/upstream.go @@ -259,22 +259,30 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { } // GetFreeSpace get the free space of the fs +// +// This is returned as 0..math.MaxInt64-1 leaving math.MaxInt64 as a sentinel func (f *Fs) GetFreeSpace() (int64, error) { if atomic.LoadInt64(&f.cacheExpiry) <= time.Now().Unix() { err := f.updateUsage() if err != nil { - return math.MaxInt64, ErrUsageFieldNotSupported + return math.MaxInt64 - 1, ErrUsageFieldNotSupported } } f.cacheMutex.RLock() defer f.cacheMutex.RUnlock() if f.usage.Free == nil { - return math.MaxInt64, ErrUsageFieldNotSupported + return math.MaxInt64 - 1, ErrUsageFieldNotSupported + } + free := *f.usage.Free + if free >= math.MaxInt64 { + free = math.MaxInt64 - 1 } - return *f.usage.Free, nil + return free, nil } // GetUsedSpace get the used space of the fs +// +// This is returned as 0..math.MaxInt64-1 leaving math.MaxInt64 as a sentinel func (f *Fs) GetUsedSpace() (int64, error) { if atomic.LoadInt64(&f.cacheExpiry) <= time.Now().Unix() { err := f.updateUsage() @@ -287,7 +295,11 @@ func (f *Fs) GetUsedSpace() (int64, error) { if f.usage.Used == nil { return 0, ErrUsageFieldNotSupported } - return *f.usage.Used, nil + used := *f.usage.Used + if used >= math.MaxInt64 { + used = math.MaxInt64 - 1 + } + return used, nil } // GetNumObjects get the number of objects of the fs From 29e37749b3f83600d26b17c0fd62971ad93cb1f8 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 4 Apr 2022 09:51:56 +0100 Subject: [PATCH 057/560] union: fix eplus policy to select correct entry for existing files #6071 --- backend/union/policy/eplus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/union/policy/eplus.go b/backend/union/policy/eplus.go index d1aad2c9e0708..57f97c23190f4 100644 --- a/backend/union/policy/eplus.go +++ b/backend/union/policy/eplus.go @@ -42,7 +42,7 @@ func (p *EpLus) lusEntries(entries []upstream.Entry) (upstream.Entry, error) { var minUsedSpace int64 = math.MaxInt64 var lusEntry upstream.Entry for _, e := range entries { - space, err := e.UpstreamFs().GetFreeSpace() + space, err := e.UpstreamFs().GetUsedSpace() if err != nil { fs.LogPrintf(fs.LogLevelNotice, nil, "Used Space is not supported for upstream %s, treating as 0", e.UpstreamFs().Name()) From 16514290411b99b5887653043647429e8c6cba91 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 4 Apr 2022 09:55:05 +0100 Subject: [PATCH 058/560] union: add min_free_space option for lfs/eplfs policies - fixes #6071 --- backend/union/common/options.go | 1 + backend/union/policy/eplfs.go | 19 +++++++++++++------ backend/union/union.go | 8 ++++++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/backend/union/common/options.go b/backend/union/common/options.go index cc275cc3722b1..edc715203f430 100644 --- a/backend/union/common/options.go +++ b/backend/union/common/options.go @@ -13,4 +13,5 @@ type Options struct { CreatePolicy string `config:"create_policy"` SearchPolicy string `config:"search_policy"` CacheTime int `config:"cache_time"` + MinFreeSpace fs.SizeSuffix `config:"min_free_space"` } diff --git a/backend/union/policy/eplfs.go b/backend/union/policy/eplfs.go index a2788138c2406..28d84ff5c7bb6 100644 --- a/backend/union/policy/eplfs.go +++ b/backend/union/policy/eplfs.go @@ -2,6 +2,7 @@ package policy import ( "context" + "errors" "math" "github.com/rclone/rclone/backend/union/upstream" @@ -18,6 +19,8 @@ type EpLfs struct { EpAll } +var errNoUpstreamsFound = errors.New("no upstreams found with more than min_free_space space spare") + func (p *EpLfs) lfs(upstreams []*upstream.Fs) (*upstream.Fs, error) { var minFreeSpace int64 = math.MaxInt64 var lfsupstream *upstream.Fs @@ -27,31 +30,35 @@ func (p *EpLfs) lfs(upstreams []*upstream.Fs) (*upstream.Fs, error) { fs.LogPrintf(fs.LogLevelNotice, nil, "Free Space is not supported for upstream %s, treating as infinite", u.Name()) } - if space < minFreeSpace { + if space < minFreeSpace && space > int64(u.Opt.MinFreeSpace) { minFreeSpace = space lfsupstream = u } } if lfsupstream == nil { - return nil, fs.ErrorObjectNotFound + return nil, errNoUpstreamsFound } return lfsupstream, nil } func (p *EpLfs) lfsEntries(entries []upstream.Entry) (upstream.Entry, error) { - var minFreeSpace int64 + var minFreeSpace int64 = math.MaxInt64 var lfsEntry upstream.Entry for _, e := range entries { - space, err := e.UpstreamFs().GetFreeSpace() + u := e.UpstreamFs() + space, err := u.GetFreeSpace() if err != nil { fs.LogPrintf(fs.LogLevelNotice, nil, - "Free Space is not supported for upstream %s, treating as infinite", e.UpstreamFs().Name()) + "Free Space is not supported for upstream %s, treating as infinite", u.Name()) } - if space < minFreeSpace { + if space < minFreeSpace && space > int64(u.Opt.MinFreeSpace) { minFreeSpace = space lfsEntry = e } } + if lfsEntry == nil { + return nil, errNoUpstreamsFound + } return lfsEntry, nil } diff --git a/backend/union/union.go b/backend/union/union.go index dc4e2c5896e9d..9776c55d2a54f 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -50,6 +50,14 @@ func init() { Name: "cache_time", Help: "Cache time of usage and free space (in seconds).\n\nThis option is only useful when a path preserving policy is used.", Default: 120, + }, { + Name: "min_free_space", + Help: `Minimum viable free space for lfs/eplfs policies. + +If a remote has less than this much free space then it won't be +considered for use in lfs or eplfs policies.`, + Advanced: true, + Default: fs.Gibi, }}, } fs.Register(fsi) From 95e093475596a1764931908abb0426bb3ea0fe7c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 28 Mar 2022 12:47:22 +0100 Subject: [PATCH 059/560] sftp: add --sftp-chunk-size to control packets sizes for high latency links See: https://forum.rclone.org/t/increasing-sftp-transfer-speed/29928 --- backend/sftp/sftp.go | 72 +++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 305fbde983925..ae617069d38ec 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -272,6 +272,26 @@ given, rclone will empty the connection pool. Set to 0 to keep connections indefinitely. `, Advanced: true, + }, { + Name: "chunk_size", + Help: `Upload and download chunk size. + +This controls the maximum packet size used in the SFTP protocol. The +RFC limits this to 32768 bytes (32k), however a lot of servers +support larger sizes and setting it larger will increase transfer +speed dramatically on high latency links. + +Only use a setting higher than 32k if you always connect to the same +server or after sufficiently broad testing. + +For example using the value of 252k with OpenSSH works well with its +maximum packet size of 256k. + +If you get the error "failed to send packet header: EOF" when copying +a large file, try lowering this number. +`, + Default: 32 * fs.Kibi, + Advanced: true, }}, } fs.Register(fsi) @@ -279,31 +299,32 @@ Set to 0 to keep connections indefinitely. // Options defines the configuration for this backend type Options struct { - Host string `config:"host"` - User string `config:"user"` - Port string `config:"port"` - Pass string `config:"pass"` - KeyPem string `config:"key_pem"` - KeyFile string `config:"key_file"` - KeyFilePass string `config:"key_file_pass"` - PubKeyFile string `config:"pubkey_file"` - KnownHostsFile string `config:"known_hosts_file"` - KeyUseAgent bool `config:"key_use_agent"` - UseInsecureCipher bool `config:"use_insecure_cipher"` - DisableHashCheck bool `config:"disable_hashcheck"` - AskPassword bool `config:"ask_password"` - PathOverride string `config:"path_override"` - SetModTime bool `config:"set_modtime"` - ShellType string `config:"shell_type"` - Md5sumCommand string `config:"md5sum_command"` - Sha1sumCommand string `config:"sha1sum_command"` - SkipLinks bool `config:"skip_links"` - Subsystem string `config:"subsystem"` - ServerCommand string `config:"server_command"` - UseFstat bool `config:"use_fstat"` - DisableConcurrentReads bool `config:"disable_concurrent_reads"` - DisableConcurrentWrites bool `config:"disable_concurrent_writes"` - IdleTimeout fs.Duration `config:"idle_timeout"` + Host string `config:"host"` + User string `config:"user"` + Port string `config:"port"` + Pass string `config:"pass"` + KeyPem string `config:"key_pem"` + KeyFile string `config:"key_file"` + KeyFilePass string `config:"key_file_pass"` + PubKeyFile string `config:"pubkey_file"` + KnownHostsFile string `config:"known_hosts_file"` + KeyUseAgent bool `config:"key_use_agent"` + UseInsecureCipher bool `config:"use_insecure_cipher"` + DisableHashCheck bool `config:"disable_hashcheck"` + AskPassword bool `config:"ask_password"` + PathOverride string `config:"path_override"` + SetModTime bool `config:"set_modtime"` + ShellType string `config:"shell_type"` + Md5sumCommand string `config:"md5sum_command"` + Sha1sumCommand string `config:"sha1sum_command"` + SkipLinks bool `config:"skip_links"` + Subsystem string `config:"subsystem"` + ServerCommand string `config:"server_command"` + UseFstat bool `config:"use_fstat"` + DisableConcurrentReads bool `config:"disable_concurrent_reads"` + DisableConcurrentWrites bool `config:"disable_concurrent_writes"` + IdleTimeout fs.Duration `config:"idle_timeout"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` } // Fs stores the interface to the remote SFTP files @@ -481,6 +502,7 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C sftp.UseFstat(f.opt.UseFstat), sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads), sftp.UseConcurrentWrites(!f.opt.DisableConcurrentWrites), + sftp.MaxPacketUnchecked(int(f.opt.ChunkSize)), ) return sftp.NewClientPipe(pr, pw, opts...) } From 78120d40d9bd5171d58aa4839c4766a4eaef4461 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 28 Mar 2022 12:57:34 +0100 Subject: [PATCH 060/560] sftp: add --sftp-concurrency to improve high latency transfers See: https://forum.rclone.org/t/increasing-sftp-transfer-speed/29928 --- backend/sftp/sftp.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index ae617069d38ec..a228dece18182 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -292,6 +292,16 @@ a large file, try lowering this number. `, Default: 32 * fs.Kibi, Advanced: true, + }, { + Name: "concurrency", + Help: `The maximum number of outstanding requests for one file + +This controls the maximum number of outstanding requests for one file. +Increasing it will increase throughput on high latency links at the +cost of using more memory. +`, + Default: 64, + Advanced: true, }}, } fs.Register(fsi) @@ -325,6 +335,7 @@ type Options struct { DisableConcurrentWrites bool `config:"disable_concurrent_writes"` IdleTimeout fs.Duration `config:"idle_timeout"` ChunkSize fs.SizeSuffix `config:"chunk_size"` + Concurrency int `config:"concurrency"` } // Fs stores the interface to the remote SFTP files @@ -503,6 +514,7 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C sftp.UseConcurrentReads(!f.opt.DisableConcurrentReads), sftp.UseConcurrentWrites(!f.opt.DisableConcurrentWrites), sftp.MaxPacketUnchecked(int(f.opt.ChunkSize)), + sftp.MaxConcurrentRequestsPerFile(f.opt.Concurrency), ) return sftp.NewClientPipe(pr, pw, opts...) } From 60d87185e154109d178e2b92d185877f305253a1 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 11 Apr 2022 10:41:17 +0100 Subject: [PATCH 061/560] sftp: add --sftp-set-env option to set environment variables Fixes #6094 --- backend/sftp/sftp.go | 100 +++++++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 27 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index a228dece18182..5e9cbe18dfd95 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -302,6 +302,27 @@ cost of using more memory. `, Default: 64, Advanced: true, + }, { + Name: "set_env", + Default: fs.SpaceSepList{}, + Help: `Environment variables to pass to sftp and commands + +Set environment variables in the form: + + VAR=value + +to be passed to the sftp client and to any commands run (eg md5sum). + +Pass multiple variables space separated, eg + + VAR1=value VAR2=value + +and pass variables with spaces in in quotes, eg + + "VAR3=value with space" "VAR4=value with space" VAR5=nospacehere + +`, + Advanced: true, }}, } fs.Register(fsi) @@ -309,33 +330,34 @@ cost of using more memory. // Options defines the configuration for this backend type Options struct { - Host string `config:"host"` - User string `config:"user"` - Port string `config:"port"` - Pass string `config:"pass"` - KeyPem string `config:"key_pem"` - KeyFile string `config:"key_file"` - KeyFilePass string `config:"key_file_pass"` - PubKeyFile string `config:"pubkey_file"` - KnownHostsFile string `config:"known_hosts_file"` - KeyUseAgent bool `config:"key_use_agent"` - UseInsecureCipher bool `config:"use_insecure_cipher"` - DisableHashCheck bool `config:"disable_hashcheck"` - AskPassword bool `config:"ask_password"` - PathOverride string `config:"path_override"` - SetModTime bool `config:"set_modtime"` - ShellType string `config:"shell_type"` - Md5sumCommand string `config:"md5sum_command"` - Sha1sumCommand string `config:"sha1sum_command"` - SkipLinks bool `config:"skip_links"` - Subsystem string `config:"subsystem"` - ServerCommand string `config:"server_command"` - UseFstat bool `config:"use_fstat"` - DisableConcurrentReads bool `config:"disable_concurrent_reads"` - DisableConcurrentWrites bool `config:"disable_concurrent_writes"` - IdleTimeout fs.Duration `config:"idle_timeout"` - ChunkSize fs.SizeSuffix `config:"chunk_size"` - Concurrency int `config:"concurrency"` + Host string `config:"host"` + User string `config:"user"` + Port string `config:"port"` + Pass string `config:"pass"` + KeyPem string `config:"key_pem"` + KeyFile string `config:"key_file"` + KeyFilePass string `config:"key_file_pass"` + PubKeyFile string `config:"pubkey_file"` + KnownHostsFile string `config:"known_hosts_file"` + KeyUseAgent bool `config:"key_use_agent"` + UseInsecureCipher bool `config:"use_insecure_cipher"` + DisableHashCheck bool `config:"disable_hashcheck"` + AskPassword bool `config:"ask_password"` + PathOverride string `config:"path_override"` + SetModTime bool `config:"set_modtime"` + ShellType string `config:"shell_type"` + Md5sumCommand string `config:"md5sum_command"` + Sha1sumCommand string `config:"sha1sum_command"` + SkipLinks bool `config:"skip_links"` + Subsystem string `config:"subsystem"` + ServerCommand string `config:"server_command"` + UseFstat bool `config:"use_fstat"` + DisableConcurrentReads bool `config:"disable_concurrent_reads"` + DisableConcurrentWrites bool `config:"disable_concurrent_writes"` + IdleTimeout fs.Duration `config:"idle_timeout"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` + Concurrency int `config:"concurrency"` + SetEnv fs.SpaceSepList `config:"set_env"` } // Fs stores the interface to the remote SFTP files @@ -483,6 +505,22 @@ func (f *Fs) sftpConnection(ctx context.Context) (c *conn, err error) { return c, nil } +// Set any environment variables on the ssh.Session +func (f *Fs) setEnv(s *ssh.Session) error { + for _, env := range f.opt.SetEnv { + equal := strings.IndexRune(env, '=') + if equal < 0 { + return fmt.Errorf("no = found in env var %q", env) + } + // fs.Debugf(f, "Setting env %q = %q", env[:equal], env[equal+1:]) + err := s.Setenv(env[:equal], env[equal+1:]) + if err != nil { + return fmt.Errorf("Failed to set env var %q: %w", env[:equal], err) + } + } + return nil +} + // Creates a new SFTP client on conn, using the specified subsystem // or sftp server, and zero or more option functions func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.Client, error) { @@ -490,6 +528,10 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C if err != nil { return nil, err } + err = f.setEnv(s) + if err != nil { + return nil, err + } pw, err := s.StdinPipe() if err != nil { return nil, err @@ -1243,6 +1285,10 @@ func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) { if err != nil { return nil, fmt.Errorf("run: get SFTP session: %w", err) } + err = f.setEnv(session) + if err != nil { + return nil, err + } defer func() { _ = session.Close() }() From 3f6186917992a61793a29240705b43562d5a31fb Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 15 Apr 2022 12:11:14 +0100 Subject: [PATCH 062/560] cmount: add tracing for *xattr FUSE callbacks See: https://github.com/winfsp/cgofuse/issues/66 See: https://forum.rclone.org/t/cannot-copy-files-to-mounted-azure-storage-windows/30092 --- cmd/cmount/fs.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/cmount/fs.go b/cmd/cmount/fs.go index 553712b1636b2..c097611b754fd 100644 --- a/cmd/cmount/fs.go +++ b/cmd/cmount/fs.go @@ -545,21 +545,25 @@ func (fsys *FS) Fsyncdir(path string, datasync bool, fh uint64) (errc int) { // Setxattr sets extended attributes. func (fsys *FS) Setxattr(path string, name string, value []byte, flags int) (errc int) { + defer log.Trace(path, "name=%q, value=%q, flags=%d", name, value, flags)("errc=%d", &errc) return -fuse.ENOSYS } // Getxattr gets extended attributes. func (fsys *FS) Getxattr(path string, name string) (errc int, value []byte) { + defer log.Trace(path, "name=%q", name)("errc=%d, value=%q", &errc, &value) return -fuse.ENOSYS, nil } // Removexattr removes extended attributes. func (fsys *FS) Removexattr(path string, name string) (errc int) { + defer log.Trace(path, "name=%q", name)("errc=%d", &errc) return -fuse.ENOSYS } // Listxattr lists extended attributes. func (fsys *FS) Listxattr(path string, fill func(name string) bool) (errc int) { + defer log.Trace(path, "fill=%p", fill)("errc=%d", &errc) return -fuse.ENOSYS } From 4ac875a811ad10589186b5a199e8b1bec33ff226 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 29 Nov 2021 16:19:00 +0000 Subject: [PATCH 063/560] sync: fix --max-duration and --cutoff-mode soft Before this change using --max-duration and --cutoff-mode soft would work like --cutoff-mode hard. This bug was introduced in this commit which made transfers be cancelable - before that transfers couldn't be canceled. 122a47fba655704b accounting: Allow transfers to be canceled with context #3257 This change adds the timeout to the input context for reading files rather than the transfer context so the files transfers themselves aren't canceled if --cutoff-mode soft is in action. This also adds a test for cutoff mode soft and max duration which was missing. See: https://forum.rclone.org/t/max-duration-and-retries-not-working-correctly/27738 --- fs/sync/sync.go | 27 +++++++++++++++++++------ fs/sync/sync_test.go | 47 ++++++++++++++++++++++++++++++-------------- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/fs/sync/sync.go b/fs/sync/sync.go index f9844ea165b66..58e74e22a6377 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -145,16 +145,29 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete if err != nil { return nil, err } - // If a max session duration has been defined add a deadline to the context if ci.MaxDuration > 0 { s.maxDurationEndTime = time.Now().Add(ci.MaxDuration) - fs.Infof(s.fdst, "Transfer session deadline: %s", s.maxDurationEndTime.Format("2006/01/02 15:04:05")) + fs.Infof(s.fdst, "Transfer session %v deadline: %s", ci.CutoffMode, s.maxDurationEndTime.Format("2006/01/02 15:04:05")) + } + // If a max session duration has been defined add a deadline + // to the main context if cutoff mode is hard. This will cut + // the transfers off. + if !s.maxDurationEndTime.IsZero() && ci.CutoffMode == fs.CutoffModeHard { s.ctx, s.cancel = context.WithDeadline(ctx, s.maxDurationEndTime) } else { s.ctx, s.cancel = context.WithCancel(ctx) } - // Input context - cancel this for graceful stop - s.inCtx, s.inCancel = context.WithCancel(s.ctx) + // Input context - cancel this for graceful stop. + // + // If a max session duration has been defined add a deadline + // to the input context if cutoff mode is graceful or soft. + // This won't stop the transfers but will cut the + // list/check/transfer pipelines. + if !s.maxDurationEndTime.IsZero() && ci.CutoffMode != fs.CutoffModeHard { + s.inCtx, s.inCancel = context.WithDeadline(s.ctx, s.maxDurationEndTime) + } else { + s.inCtx, s.inCancel = context.WithCancel(s.ctx) + } if s.noTraverse && s.deleteMode != fs.DeleteModeOff { if !fi.HaveFilesFrom() { fs.Errorf(nil, "Ignoring --no-traverse with sync") @@ -904,8 +917,9 @@ func (s *syncCopyMove) run() error { s.processError(s.deleteEmptyDirectories(s.ctx, s.fsrc, s.srcEmptyDirs)) } - // Read the error out of the context if there is one + // Read the error out of the contexts if there is one s.processError(s.ctx.Err()) + s.processError(s.inCtx.Err()) // If the duration was exceeded then add a Fatal Error so we don't retry if !s.maxDurationEndTime.IsZero() && time.Since(s.maxDurationEndTime) > 0 { @@ -918,7 +932,8 @@ func (s *syncCopyMove) run() error { fs.Infof(nil, "There was nothing to transfer") } - // cancel the context to free resources + // cancel the contexts to free resources + s.inCancel() s.cancel() return s.currentError() } diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 4bf60c47121da..72e6a65e781f3 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -1002,7 +1002,7 @@ func TestSyncWithUpdateOlder(t *testing.T) { } // Test with a max transfer duration -func TestSyncWithMaxDuration(t *testing.T) { +func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) { ctx := context.Background() ctx, ci := fs.AddConfig(ctx) if *fstest.RemoteName != "" { @@ -1013,32 +1013,49 @@ func TestSyncWithMaxDuration(t *testing.T) { maxDuration := 250 * time.Millisecond ci.MaxDuration = maxDuration - bytesPerSecond := 300 - accounting.TokenBucket.SetBwLimit(fs.BwPair{Tx: fs.SizeSuffix(bytesPerSecond), Rx: fs.SizeSuffix(bytesPerSecond)}) + ci.CutoffMode = cutoffMode + ci.CheckFirst = true + ci.OrderBy = "size" ci.Transfers = 1 + ci.Checkers = 1 + bytesPerSecond := 10 * 1024 + accounting.TokenBucket.SetBwLimit(fs.BwPair{Tx: fs.SizeSuffix(bytesPerSecond), Rx: fs.SizeSuffix(bytesPerSecond)}) defer accounting.TokenBucket.SetBwLimit(fs.BwPair{Tx: -1, Rx: -1}) - // 5 files of 60 bytes at 60 Byte/s 5 seconds - testFiles := make([]fstest.Item, 5) - for i := 0; i < len(testFiles); i++ { - testFiles[i] = r.WriteFile(fmt.Sprintf("file%d", i), "------------------------------------------------------------", t1) - } - - fstest.CheckListing(t, r.Flocal, testFiles) + // write one small file which we expect to transfer and one big one which we don't + file1 := r.WriteFile("file1", string(make([]byte, 16)), t1) + file2 := r.WriteFile("file2", string(make([]byte, 50*1024)), t1) + r.CheckLocalItems(t, file1, file2) + r.CheckRemoteItems(t) accounting.GlobalStats().ResetCounters() startTime := time.Now() err := Sync(ctx, r.Fremote, r.Flocal, false) require.True(t, errors.Is(err, errorMaxDurationReached)) + if cutoffMode == fs.CutoffModeHard { + r.CheckRemoteItems(t, file1) + assert.Equal(t, int64(1), accounting.GlobalStats().GetTransfers()) + } else { + r.CheckRemoteItems(t, file1, file2) + assert.Equal(t, int64(2), accounting.GlobalStats().GetTransfers()) + } + elapsed := time.Since(startTime) - maxTransferTime := (time.Duration(len(testFiles)) * 60 * time.Second) / time.Duration(bytesPerSecond) + const maxTransferTime = 20 * time.Second what := fmt.Sprintf("expecting elapsed time %v between %v and %v", elapsed, maxDuration, maxTransferTime) - require.True(t, elapsed >= maxDuration, what) - require.True(t, elapsed < 5*time.Second, what) - // we must not have transferred all files during the session - require.True(t, accounting.GlobalStats().GetTransfers() < int64(len(testFiles))) + assert.True(t, elapsed >= maxDuration, what) + assert.True(t, elapsed < maxTransferTime, what) +} + +func TestSyncWithMaxDuration(t *testing.T) { + t.Run("Hard", func(t *testing.T) { + testSyncWithMaxDuration(t, fs.CutoffModeHard) + }) + t.Run("Soft", func(t *testing.T) { + testSyncWithMaxDuration(t, fs.CutoffModeSoft) + }) } // Test with TrackRenames set From fa48b880c2dc27ce5c3bc38b5b84bac95e2b271e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 24 Nov 2021 12:48:57 +0000 Subject: [PATCH 064/560] s3: retry RequestTimeout errors See: https://forum.rclone.org/t/s3-failed-upload-large-files-bad-request-400/27695 --- backend/s3/s3.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index ac5e112cb2c66..dd894db18156a 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2121,6 +2121,10 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) { if fserrors.ShouldRetry(awsError.OrigErr()) { return true, err } + // If it is a timeout then we want to retry that + if awsError.Code() == "RequestTimeout" { + return true, err + } // Failing that, if it's a RequestFailure it's probably got an http status code we can check if reqErr, ok := err.(awserr.RequestFailure); ok { // 301 if wrong region for bucket - can only update if running from a bucket From e7483b40b34f7c01ad08a69c83b56d9fc79edc01 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 30 May 2022 10:14:37 +0100 Subject: [PATCH 065/560] fshttp: add --disable-http-keep-alives to disable HTTP Keep Alives See: https://forum.rclone.org/t/getting-rate-limited-before-advertised-limit-on-s3-compatible-object-storage/31010/ --- fs/config.go | 1 + fs/config/configflags/configflags.go | 1 + fs/fshttp/http.go | 1 + 3 files changed, 3 insertions(+) diff --git a/fs/config.go b/fs/config.go index 5eb85ec616fe6..e96be37d0ec9a 100644 --- a/fs/config.go +++ b/fs/config.go @@ -132,6 +132,7 @@ type ConfigInfo struct { DisableHTTP2 bool HumanReadable bool KvLockTime time.Duration // maximum time to keep key-value database locked by process + DisableHTTPKeepAlives bool } // NewConfig creates a new config with everything set to the default diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 7c2c8ade96ccd..8e3e8d5584e6f 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -137,6 +137,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &ci.DisableHTTP2, "disable-http2", "", ci.DisableHTTP2, "Disable HTTP/2 in the global transport") flags.BoolVarP(flagSet, &ci.HumanReadable, "human-readable", "", ci.HumanReadable, "Print numbers in a human-readable format, sizes with suffix Ki|Mi|Gi|Ti|Pi") flags.DurationVarP(flagSet, &ci.KvLockTime, "kv-lock-time", "", ci.KvLockTime, "Maximum time to keep key-value database locked by process") + flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.") } // ParseHeaders converts the strings passed in via the header flags into HTTPOptions diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go index ef7cf285104a0..9384d2fa4a9d1 100644 --- a/fs/fshttp/http.go +++ b/fs/fshttp/http.go @@ -53,6 +53,7 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht t.MaxIdleConns = 2 * t.MaxIdleConnsPerHost t.TLSHandshakeTimeout = ci.ConnectTimeout t.ResponseHeaderTimeout = ci.Timeout + t.DisableKeepAlives = ci.DisableHTTPKeepAlives // TLS Config t.TLSClientConfig = &tls.Config{ From 5e4caa69cefbbaa7e0f8549efc78e44728374289 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 9 Nov 2021 09:45:10 +0000 Subject: [PATCH 066/560] local: make Hash function cancelable via context --- backend/local/local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/local/local.go b/backend/local/local.go index 3c00d53892d9c..e36c56e2c68c8 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -903,7 +903,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) { return "", fmt.Errorf("hash: failed to open: %w", err) } var hashes map[hash.Type]string - hashes, err = hash.StreamTypes(in, hash.NewHashSet(r)) + hashes, err = hash.StreamTypes(readers.NewContextReader(ctx, in), hash.NewHashSet(r)) closeErr := in.Close() if err != nil { return "", fmt.Errorf("hash: failed to read: %w", err) From 115f1c2cc9f95ccf0d0b28a058f71f62424f2d0f Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 9 Nov 2021 09:45:36 +0000 Subject: [PATCH 067/560] operations: speed up hash checking by aborting the other hash if first returns nothing This speeds up hash checks when a Hash() function returns "" - this means that the hash can be canceled for the other side. In the common case of local hash vs remote hash empty this saves a lot of time. See: https://forum.rclone.org/t/rclone-s3-backend-copy-is-2x-slower-than-aws-s3-cp/27321/9 --- fs/operations/operations.go | 46 ++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/fs/operations/operations.go b/fs/operations/operations.go index c34cdf97d3f60..c21675443eb5a 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -62,36 +62,50 @@ func CheckHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object) (equal b return equal, ht, err } +var errNoHash = errors.New("no hash available") + // checkHashes does the work of CheckHashes but takes a hash.Type and // returns the effective hash type used. func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash.Type) (equal bool, htOut hash.Type, srcHash, dstHash string, err error) { // Calculate hashes in parallel g, ctx := errgroup.WithContext(ctx) + var srcErr, dstErr error g.Go(func() (err error) { - srcHash, err = src.Hash(ctx, ht) - if err != nil { - err = fs.CountError(err) - fs.Errorf(src, "Failed to calculate src hash: %v", err) + srcHash, srcErr = src.Hash(ctx, ht) + if srcErr != nil { + return srcErr } - return err + if srcHash == "" { + fs.Debugf(src, "Src hash empty - aborting Dst hash check") + return errNoHash + } + return nil }) g.Go(func() (err error) { - dstHash, err = dst.Hash(ctx, ht) - if err != nil { - err = fs.CountError(err) - fs.Errorf(dst, "Failed to calculate dst hash: %v", err) + dstHash, dstErr = dst.Hash(ctx, ht) + if dstErr != nil { + return dstErr } - return err + if dstHash == "" { + fs.Debugf(src, "Dst hash empty - aborting Src hash check") + return errNoHash + } + return nil }) err = g.Wait() - if err != nil { - return false, ht, srcHash, dstHash, err - } - if srcHash == "" { + if err == errNoHash { return true, hash.None, srcHash, dstHash, nil } - if dstHash == "" { - return true, hash.None, srcHash, dstHash, nil + if srcErr != nil { + err = fs.CountError(srcErr) + fs.Errorf(dst, "Failed to calculate src hash: %v", err) + } + if dstErr != nil { + err = fs.CountError(dstErr) + fs.Errorf(src, "Failed to calculate dst hash: %v", err) + } + if err != nil { + return false, ht, srcHash, dstHash, err } if srcHash != dstHash { fs.Debugf(src, "%v = %s (%v)", ht, srcHash, src.Fs()) From e57fe14b6105c50b805f41c52b05298da2e59c73 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 8 Jun 2022 09:08:12 +0100 Subject: [PATCH 068/560] mount: log IO errors at ERROR level - fixes #6217 --- cmd/mount/fs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/mount/fs.go b/cmd/mount/fs.go index 43d37802765c3..c89110ecc7f73 100644 --- a/cmd/mount/fs.go +++ b/cmd/mount/fs.go @@ -102,5 +102,6 @@ func translateError(err error) error { case vfs.EINVAL: return fuse.Errno(syscall.EINVAL) } + fs.Errorf(nil, "IO error: %v", err) return err } From a6ca4b38173320973e6831eb925e22ee19428669 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 7 Nov 2021 12:35:11 +0000 Subject: [PATCH 069/560] test info: check file name lengths using 1,2,3,4 byte unicode characters --- cmd/test/info/info.go | 44 +++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/cmd/test/info/info.go b/cmd/test/info/info.go index 06cdd74e984ca..c4eef5f046d2f 100644 --- a/cmd/test/info/info.go +++ b/cmd/test/info/info.go @@ -91,7 +91,7 @@ type results struct { mu sync.Mutex stringNeedsEscaping map[string]internal.Position controlResults map[string]internal.ControlResult - maxFileLength int + maxFileLength [4]int canWriteUnnormalized bool canReadUnnormalized bool canReadRenormalized bool @@ -125,7 +125,9 @@ func (r *results) Print() { fmt.Printf("}\n") } if checkLength { - fmt.Printf("maxFileLength = %d\n", r.maxFileLength) + for i := range r.maxFileLength { + fmt.Printf("maxFileLength = %d // for %d byte unicode characters\n", r.maxFileLength[i], i+1) + } } if checkNormalization { fmt.Printf("canWriteUnnormalized = %v\n", r.canWriteUnnormalized) @@ -150,7 +152,7 @@ func (r *results) WriteJSON() { report.ControlCharacters = &r.controlResults } if checkLength { - report.MaxFileLength = &r.maxFileLength + report.MaxFileLength = &r.maxFileLength[0] } if checkNormalization { report.CanWriteUnnormalized = &r.canWriteUnnormalized @@ -366,11 +368,27 @@ func (r *results) checkControlsList() { } // find the max file name size we can use -func (r *results) findMaxLength() { +func (r *results) findMaxLength(characterLength int) { + var character rune + switch characterLength { + case 1: + character = 'a' + case 2: + character = 'á' + case 3: + character = '世' + case 4: + character = '🙂' + default: + panic("Bad characterLength") + } + if characterLength != len(string(character)) { + panic(fmt.Sprintf("Chose the wrong character length %q is %d not %d", character, len(string(character)), characterLength)) + } const maxLen = 16 * 1024 - name := make([]byte, maxLen) + name := make([]rune, maxLen) for i := range name { - name[i] = 'a' + name[i] = character } // Find the first size of filename we can't write i := sort.Search(len(name), func(i int) (fail bool) { @@ -382,16 +400,20 @@ func (r *results) findMaxLength() { }() path := string(name[:i]) - _, err := r.writeFile(path) + o, err := r.writeFile(path) if err != nil { fs.Infof(r.f, "Couldn't write file with name length %d: %v", i, err) return true } fs.Infof(r.f, "Wrote file with name length %d", i) + err = o.Remove(context.Background()) + if err != nil { + fs.Errorf(o, "Failed to remove test file") + } return false }) - r.maxFileLength = i - 1 - fs.Infof(r.f, "Max file length is %d", r.maxFileLength) + r.maxFileLength[characterLength-1] = i - 1 + fs.Infof(r.f, "Max file length is %d when writing %d byte characters %q", r.maxFileLength[characterLength-1], characterLength, character) } func (r *results) checkStreaming() { @@ -447,7 +469,9 @@ func readInfo(ctx context.Context, f fs.Fs) error { r.checkControls() } if checkLength { - r.findMaxLength() + for i := range r.maxFileLength { + r.findMaxLength(i + 1) + } } if checkNormalization { r.checkUTF8Normalization() From 7d3648dc46f91bf444c5875f03cadae51cff8aa9 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 12 Nov 2021 12:48:14 +0000 Subject: [PATCH 070/560] serve ftp: check --passive-port arguments are correct See: https://forum.rclone.org/t/serve-ftp-passive-port-validity-check/27458 --- cmd/serve/ftp/ftp.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go index a2a4df159b337..b8022599e02a9 100644 --- a/cmd/serve/ftp/ftp.go +++ b/cmd/serve/ftp/ftp.go @@ -13,6 +13,7 @@ import ( "net" "os" "os/user" + "regexp" "strconv" "sync" "time" @@ -128,6 +129,8 @@ type server struct { useTLS bool } +var passivePortsRe = regexp.MustCompile(`^\s*\d+\s*-\s*\d+\s*$`) + // Make a new FTP to serve the remote func newServer(ctx context.Context, f fs.Fs, opt *Options) (*server, error) { host, port, err := net.SplitHostPort(opt.ListenAddr) @@ -151,6 +154,11 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (*server, error) { } s.useTLS = s.opt.TLSKey != "" + // Check PassivePorts format since the the server library doesn't! + if !passivePortsRe.MatchString(opt.PassivePorts) { + return nil, fmt.Errorf("invalid format for passive ports %q", opt.PassivePorts) + } + ftpopt := &ftp.ServerOpts{ Name: "Rclone FTP Server", WelcomeMessage: "Welcome to Rclone " + fs.Version + " FTP Server", From 5697dbc80f58b1a97a66e717bfd0fcbffc60ec42 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 16 Jun 2022 18:08:12 +0100 Subject: [PATCH 071/560] local: fix parsing of --local-nounc flag --- backend/local/local.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/local/local.go b/backend/local/local.go index e36c56e2c68c8..1a8908720c4f0 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -45,6 +45,7 @@ func init() { Options: []fs.Option{{ Name: "nounc", Help: "Disable UNC (long path names) conversion on Windows.", + Default: false, Advanced: runtime.GOOS != "windows", Examples: []fs.OptionExample{{ Value: "true", From e59801c69bc8a3d2b9614d9b8e7a25831d99e0bc Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 16 Jun 2022 23:26:10 +0100 Subject: [PATCH 072/560] Add Maciej Radzikowski to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 91fe78693606b..d4abb2bab956b 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -610,3 +610,4 @@ put them back in again.` >}} * Art M. Gallagher * Sven Gerber <49589423+svengerber@users.noreply.github.com> * CrossR + * Maciej Radzikowski From c85fbebce6f7166350c79e11fae763c8264ef865 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 16 Jun 2022 22:55:45 +0100 Subject: [PATCH 073/560] s3: simplify PutObject code to use the Request.SetStreamingBody method In this commit e5974ac4b0a847e8 s3: use PutObject from the aws SDK to upload single part objects rclone was made to upload objects to s3 using PUT requests rather than using signed uploads. However this change missed the fact that there is a supported way to do this in the SDK using the SetStreamingBody method on the Request. This therefore reverts a lot of the previous commit to do with making an unsigned connection and other complication and uses the SDK facility. --- backend/s3/s3.go | 107 ++++++++++++++--------------------------------- 1 file changed, 32 insertions(+), 75 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index dd894db18156a..4e2a15cdeed93 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "net/http" "net/url" "path" @@ -32,7 +33,6 @@ import ( "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" - v4 "github.com/aws/aws-sdk-go/aws/signer/v4" "github.com/aws/aws-sdk-go/service/s3" "github.com/ncw/swift/v2" "github.com/rclone/rclone/fs" @@ -2044,7 +2044,6 @@ type Fs struct { ctx context.Context // global context for reading config features *fs.Features // optional features c *s3.S3 // the connection to the s3 server - cu *s3.S3 // unsigned connection to the s3 server for PutObject ses *session.Session // the s3 session rootBucket string // bucket part of root (if any) rootDirectory string // directory part of root (if any) @@ -2181,11 +2180,7 @@ func getClient(ctx context.Context, opt *Options) *http.Client { } // s3Connection makes a connection to s3 -// -// If unsignedBody is set then the connection is configured for -// unsigned bodies which is necessary for PutObject if we don't want -// it to seek -func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *s3.S3, *session.Session, error) { +func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *session.Session, error) { ci := fs.GetConfig(ctx) // Make the auth v := credentials.Value{ @@ -2202,7 +2197,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S // start a new AWS session awsSession, err := session.NewSession() if err != nil { - return nil, nil, nil, fmt.Errorf("NewSession: %w", err) + return nil, nil, fmt.Errorf("NewSession: %w", err) } // first provider to supply a credential set "wins" @@ -2242,9 +2237,9 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S // if no access key/secret and iam is explicitly disabled then fall back to anon interaction cred = credentials.AnonymousCredentials case v.AccessKeyID == "": - return nil, nil, nil, errors.New("access_key_id not found") + return nil, nil, errors.New("access_key_id not found") case v.SecretAccessKey == "": - return nil, nil, nil, errors.New("secret_access_key not found") + return nil, nil, errors.New("secret_access_key not found") } if opt.Region == "" { @@ -2283,36 +2278,25 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S // (from the shared config file) if the passed-in Options.Config.Credentials is nil. awsSessionOpts.Config.Credentials = nil } - // Setting this stops PutObject reading the body twice and seeking - // We add our own Content-MD5 for data protection - awsSessionOpts.Config.S3DisableContentMD5Validation = aws.Bool(true) ses, err := session.NewSessionWithOptions(awsSessionOpts) if err != nil { - return nil, nil, nil, err - } - newC := func(unsignedBody bool) *s3.S3 { - c := s3.New(ses) - if opt.V2Auth || opt.Region == "other-v2-signature" { - fs.Debugf(nil, "Using v2 auth") - signer := func(req *request.Request) { - // Ignore AnonymousCredentials object - if req.Config.Credentials == credentials.AnonymousCredentials { - return - } - sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest) + return nil, nil, err + } + c := s3.New(ses) + if opt.V2Auth || opt.Region == "other-v2-signature" { + fs.Debugf(nil, "Using v2 auth") + signer := func(req *request.Request) { + // Ignore AnonymousCredentials object + if req.Config.Credentials == credentials.AnonymousCredentials { + return } - c.Handlers.Sign.Clear() - c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler) - c.Handlers.Sign.PushBack(signer) - } else if unsignedBody { - // If the body is unsigned then tell the signer that we aren't signing the payload - c.Handlers.Sign.Clear() - c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler) - c.Handlers.Sign.PushBackNamed(v4.BuildNamedHandler("v4.SignRequestHandler.WithUnsignedPayload", v4.WithUnsignedPayload)) + sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest) } - return c + c.Handlers.Sign.Clear() + c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler) + c.Handlers.Sign.PushBack(signer) } - return newC(false), newC(true), ses, nil + return c, ses, nil } func checkUploadChunkSize(cs fs.SizeSuffix) error { @@ -2501,7 +2485,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:]) } srv := getClient(ctx, opt) - c, cu, ses, err := s3Connection(ctx, opt, srv) + c, ses, err := s3Connection(ctx, opt, srv) if err != nil { return nil, err } @@ -2519,7 +2503,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e ci: ci, ctx: ctx, c: c, - cu: cu, ses: ses, pacer: pc, cache: bucket.NewCache(), @@ -2643,12 +2626,11 @@ func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error { // Make a new session with the new region oldRegion := f.opt.Region f.opt.Region = region - c, cu, ses, err := s3Connection(f.ctx, &f.opt, f.srv) + c, ses, err := s3Connection(f.ctx, &f.opt, f.srv) if err != nil { return fmt.Errorf("creating new session failed: %w", err) } f.c = c - f.cu = cu f.ses = ses fs.Logf(f, "Switched region to %q from %q", region, oldRegion) @@ -4143,48 +4125,23 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si return etag, nil } -// unWrapAwsError unwraps AWS errors, looking for a non AWS error -// -// It returns true if one was found and the error, or false and the -// error passed in. -func unWrapAwsError(err error) (found bool, outErr error) { - if awsErr, ok := err.(awserr.Error); ok { - var origErrs []error - if batchErr, ok := awsErr.(awserr.BatchError); ok { - origErrs = batchErr.OrigErrs() - } else { - origErrs = []error{awsErr.OrigErr()} - } - for _, origErr := range origErrs { - found, newErr := unWrapAwsError(origErr) - if found { - return found, newErr - } - } - return false, err - } - return true, err -} - // Upload a single part using PutObject func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) { - req.Body = readers.NewFakeSeeker(in, size) - var resp *s3.PutObjectOutput + r, resp := o.fs.c.PutObjectRequest(req) + if req.ContentLength != nil && *req.ContentLength == 0 { + // Can't upload zero length files like this for some reason + r.Body = bytes.NewReader([]byte{}) + } else { + r.SetStreamingBody(ioutil.NopCloser(in)) + } + r.SetContext(ctx) + r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD") + err = o.fs.pacer.CallNoRetry(func() (bool, error) { - resp, err = o.fs.cu.PutObject(req) + err := r.Send() return o.fs.shouldRetry(ctx, err) }) if err != nil { - // Return the underlying error if we have a Serialization error if possible - // - // Serialization errors are synthesized locally in the SDK (not returned from the - // server). We'll get one if the SDK attempts a retry, however the FakeSeeker will - // remember the previous error from Read and return that. - if do, ok := err.(awserr.Error); ok && do.Code() == request.ErrCodeSerialization { - if found, newErr := unWrapAwsError(err); found { - err = newErr - } - } return etag, lastModified, err } lastModified = time.Now() From dcc128c70d2bf4f46304e1befb3fdbbdeebd86fa Mon Sep 17 00:00:00 2001 From: Lu Wang Date: Fri, 17 Jun 2022 07:33:56 +0200 Subject: [PATCH 074/560] docs/onedrive: document creation of client ID for OneDrive Business --- docs/content/onedrive.md | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index a847d2a3879aa..f25bbe513ca22 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -120,13 +120,15 @@ To copy a local directory to an OneDrive directory called backup ### Getting your own Client ID and Key -You can use your own Client ID if the default (`client_id` left blank) -one doesn't work for you or you see lots of throttling. The default -Client ID and Key is shared by all rclone users when performing -requests. +rclone uses a default Client ID when talking to OneDrive, unless a custom `client_id` is specified in the config. +The default Client ID and Key are shared by all rclone users when performing requests. -If you are having problems with them (E.g., seeing a lot of throttling), you can get your own -Client ID and Key by following the steps below: +You may choose to create and use your own Client ID, in case the default one does not work well for you. +For example, you might see throtting. + +#### Creating Client ID for OneDrive Personal + +To create your own Client ID, please follow these steps: 1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade and then click `New registration`. 2. Enter a name for your app, choose account type `Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)`, select `Web` in `Redirect URI`, then type (do not copy and paste) `http://localhost:53682/` and click Register. Copy and keep the `Application (client) ID` under the app name for later use. @@ -142,6 +144,22 @@ See [Microsoft Docs](https://docs.microsoft.com/en-us/graph/permissions-referenc The `Sites.Read.All` permission is required if you need to [search SharePoint sites when configuring the remote](https://github.com/rclone/rclone/pull/5883). However, if that permission is not assigned, you need to exclude `Sites.Read.All` from your access scopes or set `disable_site_permission` option to true in the advanced options. +#### Creating Client ID for OneDrive Business + +The steps for OneDrive Personal may or may not work for OneDrive Business, depending on the security settings of the organization. +A common error is that the publisher of the App is not verified. + +You may try to [verify you account](https://docs.microsoft.com/en-us/azure/active-directory/develop/publisher-verification-overview), or try to limit the App to your organization only, as shown below. + +1. Make sure to create the App with your business account. +2. Follow the steps above to create an App. However, we need a different account type here: `Accounts in this organizational directory only (*** - Single tenant)`. Note that you can also change the account type aftering creating the App. +3. Find the [tenant ID](https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/active-directory-how-to-find-tenant) of your organization. +4. In the rclone config, set `auth_url` to `https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/authorize`. +5. In the rclone config, set `token_url` to `https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/token`. + +Note: If you have a special region, you may need a different host in step 4 and 5. Here are [some hints](https://github.com/rclone/rclone/blob/bc23bf11db1c78c6ebbf8ea538fbebf7058b4176/backend/onedrive/onedrive.go#L86). + + ### Modification time and hashes OneDrive allows modification times to be set on objects accurate to 1 From 295006f66226b6550cb9a936d98ff72519eaecc8 Mon Sep 17 00:00:00 2001 From: Scott Grimes Date: Mon, 24 Jan 2022 16:53:55 -0500 Subject: [PATCH 075/560] opendrive: resolve lag and truncate bugs fixes #5936 Co-authored-by: buengese --- backend/opendrive/opendrive.go | 54 ++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.go index 6e5967e626c1c..ac1a1c01df8ab 100644 --- a/backend/opendrive/opendrive.go +++ b/backend/opendrive/opendrive.go @@ -362,7 +362,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, srcPath := srcObj.fs.rootSlash() + srcObj.remote dstPath := f.rootSlash() + remote if strings.EqualFold(srcPath, dstPath) { - return nil, fmt.Errorf("Can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) + return nil, fmt.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) } // Create temporary object @@ -429,6 +429,12 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, return nil, err } + // move_copy will silently truncate new filenames + if len(leaf) > 255 { + fs.Debugf(src, "Can't move file: name (%q) exceeds 255 char", leaf) + return nil, fs.ErrorFileNameTooLong + } + // Copy the object var resp *http.Response response := moveCopyFileResponse{} @@ -1024,30 +1030,52 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { return err } var resp *http.Response - folderList := FolderList{} - err = o.fs.pacer.Call(func() (bool, error) { + fileInfo := File{} + + // If we know the object id perform a direct lookup + // because the /folder/itembyname.json endpoint is unreliable: + // newly created objects take an arbitrary amount of time to show up + if o.id != "" { opts := rest.Opts{ Method: "GET", - Path: fmt.Sprintf("/folder/itembyname.json/%s/%s?name=%s", - o.fs.session.SessionID, directoryID, url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf))), + Path: fmt.Sprintf("/file/info.json/%s?session_id=%s", + o.id, o.fs.session.SessionID), + } + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &fileInfo) + return o.fs.shouldRetry(ctx, resp, err) + }) + if err != nil { + return fmt.Errorf("failed to get fileinfo: %w", err) } + + o.id = fileInfo.FileID + o.modTime = time.Unix(fileInfo.DateModified, 0) + o.md5 = fileInfo.FileHash + o.size = fileInfo.Size + return nil + } + folderList := FolderList{} + opts := rest.Opts{ + Method: "GET", + Path: fmt.Sprintf("/folder/itembyname.json/%s/%s?name=%s", + o.fs.session.SessionID, directoryID, url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf))), + } + err = o.fs.pacer.Call(func() (bool, error) { resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList) return o.fs.shouldRetry(ctx, resp, err) }) if err != nil { return fmt.Errorf("failed to get folder list: %w", err) } - if len(folderList.Files) == 0 { return fs.ErrorObjectNotFound } - - leafFile := folderList.Files[0] - o.id = leafFile.FileID - o.modTime = time.Unix(leafFile.DateModified, 0) - o.md5 = leafFile.FileHash - o.size = leafFile.Size - + fileInfo = folderList.Files[0] + o.id = fileInfo.FileID + o.modTime = time.Unix(fileInfo.DateModified, 0) + o.md5 = fileInfo.FileHash + o.size = fileInfo.Size return nil } From 8e2d9a4cb9e857514ad0c235612b42f4ec244990 Mon Sep 17 00:00:00 2001 From: Phil Shackleton <71221528+philshacks@users.noreply.github.com> Date: Fri, 17 Jun 2022 16:50:01 +0100 Subject: [PATCH 076/560] drive: update Internal OAuth consent screen docs Updated instructions in `Making your own client_id` section to record process for selecting "Internal" OAuth consent screen. --- docs/content/drive.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/content/drive.md b/docs/content/drive.md index a2bf6cfb11c53..871fc50ef6b09 100644 --- a/docs/content/drive.md +++ b/docs/content/drive.md @@ -1458,8 +1458,9 @@ enter "Developer Contact Email" (your own email is OK); then click on "Save" (al Click again on "Credentials" on the left panel to go back to the "Credentials" screen. -(PS: if you are a GSuite user, you could also select "Internal" instead -of "External" above, but this has not been tested/documented so far). + (PS: if you are a GSuite user, you could also select "Internal" instead +of "External" above, but this will restrict API use to Google Workspace +users in your organisation). 6. Click on the "+ CREATE CREDENTIALS" button at the top of the screen, then select "OAuth client ID". @@ -1467,14 +1468,18 @@ then select "OAuth client ID". 7. Choose an application type of "Desktop app" and click "Create". (the default name is fine) 8. It will show you a client ID and client secret. Make a note of these. + + (If you selected "External" at Step 5 continue to "Publish App" in the Steps 9 and 10. + If you chose "Internal" you don't need to publish and can skip straight to + Step 11.) 9. Go to "Oauth consent screen" and press "Publish App" -10. Provide the noted client ID and client secret to rclone. - -11. Click "OAuth consent screen", then click "PUBLISH APP" button and +10. Click "OAuth consent screen", then click "PUBLISH APP" button and confirm, or add your account under "Test users". +11. Provide the noted client ID and client secret to rclone. + Be aware that, due to the "enhanced security" recently introduced by Google, you are theoretically expected to "submit your app for verification" and then wait a few weeks(!) for their response; in practice, you can go right From 2fac8fdde6480af6a8ab55d8036eb3b4188c50db Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 17 Jun 2022 16:52:17 +0100 Subject: [PATCH 077/560] Add Scott Grimes to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index d4abb2bab956b..ef130e6aa17e1 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -611,3 +611,4 @@ put them back in again.` >}} * Sven Gerber <49589423+svengerber@users.noreply.github.com> * CrossR * Maciej Radzikowski + * Scott Grimes From f829ded4567259db8585d5a73895b48adf8bb517 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 17 Jun 2022 16:52:17 +0100 Subject: [PATCH 078/560] Add Phil Shackleton to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index ef130e6aa17e1..04322ca1d9ddc 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -612,3 +612,4 @@ put them back in again.` >}} * CrossR * Maciej Radzikowski * Scott Grimes + * Phil Shackleton <71221528+philshacks@users.noreply.github.com> From f7c36ce0f9473bd065d143f3f7c1015c02bcaf08 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 17 Jun 2022 14:34:57 +0100 Subject: [PATCH 079/560] s3: unwrap SDK errors to reveal underlying errors on upload The SDK doesn't wrap errors in a Go standard way so they can't be unwrapped and tested for - eg fatal error. The code looks for a Serialization or RequestError and returns the unwrapped underlying error if possible. This fixes the fs/operations integration tests checking for fatal errors being returned. --- backend/s3/s3.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 4e2a15cdeed93..c81024ce3c60c 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -4125,6 +4125,29 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si return etag, nil } +// unWrapAwsError unwraps AWS errors, looking for a non AWS error +// +// It returns true if one was found and the error, or false and the +// error passed in. +func unWrapAwsError(err error) (found bool, outErr error) { + if awsErr, ok := err.(awserr.Error); ok { + var origErrs []error + if batchErr, ok := awsErr.(awserr.BatchError); ok { + origErrs = batchErr.OrigErrs() + } else { + origErrs = []error{awsErr.OrigErr()} + } + for _, origErr := range origErrs { + found, newErr := unWrapAwsError(origErr) + if found { + return found, newErr + } + } + return false, err + } + return true, err +} + // Upload a single part using PutObject func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) { r, resp := o.fs.c.PutObjectRequest(req) @@ -4142,6 +4165,17 @@ func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjec return o.fs.shouldRetry(ctx, err) }) if err != nil { + // Return the underlying error if we have a + // Serialization or RequestError error if possible + // + // These errors are synthesized locally in the SDK + // (not returned from the server) and we'd rather have + // the underlying error if there is one. + if do, ok := err.(awserr.Error); ok && (do.Code() == request.ErrCodeSerialization || do.Code() == request.ErrCodeRequestError) { + if found, newErr := unWrapAwsError(err); found { + err = newErr + } + } return etag, lastModified, err } lastModified = time.Now() From b9de37af80a6f794169a7029e460987149bcd28f Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 17 Jun 2022 15:02:49 +0100 Subject: [PATCH 080/560] test_all: Only run backend tests for Internet Archive as it is too slow --- fstest/test_all/config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index f0abff00739e9..3588cdddc71b2 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -139,6 +139,8 @@ backends: - backend: "internetarchive" remote: "TestIA:rclone-integration-test" fastlist: true + tests: + - backend - backend: "jottacloud" remote: "TestJottacloud:" fastlist: true From 100acc570aa42d4943d605ea007789ae0fd4f574 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 18 Jun 2022 15:05:12 +0100 Subject: [PATCH 081/560] test_all: fix -clean so it works on remotes with paths --- fstest/test_all/clean.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fstest/test_all/clean.go b/fstest/test_all/clean.go index c4d22a8a41d34..c12a4839abd72 100644 --- a/fstest/test_all/clean.go +++ b/fstest/test_all/clean.go @@ -9,6 +9,7 @@ import ( "regexp" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/fs/operations" ) @@ -39,7 +40,7 @@ func cleanFs(ctx context.Context, remote string, cleanup bool) error { } err = entries.ForDirError(func(dir fs.Directory) error { dirPath := dir.Remote() - fullPath := remote + dirPath + fullPath := fspath.JoinRootPath(remote, dirPath) if MatchTestRemote.MatchString(dirPath) { if *dryRun { log.Printf("Not Purging %s - -dry-run", fullPath) From 14e0396fcb353a2ff5e533aeea1da2307bb81d68 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 18 Jun 2022 15:13:18 +0100 Subject: [PATCH 082/560] test_all: allow internet archive backend more time --- fstest/test_all/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 3588cdddc71b2..68f17c2ddd0a5 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -141,6 +141,7 @@ backends: fastlist: true tests: - backend + extratime: 2.0 - backend: "jottacloud" remote: "TestJottacloud:" fastlist: true From 36add0afbfb908796a3f3ea6295b9b459a4ac568 Mon Sep 17 00:00:00 2001 From: eNV25 Date: Sun, 19 Jun 2022 14:22:45 +0400 Subject: [PATCH 083/560] ncdu: replace termbox with tcell's termbox wrapper The https://github.com/nsf/termbox-go library is no longer maintained so this change replaces it with the maintained github.com/gdamore/tcell library which has a termbox backwards compatibility layer. There are a few minor changes from the termbox library: - Using Clear with fg bg ColorDefault resulted in a white background for some reason. - Clear with fg ColorWhite bg ColorBlack was used instead. - tcell's termbox wrapper doesn't support ColorLightYellow. - ColorYellow + 8 was used instead. --- cmd/ncdu/ncdu.go | 16 +++++++--------- cmd/ncdu/ncdu_unsupported.go | 4 ++-- go.mod | 2 +- go.sum | 11 ++++++++--- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go index 963ddb628df15..aea2501eb7a12 100644 --- a/cmd/ncdu/ncdu.go +++ b/cmd/ncdu/ncdu.go @@ -1,7 +1,7 @@ // Package ncdu implements a text based user interface for exploring a remote -//go:build !plan9 && !solaris && !js -// +build !plan9,!solaris,!js +//go:build !plan9 && !js +// +build !plan9,!js package ncdu @@ -14,8 +14,8 @@ import ( "strings" "github.com/atotto/clipboard" + "github.com/gdamore/tcell/v2/termbox" runewidth "github.com/mattn/go-runewidth" - termbox "github.com/nsf/termbox-go" "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/cmd/ncdu/scan" "github.com/rclone/rclone/fs" @@ -333,10 +333,7 @@ func (u *UI) Draw() error { u.dirListHeight = h - 3 // Plot - err := termbox.Clear(termbox.ColorDefault, termbox.ColorDefault) - if err != nil { - return fmt.Errorf("failed to clear screen: %w", err) - } + termbox.Clear(termbox.ColorWhite, termbox.ColorBlack) // Header line Linef(0, 0, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "rclone ncdu %s - use the arrow keys to navigate, press ? for help", fs.Version) @@ -374,8 +371,9 @@ func (u *UI) Draw() error { if err != nil { fg = termbox.ColorRed } + const colorLightYellow = termbox.ColorYellow + 8 if isSelected { - fg = termbox.ColorLightYellow + fg = colorLightYellow } bg := termbox.ColorBlack if n == dirPos.entry { @@ -456,7 +454,7 @@ func (u *UI) Draw() error { if u.showBox { u.Box() } - err = termbox.Flush() + err := termbox.Flush() if err != nil { return fmt.Errorf("failed to flush screen: %w", err) } diff --git a/cmd/ncdu/ncdu_unsupported.go b/cmd/ncdu/ncdu_unsupported.go index 6f66cfb6c9cb9..b4a9b22cf0d63 100644 --- a/cmd/ncdu/ncdu_unsupported.go +++ b/cmd/ncdu/ncdu_unsupported.go @@ -1,7 +1,7 @@ // Build for ncdu for unsupported platforms to stop go complaining // about "no buildable Go source files " -//go:build plan9 || solaris || js -// +build plan9 solaris js +//go:build plan9 || js +// +build plan9 js package ncdu diff --git a/go.mod b/go.mod index 75a242ff7a701..1fe102fd68853 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00 github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.4 github.com/gabriel-vasile/mimetype v1.4.0 + github.com/gdamore/tcell/v2 v2.5.1 github.com/go-chi/chi/v5 v5.0.7 github.com/google/uuid v1.3.0 github.com/hanwen/go-fuse/v2 v2.1.0 @@ -38,7 +39,6 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 github.com/ncw/swift/v2 v2.0.1 - github.com/nsf/termbox-go v1.1.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/sftp v1.13.5-0.20211228200725-31aac3e1878d github.com/pmezard/go-difflib v1.0.0 diff --git a/go.sum b/go.sum index 8d64ccdfae170..e957043f89908 100644 --- a/go.sum +++ b/go.sum @@ -191,6 +191,10 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro= github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= +github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= +github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= +github.com/gdamore/tcell/v2 v2.5.1 h1:zc3LPdpK184lBW7syF2a5C6MV827KmErk9jGVnmsl/I= +github.com/gdamore/tcell/v2 v2.5.1/go.mod h1:wSkrPaXoiIWZqW/g7Px4xc79di6FTcpB8tvaKJ6uGBo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -403,6 +407,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= @@ -418,7 +424,6 @@ github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqf github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -444,8 +449,6 @@ github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItc github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/nsf/termbox-go v1.1.1 h1:nksUPLCb73Q++DwbYUBEglYBRPZyoXJdrj5L+TkjyZY= -github.com/nsf/termbox-go v1.1.1/go.mod h1:T0cTdVuOwf7pHQNtfhnEbzHbcNyCEcVU4YPpouCbVxo= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -877,12 +880,14 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220318055525-2edf467146b5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 21c746a56c86f20dec639afa49f321cc6c6e38b1 Mon Sep 17 00:00:00 2001 From: buengese Date: Sat, 18 Jun 2022 01:39:44 +0200 Subject: [PATCH 084/560] fichier: parse api error codes and them accordingly --- backend/fichier/api.go | 43 ++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/backend/fichier/api.go b/backend/fichier/api.go index 6a17462658dee..15c916c736314 100644 --- a/backend/fichier/api.go +++ b/backend/fichier/api.go @@ -28,25 +28,44 @@ var retryErrorCodes = []int{ 509, // Bandwidth Limit Exceeded } +var errorRegex = regexp.MustCompile(`#\d{1,3}`) + +func parseFichierError(err error) int { + matches := errorRegex.FindStringSubmatch(err.Error()) + if len(matches) == 0 { + return 0 + } + code, err := strconv.Atoi(matches[0]) + if err != nil { + fs.Debugf(nil, "failed parsing fichier error: %v", err) + return 0 + } + return code +} + // shouldRetry returns a boolean as to whether this resp and err // deserve to be retried. It returns the err as a convenience func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { if fserrors.ContextError(ctx, &err) { return false, err } - // Detect this error which the integration tests provoke - // error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}" - // - // https://1fichier.com/api.html - // - // file/ls.cgi is limited : + // 1Fichier uses HTTP error code 403 (Forbidden) for all kinds of errors with + // responses looking like this: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}" // - // Warning (can be changed in case of abuses) : - // List all files of the account is limited to 1 request per hour. - // List folders is limited to 5 000 results and 1 request per folder per 30s. - if err != nil && strings.Contains(err.Error(), "Flood detected") { - fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err) - time.Sleep(30 * time.Second) + // We attempt to parse the actual 1Fichier error code from this body and handle it accordingly + // Most importantly #374 (Flood detected: IP locked) which the integration tests provoke + // The list below is far from complete and should be expanded if we see any more error codes. + if err != nil { + switch parseFichierError(err) { + case 93: + return false, err // No such user + case 186: + return false, err // IP blocked? + case 374: + fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err) + time.Sleep(30 * time.Second) + default: + } } return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err } From f2a15a174f8cdd21950b123f102ec9f9105098e1 Mon Sep 17 00:00:00 2001 From: Caleb Date: Sun, 19 Jun 2022 07:26:53 -0600 Subject: [PATCH 085/560] docs: grammatical clarification in sync docs --- docs/content/commands/rclone_sync.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/commands/rclone_sync.md b/docs/content/commands/rclone_sync.md index 57e24dd2d9e68..5596240e1e9a9 100644 --- a/docs/content/commands/rclone_sync.md +++ b/docs/content/commands/rclone_sync.md @@ -28,7 +28,7 @@ errors at any point. Duplicate objects (files with the same name, on those providers that support it) are also not yet handled. It is always the contents of the directory that is synced, not the -directory so when source:path is a directory, it's the contents of +directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. See extended explanation in the `copy` command above if unsure. From 8b8802a0784491e6afba628aa6f7fbe78d9303db Mon Sep 17 00:00:00 2001 From: J-P Treen Date: Wed, 28 Jul 2021 17:05:21 +0100 Subject: [PATCH 086/560] copyurl: Add option to honor the HTTP header filename directive. Implemented --header-filename for use with copyurl. For specifically setting preferred download filenames for HTTP requests, RFC 6226 specifies a 'filename' directive, available within 'Content-Disposition' header. We can handle with 'mime.ParseMediaType'. See below for details: https://httpwg.org/specs/rfc6266.html#disposition.parameter.filename https://httpwg.org/specs/rfc6266.html#advice.generating Co-authored-by: buengese --- cmd/copyurl/copyurl.go | 21 ++++++++++++--------- fs/operations/operations.go | 21 ++++++++++++++++----- fs/operations/operations_test.go | 12 ++++++------ fs/operations/rc.go | 3 ++- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/cmd/copyurl/copyurl.go b/cmd/copyurl/copyurl.go index e7e31f188835d..fc3cf38cccc86 100644 --- a/cmd/copyurl/copyurl.go +++ b/cmd/copyurl/copyurl.go @@ -14,16 +14,18 @@ import ( ) var ( - autoFilename = false - printFilename = false - stdout = false - noClobber = false + autoFilename = false + headerFilename = false + printFilename = false + stdout = false + noClobber = false ) func init() { cmd.Root.AddCommand(commandDefinition) cmdFlags := commandDefinition.Flags() flags.BoolVarP(cmdFlags, &autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the URL and use it for destination file path") + flags.BoolVarP(cmdFlags, &headerFilename, "header-filename", "", headerFilename, "Get the file name from the Content-Disposition header") flags.BoolVarP(cmdFlags, &printFilename, "print-filename", "p", printFilename, "Print the resulting name from --auto-filename") flags.BoolVarP(cmdFlags, &noClobber, "no-clobber", "", noClobber, "Prevent overwriting file with same name") flags.BoolVarP(cmdFlags, &stdout, "stdout", "", stdout, "Write the output to stdout rather than a file") @@ -36,10 +38,11 @@ var commandDefinition = &cobra.Command{ Download a URL's content and copy it to the destination without saving it in temporary storage. -Setting ` + "`--auto-filename`" + ` will cause the file name to be retrieved from -the URL (after any redirections) and used in the destination -path. With ` + "`--print-filename`" + ` in addition, the resulting file name will -be printed. +Setting ` + "`--auto-filename`" + ` will attempt to automatically determine the filename from the URL +(after any redirections) and used in the destination path. +With ` + "`--auto-filename-header`" + ` in +addition, if a specific filename is set in HTTP headers, it will be used instead of the name from the URL. +With ` + "`--print-filename`" + ` in addition, the resulting file name will be printed. Setting ` + "`--no-clobber`" + ` will prevent overwriting file on the destination if there is one with the same name. @@ -69,7 +72,7 @@ will cause the output to be written to standard output. if stdout { err = operations.CopyURLToWriter(context.Background(), args[0], os.Stdout) } else { - dst, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, noClobber) + dst, err = operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename, headerFilename, noClobber) if printFilename && err == nil && dst != nil { fmt.Println(dst.Remote()) } diff --git a/fs/operations/operations.go b/fs/operations/operations.go index c21675443eb5a..5eccc44f719eb 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "io/ioutil" + "mime" "net/http" "os" "path" @@ -1757,7 +1758,7 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) // copyURLFn copies the data from the url to the function supplied -func copyURLFn(ctx context.Context, dstFileName string, url string, dstFileNameFromURL bool, fn copyURLFunc) (err error) { +func copyURLFn(ctx context.Context, dstFileName string, url string, autoFilename, dstFileNameFromHeader bool, fn copyURLFunc) (err error) { client := fshttp.NewClient(ctx) resp, err := client.Get(url) if err != nil { @@ -1771,7 +1772,17 @@ func copyURLFn(ctx context.Context, dstFileName string, url string, dstFileNameF if err != nil { modTime = time.Now() } - if dstFileNameFromURL { + if autoFilename { + if dstFileNameFromHeader { + _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) + headerFilename := path.Base(strings.Replace(params["filename"], "\\", "/", -1)) + if err != nil || headerFilename == "" { + return fmt.Errorf("copyurl failed: filename not found in the Content-Dispoition header") + } + fs.Debugf(headerFilename, "filename found in Content-Disposition header.") + return fn(ctx, headerFilename, resp.Body, resp.ContentLength, modTime) + } + dstFileName = path.Base(resp.Request.URL.Path) if dstFileName == "." || dstFileName == "/" { return fmt.Errorf("CopyURL failed: file name wasn't found in url") @@ -1782,9 +1793,9 @@ func copyURLFn(ctx context.Context, dstFileName string, url string, dstFileNameF } // CopyURL copies the data from the url to (fdst, dstFileName) -func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool, noClobber bool) (dst fs.Object, err error) { +func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, autoFilename, dstFileNameFromHeader bool, noClobber bool) (dst fs.Object, err error) { - err = copyURLFn(ctx, dstFileName, url, dstFileNameFromURL, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) { + err = copyURLFn(ctx, dstFileName, url, autoFilename, dstFileNameFromHeader, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) { if noClobber { _, err = fdst.NewObject(ctx, dstFileName) if err == nil { @@ -1799,7 +1810,7 @@ func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, ds // CopyURLToWriter copies the data from the url to the io.Writer supplied func CopyURLToWriter(ctx context.Context, url string, out io.Writer) (err error) { - return copyURLFn(ctx, "", url, false, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) { + return copyURLFn(ctx, "", url, false, false, func(ctx context.Context, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (err error) { _, err = io.Copy(out, in) return err }) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 10475562aef2d..545bcbe4dd21b 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -739,31 +739,31 @@ func TestCopyURL(t *testing.T) { ts := httptest.NewServer(handler) defer ts.Close() - o, err := operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false) + o, err := operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported) // Check file clobbering - _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, true) + _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, true) require.Error(t, err) // Check auto file naming status = 0 urlFileName := "filename.txt" - o, err = operations.CopyURL(ctx, r.Fremote, "", ts.URL+"/"+urlFileName, true, false) + o, err = operations.CopyURL(ctx, r.Fremote, "", ts.URL+"/"+urlFileName, true, false, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) assert.Equal(t, urlFileName, o.Remote()) // Check auto file naming when url without file name - _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, true, false) + _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, true, false, false) require.Error(t, err) // Check an error is returned for a 404 status = http.StatusNotFound - o, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false) + o, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, false) require.Error(t, err) assert.Contains(t, err.Error(), "Not Found") assert.Nil(t, o) @@ -776,7 +776,7 @@ func TestCopyURL(t *testing.T) { tss := httptest.NewTLSServer(handler) defer tss.Close() - o, err = operations.CopyURL(ctx, r.Fremote, "file2", tss.URL, false, false) + o, err = operations.CopyURL(ctx, r.Fremote, "file2", tss.URL, false, false, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported) diff --git a/fs/operations/rc.go b/fs/operations/rc.go index de77dae0913ef..848b56ca10a22 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -274,8 +274,9 @@ func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bo } autoFilename, _ := in.GetBool("autoFilename") noClobber, _ := in.GetBool("noClobber") + headerFilename, _ := in.GetBool("headerFilename") - _, err = CopyURL(ctx, f, remote, url, autoFilename, noClobber) + _, err = CopyURL(ctx, f, remote, url, autoFilename, headerFilename, noClobber) return nil, err case "uploadfile": From ac0dc9922ee9e5171f5e2d3e0f9a2e6efb807f78 Mon Sep 17 00:00:00 2001 From: buengese Date: Sun, 19 Jun 2022 15:39:05 +0200 Subject: [PATCH 087/560] copyurl: add tests for the option to honor the HTTP header filename directive --- fs/operations/operations_test.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 545bcbe4dd21b..04b26e8b8819e 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -729,10 +729,15 @@ func TestCopyURL(t *testing.T) { // check when reading from regular HTTP server status := 0 + nameHeader := false + headerFilename := "headerfilename.txt" handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if status != 0 { http.Error(w, "an error ocurred", status) } + if nameHeader { + w.Header().Set("Content-Disposition", `attachment; filename="folder\`+headerFilename+`"`) + } _, err := w.Write([]byte(contents)) assert.NoError(t, err) }) @@ -757,10 +762,22 @@ func TestCopyURL(t *testing.T) { assert.Equal(t, int64(len(contents)), o.Size()) assert.Equal(t, urlFileName, o.Remote()) + // Check header file naming + nameHeader = true + o, err = operations.CopyURL(ctx, r.Fremote, "", ts.URL, true, true, false) + require.NoError(t, err) + assert.Equal(t, int64(len(contents)), o.Size()) + assert.Equal(t, headerFilename, o.Remote()) + // Check auto file naming when url without file name _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, true, false, false) require.Error(t, err) + // Check header file naming without header set + nameHeader = false + _, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, true, true, false) + require.Error(t, err) + // Check an error is returned for a 404 status = http.StatusNotFound o, err = operations.CopyURL(ctx, r.Fremote, "file1", ts.URL, false, false, false) @@ -779,7 +796,7 @@ func TestCopyURL(t *testing.T) { o, err = operations.CopyURL(ctx, r.Fremote, "file2", tss.URL, false, false, false) require.NoError(t, err) assert.Equal(t, int64(len(contents)), o.Size()) - fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported) + fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1), fstest.NewItem(headerFilename, contents, t1)}, nil, fs.ModTimeNotSupported) } func TestCopyURLToWriter(t *testing.T) { From 7a909ebfb0d100187fab7949455bf74b8ce07042 Mon Sep 17 00:00:00 2001 From: Martin Czygan <53705+miku@users.noreply.github.com> Date: Mon, 20 Jun 2022 13:14:58 +0200 Subject: [PATCH 088/560] fs/cache: fix cache unpin --- fs/cache/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/cache/cache.go b/fs/cache/cache.go index f595618881744..3bdc8d37fc1b5 100644 --- a/fs/cache/cache.go +++ b/fs/cache/cache.go @@ -112,7 +112,7 @@ func PinUntilFinalized(f fs.Fs, x interface{}) { // Unpin f from the cache func Unpin(f fs.Fs) { createOnFirstUse() - c.Pin(fs.ConfigString(f)) + c.Unpin(fs.ConfigString(f)) } // Get gets an fs.Fs named fsString either from the cache or creates it afresh From d03fffdf8d9017361ecae703907d9b5ed7824ec6 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 21 Jun 2022 14:28:25 +0100 Subject: [PATCH 089/560] Add eNV25 to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 04322ca1d9ddc..dd8e186b9fec6 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -613,3 +613,4 @@ put them back in again.` >}} * Maciej Radzikowski * Scott Grimes * Phil Shackleton <71221528+philshacks@users.noreply.github.com> + * eNV25 From 3d55b537c6dcd0309cd00609ce7442ab8046be80 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 21 Jun 2022 14:28:25 +0100 Subject: [PATCH 090/560] Add Caleb to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index dd8e186b9fec6..1f2e3ec8fc501 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -614,3 +614,4 @@ put them back in again.` >}} * Scott Grimes * Phil Shackleton <71221528+philshacks@users.noreply.github.com> * eNV25 + * Caleb From ed92bf335d3649328b8b049c8b91ac0c65b861c3 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 21 Jun 2022 14:28:25 +0100 Subject: [PATCH 091/560] Add J-P Treen to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 1f2e3ec8fc501..45a272a092d4a 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -615,3 +615,4 @@ put them back in again.` >}} * Phil Shackleton <71221528+philshacks@users.noreply.github.com> * eNV25 * Caleb + * J-P Treen From 99dfe1eeae466b37cf357c219ab985914b77c7ef Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 21 Jun 2022 14:28:25 +0100 Subject: [PATCH 092/560] Add Martin Czygan to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 45a272a092d4a..90ea4b9fb8cf2 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -616,3 +616,4 @@ put them back in again.` >}} * eNV25 * Caleb * J-P Treen + * Martin Czygan <53705+miku@users.noreply.github.com> From e95dff2fa19677d926d576d96729bdf647aea46e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 20 Jun 2022 11:38:10 +0100 Subject: [PATCH 093/560] drive: add backend commands exportformats and importformats for debugging --- backend/drive/drive.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 2a4d0a64b61b4..b352b9004855e 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -3351,6 +3351,12 @@ attempted if possible. Use the -i flag to see what would be copied before copying. `, +}, { + Name: "exportformats", + Short: "Dump the export formats for debug purposes", +}, { + Name: "importformats", + Short: "Dump the import formats for debug purposes", }} // Command the backend to run a named command @@ -3465,6 +3471,10 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str } } return nil, nil + case "exportformats": + return f.exportFormats(ctx), nil + case "importformats": + return f.importFormats(ctx), nil default: return nil, fs.ErrorCommandNotFound } From ea5bb7936685e9de0ee7365a08c1ed84f6c7a9a5 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 20 Jun 2022 11:39:45 +0100 Subject: [PATCH 094/560] drive: document export for google apps scripts better This also adds some more mime types from the code. See: https://forum.rclone.org/t/how-do-i-copy-google-apps-scripts-with-rclone/31373 --- docs/content/drive.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/content/drive.md b/docs/content/drive.md index 871fc50ef6b09..da17114093f03 100644 --- a/docs/content/drive.md +++ b/docs/content/drive.md @@ -509,23 +509,28 @@ represent the currently available conversions. | Extension | Mime Type | Description | | --------- |-----------| ------------| +| bmp | image/bmp | Windows Bitmap format | | csv | text/csv | Standard CSV format for Spreadsheets | +| doc | application/msword | Classic Word file | | docx | application/vnd.openxmlformats-officedocument.wordprocessingml.document | Microsoft Office Document | | epub | application/epub+zip | E-book format | | html | text/html | An HTML Document | | jpg | image/jpeg | A JPEG Image File | -| json | application/vnd.google-apps.script+json | JSON Text Format | +| json | application/vnd.google-apps.script+json | JSON Text Format for Google Apps scripts | | odp | application/vnd.oasis.opendocument.presentation | Openoffice Presentation | | ods | application/vnd.oasis.opendocument.spreadsheet | Openoffice Spreadsheet | | ods | application/x-vnd.oasis.opendocument.spreadsheet | Openoffice Spreadsheet | | odt | application/vnd.oasis.opendocument.text | Openoffice Document | | pdf | application/pdf | Adobe PDF Format | +| pjpeg | image/pjpeg | Progressive JPEG Image | | png | image/png | PNG Image Format| | pptx | application/vnd.openxmlformats-officedocument.presentationml.presentation | Microsoft Office Powerpoint | | rtf | application/rtf | Rich Text Format | | svg | image/svg+xml | Scalable Vector Graphics Format | | tsv | text/tab-separated-values | Standard TSV format for spreadsheets | | txt | text/plain | Plain Text | +| wmf | application/x-msmetafile | Windows Meta File | +| xls | application/vnd.ms-excel | Classic Excel file | | xlsx | application/vnd.openxmlformats-officedocument.spreadsheetml.sheet | Microsoft Office Spreadsheet | | zip | application/zip | A ZIP file of HTML, Images CSS | From bc705e14d8ea53931f26a089f4dea00652f206de Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 13 Jun 2022 13:45:19 +0100 Subject: [PATCH 095/560] vfscache: fix fatal error: sync: unlock of unlocked mutex error This message is a double panic and was actually caused by an assertion panic in: vfs/vfscache/downloaders/downloaders.go This is triggered by the code added relatively recently to fix a bug with renaming files: ec72432cecdc4eee vfs: fix failed to _ensure cache internal error: downloaders is nil error So it appears that item.o may be nil at this point. This patch detects item.o being nil and fetches it again with NewObject. Fixes #6190 Fixes #6235 --- vfs/vfscache/item.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go index d6a6e20b8d215..2c81458709025 100644 --- a/vfs/vfscache/item.go +++ b/vfs/vfscache/item.go @@ -1131,6 +1131,17 @@ func (item *Item) _ensure(offset, size int64) (err error) { // Downloaders can be nil here if the file has been // renamed, so need to make some more downloaders // OK to call downloaders constructor with item.mu held + + // item.o can also be nil under some circumstances + // See: https://github.com/rclone/rclone/issues/6190 + // See: https://github.com/rclone/rclone/issues/6235 + if item.o == nil { + o, err := item.c.fremote.NewObject(context.Background(), item.name) + if err != nil { + return err + } + item.o = o + } item.downloaders = downloaders.New(item, item.c.opt, item.name, item.o) } return item.downloaders.Download(r) From 4b7dc35cf4c6d30aba2aed1390d71a2b27c1bc7e Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 19 Jun 2022 15:43:13 +0200 Subject: [PATCH 096/560] Fix sync docs incorrect merge This copies the changes from an autogenerated section in the following commit: https://github.com/rclone/rclone/commit/f2a15a174f8cdd21950b123f102ec9f9105098e1 --- cmd/sync/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 0b253b041ff8c..2bb085073f309 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -40,7 +40,7 @@ errors at any point. Duplicate objects (files with the same name, on those providers that support it) are also not yet handled. It is always the contents of the directory that is synced, not the -directory so when source:path is a directory, it's the contents of +directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. See extended explanation in the ` + "`" + `copy` + "`" + ` command above if unsure. From de5ccaab8e1dcd820fee9551c35aa3bcc5cb72b0 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 19 Jun 2022 15:51:37 +0200 Subject: [PATCH 097/560] docs: cross link doc pages for related commands --- cmd/backend/backend.go | 2 +- cmd/check/check.go | 4 +++ cmd/copy/copy.go | 9 +++-- cmd/copyto/copyto.go | 4 +-- cmd/cryptcheck/cryptcheck.go | 6 ++-- cmd/cryptdecode/cryptdecode.go | 2 +- cmd/delete/delete.go | 10 +++--- cmd/hashsum/hashsum.go | 3 ++ cmd/md5sum/md5sum.go | 4 +++ cmd/move/move.go | 6 +++- cmd/moveto/moveto.go | 2 +- cmd/ncdu/ncdu.go | 4 +++ cmd/purge/purge.go | 7 ++-- cmd/rmdir/rmdir.go | 6 ++-- cmd/rmdirs/rmdirs.go | 13 +++---- cmd/serve/webdav/webdav.go | 5 ++- cmd/sha1sum/sha1sum.go | 4 +++ cmd/sync/sync.go | 6 ++-- cmd/tree/tree.go | 8 +++-- docs/content/rc.md | 64 ++++++++++++++++------------------ fs/config/rc.go | 12 +++---- fs/operations/rc.go | 18 +++++----- fs/sync/rc.go | 2 +- 23 files changed, 117 insertions(+), 84 deletions(-) diff --git a/cmd/backend/backend.go b/cmd/backend/backend.go index 936e903c20cd0..85b7e44bd7cf2 100644 --- a/cmd/backend/backend.go +++ b/cmd/backend/backend.go @@ -145,7 +145,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](/commands/rclone_backend/) for more +See the [backend](/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command diff --git a/cmd/check/check.go b/cmd/check/check.go index ec58deb220340..886c36a58fe5e 100644 --- a/cmd/check/check.go +++ b/cmd/check/check.go @@ -139,6 +139,10 @@ Checks the files in the source and destination match. It compares sizes and hashes (MD5 or SHA1) and logs a report of files that don't match. It doesn't alter the source or destination. +For the [crypt](/crypt/) remote there is a dedicated command, +[cryptcheck](/commands/rclone_cryptcheck/), that are able to check +the checksums of the crypted files. + If you supply the |--size-only| flag, it will only compare the sizes not the hashes as well. Use this for a quick check. diff --git a/cmd/copy/copy.go b/cmd/copy/copy.go index ba2815852c10b..33175e1f77416 100644 --- a/cmd/copy/copy.go +++ b/cmd/copy/copy.go @@ -28,13 +28,18 @@ var commandDefinition = &cobra.Command{ Long: strings.ReplaceAll(` Copy the source to the destination. Does not transfer files that are identical on source and destination, testing by size and modification -time or MD5SUM. Doesn't delete files from the destination. +time or MD5SUM. Doesn't delete files from the destination. If you +want to also delete files from destination, to make it match source, +use the [sync](/commands/rclone_sync/) command instead. Note that it is always the contents of the directory that is synced, -not the directory so when source:path is a directory, it's the +not the directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. +To copy single files, use the [copyto](/commands/rclone_copyto/) +command instead. + If dest:path doesn't exist, it is created and the source:path contents go there. diff --git a/cmd/copyto/copyto.go b/cmd/copyto/copyto.go index ed6ff7b1637d8..02d2af21e2ea4 100644 --- a/cmd/copyto/copyto.go +++ b/cmd/copyto/copyto.go @@ -21,8 +21,8 @@ If source:path is a file or directory then it copies it to a file or directory named dest:path. This can be used to upload single files to other than their current -name. If the source is a directory then it acts exactly like the copy -command. +name. If the source is a directory then it acts exactly like the +[copy](/commands/rclone_copy/) command. So diff --git a/cmd/cryptcheck/cryptcheck.go b/cmd/cryptcheck/cryptcheck.go index 7ef0e86a40446..ec0597da73018 100644 --- a/cmd/cryptcheck/cryptcheck.go +++ b/cmd/cryptcheck/cryptcheck.go @@ -23,9 +23,9 @@ var commandDefinition = &cobra.Command{ Use: "cryptcheck remote:path cryptedremote:path", Short: `Cryptcheck checks the integrity of a crypted remote.`, Long: ` -rclone cryptcheck checks a remote against a crypted remote. This is -the equivalent of running rclone check, but able to check the -checksums of the crypted remote. +rclone cryptcheck checks a remote against a [crypted](/crypt/) remote. +This is the equivalent of running rclone [check](/commands/rclone_check/), +but able to check the checksums of the crypted remote. For it to work the underlying remote of the cryptedremote must support some kind of checksum. diff --git a/cmd/cryptdecode/cryptdecode.go b/cmd/cryptdecode/cryptdecode.go index bdfd3e22127bd..27ab93df131db 100644 --- a/cmd/cryptdecode/cryptdecode.go +++ b/cmd/cryptdecode/cryptdecode.go @@ -38,7 +38,7 @@ use it like this rclone cryptdecode --reverse encryptedremote: filename1 filename2 Another way to accomplish this is by using the ` + "`rclone backend encode` (or `decode`)" + `command. -See the documentation on the ` + "`crypt`" + ` overlay for more info. +See the documentation on the [crypt](/crypt/) overlay for more info. `, Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(2, 11, command, args) diff --git a/cmd/delete/delete.go b/cmd/delete/delete.go index 19816cb3a18a2..a432dbe56fb71 100644 --- a/cmd/delete/delete.go +++ b/cmd/delete/delete.go @@ -25,16 +25,16 @@ var commandDefinition = &cobra.Command{ Short: `Remove the files in path.`, // Warning! "|" will be replaced by backticks below Long: strings.ReplaceAll(` -Remove the files in path. Unlike |purge| it obeys include/exclude -filters so can be used to selectively delete files. +Remove the files in path. Unlike [purge](/commands/rclone_purge/) it +obeys include/exclude filters so can be used to selectively delete files. |rclone delete| only deletes files but leaves the directory structure alone. If you want to delete a directory and all of its contents use -the |purge| command. +the [purge](/commands/rclone_purge/) command. If you supply the |--rmdirs| flag, it will remove all empty directories along with it. -You can also use the separate command |rmdir| or |rmdirs| to -delete empty directories only. +You can also use the separate command [rmdir](/commands/rclone_rmdir/) or +[rmdirs](/commands/rclone_rmdirs/) to delete empty directories only. For example, to delete all files bigger than 100 MiB, you may first want to check what would be deleted (use either): diff --git a/cmd/hashsum/hashsum.go b/cmd/hashsum/hashsum.go index d2be8fe5f9604..64dfde4ce5685 100644 --- a/cmd/hashsum/hashsum.go +++ b/cmd/hashsum/hashsum.go @@ -93,6 +93,9 @@ not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling any hash for any remote. +For the MD5 and SHA1 algorithms there are also dedicated commands, +[md5sum](/commands/rclone_md5sum/) and [sha1sum](/commands/rclone_sha1sum/). + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, diff --git a/cmd/md5sum/md5sum.go b/cmd/md5sum/md5sum.go index 4f85c8c930f95..665ae44673e93 100644 --- a/cmd/md5sum/md5sum.go +++ b/cmd/md5sum/md5sum.go @@ -28,6 +28,10 @@ not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling MD5 for any remote. +For other algorithms, see the [hashsum](/commands/rclone_hashsum/) +command. Running ` + "`rclone md5sum remote:path`" + ` is equivalent +to running ` + "`rclone hashsum MD5 remote:path`" + `. + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, diff --git a/cmd/move/move.go b/cmd/move/move.go index 20ca4620909ab..3d7f87167eee9 100644 --- a/cmd/move/move.go +++ b/cmd/move/move.go @@ -33,6 +33,9 @@ Moves the contents of the source directory to the destination directory. Rclone will error if the source and destination overlap and the remote does not support a server-side directory move operation. +To move single files, use the [moveto](/commands/rclone_moveto/) +command instead. + If no filters are in use and if possible this will server-side move |source:path| into |dest:path|. After this |source:path| will no longer exist. @@ -43,7 +46,8 @@ move will be used, otherwise it will copy it (server-side if possible) into |dest:path| then delete the original (if no errors on copy) in |source:path|. -If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag. +If you want to delete empty source directories after move, use the +|--delete-empty-src-dirs| flag. See the [--no-traverse](/docs/#no-traverse) option for controlling whether rclone lists the destination directory or not. Supplying this diff --git a/cmd/moveto/moveto.go b/cmd/moveto/moveto.go index 8390371db3957..8d81bb3d39c3a 100644 --- a/cmd/moveto/moveto.go +++ b/cmd/moveto/moveto.go @@ -22,7 +22,7 @@ directory named dest:path. This can be used to rename files or upload single files to other than their existing name. If the source is a directory then it acts exactly -like the move command. +like the [move](/commands/rclone_move/) command. So diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go index aea2501eb7a12..7f63c8b042a1f 100644 --- a/cmd/ncdu/ncdu.go +++ b/cmd/ncdu/ncdu.go @@ -69,6 +69,10 @@ but is useful as it stands. Note that it might take some time to delete big files/directories. The UI won't respond in the meantime since the deletion is done synchronously. + +For a non-interactive listing of the remote, see the +[tree](/commands/rclone_tree/) command. To just get the total size of +the remote you can also use the [size](/commands/rclone_size/) command. `, Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(1, 1, command, args) diff --git a/cmd/purge/purge.go b/cmd/purge/purge.go index 137e3bd18a51a..335615cf96c2f 100644 --- a/cmd/purge/purge.go +++ b/cmd/purge/purge.go @@ -17,9 +17,10 @@ var commandDefinition = &cobra.Command{ Short: `Remove the path and all of its contents.`, Long: ` Remove the path and all of its contents. Note that this does not obey -include/exclude filters - everything will be removed. Use the ` + "`delete`" + ` -command if you want to selectively delete files. To delete empty directories only, -use command ` + "`rmdir`" + ` or ` + "`rmdirs`" + `. +include/exclude filters - everything will be removed. Use the +[delete](/commands/rclone_delete/) command if you want to selectively +delete files. To delete empty directories only, use command +[rmdir](/commands/rclone_rmdir/) or [rmdirs](/commands/rclone_rmdirs/). **Important**: Since this can cause data loss, test first with the ` + "`--dry-run` or the `--interactive`/`-i`" + ` flag. diff --git a/cmd/rmdir/rmdir.go b/cmd/rmdir/rmdir.go index 31b958a5bfae9..9b532ca4709b1 100644 --- a/cmd/rmdir/rmdir.go +++ b/cmd/rmdir/rmdir.go @@ -18,10 +18,10 @@ var commandDefinition = &cobra.Command{ Long: ` This removes empty directory given by path. Will not remove the path if it has any objects in it, not even empty subdirectories. Use -command ` + "`rmdirs`" + ` (or ` + "`delete`" + ` with option ` + "`--rmdirs`" + `) -to do that. +command [rmdirs](/commands/rclone_rmdirs/) (or [delete](/commands/rclone_delete/) +with option ` + "`--rmdirs`" + `) to do that. -To delete a path and any objects in it, use ` + "`purge`" + ` command. +To delete a path and any objects in it, use [purge](/commands/rclone_purge/) command. `, Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(1, 1, command, args) diff --git a/cmd/rmdirs/rmdirs.go b/cmd/rmdirs/rmdirs.go index 4631dee85b8eb..41788c5271dbe 100644 --- a/cmd/rmdirs/rmdirs.go +++ b/cmd/rmdirs/rmdirs.go @@ -26,15 +26,16 @@ that only contain empty directories), that it finds under the path. The root path itself will also be removed if it is empty, unless you supply the ` + "`--leave-root`" + ` flag. -Use command ` + "`rmdir`" + ` to delete just the empty directory -given by path, not recurse. +Use command [rmdir](/commands/rclone_rmdir/) to delete just the empty +directory given by path, not recurse. This is useful for tidying up remotes that rclone has left a lot of -empty directories in. For example the ` + "`delete`" + ` command will -delete files but leave the directory structure (unless used with -option ` + "`--rmdirs`" + `). +empty directories in. For example the [delete](/commands/rclone_delete/) +command will delete files but leave the directory structure (unless +used with option ` + "`--rmdirs`" + `). -To delete a path and any objects in it, use ` + "`purge`" + ` command. +To delete a path and any objects in it, use [purge](/commands/rclone_purge/) +command. `, Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(1, 1, command, args) diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go index a618c6c06ab68..89a76b2308b34 100644 --- a/cmd/serve/webdav/webdav.go +++ b/cmd/serve/webdav/webdav.go @@ -59,9 +59,8 @@ based on the ModTime and Size of the object. If this flag is set to "auto" then rclone will choose the first supported hash on the backend or you can use a named hash such as -"MD5" or "SHA-1". - -Use "rclone hashsum" to see the full list. +"MD5" or "SHA-1". Use the [hashsum](/commands/rclone_hashsum/) command +to see the full list. ` + httplib.Help + vfs.Help + proxy.Help, RunE: func(command *cobra.Command, args []string) error { diff --git a/cmd/sha1sum/sha1sum.go b/cmd/sha1sum/sha1sum.go index 78a70de51913c..1c3115a8b2712 100644 --- a/cmd/sha1sum/sha1sum.go +++ b/cmd/sha1sum/sha1sum.go @@ -28,6 +28,10 @@ not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling SHA-1 for any remote. +For other algorithms, see the [hashsum](/commands/rclone_hashsum/) +command. Running ` + "`rclone sha1sum remote:path`" + ` is equivalent +to running ` + "`rclone hashsum SHA1 remote:path`" + `. + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index 2bb085073f309..c0c1885ba5d5b 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -28,7 +28,9 @@ Sync the source to the destination, changing the destination only. Doesn't transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Destination is updated to match source, including deleting files -if necessary (except duplicate objects, see below). +if necessary (except duplicate objects, see below). If you don't +want to delete files from destination, use the +[copy](/commands/rclone_copy/) command instead. **Important**: Since this can cause data loss, test first with the ` + "`--dry-run` or the `--interactive`/`-i`" + ` flag. @@ -42,7 +44,7 @@ those providers that support it) are also not yet handled. It is always the contents of the directory that is synced, not the directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. See -extended explanation in the ` + "`" + `copy` + "`" + ` command above if unsure. +extended explanation in the [copy](/commands/rclone_copy/) command if unsure. If dest:path doesn't exist, it is created and the source:path contents go there. diff --git a/cmd/tree/tree.go b/cmd/tree/tree.go index 4986c4d701cc7..4bd8994626556 100644 --- a/cmd/tree/tree.go +++ b/cmd/tree/tree.go @@ -84,11 +84,15 @@ For example 1 directories, 5 files You can use any of the filtering options with the tree command (e.g. ---include and --exclude). You can also use --fast-list. +` + "`--include` and `--exclude`" + `. You can also use ` + "`--fast-list`" + `. The tree command has many options for controlling the listing which -are compatible with the tree command. Note that not all of them have +are compatible with the tree command, for example you can include file +sizes with ` + "`--size`" + `. Note that not all of them have short options as they conflict with rclone's short options. + +For a more interactive navigation of the remote see the +[ncdu](/commands/rclone_ncdu/) command. `, RunE: func(command *cobra.Command, args []string) error { cmd.CheckArgs(1, 1, command, args) diff --git a/docs/content/rc.md b/docs/content/rc.md index be642bd6294a5..8ea6b77d6a655 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -8,10 +8,10 @@ description: "Remote controlling rclone with its API" If rclone is run with the `--rc` flag then it starts an HTTP server which can be used to remote control rclone using its API. -You can either use the [rclone rc](#api-rc) command to access the API +You can either use the [rc](#api-rc) command to access the API or [use HTTP directly](#api-http). -If you just want to run a remote control then see the [rcd command](/commands/rclone_rcd/). +If you just want to run a remote control then see the [rcd](/commands/rclone_rcd/) command. ## Supported parameters @@ -544,8 +544,7 @@ This takes the following parameters: - state - state to restart with - used with continue - result - result to restart with - used with continue - -See the [config create command](/commands/rclone_config_create/) command for more information on the above. +See the [config create](/commands/rclone_config_create/) command for more information on the above. **Authentication is required for this call.** @@ -555,7 +554,7 @@ Parameters: - name - name of remote to delete -See the [config delete command](/commands/rclone_config_delete/) command for more information on the above. +See the [config delete](/commands/rclone_config_delete/) command for more information on the above. **Authentication is required for this call.** @@ -566,7 +565,7 @@ Returns a JSON object: Where keys are remote names and values are the config parameters. -See the [config dump command](/commands/rclone_config_dump/) command for more information on the above. +See the [config dump](/commands/rclone_config_dump/) command for more information on the above. **Authentication is required for this call.** @@ -576,7 +575,7 @@ Parameters: - name - name of remote to get -See the [config dump command](/commands/rclone_config_dump/) command for more information on the above. +See the [config dump](/commands/rclone_config_dump/) command for more information on the above. **Authentication is required for this call.** @@ -585,7 +584,7 @@ See the [config dump command](/commands/rclone_config_dump/) command for more in Returns - remotes - array of remote names -See the [listremotes command](/commands/rclone_listremotes/) command for more information on the above. +See the [listremotes](/commands/rclone_listremotes/) command for more information on the above. **Authentication is required for this call.** @@ -596,8 +595,7 @@ This takes the following parameters: - name - name of remote - parameters - a map of \{ "key": "value" \} pairs - -See the [config password command](/commands/rclone_config_password/) command for more information on the above. +See the [config password](/commands/rclone_config_password/) command for more information on the above. **Authentication is required for this call.** @@ -606,7 +604,7 @@ See the [config password command](/commands/rclone_config_password/) command for Returns a JSON object: - providers - array of objects -See the [config providers command](/commands/rclone_config_providers/) command for more information on the above. +See the [config providers](/commands/rclone_config_providers/) command for more information on the above. **Authentication is required for this call.** @@ -625,8 +623,7 @@ This takes the following parameters: - state - state to restart with - used with continue - result - result to restart with - used with continue - -See the [config update command](/commands/rclone_config_update/) command for more information on the above. +See the [config update](/commands/rclone_config_update/) command for more information on the above. **Authentication is required for this call.** @@ -1072,7 +1069,7 @@ This takes the following parameters: The result is as returned from rclone about --json -See the [about command](/commands/rclone_size/) command for more information on the above. +See the [about](/commands/rclone_size/) command for more information on the above. **Authentication is required for this call.** @@ -1082,7 +1079,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" -See the [cleanup command](/commands/rclone_cleanup/) command for more information on the above. +See the [cleanup](/commands/rclone_cleanup/) command for more information on the above. **Authentication is required for this call.** @@ -1104,8 +1101,9 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" - url - string, URL to read from - - autoFilename - boolean, set to true to retrieve destination file name from url -See the [copyurl command](/commands/rclone_copyurl/) command for more information on the above. +- autoFilename - boolean, set to true to retrieve destination file name from url + +See the [copyurl](/commands/rclone_copyurl/) command for more information on the above. **Authentication is required for this call.** @@ -1115,7 +1113,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" -See the [delete command](/commands/rclone_delete/) command for more information on the above. +See the [delete](/commands/rclone_delete/) command for more information on the above. **Authentication is required for this call.** @@ -1126,7 +1124,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the [deletefile command](/commands/rclone_deletefile/) command for more information on the above. +See the [deletefile](/commands/rclone_deletefile/) command for more information on the above. **Authentication is required for this call.** @@ -1209,7 +1207,7 @@ Returns: - list - This is an array of objects as described in the lsjson command -See the [lsjson command](/commands/rclone_lsjson/) for more information on the above and examples. +See the [lsjson](/commands/rclone_lsjson/) for more information on the above and examples. **Authentication is required for this call.** @@ -1220,7 +1218,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the [mkdir command](/commands/rclone_mkdir/) command for more information on the above. +See the [mkdir](/commands/rclone_mkdir/) command for more information on the above. **Authentication is required for this call.** @@ -1248,7 +1246,7 @@ Returns: - url - URL of the resource -See the [link command](/commands/rclone_link/) command for more information on the above. +See the [link](/commands/rclone_link/) command for more information on the above. **Authentication is required for this call.** @@ -1259,7 +1257,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the [purge command](/commands/rclone_purge/) command for more information on the above. +See the [purge](/commands/rclone_purge/) command for more information on the above. **Authentication is required for this call.** @@ -1270,7 +1268,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the [rmdir command](/commands/rclone_rmdir/) command for more information on the above. +See the [rmdir](/commands/rclone_rmdir/) command for more information on the above. **Authentication is required for this call.** @@ -1281,7 +1279,8 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" - leaveRoot - boolean, set to true not to delete the root -See the [rmdirs command](/commands/rclone_rmdirs/) command for more information on the above. + +See the [rmdirs](/commands/rclone_rmdirs/) command for more information on the above. **Authentication is required for this call.** @@ -1297,7 +1296,7 @@ Returns: - bytes - number of bytes in those files - sizeless - number of files with unknown size, included in count but not accounted for in bytes -See the [size command](/commands/rclone_size/) command for more information on the above. +See the [size](/commands/rclone_size/) command for more information on the above. **Authentication is required for this call.** @@ -1317,7 +1316,7 @@ The result is Note that if you are only interested in files then it is much more efficient to set the filesOnly flag in the options. -See the [lsjson command](/commands/rclone_lsjson/) for more information on the above and examples. +See the [lsjson](/commands/rclone_lsjson/) for more information on the above and examples. **Authentication is required for this call.** @@ -1328,7 +1327,8 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" - each part in body represents a file to be uploaded -See the [uploadfile command](/commands/rclone_uploadfile/) command for more information on the above. + +See the [uploadfile](/commands/rclone_uploadfile/) command for more information on the above. **Authentication is required for this call.** @@ -1542,8 +1542,7 @@ This takes the following parameters: - dstFs - a remote name string e.g. "drive:dst" for the destination - createEmptySrcDirs - create empty src directories on destination if set - -See the [copy command](/commands/rclone_copy/) command for more information on the above. +See the [copy](/commands/rclone_copy/) command for more information on the above. **Authentication is required for this call.** @@ -1556,8 +1555,7 @@ This takes the following parameters: - createEmptySrcDirs - create empty src directories on destination if set - deleteEmptySrcDirs - delete empty src directories if set - -See the [move command](/commands/rclone_move/) command for more information on the above. +See the [move](/commands/rclone_move/) command for more information on the above. **Authentication is required for this call.** @@ -1570,7 +1568,7 @@ This takes the following parameters: - createEmptySrcDirs - create empty src directories on destination if set -See the [sync command](/commands/rclone_sync/) command for more information on the above. +See the [sync](/commands/rclone_sync/) command for more information on the above. **Authentication is required for this call.** diff --git a/fs/config/rc.go b/fs/config/rc.go index db5d48ccaea32..78f6d4866d632 100644 --- a/fs/config/rc.go +++ b/fs/config/rc.go @@ -20,7 +20,7 @@ Returns a JSON object: Where keys are remote names and values are the config parameters. -See the [config dump command](/commands/rclone_config_dump/) command for more information on the above. +See the [config dump](/commands/rclone_config_dump/) command for more information on the above. `, }) } @@ -41,7 +41,7 @@ Parameters: - name - name of remote to get -See the [config dump command](/commands/rclone_config_dump/) command for more information on the above. +See the [config dump](/commands/rclone_config_dump/) command for more information on the above. `, }) } @@ -65,7 +65,7 @@ func init() { Returns - remotes - array of remote names -See the [listremotes command](/commands/rclone_listremotes/) command for more information on the above. +See the [listremotes](/commands/rclone_listremotes/) command for more information on the above. `, }) } @@ -89,7 +89,7 @@ func init() { Returns a JSON object: - providers - array of objects -See the [config providers command](/commands/rclone_config_providers/) command for more information on the above. +See the [config providers](/commands/rclone_config_providers/) command for more information on the above. `, }) } @@ -133,7 +133,7 @@ func init() { - parameters - a map of \{ "key": "value" \} pairs ` + extraHelp + ` -See the [config ` + name + ` command](/commands/rclone_config_` + name + `/) command for more information on the above.`, +See the [config ` + name + `](/commands/rclone_config_` + name + `/) command for more information on the above.`, }) } } @@ -203,7 +203,7 @@ Parameters: - name - name of remote to delete -See the [config delete command](/commands/rclone_config_delete/) command for more information on the above. +See the [config delete](/commands/rclone_config_delete/) command for more information on the above. `, }) } diff --git a/fs/operations/rc.go b/fs/operations/rc.go index 848b56ca10a22..b8c59ce2d8d87 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -41,7 +41,7 @@ Returns: - list - This is an array of objects as described in the lsjson command -See the [lsjson command](/commands/rclone_lsjson/) for more information on the above and examples. +See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples. `, }) } @@ -90,7 +90,7 @@ The result is Note that if you are only interested in files then it is much more efficient to set the filesOnly flag in the options. -See the [lsjson command](/commands/rclone_lsjson/) for more information on the above and examples. +See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples. `, }) } @@ -127,7 +127,7 @@ func init() { The result is as returned from rclone about --json -See the [about command](/commands/rclone_size/) command for more information on the above. +See the [about](/commands/rclone_about/) command for more information on the above. `, }) } @@ -202,11 +202,11 @@ func init() { {name: "mkdir", title: "Make a destination directory or container"}, {name: "rmdir", title: "Remove an empty directory or container"}, {name: "purge", title: "Remove a directory or container and all of its contents"}, - {name: "rmdirs", title: "Remove all the empty directories in the path", help: "- leaveRoot - boolean, set to true not to delete the root"}, + {name: "rmdirs", title: "Remove all the empty directories in the path", help: "- leaveRoot - boolean, set to true not to delete the root\n"}, {name: "delete", title: "Remove files in the path", noRemote: true}, {name: "deletefile", title: "Remove the single file pointed to"}, - {name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n - autoFilename - boolean, set to true to retrieve destination file name from url"}, - {name: "uploadfile", title: "Upload file using multiform/form-data", help: "- each part in body represents a file to be uploaded", needsRequest: true}, + {name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n - autoFilename - boolean, set to true to retrieve destination file name from url\n"}, + {name: "uploadfile", title: "Upload file using multiform/form-data", help: "- each part in body represents a file to be uploaded\n", needsRequest: true}, {name: "cleanup", title: "Remove trashed files in the remote or path", noRemote: true}, } { op := op @@ -226,7 +226,7 @@ func init() { - fs - a remote name string e.g. "drive:" ` + remote + op.help + ` -See the [` + op.name + ` command](/commands/rclone_` + op.name + `/) command for more information on the above. +See the [` + op.name + `](/commands/rclone_` + op.name + `/) command for more information on the above. `, }) } @@ -334,7 +334,7 @@ Returns: - count - number of files - bytes - number of bytes in those files -See the [size command](/commands/rclone_size/) command for more information on the above. +See the [size](/commands/rclone_size/) command for more information on the above. `, }) } @@ -373,7 +373,7 @@ Returns: - url - URL of the resource -See the [link command](/commands/rclone_link/) command for more information on the above. +See the [link](/commands/rclone_link/) command for more information on the above. `, }) } diff --git a/fs/sync/rc.go b/fs/sync/rc.go index 7dee9dea2dac7..42ca315f6bf5b 100644 --- a/fs/sync/rc.go +++ b/fs/sync/rc.go @@ -27,7 +27,7 @@ func init() { - createEmptySrcDirs - create empty src directories on destination if set ` + moveHelp + ` -See the [` + name + ` command](/commands/rclone_` + name + `/) command for more information on the above.`, +See the [` + name + `](/commands/rclone_` + name + `/) command for more information on the above.`, }) } } From 4f0eae366f9ea39d0f5695bba2e5df0002cb26ba Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 19 Jun 2022 15:52:23 +0200 Subject: [PATCH 098/560] docs/ncdu: fix inconsistency in key map help text Ctrl+L was listed as ^L, while Ctrl+c was listed as c-C. Changed the latter to ^c. --- cmd/ncdu/ncdu.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go index 7f63c8b042a1f..c005a94cfedbe 100644 --- a/cmd/ncdu/ncdu.go +++ b/cmd/ncdu/ncdu.go @@ -107,7 +107,7 @@ func helpText() (tr []string) { " Y display current path", " ^L refresh screen", " ? to toggle help on and off", - " q/ESC/c-C to quit", + " q/ESC/^c to quit", }...) return } From ee87e919c526e948c2d99ff0c60abbb73ed94780 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 19 Jun 2022 16:06:04 +0200 Subject: [PATCH 099/560] docs/ncdu: note that refresh screen shortcut will fix screen corruption --- cmd/ncdu/ncdu.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go index c005a94cfedbe..abb748b6d5276 100644 --- a/cmd/ncdu/ncdu.go +++ b/cmd/ncdu/ncdu.go @@ -105,7 +105,7 @@ func helpText() (tr []string) { } tr = append(tr, []string{ " Y display current path", - " ^L refresh screen", + " ^L refresh screen (fix screen corruption)", " ? to toggle help on and off", " q/ESC/^c to quit", }...) From bc70a95fca06670c4e7065d621463417698186ae Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 19 Jun 2022 18:59:54 +0200 Subject: [PATCH 100/560] docs: consistent capitalization of WebDAV, DLNA, HTTP --- README.md | 2 +- backend/http/http.go | 4 ++-- backend/webdav/webdav.go | 4 ++-- cmd/serve/http/data/data.go | 6 +++--- cmd/serve/httplib/httplib.go | 8 ++++---- cmd/serve/webdav/webdav.go | 12 ++++++------ docs/content/_index.md | 2 +- docs/content/faq.md | 2 +- docs/content/filtering.md | 2 +- docs/content/http.md | 2 +- docs/content/onedrive.md | 2 +- docs/content/webdav.md | 4 ++-- 12 files changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 7a955d31f5162..ba85e16bdd11b 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ These backends adapt or modify other storage providers * Optional encryption ([Crypt](https://rclone.org/crypt/)) * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/)) * Multi-threaded downloads to local disk - * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDav/FTP/SFTP/dlna + * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDAV/FTP/SFTP/DLNA ## Installation & documentation diff --git a/backend/http/http.go b/backend/http/http.go index e2a993a448aca..bd53da943c87a 100644 --- a/backend/http/http.go +++ b/backend/http/http.go @@ -35,11 +35,11 @@ var ( func init() { fsi := &fs.RegInfo{ Name: "http", - Description: "http Connection", + Description: "HTTP Connection", NewFs: NewFs, Options: []fs.Option{{ Name: "url", - Help: "URL of http host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.", + Help: "URL of HTTP host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.", Required: true, }, { Name: "headers", diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go index 7016b289cd042..d663ac0bfdd4d 100644 --- a/backend/webdav/webdav.go +++ b/backend/webdav/webdav.go @@ -66,7 +66,7 @@ func init() { fs.Register(&fs.RegInfo{ Name: "webdav", - Description: "Webdav", + Description: "WebDAV", NewFs: NewFs, Options: []fs.Option{{ Name: "url", @@ -74,7 +74,7 @@ func init() { Required: true, }, { Name: "vendor", - Help: "Name of the Webdav site/service/software you are using.", + Help: "Name of the WebDAV site/service/software you are using.", Examples: []fs.OptionExample{{ Value: "nextcloud", Help: "Nextcloud", diff --git a/cmd/serve/http/data/data.go b/cmd/serve/http/data/data.go index f413c05526aba..de8362100fd7d 100644 --- a/cmd/serve/http/data/data.go +++ b/cmd/serve/http/data/data.go @@ -19,8 +19,8 @@ import ( var Help = ` #### Template ---template allows a user to specify a custom markup template for http -and webdav serve functions. The server exports the following markup +--template allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: | Parameter | Description | @@ -58,7 +58,7 @@ func AfterEpoch(t time.Time) bool { return t.After(time.Time{}) } -// GetTemplate returns the HTML template for serving directories via HTTP/Webdav +// GetTemplate returns the HTML template for serving directories via HTTP/WebDAV func GetTemplate(tmpl string) (tpl *template.Template, err error) { var templateString string if tmpl == "" { diff --git a/cmd/serve/httplib/httplib.go b/cmd/serve/httplib/httplib.go index ae6583af9b697..4e3efe7d02bac 100644 --- a/cmd/serve/httplib/httplib.go +++ b/cmd/serve/httplib/httplib.go @@ -53,8 +53,8 @@ inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically. ---template allows a user to specify a custom markup template for http -and webdav serve functions. The server exports the following markup +--template allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: | Parameter | Description | @@ -99,8 +99,8 @@ Use --realm to set the authentication realm. #### SSL/TLS -By default this will serve over http. If you want you can serve over -https. You will need to supply the --cert and --key flags. If you +By default this will serve over HTTP. If you want you can serve over +HTTPS. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also. diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go index 89a76b2308b34..4d152a433238b 100644 --- a/cmd/serve/webdav/webdav.go +++ b/cmd/serve/webdav/webdav.go @@ -43,14 +43,14 @@ func init() { // Command definition for cobra var Command = &cobra.Command{ Use: "webdav remote:path", - Short: `Serve remote:path over webdav.`, + Short: `Serve remote:path over WebDAV.`, Long: ` -rclone serve webdav implements a basic webdav server to serve the -remote over HTTP via the webdav protocol. This can be viewed with a -webdav client, through a web browser, or you can make a remote of -type webdav to read and write it. +rclone serve webdav implements a basic WebDAV server to serve the +remote over HTTP via the WebDAV protocol. This can be viewed with a +WebDAV client, through a web browser, or you can make a remote of +type WebDAV to read and write it. -### Webdav options +### WebDAV options #### --etag-hash diff --git a/docs/content/_index.md b/docs/content/_index.md index c7a213bfa3970..42c8b48666586 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -95,7 +95,7 @@ Rclone helps you: - [Move](/commands/rclone_move/) files to cloud storage deleting the local after verification - [Check](/commands/rclone_check/) hashes and for missing/extra files - [Mount](/commands/rclone_mount/) your cloud storage as a network disk -- [Serve](/commands/rclone_serve/) local or remote files over [HTTP](/commands/rclone_serve_http/)/[WebDav](/commands/rclone_serve_webdav/)/[FTP](/commands/rclone_serve_ftp/)/[SFTP](/commands/rclone_serve_sftp/)/[dlna](/commands/rclone_serve_dlna/) +- [Serve](/commands/rclone_serve/) local or remote files over [HTTP](/commands/rclone_serve_http/)/[WebDav](/commands/rclone_serve_webdav/)/[FTP](/commands/rclone_serve_ftp/)/[SFTP](/commands/rclone_serve_sftp/)/[DLNA](/commands/rclone_serve_dlna/) - Experimental [Web based GUI](/gui/) ## Supported providers {#providers} diff --git a/docs/content/faq.md b/docs/content/faq.md index ecd2408829e5c..00ed0e06d3950 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -118,7 +118,7 @@ e.g. export no_proxy=localhost,127.0.0.0/8,my.host.name export NO_PROXY=$no_proxy -Note that the ftp backend does not support `ftp_proxy` yet. +Note that the FTP backend does not support `ftp_proxy` yet. ### Rclone gives x509: failed to load system roots and no roots provided error ### diff --git a/docs/content/filtering.md b/docs/content/filtering.md index de191ddc22ca4..8fc3bf109978d 100644 --- a/docs/content/filtering.md +++ b/docs/content/filtering.md @@ -274,7 +274,7 @@ every path against the supplied regular expression(s). Directory recursion optimisation occurs if either: * A source remote does not support the rclone `ListR` primitive. local, -sftp, Microsoft OneDrive and WebDav do not support `ListR`. Google +sftp, Microsoft OneDrive and WebDAV do not support `ListR`. Google Drive and most bucket type storage do. [Full list](https://rclone.org/overview/#optional-features) * On other remotes (those that support `ListR`), if the rclone command is not naturally recursive, and diff --git a/docs/content/http.md b/docs/content/http.md index 501cd5a22ade4..9affbf3584705 100644 --- a/docs/content/http.md +++ b/docs/content/http.md @@ -52,7 +52,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / http Connection +XX / HTTP Connection \ "http" [snip] Storage> http diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index f25bbe513ca22..36f3c89fbe07c 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -670,7 +670,7 @@ Description: Using application 'rclone' is currently not supported for your orga This means that rclone can't use the OneDrive for Business API with your account. You can't do much about it, maybe write an email to your admins. -However, there are other ways to interact with your OneDrive account. Have a look at the webdav backend: https://rclone.org/webdav/#sharepoint +However, there are other ways to interact with your OneDrive account. Have a look at the WebDAV backend: https://rclone.org/webdav/#sharepoint ### invalid\_grant (AADSTS50076) #### diff --git a/docs/content/webdav.md b/docs/content/webdav.md index c5612fead8d2f..766a5cc8cd28b 100644 --- a/docs/content/webdav.md +++ b/docs/content/webdav.md @@ -31,7 +31,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Webdav +XX / WebDAV \ "webdav" [snip] Storage> webdav @@ -40,7 +40,7 @@ Choose a number from below, or type in your own value 1 / Connect to example.com \ "https://example.com" url> https://example.com/remote.php/webdav/ -Name of the Webdav site/service/software you are using +Name of the WebDAV site/service/software you are using Choose a number from below, or type in your own value 1 / Nextcloud \ "nextcloud" From 70d9d75801cd3aa59a4c856fcff06dce31a05544 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 19 Jun 2022 19:18:14 +0200 Subject: [PATCH 101/560] docs: improve serve command descriptions --- cmd/serve/dlna/dlna.go | 18 ++++++++++-------- cmd/serve/ftp/ftp.go | 6 +++--- cmd/serve/http/http.go | 6 +++--- cmd/serve/restic/restic.go | 4 ++-- cmd/serve/serve.go | 4 ++-- cmd/serve/sftp/sftp.go | 5 ++--- cmd/serve/webdav/webdav.go | 8 +++----- 7 files changed, 25 insertions(+), 26 deletions(-) diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go index a6b71bf24fa44..0c9d2fbcaab51 100644 --- a/cmd/serve/dlna/dlna.go +++ b/cmd/serve/dlna/dlna.go @@ -34,14 +34,16 @@ func init() { var Command = &cobra.Command{ Use: "dlna remote:path", Short: `Serve remote:path over DLNA`, - Long: `rclone serve dlna is a DLNA media server for media stored in an rclone remote. Many -devices, such as the Xbox and PlayStation, can automatically discover this server in the LAN -and play audio/video from it. VLC is also supported. Service discovery uses UDP multicast -packets (SSDP) and will thus only work on LANs. - -Rclone will list all files present in the remote, without filtering based on media formats or -file extensions. Additionally, there is no media transcoding support. This means that some -players might show files that they are not able to play back correctly. + Long: `Run a DLNA media server for media stored in an rclone remote. Many +devices, such as the Xbox and PlayStation, can automatically discover +this server in the LAN and play audio/video from it. VLC is also +supported. Service discovery uses UDP multicast packets (SSDP) and +will thus only work on LANs. + +Rclone will list all files present in the remote, without filtering +based on media formats or file extensions. Additionally, there is no +media transcoding support. This means that some players might show +files that they are not able to play back correctly. ` + dlnaflags.Help + vfs.Help, Run: func(command *cobra.Command, args []string) { diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go index b8022599e02a9..f8ba817259cad 100644 --- a/cmd/serve/ftp/ftp.go +++ b/cmd/serve/ftp/ftp.go @@ -80,9 +80,9 @@ var Command = &cobra.Command{ Use: "ftp remote:path", Short: `Serve remote:path over FTP.`, Long: ` -rclone serve ftp implements a basic ftp server to serve the -remote over FTP protocol. This can be viewed with a ftp client -or you can make a remote of type ftp to read and write it. +Run a basic FTP server to serve a remote over FTP protocol. +This can be viewed with a FTP client or you can make a remote of +type FTP to read and write it. ### Server options diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go index 780aa9c915907..c809d6c33e3be 100644 --- a/cmd/serve/http/http.go +++ b/cmd/serve/http/http.go @@ -47,9 +47,9 @@ func init() { var Command = &cobra.Command{ Use: "http remote:path", Short: `Serve the remote over HTTP.`, - Long: `rclone serve http implements a basic web server to serve the remote -over HTTP. This can be viewed in a web browser or you can make a -remote of type http read from it. + Long: `Run a basic web server to serve a remote over HTTP. +This can be viewed in a web browser or you can make a remote of type +http read from it. You can use the filter flags (e.g. --include, --exclude) to control what is served. diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go index b1ac10393b345..65b60dc053d82 100644 --- a/cmd/serve/restic/restic.go +++ b/cmd/serve/restic/restic.go @@ -50,8 +50,8 @@ func init() { var Command = &cobra.Command{ Use: "restic remote:path", Short: `Serve the remote for restic's REST API.`, - Long: `rclone serve restic implements restic's REST backend API -over HTTP. This allows restic to use rclone as a data storage + Long: `Run a basic web server to serve a remove over restic's REST backend +API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly. [Restic](https://restic.net/) is a command-line program for doing diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 7bed14c7e6873..848cb24bb34db 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -41,8 +41,8 @@ func init() { var Command = &cobra.Command{ Use: "serve [opts] ", Short: `Serve a remote over a protocol.`, - Long: `rclone serve is used to serve a remote over a given protocol. This -command requires the use of a subcommand to specify the protocol, e.g. + Long: `Serve a remote over a given protocol. Requires the use of a +subcommand to specify the protocol, e.g. rclone serve http remote: diff --git a/cmd/serve/sftp/sftp.go b/cmd/serve/sftp/sftp.go index b1be36bb1630b..cecb757077ac6 100644 --- a/cmd/serve/sftp/sftp.go +++ b/cmd/serve/sftp/sftp.go @@ -62,9 +62,8 @@ func init() { var Command = &cobra.Command{ Use: "sftp remote:path", Short: `Serve the remote over SFTP.`, - Long: `rclone serve sftp implements an SFTP server to serve the remote -over SFTP. This can be used with an SFTP client or you can make a -remote of type sftp to use with it. + Long: `Run a SFTP server to serve a remote over SFTP. This can be used +with an SFTP client or you can make a remote of type sftp to use with it. You can use the filter flags (e.g. --include, --exclude) to control what is served. diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go index 4d152a433238b..bfd0eda31935d 100644 --- a/cmd/serve/webdav/webdav.go +++ b/cmd/serve/webdav/webdav.go @@ -44,11 +44,9 @@ func init() { var Command = &cobra.Command{ Use: "webdav remote:path", Short: `Serve remote:path over WebDAV.`, - Long: ` -rclone serve webdav implements a basic WebDAV server to serve the -remote over HTTP via the WebDAV protocol. This can be viewed with a -WebDAV client, through a web browser, or you can make a remote of -type WebDAV to read and write it. + Long: `Run a basic WebDAV server to serve a remote over HTTP via the +WebDAV protocol. This can be viewed with a WebDAV client, through a web +browser, or you can make a remote of type WebDAV to read and write it. ### WebDAV options From dbf1234edfbfbd63673562dfcaa0b566eed2fd18 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 19 Jun 2022 19:29:02 +0200 Subject: [PATCH 102/560] docs: skip "Connection" suffix from FTP, SSH/SFTP and HTTP backend names --- backend/ftp/ftp.go | 2 +- backend/http/http.go | 2 +- backend/sftp/sftp.go | 2 +- docs/content/ftp.md | 2 +- docs/content/http.md | 2 +- docs/content/s3.md | 4 ++-- docs/content/sftp.md | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index 92b84fe9c014c..c078f78b095f0 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -45,7 +45,7 @@ const ( func init() { fs.Register(&fs.RegInfo{ Name: "ftp", - Description: "FTP Connection", + Description: "FTP", NewFs: NewFs, Options: []fs.Option{{ Name: "host", diff --git a/backend/http/http.go b/backend/http/http.go index bd53da943c87a..3e6e83155426d 100644 --- a/backend/http/http.go +++ b/backend/http/http.go @@ -35,7 +35,7 @@ var ( func init() { fsi := &fs.RegInfo{ Name: "http", - Description: "HTTP Connection", + Description: "HTTP", NewFs: NewFs, Options: []fs.Option{{ Name: "url", diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 5e9cbe18dfd95..74af5fa4104a6 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -57,7 +57,7 @@ var ( func init() { fsi := &fs.RegInfo{ Name: "sftp", - Description: "SSH/SFTP Connection", + Description: "SSH/SFTP", NewFs: NewFs, Options: []fs.Option{{ Name: "host", diff --git a/docs/content/ftp.md b/docs/content/ftp.md index 203e944bcba6d..f2d4a4229ede5 100644 --- a/docs/content/ftp.md +++ b/docs/content/ftp.md @@ -39,7 +39,7 @@ Type of storage to configure. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [snip] -XX / FTP Connection +XX / FTP \ "ftp" [snip] Storage> ftp diff --git a/docs/content/http.md b/docs/content/http.md index 9affbf3584705..10ef35098324c 100644 --- a/docs/content/http.md +++ b/docs/content/http.md @@ -52,7 +52,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / HTTP Connection +XX / HTTP \ "http" [snip] Storage> http diff --git a/docs/content/s3.md b/docs/content/s3.md index 15b26e7e2944c..fbd47c76327cd 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -2901,7 +2901,7 @@ r) Rename remote c) Copy remote s) Set configuration password q) Quit config -e/n/d/r/c/s/q> q +e/n/d/r/c/s/q> q ``` ### IBM COS (S3) @@ -2937,7 +2937,7 @@ Choose a number from below, or type in your own value 4 / Backblaze B2 \ "b2" [snip] - 23 / http Connection + 23 / HTTP \ "http" Storage> 3 ``` diff --git a/docs/content/sftp.md b/docs/content/sftp.md index a0fc7271cb92e..fa0f5ba054943 100644 --- a/docs/content/sftp.md +++ b/docs/content/sftp.md @@ -49,7 +49,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / SSH/SFTP Connection +XX / SSH/SFTP \ "sftp" [snip] Storage> sftp From bf54c909c91e3aa338b379b99c3f6c55655ba930 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 19 Jun 2022 19:43:16 +0200 Subject: [PATCH 103/560] docs: improved capitalization --- docs/content/_index.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/content/_index.md b/docs/content/_index.md index 42c8b48666586..dcf51e23f8c9c 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -174,14 +174,14 @@ WebDAV or S3, that work out of the box.) These backends adapt or modify other storage providers: -{{< provider name="Alias: rename existing remotes" home="/alias/" config="/alias/" >}} -{{< provider name="Cache: cache remotes (DEPRECATED)" home="/cache/" config="/cache/" >}} -{{< provider name="Chunker: split large files" home="/chunker/" config="/chunker/" >}} -{{< provider name="Combine: combine multiple remotes into a directory tree" home="/combine/" config="/combine/" >}} -{{< provider name="Compress: compress files" home="/compress/" config="/compress/" >}} -{{< provider name="Crypt: encrypt files" home="/crypt/" config="/crypt/" >}} -{{< provider name="Hasher: hash files" home="/hasher/" config="/hasher/" >}} -{{< provider name="Union: join multiple remotes to work together" home="/union/" config="/union/" >}} +{{< provider name="Alias: Rename existing remotes" home="/alias/" config="/alias/" >}} +{{< provider name="Cache: Cache remotes (DEPRECATED)" home="/cache/" config="/cache/" >}} +{{< provider name="Chunker: Split large files" home="/chunker/" config="/chunker/" >}} +{{< provider name="Combine: Combine multiple remotes into a directory tree" home="/combine/" config="/combine/" >}} +{{< provider name="Compress: Compress files" home="/compress/" config="/compress/" >}} +{{< provider name="Crypt: Encrypt files" home="/crypt/" config="/crypt/" >}} +{{< provider name="Hasher: Hash files" home="/hasher/" config="/hasher/" >}} +{{< provider name="Union: Join multiple remotes to work together" home="/union/" config="/union/" >}} ## Links From 9f81b4df4f65e277c8d4363724a448bfc5608c8c Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 19 Jun 2022 19:45:38 +0200 Subject: [PATCH 104/560] docs/lsjson: fix code block indentation --- cmd/lsjson/lsjson.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lsjson/lsjson.go b/cmd/lsjson/lsjson.go index 47bafa4e3b876..aa23c210d9479 100644 --- a/cmd/lsjson/lsjson.go +++ b/cmd/lsjson/lsjson.go @@ -41,7 +41,7 @@ var commandDefinition = &cobra.Command{ The output is an array of Items, where each Item looks like this - { + { "Hashes" : { "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f", "MD5" : "b1946ac92492d2347c6235b4d2611184", @@ -59,7 +59,7 @@ The output is an array of Items, where each Item looks like this "Path" : "full/path/goes/here/file.txt", "Size" : 6, "Tier" : "hot", - } + } If --hash is not specified the Hashes property won't be emitted. The types of hash can be specified with the --hash-type parameter (which From 53f831f40a7341cc3da199b0266ea573b6d94551 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 19 Jun 2022 19:55:37 +0200 Subject: [PATCH 105/560] docs: add missing code section formatting to commands and flags --- cmd/cryptdecode/cryptdecode.go | 4 +-- cmd/dedupe/dedupe.go | 2 +- cmd/genautocomplete/genautocomplete.go | 2 +- cmd/listremotes/listremotes.go | 2 +- cmd/lsd/lsd.go | 4 +-- cmd/lsf/lsf.go | 12 ++++---- cmd/lsjson/lsjson.go | 20 ++++++------- cmd/obscure/obscure.go | 2 +- cmd/rc/rc.go | 20 ++++++------- cmd/rcat/rcat.go | 6 ++-- cmd/serve/http/data/data.go | 2 +- cmd/serve/http/http.go | 6 ++-- cmd/serve/httplib/httplib.go | 40 +++++++++++++------------- cmd/serve/restic/restic.go | 10 +++---- cmd/serve/sftp/sftp.go | 29 ++++++++++--------- lib/http/auth/auth.go | 8 +++--- lib/http/http.go | 30 +++++++++---------- 17 files changed, 100 insertions(+), 99 deletions(-) diff --git a/cmd/cryptdecode/cryptdecode.go b/cmd/cryptdecode/cryptdecode.go index 27ab93df131db..2f938cbfa7a0a 100644 --- a/cmd/cryptdecode/cryptdecode.go +++ b/cmd/cryptdecode/cryptdecode.go @@ -29,7 +29,7 @@ var commandDefinition = &cobra.Command{ rclone cryptdecode returns unencrypted file names when provided with a list of encrypted file names. List limit is 10 items. -If you supply the --reverse flag, it will return encrypted file names. +If you supply the ` + "`--reverse`" + ` flag, it will return encrypted file names. use it like this @@ -37,7 +37,7 @@ use it like this rclone cryptdecode --reverse encryptedremote: filename1 filename2 -Another way to accomplish this is by using the ` + "`rclone backend encode` (or `decode`)" + `command. +Another way to accomplish this is by using the ` + "`rclone backend encode` (or `decode`)" + ` command. See the documentation on the [crypt](/crypt/) overlay for more info. `, Run: func(command *cobra.Command, args []string) { diff --git a/cmd/dedupe/dedupe.go b/cmd/dedupe/dedupe.go index 270634aede92f..b43c4c2795663 100644 --- a/cmd/dedupe/dedupe.go +++ b/cmd/dedupe/dedupe.go @@ -37,7 +37,7 @@ Opendrive) that can have duplicate file names. It can be run on wrapping backend (e.g. crypt) if they wrap a backend which supports duplicate file names. -However if --by-hash is passed in then dedupe will find files with +However if ` + "`--by-hash`" + ` is passed in then dedupe will find files with duplicate hashes instead which will work on any backend which supports at least one hash. This can be used to find files with duplicate content. This is known as deduping by hash. diff --git a/cmd/genautocomplete/genautocomplete.go b/cmd/genautocomplete/genautocomplete.go index 760d17dc05779..5971135770d54 100644 --- a/cmd/genautocomplete/genautocomplete.go +++ b/cmd/genautocomplete/genautocomplete.go @@ -14,6 +14,6 @@ var completionDefinition = &cobra.Command{ Short: `Output completion script for a given shell.`, Long: ` Generates a shell completion script for rclone. -Run with --help to list the supported shells. +Run with ` + "`--help`" + ` to list the supported shells. `, } diff --git a/cmd/listremotes/listremotes.go b/cmd/listremotes/listremotes.go index 9d1e7f0b8211f..908979a4fa34f 100644 --- a/cmd/listremotes/listremotes.go +++ b/cmd/listremotes/listremotes.go @@ -27,7 +27,7 @@ var commandDefinition = &cobra.Command{ Long: ` rclone listremotes lists all the available remotes from the config file. -When uses with the -l flag it lists the types too. +When used with the ` + "`--long`" + ` flag it lists the types too. `, Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(0, 0, command, args) diff --git a/cmd/lsd/lsd.go b/cmd/lsd/lsd.go index 75db3bb7335b5..bd723daadb5aa 100644 --- a/cmd/lsd/lsd.go +++ b/cmd/lsd/lsd.go @@ -27,7 +27,7 @@ var commandDefinition = &cobra.Command{ Short: `List all directories/containers/buckets in the path.`, Long: ` Lists the directories in the source path to standard output. Does not -recurse by default. Use the -R flag to recurse. +recurse by default. Use the ` + "`-R`" + ` flag to recurse. This command lists the total size of the directory (if known, -1 if not), the modification time (if known, the current time if not), the @@ -45,7 +45,7 @@ Or -1 2017-01-03 14:40:54 -1 2500files -1 2017-07-08 14:39:28 -1 4000files -If you just want the directory names use "rclone lsf --dirs-only". +If you just want the directory names use ` + "`rclone lsf --dirs-only`" + `. ` + lshelp.Help, Run: func(command *cobra.Command, args []string) { diff --git a/cmd/lsf/lsf.go b/cmd/lsf/lsf.go index a461a5c9cd961..812edf8eab748 100644 --- a/cmd/lsf/lsf.go +++ b/cmd/lsf/lsf.go @@ -59,7 +59,7 @@ Eg ferejej3gux/ fubuwic -Use the --format option to control what gets listed. By default this +Use the ` + "`--format`" + ` option to control what gets listed. By default this is just the path, but you can use these parameters to control the output: @@ -74,7 +74,7 @@ output: T - tier of storage if known, e.g. "Hot" or "Cool" So if you wanted the path, size and modification time, you would use ---format "pst", or maybe --format "tsp" to put the path last. +` + "`--format \"pst\"`, or maybe `--format \"tsp\"`" + ` to put the path last. Eg @@ -86,7 +86,7 @@ Eg 2016-06-25 18:55:40;37600;fubuwic If you specify "h" in the format you will get the MD5 hash by default, -use the "--hash" flag to change which hash you want. Note that this +use the ` + "`--hash`" + ` flag to change which hash you want. Note that this can be returned as an empty string if it isn't available on the object (and for directories), "ERROR" if there was an error reading it from the object and "UNSUPPORTED" if that object does not support that hash @@ -108,7 +108,7 @@ Eg (Though "rclone md5sum ." is an easier way of typing this.) By default the separator is ";" this can be changed with the ---separator flag. Note that separators aren't escaped in the path so +` + "`--separator`" + ` flag. Note that separators aren't escaped in the path so putting it last is a good strategy. Eg @@ -130,8 +130,8 @@ Eg test.sh,449 "this file contains a comma, in the file name.txt",6 -Note that the --absolute parameter is useful for making lists of files -to pass to an rclone copy with the --files-from-raw flag. +Note that the ` + "`--absolute`" + ` parameter is useful for making lists of files +to pass to an rclone copy with the ` + "`--files-from-raw`" + ` flag. For example, to find all the files modified within one day and copy those only (without traversing the whole directory structure): diff --git a/cmd/lsjson/lsjson.go b/cmd/lsjson/lsjson.go index aa23c210d9479..70b96a0f76513 100644 --- a/cmd/lsjson/lsjson.go +++ b/cmd/lsjson/lsjson.go @@ -61,27 +61,27 @@ The output is an array of Items, where each Item looks like this "Tier" : "hot", } -If --hash is not specified the Hashes property won't be emitted. The -types of hash can be specified with the --hash-type parameter (which -may be repeated). If --hash-type is set then it implies --hash. +If ` + "`--hash`" + ` is not specified the Hashes property won't be emitted. The +types of hash can be specified with the ` + "`--hash-type`" + ` parameter (which +may be repeated). If ` + "`--hash-type`" + ` is set then it implies ` + "`--hash`" + `. -If --no-modtime is specified then ModTime will be blank. This can +If ` + "`--no-modtime`" + ` is specified then ModTime will be blank. This can speed things up on remotes where reading the ModTime takes an extra request (e.g. s3, swift). -If --no-mimetype is specified then MimeType will be blank. This can +If ` + "`--no-mimetype`" + ` is specified then MimeType will be blank. This can speed things up on remotes where reading the MimeType takes an extra request (e.g. s3, swift). -If --encrypted is not specified the Encrypted won't be emitted. +If ` + "`--encrypted`" + ` is not specified the Encrypted won't be emitted. -If --dirs-only is not specified files in addition to directories are +If ` + "`--dirs-only`" + ` is not specified files in addition to directories are returned -If --files-only is not specified directories in addition to the files +If ` + "`--files-only`" + ` is not specified directories in addition to the files will be returned. -if --stat is set then a single JSON blob will be returned about the +if ` + "`--stat`" + ` is set then a single JSON blob will be returned about the item pointed to. This will return an error if the item isn't found. However on bucket based backends (like s3, gcs, b2, azureblob etc) if the item isn't found it will return an empty directory as it isn't @@ -90,7 +90,7 @@ possible to tell empty directories from missing directories there. The Path field will only show folders below the remote path being listed. If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt" will be "subfolder/file.txt", not "remote:path/subfolder/file.txt". -When used without --recursive the Path will always be the same as Name. +When used without ` + "`--recursive`" + ` the Path will always be the same as Name. If the directory is a bucket in a bucket-based backend, then "IsBucket" will be set to true. This key won't be present unless it is diff --git a/cmd/obscure/obscure.go b/cmd/obscure/obscure.go index 71987079d30f5..d26f1d72b660b 100644 --- a/cmd/obscure/obscure.go +++ b/cmd/obscure/obscure.go @@ -33,7 +33,7 @@ This command can also accept a password through STDIN instead of an argument by passing a hyphen as an argument. This will use the first line of STDIN as the password not including the trailing newline. -echo "secretpassword" | rclone obscure - + echo "secretpassword" | rclone obscure - If there is no data on STDIN to read, rclone obscure will default to obfuscating the hyphen itself. diff --git a/cmd/rc/rc.go b/cmd/rc/rc.go index fec0d5c403b61..6b123f97e3ffb 100644 --- a/cmd/rc/rc.go +++ b/cmd/rc/rc.go @@ -50,26 +50,26 @@ var commandDefinition = &cobra.Command{ Short: `Run a command against a running rclone.`, Long: ` -This runs a command against a running rclone. Use the --url flag to +This runs a command against a running rclone. Use the ` + "`--url`" + ` flag to specify an non default URL to connect on. This can be either a ":port" which is taken to mean "http://localhost:port" or a "host:port" which is taken to mean "http://host:port" -A username and password can be passed in with --user and --pass. +A username and password can be passed in with ` + "`--user`" + ` and ` + "`--pass`" + `. -Note that --rc-addr, --rc-user, --rc-pass will be read also for --url, ---user, --pass. +Note that ` + "`--rc-addr`, `--rc-user`, `--rc-pass`" + ` will be read also for +` + "`--url`, `--user`, `--pass`" + `. Arguments should be passed in as parameter=value. The result will be returned as a JSON object by default. -The --json parameter can be used to pass in a JSON blob as an input +The ` + "`--json`" + ` parameter can be used to pass in a JSON blob as an input instead of key=value arguments. This is the only way of passing in more complicated values. -The -o/--opt option can be used to set a key "opt" with key, value -options in the form "-o key=value" or "-o key". It can be repeated as +The ` + "`-o`/`--opt`" + ` option can be used to set a key "opt" with key, value +options in the form ` + "`-o key=value` or `-o key`" + `. It can be repeated as many times as required. This is useful for rc commands which take the "opt" parameter which by convention is a dictionary of strings. @@ -80,7 +80,7 @@ Will place this in the "opt" value {"key":"value", "key2","") -The -a/--arg option can be used to set strings in the "arg" value. It +The ` + "`-a`/`--arg`" + ` option can be used to set strings in the "arg" value. It can be repeated as many times as required. This is useful for rc commands which take the "arg" parameter which by convention is a list of strings. @@ -91,13 +91,13 @@ Will place this in the "arg" value ["value", "value2"] -Use --loopback to connect to the rclone instance running "rclone rc". +Use ` + "`--loopback`" + ` to connect to the rclone instance running ` + "`rclone rc`" + `. This is very useful for testing commands without having to run an rclone rc server, e.g.: rclone rc --loopback operations/about fs=/ -Use "rclone rc" to see a list of all possible commands.`, +Use ` + "`rclone rc`" + ` to see a list of all possible commands.`, Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(0, 1e9, command, args) cmd.Run(false, false, command, func() error { diff --git a/cmd/rcat/rcat.go b/cmd/rcat/rcat.go index ef969b170f6ab..f082fa95d34b5 100644 --- a/cmd/rcat/rcat.go +++ b/cmd/rcat/rcat.go @@ -44,11 +44,11 @@ must fit into RAM. The cutoff needs to be small enough to adhere the limits of your remote, please see there. Generally speaking, setting this cutoff too high will decrease your performance. -Use the |--size| flag to preallocate the file in advance at the remote end +Use the ` + "`--size`" + ` flag to preallocate the file in advance at the remote end and actually stream it, even if remote backend doesn't support streaming. -|--size| should be the exact size of the input stream in bytes. If the -size of the stream is different in length to the |--size| passed in +` + "`--size`" + ` should be the exact size of the input stream in bytes. If the +size of the stream is different in length to the ` + "`--size`" + ` passed in then the transfer will likely fail. Note that the upload can also not be retried because the data is diff --git a/cmd/serve/http/data/data.go b/cmd/serve/http/data/data.go index de8362100fd7d..73f6a1ff28b91 100644 --- a/cmd/serve/http/data/data.go +++ b/cmd/serve/http/data/data.go @@ -19,7 +19,7 @@ import ( var Help = ` #### Template ---template allows a user to specify a custom markup template for HTTP +` + "`--template`" + ` allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go index c809d6c33e3be..8712f8ac82108 100644 --- a/cmd/serve/http/http.go +++ b/cmd/serve/http/http.go @@ -51,12 +51,12 @@ var Command = &cobra.Command{ This can be viewed in a web browser or you can make a remote of type http read from it. -You can use the filter flags (e.g. --include, --exclude) to control what +You can use the filter flags (e.g. ` + "`--include`, `--exclude`" + `) to control what is served. -The server will log errors. Use -v to see access logs. +The server will log errors. Use ` + "`-v`" + ` to see access logs. ---bwlimit will be respected for file transfers. Use --stats to +` + "`--bwlimit`" + ` will be respected for file transfers. Use ` + "`--stats`" + ` to control the stats printing. ` + httplib.Help + data.Help + auth.Help + vfs.Help, Run: func(command *cobra.Command, args []string) { diff --git a/cmd/serve/httplib/httplib.go b/cmd/serve/httplib/httplib.go index 4e3efe7d02bac..82a29817f86de 100644 --- a/cmd/serve/httplib/httplib.go +++ b/cmd/serve/httplib/httplib.go @@ -30,30 +30,30 @@ var () var Help = ` ### Server options -Use --addr to specify which IP address and port the server should -listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all -IPs. By default it only listens on localhost. You can use port +Use ` + "`--addr`" + ` to specify which IP address and port the server should +listen on, e.g. ` + "`--addr 1.2.3.4:8000` or `--addr :8080`" + ` to +listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. -If you set --addr to listen on a public or LAN accessible IP address +If you set ` + "`--addr`" + ` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. ---server-read-timeout and --server-write-timeout can be used to +` + "`--server-read-timeout` and `--server-write-timeout`" + ` can be used to control the timeouts on the server. Note that this is the total time for a transfer. ---max-header-bytes controls the maximum number of bytes the server will +` + "`--max-header-bytes`" + ` controls the maximum number of bytes the server will accept in the HTTP header. ---baseurl controls the URL prefix that rclone serves from. By default -rclone will serve from the root. If you used --baseurl "/rclone" then +` + "`--baseurl`" + ` controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used ` + "`--baseurl \"/rclone\"`" + ` then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically -inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", ---baseurl "/rclone" and --baseurl "/rclone/" are all treated +inserts leading and trailing "/" on ` + "`--baseurl`" + `, so ` + "`--baseurl \"rclone\"`" + `, +` + "`--baseurl \"/rclone\"` and `--baseurl \"/rclone/\"`" + ` are all treated identically. ---template allows a user to specify a custom markup template for HTTP +` + "`--template`" + ` allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: @@ -81,9 +81,9 @@ to be used within the template to server pages: By default this will serve files without needing a login. You can either use an htpasswd file which can take lots of users, or -set a single username and password with the --user and --pass flags. +set a single username and password with the ` + "`--user` and `--pass`" + ` flags. -Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is +Use ` + "`--htpasswd /path/to/htpasswd`" + ` to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. @@ -95,18 +95,18 @@ To create an htpasswd file: The password file can be updated while rclone is running. -Use --realm to set the authentication realm. +Use ` + "`--realm`" + ` to set the authentication realm. #### SSL/TLS By default this will serve over HTTP. If you want you can serve over -HTTPS. You will need to supply the --cert and --key flags. If you -wish to do client side certificate validation then you will need to -supply --client-ca also. +HTTPS. You will need to supply the ` + "`--cert` and `--key`" + ` flags. +If you wish to do client side certificate validation then you will need to +supply ` + "`--client-ca`" + ` also. ---cert should be either a PEM encoded certificate or a concatenation -of that with the CA certificate. --key should be the PEM encoded -private key and --client-ca should be the PEM encoded client +` + "`--cert`" + ` should be either a PEM encoded certificate or a concatenation +of that with the CA certificate. ` + "`--key`" + ` should be the PEM encoded +private key and ` + "`--client-ca`" + ` should be the PEM encoded client certificate authority certificate. ` diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go index 65b60dc053d82..a8eb7b483a663 100644 --- a/cmd/serve/restic/restic.go +++ b/cmd/serve/restic/restic.go @@ -59,8 +59,8 @@ backups. The server will log errors. Use -v to see access logs. ---bwlimit will be respected for file transfers. Use --stats to -control the stats printing. +` + "`--bwlimit`" + ` will be respected for file transfers. +Use ` + "`--stats`" + ` to control the stats printing. ### Setting up rclone for use by restic ### @@ -79,11 +79,11 @@ Where you can replace "backup" in the above by whatever path in the remote you wish to use. By default this will serve on "localhost:8080" you can change this -with use of the "--addr" flag. +with use of the ` + "`--addr`" + ` flag. You might wish to start this server on boot. -Adding --cache-objects=false will cause rclone to stop caching objects +Adding ` + "`--cache-objects=false`" + ` will cause rclone to stop caching objects returned from the List call. Caching is normally desirable as it speeds up downloading objects, saves transactions and uses very little memory. @@ -129,7 +129,7 @@ these **must** end with /. Eg #### Private repositories #### -The "--private-repos" flag can be used to limit users to repositories starting +The` + "`--private-repos`" + ` flag can be used to limit users to repositories starting with a path of ` + "`//`" + `. ` + httplib.Help, Run: func(command *cobra.Command, args []string) { diff --git a/cmd/serve/sftp/sftp.go b/cmd/serve/sftp/sftp.go index cecb757077ac6..c77450a5e2a5d 100644 --- a/cmd/serve/sftp/sftp.go +++ b/cmd/serve/sftp/sftp.go @@ -65,17 +65,18 @@ var Command = &cobra.Command{ Long: `Run a SFTP server to serve a remote over SFTP. This can be used with an SFTP client or you can make a remote of type sftp to use with it. -You can use the filter flags (e.g. --include, --exclude) to control what +You can use the filter flags (e.g. ` + "`--include`, `--exclude`" + `) to control what is served. -The server will log errors. Use -v to see access logs. +The server will log errors. Use ` + "`-v`" + ` to see access logs. ---bwlimit will be respected for file transfers. Use --stats to -control the stats printing. +` + "`--bwlimit`" + ` will be respected for file transfers. +Use ` + "`--stats`" + ` to control the stats printing. -You must provide some means of authentication, either with --user/--pass, -an authorized keys file (specify location with --authorized-keys - the -default is the same as ssh), an --auth-proxy, or set the --no-auth flag for no +You must provide some means of authentication, either with +` + "`--user`/`--pass`" + `, an authorized keys file (specify location with +` + "`--authorized-keys`" + ` - the default is the same as ssh), an +` + "`--auth-proxy`" + `, or set the ` + "`--no-auth`" + ` flag for no authentication when logging in. Note that this also implements a small number of shell commands so @@ -83,30 +84,30 @@ that it can provide md5sum/sha1sum/df information for the rclone sftp backend. This means that is can support SHA1SUMs, MD5SUMs and the about command when paired with the rclone sftp backend. -If you don't supply a host --key then rclone will generate rsa, ecdsa +If you don't supply a host ` + "`--key`" + ` then rclone will generate rsa, ecdsa and ed25519 variants, and cache them for later use in rclone's cache -directory (see "rclone help flags cache-dir") in the "serve-sftp" +directory (see ` + "`rclone help flags cache-dir`" + `) in the "serve-sftp" directory. By default the server binds to localhost:2022 - if you want it to be -reachable externally then supply "--addr :2022" for example. +reachable externally then supply ` + "`--addr :2022`" + ` for example. -Note that the default of "--vfs-cache-mode off" is fine for the rclone +Note that the default of ` + "`--vfs-cache-mode off`" + ` is fine for the rclone sftp backend, but it may not be with other SFTP clients. -If --stdio is specified, rclone will serve SFTP over stdio, which can +If ` + "`--stdio`" + ` is specified, rclone will serve SFTP over stdio, which can be used with sshd via ~/.ssh/authorized_keys, for example: restrict,command="rclone serve sftp --stdio ./photos" ssh-rsa ... -On the client you need to set "--transfers 1" when using --stdio. +On the client you need to set ` + "`--transfers 1`" + ` when using ` + "`--stdio`" + `. Otherwise multiple instances of the rclone server are started by OpenSSH which can lead to "corrupted on transfer" errors. This is the case because the client chooses indiscriminately which server to send commands to while the servers all have different views of the state of the filing system. The "restrict" in authorized_keys prevents SHA1SUMs and MD5SUMs from beeing -used. Omitting "restrict" and using --sftp-path-override to enable +used. Omitting "restrict" and using ` + "`--sftp-path-override`" + ` to enable checksumming is possible but less secure and you could use the SFTP server provided by OpenSSH in this case. diff --git a/lib/http/auth/auth.go b/lib/http/auth/auth.go index 222d09d2e6f52..e30ee9383f8cc 100644 --- a/lib/http/auth/auth.go +++ b/lib/http/auth/auth.go @@ -14,9 +14,9 @@ var Help = ` By default this will serve files without needing a login. You can either use an htpasswd file which can take lots of users, or -set a single username and password with the --user and --pass flags. +set a single username and password with the ` + "`--user` and `--pass`" + ` flags. -Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is +Use ` + "`--htpasswd /path/to/htpasswd`" + ` to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. @@ -28,9 +28,9 @@ To create an htpasswd file: The password file can be updated while rclone is running. -Use --realm to set the authentication realm. +Use ` + "`--realm`" + ` to set the authentication realm. -Use --salt to change the password hashing salt from the default. +Use ` + "`--salt`" + ` to change the password hashing salt from the default. ` // CustomAuthFn if used will be used to authenticate user, pass. If an error diff --git a/lib/http/http.go b/lib/http/http.go index d6ff1452aa52b..ac5befd3719f4 100644 --- a/lib/http/http.go +++ b/lib/http/http.go @@ -25,39 +25,39 @@ import ( var Help = ` ### Server options -Use --addr to specify which IP address and port the server should -listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all +Use ` + "`--addr`" + ` to specify which IP address and port the server should +listen on, eg ` + "`--addr 1.2.3.4:8000` or `--addr :8080`" + ` to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. -If you set --addr to listen on a public or LAN accessible IP address +If you set ` + "`--addr`" + ` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. ---server-read-timeout and --server-write-timeout can be used to +` + "`--server-read-timeout` and `--server-write-timeout`" + ` can be used to control the timeouts on the server. Note that this is the total time for a transfer. ---max-header-bytes controls the maximum number of bytes the server will +` + "`--max-header-bytes`" + ` controls the maximum number of bytes the server will accept in the HTTP header. ---baseurl controls the URL prefix that rclone serves from. By default -rclone will serve from the root. If you used --baseurl "/rclone" then +` + "`--baseurl`" + ` controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used ` + "`--baseurl \"/rclone\"`" + ` then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically -inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", ---baseurl "/rclone" and --baseurl "/rclone/" are all treated +inserts leading and trailing "/" on ` + "`--baseurl`" + `, so ` + "`--baseurl \"rclone\"`" + `, +` + "`--baseurl \"/rclone\"` and `--baseurl \"/rclone/\"`" + ` are all treated identically. #### SSL/TLS By default this will serve over http. If you want you can serve over -https. You will need to supply the --cert and --key flags. If you -wish to do client side certificate validation then you will need to -supply --client-ca also. +https. You will need to supply the ` + "`--cert` and `--key`" + ` flags. +If you wish to do client side certificate validation then you will need to +supply ` + "`--client-ca`" + ` also. ---cert should be a either a PEM encoded certificate or a concatenation -of that with the CA certificate. --key should be the PEM encoded -private key and --client-ca should be the PEM encoded client +` + "`--cert`" + ` should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. ` + "`--key`" + ` should be the PEM encoded +private key and ` + "`--client-ca`" + ` should be the PEM encoded client certificate authority certificate. ` From 027746ef6e6fbaf5bee33bdd6481dc9c3e8ffa45 Mon Sep 17 00:00:00 2001 From: Abhiraj Date: Fri, 24 Jun 2022 01:38:09 +0530 Subject: [PATCH 106/560] drive: moved rclone_folder_id to advanced section - fixes #3463 --- backend/drive/drive.go | 1 + docs/content/drive.md | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index b352b9004855e..9eac37a0efab4 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -277,6 +277,7 @@ Leave blank normally. Fill in to access "Computers" folders (see docs), or for rclone to use a non root folder as its starting point. `, + Advanced: true, }, { Name: "service_account_file", Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp, diff --git a/docs/content/drive.md b/docs/content/drive.md index da17114093f03..08d1bb8330cf5 100644 --- a/docs/content/drive.md +++ b/docs/content/drive.md @@ -58,8 +58,6 @@ Choose a number from below, or type in your own value 5 | does not allow any access to read or download file content. \ "drive.metadata.readonly" scope> 1 -ID of the root folder - leave blank normally. Fill in to access "Computers" folders. (see docs). -root_folder_id> Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. service_account_file> Remote config @@ -161,7 +159,7 @@ directories. ### Root folder ID -You can set the `root_folder_id` for rclone. This is the directory +This option has been moved to the advanced section. You can set the `root_folder_id` for rclone. This is the directory (identified by its `Folder ID`) that rclone considers to be the root of your drive. From fdd2f8e6d21d1cf95b33fd2bb09f9c1caef0069d Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 8 Jun 2022 22:54:39 +0200 Subject: [PATCH 107/560] Error strings should not be capitalized Reported by staticcheck 2022.1.2 (v0.3.2) See: staticcheck.io --- backend/azureblob/azureblob.go | 26 +++++++++++++------------- backend/azureblob/imds.go | 4 ++-- backend/cache/cache.go | 2 +- backend/cache/plex.go | 2 +- backend/cache/storage_persistent.go | 14 +++++++------- backend/compress/compress.go | 16 ++++++++-------- backend/crypt/cipher.go | 4 ++-- backend/crypt/crypt.go | 2 +- backend/crypt/pkcs7/pkcs7.go | 10 +++++----- backend/drive/drive.go | 8 ++++---- backend/dropbox/dropbox.go | 4 ++-- backend/fichier/api.go | 4 ++-- backend/fichier/fichier.go | 2 +- backend/ftp/ftp.go | 4 ++-- backend/hdfs/fs.go | 2 +- backend/koofr/koofr.go | 4 ++-- backend/mailru/api/helpers.go | 6 +++--- backend/mailru/mailru.go | 12 ++++++------ backend/qingstor/qingstor.go | 2 +- backend/qingstor/upload.go | 2 +- backend/seafile/seafile.go | 8 ++++---- backend/seafile/webapi.go | 2 +- backend/sftp/sftp.go | 2 +- backend/swift/swift.go | 2 +- backend/uptobox/uptobox.go | 6 +++--- cmd/cmd.go | 2 +- cmd/config/config.go | 2 +- cmd/cryptdecode/cryptdecode.go | 2 +- cmd/lsf/lsf.go | 2 +- cmd/mount/dir.go | 2 +- cmd/mountlib/check_linux.go | 2 +- cmd/mountlib/check_other.go | 4 ++-- cmd/rc/rc.go | 2 +- cmd/serve/ftp/ftp.go | 20 ++++++++++---------- cmd/serve/restic/restic.go | 2 +- cmd/serve/sftp/connection.go | 2 +- cmd/settier/settier.go | 2 +- cmd/tree/tree.go | 6 +++--- fs/accounting/accounting.go | 2 +- fs/config/configfile/configfile.go | 16 ++++++++-------- fs/config/crypt.go | 6 +++--- fs/cutoffmode.go | 4 ++-- fs/dump.go | 2 +- fs/dump_test.go | 4 ++-- fs/filter/filter.go | 4 ++-- fs/filter/filter_test.go | 4 ++-- fs/hash/hash.go | 2 +- fs/log.go | 4 ++-- fs/open_options.go | 10 +++++----- fs/operations/check.go | 8 ++++---- fs/operations/lsjson.go | 2 +- fs/operations/operations.go | 10 +++++----- fs/operations/operations_test.go | 4 ++-- fs/rc/internal.go | 2 +- fs/rc/rcserver/rcserver_test.go | 2 +- fs/rc/webgui/webgui.go | 2 +- fs/sync/sync.go | 2 +- fstest/mockobject/mockobject.go | 2 +- fstest/test_all/clean.go | 2 +- lib/http/http.go | 12 ++++++------ lib/jwtutil/jwtutil.go | 2 +- lib/rest/url.go | 2 +- vfs/file.go | 2 +- vfs/vfscommon/cachemode.go | 4 ++-- vfs/vfsflags/filemode.go | 2 +- 65 files changed, 159 insertions(+), 159 deletions(-) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 58f4552b9bd4f..971bdc1d14fd5 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -539,10 +539,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e err = checkUploadChunkSize(opt.ChunkSize) if err != nil { - return nil, fmt.Errorf("azure: chunk size: %w", err) + return nil, fmt.Errorf("chunk size: %w", err) } if opt.ListChunkSize > maxListChunkSize { - return nil, fmt.Errorf("azure: blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize) + return nil, fmt.Errorf("blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize) } if opt.Endpoint == "" { opt.Endpoint = storageDefaultBaseURL @@ -551,12 +551,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e if opt.AccessTier == "" { opt.AccessTier = string(defaultAccessTier) } else if !validateAccessTier(opt.AccessTier) { - return nil, fmt.Errorf("Azure Blob: Supported access tiers are %s, %s and %s", + return nil, fmt.Errorf("supported access tiers are %s, %s and %s", string(azblob.AccessTierHot), string(azblob.AccessTierCool), string(azblob.AccessTierArchive)) } if !validatePublicAccess((opt.PublicAccess)) { - return nil, fmt.Errorf("Azure Blob: Supported public access level are %s and %s", + return nil, fmt.Errorf("supported public access level are %s and %s", string(azblob.PublicAccessBlob), string(azblob.PublicAccessContainer)) } @@ -598,7 +598,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e case opt.UseEmulator: credential, err := azblob.NewSharedKeyCredential(emulatorAccount, emulatorAccountKey) if err != nil { - return nil, fmt.Errorf("Failed to parse credentials: %w", err) + return nil, fmt.Errorf("failed to parse credentials: %w", err) } u, err = url.Parse(emulatorBlobEndpoint) if err != nil { @@ -644,7 +644,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e }) if err != nil { - return nil, fmt.Errorf("Failed to acquire MSI token: %w", err) + return nil, fmt.Errorf("failed to acquire MSI token: %w", err) } u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint)) @@ -679,7 +679,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e case opt.Account != "" && opt.Key != "": credential, err := azblob.NewSharedKeyCredential(opt.Account, opt.Key) if err != nil { - return nil, fmt.Errorf("Failed to parse credentials: %w", err) + return nil, fmt.Errorf("failed to parse credentials: %w", err) } u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint)) @@ -699,7 +699,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e parts := azblob.NewBlobURLParts(*u) if parts.ContainerName != "" { if f.rootContainer != "" && parts.ContainerName != f.rootContainer { - return nil, errors.New("Container name in SAS URL and container provided in command do not match") + return nil, errors.New("container name in SAS URL and container provided in command do not match") } containerURL := azblob.NewContainerURL(*u, pipeline) f.cntURLcache[parts.ContainerName] = &containerURL @@ -727,7 +727,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e pipe := f.newPipeline(azblob.NewTokenCredential("", tokenRefresher), options) serviceURL = azblob.NewServiceURL(*u, pipe) default: - return nil, errors.New("No authentication method configured") + return nil, errors.New("no authentication method configured") } f.svcURL = &serviceURL @@ -1337,7 +1337,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { } data, err := base64.StdEncoding.DecodeString(o.md5) if err != nil { - return "", fmt.Errorf("Failed to decode Content-MD5: %q: %w", o.md5, err) + return "", fmt.Errorf("failed to decode Content-MD5: %q: %w", o.md5, err) } return hex.EncodeToString(data), nil } @@ -1527,7 +1527,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read var offset int64 var count int64 if o.AccessTier() == azblob.AccessTierArchive { - return nil, fmt.Errorf("Blob in archive tier, you need to set tier to hot or cool first") + return nil, fmt.Errorf("blob in archive tier, you need to set tier to hot or cool first") } fs.FixRangeOption(options, o.size) for _, option := range options { @@ -1752,7 +1752,7 @@ func (o *Object) AccessTier() azblob.AccessTierType { // SetTier performs changing object tier func (o *Object) SetTier(tier string) error { if !validateAccessTier(tier) { - return fmt.Errorf("Tier %s not supported by Azure Blob Storage", tier) + return fmt.Errorf("tier %s not supported by Azure Blob Storage", tier) } // Check if current tier already matches with desired tier @@ -1768,7 +1768,7 @@ func (o *Object) SetTier(tier string) error { }) if err != nil { - return fmt.Errorf("Failed to set Blob Tier: %w", err) + return fmt.Errorf("failed to set Blob Tier: %w", err) } // Set access tier on local object also, this typically diff --git a/backend/azureblob/imds.go b/backend/azureblob/imds.go index b23e91d626f3e..84c6379e6d821 100644 --- a/backend/azureblob/imds.go +++ b/backend/azureblob/imds.go @@ -119,7 +119,7 @@ func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) { b, err := ioutil.ReadAll(resp.Body) if err != nil { - return result, fmt.Errorf("Couldn't read IMDS response: %w", err) + return result, fmt.Errorf("couldn't read IMDS response: %w", err) } // Remove BOM, if any. azcopy does this so I'm following along. b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf")) @@ -130,7 +130,7 @@ func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) { // storage API call. err = json.Unmarshal(b, &result) if err != nil { - return result, fmt.Errorf("Couldn't unmarshal IMDS response: %w", err) + return result, fmt.Errorf("couldn't unmarshal IMDS response: %w", err) } return result, nil diff --git a/backend/cache/cache.go b/backend/cache/cache.go index fc038f6adc5b4..1c75aea4a8529 100644 --- a/backend/cache/cache.go +++ b/backend/cache/cache.go @@ -1128,7 +1128,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( case fs.Directory: _ = f.cache.AddDir(DirectoryFromOriginal(ctx, f, o)) default: - return fmt.Errorf("Unknown object type %T", entry) + return fmt.Errorf("unknown object type %T", entry) } } diff --git a/backend/cache/plex.go b/backend/cache/plex.go index e3a333c73a1b2..09ea472336886 100644 --- a/backend/cache/plex.go +++ b/backend/cache/plex.go @@ -213,7 +213,7 @@ func (p *plexConnector) authenticate() error { var data map[string]interface{} err = json.NewDecoder(resp.Body).Decode(&data) if err != nil { - return fmt.Errorf("failed to obtain token: %v", err) + return fmt.Errorf("failed to obtain token: %w", err) } tokenGen, ok := get(data, "user", "authToken") if !ok { diff --git a/backend/cache/storage_persistent.go b/backend/cache/storage_persistent.go index 0de9a39633396..eb36a08063e44 100644 --- a/backend/cache/storage_persistent.go +++ b/backend/cache/storage_persistent.go @@ -250,7 +250,7 @@ func (b *Persistent) GetDirEntries(cachedDir *Directory) (fs.DirEntries, error) if val != nil { err := json.Unmarshal(val, cachedDir) if err != nil { - return fmt.Errorf("error during unmarshalling obj: %v", err) + return fmt.Errorf("error during unmarshalling obj: %w", err) } } else { return fmt.Errorf("missing cached dir: %v", cachedDir) @@ -551,7 +551,7 @@ func (b *Persistent) CleanChunksBySize(maxSize int64) { err := b.db.Update(func(tx *bolt.Tx) error { dataTsBucket := tx.Bucket([]byte(DataTsBucket)) if dataTsBucket == nil { - return fmt.Errorf("Couldn't open (%v) bucket", DataTsBucket) + return fmt.Errorf("couldn't open (%v) bucket", DataTsBucket) } // iterate through ts c := dataTsBucket.Cursor() @@ -901,16 +901,16 @@ func (b *Persistent) rollbackPendingUpload(remote string) error { v := bucket.Get([]byte(remote)) err = json.Unmarshal(v, tempObj) if err != nil { - return fmt.Errorf("pending upload (%v) not found %v", remote, err) + return fmt.Errorf("pending upload (%v) not found: %w", remote, err) } tempObj.Started = false v2, err := json.Marshal(tempObj) if err != nil { - return fmt.Errorf("pending upload not updated %v", err) + return fmt.Errorf("pending upload not updated: %w", err) } err = bucket.Put([]byte(tempObj.DestPath), v2) if err != nil { - return fmt.Errorf("pending upload not updated %v", err) + return fmt.Errorf("pending upload not updated: %w", err) } return nil }) @@ -966,11 +966,11 @@ func (b *Persistent) updatePendingUpload(remote string, fn func(item *tempUpload } v2, err := json.Marshal(tempObj) if err != nil { - return fmt.Errorf("pending upload not updated %v", err) + return fmt.Errorf("pending upload not updated: %w", err) } err = bucket.Put([]byte(tempObj.DestPath), v2) if err != nil { - return fmt.Errorf("pending upload not updated %v", err) + return fmt.Errorf("pending upload not updated: %w", err) } return nil diff --git a/backend/compress/compress.go b/backend/compress/compress.go index 6d3397dfb0019..dcc54a11c0e9d 100644 --- a/backend/compress/compress.go +++ b/backend/compress/compress.go @@ -222,7 +222,7 @@ func processFileName(compressedFileName string) (origFileName string, extension // Separate the filename and size from the extension extensionPos := strings.LastIndex(compressedFileName, ".") if extensionPos == -1 { - return "", "", 0, errors.New("File name has no extension") + return "", "", 0, errors.New("file name has no extension") } extension = compressedFileName[extensionPos:] nameWithSize := compressedFileName[:extensionPos] @@ -231,11 +231,11 @@ func processFileName(compressedFileName string) (origFileName string, extension } match := nameRegexp.FindStringSubmatch(nameWithSize) if match == nil || len(match) != 3 { - return "", "", 0, errors.New("Invalid filename") + return "", "", 0, errors.New("invalid filename") } size, err := base64ToInt64(match[2]) if err != nil { - return "", "", 0, errors.New("Could not decode size") + return "", "", 0, errors.New("could not decode size") } return match[1], gzFileExt, size, nil } @@ -304,7 +304,7 @@ func (f *Fs) processEntries(entries fs.DirEntries) (newEntries fs.DirEntries, er case fs.Directory: f.addDir(&newEntries, x) default: - return nil, fmt.Errorf("Unknown object type %T", entry) + return nil, fmt.Errorf("unknown object type %T", entry) } } return newEntries, nil @@ -466,10 +466,10 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod _ = os.Remove(tempFile.Name()) }() if err != nil { - return nil, fmt.Errorf("Failed to create temporary local FS to spool file: %w", err) + return nil, fmt.Errorf("failed to create temporary local FS to spool file: %w", err) } if _, err = io.Copy(tempFile, in); err != nil { - return nil, fmt.Errorf("Failed to write temporary local file: %w", err) + return nil, fmt.Errorf("failed to write temporary local file: %w", err) } if _, err = tempFile.Seek(0, 0); err != nil { return nil, err @@ -720,7 +720,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt if found && (oldObj.(*Object).meta.Mode != Uncompressed || compressible) { err = oldObj.(*Object).Object.Remove(ctx) if err != nil { - return nil, fmt.Errorf("Could remove original object: %w", err) + return nil, fmt.Errorf("couldn't remove original object: %w", err) } } @@ -729,7 +729,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt if compressible { wrapObj, err := operations.Move(ctx, f.Fs, nil, f.dataName(src.Remote(), newObj.size, compressible), newObj.Object) if err != nil { - return nil, fmt.Errorf("Couldn't rename streamed Object.: %w", err) + return nil, fmt.Errorf("couldn't rename streamed object: %w", err) } newObj.Object = wrapObj } diff --git a/backend/crypt/cipher.go b/backend/crypt/cipher.go index 4a3556295b69e..1b265ad19e918 100644 --- a/backend/crypt/cipher.go +++ b/backend/crypt/cipher.go @@ -96,7 +96,7 @@ func NewNameEncryptionMode(s string) (mode NameEncryptionMode, err error) { case "obfuscate": mode = NameEncryptionObfuscated default: - err = fmt.Errorf("Unknown file name encryption mode %q", s) + err = fmt.Errorf("unknown file name encryption mode %q", s) } return mode, err } @@ -162,7 +162,7 @@ func NewNameEncoding(s string) (enc fileNameEncoding, err error) { case "base32768": enc = base32768.SafeEncoding default: - err = fmt.Errorf("Unknown file name encoding mode %q", s) + err = fmt.Errorf("unknown file name encoding mode %q", s) } return enc, err } diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go index 11dd55dad9954..33e57cdb9b513 100644 --- a/backend/crypt/crypt.go +++ b/backend/crypt/crypt.go @@ -328,7 +328,7 @@ func (f *Fs) encryptEntries(ctx context.Context, entries fs.DirEntries) (newEntr case fs.Directory: f.addDir(ctx, &newEntries, x) default: - return nil, fmt.Errorf("Unknown object type %T", entry) + return nil, fmt.Errorf("unknown object type %T", entry) } } return newEntries, nil diff --git a/backend/crypt/pkcs7/pkcs7.go b/backend/crypt/pkcs7/pkcs7.go index db604ae4b6a06..d2a547042ad5e 100644 --- a/backend/crypt/pkcs7/pkcs7.go +++ b/backend/crypt/pkcs7/pkcs7.go @@ -8,11 +8,11 @@ import "errors" // Errors Unpad can return var ( - ErrorPaddingNotFound = errors.New("Bad PKCS#7 padding - not padded") - ErrorPaddingNotAMultiple = errors.New("Bad PKCS#7 padding - not a multiple of blocksize") - ErrorPaddingTooLong = errors.New("Bad PKCS#7 padding - too long") - ErrorPaddingTooShort = errors.New("Bad PKCS#7 padding - too short") - ErrorPaddingNotAllTheSame = errors.New("Bad PKCS#7 padding - not all the same") + ErrorPaddingNotFound = errors.New("bad PKCS#7 padding - not padded") + ErrorPaddingNotAMultiple = errors.New("bad PKCS#7 padding - not a multiple of blocksize") + ErrorPaddingTooLong = errors.New("bad PKCS#7 padding - too long") + ErrorPaddingTooShort = errors.New("bad PKCS#7 padding - too short") + ErrorPaddingNotAllTheSame = errors.New("bad PKCS#7 padding - not all the same") ) // Pad buf using PKCS#7 to a multiple of n. diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 9eac37a0efab4..918faddf95b4e 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -2218,10 +2218,10 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, exportExt, _, _ = f.findExportFormatByMimeType(ctx, importMimeType) if exportExt == "" { - return nil, fmt.Errorf("No export format found for %q", importMimeType) + return nil, fmt.Errorf("no export format found for %q", importMimeType) } if exportExt != srcExt && !f.opt.AllowImportNameChange { - return nil, fmt.Errorf("Can't convert %q to a document with a different export filetype (%q)", srcExt, exportExt) + return nil, fmt.Errorf("can't convert %q to a document with a different export filetype (%q)", srcExt, exportExt) } } } @@ -2526,7 +2526,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // result of List() func (f *Fs) Purge(ctx context.Context, dir string) error { if f.opt.TrashedOnly { - return errors.New("Can't purge with --drive-trashed-only. Use delete if you want to selectively delete files") + return errors.New("can't purge with --drive-trashed-only, use delete if you want to selectively delete files") } return f.purgeCheck(ctx, dir, false) } @@ -3715,7 +3715,7 @@ func (o *baseObject) open(ctx context.Context, url string, options ...fs.OpenOpt url += "acknowledgeAbuse=true" _, res, err = o.httpResponse(ctx, url, "GET", options) } else { - err = fmt.Errorf("Use the --drive-acknowledge-abuse flag to download this file: %w", err) + err = fmt.Errorf("use the --drive-acknowledge-abuse flag to download this file: %w", err) } } if err != nil { diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 69d0b4e7d326c..bcee49821d9af 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -1199,7 +1199,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, return } if len(listRes.Links) == 0 { - err = errors.New("Dropbox says the sharing link already exists, but list came back empty") + err = errors.New("sharing link already exists, but list came back empty") return } linkRes = listRes.Links[0] @@ -1211,7 +1211,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, case *sharing.FolderLinkMetadata: link = res.Url default: - err = fmt.Errorf("Don't know how to extract link, response has unknown format: %T", res) + err = fmt.Errorf("don't know how to extract link, response has unknown format: %T", res) } } return diff --git a/backend/fichier/api.go b/backend/fichier/api.go index 15c916c736314..1ee812c47de5d 100644 --- a/backend/fichier/api.go +++ b/backend/fichier/api.go @@ -487,7 +487,7 @@ func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName, fileName = f.opt.Enc.FromStandardName(fileName) if len(uploadID) > 10 || !isAlphaNumeric(uploadID) { - return nil, errors.New("Invalid UploadID") + return nil, errors.New("invalid UploadID") } opts := rest.Opts{ @@ -529,7 +529,7 @@ func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (re // fs.Debugf(f, "Ending File Upload `%s`", uploadID) if len(uploadID) > 10 || !isAlphaNumeric(uploadID) { - return nil, errors.New("Invalid UploadID") + return nil, errors.New("invalid UploadID") } opts := rest.Opts{ diff --git a/backend/fichier/fichier.go b/backend/fichier/fichier.go index 8548d00c2b7d3..454e391ff61ca 100644 --- a/backend/fichier/fichier.go +++ b/backend/fichier/fichier.go @@ -294,7 +294,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { path, ok := f.dirCache.GetInv(directoryID) if !ok { - return nil, errors.New("Cannot find dir in dircache") + return nil, errors.New("cannot find dir in dircache") } return f.newObjectFromFile(ctx, path, file), nil diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index c078f78b095f0..d92678f08a71b 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -487,7 +487,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs protocol = "ftps://" } if opt.TLS && opt.ExplicitTLS { - return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config") + return nil, errors.New("implicit TLS and explicit TLS are mutually incompatible, please revise your config") } var tlsConfig *tls.Config if opt.TLS || opt.ExplicitTLS { @@ -718,7 +718,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e case <-timer.C: // if timer fired assume no error but connection dead fs.Errorf(f, "Timeout when waiting for List") - return nil, errors.New("Timeout when waiting for List") + return nil, errors.New("timeout when waiting for List") } // Annoyingly FTP returns success for a directory which diff --git a/backend/hdfs/fs.go b/backend/hdfs/fs.go index 8de9ddbacf03d..a68372db69aaf 100644 --- a/backend/hdfs/fs.go +++ b/backend/hdfs/fs.go @@ -92,7 +92,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e if opt.ServicePrincipalName != "" { options.KerberosClient, err = getKerberosClient() if err != nil { - return nil, fmt.Errorf("Problem with kerberos authentication: %s", err) + return nil, fmt.Errorf("problem with kerberos authentication: %w", err) } options.KerberosServicePrincipleName = opt.ServicePrincipalName diff --git a/backend/koofr/koofr.go b/backend/koofr/koofr.go index f77ae019ff9df..f893b898b4896 100644 --- a/backend/koofr/koofr.go +++ b/backend/koofr/koofr.go @@ -351,9 +351,9 @@ func NewFsFromOptions(ctx context.Context, name, root string, opt *Options) (ff } if f.mountID == "" { if opt.MountID == "" { - return nil, errors.New("Failed to find primary mount") + return nil, errors.New("failed to find primary mount") } - return nil, errors.New("Failed to find mount " + opt.MountID) + return nil, errors.New("failed to find mount " + opt.MountID) } rootFile, err := f.client.FilesInfo(f.mountID, f.opt.Enc.FromStandardPath("/"+f.root)) if err == nil && rootFile.Type != "dir" { diff --git a/backend/mailru/api/helpers.go b/backend/mailru/api/helpers.go index 1b43eb182a8f2..d7164185fe2dd 100644 --- a/backend/mailru/api/helpers.go +++ b/backend/mailru/api/helpers.go @@ -16,9 +16,9 @@ import ( // protocol errors var ( - ErrorPrematureEOF = errors.New("Premature EOF") - ErrorInvalidLength = errors.New("Invalid length") - ErrorZeroTerminate = errors.New("String must end with zero") + ErrorPrematureEOF = errors.New("premature EOF") + ErrorInvalidLength = errors.New("invalid length") + ErrorZeroTerminate = errors.New("string must end with zero") ) // BinWriter is a binary protocol writer diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go index 24693c67cfcb8..a9a9bab2dc38d 100644 --- a/backend/mailru/mailru.go +++ b/backend/mailru/mailru.go @@ -435,10 +435,10 @@ func (f *Fs) authorize(ctx context.Context, force bool) (err error) { t, err = oauthConfig.PasswordCredentialsToken(ctx, f.opt.Username, f.opt.Password) } if err == nil && !tokenIsValid(t) { - err = errors.New("Invalid token") + err = errors.New("invalid token") } if err != nil { - return fmt.Errorf("Failed to authorize: %w", err) + return fmt.Errorf("failed to authorize: %w", err) } if err = oauthutil.PutToken(f.name, f.m, t, false); err != nil { @@ -580,7 +580,7 @@ func readBodyWord(res *http.Response) (word string, err error) { word = strings.Split(line, " ")[0] } if word == "" { - return "", errors.New("Empty reply from dispatcher") + return "", errors.New("empty reply from dispatcher") } return word, nil } @@ -1684,7 +1684,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op spoolFile, mrHash, err := makeTempFile(ctx, tmpFs, wrapIn, src) if err != nil { - return fmt.Errorf("Failed to create spool file: %w", err) + return fmt.Errorf("failed to create spool file: %w", err) } if o.putByHash(ctx, mrHash, src, "spool") { // If put by hash is successful, ignore transitive error @@ -1966,7 +1966,7 @@ func (o *Object) readMetaData(ctx context.Context, force bool) error { return fs.ErrorIsDir } if newObj.remote != o.remote { - return fmt.Errorf("File %q path has changed to %q", o.remote, newObj.remote) + return fmt.Errorf("file %q path has changed to %q", o.remote, newObj.remote) } o.hasMetaData = true o.size = newObj.size @@ -2318,7 +2318,7 @@ func (p *serverPool) Dispatch(ctx context.Context, current string) (string, erro }) if err != nil || url == "" { closeBody(res) - return "", fmt.Errorf("Failed to request file server: %w", err) + return "", fmt.Errorf("failed to request file server: %w", err) } p.addServer(url, now) diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go index 5f5134099be09..9e9b0c6cc9b0a 100644 --- a/backend/qingstor/qingstor.go +++ b/backend/qingstor/qingstor.go @@ -253,7 +253,7 @@ func qsServiceConnection(ctx context.Context, opt *Options) (*qs.Service, error) _protocol, _host, _port, err := qsParseEndpoint(endpoint) if err != nil { - return nil, fmt.Errorf("The endpoint \"%s\" format error", endpoint) + return nil, fmt.Errorf("the endpoint \"%s\" format error", endpoint) } if _protocol != "" { diff --git a/backend/qingstor/upload.go b/backend/qingstor/upload.go index d9a81801950d4..797e718f89b79 100644 --- a/backend/qingstor/upload.go +++ b/backend/qingstor/upload.go @@ -184,7 +184,7 @@ func (u *uploader) upload() error { fs.Debugf(u, "Uploading as single part object to QingStor") return u.singlePartUpload(reader, u.readerPos) } else if err != nil { - return fmt.Errorf("read upload data failed: %s", err) + return fmt.Errorf("read upload data failed: %w", err) } fs.Debugf(u, "Uploading as multi-part object to QingStor") diff --git a/backend/seafile/seafile.go b/backend/seafile/seafile.go index 15a7086411484..ffae61a5ae1dc 100644 --- a/backend/seafile/seafile.go +++ b/backend/seafile/seafile.go @@ -886,7 +886,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string // 1- rename source err = srcFs.renameDir(ctx, srcLibraryID, srcPath, tempName) if err != nil { - return fmt.Errorf("Cannot rename source directory to a temporary name: %w", err) + return fmt.Errorf("cannot rename source directory to a temporary name: %w", err) } // 2- move source to destination @@ -900,7 +900,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string // 3- rename destination back to source name err = f.renameDir(ctx, dstLibraryID, path.Join(dstDir, tempName), dstName) if err != nil { - return fmt.Errorf("Cannot rename temporary directory to destination name: %w", err) + return fmt.Errorf("cannot rename temporary directory to destination name: %w", err) } return nil @@ -923,7 +923,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // CleanUp the trash in the Fs func (f *Fs) CleanUp(ctx context.Context) error { if f.libraryName == "" { - return errors.New("Cannot clean up at the root of the seafile server: please select a library to clean up") + return errors.New("cannot clean up at the root of the seafile server, please select a library to clean up") } libraryID, err := f.getLibraryID(ctx, f.libraryName) if err != nil { @@ -972,7 +972,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, libraryName, filePath := f.splitPath(remote) if libraryName == "" { // We cannot share the whole seafile server, we need at least a library - return "", errors.New("Cannot share the root of the seafile server. Please select a library to share") + return "", errors.New("cannot share the root of the seafile server, please select a library to share") } libraryID, err := f.getLibraryID(ctx, libraryName) if err != nil { diff --git a/backend/seafile/webapi.go b/backend/seafile/webapi.go index 29d2f8645be1b..c1eb33da0a934 100644 --- a/backend/seafile/webapi.go +++ b/backend/seafile/webapi.go @@ -26,7 +26,7 @@ const ( // Errors specific to seafile fs var ( - ErrorInternalDuringUpload = errors.New("Internal server error during file upload") + ErrorInternalDuringUpload = errors.New("internal server error during file upload") ) // ==================== Seafile API ==================== diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 74af5fa4104a6..bb5add94ddc7c 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -515,7 +515,7 @@ func (f *Fs) setEnv(s *ssh.Session) error { // fs.Debugf(f, "Setting env %q = %q", env[:equal], env[equal+1:]) err := s.Setenv(env[:equal], env[equal+1:]) if err != nil { - return fmt.Errorf("Failed to set env var %q: %w", env[:equal], err) + return fmt.Errorf("failed to set env var %q: %w", env[:equal], err) } } return nil diff --git a/backend/swift/swift.go b/backend/swift/swift.go index 074e2daf7f9c6..e4bae345e8be6 100644 --- a/backend/swift/swift.go +++ b/backend/swift/swift.go @@ -1303,7 +1303,7 @@ func (o *Object) getSegmentsDlo(ctx context.Context) (segmentsContainer string, } delimiter := strings.Index(dirManifest, "/") if len(dirManifest) == 0 || delimiter < 0 { - err = errors.New("Missing or wrong structure of manifest of Dynamic large object") + err = errors.New("missing or wrong structure of manifest of Dynamic large object") return } return dirManifest[:delimiter], dirManifest[delimiter+1:], nil diff --git a/backend/uptobox/uptobox.go b/backend/uptobox/uptobox.go index 56c4f80a1ec4d..e232ac83cd460 100644 --- a/backend/uptobox/uptobox.go +++ b/backend/uptobox/uptobox.go @@ -474,7 +474,7 @@ func (f *Fs) updateFileInformation(ctx context.Context, update *api.UpdateFileIn func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) { if size > int64(200e9) { // max size 200GB - return nil, errors.New("File too big, cant upload") + return nil, errors.New("file too big, can't upload") } else if size == 0 { return nil, fs.ErrorCantUploadEmptyFiles } @@ -497,7 +497,7 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size return nil, err } if info.StatusCode != 0 { - return nil, fmt.Errorf("putUnchecked: api error: %d - %s", info.StatusCode, info.Message) + return nil, fmt.Errorf("putUnchecked api error: %d - %s", info.StatusCode, info.Message) } // we need to have a safe name for the upload to work tmpName := "rcloneTemp" + random.String(8) @@ -506,7 +506,7 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size return nil, err } if len(upload.Files) != 1 { - return nil, errors.New("Upload: unexpected response") + return nil, errors.New("upload unexpected response") } match := f.IDRegexp.FindStringSubmatch(upload.Files[0].URL) diff --git a/cmd/cmd.go b/cmd/cmd.go index 82ae1ad838712..5dd120be7fd6a 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -117,7 +117,7 @@ func newFsFileAddFilter(remote string) (fs.Fs, string) { f, fileName := NewFsFile(remote) if fileName != "" { if !fi.InActive() { - err := fmt.Errorf("Can't limit to single files when using filters: %v", remote) + err := fmt.Errorf("can't limit to single files when using filters: %v", remote) err = fs.CountError(err) log.Fatalf(err.Error()) } diff --git a/cmd/config/config.go b/cmd/config/config.go index 96734e5528686..ed57003278043 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -402,7 +402,7 @@ To reconnect use "rclone config reconnect". } err := doDisconnect(context.Background()) if err != nil { - return fmt.Errorf("Disconnect call failed: %w", err) + return fmt.Errorf("disconnect call failed: %w", err) } return nil }, diff --git a/cmd/cryptdecode/cryptdecode.go b/cmd/cryptdecode/cryptdecode.go index 2f938cbfa7a0a..d36c70ca47f52 100644 --- a/cmd/cryptdecode/cryptdecode.go +++ b/cmd/cryptdecode/cryptdecode.go @@ -48,7 +48,7 @@ See the documentation on the [crypt](/crypt/) overlay for more info. return err } if fsInfo.Name != "crypt" { - return errors.New("The remote needs to be of type \"crypt\"") + return errors.New("the remote needs to be of type \"crypt\"") } cipher, err := crypt.NewCipher(config) if err != nil { diff --git a/cmd/lsf/lsf.go b/cmd/lsf/lsf.go index 812edf8eab748..732d2b5e8980a 100644 --- a/cmd/lsf/lsf.go +++ b/cmd/lsf/lsf.go @@ -199,7 +199,7 @@ func Lsf(ctx context.Context, fsrc fs.Fs, out io.Writer) error { case 'T': list.AddTier() default: - return fmt.Errorf("Unknown format character %q", char) + return fmt.Errorf("unknown format character %q", char) } } diff --git a/cmd/mount/dir.go b/cmd/mount/dir.go index 47ec4bf2da5d2..b82b521796f13 100644 --- a/cmd/mount/dir.go +++ b/cmd/mount/dir.go @@ -199,7 +199,7 @@ func (d *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fusefs defer log.Trace(d, "oldName=%q, newName=%q, newDir=%+v", req.OldName, req.NewName, newDir)("err=%v", &err) destDir, ok := newDir.(*Dir) if !ok { - return fmt.Errorf("Unknown Dir type %T", newDir) + return fmt.Errorf("unknown Dir type %T", newDir) } err = d.Dir.Rename(req.OldName, req.NewName, destDir.Dir) diff --git a/cmd/mountlib/check_linux.go b/cmd/mountlib/check_linux.go index 3cdb346905d8a..1f241af8723bc 100644 --- a/cmd/mountlib/check_linux.go +++ b/cmd/mountlib/check_linux.go @@ -22,7 +22,7 @@ const ( // On Linux we use the OS-specific /proc/mount API so the check won't access the path. // Directories marked as "mounted" by autofs are considered not mounted. func CheckMountEmpty(mountpoint string) error { - const msg = "Directory already mounted, use --allow-non-empty to mount anyway: %s" + const msg = "directory already mounted, use --allow-non-empty to mount anyway: %s" mountpointAbs, err := filepath.Abs(mountpoint) if err != nil { diff --git a/cmd/mountlib/check_other.go b/cmd/mountlib/check_other.go index 0b7eeede71eb8..c8876632cb3d2 100644 --- a/cmd/mountlib/check_other.go +++ b/cmd/mountlib/check_other.go @@ -17,7 +17,7 @@ import ( func CheckMountEmpty(mountpoint string) error { fp, err := os.Open(mountpoint) if err != nil { - return fmt.Errorf("Can not open: %s: %w", mountpoint, err) + return fmt.Errorf("cannot open: %s: %w", mountpoint, err) } defer fs.CheckClose(fp, &err) @@ -26,7 +26,7 @@ func CheckMountEmpty(mountpoint string) error { return nil } - const msg = "Directory is not empty, use --allow-non-empty to mount anyway: %s" + const msg = "directory is not empty, use --allow-non-empty to mount anyway: %s" if err == nil { return fmt.Errorf(msg, mountpoint) } diff --git a/cmd/rc/rc.go b/cmd/rc/rc.go index 6b123f97e3ffb..53e8697d80095 100644 --- a/cmd/rc/rc.go +++ b/cmd/rc/rc.go @@ -211,7 +211,7 @@ func doCall(ctx context.Context, path string, in rc.Params) (out rc.Params, err bodyString = err.Error() } bodyString = strings.TrimSpace(bodyString) - return nil, fmt.Errorf("Failed to read rc response: %s: %s", resp.Status, bodyString) + return nil, fmt.Errorf("failed to read rc response: %s: %s", resp.Status, bodyString) } // Parse output diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go index f8ba817259cad..05f7b092405d9 100644 --- a/cmd/serve/ftp/ftp.go +++ b/cmd/serve/ftp/ftp.go @@ -135,11 +135,11 @@ var passivePortsRe = regexp.MustCompile(`^\s*\d+\s*-\s*\d+\s*$`) func newServer(ctx context.Context, f fs.Fs, opt *Options) (*server, error) { host, port, err := net.SplitHostPort(opt.ListenAddr) if err != nil { - return nil, errors.New("Failed to parse host:port") + return nil, errors.New("failed to parse host:port") } portNum, err := strconv.Atoi(port) if err != nil { - return nil, errors.New("Failed to parse host:port") + return nil, errors.New("failed to parse host:port") } s := &server{ @@ -284,7 +284,7 @@ func (d *Driver) ChangeDir(path string) (err error) { return err } if !n.IsDir() { - return errors.New("Not a directory") + return errors.New("not a directory") } return nil } @@ -296,12 +296,12 @@ func (d *Driver) ListDir(path string, callback func(ftp.FileInfo) error) (err er defer log.Trace(path, "")("err = %v", &err) node, err := d.vfs.Stat(path) if err == vfs.ENOENT { - return errors.New("Directory not found") + return errors.New("directory not found") } else if err != nil { return err } if !node.IsDir() { - return errors.New("Not a directory") + return errors.New("not a directory") } dir := node.(*vfs.Dir) @@ -335,7 +335,7 @@ func (d *Driver) DeleteDir(path string) (err error) { return err } if !node.IsDir() { - return errors.New("Not a directory") + return errors.New("not a directory") } err = node.Remove() if err != nil { @@ -354,7 +354,7 @@ func (d *Driver) DeleteFile(path string) (err error) { return err } if !node.IsFile() { - return errors.New("Not a file") + return errors.New("not a file") } err = node.Remove() if err != nil { @@ -392,12 +392,12 @@ func (d *Driver) GetFile(path string, offset int64) (size int64, fr io.ReadClose node, err := d.vfs.Stat(path) if err == vfs.ENOENT { fs.Infof(path, "File not found") - return 0, nil, errors.New("File not found") + return 0, nil, errors.New("file not found") } else if err != nil { return 0, nil, err } if !node.IsFile() { - return 0, nil, errors.New("Not a file") + return 0, nil, errors.New("not a file") } handle, err := node.Open(os.O_RDONLY) @@ -426,7 +426,7 @@ func (d *Driver) PutFile(path string, data io.Reader, appendData bool) (n int64, if err == nil { isExist = true if node.IsDir() { - return 0, errors.New("A dir has the same name") + return 0, errors.New("a dir has the same name") } } else { if os.IsNotExist(err) { diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go index a8eb7b483a663..7139b084b8e5d 100644 --- a/cmd/serve/restic/restic.go +++ b/cmd/serve/restic/restic.go @@ -139,7 +139,7 @@ with a path of ` + "`//`" + `. s := NewServer(f, &httpflags.Opt) if stdio { if terminal.IsTerminal(int(os.Stdout.Fd())) { - return errors.New("Refusing to run HTTP2 server directly on a terminal, please let restic start rclone") + return errors.New("refusing to run HTTP2 server directly on a terminal, please let restic start rclone") } conn := &StdioConn{ diff --git a/cmd/serve/sftp/connection.go b/cmd/serve/sftp/connection.go index 693b5fbb06a8c..d8be2bce5831b 100644 --- a/cmd/serve/sftp/connection.go +++ b/cmd/serve/sftp/connection.go @@ -74,7 +74,7 @@ func (c *conn) execCommand(ctx context.Context, out io.Writer, command string) ( } usage, err := about(ctx) if err != nil { - return fmt.Errorf("About failed: %w", err) + return fmt.Errorf("about failed: %w", err) } total, used, free := int64(-1), int64(-1), int64(-1) if usage.Total != nil { diff --git a/cmd/settier/settier.go b/cmd/settier/settier.go index 97953727a09be..6be1c1c6a2951 100644 --- a/cmd/settier/settier.go +++ b/cmd/settier/settier.go @@ -47,7 +47,7 @@ Or just provide remote directory and all files in directory will be tiered cmd.Run(false, false, command, func() error { isSupported := fsrc.Features().SetTier if !isSupported { - return fmt.Errorf("Remote %s does not support settier", fsrc.Name()) + return fmt.Errorf("remote %s does not support settier", fsrc.Name()) } return operations.SetTier(context.Background(), fsrc, tier) diff --git a/cmd/tree/tree.go b/cmd/tree/tree.go index 4bd8994626556..2528d9dc2a42d 100644 --- a/cmd/tree/tree.go +++ b/cmd/tree/tree.go @@ -102,7 +102,7 @@ For a more interactive navigation of the remote see the var err error outFile, err = os.Create(outFileName) if err != nil { - return fmt.Errorf("failed to create output file: %v", err) + return fmt.Errorf("failed to create output file: %w", err) } } opts.VerSort = opts.VerSort || sort == "version" @@ -209,7 +209,7 @@ func (dirs Fs) Stat(filePath string) (fi os.FileInfo, err error) { } _, entry := dirtree.DirTree(dirs).Find(filePath) if entry == nil { - return nil, fmt.Errorf("Couldn't find %q in directory cache", filePath) + return nil, fmt.Errorf("couldn't find %q in directory cache", filePath) } return &FileInfo{entry}, nil } @@ -221,7 +221,7 @@ func (dirs Fs) ReadDir(dir string) (names []string, err error) { dir = strings.TrimLeft(dir, "/") entries, ok := dirs[dir] if !ok { - return nil, fmt.Errorf("Couldn't find directory %q", dir) + return nil, fmt.Errorf("couldn't find directory %q", dir) } for _, entry := range entries { names = append(names, path.Base(entry.Remote())) diff --git a/fs/accounting/accounting.go b/fs/accounting/accounting.go index 53d6549fb65d0..a019232fb27e8 100644 --- a/fs/accounting/accounting.go +++ b/fs/accounting/accounting.go @@ -19,7 +19,7 @@ import ( // ErrorMaxTransferLimitReached defines error when transfer limit is reached. // Used for checking on exit and matching to correct exit code. -var ErrorMaxTransferLimitReached = errors.New("Max transfer limit reached as set by --max-transfer") +var ErrorMaxTransferLimitReached = errors.New("max transfer limit reached as set by --max-transfer") // ErrorMaxTransferLimitReachedFatal is returned from Read when the max // transfer limit is reached. diff --git a/fs/config/configfile/configfile.go b/fs/config/configfile/configfile.go index df01f25a5499e..e9c498995f3a3 100644 --- a/fs/config/configfile/configfile.go +++ b/fs/config/configfile/configfile.go @@ -106,7 +106,7 @@ func (s *Storage) Save() error { configPath := config.GetConfigPath() if configPath == "" { - return fmt.Errorf("Failed to save config file: Path is empty") + return fmt.Errorf("failed to save config file, path is empty") } dir, name := filepath.Split(configPath) @@ -116,18 +116,18 @@ func (s *Storage) Save() error { } f, err := ioutil.TempFile(dir, name) if err != nil { - return fmt.Errorf("Failed to create temp file for new config: %v", err) + return fmt.Errorf("failed to create temp file for new config: %w", err) } defer func() { _ = f.Close() if err := os.Remove(f.Name()); err != nil && !os.IsNotExist(err) { - fs.Errorf(nil, "Failed to remove temp config file: %v", err) + fs.Errorf(nil, "failed to remove temp config file: %v", err) } }() var buf bytes.Buffer if err := goconfig.SaveConfigData(s.gc, &buf); err != nil { - return fmt.Errorf("Failed to save config file: %v", err) + return fmt.Errorf("failed to save config file: %w", err) } if err := config.Encrypt(&buf, f); err != nil { @@ -137,7 +137,7 @@ func (s *Storage) Save() error { _ = f.Sync() err = f.Close() if err != nil { - return fmt.Errorf("Failed to close config file: %v", err) + return fmt.Errorf("failed to close config file: %w", err) } var fileMode os.FileMode = 0600 @@ -157,10 +157,10 @@ func (s *Storage) Save() error { } if err = os.Rename(configPath, configPath+".old"); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("Failed to move previous config to backup location: %v", err) + return fmt.Errorf("failed to move previous config to backup location: %w", err) } if err = os.Rename(f.Name(), configPath); err != nil { - return fmt.Errorf("Failed to move newly written config from %s to final location: %v", f.Name(), err) + return fmt.Errorf("failed to move newly written config from %s to final location: %v", f.Name(), err) } if err := os.Remove(configPath + ".old"); err != nil && !os.IsNotExist(err) { fs.Errorf(nil, "Failed to remove backup config file: %v", err) @@ -177,7 +177,7 @@ func (s *Storage) Serialize() (string, error) { s.check() var buf bytes.Buffer if err := goconfig.SaveConfigData(s.gc, &buf); err != nil { - return "", fmt.Errorf("Failed to save config file: %v", err) + return "", fmt.Errorf("failed to save config file: %w", err) } return buf.String(), nil diff --git a/fs/config/crypt.go b/fs/config/crypt.go index 002e6c49fd71d..6792811dcef5c 100644 --- a/fs/config/crypt.go +++ b/fs/config/crypt.go @@ -133,7 +133,7 @@ func Decrypt(b io.ReadSeeker) (io.Reader, error) { return nil, fmt.Errorf("failed to load base64 encoded data: %w", err) } if len(box) < 24+secretbox.Overhead { - return nil, errors.New("Configuration data too short") + return nil, errors.New("configuration data too short") } var out []byte @@ -206,7 +206,7 @@ func Encrypt(src io.Reader, dst io.Writer) error { enc := base64.NewEncoder(base64.StdEncoding, dst) _, err := enc.Write(nonce[:]) if err != nil { - return fmt.Errorf("Failed to write config file: %v", err) + return fmt.Errorf("failed to write config file: %w", err) } var key [32]byte @@ -219,7 +219,7 @@ func Encrypt(src io.Reader, dst io.Writer) error { b := secretbox.Seal(nil, data, &nonce, &key) _, err = enc.Write(b) if err != nil { - return fmt.Errorf("Failed to write config file: %v", err) + return fmt.Errorf("failed to write config file: %w", err) } return enc.Close() } diff --git a/fs/cutoffmode.go b/fs/cutoffmode.go index a58c5c4520697..9f6390663c70d 100644 --- a/fs/cutoffmode.go +++ b/fs/cutoffmode.go @@ -38,7 +38,7 @@ func (m *CutoffMode) Set(s string) error { return nil } } - return fmt.Errorf("Unknown cutoff mode %q", s) + return fmt.Errorf("unknown cutoff mode %q", s) } // Type of the value @@ -50,7 +50,7 @@ func (m *CutoffMode) Type() string { func (m *CutoffMode) UnmarshalJSON(in []byte) error { return UnmarshalJSONFlag(in, m, func(i int64) error { if i < 0 || i >= int64(len(cutoffModeToString)) { - return fmt.Errorf("Out of range cutoff mode %d", i) + return fmt.Errorf("out of range cutoff mode %d", i) } *m = (CutoffMode)(i) return nil diff --git a/fs/dump.go b/fs/dump.go index f0c72bcaba789..a156712533c36 100644 --- a/fs/dump.go +++ b/fs/dump.go @@ -78,7 +78,7 @@ func (f *DumpFlags) Set(s string) error { } } if !found { - return fmt.Errorf("Unknown dump flag %q", part) + return fmt.Errorf("unknown dump flag %q", part) } } *f = flags diff --git a/fs/dump_test.go b/fs/dump_test.go index c0ea248dd279c..ede6125e8c731 100644 --- a/fs/dump_test.go +++ b/fs/dump_test.go @@ -30,7 +30,7 @@ func TestDumpFlagsSet(t *testing.T) { {"bodies,headers,auth", DumpBodies | DumpHeaders | DumpAuth, ""}, {"bodies,headers,auth", DumpBodies | DumpHeaders | DumpAuth, ""}, {"headers,bodies,requests,responses,auth,filters", DumpHeaders | DumpBodies | DumpRequests | DumpResponses | DumpAuth | DumpFilters, ""}, - {"headers,bodies,unknown,auth", 0, "Unknown dump flag \"unknown\""}, + {"headers,bodies,unknown,auth", 0, "unknown dump flag \"unknown\""}, } { f := DumpFlags(-1) initial := f @@ -69,7 +69,7 @@ func TestDumpFlagsUnmarshallJSON(t *testing.T) { {`"bodies,headers,auth"`, DumpBodies | DumpHeaders | DumpAuth, ""}, {`"bodies,headers,auth"`, DumpBodies | DumpHeaders | DumpAuth, ""}, {`"headers,bodies,requests,responses,auth,filters"`, DumpHeaders | DumpBodies | DumpRequests | DumpResponses | DumpAuth | DumpFilters, ""}, - {`"headers,bodies,unknown,auth"`, 0, "Unknown dump flag \"unknown\""}, + {`"headers,bodies,unknown,auth"`, 0, "unknown dump flag \"unknown\""}, {`0`, DumpFlags(0), ""}, {strconv.Itoa(int(DumpBodies)), DumpBodies, ""}, {strconv.Itoa(int(DumpBodies | DumpHeaders | DumpAuth)), DumpBodies | DumpHeaders | DumpAuth, ""}, diff --git a/fs/filter/filter.go b/fs/filter/filter.go index e35018fa50112..741580fffec9e 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -199,7 +199,7 @@ func NewFilter(opt *Opt) (f *Filter, err error) { for _, rule := range f.Opt.FilesFrom { if !inActive { - return nil, fmt.Errorf("The usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw") + return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw") } f.initAddFile() // init to show --files-from set even if no files within err := forEachLine(rule, false, func(line string) error { @@ -214,7 +214,7 @@ func NewFilter(opt *Opt) (f *Filter, err error) { // --files-from-raw can be used with --files-from, hence we do // not need to get the value of f.InActive again if !inActive { - return nil, fmt.Errorf("The usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from") + return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from") } f.initAddFile() // init to show --files-from set even if no files within err := forEachLine(rule, true, func(line string) error { diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go index 81a01af3d1dac..27da120c4e5e1 100644 --- a/fs/filter/filter_test.go +++ b/fs/filter/filter_test.go @@ -62,7 +62,7 @@ func TestNewFilterForbiddenMixOfFilesFromAndFilterRule(t *testing.T) { _, err := NewFilter(&Opt) require.Error(t, err) - require.Contains(t, err.Error(), "The usage of --files-from overrides all other filters") + require.Contains(t, err.Error(), "the usage of --files-from overrides all other filters") } func TestNewFilterForbiddenMixOfFilesFromRawAndFilterRule(t *testing.T) { @@ -85,7 +85,7 @@ func TestNewFilterForbiddenMixOfFilesFromRawAndFilterRule(t *testing.T) { _, err := NewFilter(&Opt) require.Error(t, err) - require.Contains(t, err.Error(), "The usage of --files-from-raw overrides all other filters") + require.Contains(t, err.Error(), "the usage of --files-from-raw overrides all other filters") } func TestNewFilterWithFilesFromAlone(t *testing.T) { diff --git a/fs/hash/hash.go b/fs/hash/hash.go index 48ecdcd4cb8af..93b1de3b22b03 100644 --- a/fs/hash/hash.go +++ b/fs/hash/hash.go @@ -153,7 +153,7 @@ func (h *Type) Set(s string) error { *h = hash.hashType return nil } - return fmt.Errorf("Unknown hash type %q", s) + return fmt.Errorf("unknown hash type %q", s) } // Type of the value diff --git a/fs/log.go b/fs/log.go index 423352344f7a4..8328b9106e01d 100644 --- a/fs/log.go +++ b/fs/log.go @@ -61,7 +61,7 @@ func (l *LogLevel) Set(s string) error { return nil } } - return fmt.Errorf("Unknown log level %q", s) + return fmt.Errorf("unknown log level %q", s) } // Type of the value @@ -73,7 +73,7 @@ func (l *LogLevel) Type() string { func (l *LogLevel) UnmarshalJSON(in []byte) error { return UnmarshalJSONFlag(in, l, func(i int64) error { if i < 0 || i >= int64(LogLevel(len(logLevelToString))) { - return fmt.Errorf("Unknown log level %d", i) + return fmt.Errorf("unknown log level %d", i) } *l = (LogLevel)(i) return nil diff --git a/fs/open_options.go b/fs/open_options.go index 938e38d04fd6f..e357a9a6990d0 100644 --- a/fs/open_options.go +++ b/fs/open_options.go @@ -72,28 +72,28 @@ func (o *RangeOption) Header() (key string, value string) { func ParseRangeOption(s string) (po *RangeOption, err error) { const preamble = "bytes=" if !strings.HasPrefix(s, preamble) { - return nil, errors.New("Range: header invalid: doesn't start with " + preamble) + return nil, errors.New("range: header invalid: doesn't start with " + preamble) } s = s[len(preamble):] if strings.ContainsRune(s, ',') { - return nil, errors.New("Range: header invalid: contains multiple ranges which isn't supported") + return nil, errors.New("range: header invalid: contains multiple ranges which isn't supported") } dash := strings.IndexRune(s, '-') if dash < 0 { - return nil, errors.New("Range: header invalid: contains no '-'") + return nil, errors.New("range: header invalid: contains no '-'") } start, end := strings.TrimSpace(s[:dash]), strings.TrimSpace(s[dash+1:]) o := RangeOption{Start: -1, End: -1} if start != "" { o.Start, err = strconv.ParseInt(start, 10, 64) if err != nil || o.Start < 0 { - return nil, errors.New("Range: header invalid: bad start") + return nil, errors.New("range: header invalid: bad start") } } if end != "" { o.End, err = strconv.ParseInt(end, 10, 64) if err != nil || o.End < 0 { - return nil, errors.New("Range: header invalid: bad end") + return nil, errors.New("range: header invalid: bad end") } } return &o, nil diff --git a/fs/operations/check.go b/fs/operations/check.go index a7a52e299b28d..5cb3b0cf998b3 100644 --- a/fs/operations/check.go +++ b/fs/operations/check.go @@ -80,7 +80,7 @@ func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) { if c.opt.OneWay { return false } - err := fmt.Errorf("File not in %v", c.opt.Fsrc) + err := fmt.Errorf("file not in %v", c.opt.Fsrc) fs.Errorf(dst, "%v", err) _ = fs.CountError(err) atomic.AddInt32(&c.differences, 1) @@ -102,7 +102,7 @@ func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) { func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) { switch src.(type) { case fs.Object: - err := fmt.Errorf("File not in %v", c.opt.Fdst) + err := fmt.Errorf("file not in %v", c.opt.Fdst) fs.Errorf(src, "%v", err) _ = fs.CountError(err) atomic.AddInt32(&c.differences, 1) @@ -125,7 +125,7 @@ func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (di tr.Done(ctx, err) }() if sizeDiffers(ctx, src, dst) { - err = fmt.Errorf("Sizes differ") + err = fmt.Errorf("sizes differ") fs.Errorf(src, "%v", err) return true, false, nil } @@ -424,7 +424,7 @@ func CheckSum(ctx context.Context, fsrc, fsum fs.Fs, sumFile string, hashType ha continue } // filesystem missed the file, sum wasn't consumed - err := fmt.Errorf("File not in %v", opt.Fdst) + err := fmt.Errorf("file not in %v", opt.Fdst) fs.Errorf(filename, "%v", err) _ = fs.CountError(err) if lastErr == nil { diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.go index 5af5a4295808b..144a47bb6e13b 100644 --- a/fs/operations/lsjson.go +++ b/fs/operations/lsjson.go @@ -122,7 +122,7 @@ func newListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOp return nil, fmt.Errorf("ListJSON failed to load config for crypt remote: %w", err) } if fsInfo.Name != "crypt" { - return nil, errors.New("The remote needs to be of type \"crypt\"") + return nil, errors.New("the remote needs to be of type \"crypt\"") } lj.cipher, err = crypt.NewCipher(config) if err != nil { diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 5eccc44f719eb..bdeeaf03686e3 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1407,7 +1407,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, fs.Debugf(fdst, "Target remote doesn't support streaming uploads, creating temporary local FS to spool file") tmpLocalFs, err := fs.TemporaryLocalFs(ctx) if err != nil { - return nil, fmt.Errorf("Failed to create temporary local FS to spool file: %w", err) + return nil, fmt.Errorf("failed to create temporary local FS to spool file: %w", err) } defer func() { err := Purge(ctx, tmpLocalFs, "") @@ -1523,7 +1523,7 @@ func GetCompareDest(ctx context.Context) (CompareDest []fs.Fs, err error) { ci := fs.GetConfig(ctx) CompareDest, err = cache.GetArr(ctx, ci.CompareDest) if err != nil { - return nil, fserrors.FatalError(fmt.Errorf("Failed to make fs for --compare-dest %q: %v", ci.CompareDest, err)) + return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --compare-dest %q: %w", ci.CompareDest, err)) } return CompareDest, nil } @@ -1562,7 +1562,7 @@ func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest []fs.Fs, err error) ci := fs.GetConfig(ctx) CopyDest, err = cache.GetArr(ctx, ci.CopyDest) if err != nil { - return nil, fserrors.FatalError(fmt.Errorf("Failed to make fs for --copy-dest %q: %v", ci.CopyDest, err)) + return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --copy-dest %q: %w", ci.CopyDest, err)) } if !SameConfigArr(fdst, CopyDest) { return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination")) @@ -1777,7 +1777,7 @@ func copyURLFn(ctx context.Context, dstFileName string, url string, autoFilename _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) headerFilename := path.Base(strings.Replace(params["filename"], "\\", "/", -1)) if err != nil || headerFilename == "" { - return fmt.Errorf("copyurl failed: filename not found in the Content-Dispoition header") + return fmt.Errorf("CopyURL failed: filename not found in the Content-Dispoition header") } fs.Debugf(headerFilename, "filename found in Content-Disposition header.") return fn(ctx, headerFilename, resp.Body, resp.ContentLength, modTime) @@ -1822,7 +1822,7 @@ func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string) if ci.BackupDir != "" { backupDir, err = cache.Get(ctx, ci.BackupDir) if err != nil { - return nil, fserrors.FatalError(fmt.Errorf("Failed to make fs for --backup-dir %q: %v", ci.BackupDir, err)) + return nil, fserrors.FatalError(fmt.Errorf("failed to make fs for --backup-dir %q: %w", ci.BackupDir, err)) } if !SameConfig(fdst, backupDir) { return nil, fserrors.FatalError(errors.New("parameter to --backup-dir has to be on the same remote as destination")) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 04b26e8b8819e..02a44d385cf3e 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1696,7 +1696,7 @@ func TestCopyFileMaxTransfer(t *testing.T) { accounting.Stats(ctx).ResetCounters() err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file2.Path, file2.Path) require.NotNil(t, err, "Did not get expected max transfer limit error") - assert.Contains(t, err.Error(), "Max transfer limit reached") + assert.Contains(t, err.Error(), "max transfer limit reached") assert.True(t, fserrors.IsFatalError(err), fmt.Sprintf("Not fatal error: %v: %#v:", err, err)) r.CheckLocalItems(t, file1, file2, file3, file4) r.CheckRemoteItems(t, file1) @@ -1708,7 +1708,7 @@ func TestCopyFileMaxTransfer(t *testing.T) { accounting.Stats(ctx).ResetCounters() err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file3.Path, file3.Path) require.NotNil(t, err) - assert.Contains(t, err.Error(), "Max transfer limit reached") + assert.Contains(t, err.Error(), "max transfer limit reached") assert.True(t, fserrors.IsNoRetryError(err)) r.CheckLocalItems(t, file1, file2, file3, file4) r.CheckRemoteItems(t, file1) diff --git a/fs/rc/internal.go b/fs/rc/internal.go index 10077ae098651..24e67eabb0a94 100644 --- a/fs/rc/internal.go +++ b/fs/rc/internal.go @@ -475,7 +475,7 @@ func rcRunCommand(ctx context.Context, in Params) (out Params, err error) { cmd.Stdout = httpResponse cmd.Stderr = httpResponse } else { - return nil, fmt.Errorf("Unknown returnType %q", returnType) + return nil, fmt.Errorf("unknown returnType %q", returnType) } err = cmd.Run() diff --git a/fs/rc/rcserver/rcserver_test.go b/fs/rc/rcserver/rcserver_test.go index f75cfe52f2042..34181b631ed0c 100644 --- a/fs/rc/rcserver/rcserver_test.go +++ b/fs/rc/rcserver/rcserver_test.go @@ -497,7 +497,7 @@ func TestRCWithAuth(t *testing.T) { ContentType: "application/x-www-form-urlencoded", Status: http.StatusInternalServerError, Expected: `{ - "error": "Unknown returnType \"POTATO\"", + "error": "unknown returnType \"POTATO\"", "input": { "command": "version", "returnType": "POTATO" diff --git a/fs/rc/webgui/webgui.go b/fs/rc/webgui/webgui.go index ea37382eea511..8153024d173de 100644 --- a/fs/rc/webgui/webgui.go +++ b/fs/rc/webgui/webgui.go @@ -63,7 +63,7 @@ func CheckAndDownloadWebGUIRelease(checkUpdate bool, forceUpdate bool, fetchURL // Get the latest release details WebUIURL, tag, size, err := GetLatestReleaseURL(fetchURL) if err != nil { - return fmt.Errorf("Error checking for web gui release update, skipping update: %w", err) + return fmt.Errorf("error checking for web gui release update, skipping update: %w", err) } dat, err := ioutil.ReadFile(tagPath) tagsMatch := false diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 58e74e22a6377..052b867e8d60c 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -825,7 +825,7 @@ func (s *syncCopyMove) tryRename(src fs.Object) bool { // errorMaxDurationReached defines error when transfer duration is reached // Used for checking on exit and matching to correct exit code. -var errorMaxDurationReached = fserrors.FatalError(errors.New("Max transfer duration reached as set by --max-duration")) +var errorMaxDurationReached = fserrors.FatalError(errors.New("max transfer duration reached as set by --max-duration")) // Syncs fsrc into fdst // diff --git a/fstest/mockobject/mockobject.go b/fstest/mockobject/mockobject.go index 0d0c4516cb049..084536b85a1a5 100644 --- a/fstest/mockobject/mockobject.go +++ b/fstest/mockobject/mockobject.go @@ -140,7 +140,7 @@ func (o *ContentMockObject) Open(ctx context.Context, options ...fs.OpenOption) offset, limit = x.Decode(size) default: if option.Mandatory() { - return nil, fmt.Errorf("Unsupported mandatory option: %v", option) + return nil, fmt.Errorf("unsupported mandatory option: %v", option) } } } diff --git a/fstest/test_all/clean.go b/fstest/test_all/clean.go index c12a4839abd72..6da23b7e1cc46 100644 --- a/fstest/test_all/clean.go +++ b/fstest/test_all/clean.go @@ -56,7 +56,7 @@ func cleanFs(ctx context.Context, remote string, cleanup bool) error { } err = operations.Purge(ctx, dir, "") if err != nil { - err = fmt.Errorf("Purge failed: %w", err) + err = fmt.Errorf("purge failed: %w", err) lastErr = err fs.Errorf(dir, "%v", err) return nil diff --git a/lib/http/http.go b/lib/http/http.go index ac5befd3719f4..1ff7c11489271 100644 --- a/lib/http/http.go +++ b/lib/http/http.go @@ -122,7 +122,7 @@ func useSSL(opt Options) bool { func NewServer(listeners, tlsListeners []net.Listener, opt Options) (Server, error) { // Validate input if len(listeners) == 0 && len(tlsListeners) == 0 { - return nil, errors.New("Can't create server without listeners") + return nil, errors.New("can't create server without listeners") } // Prepare TLS config @@ -130,12 +130,12 @@ func NewServer(listeners, tlsListeners []net.Listener, opt Options) (Server, err useSSL := useSSL(opt) if (len(opt.SslCertBody) > 0) != (len(opt.SslKeyBody) > 0) { - err := errors.New("Need both SslCertBody and SslKeyBody to use SSL") + err := errors.New("need both SslCertBody and SslKeyBody to use SSL") log.Fatalf(err.Error()) return nil, err } if (opt.SslCert != "") != (opt.SslKey != "") { - err := errors.New("Need both -cert and -key to use SSL") + err := errors.New("need both -cert and -key to use SSL") log.Fatalf(err.Error()) return nil, err } @@ -156,12 +156,12 @@ func NewServer(listeners, tlsListeners []net.Listener, opt Options) (Server, err Certificates: []tls.Certificate{cert}, } } else if len(listeners) == 0 && len(tlsListeners) != 0 { - return nil, errors.New("No SslKey or non-tlsListeners") + return nil, errors.New("no SslKey or non-tlsListeners") } if opt.ClientCA != "" { if !useSSL { - err := errors.New("Can't use --client-ca without --cert and --key") + err := errors.New("can't use --client-ca without --cert and --key") log.Fatalf(err.Error()) return nil, err } @@ -172,7 +172,7 @@ func NewServer(listeners, tlsListeners []net.Listener, opt Options) (Server, err return nil, err } if !certpool.AppendCertsFromPEM(pem) { - err := errors.New("Can't parse client certificate authority") + err := errors.New("can't parse client certificate authority") log.Fatalf(err.Error()) return nil, err } diff --git a/lib/jwtutil/jwtutil.go b/lib/jwtutil/jwtutil.go index 8473c45bddbb6..af99ed3f80606 100644 --- a/lib/jwtutil/jwtutil.go +++ b/lib/jwtutil/jwtutil.go @@ -78,7 +78,7 @@ func Config(id, name string, claims *jws.ClaimSet, header *jws.Header, queryPara result := &response{} err = json.NewDecoder(strings.NewReader(s)).Decode(result) if result.AccessToken == "" && err == nil { - err = errors.New("No AccessToken in Response") + err = errors.New("no AccessToken in Response") } if err != nil { return fmt.Errorf("jwtutil: failed to get token: %w", err) diff --git a/lib/rest/url.go b/lib/rest/url.go index 4a1d7139091eb..8f443d59d3a93 100644 --- a/lib/rest/url.go +++ b/lib/rest/url.go @@ -11,7 +11,7 @@ import ( func URLJoin(base *url.URL, path string) (*url.URL, error) { rel, err := url.Parse(path) if err != nil { - return nil, fmt.Errorf("Error parsing %q as URL: %w", path, err) + return nil, fmt.Errorf("error parsing %q as URL: %w", path, err) } return base.ResolveReference(rel), nil } diff --git a/vfs/file.go b/vfs/file.go index 4c4fc214ac2dd..2d3199f137dc2 100644 --- a/vfs/file.go +++ b/vfs/file.go @@ -400,7 +400,7 @@ func (f *File) _applyPendingModTime() error { defer func() { f.pendingModTime = time.Time{} }() if f.o == nil { - return errors.New("Cannot apply ModTime, file object is not available") + return errors.New("cannot apply ModTime, file object is not available") } dt := f.pendingModTime.Sub(f.o.ModTime(context.Background())) diff --git a/vfs/vfscommon/cachemode.go b/vfs/vfscommon/cachemode.go index a7f35fe9c14db..a2d91bd4ea38b 100644 --- a/vfs/vfscommon/cachemode.go +++ b/vfs/vfscommon/cachemode.go @@ -40,7 +40,7 @@ func (l *CacheMode) Set(s string) error { return nil } } - return fmt.Errorf("Unknown cache mode level %q", s) + return fmt.Errorf("unknown cache mode level %q", s) } // Type of the value @@ -52,7 +52,7 @@ func (l *CacheMode) Type() string { func (l *CacheMode) UnmarshalJSON(in []byte) error { return fs.UnmarshalJSONFlag(in, l, func(i int64) error { if i < 0 || i >= int64(len(cacheModeToString)) { - return fmt.Errorf("Unknown cache mode level %d", i) + return fmt.Errorf("unknown cache mode level %d", i) } *l = CacheMode(i) return nil diff --git a/vfs/vfsflags/filemode.go b/vfs/vfsflags/filemode.go index 50a8bc4233c8e..b1c2e8bcd8e97 100644 --- a/vfs/vfsflags/filemode.go +++ b/vfs/vfsflags/filemode.go @@ -20,7 +20,7 @@ func (x *FileMode) String() string { func (x *FileMode) Set(s string) error { i, err := strconv.ParseInt(s, 8, 64) if err != nil { - return fmt.Errorf("Bad FileMode - must be octal digits: %w", err) + return fmt.Errorf("bad FileMode - must be octal digits: %w", err) } *x.Mode = (os.FileMode)(i) return nil From 517e7d927199f6bfe1ac8e7b06c5308bc2b7c022 Mon Sep 17 00:00:00 2001 From: buda Date: Mon, 27 Jun 2022 20:56:03 +0400 Subject: [PATCH 108/560] accounting: fix unknown length file transfers count 3 transfers each #6213 This was caused by nested calls to NewTransfer/Done. This fixes the problem by only incrementing transfers if the remote is present in the transferMap which means we only increment it once. --- fs/accounting/stats.go | 6 +++--- fs/accounting/transfermap.go | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go index b4b32a8eecec2..fc17f95ac23ca 100644 --- a/fs/accounting/stats.go +++ b/fs/accounting/stats.go @@ -714,10 +714,10 @@ func (s *StatsInfo) NewTransferRemoteSize(remote string, size int64) *Transfer { // DoneTransferring removes a transfer from the stats // -// if ok is true then it increments the transfers count +// if ok is true and it was in the transfermap (to avoid incrementing in case of nested calls, #6213) then it increments the transfers count func (s *StatsInfo) DoneTransferring(remote string, ok bool) { - s.transferring.del(remote) - if ok { + existed := s.transferring.del(remote) + if ok && existed { s.mu.Lock() s.transfers++ s.mu.Unlock() diff --git a/fs/accounting/transfermap.go b/fs/accounting/transfermap.go index ed64bf3697408..4fb30a07980b5 100644 --- a/fs/accounting/transfermap.go +++ b/fs/accounting/transfermap.go @@ -34,10 +34,13 @@ func (tm *transferMap) add(tr *Transfer) { } // del removes a transfer from the map by name -func (tm *transferMap) del(remote string) { +func (tm *transferMap) del(remote string) bool { tm.mu.Lock() + _, exists := tm.items[remote] delete(tm.items, remote) tm.mu.Unlock() + + return exists } // merge adds items from another map From 32006033e6d576630207514cbe3f4779d65133bd Mon Sep 17 00:00:00 2001 From: mirekphd <36706320+mirekphd@users.noreply.github.com> Date: Mon, 27 Jun 2022 20:33:07 +0200 Subject: [PATCH 109/560] docs: note wider impact of --checkers=N on parallelism #6280 --- docs/content/docs.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index 50bc83cfae7a0..66777e1b8981d 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -639,12 +639,22 @@ objects to transfer is held in memory before the transfers start. ### --checkers=N ### -The number of checkers to run in parallel. Checkers do the equality -checking of files during a sync. For some storage systems (e.g. S3, -Swift, Dropbox) this can take a significant amount of time so they are -run in parallel. - -The default is to run 8 checkers in parallel. +Originally controlling just the number of file checkers to run in parallel, +e.g. by `rclone copy`. Now a fairly universal parallelism control +used by `rclone` in several places. + +Note: checkers do the equality checking of files during a sync. +For some storage systems (e.g. S3, Swift, Dropbox) this can take +a significant amount of time so they are run in parallel. + +The default is to run 8 checkers in parallel. However, in case +of slow-reacting backends you may need to lower (rather than increase) +this default by setting `--checkers` to 4 or less threads. This is +especially advised if you are experiencing backend server crashes +during file checking phase (e.g. on subsequent or top-up backups +where little or no file copying is done and checking takes up +most of the time). Increase this setting only with utmost care, +while monitoring your server health and file checking throughput. ### -c, --checksum ### From 326c43ab3f5c7c8570bf87b4656b41846ce00877 Mon Sep 17 00:00:00 2001 From: vyloy Date: Sat, 18 Jun 2022 15:29:21 +0800 Subject: [PATCH 110/560] s3: add IDrive e2 to provider list --- backend/s3/s3.go | 55 +++++++++++++++++-- docs/content/_index.md | 1 + docs/content/s3.md | 121 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 169 insertions(+), 8 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index c81024ce3c60c..c5dfbcb74c25d 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "encoding/base64" "encoding/hex" + "encoding/json" "encoding/xml" "errors" "fmt" @@ -60,9 +61,16 @@ import ( func init() { fs.Register(&fs.RegInfo{ Name: "s3", - Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi", + Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi", NewFs: NewFs, CommandHelp: commandHelp, + Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { + switch config.State { + case "": + return nil, setEndpointValueForIDriveE2(m) + } + return nil, fmt.Errorf("unknown state %q", config.State) + }, Options: []fs.Option{{ Name: fs.ConfigProvider, Help: "Choose your S3 provider.", @@ -98,6 +106,9 @@ func init() { }, { Value: "IBMCOS", Help: "IBM COS S3", + }, { + Value: "IDrive", + Help: "IDrive e2", }, { Value: "LyveCloud", Help: "Seagate Lyve Cloud", @@ -369,7 +380,7 @@ func init() { }, { Name: "region", Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.", - Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS", + Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive", Examples: []fs.OptionExample{{ Value: "", Help: "Use this if unsure.\nWill use v4 signatures and an empty region.", @@ -983,7 +994,7 @@ func init() { }, { Name: "endpoint", Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.", - Provider: "!AWS,IBMCOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp", + Provider: "!AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp", Examples: []fs.OptionExample{{ Value: "objects-us-east-1.dream.io", Help: "Dream Objects endpoint", @@ -1393,7 +1404,7 @@ func init() { }, { Name: "location_constraint", Help: "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.", - Provider: "!AWS,IBMCOS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS", + Provider: "!AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS", }, { Name: "acl", Help: `Canned ACL used when creating buckets and storing or copying objects. @@ -2329,6 +2340,37 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) { return } +// setEndpointValueForIDriveE2 gets user region endpoint against the Access Key details by calling the API +func setEndpointValueForIDriveE2(m configmap.Mapper) (err error) { + value, ok := m.Get(fs.ConfigProvider) + if !ok || value != "IDrive" { + return + } + value, ok = m.Get("access_key_id") + if !ok || value == "" { + return + } + client := &http.Client{Timeout: time.Second * 3} + // API to get user region endpoint against the Access Key details: https://www.idrive.com/e2/guides/get_region_endpoint + resp, err := client.Post("https://api.idrivee2.com/api/service/get_region_end_point", + "application/json", + strings.NewReader(`{"access_key": "`+value+`"}`)) + if err != nil { + return + } + defer fs.CheckClose(resp.Body, &err) + decoder := json.NewDecoder(resp.Body) + var data = &struct { + RespCode int `json:"resp_code"` + RespMsg string `json:"resp_msg"` + DomainName string `json:"domain_name"` + }{} + if err = decoder.Decode(data); err == nil && data.RespCode == 0 { + m.Set("endpoint", data.DomainName) + } + return +} + // Set the provider quirks // // There should be no testing against opt.Provider anywhere in the @@ -2375,6 +2417,8 @@ func setQuirks(opt *Options) { virtualHostStyle = false urlEncodeListings = false useMultipartEtag = false // untested + case "IDrive": + virtualHostStyle = false case "LyveCloud": useMultipartEtag = false // LyveCloud seems to calculate multipart Etags differently from AWS case "Minio": @@ -2554,6 +2598,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e f.features.SetTier = false f.features.GetTier = false } + if opt.Provider == "IDrive" { + f.features.SetTier = false + } // f.listMultipartUploads() return f, nil } diff --git a/docs/content/_index.md b/docs/content/_index.md index dcf51e23f8c9c..58410d6d4330d 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -132,6 +132,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}} {{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}} {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} +{{< provider name="IDrive e2" home="https://www.idrive.com/e2/" config="/s3/#idrive-e2" >}} {{< provider name="Koofr" home="https://koofr.eu/" config="/koofr/" >}} {{< provider name="Mail.ru Cloud" home="https://cloud.mail.ru/" config="/mailru/" >}} {{< provider name="Memset Memstore" home="https://www.memset.com/cloud/storage/" config="/swift/" >}} diff --git a/docs/content/s3.md b/docs/content/s3.md index fbd47c76327cd..c2adbfe95222c 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -18,6 +18,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} {{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}} {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} +{{< provider name="IDrive e2" home="https://www.idrive.com/e2/" config="/s3/#idrive-e2" >}} {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}} {{< provider name="RackCorp Object Storage" home="https://www.rackcorp.com/" config="/s3/#RackCorp" >}} {{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}} @@ -570,7 +571,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS). +Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS). #### --s3-provider @@ -601,6 +602,8 @@ Properties: - Huawei Object Storage Service - "IBMCOS" - IBM COS S3 + - "IDrive" + - IDrive e2 - "LyveCloud" - Seagate Lyve Cloud - "Minio" @@ -836,7 +839,7 @@ Properties: - Config: region - Env Var: RCLONE_S3_REGION -- Provider: !AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS +- Provider: !AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive - Type: string - Required: false - Examples: @@ -1362,7 +1365,7 @@ Properties: - Config: endpoint - Env Var: RCLONE_S3_ENDPOINT -- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,HuaweiOBS +- Provider: !AWS,IBMCOS,IDrive,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,HuaweiOBS - Type: string - Required: false - Examples: @@ -1601,7 +1604,7 @@ Properties: - Config: location_constraint - Env Var: RCLONE_S3_LOCATION_CONSTRAINT -- Provider: !AWS,IBMCOS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS,HuaweiOBS +- Provider: !AWS,IBMCOS,IDrive,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS,HuaweiOBS - Type: string - Required: false @@ -3076,6 +3079,116 @@ acl> 1 rclone delete IBM-COS-XREGION:newbucket/file.txt ``` +### IDrive e2 {#idrive-e2} + +Here is an example of making an [IDrive e2](https://www.idrive.com/e2/) +configuration. First run: + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n + +Enter name for new remote. +name> e2 + +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +[snip] +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \ (s3) +[snip] +Storage> s3 + +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. +[snip] +XX / IDrive e2 + \ (IDrive) +[snip] +provider> IDrive + +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) +env_auth> + +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> YOUR_ACCESS_KEY + +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> YOUR_SECRET_KEY + +Option acl. +Canned ACL used when creating buckets and storing or copying objects. +This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Note that this ACL is applied when server-side copying objects as S3 +doesn't copy the ACL from the source but rather writes a fresh one. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \ (private) + / Owner gets FULL_CONTROL. + 2 | The AllUsers group gets READ access. + \ (public-read) + / Owner gets FULL_CONTROL. + 3 | The AllUsers group gets READ and WRITE access. + | Granting this on a bucket is generally not recommended. + \ (public-read-write) + / Owner gets FULL_CONTROL. + 4 | The AuthenticatedUsers group gets READ access. + \ (authenticated-read) + / Object owner gets FULL_CONTROL. + 5 | Bucket owner gets READ access. + | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ (bucket-owner-read) + / Both the object owner and the bucket owner get FULL_CONTROL over the object. + 6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ (bucket-owner-full-control) +acl> + +Edit advanced config? +y) Yes +n) No (default) +y/n> + +Configuration complete. +Options: +- type: s3 +- provider: IDrive +- access_key_id: YOUR_ACCESS_KEY +- secret_access_key: YOUR_SECRET_KEY +- endpoint: q9d9.la12.idrivee2-5.com +Keep this "e2" remote? +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + ### Minio [Minio](https://minio.io/) is an object storage server built for cloud application developers and devops. From 5de9278650584ddcb757fd82d28fdcede3927752 Mon Sep 17 00:00:00 2001 From: Martin Czygan <53705+miku@users.noreply.github.com> Date: Tue, 28 Jun 2022 13:51:59 +0200 Subject: [PATCH 111/560] fs/cache: make sure we call the Shutdown method on backends This change ensures we call the Shutdown method on backends when they drop out of the fs/cache and at program exit. Some backends implement the optional fs.Shutdowner interface. Until now, Shutdown is only checked and called, when a backend is wrapped (e.g. crypt, compress, ...). To have a general way to perform operations at the end of the backend lifecycle with proper error handling, we can call Shutdown at cache clear time. We add a finalize hook to the cache which will be called when values drop out of the cache. Previous discussion: https://forum.rclone.org/t/31336 --- cmd/cmd.go | 6 ++++++ fs/cache/cache.go | 5 +++++ lib/cache/cache.go | 36 +++++++++++++++++++++++++++++------- lib/cache/cache_test.go | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 5dd120be7fd6a..6209fad55d949 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -321,6 +321,12 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) { } } + // clear cache and shutdown backends + cache.Clear() + if lastErr := accounting.GlobalStats().GetLastError(); cmdErr == nil { + cmdErr = lastErr + } + // Log the final error message and exit if cmdErr != nil { nerrs := accounting.GlobalStats().GetErrors() diff --git a/fs/cache/cache.go b/fs/cache/cache.go index 3bdc8d37fc1b5..4e2c4638e9924 100644 --- a/fs/cache/cache.go +++ b/fs/cache/cache.go @@ -25,6 +25,11 @@ func createOnFirstUse() { c = cache.New() c.SetExpireDuration(ci.FsCacheExpireDuration) c.SetExpireInterval(ci.FsCacheExpireInterval) + c.SetFinalizer(func(value interface{}) { + if s, ok := value.(fs.Shutdowner); ok { + _ = fs.CountError(s.Shutdown(context.Background())) + } + }) }) } diff --git a/lib/cache/cache.go b/lib/cache/cache.go index f9533c13718ca..b8d323ff1276a 100644 --- a/lib/cache/cache.go +++ b/lib/cache/cache.go @@ -16,6 +16,7 @@ type Cache struct { expireRunning bool expireDuration time.Duration // expire the cache entry when it is older than this expireInterval time.Duration // interval to run the cache expire + finalize func(value interface{}) } // New creates a new cache with the default expire duration and interval @@ -25,6 +26,7 @@ func New() *Cache { expireRunning: false, expireDuration: 300 * time.Second, expireInterval: 60 * time.Second, + finalize: func(_ interface{}) {}, } } @@ -154,7 +156,10 @@ func (c *Cache) GetMaybe(key string) (value interface{}, found bool) { // Returns true if the entry was found func (c *Cache) Delete(key string) bool { c.mu.Lock() - _, found := c.cache[key] + entry, found := c.cache[key] + if found { + c.finalize(entry.value) + } delete(c.cache, key) c.mu.Unlock() return found @@ -165,11 +170,13 @@ func (c *Cache) Delete(key string) bool { // Returns number of entries deleted func (c *Cache) DeletePrefix(prefix string) (deleted int) { c.mu.Lock() - for k := range c.cache { - if strings.HasPrefix(k, prefix) { - delete(c.cache, k) - deleted++ + for key, entry := range c.cache { + if !strings.HasPrefix(key, prefix) { + continue } + c.finalize(entry.value) + delete(c.cache, key) + deleted++ } c.mu.Unlock() return deleted @@ -183,12 +190,17 @@ func (c *Cache) Rename(oldKey, newKey string) (value interface{}, found bool) { c.mu.Lock() if newEntry, newFound := c.cache[newKey]; newFound { // If new entry is found use that + if _, oldFound := c.cache[oldKey]; oldFound { + // If there's an old entry, we drop it and also try shutdown. + c.finalize(c.cache[oldKey].value) + } delete(c.cache, oldKey) value, found = newEntry.value, newFound c.used(newEntry) } else if oldEntry, oldFound := c.cache[oldKey]; oldFound { // If old entry is found rename it to new and use that c.cache[newKey] = oldEntry + // No need to shutdown here, as value lives on under newKey delete(c.cache, oldKey) c.used(oldEntry) value, found = oldEntry.value, oldFound @@ -204,6 +216,7 @@ func (c *Cache) cacheExpire() { now := time.Now() for key, entry := range c.cache { if entry.pinCount <= 0 && now.Sub(entry.lastUsed) > c.expireDuration { + c.finalize(entry.value) delete(c.cache, key) } } @@ -218,10 +231,12 @@ func (c *Cache) cacheExpire() { // Clear removes everything from the cache func (c *Cache) Clear() { c.mu.Lock() - for k := range c.cache { - delete(c.cache, k) + for key, entry := range c.cache { + c.finalize(entry.value) + delete(c.cache, key) } c.mu.Unlock() + return } // Entries returns the number of entries in the cache @@ -231,3 +246,10 @@ func (c *Cache) Entries() int { c.mu.Unlock() return entries } + +// SetFinalizer sets a function to be called when a value drops out of the cache +func (c *Cache) SetFinalizer(finalize func(interface{})) { + c.mu.Lock() + c.finalize = finalize + c.mu.Unlock() +} diff --git a/lib/cache/cache_test.go b/lib/cache/cache_test.go index 089db07ef618e..415e167c3f1c1 100644 --- a/lib/cache/cache_test.go +++ b/lib/cache/cache_test.go @@ -333,3 +333,37 @@ func TestCacheRename(t *testing.T) { assert.Equal(t, 1, c.Entries()) } + +func TestCacheFinalize(t *testing.T) { + c := New() + numCalled := 0 + c.SetFinalizer(func(v interface{}) { + numCalled++ + }) + create := func(path string) (interface{}, bool, error) { + return path, true, nil + } + _, _ = c.Get("ok", create) + assert.Equal(t, 0, numCalled) + c.Clear() + assert.Equal(t, 1, numCalled) + + _, _ = c.Get("ok", create) + c.Delete("ok") + assert.Equal(t, 2, numCalled) + + _, _ = c.Get("ok", create) + c.DeletePrefix("ok") + assert.Equal(t, 3, numCalled) + + _, _ = c.Get("old", create) + _, _ = c.Get("new", create) + c.Rename("old", "new") + assert.Equal(t, 4, numCalled) + + c.expireDuration = 1 * time.Millisecond + _, _ = c.Get("ok", create) + time.Sleep(2 * time.Millisecond) + c.cacheExpire() // "ok" and "new" fall out of cache + assert.Equal(t, 6, numCalled) +} From 0fca4d2c864732ddb6ba682f258c5bc29d317edf Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 29 Jun 2022 10:55:03 +0100 Subject: [PATCH 112/560] Add buda to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 90ea4b9fb8cf2..7b7be2b84cdcc 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -617,3 +617,4 @@ put them back in again.` >}} * Caleb * J-P Treen * Martin Czygan <53705+miku@users.noreply.github.com> + * buda From 370c8fa220933b05c3e8479f4009c2ecf8c46269 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 29 Jun 2022 10:55:03 +0100 Subject: [PATCH 113/560] Add mirekphd to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 7b7be2b84cdcc..9b8f3ae6cfa3a 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -618,3 +618,4 @@ put them back in again.` >}} * J-P Treen * Martin Czygan <53705+miku@users.noreply.github.com> * buda + * mirekphd <36706320+mirekphd@users.noreply.github.com> From 35f24d5b84f9c4e98795e52d0b46fa85e22744ae Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 29 Jun 2022 10:55:03 +0100 Subject: [PATCH 114/560] Add vyloy to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 9b8f3ae6cfa3a..66f33c154de12 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -619,3 +619,4 @@ put them back in again.` >}} * Martin Czygan <53705+miku@users.noreply.github.com> * buda * mirekphd <36706320+mirekphd@users.noreply.github.com> + * vyloy From 461d041c4df26f290329864b295a2c2ee7fee1fd Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 27 Jun 2022 12:29:13 +0100 Subject: [PATCH 115/560] fstest: remove spurious contents return from PutTestContents and friends --- backend/chunker/chunker_internal_test.go | 14 +++++++------- backend/drive/drive_internal_test.go | 6 +++--- backend/hasher/hasher_internal_test.go | 2 +- backend/union/union_internal_test.go | 6 +++--- fs/operations/operations_test.go | 2 +- fstest/fstests/fstests.go | 12 +++++++----- 6 files changed, 22 insertions(+), 20 deletions(-) diff --git a/backend/chunker/chunker_internal_test.go b/backend/chunker/chunker_internal_test.go index 9299305dc39ba..c875c6762e5ba 100644 --- a/backend/chunker/chunker_internal_test.go +++ b/backend/chunker/chunker_internal_test.go @@ -59,7 +59,7 @@ var mtime1 = fstest.Time("2001-02-03T04:05:06.499999999Z") func testPutFile(ctx context.Context, t *testing.T, f fs.Fs, name, contents, message string, check bool) fs.Object { item := fstest.Item{Path: name, ModTime: mtime1} - _, obj := fstests.PutTestContents(ctx, t, f, &item, contents, check) + obj := fstests.PutTestContents(ctx, t, f, &item, contents, check) assert.NotNil(t, obj, message) return obj } @@ -440,7 +440,7 @@ func testSmallFileInternals(t *testing.T, f *Fs) { checkSmallFile := func(name, contents string) { filename := path.Join(dir, name) item := fstest.Item{Path: filename, ModTime: modTime} - _, put := fstests.PutTestContents(ctx, t, f, &item, contents, false) + put := fstests.PutTestContents(ctx, t, f, &item, contents, false) assert.NotNil(t, put) checkSmallFileInternals(put) checkContents(put, contents) @@ -489,7 +489,7 @@ func testPreventCorruption(t *testing.T, f *Fs) { newFile := func(name string) fs.Object { item := fstest.Item{Path: path.Join(dir, name), ModTime: modTime} - _, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) + obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) require.NotNil(t, obj) return obj } @@ -599,7 +599,7 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) { newFile := func(f fs.Fs, name string) (obj fs.Object, filename string, txnID string) { filename = path.Join(dir, name) item := fstest.Item{Path: filename, ModTime: modTime} - _, obj = fstests.PutTestContents(ctx, t, f, &item, contents, true) + obj = fstests.PutTestContents(ctx, t, f, &item, contents, true) require.NotNil(t, obj) if chunkObj, isChunkObj := obj.(*Object); isChunkObj { txnID = chunkObj.xactID @@ -716,7 +716,7 @@ func testFutureProof(t *testing.T, f *Fs) { name = f.makeChunkName(name, part-1, "", "") } item := fstest.Item{Path: name, ModTime: modTime} - _, obj := fstests.PutTestContents(ctx, t, f.base, &item, data, true) + obj := fstests.PutTestContents(ctx, t, f.base, &item, data, true) assert.NotNil(t, obj, msg) } @@ -790,7 +790,7 @@ func testBackwardsCompatibility(t *testing.T, f *Fs) { newFile := func(f fs.Fs, name string) (fs.Object, string) { filename := path.Join(dir, name) item := fstest.Item{Path: filename, ModTime: modTime} - _, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) + obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) require.NotNil(t, obj) return obj, filename } @@ -844,7 +844,7 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) { modTime := fstest.Time("2001-02-03T04:05:06.499999999Z") item := fstest.Item{Path: "movefile", ModTime: modTime} contents := "abcdef" - _, file := fstests.PutTestContents(ctx, t, fs1, &item, contents, true) + file := fstests.PutTestContents(ctx, t, fs1, &item, contents, true) dstOverwritten, _ := fs2.NewObject(ctx, "movefile") dstFile, err := operations.Move(ctx, fs2, dstOverwritten, "movefile", file) diff --git a/backend/drive/drive_internal_test.go b/backend/drive/drive_internal_test.go index 2a21fd5748e43..8df52716c3a49 100644 --- a/backend/drive/drive_internal_test.go +++ b/backend/drive/drive_internal_test.go @@ -378,9 +378,9 @@ func (f *Fs) InternalTestUnTrash(t *testing.T) { // Make some objects, one in a subdir contents := random.String(100) file1 := fstest.NewItem("trashDir/toBeTrashed", contents, time.Now()) - _, obj1 := fstests.PutTestContents(ctx, t, f, &file1, contents, false) + obj1 := fstests.PutTestContents(ctx, t, f, &file1, contents, false) file2 := fstest.NewItem("trashDir/subdir/toBeTrashed", contents, time.Now()) - _, _ = fstests.PutTestContents(ctx, t, f, &file2, contents, false) + _ = fstests.PutTestContents(ctx, t, f, &file2, contents, false) // Check objects checkObjects := func() { @@ -496,7 +496,7 @@ func (f *Fs) InternalTestAgeQuery(t *testing.T) { require.NoError(t, err) file1 := fstest.Item{ModTime: time.Now(), Path: "agequery.txt"} - _, _ = fstests.PutTestContents(defCtx, t, tempFs1, &file1, "abcxyz", true) + _ = fstests.PutTestContents(defCtx, t, tempFs1, &file1, "abcxyz", true) // validate sync/copy const timeQuery = "(modifiedTime >= '" diff --git a/backend/hasher/hasher_internal_test.go b/backend/hasher/hasher_internal_test.go index 3ec7a11f94eae..5332318344882 100644 --- a/backend/hasher/hasher_internal_test.go +++ b/backend/hasher/hasher_internal_test.go @@ -19,7 +19,7 @@ import ( func putFile(ctx context.Context, t *testing.T, f fs.Fs, name, data string) fs.Object { mtime1 := fstest.Time("2001-02-03T04:05:06.499999999Z") item := fstest.Item{Path: name, ModTime: mtime1} - _, o := fstests.PutTestContents(ctx, t, f, &item, data, true) + o := fstests.PutTestContents(ctx, t, f, &item, data, true) require.NotNil(t, o) return o } diff --git a/backend/union/union_internal_test.go b/backend/union/union_internal_test.go index 22443dd053572..b6f78ec8d4900 100644 --- a/backend/union/union_internal_test.go +++ b/backend/union/union_internal_test.go @@ -38,7 +38,7 @@ func (f *Fs) TestInternalReadOnly(t *testing.T) { // Put a file onto the read only fs contents := random.String(50) file1 := fstest.NewItem(dir+"/file.txt", contents, time.Now()) - _, obj1 := fstests.PutTestContents(ctx, t, rofs, &file1, contents, true) + obj1 := fstests.PutTestContents(ctx, t, rofs, &file1, contents, true) // Check read from readonly fs via union o, err := f.NewObject(ctx, file1.Path) @@ -109,14 +109,14 @@ func TestMoveCopy(t *testing.T) { // Put a file onto the local fs contentsLocal := random.String(50) fileLocal := fstest.NewItem("local.txt", contentsLocal, time.Now()) - _, _ = fstests.PutTestContents(ctx, t, fLocal, &fileLocal, contentsLocal, true) + _ = fstests.PutTestContents(ctx, t, fLocal, &fileLocal, contentsLocal, true) objLocal, err := f.NewObject(ctx, fileLocal.Path) require.NoError(t, err) // Put a file onto the memory fs contentsMemory := random.String(60) fileMemory := fstest.NewItem("memory.txt", contentsMemory, time.Now()) - _, _ = fstests.PutTestContents(ctx, t, fMemory, &fileMemory, contentsMemory, true) + _ = fstests.PutTestContents(ctx, t, fMemory, &fileMemory, contentsMemory, true) objMemory, err := f.NewObject(ctx, fileMemory.Path) require.NoError(t, err) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 02a44d385cf3e..4b65f689409cb 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -297,7 +297,7 @@ func TestHashSumsWithErrors(t *testing.T) { // Make a test file content := "-" item1 := fstest.NewItem("file1", content, t1) - _, _ = fstests.PutTestContents(ctx, t, memFs, &item1, content, true) + _ = fstests.PutTestContents(ctx, t, memFs, &item1, content, true) // MemoryFS supports MD5 buf := &bytes.Buffer{} diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 58481416a56b9..d9de908c1dd1a 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -178,7 +178,7 @@ var _ fs.MimeTyper = (*objectInfoWithMimeType)(nil) // putTestContentsMimeType puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove // // it uploads the object with the mimeType passed in if set -func putTestContentsMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool, mimeType string) (string, fs.Object) { +func putTestContentsMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool, mimeType string) fs.Object { var ( err error obj fs.Object @@ -204,22 +204,24 @@ func putTestContentsMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *f obj = findObject(ctx, t, f, file.Path) file.Check(t, obj, f.Precision()) } - return contents, obj + return obj } // PutTestContents puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove -func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool) (string, fs.Object) { +func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool) fs.Object { return putTestContentsMimeType(ctx, t, f, file, contents, check, "") } // testPut puts file with random contents to the remote func testPut(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) (string, fs.Object) { - return PutTestContents(ctx, t, f, file, random.String(100), true) + contents := random.String(100) + return contents, PutTestContents(ctx, t, f, file, contents, true) } // testPutMimeType puts file with random contents to the remote and the mime type given func testPutMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, mimeType string) (string, fs.Object) { - return putTestContentsMimeType(ctx, t, f, file, random.String(100), true, mimeType) + contents := random.String(100) + return contents, putTestContentsMimeType(ctx, t, f, file, contents, true, mimeType) } // TestPutLarge puts file to the remote, checks it and removes it on success. From 6a0e021dacb5a3c4840cde35ec26e3f254121de5 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 24 May 2022 11:00:00 +0100 Subject: [PATCH 116/560] fs: implement optional Metadata interface for Objects #111 This implements integration tests for the feature also. --- docs/content/docs.md | 126 ++++++++++++++++++++++++++++++++++++ docs/content/overview.md | 100 ++++++++++++++++------------ fs/features.go | 6 ++ fs/metadata.go | 29 +++++++++ fs/metadata_test.go | 17 +++++ fs/operations/operations.go | 10 +++ fs/types.go | 13 ++++ fstest/fstests/fstests.go | 114 ++++++++++++++++++++++++++++---- 8 files changed, 362 insertions(+), 53 deletions(-) create mode 100644 fs/metadata.go create mode 100644 fs/metadata_test.go diff --git a/docs/content/docs.md b/docs/content/docs.md index 66777e1b8981d..7cfffe2b44d94 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -425,6 +425,126 @@ This can be used when scripting to make aged backups efficiently, e.g. rclone sync -i remote:current-backup remote:previous-backup rclone sync -i /path/to/files remote:current-backup +## Metadata support {#metadata} + +Metadata is data about a file which isn't the contents of the file. +Normally rclone only preserves the modification time and the content +(MIME) type where possible. + +Rclone supports preserving all the available metadata on files (not +directories) when using the `--metadata` or `-M` flag. + +Exactly what metadata is supported and what that support means depends +on the backend. Backends that support metadata have a metadata section +in their docs and are listed in the [features table](/overview/#features) +(Eg [local](/local/#metadata), [s3](/s3/#metadata)) + +Rclone only supports a one-time sync of metadata. This means that +metadata will be synced from the source object to the destination +object only when the source object has changed and needs to be +re-uploaded. If the metadata subsequently changes on the source object +without changing the object itself then it won't be synced to the +destination object. This is in line with the way rclone syncs +`Content-Type` without the `--metadata` flag. + +Using `--metadata` when syncing from local to local will preserve file +attributes such as file mode, owner, extended attributes (not +Windows). + +Note that arbitrary metadata may be added to objects using the +`--upload-metadata key=value` flag when the object is first uploaded. +This flag can be repeated as many times as necessary. + +### Types of metadata + +Metadata is divided into two type. System metadata and User metadata. + +Metadata which the backend uses itself is called system metadata. For +example on the local backend the system metadata `uid` will store the +user ID of the file when used on a unix based platform. + +Arbitrary metadata is called user metadata and this can be set however +is desired. + +When objects are copied from backend to backend, they will attempt to +interpret system metadata if it is supplied. Metadata may change from +being user metadata to system metadata as objects are copied between +different backends. For example copying an object from s3 sets the +`content-type` metadata. In a backend which understands this (like +`azureblob`) this will become the Content-Type of the object. In a +backend which doesn't understand this (like the `local` backend) this +will become user metadata. However should the local object be copied +back to s3, the Content-Type will be set correctly. + +### Metadata framework + +Rclone implements a metadata framework which can read metadata from an +object and write it to the object when (and only when) it is being +uploaded. + +This metadata is stored as a dictionary with string keys and string +values. + +There are some limits on the names of the keys (these may be clarified +further in the future). + +- must be lower case +- may be `a-z` `0-9` containing `.` `-` or `_` +- length is backend dependent + +Each backend can provide system metadata that it understands. Some +backends can also store arbitrary user metadata. + +Where possible the key names are standardized, so, for example, it is +possible to copy object metadata from s3 to azureblob for example and +metadata will be translated apropriately. + +Some backends have limits on the size of the metadata and rclone will +give errors on upload if they are exceeded. + +### Metadata preservation + +The goal of the implementation is to + +1. Preserve metadata if at all possible +2. Interpret metadata if at all possible + +The consequences of 1 is that you can copy an S3 object to a local +disk then back to S3 losslessly. Likewise you can copy a local file +with file attributes and xattrs from local disk to s3 and back again +losslessly. + +The consequence of 2 is that you can copy an S3 object with metadata +to Azureblob (say) and have the metadata appear on the Azureblob +object also. + +### Standard system metadata + +Here is a table of standard system metadata which, if appropriate, a +backend may implement. + +| key | description | example | +|---------------------|-------------|---------| +| mode | File type and mode: octal, unix style | 0100664 | +| uid | User ID of owner: decimal number | 500 | +| gid | Group ID of owner: decimal number | 500 | +| rdev | Device ID (if special file) => hexadecimal | 0 | +| atime | Time of last access: RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | +| mtime | Time of last modification: RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | +| btime | Time of file creation (birth): RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | +| cache-control | Cache-Control header | no-cache | +| content-disposition | Content-Disposition header | inline | +| content-encoding | Content-Encoding header | gzip | +| content-language | Content-Language header | en-US | +| content-type | Content-Type header | text/plain | + +The metadata keys `mtime` and `content-type` will take precedence if +supplied in the metadata over reading the `Content-Type` or +modification time of the source object. + +Hashes are not included in system metadata as there is a well defined +way of reading those already. + Options ------- @@ -1206,6 +1326,12 @@ When the limit is reached all transfers will stop immediately. Rclone will exit with exit code 8 if the transfer limit is reached. +## --metadata / -M + +Setting this flag enables rclone to copy the metadata from the source +to the destination. For local backends this is ownership, permissions, +xattr etc. See the [#metadata](metadata section) for more info. + ### --cutoff-mode=hard|soft|cautious ### This modifies the behavior of `--max-transfer` diff --git a/docs/content/overview.md b/docs/content/overview.md index c107723eab65c..74194f28f3f61 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -14,48 +14,48 @@ show through. Here is an overview of the major features of each cloud storage system. -| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | -| ---------------------------- |:----------------:|:-------:|:----------------:|:---------------:|:---------:| -| 1Fichier | Whirlpool | - | No | Yes | R | -| Akamai Netstorage | MD5, SHA256 | R/W | No | No | R | -| Amazon Drive | MD5 | - | Yes | No | R | -| Amazon S3 (or S3 compatible) | MD5 | R/W | No | No | R/W | -| Backblaze B2 | SHA1 | R/W | No | No | R/W | -| Box | SHA1 | R/W | Yes | No | - | -| Citrix ShareFile | MD5 | R/W | Yes | No | - | -| Dropbox | DBHASH ¹ | R | Yes | No | - | -| Enterprise File Fabric | - | R/W | Yes | No | R/W | -| FTP | - | R/W ¹⁰ | No | No | - | -| Google Cloud Storage | MD5 | R/W | No | No | R/W | -| Google Drive | MD5 | R/W | No | Yes | R/W | -| Google Photos | - | - | No | Yes | R | -| HDFS | - | R/W | No | No | - | -| HTTP | - | R | No | No | R | -| Hubic | MD5 | R/W | No | No | R/W | -| Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | -| Jottacloud | MD5 | R/W | Yes | No | R | -| Koofr | MD5 | - | Yes | No | - | -| Mail.ru Cloud | Mailru ⁶ | R/W | Yes | No | - | -| Mega | - | - | No | Yes | - | -| Memory | MD5 | R/W | No | No | - | -| Microsoft Azure Blob Storage | MD5 | R/W | No | No | R/W | -| Microsoft OneDrive | SHA1 ⁵ | R/W | Yes | No | R | -| OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | -| OpenStack Swift | MD5 | R/W | No | No | R/W | -| pCloud | MD5, SHA1 ⁷ | R | No | No | W | -| premiumize.me | - | - | Yes | No | R | -| put.io | CRC-32 | R/W | No | Yes | R | -| QingStor | MD5 | - ⁹ | No | No | R/W | -| Seafile | - | - | No | No | - | -| SFTP | MD5, SHA1 ² | R/W | Depends | No | - | -| Sia | - | - | No | No | - | -| SugarSync | - | - | No | No | - | -| Storj | - | R | No | No | - | -| Uptobox | - | - | No | Yes | - | -| WebDAV | MD5, SHA1 ³ | R ⁴ | Depends | No | - | -| Yandex Disk | MD5 | R/W | No | No | R | -| Zoho WorkDrive | - | - | No | No | - | -| The local filesystem | All | R/W | Depends | No | - | +| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | Metadata | +| ---------------------------- |:----------------:|:-------:|:----------------:|:---------------:|:---------:|:--------:| +| 1Fichier | Whirlpool | - | No | Yes | R | - | +| Akamai Netstorage | MD5, SHA256 | R/W | No | No | R | - | +| Amazon Drive | MD5 | - | Yes | No | R | - | +| Amazon S3 (or S3 compatible) | MD5 | R/W | No | No | R/W | RWU | +| Backblaze B2 | SHA1 | R/W | No | No | R/W | - | +| Box | SHA1 | R/W | Yes | No | - | - | +| Citrix ShareFile | MD5 | R/W | Yes | No | - | - | +| Dropbox | DBHASH ¹ | R | Yes | No | - | - | +| Enterprise File Fabric | - | R/W | Yes | No | R/W | - | +| FTP | - | R/W ¹⁰ | No | No | - | - | +| Google Cloud Storage | MD5 | R/W | No | No | R/W | - | +| Google Drive | MD5 | R/W | No | Yes | R/W | - | +| Google Photos | - | - | No | Yes | R | - | +| HDFS | - | R/W | No | No | - | - | +| HTTP | - | R | No | No | R | - | +| Hubic | MD5 | R/W | No | No | R/W | - | +| Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | - | +| Jottacloud | MD5 | R/W | Yes | No | R | - | +| Koofr | MD5 | - | Yes | No | - | - | +| Mail.ru Cloud | Mailru ⁶ | R/W | Yes | No | - | - | +| Mega | - | - | No | Yes | - | - | +| Memory | MD5 | R/W | No | No | - | - | +| Microsoft Azure Blob Storage | MD5 | R/W | No | No | R/W | - | +| Microsoft OneDrive | SHA1 ⁵ | R/W | Yes | No | R | - | +| OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | - | +| OpenStack Swift | MD5 | R/W | No | No | R/W | - | +| pCloud | MD5, SHA1 ⁷ | R | No | No | W | - | +| premiumize.me | - | - | Yes | No | R | - | +| put.io | CRC-32 | R/W | No | Yes | R | - | +| QingStor | MD5 | - ⁹ | No | No | R/W | - | +| Seafile | - | - | No | No | - | - | +| SFTP | MD5, SHA1 ² | R/W | Depends | No | - | - | +| Sia | - | - | No | No | - | - | +| SugarSync | - | - | No | No | - | - | +| Storj | - | R | No | No | - | - | +| Uptobox | - | - | No | Yes | - | - | +| WebDAV | MD5, SHA1 ³ | R ⁴ | Depends | No | - | - | +| Yandex Disk | MD5 | R/W | No | No | R | - | +| Zoho WorkDrive | - | - | No | No | - | - | +| The local filesystem | All | R/W | Depends | No | - | RWU | ### Notes @@ -438,6 +438,22 @@ remote which supports writing (`W`) then rclone will preserve the MIME types. Otherwise they will be guessed from the extension, or the remote itself may assign the MIME type. +### Metadata + +Backends may or may support reading or writing metadata. They may +support reading and writing system metadata (metadata intrinsic to +that backend) and/or user metadata (general purpose metadata). + +The levels of metadata support are + +| Key | Explanation | +|-----|-------------| +| `R` | Read only System Metadata | +| `RW` | Read and write System Metadata | +| `RWU` | Read and write System Metadata and read and write User Metadata | + +See [the metadata docs](/docs/#metadata) for more info. + ## Optional Features ## All rclone remotes support a base command set. Other features depend diff --git a/fs/features.go b/fs/features.go index 6db6bd548c07c..6fce02d608df8 100644 --- a/fs/features.go +++ b/fs/features.go @@ -26,6 +26,9 @@ type Features struct { IsLocal bool // is the local backend SlowModTime bool // if calling ModTime() generally takes an extra transaction SlowHash bool // if calling Hash() generally takes an extra transaction + ReadMetadata bool // can read metadata from objects + WriteMetadata bool // can write metadata to objects + UserMetadata bool // can read/write general purpose metadata // Purge all files in the directory specified // @@ -305,6 +308,9 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features { ft.DuplicateFiles = ft.DuplicateFiles && mask.DuplicateFiles ft.ReadMimeType = ft.ReadMimeType && mask.ReadMimeType ft.WriteMimeType = ft.WriteMimeType && mask.WriteMimeType + ft.ReadMetadata = ft.ReadMetadata && mask.ReadMetadata + ft.WriteMetadata = ft.WriteMetadata && mask.WriteMetadata + ft.UserMetadata = ft.UserMetadata && mask.UserMetadata ft.CanHaveEmptyDirectories = ft.CanHaveEmptyDirectories && mask.CanHaveEmptyDirectories ft.BucketBased = ft.BucketBased && mask.BucketBased ft.BucketBasedRootOK = ft.BucketBasedRootOK && mask.BucketBasedRootOK diff --git a/fs/metadata.go b/fs/metadata.go new file mode 100644 index 0000000000000..dcdb517dc7d73 --- /dev/null +++ b/fs/metadata.go @@ -0,0 +1,29 @@ +package fs + +import "context" + +// Metadata represents Object metadata in a standardised form +// +// See docs/content/metadata.md for the interpretation of the keys +type Metadata map[string]string + +// Set k to v on m +// +// If m is nil, then it will get made +func (m *Metadata) Set(k, v string) { + if *m == nil { + *m = make(Metadata, 1) + } + (*m)[k] = v +} + +// GetMetadata from an ObjectInfo +// +// If the object has no metadata then metadata will be nil +func GetMetadata(ctx context.Context, o ObjectInfo) (metadata Metadata, err error) { + do, ok := o.(Metadataer) + if !ok { + return nil, nil + } + return do.Metadata(ctx) +} diff --git a/fs/metadata_test.go b/fs/metadata_test.go new file mode 100644 index 0000000000000..bc72c3964d310 --- /dev/null +++ b/fs/metadata_test.go @@ -0,0 +1,17 @@ +package fs + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMetadataSet(t *testing.T) { + var m Metadata + assert.Nil(t, m) + m.Set("key", "value") + assert.NotNil(t, m) + assert.Equal(t, "value", m["key"]) + m.Set("key", "value2") + assert.Equal(t, "value2", m["key"]) +} diff --git a/fs/operations/operations.go b/fs/operations/operations.go index bdeeaf03686e3..a9c2b524c3582 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -352,6 +352,16 @@ func (o *OverrideRemote) GetTier() string { return "" } +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *OverrideRemote) Metadata(ctx context.Context) (fs.Metadata, error) { + if do, ok := o.ObjectInfo.(fs.Metadataer); ok { + return do.Metadata(ctx) + } + return nil, nil +} + // Check all optional interfaces satisfied var _ fs.FullObjectInfo = (*OverrideRemote)(nil) diff --git a/fs/types.go b/fs/types.go index b340ce67b98e7..88ff5e8cd390e 100644 --- a/fs/types.go +++ b/fs/types.go @@ -183,6 +183,14 @@ type GetTierer interface { GetTier() string } +// Metadataer is an optional interface for Object +type Metadataer interface { + // Metadata returns metadata for an object + // + // It should return nil if there is no Metadata + Metadata(ctx context.Context) (Metadata, error) +} + // FullObjectInfo contains all the read-only optional interfaces // // Use for checking making wrapping ObjectInfos implement everything @@ -192,6 +200,7 @@ type FullObjectInfo interface { IDer ObjectUnWrapper GetTierer + Metadataer } // FullObject contains all the optional interfaces for Object @@ -204,6 +213,7 @@ type FullObject interface { ObjectUnWrapper GetTierer SetTierer + Metadataer } // ObjectOptionalInterfaces returns the names of supported and @@ -232,6 +242,9 @@ func ObjectOptionalInterfaces(o Object) (supported, unsupported []string) { _, ok = o.(GetTierer) store(ok, "GetTier") + _, ok = o.(Metadataer) + store(ok, "Metadata") + return supported, unsupported } diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index d9de908c1dd1a..47b841bcaf327 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -157,13 +157,15 @@ func retry(t *testing.T, what string, f func() error) { type objectInfoWithMimeType struct { fs.ObjectInfo mimeType string + metadata fs.Metadata } // Return a wrapped fs.ObjectInfo which returns the mime type given -func overrideMimeType(o fs.ObjectInfo, mimeType string) fs.ObjectInfo { +func overrideMimeType(o fs.ObjectInfo, mimeType string, metadata fs.Metadata) fs.ObjectInfo { return &objectInfoWithMimeType{ ObjectInfo: o, mimeType: mimeType, + metadata: metadata, } } @@ -172,13 +174,25 @@ func (o *objectInfoWithMimeType) MimeType(ctx context.Context) string { return o.mimeType } +// Metadata that was overridden +func (o *objectInfoWithMimeType) Metadata(ctx context.Context) (fs.Metadata, error) { + return o.metadata, nil +} + +// check interfaces +var ( + _ fs.MimeTyper = (*objectInfoWithMimeType)(nil) + _ fs.Metadataer = (*objectInfoWithMimeType)(nil) +) + // check interface -var _ fs.MimeTyper = (*objectInfoWithMimeType)(nil) -// putTestContentsMimeType puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove +// PutTestContentsMetadata puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove // -// it uploads the object with the mimeType passed in if set -func putTestContentsMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool, mimeType string) fs.Object { +// It uploads the object with the mimeType and metadata passed in if set +// +// It returns the object which will have been checked if check is set +func PutTestContentsMetadata(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool, mimeType string, metadata fs.Metadata) fs.Object { var ( err error obj fs.Object @@ -191,14 +205,21 @@ func putTestContentsMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *f file.Size = int64(buf.Len()) obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil) - if mimeType != "" { - obji = overrideMimeType(obji, mimeType) + if mimeType != "" || metadata != nil { + obji = overrideMimeType(obji, mimeType, metadata) } obj, err = f.Put(ctx, in, obji) return err }) file.Hashes = uploadHash.Sums() if check { + // Overwrite time with that in metadata if it is already specified + mtime, ok := metadata["mtime"] + if ok { + modTime, err := time.Parse(time.RFC3339Nano, mtime) + require.NoError(t, err) + file.ModTime = modTime + } file.Check(t, obj, f.Precision()) // Re-read the object and check again obj = findObject(ctx, t, f, file.Path) @@ -209,7 +230,7 @@ func putTestContentsMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *f // PutTestContents puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove func PutTestContents(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool) fs.Object { - return putTestContentsMimeType(ctx, t, f, file, contents, check, "") + return PutTestContentsMetadata(ctx, t, f, file, contents, check, "", nil) } // testPut puts file with random contents to the remote @@ -219,9 +240,9 @@ func testPut(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) (str } // testPutMimeType puts file with random contents to the remote and the mime type given -func testPutMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, mimeType string) (string, fs.Object) { +func testPutMimeType(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, mimeType string, metadata fs.Metadata) (string, fs.Object) { contents := random.String(100) - return contents, putTestContentsMimeType(ctx, t, f, file, contents, true, mimeType) + return contents, PutTestContentsMetadata(ctx, t, f, file, contents, true, mimeType, metadata) } // TestPutLarge puts file to the remote, checks it and removes it on success. @@ -350,6 +371,7 @@ func Run(t *testing.T, opt *Opt) { } file1Contents string file1MimeType = "text/csv" + file1Metadata = fs.Metadata{"rclone-test": "potato"} file2 = fstest.Item{ ModTime: fstest.Time("2001-02-03T04:05:10.123123123Z"), Path: `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`, @@ -868,7 +890,7 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) file1Contents, _ = testPut(ctx, t, f, &file1) /* file2Contents = */ testPut(ctx, t, f, &file2) - file1Contents, _ = testPutMimeType(ctx, t, f, &file1, file1MimeType) + file1Contents, _ = testPutMimeType(ctx, t, f, &file1, file1MimeType, file1Metadata) // Note that the next test will check there are no duplicated file names // TestFsListDirFile2 tests the files are correctly uploaded by doing @@ -1357,6 +1379,76 @@ func Run(t *testing.T, opt *Opt) { } }) + // TestObjectMetadata tests the Metadata of the object is correct + t.Run("ObjectMetadata", func(t *testing.T) { + skipIfNotOk(t) + features := f.Features() + obj := findObject(ctx, t, f, file1.Path) + do, ok := obj.(fs.Metadataer) + if !ok { + require.False(t, features.ReadMetadata, "Features.ReadMetadata is set but Object.Metadata method not found") + t.Skip("Metadata method not supported") + } + metadata, err := do.Metadata(ctx) + require.NoError(t, err) + // check standard metadata + for k, v := range metadata { + switch k { + case "atime", "btime", "mtime": + mtime, err := time.Parse(time.RFC3339Nano, v) + require.NoError(t, err) + if k == "mtime" { + fstest.AssertTimeEqualWithPrecision(t, file1.Path, file1.ModTime, mtime, f.Precision()) + } + } + } + if !features.ReadMetadata { + if metadata != nil { + require.Equal(t, "", metadata, "Features.ReadMetadata is not set but Object.Metadata returned a non nil Metadata") + } + } else if features.WriteMetadata { + require.NotNil(t, metadata) + if features.UserMetadata { + // check all the metadata bits we uploaded are present - there may be more we didn't write + for k, v := range file1Metadata { + assert.Equal(t, v, metadata[k], "can read and write metadata but failed on key %q", k) + } + } + // Now test we can set the mtime and content-type via the metadata and these take precedence + t.Run("mtime", func(t *testing.T) { + path := "metadatatest" + mtimeModTime := fstest.Time("2002-02-03T04:05:06.499999999Z") + modTime := fstest.Time("2003-02-03T04:05:06.499999999Z") + item := fstest.NewItem(path, path, modTime) + metaMimeType := "application/zip" + mimeType := "application/gzip" + metadata := fs.Metadata{ + "mtime": mtimeModTime.Format(time.RFC3339Nano), + "content-type": metaMimeType, + } + // This checks the mtime is correct also and returns the re-read object + _, obj := testPutMimeType(ctx, t, f, &item, mimeType, metadata) + defer func() { + assert.NoError(t, obj.Remove(ctx)) + }() + // Check content-type got updated too + if features.ReadMimeType && features.WriteMimeType { + // read the object from scratch + o, err := f.NewObject(ctx, path) + require.NoError(t, err) + + // Check the mimetype is correct + do, ok := o.(fs.MimeTyper) + require.True(t, ok) + gotMimeType := do.MimeType(ctx) + assert.Equal(t, metaMimeType, gotMimeType) + } + }) + } else { + // Have some metadata here we didn't write - can't really check it! + } + }) + // TestObjectSetModTime tests that SetModTime works t.Run("ObjectSetModTime", func(t *testing.T) { skipIfNotOk(t) From 0652ec95db726f8895cb4aa402df04417439c270 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 22 Jun 2022 15:56:41 +0100 Subject: [PATCH 117/560] fs: implement MetadataInfo to show info about metadata in help and rc Info about this will appear in operations/fsinfo and in the backend help (`rclone help backend s3`). --- cmd/help.go | 25 +++++++ fs/metadata.go | 14 ++++ fs/operations/operations.go | 21 ++++-- fs/operations/rc.go | 137 +++++++++++++++++++++++++----------- fs/registry.go | 2 + fstest/fstests/fstests.go | 9 ++- 6 files changed, 160 insertions(+), 48 deletions(-) diff --git a/cmd/help.go b/cmd/help.go index db7e95da40e7f..b5f1c222707dd 100644 --- a/cmd/help.go +++ b/cmd/help.go @@ -6,6 +6,7 @@ import ( "log" "os" "regexp" + "sort" "strings" "github.com/rclone/rclone/fs" @@ -362,4 +363,28 @@ func showBackend(name string) { fmt.Printf("\n") } } + if backend.MetadataInfo != nil { + fmt.Printf("### Metadata\n\n") + fmt.Printf("%s\n\n", strings.TrimSpace(backend.MetadataInfo.Help)) + if len(backend.MetadataInfo.System) > 0 { + fmt.Printf("Here are the possible system metadata items for the %s backend.\n\n", backend.Name) + keys := []string{} + for k := range backend.MetadataInfo.System { + keys = append(keys, k) + } + sort.Strings(keys) + fmt.Printf("| Name | Help | Type | Example | Read Only |\n") + fmt.Printf("|------|------|------|---------|-----------|\n") + for _, k := range keys { + v := backend.MetadataInfo.System[k] + ro := "N" + if v.ReadOnly { + ro = "**Y**" + } + fmt.Printf("| %s | %s | %s | %s | %s |\n", k, v.Help, v.Type, v.Example, ro) + } + fmt.Printf("\n") + } + fmt.Printf("See the [metadata](/docs/#metadata) docs for more info.\n\n") + } } diff --git a/fs/metadata.go b/fs/metadata.go index dcdb517dc7d73..3df5a29f5d33b 100644 --- a/fs/metadata.go +++ b/fs/metadata.go @@ -7,6 +7,20 @@ import "context" // See docs/content/metadata.md for the interpretation of the keys type Metadata map[string]string +// MetadataHelp represents help for a bit of system metadata +type MetadataHelp struct { + Help string + Type string + Example string + ReadOnly bool +} + +// MetadataInfo is help for the whole metadata for this backend. +type MetadataInfo struct { + System map[string]MetadataHelp + Help string +} + // Set k to v on m // // If m is nil, then it will get made diff --git a/fs/operations/operations.go b/fs/operations/operations.go index a9c2b524c3582..aa69a0cbf692c 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -2278,21 +2278,30 @@ type FsInfo struct { // Features returns the optional features of this Fs Features map[string]bool + + // MetadataInfo returns info about the metadata for this backend + MetadataInfo *fs.MetadataInfo } // GetFsInfo gets the information (FsInfo) about a given Fs func GetFsInfo(f fs.Fs) *FsInfo { + features := f.Features() info := &FsInfo{ - Name: f.Name(), - Root: f.Root(), - String: f.String(), - Precision: f.Precision(), - Hashes: make([]string, 0, 4), - Features: f.Features().Enabled(), + Name: f.Name(), + Root: f.Root(), + String: f.String(), + Precision: f.Precision(), + Hashes: make([]string, 0, 4), + Features: features.Enabled(), + MetadataInfo: nil, } for _, hashType := range f.Hashes().Array() { info.Hashes = append(info.Hashes, hashType.String()) } + fsInfo, _, _, _, err := fs.ParseRemote(fs.ConfigString(f)) + if err == nil && fsInfo != nil && fsInfo.MetadataInfo != nil { + info.MetadataInfo = fsInfo.MetadataInfo + } return info } diff --git a/fs/operations/rc.go b/fs/operations/rc.go index b8c59ce2d8d87..bfa7e2599133d 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -413,46 +413,103 @@ This returns info about the remote passed in; ` + "```" + ` { - // optional features and whether they are available or not - "Features": { - "About": true, - "BucketBased": false, - "CanHaveEmptyDirectories": true, - "CaseInsensitive": false, - "ChangeNotify": false, - "CleanUp": false, - "Copy": false, - "DirCacheFlush": false, - "DirMove": true, - "DuplicateFiles": false, - "GetTier": false, - "ListR": false, - "MergeDirs": false, - "Move": true, - "OpenWriterAt": true, - "PublicLink": false, - "Purge": true, - "PutStream": true, - "PutUnchecked": false, - "ReadMimeType": false, - "ServerSideAcrossConfigs": false, - "SetTier": false, - "SetWrapper": false, - "UnWrap": false, - "WrapFs": false, - "WriteMimeType": false - }, - // Names of hashes available - "Hashes": [ - "MD5", - "SHA-1", - "DropboxHash", - "QuickXorHash" - ], - "Name": "local", // Name as created - "Precision": 1, // Precision of timestamps in ns - "Root": "/", // Path as created - "String": "Local file system at /" // how the remote will appear in logs + // optional features and whether they are available or not + "Features": { + "About": true, + "BucketBased": false, + "BucketBasedRootOK": false, + "CanHaveEmptyDirectories": true, + "CaseInsensitive": false, + "ChangeNotify": false, + "CleanUp": false, + "Command": true, + "Copy": false, + "DirCacheFlush": false, + "DirMove": true, + "Disconnect": false, + "DuplicateFiles": false, + "GetTier": false, + "IsLocal": true, + "ListR": false, + "MergeDirs": false, + "MetadataInfo": true, + "Move": true, + "OpenWriterAt": true, + "PublicLink": false, + "Purge": true, + "PutStream": true, + "PutUnchecked": false, + "ReadMetadata": true, + "ReadMimeType": false, + "ServerSideAcrossConfigs": false, + "SetTier": false, + "SetWrapper": false, + "Shutdown": false, + "SlowHash": true, + "SlowModTime": false, + "UnWrap": false, + "UserInfo": false, + "UserMetadata": true, + "WrapFs": false, + "WriteMetadata": true, + "WriteMimeType": false + }, + // Names of hashes available + "Hashes": [ + "md5", + "sha1", + "whirlpool", + "crc32", + "sha256", + "dropbox", + "mailru", + "quickxor" + ], + "Name": "local", // Name as created + "Precision": 1, // Precision of timestamps in ns + "Root": "/", // Path as created + "String": "Local file system at /", // how the remote will appear in logs + // Information about the system metadata for this backend + "MetadataInfo": { + "System": { + "atime": { + "Help": "Time of last access", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "btime": { + "Help": "Time of file birth (creation)", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "gid": { + "Help": "Group ID of owner", + "Type": "decimal number", + "Example": "500" + }, + "mode": { + "Help": "File type and mode", + "Type": "octal, unix style", + "Example": "0100664" + }, + "mtime": { + "Help": "Time of last modification", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "rdev": { + "Help": "Device ID (if special file)", + "Type": "hexadecimal", + "Example": "1abc" + }, + "uid": { + "Help": "User ID of owner", + "Type": "decimal number", + "Example": "500" + } + }, + "Help": "Textual help string\n" + } } ` + "```" + ` diff --git a/fs/registry.go b/fs/registry.go index 2f4afc7fe7c99..d380c7e0cb658 100644 --- a/fs/registry.go +++ b/fs/registry.go @@ -40,6 +40,8 @@ type RegInfo struct { Aliases []string // Hide - if set don't show in the configurator Hide bool + // MetadataInfo help about the metadata in use in this backend + MetadataInfo *MetadataInfo } // FileName returns the on disk file name for this backend diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 47b841bcaf327..5d24296a04c61 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -1384,8 +1384,13 @@ func Run(t *testing.T, opt *Opt) { skipIfNotOk(t) features := f.Features() obj := findObject(ctx, t, f, file1.Path) - do, ok := obj.(fs.Metadataer) - if !ok { + do, objectHasMetadata := obj.(fs.Metadataer) + if objectHasMetadata || features.ReadMetadata || features.WriteMetadata || features.UserMetadata { + fsInfo, _, _, _, err := fs.ParseRemote(fs.ConfigString(f)) + require.NoError(t, err) + require.NotNil(t, fsInfo.MetadataInfo, "Object declares metadata support but no MetadataInfo in RegInfo") + } + if !objectHasMetadata { require.False(t, features.ReadMetadata, "Features.ReadMetadata is set but Object.Metadata method not found") t.Skip("Metadata method not supported") } From c4451bc43a32a5117c0cbe5a96140c65ec6cf507 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 24 May 2022 15:46:07 +0100 Subject: [PATCH 118/560] fs: add --metadata-set flag to specify metadata for uploads --- docs/content/docs.md | 8 ++- fs/config.go | 1 + fs/config/configflags/configflags.go | 16 ++++++ fs/metadata.go | 36 +++++++++++++ fs/metadata_test.go | 79 ++++++++++++++++++++++++++++ fs/open_options.go | 18 +++++++ fs/open_options_test.go | 10 ++++ fs/operations/operations.go | 6 +++ 8 files changed, 173 insertions(+), 1 deletion(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index 7cfffe2b44d94..a2bc38b078249 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -452,7 +452,7 @@ attributes such as file mode, owner, extended attributes (not Windows). Note that arbitrary metadata may be added to objects using the -`--upload-metadata key=value` flag when the object is first uploaded. +`--metadata-set key=value` flag when the object is first uploaded. This flag can be repeated as many times as necessary. ### Types of metadata @@ -1332,6 +1332,12 @@ Setting this flag enables rclone to copy the metadata from the source to the destination. For local backends this is ownership, permissions, xattr etc. See the [#metadata](metadata section) for more info. +### --metadata-set key=value + +Add metadata `key` = `value` when uploading. This can be repeated as +many times as required. See the [#metadata](metadata section) for more +info. + ### --cutoff-mode=hard|soft|cautious ### This modifies the behavior of `--max-transfer` diff --git a/fs/config.go b/fs/config.go index e96be37d0ec9a..dceb063bb91d4 100644 --- a/fs/config.go +++ b/fs/config.go @@ -124,6 +124,7 @@ type ConfigInfo struct { UploadHeaders []*HTTPOption DownloadHeaders []*HTTPOption Headers []*HTTPOption + MetadataSet Metadata // extra metadata to write when uploading RefreshTimes bool NoConsole bool TrafficClass uint8 diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 8e3e8d5584e6f..de9d08073097a 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -37,6 +37,7 @@ var ( uploadHeaders []string downloadHeaders []string headers []string + metadataSet []string ) // AddFlags adds the non filing system specific flags to the command @@ -129,6 +130,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions") flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions") flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions") + flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading") flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files") flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)") flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21") @@ -271,6 +273,20 @@ func SetFlags(ci *fs.ConfigInfo) { if len(headers) != 0 { ci.Headers = ParseHeaders(headers) } + if len(headers) != 0 { + ci.Headers = ParseHeaders(headers) + } + if len(metadataSet) != 0 { + ci.MetadataSet = make(fs.Metadata, len(metadataSet)) + for _, kv := range metadataSet { + equal := strings.IndexRune(kv, '=') + if equal < 0 { + log.Fatalf("Failed to parse '%s' as metadata key=value.", kv) + } + ci.MetadataSet[strings.ToLower(kv[:equal])] = kv[equal+1:] + } + fs.Debugf(nil, "MetadataUpload %v", ci.MetadataSet) + } if len(dscp) != 0 { if value, ok := parseDSCP(dscp); ok { ci.TrafficClass = value << 2 diff --git a/fs/metadata.go b/fs/metadata.go index 3df5a29f5d33b..50d0340ed05fd 100644 --- a/fs/metadata.go +++ b/fs/metadata.go @@ -31,6 +31,30 @@ func (m *Metadata) Set(k, v string) { (*m)[k] = v } +// Merge other into m +// +// If m is nil, then it will get made +func (m *Metadata) Merge(other Metadata) { + for k, v := range other { + if *m == nil { + *m = make(Metadata, len(other)) + } + (*m)[k] = v + } +} + +// MergeOptions gets any Metadata from the options passed in and +// stores it in m (which may be nil). +// +// If there is no m then metadata will be nil +func (m *Metadata) MergeOptions(options []OpenOption) { + for _, opt := range options { + if metadataOption, ok := opt.(MetadataOption); ok { + m.Merge(Metadata(metadataOption)) + } + } +} + // GetMetadata from an ObjectInfo // // If the object has no metadata then metadata will be nil @@ -41,3 +65,15 @@ func GetMetadata(ctx context.Context, o ObjectInfo) (metadata Metadata, err erro } return do.Metadata(ctx) } + +// GetMetadataOptions from an ObjectInfo and merge it with any in options +// +// If the object has no metadata then metadata will be nil +func GetMetadataOptions(ctx context.Context, o ObjectInfo, options []OpenOption) (metadata Metadata, err error) { + metadata, err = GetMetadata(ctx, o) + if err != nil { + return nil, err + } + metadata.MergeOptions(options) + return metadata, nil +} diff --git a/fs/metadata_test.go b/fs/metadata_test.go index bc72c3964d310..ba982cbc1d0ee 100644 --- a/fs/metadata_test.go +++ b/fs/metadata_test.go @@ -1,6 +1,7 @@ package fs import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -15,3 +16,81 @@ func TestMetadataSet(t *testing.T) { m.Set("key", "value2") assert.Equal(t, "value2", m["key"]) } + +func TestMetadataMerge(t *testing.T) { + for _, test := range []struct { + in Metadata + merge Metadata + want Metadata + }{ + { + in: Metadata{}, + merge: Metadata{}, + want: Metadata{}, + }, { + in: nil, + merge: nil, + want: nil, + }, { + in: nil, + merge: Metadata{}, + want: nil, + }, { + in: nil, + merge: Metadata{"a": "1", "b": "2"}, + want: Metadata{"a": "1", "b": "2"}, + }, { + in: Metadata{"a": "1", "b": "2"}, + merge: nil, + want: Metadata{"a": "1", "b": "2"}, + }, { + in: Metadata{"a": "1", "b": "2"}, + merge: Metadata{"b": "B", "c": "3"}, + want: Metadata{"a": "1", "b": "B", "c": "3"}, + }, + } { + what := fmt.Sprintf("in=%v, merge=%v", test.in, test.merge) + test.in.Merge(test.merge) + assert.Equal(t, test.want, test.in, what) + } +} + +func TestMetadataMergeOptions(t *testing.T) { + for _, test := range []struct { + in Metadata + opts []OpenOption + want Metadata + }{ + { + opts: []OpenOption{}, + want: nil, + }, { + opts: []OpenOption{&HTTPOption{}}, + want: nil, + }, { + opts: []OpenOption{MetadataOption{"a": "1", "b": "2"}}, + want: Metadata{"a": "1", "b": "2"}, + }, { + opts: []OpenOption{ + &HTTPOption{}, + MetadataOption{"a": "1", "b": "2"}, + MetadataOption{"b": "B", "c": "3"}, + &HTTPOption{}, + }, + want: Metadata{"a": "1", "b": "B", "c": "3"}, + }, { + in: Metadata{"a": "first", "z": "OK"}, + opts: []OpenOption{ + &HTTPOption{}, + MetadataOption{"a": "1", "b": "2"}, + MetadataOption{"b": "B", "c": "3"}, + &HTTPOption{}, + }, + want: Metadata{"a": "1", "b": "B", "c": "3", "z": "OK"}, + }, + } { + what := fmt.Sprintf("in=%v, opts=%v", test.in, test.opts) + test.in.MergeOptions(test.opts) + assert.Equal(t, test.want, test.in, what) + } +} diff --git a/fs/open_options.go b/fs/open_options.go index e357a9a6990d0..57daa132ab2c5 100644 --- a/fs/open_options.go +++ b/fs/open_options.go @@ -258,6 +258,24 @@ func (o NullOption) Mandatory() bool { return false } +// MetadataOption defines an Option which does nothing +type MetadataOption Metadata + +// Header formats the option as an http header +func (o MetadataOption) Header() (key string, value string) { + return "", "" +} + +// String formats the option into human-readable form +func (o MetadataOption) String() string { + return fmt.Sprintf("MetadataOption(%v)", Metadata(o)) +} + +// Mandatory returns whether the option must be parsed or can be ignored +func (o MetadataOption) Mandatory() bool { + return false +} + // OpenOptionAddHeaders adds each header found in options to the // headers map provided the key was non empty. func OpenOptionAddHeaders(options []OpenOption, headers map[string]string) { diff --git a/fs/open_options_test.go b/fs/open_options_test.go index 829f45c1974c4..37463adeff7eb 100644 --- a/fs/open_options_test.go +++ b/fs/open_options_test.go @@ -132,6 +132,16 @@ func TestNullOption(t *testing.T) { assert.Equal(t, false, opt.Mandatory()) } +func TestMetadataOption(t *testing.T) { + opt := MetadataOption{"onion": "ice cream"} + var _ OpenOption = opt // check interface + assert.Equal(t, "MetadataOption(map[onion:ice cream])", opt.String()) + key, value := opt.Header() + assert.Equal(t, "", key) + assert.Equal(t, "", value) + assert.Equal(t, false, opt.Mandatory()) +} + func TestFixRangeOptions(t *testing.T) { for _, test := range []struct { name string diff --git a/fs/operations/operations.go b/fs/operations/operations.go index aa69a0cbf692c..a41a0a009a160 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -489,6 +489,9 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj for _, option := range ci.UploadHeaders { options = append(options, option) } + if ci.MetadataSet != nil { + options = append(options, fs.MetadataOption(ci.MetadataSet)) + } if doUpdate { actionTaken = "Copied (replaced existing)" err = dst.Update(ctx, in, wrappedSrc, options...) @@ -1381,6 +1384,9 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, for _, option := range ci.UploadHeaders { options = append(options, option) } + if ci.MetadataSet != nil { + options = append(options, fs.MetadataOption(ci.MetadataSet)) + } compare := func(dst fs.Object) error { var sums map[hash.Type]string From 78d52882ca9fe3d35a6afe34d212281efb89a91e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 25 May 2022 12:18:49 +0100 Subject: [PATCH 119/560] fs: add --metadata/-M flag to control whether metadata is copied --- fs/config.go | 1 + fs/config/configflags/configflags.go | 1 + fs/metadata.go | 6 ++++++ fstest/fstests/fstests.go | 9 +++++++++ 4 files changed, 17 insertions(+) diff --git a/fs/config.go b/fs/config.go index dceb063bb91d4..a39779bae7b4d 100644 --- a/fs/config.go +++ b/fs/config.go @@ -134,6 +134,7 @@ type ConfigInfo struct { HumanReadable bool KvLockTime time.Duration // maximum time to keep key-value database locked by process DisableHTTPKeepAlives bool + Metadata bool } // NewConfig creates a new config with everything set to the default diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index de9d08073097a..61aa65f91fe97 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -140,6 +140,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &ci.HumanReadable, "human-readable", "", ci.HumanReadable, "Print numbers in a human-readable format, sizes with suffix Ki|Mi|Gi|Ti|Pi") flags.DurationVarP(flagSet, &ci.KvLockTime, "kv-lock-time", "", ci.KvLockTime, "Maximum time to keep key-value database locked by process") flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.") + flags.BoolVarP(flagSet, &ci.Metadata, "metadata", "M", ci.Metadata, "If set, preserve metadata when copying objects") } // ParseHeaders converts the strings passed in via the header flags into HTTPOptions diff --git a/fs/metadata.go b/fs/metadata.go index 50d0340ed05fd..1fa48dc77e76a 100644 --- a/fs/metadata.go +++ b/fs/metadata.go @@ -68,8 +68,14 @@ func GetMetadata(ctx context.Context, o ObjectInfo) (metadata Metadata, err erro // GetMetadataOptions from an ObjectInfo and merge it with any in options // +// If --metadata isn't in use it will return nil +// // If the object has no metadata then metadata will be nil func GetMetadataOptions(ctx context.Context, o ObjectInfo, options []OpenOption) (metadata Metadata, err error) { + ci := GetConfig(ctx) + if !ci.Metadata { + return nil, nil + } metadata, err = GetMetadata(ctx, o) if err != nil { return nil, err diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 5d24296a04c61..9cc95896eac41 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -206,6 +206,15 @@ func PutTestContentsMetadata(ctx context.Context, t *testing.T, f fs.Fs, file *f file.Size = int64(buf.Len()) obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil) if mimeType != "" || metadata != nil { + // force the --metadata flag on temporarily + if metadata != nil { + ci := fs.GetConfig(ctx) + previousMetadata := ci.Metadata + ci.Metadata = true + defer func() { + ci.Metadata = previousMetadata + }() + } obji = overrideMimeType(obji, mimeType, metadata) } obj, err = f.Put(ctx, in, obji) From d823a38ce57b22fc622946b5e6aaf07cb998c8be Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 24 May 2022 11:16:29 +0100 Subject: [PATCH 120/560] lsjson: add --metadata/-M flag Note that this removes the `-M` flag from `--encrypted` as it conflicted with the global flag and adds it to `--metadata`. --- cmd/lsjson/lsjson.go | 6 +++++- fs/operations/lsjson.go | 10 ++++++++++ fs/operations/lsjson_test.go | 13 +++++++++++++ fs/operations/rc.go | 1 + 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/cmd/lsjson/lsjson.go b/cmd/lsjson/lsjson.go index 70b96a0f76513..8e4e343da0f68 100644 --- a/cmd/lsjson/lsjson.go +++ b/cmd/lsjson/lsjson.go @@ -26,10 +26,11 @@ func init() { flags.BoolVarP(cmdFlags, &opt.ShowHash, "hash", "", false, "Include hashes in the output (may take longer)") flags.BoolVarP(cmdFlags, &opt.NoModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up)") flags.BoolVarP(cmdFlags, &opt.NoMimeType, "no-mimetype", "", false, "Don't read the mime type (can speed things up)") - flags.BoolVarP(cmdFlags, &opt.ShowEncrypted, "encrypted", "M", false, "Show the encrypted names") + flags.BoolVarP(cmdFlags, &opt.ShowEncrypted, "encrypted", "", false, "Show the encrypted names") flags.BoolVarP(cmdFlags, &opt.ShowOrigIDs, "original", "", false, "Show the ID of the underlying Object") flags.BoolVarP(cmdFlags, &opt.FilesOnly, "files-only", "", false, "Show only files in the listing") flags.BoolVarP(cmdFlags, &opt.DirsOnly, "dirs-only", "", false, "Show only directories in the listing") + flags.BoolVarP(cmdFlags, &opt.Metadata, "metadata", "M", false, "Add metadata to the listing") flags.StringArrayVarP(cmdFlags, &opt.HashTypes, "hash-type", "", nil, "Show only this hash type (may be repeated)") flags.BoolVarP(cmdFlags, &statOnly, "stat", "", false, "Just return the info for the pointed to file") } @@ -81,6 +82,9 @@ returned If ` + "`--files-only`" + ` is not specified directories in addition to the files will be returned. +If ` + "`--metadata`" + ` is set then an additional Metadata key will be returned. +This will have metdata in rclone standard format as a JSON object. + if ` + "`--stat`" + ` is set then a single JSON blob will be returned about the item pointed to. This will return an error if the item isn't found. However on bucket based backends (like s3, gcs, b2, azureblob etc) if diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.go index 144a47bb6e13b..067681956747e 100644 --- a/fs/operations/lsjson.go +++ b/fs/operations/lsjson.go @@ -29,6 +29,7 @@ type ListJSONItem struct { OrigID string `json:",omitempty"` Tier string `json:",omitempty"` IsBucket bool `json:",omitempty"` + Metadata fs.Metadata `json:",omitempty"` } // Timestamp a time in the provided format @@ -80,6 +81,7 @@ type ListJSONOpt struct { ShowHash bool `json:"showHash"` DirsOnly bool `json:"dirsOnly"` FilesOnly bool `json:"filesOnly"` + Metadata bool `json:"metadata"` HashTypes []string `json:"hashTypes"` // hash types to show if ShowHash is set, e.g. "MD5", "SHA-1" } @@ -222,6 +224,14 @@ func (lj *listJSON) entry(ctx context.Context, entry fs.DirEntry) (*ListJSONItem item.Tier = do.GetTier() } } + if lj.opt.Metadata { + metadata, err := fs.GetMetadata(ctx, x) + if err != nil { + fs.Errorf(x, "Failed to read metadata: %v", err) + } else if metadata != nil { + item.Metadata = metadata + } + } default: fs.Errorf(nil, "Unknown type %T in listing in ListJSON", entry) } diff --git a/fs/operations/lsjson_test.go b/fs/operations/lsjson_test.go index ebc2c3f479867..d5d7ca319c160 100644 --- a/fs/operations/lsjson_test.go +++ b/fs/operations/lsjson_test.go @@ -172,6 +172,19 @@ func TestListJSON(t *testing.T) { ModTime: operations.Timestamp{When: t1}, IsDir: false, }}, + }, { + name: "Metadata", + opt: operations.ListJSONOpt{ + FilesOnly: true, + Metadata: true, + }, + want: []*operations.ListJSONItem{{ + Path: "file1", + Name: "file1", + Size: 5, + ModTime: operations.Timestamp{When: t1}, + IsDir: false, + }}, }, } { t.Run(test.name, func(t *testing.T) { diff --git a/fs/operations/rc.go b/fs/operations/rc.go index bfa7e2599133d..39705b11958a5 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -34,6 +34,7 @@ func init() { - noMimeType - If set don't show mime types - dirsOnly - If set only show directories - filesOnly - If set only show files + - metadata - If set return metadata of objects also - hashTypes - array of strings of hash types to show if showHash set Returns: From 776a08389244335ad05d5b9a0825c1f4ade555e1 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 24 May 2022 11:31:48 +0100 Subject: [PATCH 121/560] lsf: add metadata support with `M` flag --- cmd/lsf/lsf.go | 4 ++++ fs/operations/operations.go | 16 ++++++++++++++++ fs/operations/operations_test.go | 5 +++++ 3 files changed, 25 insertions(+) diff --git a/cmd/lsf/lsf.go b/cmd/lsf/lsf.go index 732d2b5e8980a..e370f930c2b48 100644 --- a/cmd/lsf/lsf.go +++ b/cmd/lsf/lsf.go @@ -72,6 +72,7 @@ output: m - MimeType of object if known e - encrypted name T - tier of storage if known, e.g. "Hot" or "Cool" + M - Metadata of object in JSON blob format, eg {"key":"value"} So if you wanted the path, size and modification time, you would use ` + "`--format \"pst\"`, or maybe `--format \"tsp\"`" + ` to put the path last. @@ -198,6 +199,9 @@ func Lsf(ctx context.Context, fsrc fs.Fs, out io.Writer) error { opt.ShowOrigIDs = true case 'T': list.AddTier() + case 'M': + list.AddMetadata() + opt.Metadata = true default: return fmt.Errorf("unknown format character %q", char) } diff --git a/fs/operations/operations.go b/fs/operations/operations.go index a41a0a009a160..1ea41853fbe6d 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "encoding/csv" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -2154,6 +2155,21 @@ func (l *ListFormat) AddMimeType() { }) } +// AddMetadata adds file's Metadata to the output if known +func (l *ListFormat) AddMetadata() { + l.AppendOutput(func(entry *ListJSONItem) string { + metadata := entry.Metadata + if metadata == nil { + metadata = make(fs.Metadata) + } + out, err := json.Marshal(metadata) + if err != nil { + return fmt.Sprintf("Failed to read metadata: %v", err.Error()) + } + return string(out) + }) +} + // AppendOutput adds string generated by specific function to printed output func (l *ListFormat) AppendOutput(functionToAppend func(item *ListJSONItem) string) { l.output = append(l.output, functionToAppend) diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 4b65f689409cb..8cd7296e4c2b2 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1428,6 +1428,11 @@ func TestListFormat(t *testing.T) { assert.Contains(t, list.Format(item0), "/") assert.Equal(t, "inode/directory", list.Format(item1)) + list.SetOutput(nil) + list.AddMetadata() + assert.Equal(t, "{}", list.Format(item0)) + assert.Equal(t, "{}", list.Format(item1)) + list.SetOutput(nil) list.AddPath() list.SetAbsolute(true) From a692bd2cd45d5f4ae8799808bc4e2267c96ec94c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 22 Jun 2022 15:40:30 +0100 Subject: [PATCH 122/560] s3: change metadata storage to normal map with lowercase keys --- backend/s3/s3.go | 58 +++++++++++++++++++++++++++++++----------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index c5dfbcb74c25d..7a28982640fad 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2072,14 +2072,14 @@ type Object struct { // // List will read everything but meta & mimeType - to fill // that in you need to call readMetaData - fs *Fs // what this object is part of - remote string // The remote path - md5 string // md5sum of the object - bytes int64 // size of the object - lastModified time.Time // Last modified - meta map[string]*string // The object metadata if known - may be nil - mimeType string // MimeType of object - may be "" - storageClass string // e.g. GLACIER + fs *Fs // what this object is part of + remote string // The remote path + md5 string // md5sum of the object + bytes int64 // size of the object + lastModified time.Time // Last modified + meta map[string]string // The object metadata if known - may be nil - with lower case keys + mimeType string // MimeType of object - may be "" + storageClass string // e.g. GLACIER } // ------------------------------------------------------------ @@ -3753,6 +3753,27 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { return nil } +// Convert S3 metadata with pointers into a map[string]string +// while lowercasing the keys +func s3MetadataToMap(s3Meta map[string]*string) map[string]string { + meta := make(map[string]string, len(s3Meta)) + for k, v := range s3Meta { + if v != nil { + meta[strings.ToLower(k)] = *v + } + } + return meta +} + +// Convert our metadata back into S3 metadata +func mapToS3Metadata(meta map[string]string) map[string]*string { + s3Meta := make(map[string]*string, len(meta)) + for k, v := range meta { + s3Meta[k] = aws.String(v) + } + return s3Meta +} + func (o *Object) setMetaData(etag *string, contentLength *int64, lastModified *time.Time, meta map[string]*string, mimeType *string, storageClass *string) { // Ignore missing Content-Length assuming it is 0 // Some versions of ceph do this due their apache proxies @@ -3760,17 +3781,14 @@ func (o *Object) setMetaData(etag *string, contentLength *int64, lastModified *t o.bytes = *contentLength } o.setMD5FromEtag(aws.StringValue(etag)) - o.meta = meta - if o.meta == nil { - o.meta = map[string]*string{} - } + o.meta = s3MetadataToMap(meta) // Read MD5 from metadata if present if md5sumBase64, ok := o.meta[metaMD5Hash]; ok { - md5sumBytes, err := base64.StdEncoding.DecodeString(*md5sumBase64) + md5sumBytes, err := base64.StdEncoding.DecodeString(md5sumBase64) if err != nil { - fs.Debugf(o, "Failed to read md5sum from metadata %q: %v", *md5sumBase64, err) + fs.Debugf(o, "Failed to read md5sum from metadata %q: %v", md5sumBase64, err) } else if len(md5sumBytes) != 16 { - fs.Debugf(o, "Failed to read md5sum from metadata %q: wrong length", *md5sumBase64) + fs.Debugf(o, "Failed to read md5sum from metadata %q: wrong length", md5sumBase64) } else { o.md5 = hex.EncodeToString(md5sumBytes) } @@ -3800,11 +3818,11 @@ func (o *Object) ModTime(ctx context.Context) time.Time { } // read mtime out of metadata if available d, ok := o.meta[metaMtime] - if !ok || d == nil { + if !ok { // fs.Debugf(o, "No metadata") return o.lastModified } - modTime, err := swift.FloatStringToTime(*d) + modTime, err := swift.FloatStringToTime(d) if err != nil { fs.Logf(o, "Failed to read mtime from object: %v", err) return o.lastModified @@ -3818,7 +3836,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { if err != nil { return err } - o.meta[metaMtime] = aws.String(swift.TimeToFloatString(modTime)) + o.meta[metaMtime] = swift.TimeToFloatString(modTime) // Can't update metadata here, so return this error to force a recopy if o.storageClass == "GLACIER" || o.storageClass == "DEEP_ARCHIVE" { @@ -3829,7 +3847,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { bucket, bucketPath := o.split() req := s3.CopyObjectInput{ ContentType: aws.String(fs.MimeType(ctx, o)), // Guess the content type - Metadata: o.meta, + Metadata: mapToS3Metadata(o.meta), MetadataDirective: aws.String(s3.MetadataDirectiveReplace), // replace metadata with that passed in } if o.fs.opt.RequesterPays { @@ -4423,7 +4441,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op o.md5 = md5sumHex o.bytes = size o.lastModified = time.Now() - o.meta = req.Metadata + o.meta = s3MetadataToMap(req.Metadata) o.mimeType = aws.StringValue(req.ContentType) o.storageClass = aws.StringValue(req.StorageClass) // If we have done a single part PUT request then we can read these From 22abd785eba35ca29cfcd6c162ca46e933344a1a Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 24 May 2022 12:32:39 +0100 Subject: [PATCH 123/560] s3: implement reading and writing of metadata #111 --- backend/s3/s3.go | 275 +++++++++++++++++++++++++++------ backend/s3/s3_internal_test.go | 61 ++++++++ docs/content/s3.md | 20 +++ 3 files changed, 307 insertions(+), 49 deletions(-) create mode 100644 backend/s3/s3_internal_test.go diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 7a28982640fad..1693446b08640 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -71,6 +71,10 @@ func init() { } return nil, fmt.Errorf("unknown state %q", config.State) }, + MetadataInfo: &fs.MetadataInfo{ + System: systemMetadataInfo, + Help: `User metadata is stored as x-amz-meta- keys. S3 metadata keys are case insensitive and are always returned in lower case.`, + }, Options: []fs.Option{{ Name: fs.ConfigProvider, Help: "Choose your S3 provider.", @@ -1984,8 +1988,8 @@ circumstances or for testing. // Constants const ( - metaMtime = "Mtime" // the meta key to store mtime in - e.g. X-Amz-Meta-Mtime - metaMD5Hash = "Md5chksum" // the meta key to store md5hash in + metaMtime = "mtime" // the meta key to store mtime in - e.g. X-Amz-Meta-Mtime + metaMD5Hash = "md5chksum" // the meta key to store md5hash in // The maximum size of object we can COPY - this should be 5 GiB but is < 5 GB for b2 compatibility // See https://forum.rclone.org/t/copying-files-within-a-b2-bucket/16680/76 maxSizeForCopy = 4768 * 1024 * 1024 @@ -2000,6 +2004,57 @@ const ( maxExpireDuration = fs.Duration(7 * 24 * time.Hour) // max expiry is 1 week ) +// system metadata keys which this backend owns +var systemMetadataInfo = map[string]fs.MetadataHelp{ + "cache-control": { + Help: "Cache-Control header", + Type: "string", + Example: "no-cache", + }, + "content-disposition": { + Help: "Content-Disposition header", + Type: "string", + Example: "inline", + }, + "content-encoding": { + Help: "Content-Encoding header", + Type: "string", + Example: "gzip", + }, + "content-language": { + Help: "Content-Language header", + Type: "string", + Example: "en-US", + }, + "content-type": { + Help: "Content-Type header", + Type: "string", + Example: "text/plain", + }, + // "tagging": { + // Help: "x-amz-tagging header", + // Type: "string", + // Example: "tag1=value1&tag2=value2", + // }, + "tier": { + Help: "Tier of the object", + Type: "string", + Example: "GLACIER", + ReadOnly: true, + }, + "mtime": { + Help: "Time of last modification, read from rclone metadata", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999999999Z07:00", + }, + "btime": { + Help: "Time of file birth (creation) read from Last-Modified header", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999999999Z07:00", + ReadOnly: true, + }, +} + // Options defines the configuration for this backend type Options struct { Provider string `config:"provider"` @@ -2079,7 +2134,13 @@ type Object struct { lastModified time.Time // Last modified meta map[string]string // The object metadata if known - may be nil - with lower case keys mimeType string // MimeType of object - may be "" - storageClass string // e.g. GLACIER + + // Metadata as pointers to strings as they often won't be present + storageClass *string // e.g. GLACIER + cacheControl *string // Cache-Control: header + contentDisposition *string // Content-Disposition: header + contentEncoding *string // Content-Encoding: header + contentLanguage *string // Content-Language: header } // ------------------------------------------------------------ @@ -2573,6 +2634,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e f.features = (&fs.Features{ ReadMimeType: true, WriteMimeType: true, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, BucketBased: true, BucketBasedRootOK: true, SetTier: true, @@ -2623,7 +2687,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje } o.setMD5FromEtag(aws.StringValue(info.ETag)) o.bytes = aws.Int64Value(info.Size) - o.storageClass = aws.StringValue(info.StorageClass) + o.storageClass = info.StorageClass } else if !o.fs.opt.NoHeadObject { err := o.readMetaData(ctx) // reads info and meta, returning an error if err != nil { @@ -3491,7 +3555,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str st.Status = "Not an S3 object" return } - if o.storageClass != "GLACIER" && o.storageClass != "DEEP_ARCHIVE" { + if o.storageClass == nil || (*o.storageClass != "GLACIER" && *o.storageClass != "DEEP_ARCHIVE") { st.Status = "Not GLACIER or DEEP_ARCHIVE storage class" return } @@ -3749,7 +3813,8 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { if err != nil { return err } - o.setMetaData(resp.ETag, resp.ContentLength, resp.LastModified, resp.Metadata, resp.ContentType, resp.StorageClass) + o.setMetaData(resp) + // resp.ETag, resp.ContentLength, resp.LastModified, resp.Metadata, resp.ContentType, resp.StorageClass) return nil } @@ -3774,14 +3839,14 @@ func mapToS3Metadata(meta map[string]string) map[string]*string { return s3Meta } -func (o *Object) setMetaData(etag *string, contentLength *int64, lastModified *time.Time, meta map[string]*string, mimeType *string, storageClass *string) { +func (o *Object) setMetaData(resp *s3.HeadObjectOutput) { // Ignore missing Content-Length assuming it is 0 // Some versions of ceph do this due their apache proxies - if contentLength != nil { - o.bytes = *contentLength + if resp.ContentLength != nil { + o.bytes = *resp.ContentLength } - o.setMD5FromEtag(aws.StringValue(etag)) - o.meta = s3MetadataToMap(meta) + o.setMD5FromEtag(aws.StringValue(resp.ETag)) + o.meta = s3MetadataToMap(resp.Metadata) // Read MD5 from metadata if present if md5sumBase64, ok := o.meta[metaMD5Hash]; ok { md5sumBytes, err := base64.StdEncoding.DecodeString(md5sumBase64) @@ -3793,14 +3858,20 @@ func (o *Object) setMetaData(etag *string, contentLength *int64, lastModified *t o.md5 = hex.EncodeToString(md5sumBytes) } } - o.storageClass = aws.StringValue(storageClass) - if lastModified == nil { + if resp.LastModified == nil { o.lastModified = time.Now() fs.Logf(o, "Failed to read last modified") } else { - o.lastModified = *lastModified + o.lastModified = *resp.LastModified } - o.mimeType = aws.StringValue(mimeType) + o.mimeType = aws.StringValue(resp.ContentType) + + // Set system metadata + o.storageClass = resp.StorageClass + o.cacheControl = resp.CacheControl + o.contentDisposition = resp.ContentDisposition + o.contentEncoding = resp.ContentEncoding + o.contentLanguage = resp.ContentLanguage } // ModTime returns the modification time of the object @@ -3839,7 +3910,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { o.meta[metaMtime] = swift.TimeToFloatString(modTime) // Can't update metadata here, so return this error to force a recopy - if o.storageClass == "GLACIER" || o.storageClass == "DEEP_ARCHIVE" { + if o.storageClass != nil && (*o.storageClass == "GLACIER" || *o.storageClass == "DEEP_ARCHIVE") { return fs.ErrorCantSetModTime } @@ -3900,17 +3971,34 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options metaData := make(map[string]*string) for key, value := range resp.Header { - if strings.HasPrefix(key, "x-amz-meta") { + key = strings.ToLower(key) + if strings.HasPrefix(key, "x-amz-meta-") { metaKey := strings.TrimPrefix(key, "x-amz-meta-") - metaData[strings.Title(metaKey)] = &value[0] + metaData[metaKey] = &value[0] } } - storageClass := resp.Header.Get("X-Amz-Storage-Class") - contentType := resp.Header.Get("Content-Type") - etag := resp.Header.Get("Etag") - - o.setMetaData(&etag, contentLength, &lastModified, metaData, &contentType, &storageClass) + header := func(k string) *string { + v := resp.Header.Get(k) + if v == "" { + return nil + } + return &v + } + + var head = s3.HeadObjectOutput{ + ETag: header("Etag"), + ContentLength: contentLength, + LastModified: &lastModified, + Metadata: metaData, + CacheControl: header("Cache-Control"), + ContentDisposition: header("Content-Disposition"), + ContentEncoding: header("Content-Encoding"), + ContentLanguage: header("Content-Language"), + ContentType: header("Content-Type"), + StorageClass: header("X-Amz-Storage-Class"), + } + o.setMetaData(&head) return resp.Body, err } @@ -3985,7 +4073,10 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read fs.Debugf(o, "Failed to find length in %q", contentRange) } } - o.setMetaData(resp.ETag, size, resp.LastModified, resp.Metadata, resp.ContentType, resp.StorageClass) + var head s3.HeadObjectOutput + structs.SetFrom(&head, resp) + head.ContentLength = size + o.setMetaData(&head) return resp.Body, nil } @@ -4322,11 +4413,56 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff) - // Set the mtime in the meta data - metadata := map[string]*string{ - metaMtime: aws.String(swift.TimeToFloatString(modTime)), + req := s3.PutObjectInput{ + Bucket: &bucket, + ACL: &o.fs.opt.ACL, + Key: &bucketPath, + } + + // Fetch metadata if --metadata is in use + meta, err := fs.GetMetadataOptions(ctx, src, options) + if err != nil { + return fmt.Errorf("failed to read metadata from source object: %w", err) + } + req.Metadata = make(map[string]*string, len(meta)+2) + // merge metadata into request and user metadata + for k, v := range meta { + pv := aws.String(v) + k = strings.ToLower(k) + switch k { + case "cache-control": + req.CacheControl = pv + case "content-disposition": + req.ContentDisposition = pv + case "content-encoding": + req.ContentEncoding = pv + case "content-language": + req.ContentLanguage = pv + case "content-type": + req.ContentType = pv + case "x-amz-tagging": + req.Tagging = pv + case "tier": + // ignore + case "mtime": + // mtime in meta overrides source ModTime + metaModTime, err := time.Parse(time.RFC3339Nano, v) + if err != nil { + fs.Debugf(o, "failed to parse metadata %s: %q: %v", k, v, err) + } else { + modTime = metaModTime + } + case "btime": + // write as metadata since we can't set it + req.Metadata[k] = pv + default: + req.Metadata[k] = pv + } } + // Set the mtime in the meta data + req.Metadata[metaMtime] = aws.String(swift.TimeToFloatString(modTime)) + // read the md5sum if available // - for non multipart // - so we can add a ContentMD5 @@ -4346,20 +4482,15 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // - a multipart upload // - the Etag is not an MD5, eg when using SSE/SSE-C // provided checksums aren't disabled - metadata[metaMD5Hash] = &md5sumBase64 + req.Metadata[metaMD5Hash] = &md5sumBase64 } } } } - // Guess the content type - mimeType := fs.MimeType(ctx, src) - req := s3.PutObjectInput{ - Bucket: &bucket, - ACL: &o.fs.opt.ACL, - Key: &bucketPath, - ContentType: &mimeType, - Metadata: metadata, + // Set the content type it it isn't set already + if req.ContentType == nil { + req.ContentType = aws.String(fs.MimeType(ctx, src)) } if size >= 0 { req.ContentLength = &size @@ -4438,19 +4569,19 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // so make up the object as best we can assuming it got // uploaded properly. If size < 0 then we need to do the HEAD. if o.fs.opt.NoHead && size >= 0 { - o.md5 = md5sumHex - o.bytes = size - o.lastModified = time.Now() - o.meta = s3MetadataToMap(req.Metadata) - o.mimeType = aws.StringValue(req.ContentType) - o.storageClass = aws.StringValue(req.StorageClass) + var head s3.HeadObjectOutput + structs.SetFrom(&head, req) + head.ETag = &md5sumHex // doesn't matter quotes are misssing + head.ContentLength = &size // If we have done a single part PUT request then we can read these if gotEtag != "" { - o.setMD5FromEtag(gotEtag) + head.ETag = &gotEtag } - if !o.lastModified.IsZero() { - o.lastModified = lastModified + if lastModified.IsZero() { + lastModified = time.Now() } + head.LastModified = &lastModified + o.setMetaData(&head) return nil } @@ -4460,7 +4591,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op if err != nil { return err } - o.setMetaData(head.ETag, head.ContentLength, head.LastModified, head.Metadata, head.ContentType, head.StorageClass) + o.setMetaData(head) if o.fs.opt.UseMultipartEtag.Value && !o.fs.etagIsNotMD5 && wantETag != "" && head.ETag != nil && *head.ETag != "" { gotETag := strings.Trim(strings.ToLower(*head.ETag), `"`) if wantETag != gotETag { @@ -4511,16 +4642,61 @@ func (o *Object) SetTier(tier string) (err error) { if err != nil { return err } - o.storageClass = tier + o.storageClass = &tier return err } // GetTier returns storage class as string func (o *Object) GetTier() string { - if o.storageClass == "" { + if o.storageClass == nil || *o.storageClass == "" { return "STANDARD" } - return o.storageClass + return *o.storageClass +} + +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) { + err = o.readMetaData(ctx) + if err != nil { + return nil, err + } + metadata = make(fs.Metadata, len(o.meta)+7) + for k, v := range o.meta { + switch k { + case metaMtime: + if modTime, err := swift.FloatStringToTime(v); err == nil { + metadata["mtime"] = modTime.Format(time.RFC3339Nano) + } + case metaMD5Hash: + // don't write hash metadata + default: + metadata[k] = v + } + } + if o.mimeType != "" { + metadata["content-type"] = o.mimeType + } + // metadata["x-amz-tagging"] = "" + if !o.lastModified.IsZero() { + metadata["btime"] = o.lastModified.Format(time.RFC3339Nano) + } + + // Set system metadata + setMetadata := func(k string, v *string) { + if v == nil || *v == "" { + return + } + metadata[k] = *v + } + setMetadata("cache-control", o.cacheControl) + setMetadata("content-disposition", o.contentDisposition) + setMetadata("content-encoding", o.contentEncoding) + setMetadata("content-language", o.contentLanguage) + setMetadata("tier", o.storageClass) + + return metadata, nil } // Check the interfaces are satisfied @@ -4535,4 +4711,5 @@ var ( _ fs.MimeTyper = &Object{} _ fs.GetTierer = &Object{} _ fs.SetTierer = &Object{} + _ fs.Metadataer = &Object{} ) diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go new file mode 100644 index 0000000000000..ecd39f41f3ec8 --- /dev/null +++ b/backend/s3/s3_internal_test.go @@ -0,0 +1,61 @@ +package s3 + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/fstest/fstests" + "github.com/rclone/rclone/lib/random" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func (f *Fs) InternalTestMetadata(t *testing.T) { + ctx := context.Background() + contents := random.String(100) + item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) + btime := time.Now() + metadata := fs.Metadata{ + "cache-control": "no-cache", + "content-disposition": "inline", + "content-encoding": "gzip", + "content-language": "en-US", + "content-type": "text/plain", + "mtime": "2009-05-06T04:05:06.499999999Z", + // "tier" - read only + // "btime" - read only + } + obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, "text/html", metadata) + defer func() { + assert.NoError(t, obj.Remove(ctx)) + }() + o := obj.(*Object) + gotMetadata, err := o.Metadata(ctx) + require.NoError(t, err) + for k, v := range metadata { + got := gotMetadata[k] + switch k { + case "mtime": + assert.True(t, fstest.Time(v).Equal(fstest.Time(got))) + case "btime": + gotBtime := fstest.Time(got) + dt := gotBtime.Sub(btime) + assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("btime more than 1 minute out want %v got %v delta %v", btime, gotBtime, dt)) + assert.True(t, fstest.Time(v).Equal(fstest.Time(got))) + case "tier": + assert.NotEqual(t, "", got) + default: + assert.Equal(t, v, got, k) + } + } +} + +func (f *Fs) InternalTest(t *testing.T) { + t.Run("Metadata", f.InternalTestMetadata) +} + +var _ fstests.InternalTester = (*Fs)(nil) diff --git a/docs/content/s3.md b/docs/content/s3.md index c2adbfe95222c..ddf9fd23dec58 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -2369,6 +2369,26 @@ Properties: - Type: Tristate - Default: unset +### Metadata + +User metadata is stored as x-amz-meta- keys. S3 metadata keys are case insensitive and are always returned in lower case. + +Here are the possible system metadata items for the s3 backend. + +| Name | Help | Type | Example | Read Only | +|------|------|------|---------|-----------| +| btime | Time of file birth (creation) read from Last-Modified header | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | **Y** | +| cache-control | Cache-Control header | string | no-cache | N | +| content-disposition | Content-Disposition header | string | inline | N | +| content-encoding | Content-Encoding header | string | gzip | N | +| content-language | Content-Language header | string | en-US | N | +| content-type | Content-Type header | string | text/plain | N | +| mtime | Time of last modification, read from rclone metadata | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N | +| tier | Tier of the object | string | GLACIER | **Y** | + + +See the [metadata](/docs/#metadata) docs for more info. + ## Backend commands Here are the commands specific to the s3 backend. From c556e98f496c0735fec487b4752e0c458420f331 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 24 May 2022 18:06:16 +0100 Subject: [PATCH 124/560] local: add Metadata support #111 --- backend/local/local.go | 71 ++++++++++++-- backend/local/local_internal_test.go | 137 ++++++++++++++++++++++++++ backend/local/metadata.go | 138 +++++++++++++++++++++++++++ backend/local/metadata_bsd.go | 38 ++++++++ backend/local/metadata_linux.go | 47 +++++++++ backend/local/metadata_other.go | 21 ++++ backend/local/metadata_unix.go | 37 +++++++ backend/local/metadata_windows.go | 34 +++++++ backend/local/setbtime.go | 16 ++++ backend/local/setbtime_windows.go | 28 ++++++ backend/local/xattr.go | 87 +++++++++++++++++ backend/local/xattr_unsupported.go | 21 ++++ docs/content/local.md | 26 +++++ go.mod | 1 + go.sum | 3 + 15 files changed, 699 insertions(+), 6 deletions(-) create mode 100644 backend/local/metadata.go create mode 100644 backend/local/metadata_bsd.go create mode 100644 backend/local/metadata_linux.go create mode 100644 backend/local/metadata_other.go create mode 100644 backend/local/metadata_unix.go create mode 100644 backend/local/metadata_windows.go create mode 100644 backend/local/setbtime.go create mode 100644 backend/local/setbtime_windows.go create mode 100644 backend/local/xattr.go create mode 100644 backend/local/xattr_unsupported.go diff --git a/backend/local/local.go b/backend/local/local.go index 1a8908720c4f0..3a83bd2623442 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -42,6 +42,18 @@ func init() { Description: "Local Disk", NewFs: NewFs, CommandHelp: commandHelp, + MetadataInfo: &fs.MetadataInfo{ + System: systemMetadataInfo, + Help: `Depending on which OS is in use the local backend may return only some +of the system metadata. Setting system metadata is supported on all +OSes but setting user metadata is only supported on linux, freebsd, +netbsd, macOS and Solaris. It is **not** supported on Windows yet +([see pkg/attrs#47](https://github.com/pkg/xattr/issues/47)). + +User metadata is stored as extended attributes (which may not be +supported by all file systems) under the "user.*" prefix. +`, + }, Options: []fs.Option{{ Name: "nounc", Help: "Disable UNC (long path names) conversion on Windows.", @@ -280,6 +292,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e CanHaveEmptyDirectories: true, IsLocal: true, SlowHash: true, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: xattrSupported, // can only R/W general purpose metadata if xattrs are supported }).Fill(ctx, f) if opt.FollowSymlinks { f.lstat = os.Stat @@ -938,17 +953,22 @@ func (o *Object) ModTime(ctx context.Context) time.Time { return o.modTime } +// Set the atime and ltime of the object +func (o *Object) setTimes(atime, mtime time.Time) (err error) { + if o.translatedLink { + err = lChtimes(o.path, atime, mtime) + } else { + err = os.Chtimes(o.path, atime, mtime) + } + return err +} + // SetModTime sets the modification time of the local fs object func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { if o.fs.opt.NoSetModTime { return nil } - var err error - if o.translatedLink { - err = lChtimes(o.path, modTime, modTime) - } else { - err = os.Chtimes(o.path, modTime, modTime) - } + err := o.setTimes(modTime, modTime) if err != nil { return err } @@ -1223,6 +1243,16 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return err } + // Fetch and set metadata if --metadata is in use + meta, err := fs.GetMetadataOptions(ctx, src, options) + if err != nil { + return fmt.Errorf("failed to read metadata from source object: %w", err) + } + err = o.writeMetadata(meta) + if err != nil { + return fmt.Errorf("failed to set metadata: %w", err) + } + // ReRead info now that we have finished return o.lstat() } @@ -1321,6 +1351,34 @@ func (o *Object) Remove(ctx context.Context) error { return remove(o.path) } +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) { + metadata, err = o.getXattr() + if err != nil { + return nil, err + } + err = o.readMetadataFromFile(&metadata) + if err != nil { + return nil, err + } + return metadata, nil +} + +// Write the metadata on the object +func (o *Object) writeMetadata(metadata fs.Metadata) (err error) { + err = o.setXattr(metadata) + if err != nil { + return err + } + err = o.writeMetadataToFile(metadata) + if err != nil { + return err + } + return err +} + func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string { if runtime.GOOS == "windows" { if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") { @@ -1360,4 +1418,5 @@ var ( _ fs.Commander = &Fs{} _ fs.OpenWriterAter = &Fs{} _ fs.Object = &Object{} + _ fs.Metadataer = &Object{} ) diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go index 2957dcd1cfd55..55b98bb53728b 100644 --- a/backend/local/local_internal_test.go +++ b/backend/local/local_internal_test.go @@ -3,10 +3,12 @@ package local import ( "bytes" "context" + "fmt" "io/ioutil" "os" "path" "path/filepath" + "runtime" "testing" "time" @@ -229,3 +231,138 @@ func TestHashOnDelete(t *testing.T) { _, err = o.Hash(ctx, hash.MD5) require.Error(t, err) } + +func TestMetadata(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + defer r.Finalise() + const filePath = "metafile.txt" + when := time.Now() + const dayLength = len("2001-01-01") + whenRFC := when.Format(time.RFC3339Nano) + r.WriteFile(filePath, "metadata file contents", when) + f := r.Flocal.(*Fs) + + // Get the object + obj, err := f.NewObject(ctx, filePath) + require.NoError(t, err) + o := obj.(*Object) + + features := f.Features() + + var hasXID, hasAtime, hasBtime bool + switch runtime.GOOS { + case "darwin", "freebsd", "netbsd", "linux": + hasXID, hasAtime, hasBtime = true, true, true + case "openbsd", "solaris": + hasXID, hasAtime = true, true + case "windows": + hasAtime, hasBtime = true, true + case "plan9", "js": + // nada + default: + t.Errorf("No test cases for OS %q", runtime.GOOS) + } + + assert.True(t, features.ReadMetadata) + assert.True(t, features.WriteMetadata) + assert.Equal(t, xattrSupported, features.UserMetadata) + + t.Run("Xattr", func(t *testing.T) { + if !xattrSupported { + t.Skip() + } + m, err := o.getXattr() + require.NoError(t, err) + assert.Nil(t, m) + + inM := fs.Metadata{ + "potato": "chips", + "cabbage": "soup", + } + err = o.setXattr(inM) + require.NoError(t, err) + + m, err = o.getXattr() + require.NoError(t, err) + assert.NotNil(t, m) + assert.Equal(t, inM, m) + }) + + checkTime := func(m fs.Metadata, key string, when time.Time) { + mt, ok := o.parseMetadataTime(m, key) + assert.True(t, ok) + dt := mt.Sub(when) + precision := time.Second + assert.True(t, dt >= -precision && dt <= precision, fmt.Sprintf("%s: dt %v outside +/- precision %v", key, dt, precision)) + } + + checkInt := func(m fs.Metadata, key string, base int) int { + value, ok := o.parseMetadataInt(m, key, base) + assert.True(t, ok) + return value + } + t.Run("Read", func(t *testing.T) { + m, err := o.Metadata(ctx) + require.NoError(t, err) + assert.NotNil(t, m) + + // All OSes have these + checkInt(m, "mode", 8) + checkTime(m, "mtime", when) + + assert.Equal(t, len(whenRFC), len(m["mtime"])) + assert.Equal(t, whenRFC[:dayLength], m["mtime"][:dayLength]) + + if hasAtime { + checkTime(m, "atime", when) + } + if hasBtime { + checkTime(m, "btime", when) + } + if hasXID { + checkInt(m, "uid", 10) + checkInt(m, "gid", 10) + } + }) + + t.Run("Write", func(t *testing.T) { + newAtimeString := "2011-12-13T14:15:16.999999999Z" + newAtime := fstest.Time(newAtimeString) + newMtimeString := "2011-12-12T14:15:16.999999999Z" + newMtime := fstest.Time(newMtimeString) + newBtimeString := "2011-12-11T14:15:16.999999999Z" + newBtime := fstest.Time(newBtimeString) + newM := fs.Metadata{ + "mtime": newMtimeString, + "atime": newAtimeString, + "btime": newBtimeString, + // Can't test uid, gid without being root + "mode": "0767", + "potato": "wedges", + } + err := o.writeMetadata(newM) + require.NoError(t, err) + + m, err := o.Metadata(ctx) + require.NoError(t, err) + assert.NotNil(t, m) + + mode := checkInt(m, "mode", 8) + if runtime.GOOS != "windows" { + assert.Equal(t, 0767, mode&0777, fmt.Sprintf("mode wrong - expecting 0767 got 0%o", mode&0777)) + } + + checkTime(m, "mtime", newMtime) + if hasAtime { + checkTime(m, "atime", newAtime) + } + if haveSetBTime { + checkTime(m, "btime", newBtime) + } + if xattrSupported { + assert.Equal(t, "wedges", m["potato"]) + } + }) + +} diff --git a/backend/local/metadata.go b/backend/local/metadata.go new file mode 100644 index 0000000000000..097abedf18ffc --- /dev/null +++ b/backend/local/metadata.go @@ -0,0 +1,138 @@ +package local + +import ( + "fmt" + "os" + "runtime" + "strconv" + "time" + + "github.com/rclone/rclone/fs" +) + +const metadataTimeFormat = time.RFC3339Nano + +// system metadata keys which this backend owns +// +// not all values supported on all OSes +var systemMetadataInfo = map[string]fs.MetadataHelp{ + "mode": { + Help: "File type and mode", + Type: "octal, unix style", + Example: "0100664", + }, + "uid": { + Help: "User ID of owner", + Type: "decimal number", + Example: "500", + }, + "gid": { + Help: "Group ID of owner", + Type: "decimal number", + Example: "500", + }, + "rdev": { + Help: "Device ID (if special file)", + Type: "hexadecimal", + Example: "1abc", + }, + "atime": { + Help: "Time of last access", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999999999Z07:00", + }, + "mtime": { + Help: "Time of last modification", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999999999Z07:00", + }, + "btime": { + Help: "Time of file birth (creation)", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999999999Z07:00", + }, +} + +// parse a time string from metadata with key +func (o *Object) parseMetadataTime(m fs.Metadata, key string) (t time.Time, ok bool) { + value, ok := m[key] + if ok { + var err error + t, err = time.Parse(metadataTimeFormat, value) + if err != nil { + fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err) + ok = false + } + } + return t, ok +} + +// parse am int from metadata with key and base +func (o *Object) parseMetadataInt(m fs.Metadata, key string, base int) (result int, ok bool) { + value, ok := m[key] + if ok { + var err error + result64, err := strconv.ParseInt(value, base, 64) + if err != nil { + fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err) + ok = false + } + result = int(result64) + } + return result, ok +} + +// Write the metadata into the file +// +// It isn't possible to set the ctime and btime under Unix +func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) { + var err error + atime, atimeOK := o.parseMetadataTime(m, "atime") + mtime, mtimeOK := o.parseMetadataTime(m, "mtime") + btime, btimeOK := o.parseMetadataTime(m, "btime") + if atimeOK || mtimeOK { + if atimeOK && !mtimeOK { + mtime = atime + } + if !atimeOK && mtimeOK { + atime = mtime + } + err = o.setTimes(atime, mtime) + if err != nil { + outErr = fmt.Errorf("failed to set times: %w", err) + } + } + if haveSetBTime { + if btimeOK { + err = setBTime(o.path, btime) + if err != nil { + outErr = fmt.Errorf("failed to set birth (creation) time: %w", err) + } + } + } + uid, hasUID := o.parseMetadataInt(m, "uid", 10) + gid, hasGID := o.parseMetadataInt(m, "gid", 10) + if hasUID { + // FIXME should read UID and GID of current user and only attempt to set it if different + if !hasGID { + gid = uid + } + if runtime.GOOS == "windows" || runtime.GOOS == "plan9" { + fs.Debugf(o, "Ignoring request to set ownership %o.%o on this OS", gid, uid) + } else { + err = os.Chown(o.path, uid, gid) + if err != nil { + outErr = fmt.Errorf("failed to change ownership: %w", err) + } + } + } + mode, hasMode := o.parseMetadataInt(m, "mode", 8) + if hasMode { + err = os.Chmod(o.path, os.FileMode(mode)) + if err != nil { + outErr = fmt.Errorf("failed to change permissions: %w", err) + } + } + // FIXME not parsing rdev yet + return outErr +} diff --git a/backend/local/metadata_bsd.go b/backend/local/metadata_bsd.go new file mode 100644 index 0000000000000..c4c90d2266221 --- /dev/null +++ b/backend/local/metadata_bsd.go @@ -0,0 +1,38 @@ +//go:build darwin || freebsd || netbsd +// +build darwin freebsd netbsd + +package local + +import ( + "fmt" + "syscall" + "time" + + "github.com/rclone/rclone/fs" +) + +// Read the metadata from the file into metadata where possible +func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) { + info, err := o.fs.lstat(o.path) + if err != nil { + return err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + fs.Debugf(o, "didn't return Stat_t as expected") + return nil + } + m.Set("mode", fmt.Sprintf("%0o", stat.Mode)) + m.Set("uid", fmt.Sprintf("%d", stat.Uid)) + m.Set("gid", fmt.Sprintf("%d", stat.Gid)) + if stat.Rdev != 0 { + m.Set("rdev", fmt.Sprintf("%x", stat.Rdev)) + } + setTime := func(key string, t syscall.Timespec) { + m.Set(key, time.Unix(t.Unix()).Format(metadataTimeFormat)) + } + setTime("atime", stat.Atimespec) + setTime("mtime", stat.Mtimespec) + setTime("btime", stat.Birthtimespec) + return nil +} diff --git a/backend/local/metadata_linux.go b/backend/local/metadata_linux.go new file mode 100644 index 0000000000000..17d32c06bfde0 --- /dev/null +++ b/backend/local/metadata_linux.go @@ -0,0 +1,47 @@ +//go:build linux +// +build linux + +package local + +import ( + "fmt" + "time" + + "github.com/rclone/rclone/fs" + "golang.org/x/sys/unix" +) + +// Read the metadata from the file into metadata where possible +func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) { + flags := unix.AT_SYMLINK_NOFOLLOW + if o.fs.opt.FollowSymlinks { + flags = 0 + } + var stat unix.Statx_t + err = unix.Statx(unix.AT_FDCWD, o.path, flags, (0 | + unix.STATX_TYPE | // Want stx_mode & S_IFMT + unix.STATX_MODE | // Want stx_mode & ~S_IFMT + unix.STATX_UID | // Want stx_uid + unix.STATX_GID | // Want stx_gid + unix.STATX_ATIME | // Want stx_atime + unix.STATX_MTIME | // Want stx_mtime + unix.STATX_CTIME | // Want stx_ctime + unix.STATX_BTIME), // Want stx_btime + &stat) + if err != nil { + return err + } + m.Set("mode", fmt.Sprintf("%0o", stat.Mode)) + m.Set("uid", fmt.Sprintf("%d", stat.Uid)) + m.Set("gid", fmt.Sprintf("%d", stat.Gid)) + if stat.Rdev_major != 0 || stat.Rdev_minor != 0 { + m.Set("rdev", fmt.Sprintf("%x", uint64(stat.Rdev_major)<<32|uint64(stat.Rdev_minor))) + } + setTime := func(key string, t unix.StatxTimestamp) { + m.Set(key, time.Unix(t.Sec, int64(t.Nsec)).Format(metadataTimeFormat)) + } + setTime("atime", stat.Atime) + setTime("mtime", stat.Mtime) + setTime("btime", stat.Btime) + return nil +} diff --git a/backend/local/metadata_other.go b/backend/local/metadata_other.go new file mode 100644 index 0000000000000..f2ce80956f6a6 --- /dev/null +++ b/backend/local/metadata_other.go @@ -0,0 +1,21 @@ +//go:build plan9 || js +// +build plan9 js + +package local + +import ( + "fmt" + + "github.com/rclone/rclone/fs" +) + +// Read the metadata from the file into metadata where possible +func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) { + info, err := o.fs.lstat(o.path) + if err != nil { + return err + } + m.Set("mode", fmt.Sprintf("%0o", info.Mode())) + m.Set("mtime", info.ModTime().Format(metadataTimeFormat)) + return nil +} diff --git a/backend/local/metadata_unix.go b/backend/local/metadata_unix.go new file mode 100644 index 0000000000000..48bf7faa36a3f --- /dev/null +++ b/backend/local/metadata_unix.go @@ -0,0 +1,37 @@ +//go:build openbsd || solaris +// +build openbsd solaris + +package local + +import ( + "fmt" + "syscall" + "time" + + "github.com/rclone/rclone/fs" +) + +// Read the metadata from the file into metadata where possible +func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) { + info, err := o.fs.lstat(o.path) + if err != nil { + return err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + fs.Debugf(o, "didn't return Stat_t as expected") + return nil + } + m.Set("mode", fmt.Sprintf("%0o", stat.Mode)) + m.Set("uid", fmt.Sprintf("%d", stat.Uid)) + m.Set("gid", fmt.Sprintf("%d", stat.Gid)) + if stat.Rdev != 0 { + m.Set("rdev", fmt.Sprintf("%x", stat.Rdev)) + } + setTime := func(key string, t syscall.Timespec) { + m.Set(key, time.Unix(t.Unix()).Format(metadataTimeFormat)) + } + setTime("atime", stat.Atim) + setTime("mtime", stat.Mtim) + return nil +} diff --git a/backend/local/metadata_windows.go b/backend/local/metadata_windows.go new file mode 100644 index 0000000000000..b2588953e95a9 --- /dev/null +++ b/backend/local/metadata_windows.go @@ -0,0 +1,34 @@ +//go:build windows +// +build windows + +package local + +import ( + "fmt" + "syscall" + "time" + + "github.com/rclone/rclone/fs" +) + +// Read the metadata from the file into metadata where possible +func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) { + info, err := o.fs.lstat(o.path) + if err != nil { + return err + } + stat, ok := info.Sys().(*syscall.Win32FileAttributeData) + if !ok { + fs.Debugf(o, "didn't return Win32FileAttributeData as expected") + return nil + } + // FIXME do something with stat.FileAttributes ? + m.Set("mode", fmt.Sprintf("%0o", info.Mode())) + setTime := func(key string, t syscall.Filetime) { + m.Set(key, time.Unix(0, t.Nanoseconds()).Format(metadataTimeFormat)) + } + setTime("atime", stat.LastAccessTime) + setTime("mtime", stat.LastWriteTime) + setTime("btime", stat.CreationTime) + return nil +} diff --git a/backend/local/setbtime.go b/backend/local/setbtime.go new file mode 100644 index 0000000000000..20a914bf2a38f --- /dev/null +++ b/backend/local/setbtime.go @@ -0,0 +1,16 @@ +//go:build !windows +// +build !windows + +package local + +import ( + "time" +) + +const haveSetBTime = false + +// setBTime changes the the birth time of the file passed in +func setBTime(name string, btime time.Time) error { + // Does nothing + return nil +} diff --git a/backend/local/setbtime_windows.go b/backend/local/setbtime_windows.go new file mode 100644 index 0000000000000..3734c2e7b5584 --- /dev/null +++ b/backend/local/setbtime_windows.go @@ -0,0 +1,28 @@ +//go:build windows +// +build windows + +package local + +import ( + "os" + "time" + "syscall" +) + +const haveSetBTime = true + +// setBTime sets the the birth time of the file passed in +func setBTime(name string, btime time.Time) (err error) { + h, err := syscall.Open(name, os.O_RDWR, 0755) + if err != nil { + return err + } + defer func() { + closeErr := syscall.Close(h) + if err == nil { + err = closeErr + } + }() + bFileTime := syscall.NsecToFiletime(btime.UnixNano()) + return syscall.SetFileTime(h, &bFileTime, nil, nil) +} diff --git a/backend/local/xattr.go b/backend/local/xattr.go new file mode 100644 index 0000000000000..a2c0cc5e868e7 --- /dev/null +++ b/backend/local/xattr.go @@ -0,0 +1,87 @@ +//go:build !openbsd && !plan9 +// +build !openbsd,!plan9 + +package local + +import ( + "fmt" + "strings" + + "github.com/pkg/xattr" + "github.com/rclone/rclone/fs" +) + +const ( + xattrPrefix = "user." // FIXME is this correct for all unixes? + xattrSupported = xattr.XATTR_SUPPORTED +) + +// getXattr returns the extended attributes for an object +// +// It doesn't return any attributes owned by this backend in +// metadataKeys +func (o *Object) getXattr() (metadata fs.Metadata, err error) { + if !xattrSupported { + return nil, nil + } + var list []string + if o.fs.opt.FollowSymlinks { + list, err = xattr.List(o.path) + } else { + list, err = xattr.LList(o.path) + } + if err != nil { + return nil, fmt.Errorf("failed to read xattr: %w", err) + } + if len(list) == 0 { + return nil, nil + } + metadata = make(fs.Metadata, len(list)) + for _, k := range list { + var v []byte + if o.fs.opt.FollowSymlinks { + v, err = xattr.Get(o.path, k) + } else { + v, err = xattr.LGet(o.path, k) + } + if err != nil { + return nil, fmt.Errorf("failed to read xattr key %q: %w", k, err) + } + k = strings.ToLower(k) + if !strings.HasPrefix(k, xattrPrefix) { + continue + } + k = k[len(xattrPrefix):] + if _, found := systemMetadataInfo[k]; found { + continue + } + metadata[k] = string(v) + } + return metadata, nil +} + +// setXattr sets the metadata on the file Xattrs +// +// It doesn't set any attributes owned by this backend in metadataKeys +func (o *Object) setXattr(metadata fs.Metadata) (err error) { + if !xattrSupported { + return nil + } + for k, value := range metadata { + k = strings.ToLower(k) + if _, found := systemMetadataInfo[k]; found { + continue + } + k = xattrPrefix + k + v := []byte(value) + if o.fs.opt.FollowSymlinks { + err = xattr.Set(o.path, k, v) + } else { + err = xattr.LSet(o.path, k, v) + } + if err != nil { + return fmt.Errorf("failed to set xattr key %q: %w", k, err) + } + } + return nil +} diff --git a/backend/local/xattr_unsupported.go b/backend/local/xattr_unsupported.go new file mode 100644 index 0000000000000..88455939a2281 --- /dev/null +++ b/backend/local/xattr_unsupported.go @@ -0,0 +1,21 @@ +//go:build openbsd || plan9 +// +build openbsd plan9 + +// The pkg/xattr module doesn't compile for openbsd or plan9 +package local + +import "github.com/rclone/rclone/fs" + +const ( + xattrSupported = false +) + +// getXattr returns the extended attributes for an object +func (o *Object) getXattr() (metadata fs.Metadata, err error) { + return nil, nil +} + +// setXattr sets the metadata on the file Xattrs +func (o *Object) setXattr(metadata fs.Metadata) (err error) { + return nil +} diff --git a/docs/content/local.md b/docs/content/local.md index a5004108fd4ba..c17a22c1a8a70 100644 --- a/docs/content/local.md +++ b/docs/content/local.md @@ -563,6 +563,32 @@ Properties: - Type: MultiEncoder - Default: Slash,Dot +### Metadata + +Depending on which OS is in use the local backend may return only some +of the system metadata. Setting system metadata is supported on all +OSes but setting user metadata is only supported on linux, freebsd, +netbsd, macOS and Solaris. It is **not** supported on Windows yet +([see pkg/attrs#47](https://github.com/pkg/xattr/issues/47)). + +User metadata is stored as extended attributes (which may not be +supported by all file systems) under the "user.*" prefix. + +Here are the possible system metadata items for the local backend. + +| Name | Help | Type | Example | Read Only | +|------|------|------|---------|-----------| +| atime | Time of last access | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N | +| btime | Time of file birth (creation) | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N | +| gid | Group ID of owner | decimal number | 500 | N | +| mode | File type and mode | octal, unix style | 0100664 | N | +| mtime | Time of last modification | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N | +| rdev | Device ID (if special file) | hexadecimal | 1abc | N | +| uid | User ID of owner | decimal number | 500 | N | + + +See the [metadata](/docs/#metadata) docs for more info. + ## Backend commands Here are the commands specific to the local backend. diff --git a/go.mod b/go.mod index 1fe102fd68853..e61cdce97dbc6 100644 --- a/go.mod +++ b/go.mod @@ -76,6 +76,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af + github.com/pkg/xattr v0.4.7 // indirect golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 ) diff --git a/go.sum b/go.sum index e957043f89908..9cc774dd30d10 100644 --- a/go.sum +++ b/go.sum @@ -478,6 +478,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.5-0.20211228200725-31aac3e1878d h1:7cHNeARnMq3icpbMdvyUELykWM4zOj5NRhH2Y3sfgBc= github.com/pkg/sftp v1.13.5-0.20211228200725-31aac3e1878d/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= +github.com/pkg/xattr v0.4.7 h1:XoA3KzmFvyPlH4RwX5eMcgtzcaGBaSvgt3IoFQfbrmQ= +github.com/pkg/xattr v0.4.7/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= @@ -882,6 +884,7 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220318055525-2edf467146b5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= From 866c873daa97f90f0bf6d303a1ea92d248f17b1d Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 29 Jun 2022 11:51:46 +0100 Subject: [PATCH 125/560] backend: allow wrapping backend tests to run in make quicktest --- backend/chunker/chunker_test.go | 1 + backend/combine/combine_test.go | 2 ++ backend/compress/compress_test.go | 1 + backend/crypt/crypt_test.go | 13 +++++++++++++ backend/hasher/hasher_test.go | 1 + backend/local/local_test.go | 5 +++-- backend/memory/memory_test.go | 5 +++-- backend/union/union_test.go | 6 ++++++ fstest/fstests/fstests.go | 3 ++- 9 files changed, 32 insertions(+), 5 deletions(-) diff --git a/backend/chunker/chunker_test.go b/backend/chunker/chunker_test.go index 4acdf5b5aab18..cca76f9e72776 100644 --- a/backend/chunker/chunker_test.go +++ b/backend/chunker/chunker_test.go @@ -53,6 +53,7 @@ func TestIntegration(t *testing.T) { {Name: name, Key: "type", Value: "chunker"}, {Name: name, Key: "remote", Value: tempDir}, } + opt.QuickTestOK = true } fstests.Run(t, &opt) } diff --git a/backend/combine/combine_test.go b/backend/combine/combine_test.go index c6274c562097a..46b49bcf0305f 100644 --- a/backend/combine/combine_test.go +++ b/backend/combine/combine_test.go @@ -35,6 +35,7 @@ func TestLocal(t *testing.T) { {Name: name, Key: "type", Value: "combine"}, {Name: name, Key: "upstreams", Value: upstreams}, }, + QuickTestOK: true, }) } @@ -50,6 +51,7 @@ func TestMemory(t *testing.T) { {Name: name, Key: "type", Value: "combine"}, {Name: name, Key: "upstreams", Value: upstreams}, }, + QuickTestOK: true, }) } diff --git a/backend/compress/compress_test.go b/backend/compress/compress_test.go index 76baf22aa91cd..356e9dd438295 100644 --- a/backend/compress/compress_test.go +++ b/backend/compress/compress_test.go @@ -61,5 +61,6 @@ func TestRemoteGzip(t *testing.T) { {Name: name, Key: "remote", Value: tempdir}, {Name: name, Key: "compression_mode", Value: "gzip"}, }, + QuickTestOK: true, }) } diff --git a/backend/crypt/crypt_test.go b/backend/crypt/crypt_test.go index 8369a60f7d2dd..d4b9dc1e43283 100644 --- a/backend/crypt/crypt_test.go +++ b/backend/crypt/crypt_test.go @@ -4,6 +4,7 @@ package crypt_test import ( "os" "path/filepath" + "runtime" "testing" "github.com/rclone/rclone/backend/crypt" @@ -46,6 +47,7 @@ func TestStandardBase32(t *testing.T) { }, UnimplementableFsMethods: []string{"OpenWriterAt"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } @@ -67,6 +69,7 @@ func TestStandardBase64(t *testing.T) { }, UnimplementableFsMethods: []string{"OpenWriterAt"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } @@ -88,6 +91,7 @@ func TestStandardBase32768(t *testing.T) { }, UnimplementableFsMethods: []string{"OpenWriterAt"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } @@ -109,6 +113,7 @@ func TestOff(t *testing.T) { }, UnimplementableFsMethods: []string{"OpenWriterAt"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } @@ -117,6 +122,9 @@ func TestObfuscate(t *testing.T) { if *fstest.RemoteName != "" { t.Skip("Skipping as -remote set") } + if runtime.GOOS == "darwin" { + t.Skip("Skipping on macOS as obfuscating control characters makes filenames macOS can't cope with") + } tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate") name := "TestCrypt3" fstests.Run(t, &fstests.Opt{ @@ -131,6 +139,7 @@ func TestObfuscate(t *testing.T) { SkipBadWindowsCharacters: true, UnimplementableFsMethods: []string{"OpenWriterAt"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } @@ -139,6 +148,9 @@ func TestNoDataObfuscate(t *testing.T) { if *fstest.RemoteName != "" { t.Skip("Skipping as -remote set") } + if runtime.GOOS == "darwin" { + t.Skip("Skipping on macOS as obfuscating control characters makes filenames macOS can't cope with") + } tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate") name := "TestCrypt4" fstests.Run(t, &fstests.Opt{ @@ -154,5 +166,6 @@ func TestNoDataObfuscate(t *testing.T) { SkipBadWindowsCharacters: true, UnimplementableFsMethods: []string{"OpenWriterAt"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } diff --git a/backend/hasher/hasher_test.go b/backend/hasher/hasher_test.go index 09d60f33d71c4..23feb9b96ca7f 100644 --- a/backend/hasher/hasher_test.go +++ b/backend/hasher/hasher_test.go @@ -33,6 +33,7 @@ func TestIntegration(t *testing.T) { {Name: "TestHasher", Key: "remote", Value: tempDir}, } opt.RemoteName = "TestHasher:" + opt.QuickTestOK = true } fstests.Run(t, &opt) } diff --git a/backend/local/local_test.go b/backend/local/local_test.go index 8ecab2dbdc3d0..d0d54ec26c4c1 100644 --- a/backend/local/local_test.go +++ b/backend/local/local_test.go @@ -11,7 +11,8 @@ import ( // TestIntegration runs integration tests against the remote func TestIntegration(t *testing.T) { fstests.Run(t, &fstests.Opt{ - RemoteName: "", - NilObject: (*local.Object)(nil), + RemoteName: "", + NilObject: (*local.Object)(nil), + QuickTestOK: true, }) } diff --git a/backend/memory/memory_test.go b/backend/memory/memory_test.go index 3d768fec0efbe..9989bc60de2df 100644 --- a/backend/memory/memory_test.go +++ b/backend/memory/memory_test.go @@ -10,7 +10,8 @@ import ( // TestIntegration runs integration tests against the remote func TestIntegration(t *testing.T) { fstests.Run(t, &fstests.Opt{ - RemoteName: ":memory:", - NilObject: (*Object)(nil), + RemoteName: ":memory:", + NilObject: (*Object)(nil), + QuickTestOK: true, }) } diff --git a/backend/union/union_test.go b/backend/union/union_test.go index 8b69c4c27f8b3..f5a9948f61d4f 100644 --- a/backend/union/union_test.go +++ b/backend/union/union_test.go @@ -41,6 +41,7 @@ func TestStandard(t *testing.T) { }, UnimplementableFsMethods: []string{"OpenWriterAt", "DuplicateFiles"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } @@ -62,6 +63,7 @@ func TestRO(t *testing.T) { }, UnimplementableFsMethods: []string{"OpenWriterAt", "DuplicateFiles"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } @@ -83,6 +85,7 @@ func TestNC(t *testing.T) { }, UnimplementableFsMethods: []string{"OpenWriterAt", "DuplicateFiles"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } @@ -104,6 +107,7 @@ func TestPolicy1(t *testing.T) { }, UnimplementableFsMethods: []string{"OpenWriterAt", "DuplicateFiles"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } @@ -125,6 +129,7 @@ func TestPolicy2(t *testing.T) { }, UnimplementableFsMethods: []string{"OpenWriterAt", "DuplicateFiles"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } @@ -146,5 +151,6 @@ func TestPolicy3(t *testing.T) { }, UnimplementableFsMethods: []string{"OpenWriterAt", "DuplicateFiles"}, UnimplementableObjectMethods: []string{"MimeType"}, + QuickTestOK: true, }) } diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 9cc95896eac41..7a72ef5f518d2 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -327,6 +327,7 @@ type Opt struct { SkipFsCheckWrap bool // if set skip FsCheckWrap SkipObjectCheckWrap bool // if set skip ObjectCheckWrap SkipInvalidUTF8 bool // if set skip invalid UTF-8 checks + QuickTestOK bool // if set, run this test with make quicktest } // returns true if x is found in ss @@ -392,7 +393,7 @@ func Run(t *testing.T, opt *Opt) { unwrappableFsMethods = []string{"Command"} // these Fs methods don't need to be wrapped ever ) - if strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" { + if strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" && !opt.QuickTestOK { t.Skip("quicktest only") } From cd1735bb10bbacec4d8f8a0e8c6d92e427098463 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 29 Jun 2022 14:27:25 +0100 Subject: [PATCH 126/560] cache: mark as not supporting metadata --- backend/cache/cache_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cache/cache_test.go b/backend/cache/cache_test.go index 44307ea6d58b3..9ad4e31e5afe8 100644 --- a/backend/cache/cache_test.go +++ b/backend/cache/cache_test.go @@ -19,7 +19,7 @@ func TestIntegration(t *testing.T) { RemoteName: "TestCache:", NilObject: (*cache.Object)(nil), UnimplementableFsMethods: []string{"PublicLink", "OpenWriterAt"}, - UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"}, + UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata"}, SkipInvalidUTF8: true, // invalid UTF-8 confuses the cache }) } From ba5760ff3833a064dcabb498553a8616a4d24a0e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 29 Jun 2022 14:27:43 +0100 Subject: [PATCH 127/560] chunker: mark as not supporting metadata --- backend/chunker/chunker_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/chunker/chunker_test.go b/backend/chunker/chunker_test.go index cca76f9e72776..7f0c0310d0856 100644 --- a/backend/chunker/chunker_test.go +++ b/backend/chunker/chunker_test.go @@ -35,6 +35,7 @@ func TestIntegration(t *testing.T) { "MimeType", "GetTier", "SetTier", + "Metadata", }, UnimplementableFsMethods: []string{ "PublicLink", From 8c483daf85d6a3644b87772bbedfa06e51842b95 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 29 Jun 2022 14:27:58 +0100 Subject: [PATCH 128/560] combine: support metadata --- backend/combine/combine.go | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/backend/combine/combine.go b/backend/combine/combine.go index 934ed76f44ada..36fdc69c6ea56 100644 --- a/backend/combine/combine.go +++ b/backend/combine/combine.go @@ -31,6 +31,9 @@ func init() { Name: "combine", Description: "Combine several remotes into one", NewFs: NewFs, + MetadataInfo: &fs.MetadataInfo{ + Help: `Any metadata supported by the underlying remote is read and written.`, + }, Options: []fs.Option{{ Name: "upstreams", Help: `Upstreams for combining @@ -222,6 +225,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs BucketBased: true, SetTier: true, GetTier: true, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, }).Fill(ctx, f) canMove := true for _, u := range f.upstreams { @@ -925,6 +931,45 @@ func (o *Object) UnWrap() fs.Object { return o.Object } +// GetTier returns storage tier or class of the Object +func (o *Object) GetTier() string { + do, ok := o.Object.(fs.GetTierer) + if !ok { + return "" + } + return do.GetTier() +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + do, ok := o.Object.(fs.IDer) + if !ok { + return "" + } + return do.ID() +} + +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) { + do, ok := o.Object.(fs.Metadataer) + if !ok { + return nil, nil + } + return do.Metadata(ctx) +} + +// SetTier performs changing storage tier of the Object if +// multiple storage classes supported +func (o *Object) SetTier(tier string) error { + do, ok := o.Object.(fs.SetTierer) + if !ok { + return errors.New("underlying remote does not support SetTier") + } + return do.SetTier(tier) +} + // Check the interfaces are satisfied var ( _ fs.Fs = (*Fs)(nil) @@ -938,4 +983,5 @@ var ( _ fs.Abouter = (*Fs)(nil) _ fs.ListRer = (*Fs)(nil) _ fs.Shutdowner = (*Fs)(nil) + _ fs.FullObject = (*Object)(nil) ) From c198700812030012e21d18886ec0589daf1392ad Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 29 Jun 2022 14:28:28 +0100 Subject: [PATCH 129/560] compress: support metadata --- backend/compress/compress.go | 75 ++++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 7 deletions(-) diff --git a/backend/compress/compress.go b/backend/compress/compress.go index dcc54a11c0e9d..9033b38ff4f11 100644 --- a/backend/compress/compress.go +++ b/backend/compress/compress.go @@ -70,6 +70,9 @@ func init() { Name: "compress", Description: "Compress a remote", NewFs: NewFs, + MetadataInfo: &fs.MetadataInfo{ + Help: `Any metadata supported by the underlying remote is read and written.`, + }, Options: []fs.Option{{ Name: "remote", Help: "Remote to compress.", @@ -180,6 +183,9 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, SetTier: true, BucketBased: true, CanHaveEmptyDirectories: true, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, }).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs) // We support reading MIME types no matter the wrapped fs f.features.ReadMimeType = true @@ -1214,6 +1220,21 @@ func (o *Object) MimeType(ctx context.Context) string { return o.meta.MimeType } +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) { + err := o.loadMetadataIfNotLoaded(ctx) + if err != nil { + return nil, err + } + do, ok := o.mo.(fs.Metadataer) + if !ok { + return nil, nil + } + return do.Metadata(ctx) +} + // Hash returns the selected checksum of the file // If no checksum is available it returns "" func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) { @@ -1360,6 +1381,51 @@ func (o *ObjectInfo) Hash(ctx context.Context, ht hash.Type) (string, error) { return "", nil // cannot know the checksum } +// ID returns the ID of the Object if known, or "" if not +func (o *ObjectInfo) ID() string { + do, ok := o.src.(fs.IDer) + if !ok { + return "" + } + return do.ID() +} + +// MimeType returns the content type of the Object if +// known, or "" if not +func (o *ObjectInfo) MimeType(ctx context.Context) string { + do, ok := o.src.(fs.MimeTyper) + if !ok { + return "" + } + return do.MimeType(ctx) +} + +// UnWrap returns the Object that this Object is wrapping or +// nil if it isn't wrapping anything +func (o *ObjectInfo) UnWrap() fs.Object { + return fs.UnWrapObjectInfo(o.src) +} + +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *ObjectInfo) Metadata(ctx context.Context) (fs.Metadata, error) { + do, ok := o.src.(fs.Metadataer) + if !ok { + return nil, nil + } + return do.Metadata(ctx) +} + +// GetTier returns storage tier or class of the Object +func (o *ObjectInfo) GetTier() string { + do, ok := o.src.(fs.GetTierer) + if !ok { + return "" + } + return do.GetTier() +} + // ID returns the ID of the Object if known, or "" if not func (o *Object) ID() string { do, ok := o.Object.(fs.IDer) @@ -1412,11 +1478,6 @@ var ( _ fs.ChangeNotifier = (*Fs)(nil) _ fs.PublicLinker = (*Fs)(nil) _ fs.Shutdowner = (*Fs)(nil) - _ fs.ObjectInfo = (*ObjectInfo)(nil) - _ fs.GetTierer = (*Object)(nil) - _ fs.SetTierer = (*Object)(nil) - _ fs.Object = (*Object)(nil) - _ fs.ObjectUnWrapper = (*Object)(nil) - _ fs.IDer = (*Object)(nil) - _ fs.MimeTyper = (*Object)(nil) + _ fs.FullObjectInfo = (*ObjectInfo)(nil) + _ fs.FullObject = (*Object)(nil) ) From bf4a16ae308df7c6e3ad1305e60343d75384ace3 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 29 Jun 2022 14:29:00 +0100 Subject: [PATCH 130/560] crypt: support metadata --- backend/crypt/crypt.go | 78 +++++++++++++++++++++++++--- backend/crypt/crypt_internal_test.go | 4 +- 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go index 33e57cdb9b513..6f599cafb2c90 100644 --- a/backend/crypt/crypt.go +++ b/backend/crypt/crypt.go @@ -28,6 +28,9 @@ func init() { Description: "Encrypt/Decrypt a remote", NewFs: NewFs, CommandHelp: commandHelp, + MetadataInfo: &fs.MetadataInfo{ + Help: `Any metadata supported by the underlying remote is read and written.`, + }, Options: []fs.Option{{ Name: "remote", Help: "Remote to encrypt/decrypt.\n\nNormally should contain a ':' and a path, e.g. \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).", @@ -241,6 +244,9 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, SetTier: true, GetTier: true, ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, }).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs) return f, err @@ -1056,6 +1062,50 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) { return "", nil } +// GetTier returns storage tier or class of the Object +func (o *ObjectInfo) GetTier() string { + do, ok := o.ObjectInfo.(fs.GetTierer) + if !ok { + return "" + } + return do.GetTier() +} + +// ID returns the ID of the Object if known, or "" if not +func (o *ObjectInfo) ID() string { + do, ok := o.ObjectInfo.(fs.IDer) + if !ok { + return "" + } + return do.ID() +} + +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *ObjectInfo) Metadata(ctx context.Context) (fs.Metadata, error) { + do, ok := o.ObjectInfo.(fs.Metadataer) + if !ok { + return nil, nil + } + return do.Metadata(ctx) +} + +// MimeType returns the content type of the Object if +// known, or "" if not +// +// This is deliberately unsupported so we don't leak mime type info by +// default. +func (o *ObjectInfo) MimeType(ctx context.Context) string { + return "" +} + +// UnWrap returns the Object that this Object is wrapping or +// nil if it isn't wrapping anything +func (o *ObjectInfo) UnWrap() fs.Object { + return fs.UnWrapObjectInfo(o.ObjectInfo) +} + // ID returns the ID of the Object if known, or "" if not func (o *Object) ID() string { do, ok := o.Object.(fs.IDer) @@ -1084,6 +1134,26 @@ func (o *Object) GetTier() string { return do.GetTier() } +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) { + do, ok := o.Object.(fs.Metadataer) + if !ok { + return nil, nil + } + return do.Metadata(ctx) +} + +// MimeType returns the content type of the Object if +// known, or "" if not +// +// This is deliberately unsupported so we don't leak mime type info by +// default. +func (o *Object) MimeType(ctx context.Context) string { + return "" +} + // Check the interfaces are satisfied var ( _ fs.Fs = (*Fs)(nil) @@ -1106,10 +1176,6 @@ var ( _ fs.UserInfoer = (*Fs)(nil) _ fs.Disconnecter = (*Fs)(nil) _ fs.Shutdowner = (*Fs)(nil) - _ fs.ObjectInfo = (*ObjectInfo)(nil) - _ fs.Object = (*Object)(nil) - _ fs.ObjectUnWrapper = (*Object)(nil) - _ fs.IDer = (*Object)(nil) - _ fs.SetTierer = (*Object)(nil) - _ fs.GetTierer = (*Object)(nil) + _ fs.FullObjectInfo = (*ObjectInfo)(nil) + _ fs.FullObject = (*Object)(nil) ) diff --git a/backend/crypt/crypt_internal_test.go b/backend/crypt/crypt_internal_test.go index 2f7f1f8b36d0f..82587763061d7 100644 --- a/backend/crypt/crypt_internal_test.go +++ b/backend/crypt/crypt_internal_test.go @@ -91,7 +91,9 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) { src := f.newObjectInfo(oi, nonce) // Test ObjectInfo methods - assert.Equal(t, int64(outBuf.Len()), src.Size()) + if !f.opt.NoDataEncryption { + assert.Equal(t, int64(outBuf.Len()), src.Size()) + } assert.Equal(t, f, src.Fs()) assert.NotEqual(t, path, src.Remote()) From ed87ae51c0fa42f9523a9df4d87d452452d1fd84 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 29 Jun 2022 14:29:15 +0100 Subject: [PATCH 131/560] union: support metadata --- backend/union/entry.go | 47 +++++++++++++++++++++++++-- backend/union/union.go | 8 ++++- backend/union/upstream/upstream.go | 52 ++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 4 deletions(-) diff --git a/backend/union/entry.go b/backend/union/entry.go index efd2cbaef71e8..023f3bb456d7c 100644 --- a/backend/union/entry.go +++ b/backend/union/entry.go @@ -2,6 +2,7 @@ package union import ( "context" + "errors" "fmt" "io" "io/ioutil" @@ -34,9 +35,8 @@ type entry interface { candidates() []upstream.Entry } -// UnWrap returns the Object that this Object is wrapping or -// nil if it isn't wrapping anything -func (o *Object) UnWrap() *upstream.Object { +// UnWrapUpstream returns the upstream Object that this Object is wrapping +func (o *Object) UnWrapUpstream() *upstream.Object { return o.Object } @@ -140,6 +140,42 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) error { return errs.Err() } +// GetTier returns storage tier or class of the Object +func (o *Object) GetTier() string { + do, ok := o.Object.Object.(fs.GetTierer) + if !ok { + return "" + } + return do.GetTier() +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + do, ok := o.Object.Object.(fs.IDer) + if !ok { + return "" + } + return do.ID() +} + +// MimeType returns the content type of the Object if known +func (o *Object) MimeType(ctx context.Context) (mimeType string) { + if do, ok := o.Object.Object.(fs.MimeTyper); ok { + mimeType = do.MimeType(ctx) + } + return mimeType +} + +// SetTier performs changing storage tier of the Object if +// multiple storage classes supported +func (o *Object) SetTier(tier string) error { + do, ok := o.Object.Object.(fs.SetTierer) + if !ok { + return errors.New("underlying remote does not support SetTier") + } + return do.SetTier(tier) +} + // ModTime returns the modification date of the directory // It returns the latest ModTime of all candidates func (d *Directory) ModTime(ctx context.Context) (t time.Time) { @@ -164,3 +200,8 @@ func (d *Directory) Size() (s int64) { } return s } + +// Check the interfaces are satisfied +var ( + _ fs.FullObject = (*Object)(nil) +) diff --git a/backend/union/union.go b/backend/union/union.go index 9776c55d2a54f..e1ec53e85eacc 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -30,6 +30,9 @@ func init() { Name: "union", Description: "Union merges the contents of several upstream fs", NewFs: NewFs, + MetadataInfo: &fs.MetadataInfo{ + Help: `Any metadata supported by the underlying remote is read and written.`, + }, Options: []fs.Option{{ Name: "upstreams", Help: "List of space separated upstreams.\n\nCan be 'upstreama:test/dir upstreamb:', '\"upstreama:test/space:ro dir\" upstreamb:', etc.", @@ -219,7 +222,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, fs.Debugf(src, "Can't copy - not same remote type") return nil, fs.ErrorCantCopy } - o := srcObj.UnWrap() + o := srcObj.UnWrapUpstream() su := o.UpstreamFs() if su.Features().Copy == nil { return nil, fs.ErrorCantCopy @@ -881,6 +884,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e BucketBased: true, SetTier: true, GetTier: true, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, }).Fill(ctx, f) canMove := true for _, f := range upstreams { diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go index 1a85d783fe44b..c6e906107620b 100644 --- a/backend/union/upstream/upstream.go +++ b/backend/union/upstream/upstream.go @@ -245,6 +245,53 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return nil } +// GetTier returns storage tier or class of the Object +func (o *Object) GetTier() string { + do, ok := o.Object.(fs.GetTierer) + if !ok { + return "" + } + return do.GetTier() +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + do, ok := o.Object.(fs.IDer) + if !ok { + return "" + } + return do.ID() +} + +// MimeType returns the content type of the Object if known +func (o *Object) MimeType(ctx context.Context) (mimeType string) { + if do, ok := o.Object.(fs.MimeTyper); ok { + mimeType = do.MimeType(ctx) + } + return mimeType +} + +// SetTier performs changing storage tier of the Object if +// multiple storage classes supported +func (o *Object) SetTier(tier string) error { + do, ok := o.Object.(fs.SetTierer) + if !ok { + return errors.New("underlying remote does not support SetTier") + } + return do.SetTier(tier) +} + +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) { + do, ok := o.Object.(fs.Metadataer) + if !ok { + return nil, nil + } + return do.Metadata(ctx) +} + // About gets quota information from the Fs func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { if atomic.LoadInt64(&f.cacheExpiry) <= time.Now().Unix() { @@ -363,3 +410,8 @@ func (f *Fs) updateUsageCore(lock bool) error { f.usage = usage return nil } + +// Check the interfaces are satisfied +var ( + _ fs.FullObject = (*Object)(nil) +) From 7e7a8a95e974ba40ed5154a93e268283d488e3b8 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 29 Jun 2022 14:52:16 +0100 Subject: [PATCH 132/560] hasher: support metadata --- backend/hasher/hasher.go | 26 ++++++++++++++++++++------ backend/hasher/hasher_internal_test.go | 2 +- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/backend/hasher/hasher.go b/backend/hasher/hasher.go index 1870983e043d8..e80daedee6a35 100644 --- a/backend/hasher/hasher.go +++ b/backend/hasher/hasher.go @@ -27,6 +27,9 @@ func init() { Name: "hasher", Description: "Better checksums for other remotes", NewFs: NewFs, + MetadataInfo: &fs.MetadataInfo{ + Help: `Any metadata supported by the underlying remote is read and written.`, + }, CommandHelp: commandHelp, Options: []fs.Option{{ Name: "remote", @@ -158,6 +161,11 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs IsLocal: true, ReadMimeType: true, WriteMimeType: true, + SetTier: true, + GetTier: true, + ReadMetadata: true, + WriteMetadata: true, + UserMetadata: true, } f.features = stubFeatures.Fill(ctx, f).Mask(ctx, f.Fs).WrapsFs(f, f.Fs) @@ -485,6 +493,17 @@ func (o *Object) MimeType(ctx context.Context) string { return "" } +// Metadata returns metadata for an object +// +// It should return nil if there is no Metadata +func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) { + do, ok := o.Object.(fs.Metadataer) + if !ok { + return nil, nil + } + return do.Metadata(ctx) +} + // Check the interfaces are satisfied var ( _ fs.Fs = (*Fs)(nil) @@ -507,10 +526,5 @@ var ( _ fs.UserInfoer = (*Fs)(nil) _ fs.Disconnecter = (*Fs)(nil) _ fs.Shutdowner = (*Fs)(nil) - _ fs.Object = (*Object)(nil) - _ fs.ObjectUnWrapper = (*Object)(nil) - _ fs.IDer = (*Object)(nil) - _ fs.SetTierer = (*Object)(nil) - _ fs.GetTierer = (*Object)(nil) - _ fs.MimeTyper = (*Object)(nil) + _ fs.FullObject = (*Object)(nil) ) diff --git a/backend/hasher/hasher_internal_test.go b/backend/hasher/hasher_internal_test.go index 5332318344882..0ca418e20a965 100644 --- a/backend/hasher/hasher_internal_test.go +++ b/backend/hasher/hasher_internal_test.go @@ -35,7 +35,7 @@ func (f *Fs) testUploadFromCrypt(t *testing.T) { // make a temporary crypt remote ctx := context.Background() pass := obscure.MustObscure("crypt") - remote := fmt.Sprintf(":crypt,remote=%s,password=%s:", tempRoot, pass) + remote := fmt.Sprintf(`:crypt,remote="%s",password="%s":`, tempRoot, pass) cryptFs, err := fs.NewFs(ctx, remote) require.NoError(t, err) From 73e3bb09d71c31814e00ebcad2eec9be3b74ccf0 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 23:13:12 +0200 Subject: [PATCH 133/560] http: fix missing response when using custom auth handler --- lib/http/auth/basic.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/http/auth/basic.go b/lib/http/auth/basic.go index b2a2c658b45a3..69907c009371e 100644 --- a/lib/http/auth/basic.go +++ b/lib/http/auth/basic.go @@ -113,6 +113,7 @@ func CustomAuth(fn CustomAuthFn, realm string) httplib.Middleware { if value != nil { r = r.WithContext(context.WithValue(r.Context(), ContextAuthKey, value)) } + next.ServeHTTP(w, r) } }) } From 9dbed02329456c71d2052a6365041deda94bc020 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Tue, 21 Jun 2022 01:14:06 +0200 Subject: [PATCH 134/560] jottacloud: always store username in config and use it to avoid initial api request Existing version did save username in config, but only when entering the custom device/mountpoint sequence in config. Regardless of that, it did always look up the username at startup with an api request. This commit improves it so that the username will always be stored in config, and when using standard authentication it picks it from the login token instead of requesting it from the remote api, and also in fs constructor it picks it from config instead of requesting it from remote api (again). --- backend/jottacloud/jottacloud.go | 53 ++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 735b5181fb080..5998a04a2780e 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -152,7 +152,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf m.Set(configClientSecret, "") srv := rest.NewClient(fshttp.NewClient(ctx)) - token, tokenEndpoint, err := doTokenAuth(ctx, srv, loginToken) + token, tokenEndpoint, username, err := doTokenAuth(ctx, srv, loginToken) if err != nil { return nil, fmt.Errorf("failed to get oauth token: %w", err) } @@ -161,6 +161,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf if err != nil { return nil, fmt.Errorf("error while saving token: %w", err) } + m.Set(configUsername, username) return fs.ConfigGoto("choose_device") case "legacy": // configure a jottacloud backend using legacy authentication m.Set("configVersion", fmt.Sprint(legacyConfigVersion)) @@ -271,22 +272,30 @@ sync or the backup section, for example, you must choose yes.`) if config.Result != "true" { m.Set(configDevice, "") m.Set(configMountpoint, "") + } + username, userOk := m.Get(configUsername) + if userOk && config.Result != "true" { return fs.ConfigGoto("end") } oAuthClient, _, err := getOAuthClient(ctx, name, m) if err != nil { return nil, err } - jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL) - apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL) - - cust, err := getCustomerInfo(ctx, apiSrv) - if err != nil { - return nil, err + if !userOk { + apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL) + cust, err := getCustomerInfo(ctx, apiSrv) + if err != nil { + return nil, err + } + username = cust.Username + m.Set(configUsername, username) + if config.Result != "true" { + return fs.ConfigGoto("end") + } } - m.Set(configUsername, cust.Username) - acc, err := getDriveInfo(ctx, jfsSrv, cust.Username) + jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL) + acc, err := getDriveInfo(ctx, jfsSrv, username) if err != nil { return nil, err } @@ -582,10 +591,10 @@ func doLegacyAuth(ctx context.Context, srv *rest.Client, oauthConfig *oauth2.Con } // doTokenAuth runs the actual token request for V2 authentication -func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 string) (token oauth2.Token, tokenEndpoint string, err error) { +func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 string) (token oauth2.Token, tokenEndpoint string, username string, err error) { loginTokenBytes, err := base64.RawURLEncoding.DecodeString(loginTokenBase64) if err != nil { - return token, "", err + return token, "", "", err } // decode login token @@ -593,7 +602,7 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri decoder := json.NewDecoder(bytes.NewReader(loginTokenBytes)) err = decoder.Decode(&loginToken) if err != nil { - return token, "", err + return token, "", "", err } // retrieve endpoint urls @@ -604,7 +613,7 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri var wellKnown api.WellKnown _, err = apiSrv.CallJSON(ctx, &opts, nil, &wellKnown) if err != nil { - return token, "", err + return token, "", "", err } // prepare out token request with username and password @@ -626,14 +635,14 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri var jsonToken api.TokenJSON _, err = apiSrv.CallJSON(ctx, &opts, nil, &jsonToken) if err != nil { - return token, "", err + return token, "", "", err } token.AccessToken = jsonToken.AccessToken token.RefreshToken = jsonToken.RefreshToken token.TokenType = jsonToken.TokenType token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second) - return token, wellKnown.TokenEndpoint, err + return token, wellKnown.TokenEndpoint, loginToken.Username, err } // getCustomerInfo queries general information about the account @@ -935,11 +944,17 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e return err }) - cust, err := getCustomerInfo(ctx, f.apiSrv) - if err != nil { - return nil, err + user, userOk := m.Get(configUsername) + if userOk { + f.user = user + } else { + fs.Infof(nil, "Username not found in config and must be looked up, reconfigure to avoid the extra request") + cust, err := getCustomerInfo(ctx, f.apiSrv) + if err != nil { + return nil, err + } + f.user = cust.Username } - f.user = cust.Username f.setEndpoints() if root != "" && !rootIsDir { From accf91742c662f9bfe4202537b4235ac76f6c20d Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 4 Jul 2022 09:25:10 +0100 Subject: [PATCH 135/560] fs: add Type and FindFromFs to manage Fs and RegInfo --- fs/newfs.go | 6 +++++- fs/registry.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/fs/newfs.go b/fs/newfs.go index 1ac5e48071280..f64cb2c8dfa75 100644 --- a/fs/newfs.go +++ b/fs/newfs.go @@ -48,7 +48,11 @@ func NewFs(ctx context.Context, path string) (Fs, error) { // These need to work as filesystem names as the VFS cache will use them configName += suffix } - return fsInfo.NewFs(ctx, configName, fsPath, config) + f, err := fsInfo.NewFs(ctx, configName, fsPath, config) + if f != nil && (err == nil || err == ErrorIsFile) { + addReverse(f, fsInfo) + } + return f, err } // ConfigFs makes the config for calling NewFs with. diff --git a/fs/registry.go b/fs/registry.go index d380c7e0cb658..ca2c3d976f05c 100644 --- a/fs/registry.go +++ b/fs/registry.go @@ -10,6 +10,7 @@ import ( "reflect" "sort" "strings" + "sync" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" @@ -301,3 +302,33 @@ func MustFind(name string) *RegInfo { } return fs } + +// Type returns a textual string to identify the type of the remote +func Type(f Fs) string { + typeName := fmt.Sprintf("%T", f) + typeName = strings.TrimPrefix(typeName, "*") + typeName = strings.TrimSuffix(typeName, ".Fs") + return typeName +} + +var ( + typeToRegInfoMu sync.Mutex + typeToRegInfo = map[string]*RegInfo{} +) + +// Add the RegInfo to the reverse map +func addReverse(f Fs, fsInfo *RegInfo) { + typeToRegInfoMu.Lock() + defer typeToRegInfoMu.Unlock() + typeToRegInfo[Type(f)] = fsInfo +} + +// FindFromFs finds the *RegInfo used to create this Fs, provided +// it was created by fs.NewFs or cache.Get +// +// It returns nil if not found +func FindFromFs(f Fs) *RegInfo { + typeToRegInfoMu.Lock() + defer typeToRegInfoMu.Unlock() + return typeToRegInfo[Type(f)] +} From a58b482061bce6dda4f65c811c499d1aefbbd481 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 4 Jul 2022 09:25:49 +0100 Subject: [PATCH 136/560] fstests: fix Metadata tests on remotes with additional config --- fstest/fstests/fstests.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 7a72ef5f518d2..286a6787102d2 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -1396,8 +1396,8 @@ func Run(t *testing.T, opt *Opt) { obj := findObject(ctx, t, f, file1.Path) do, objectHasMetadata := obj.(fs.Metadataer) if objectHasMetadata || features.ReadMetadata || features.WriteMetadata || features.UserMetadata { - fsInfo, _, _, _, err := fs.ParseRemote(fs.ConfigString(f)) - require.NoError(t, err) + fsInfo := fs.FindFromFs(f) + require.NotNil(t, fsInfo) require.NotNil(t, fsInfo.MetadataInfo, "Object declares metadata support but no MetadataInfo in RegInfo") } if !objectHasMetadata { From 06182a344340758d9d4f499ac5808d1193f9600a Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 4 Jul 2022 09:41:46 +0100 Subject: [PATCH 137/560] s3: actually compress the payload for content-type gzip test --- backend/s3/s3_internal_test.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go index ecd39f41f3ec8..2568968d6700d 100644 --- a/backend/s3/s3_internal_test.go +++ b/backend/s3/s3_internal_test.go @@ -1,6 +1,8 @@ package s3 import ( + "bytes" + "compress/gzip" "context" "fmt" "testing" @@ -14,9 +16,20 @@ import ( "github.com/stretchr/testify/require" ) +func gz(t *testing.T, s string) string { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, err := zw.Write([]byte(s)) + require.NoError(t, err) + err = zw.Close() + require.NoError(t, err) + return buf.String() +} + func (f *Fs) InternalTestMetadata(t *testing.T) { ctx := context.Background() - contents := random.String(100) + contents := gz(t, random.String(1000)) + item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) btime := time.Now() metadata := fs.Metadata{ From 424a1f39eb376ccfc524b56f5dd835d1fa6a0405 Mon Sep 17 00:00:00 2001 From: Anthrazz <25553648+Anthrazz@users.noreply.github.com> Date: Mon, 4 Jul 2022 08:54:37 +0200 Subject: [PATCH 138/560] sftp: add Hetzner Storage Boxes to supported sftp backends --- docs/content/_index.md | 1 + docs/content/sftp.md | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/content/_index.md b/docs/content/_index.md index 58410d6d4330d..b6e2ac4d1c0ac 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -127,6 +127,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="Google Drive" home="https://www.google.com/drive/" config="/drive/" >}} {{< provider name="Google Photos" home="https://www.google.com/photos/about/" config="/googlephotos/" >}} {{< provider name="HDFS" home="https://hadoop.apache.org/" config="/hdfs/" >}} +{{< provider name="Hetzner Storage Box" home="https://www.hetzner.com/storage/storage-box" config="/sftp/#hetzner-storage-box" >}} {{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}} {{< provider name="Hubic" home="https://hubic.com/" config="/hubic/" >}} {{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}} diff --git a/docs/content/sftp.md b/docs/content/sftp.md index fa0f5ba054943..ed5eec9de3d68 100644 --- a/docs/content/sftp.md +++ b/docs/content/sftp.md @@ -11,6 +11,7 @@ Protocol](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol). The SFTP backend can be used with a number of different providers: {{< provider_list >}} +{{< provider name="Hetzner Storage Box" home="https://www.hetzner.com/storage/storage-box" config="/sftp/#hetzner-storage-box">}} {{< provider name="rsync.net" home="https://rsync.net/products/rclone.html" config="/sftp/#rsync-net">}} {{< /provider_list >}} @@ -25,7 +26,7 @@ would list the home directory of the user cofigured in the rclone remote config directory for remote machine (i.e. `/`) Note that some SFTP servers will need the leading / - Synology is a -good example of this. rsync.net, on the other hand, requires users to +good example of this. rsync.net and Hetzner, on the other hand, requires users to OMIT the leading /. Note that by default rclone will try to execute shell commands on @@ -792,3 +793,9 @@ Note that `--timeout` and `--contimeout` are both supported. rsync.net is supported through the SFTP backend. See [rsync.net's documentation of rclone examples](https://www.rsync.net/products/rclone.html). + +## Hetzner Storage Box {#hetzner-storage-box} + +Hetzner Storage Boxes are supported through the SFTP backend on port 23. + +See [Hetzner's documentation for details](https://docs.hetzner.com/robot/storage-box/access/access-ssh-rsync-borg#rclone) From 060c8dfff092f732aef79b9b46a46e32d07578ee Mon Sep 17 00:00:00 2001 From: zzr93 <34027824+zzr93@users.noreply.github.com> Date: Mon, 4 Jul 2022 17:18:04 +0800 Subject: [PATCH 139/560] operations: use correct src/dst in some log messages Most of the time this will make no difference to user logs, however the difference may be visible in JSON logs and on the rare occasions src and dst are pointing to different file names. --- fs/operations/operations.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 1ea41853fbe6d..e00a20a9e066f 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -89,7 +89,7 @@ func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash. return dstErr } if dstHash == "" { - fs.Debugf(src, "Dst hash empty - aborting Src hash check") + fs.Debugf(dst, "Dst hash empty - aborting Src hash check") return errNoHash } return nil @@ -100,11 +100,11 @@ func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash. } if srcErr != nil { err = fs.CountError(srcErr) - fs.Errorf(dst, "Failed to calculate src hash: %v", err) + fs.Errorf(src, "Failed to calculate src hash: %v", err) } if dstErr != nil { err = fs.CountError(dstErr) - fs.Errorf(src, "Failed to calculate dst hash: %v", err) + fs.Errorf(dst, "Failed to calculate dst hash: %v", err) } if err != nil { return false, ht, srcHash, dstHash, err From 0772cae3142dc153c056157dbba624ddecad2e02 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 16:01:19 +0200 Subject: [PATCH 140/560] staticcheck: use result of type assertion to simplify cases --- backend/union/union.go | 6 +++--- backend/union/upstream/upstream.go | 6 +++--- fs/operations/reopen.go | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/union/union.go b/backend/union/union.go index e1ec53e85eacc..c93fc06497927 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -85,16 +85,16 @@ func (f *Fs) wrapEntries(entries ...upstream.Entry) (entry, error) { if err != nil { return nil, err } - switch e.(type) { + switch e := e.(type) { case *upstream.Object: return &Object{ - Object: e.(*upstream.Object), + Object: e, fs: f, co: entries, }, nil case *upstream.Directory: return &Directory{ - Directory: e.(*upstream.Directory), + Directory: e, cd: entries, }, nil default: diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go index c6e906107620b..06f3b9014dc52 100644 --- a/backend/union/upstream/upstream.go +++ b/backend/union/upstream/upstream.go @@ -129,11 +129,11 @@ func (f *Fs) WrapObject(o fs.Object) *Object { // WrapEntry wraps an fs.DirEntry to include the info // of the upstream Fs func (f *Fs) WrapEntry(e fs.DirEntry) (Entry, error) { - switch e.(type) { + switch e := e.(type) { case fs.Object: - return f.WrapObject(e.(fs.Object)), nil + return f.WrapObject(e), nil case fs.Directory: - return f.WrapDirectory(e.(fs.Directory)), nil + return f.WrapDirectory(e), nil default: return nil, fmt.Errorf("unknown object type %T", e) } diff --git a/fs/operations/reopen.go b/fs/operations/reopen.go index 1f575d7cc6c33..e0f38cfcb1bf6 100644 --- a/fs/operations/reopen.go +++ b/fs/operations/reopen.go @@ -59,11 +59,11 @@ func (h *ReOpen) open() error { var hashOption *fs.HashesOption var rangeOption *fs.RangeOption for _, option := range h.options { - switch option.(type) { + switch option := option.(type) { case *fs.HashesOption: - hashOption = option.(*fs.HashesOption) + hashOption = option case *fs.RangeOption: - rangeOption = option.(*fs.RangeOption) + rangeOption = option case *fs.HTTPOption: opts = append(opts, option) default: From 3435bf7f34ed0f7821d4119630496ad91427378f Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 15:52:30 +0200 Subject: [PATCH 141/560] staticcheck: no value of type int64 is greater than math.MaxInt64 --- backend/union/upstream/upstream.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go index 06f3b9014dc52..35b477f7930ab 100644 --- a/backend/union/upstream/upstream.go +++ b/backend/union/upstream/upstream.go @@ -320,11 +320,7 @@ func (f *Fs) GetFreeSpace() (int64, error) { if f.usage.Free == nil { return math.MaxInt64 - 1, ErrUsageFieldNotSupported } - free := *f.usage.Free - if free >= math.MaxInt64 { - free = math.MaxInt64 - 1 - } - return free, nil + return *f.usage.Free, nil } // GetUsedSpace get the used space of the fs @@ -342,11 +338,7 @@ func (f *Fs) GetUsedSpace() (int64, error) { if f.usage.Used == nil { return 0, ErrUsageFieldNotSupported } - used := *f.usage.Used - if used >= math.MaxInt64 { - used = math.MaxInt64 - 1 - } - return used, nil + return *f.usage.Used, nil } // GetNumObjects get the number of objects of the fs From 7822df565e252a7df89e40d65e6d61a357b28150 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 16:02:02 +0200 Subject: [PATCH 142/560] staticcheck: unused func --- backend/azureblob/azureblob.go | 13 ------------- backend/drive/drive.go | 6 ------ backend/onedrive/onedrive.go | 10 ---------- backend/uptobox/uptobox.go | 5 ----- 4 files changed, 34 deletions(-) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 971bdc1d14fd5..a852bac565efd 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -1293,19 +1293,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, return f.NewObject(ctx, remote) } -func (f *Fs) getMemoryPool(size int64) *pool.Pool { - if size == int64(f.opt.ChunkSize) { - return f.pool - } - - return pool.New( - time.Duration(f.opt.MemoryPoolFlushTime), - int(size), - f.ci.Transfers, - f.opt.MemoryPoolUseMmap, - ) -} - // ------------------------------------------------------------ // Fs returns the parent Fs diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 918faddf95b4e..4e17f09df297d 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -3525,12 +3525,6 @@ func (o *baseObject) Size() int64 { return o.bytes } -// getRemoteInfo returns a drive.File for the remote -func (f *Fs) getRemoteInfo(ctx context.Context, remote string) (info *drive.File, err error) { - info, _, _, _, _, err = f.getRemoteInfoWithExport(ctx, remote) - return -} - // getRemoteInfoWithExport returns a drive.File and the export settings for the remote func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) ( info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) { diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index f8ab22a27480d..4e899addc8601 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -1760,16 +1760,6 @@ func (o *Object) rootPath() string { return o.fs.rootPath(o.remote) } -// srvPath returns a path for use in server given a remote -func (f *Fs) srvPath(remote string) string { - return f.opt.Enc.FromStandardPath(f.rootSlash() + remote) -} - -// srvPath returns a path for use in server -func (o *Object) srvPath() string { - return o.fs.srvPath(o.remote) -} - // Hash returns the SHA-1 of an object returning a lowercase hex string func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { if o.fs.driveType == driveTypePersonal { diff --git a/backend/uptobox/uptobox.go b/backend/uptobox/uptobox.go index e232ac83cd460..88faefce416f7 100644 --- a/backend/uptobox/uptobox.go +++ b/backend/uptobox/uptobox.go @@ -918,11 +918,6 @@ func (o *Object) Remote() string { return o.remote } -// Returns the full remote path for the object -func (o *Object) filePath() string { - return o.fs.dirPath(o.remote) -} - // ModTime returns the modification time of the object // // It attempts to read the objects mtime and if that isn't present the From 5b579cea47ce46fef393336042b4f1b8184c2640 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 16:33:45 +0200 Subject: [PATCH 143/560] staticcheck: use golang.org/x/text/cases instead of deprecated strings.Title strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0. The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead. --- cmd/help.go | 5 ++++- fs/operations/operations_test.go | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/help.go b/cmd/help.go index b5f1c222707dd..0d1295d34050f 100644 --- a/cmd/help.go +++ b/cmd/help.go @@ -17,6 +17,8 @@ import ( "github.com/rclone/rclone/lib/atexit" "github.com/spf13/cobra" "github.com/spf13/pflag" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) // Root is the main rclone command @@ -316,7 +318,8 @@ func showBackend(name string) { optionsType = "advanced" continue } - fmt.Printf("### %s options\n\n", strings.Title(optionsType)) + optionsType = cases.Title(language.Und, cases.NoLower).String(optionsType) + fmt.Printf("### %s options\n\n", optionsType) fmt.Printf("Here are the %s options specific to %s (%s).\n\n", optionsType, backend.Name, backend.Description) optionsType = "advanced" for _, opt := range opts { diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 8cd7296e4c2b2..34b88d6d14c03 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -47,6 +47,8 @@ import ( "github.com/rclone/rclone/fstest/fstests" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) // Some times used in the tests @@ -270,7 +272,7 @@ func TestHashSums(t *testing.T) { if !hashes.Contains(test.ht) { continue } - name := strings.Title(test.ht.String()) + name := cases.Title(language.Und, cases.NoLower).String(test.ht.String()) if test.download { name += "Download" } From 7b8c974decee9f9f673865798a06ac60a0e1ac79 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 17:20:08 +0200 Subject: [PATCH 144/560] staticcheck: ineffective break statement --- vfs/vfs.go | 1 - 1 file changed, 1 deletion(-) diff --git a/vfs/vfs.go b/vfs/vfs.go index fc4eb126442a1..304e23756e3ea 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -369,7 +369,6 @@ func (vfs *VFS) WaitForWriters(timeout time.Duration) { tick.Reset(tickTime) select { case <-tick.C: - break case <-deadline.C: fs.Errorf(nil, "Exiting even though %d writers active and %d cache items in use after %v\n%s", writers, cacheInUse, timeout, vfs.cache.Dump()) return From a1fd60ec2b296918befca263fa1986249a4840ef Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 17:20:31 +0200 Subject: [PATCH 145/560] staticcheck: empty branch --- backend/yandex/yandex.go | 8 ++++---- fstest/fstests/fstests.go | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.go index 7eea7187f2223..3097a8bc9ee40 100644 --- a/backend/yandex/yandex.go +++ b/backend/yandex/yandex.go @@ -782,11 +782,11 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string } _, err = f.readMetaDataForPath(ctx, dstPath, &api.ResourceInfoRequestOptions{}) - if apiErr, ok := err.(*api.ErrorResponse); ok { + if _, ok := err.(*api.ErrorResponse); ok { // does not exist - if apiErr.ErrorName == "DiskNotFoundError" { - // OK - } + //if apiErr.ErrorName == "DiskNotFoundError" { + // // OK + //} } else if err != nil { return err } else { diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 286a6787102d2..97e44ea1a298e 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -1459,9 +1459,7 @@ func Run(t *testing.T, opt *Opt) { assert.Equal(t, metaMimeType, gotMimeType) } }) - } else { - // Have some metadata here we didn't write - can't really check it! - } + } // else: Have some metadata here we didn't write - can't really check it! }) // TestObjectSetModTime tests that SetModTime works From 3e9c5eca3bfb3a4d3203126d4b26004fea78146f Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sat, 2 Jul 2022 16:33:04 +0200 Subject: [PATCH 146/560] yandex: handle api error on server-side move --- backend/yandex/yandex.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.go index 3097a8bc9ee40..7f0ddc5d8d4df 100644 --- a/backend/yandex/yandex.go +++ b/backend/yandex/yandex.go @@ -782,11 +782,10 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string } _, err = f.readMetaDataForPath(ctx, dstPath, &api.ResourceInfoRequestOptions{}) - if _, ok := err.(*api.ErrorResponse); ok { - // does not exist - //if apiErr.ErrorName == "DiskNotFoundError" { - // // OK - //} + if apiErr, ok := err.(*api.ErrorResponse); ok { + if apiErr.ErrorName != "DiskNotFoundError" { + return err + } } else if err != nil { return err } else { From c9d67c86fb69e9b0da0087963f9d722a223a2306 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 17:42:18 +0200 Subject: [PATCH 147/560] staticcheck: ignore suggestion to use context.TODO instead of nil when testing nil Context --- fs/config_test.go | 2 +- fs/filter/filter_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/config_test.go b/fs/config_test.go index 9813dc218990c..f9157bbf6e5f3 100644 --- a/fs/config_test.go +++ b/fs/config_test.go @@ -11,7 +11,7 @@ func TestGetConfig(t *testing.T) { ctx := context.Background() // Check nil - config := GetConfig(nil) + config := GetConfig(nil) //lint:ignore SA1012 we want to test passing a nil Context and therefore ignore lint suggestion of using context.TODO assert.Equal(t, globalConfig, config) // Check empty config diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go index 27da120c4e5e1..aada638fc5031 100644 --- a/fs/filter/filter_test.go +++ b/fs/filter/filter_test.go @@ -798,7 +798,7 @@ func TestGetConfig(t *testing.T) { ctx := context.Background() // Check nil - config := GetConfig(nil) + config := GetConfig(nil) //lint:ignore SA1012 we want to test passing a nil Context and therefore ignore lint suggestion of using context.TODO assert.Equal(t, globalConfig, config) // Check empty config From 1f9560e8739f14bacaa213ef54dbdcadc661e174 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 18:11:45 +0200 Subject: [PATCH 148/560] selfupdate: replace deprecated x/crypto/openpgp package with ProtonMail/go-crypto --- cmd/selfupdate/verify.go | 6 +++--- go.mod | 1 + go.sum | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/selfupdate/verify.go b/cmd/selfupdate/verify.go index 74f87eab538f5..503bbd7f6368d 100644 --- a/cmd/selfupdate/verify.go +++ b/cmd/selfupdate/verify.go @@ -10,9 +10,9 @@ import ( "fmt" "strings" + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/clearsign" "github.com/rclone/rclone/fs" - "golang.org/x/crypto/openpgp" - "golang.org/x/crypto/openpgp/clearsign" ) var ncwPublicKeyPGP = `-----BEGIN PGP PUBLIC KEY BLOCK----- @@ -59,7 +59,7 @@ func verifyHashsum(ctx context.Context, siteURL, version, archive string, hash [ } block, rest := clearsign.Decode(sumsBuf) // block.Bytes = block.Bytes[1:] // uncomment to test invalid signature - _, err = openpgp.CheckDetachedSignature(keyRing, bytes.NewReader(block.Bytes), block.ArmoredSignature.Body) + _, err = openpgp.CheckDetachedSignature(keyRing, bytes.NewReader(block.Bytes), block.ArmoredSignature.Body, nil) if err != nil || len(rest) > 0 { return errors.New("invalid hashsum signature") } diff --git a/go.mod b/go.mod index e61cdce97dbc6..53eff1fed2622 100644 --- a/go.mod +++ b/go.mod @@ -72,6 +72,7 @@ require ( require ( github.com/Microsoft/go-winio v0.5.1 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20220623141421-5afb4c282135 github.com/golang-jwt/jwt/v4 v4.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect diff --git a/go.sum b/go.sum index 9cc774dd30d10..008e5a7f9652c 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpz github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20220623141421-5afb4c282135 h1:xDc/cFH/hwyr9KyWc0sm26lpsscqtfZBvU8NpRLHwJ0= +github.com/ProtonMail/go-crypto v0.0.0-20220623141421-5afb4c282135/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= @@ -660,6 +662,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= From 9612ca61105c0138eeb939c35cd711f4dc955ced Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 18:34:37 +0200 Subject: [PATCH 149/560] staticcheck: ignore unused if platform dependent --- cmd/serve/docker/docker.go | 2 +- cmd/serve/docker/systemd_unsupported.go | 1 + cmd/serve/ftp/ftp.go | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/serve/docker/docker.go b/cmd/serve/docker/docker.go index b9702d87f3203..5e49718016f0d 100644 --- a/cmd/serve/docker/docker.go +++ b/cmd/serve/docker/docker.go @@ -20,7 +20,7 @@ var ( pluginName = "rclone" pluginScope = "local" baseDir = "/var/lib/docker-volumes/rclone" - sockDir = "/run/docker/plugins" // location of unix sockets (only relevant on Linux and FreeBSD) + sockDir = "/run/docker/plugins" //lint:ignore U1000 unused when not building linux defSpecDir = "/etc/docker/plugins" stateFile = "docker-plugin.state" socketAddr = "" // TCP listening address or empty string for Unix socket diff --git a/cmd/serve/docker/systemd_unsupported.go b/cmd/serve/docker/systemd_unsupported.go index e25d784c035eb..02b0b81a445c6 100644 --- a/cmd/serve/docker/systemd_unsupported.go +++ b/cmd/serve/docker/systemd_unsupported.go @@ -7,6 +7,7 @@ import ( "os" ) +//lint:ignore U1000 unused when not building linux func systemdActivationFiles() []*os.File { return nil } diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go index 05f7b092405d9..7ed96e3c0e385 100644 --- a/cmd/serve/ftp/ftp.go +++ b/cmd/serve/ftp/ftp.go @@ -184,7 +184,8 @@ func (s *server) serve() error { return s.srv.ListenAndServe() } -// serve runs the ftp server +// close stops the ftp server +//lint:ignore U1000 unused when not building linux func (s *server) close() error { fs.Logf(s.f, "Stopping FTP on %s", s.srv.Hostname+":"+strconv.Itoa(s.srv.Port)) return s.srv.Shutdown() From 92a43c5f7b554de970155e7cf20a693b49c72200 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 18:35:01 +0200 Subject: [PATCH 150/560] staticcheck: should use a simple channel send/receive instead of select with a single case --- fs/rc/jobs/job_test.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/fs/rc/jobs/job_test.go b/fs/rc/jobs/job_test.go index 3fcd4530bf94a..cb92e602aeaa8 100644 --- a/fs/rc/jobs/job_test.go +++ b/fs/rc/jobs/job_test.go @@ -107,21 +107,17 @@ var shortFn = func(ctx context.Context, in rc.Params) (rc.Params, error) { } var ctxFn = func(ctx context.Context, in rc.Params) (rc.Params, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - } + <-ctx.Done() + return nil, ctx.Err() } var ctxParmFn = func(paramCtx context.Context, returnError bool) func(ctx context.Context, in rc.Params) (rc.Params, error) { return func(ctx context.Context, in rc.Params) (rc.Params, error) { - select { - case <-paramCtx.Done(): - if returnError { - return nil, ctx.Err() - } - return rc.Params{}, nil + <-paramCtx.Done() + if returnError { + return nil, ctx.Err() } + return rc.Params{}, nil } } From 986bb176561be434a7cdb19df310ebb7d5f45296 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 18:36:46 +0200 Subject: [PATCH 151/560] staticcheck: awserr.BatchError is deprecated: Replaced with BatchedErrors --- backend/s3/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 1693446b08640..448b7c08f00b4 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -4288,7 +4288,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si func unWrapAwsError(err error) (found bool, outErr error) { if awsErr, ok := err.(awserr.Error); ok { var origErrs []error - if batchErr, ok := awsErr.(awserr.BatchError); ok { + if batchErr, ok := awsErr.(awserr.BatchedErrors); ok { origErrs = batchErr.OrigErrs() } else { origErrs = []error{awsErr.OrigErr()} From c70e890966debf03c8563d76e9ff865edfeede04 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 18:40:43 +0200 Subject: [PATCH 152/560] staticcheck: TLS config NameToCertificate is deprecated, should instead let library select the first compatible chain from Certificates --- fs/fshttp/http.go | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go index 9384d2fa4a9d1..953a313dd59f4 100644 --- a/fs/fshttp/http.go +++ b/fs/fshttp/http.go @@ -70,7 +70,6 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht log.Fatalf("Failed to load --client-cert/--client-key pair: %v", err) } t.TLSClientConfig.Certificates = []tls.Certificate{cert} - t.TLSClientConfig.BuildNameToCertificate() } // Load CA cert From f18095b0044d657719da9b188f0b0e15ed09a358 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 21:43:08 +0200 Subject: [PATCH 153/560] staticcheck: ignore deprecations that are not relevant --- lib/structs/structs_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/structs/structs_test.go b/lib/structs/structs_test.go index 9dffecf3f155c..d9c3bc67833b1 100644 --- a/lib/structs/structs_test.go +++ b/lib/structs/structs_test.go @@ -22,8 +22,8 @@ func TestSetDefaults(t *testing.T) { assert.Equal(t, ptr(old.Proxy), ptr(newT.Proxy), "when checking .Proxy") assert.Equal(t, ptr(old.DialContext), ptr(newT.DialContext), "when checking .DialContext") // Check the other public fields - assert.Equal(t, ptr(old.Dial), ptr(newT.Dial), "when checking .Dial") - assert.Equal(t, ptr(old.DialTLS), ptr(newT.DialTLS), "when checking .DialTLS") + assert.Equal(t, ptr(old.Dial), ptr(newT.Dial), "when checking .Dial") //lint:ignore SA1019 newT.Dial has been deprecated since Go 1.7: Use DialContext instead, which allows the transport to cancel dials as soon as they are no longer needed. If both are set, DialContext takes priority. + assert.Equal(t, ptr(old.DialTLS), ptr(newT.DialTLS), "when checking .DialTLS") //lint:ignore SA1019 old.DialTLS has been deprecated since Go 1.14: Use DialTLSContext instead, which allows the transport to cancel dials as soon as they are no longer needed. If both are set, DialTLSContext takes priority. assert.Equal(t, old.TLSClientConfig, newT.TLSClientConfig, "when checking .TLSClientConfig") assert.Equal(t, old.TLSHandshakeTimeout, newT.TLSHandshakeTimeout, "when checking .TLSHandshakeTimeout") assert.Equal(t, old.DisableKeepAlives, newT.DisableKeepAlives, "when checking .DisableKeepAlives") From e5bf6a813c75a6436041b15b97d624060b598eb1 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 24 Jun 2022 21:45:38 +0200 Subject: [PATCH 154/560] staticcheck: google api New is deprecated: please use NewService instead --- backend/drive/drive.go | 9 +++++---- backend/googlecloudstorage/googlecloudstorage.go | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 4e17f09df297d..baf2be267e839 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -51,6 +51,7 @@ import ( drive_v2 "google.golang.org/api/drive/v2" drive "google.golang.org/api/drive/v3" "google.golang.org/api/googleapi" + "google.golang.org/api/option" ) // Constants @@ -1210,13 +1211,13 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err // Create a new authorized Drive client. f.client = oAuthClient - f.svc, err = drive.New(f.client) + f.svc, err = drive.NewService(context.Background(), option.WithHTTPClient(f.client)) if err != nil { return nil, fmt.Errorf("couldn't create Drive client: %w", err) } if f.opt.V2DownloadMinSize >= 0 { - f.v2Svc, err = drive_v2.New(f.client) + f.v2Svc, err = drive_v2.NewService(context.Background(), option.WithHTTPClient(f.client)) if err != nil { return nil, fmt.Errorf("couldn't create Drive v2 client: %w", err) } @@ -2992,12 +2993,12 @@ func (f *Fs) changeServiceAccountFile(ctx context.Context, file string) (err err return fmt.Errorf("drive: failed when making oauth client: %w", err) } f.client = oAuthClient - f.svc, err = drive.New(f.client) + f.svc, err = drive.NewService(context.Background(), option.WithHTTPClient(f.client)) if err != nil { return fmt.Errorf("couldn't create Drive client: %w", err) } if f.opt.V2DownloadMinSize >= 0 { - f.v2Svc, err = drive_v2.New(f.client) + f.v2Svc, err = drive_v2.NewService(context.Background(), option.WithHTTPClient(f.client)) if err != nil { return fmt.Errorf("couldn't create Drive v2 client: %w", err) } diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index e010dfd88bdd7..b11771d1b8351 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -44,6 +44,7 @@ import ( "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/api/googleapi" + option "google.golang.org/api/option" // NOTE: This API is deprecated storage "google.golang.org/api/storage/v1" @@ -524,7 +525,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // Create a new authorized Drive client. f.client = oAuthClient - f.svc, err = storage.New(f.client) + f.svc, err = storage.NewService(context.Background(), option.WithHTTPClient(f.client)) if err != nil { return nil, fmt.Errorf("couldn't create Google Cloud Storage client: %w", err) } From ad8c94e9820111b612c7d2984be02120f9da0511 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sat, 2 Jul 2022 16:07:54 +0200 Subject: [PATCH 155/560] staticcheck: redundant return statement --- lib/cache/cache.go | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cache/cache.go b/lib/cache/cache.go index b8d323ff1276a..8e03819bc2152 100644 --- a/lib/cache/cache.go +++ b/lib/cache/cache.go @@ -236,7 +236,6 @@ func (c *Cache) Clear() { delete(c.cache, key) } c.mu.Unlock() - return } // Entries returns the number of entries in the cache From 5c6a958ad88db85471fc8d1975a5378f67e16fc1 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sat, 2 Jul 2022 16:11:22 +0200 Subject: [PATCH 156/560] go mod tidy: github.com/pkg/xattr should be direct --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 53eff1fed2622..8f301a4df5ce3 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af - github.com/pkg/xattr v0.4.7 // indirect + github.com/pkg/xattr v0.4.7 golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 ) From 3ec07d5db9f932335c1a33e30d08042012a5753d Mon Sep 17 00:00:00 2001 From: Paul Norman Date: Tue, 5 Jul 2022 20:12:57 -0700 Subject: [PATCH 157/560] docs: fix typo in license webpage --- MANUAL.html | 2 +- MANUAL.md | 2 +- MANUAL.txt | 2 +- README.md | 2 +- docs/content/licence.md | 2 +- rclone.1 | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/MANUAL.html b/MANUAL.html index 7b39794fbedda..eb2a7d1970566 100644 --- a/MANUAL.html +++ b/MANUAL.html @@ -30541,7 +30541,7 @@

Rclone

For example: On a Windows system, you have a file with name Test：1.jpg, where ： is the Unicode fullwidth colon symbol. When using rclone to copy this to your Google Drive, you will notice that the file gets renamed to Test:1.jpg, where : is the regular (halfwidth) colon.

The reason for such renames is the way rclone handles different restricted filenames on different cloud storage systems. It tries to avoid ambiguous file names as much and allow moving files between many cloud storage systems transparently, by replacing invalid characters with similar looking Unicode characters when transferring to one storage system, and replacing back again when transferring to a different storage system where the original characters are supported. When the same Unicode characters are intentionally used in file names, this replacement strategy leads to unwanted renames. Read more here.

License

-

This is free software under the terms of MIT the license (check the COPYING file included with the source code).

+

This is free software under the terms of the MIT license (check the COPYING file included with the source code).

Copyright (C) 2019 by Nick Craig-Wood https://www.craig-wood.com/nick/
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
diff --git a/MANUAL.md b/MANUAL.md
index 06b37f0d201e5..58e4ddad6a03d 100644
--- a/MANUAL.md
+++ b/MANUAL.md
@@ -40150,7 +40150,7 @@ to unwanted renames. Read more [here](https://rclone.org/overview/#restricted-fi
 
 # License
 
-This is free software under the terms of MIT the license (check the
+This is free software under the terms of the MIT license (check the
 COPYING file included with the source code).
 
 ```
diff --git a/MANUAL.txt b/MANUAL.txt
index 994bd6612bec6..d25a312cb36eb 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -40651,7 +40651,7 @@ replacement strategy leads to unwanted renames. Read more here.
 
 License
 
-This is free software under the terms of MIT the license (check the
+This is free software under the terms of the MIT license (check the
 COPYING file included with the source code).
 
     Copyright (C) 2019 by Nick Craig-Wood https://www.craig-wood.com/nick/
diff --git a/README.md b/README.md
index ba85e16bdd11b..01776699e2f1f 100644
--- a/README.md
+++ b/README.md
@@ -132,5 +132,5 @@ Please see the [rclone website](https://rclone.org/) for:
 License
 -------
 
-This is free software under the terms of MIT the license (check the
+This is free software under the terms of the MIT license (check the
 [COPYING file](/COPYING) included in this package).
diff --git a/docs/content/licence.md b/docs/content/licence.md
index a12d71de4ea5a..004522223c0c2 100644
--- a/docs/content/licence.md
+++ b/docs/content/licence.md
@@ -5,7 +5,7 @@ description: "Rclone Licence"
 
 # License
 
-This is free software under the terms of MIT the license (check the
+This is free software under the terms of the MIT license (check the
 COPYING file included with the source code).
 
 ```
diff --git a/rclone.1 b/rclone.1
index 157f41d569313..693b003af1d88 100644
--- a/rclone.1
+++ b/rclone.1
@@ -59039,7 +59039,7 @@ Read more
 here (https://rclone.org/overview/#restricted-filenames-caveats).
 .SH License
 .PP
-This is free software under the terms of MIT the license (check the
+This is free software under the terms of the MIT license (check the
 COPYING file included with the source code).
 .IP
 .nf

From b5efffee9d53eb6e7801011846cb248feb9c83a7 Mon Sep 17 00:00:00 2001
From: Lorenzo Maiorfi 
Date: Wed, 6 Jul 2022 12:54:04 +0200
Subject: [PATCH 158/560] azureblob: allow remote emulator (azurite) - fixes
 #6290

---
 backend/azureblob/azureblob.go | 6 +++++-
 docs/content/azureblob.md      | 9 +++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index a852bac565efd..5597ed21fd6c1 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -600,7 +600,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed to parse credentials: %w", err)
 		}
-		u, err = url.Parse(emulatorBlobEndpoint)
+		var actualEmulatorEndpoint = emulatorBlobEndpoint
+		if opt.Endpoint != "" {
+			actualEmulatorEndpoint = opt.Endpoint
+		}
+		u, err = url.Parse(actualEmulatorEndpoint)
 		if err != nil {
 			return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
 		}
diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md
index a8151912bc44d..fa2205ae58993 100644
--- a/docs/content/azureblob.md
+++ b/docs/content/azureblob.md
@@ -526,7 +526,8 @@ See [List of backends that do not support rclone about](https://rclone.org/overv
 
 ## Azure Storage Emulator Support
 
-You can test rclone with storage emulator locally, to do this make sure azure storage emulator
-installed locally and set up a new remote with `rclone config` follow instructions described in
-introduction, set `use_emulator` config as `true`, you do not need to provide default account name
-or key if using emulator.
+You can run rclone with storage emulator (usually _azurite_).
+
+To do this, just set up a new remote with `rclone config` following instructions described in introduction and set `use_emulator` config as `true`. You do not need to provide default account name neither an account key.
+
+Also, if you want to access a storage emulator instance running on a different machine, you can override _Endpoint_ parameter in advanced settings, setting it to `http(s)://:/devstoreaccount1` (e.g. `http://10.254.2.5:10000/devstoreaccount1`).

From 388da82762ba24cef13c995d572ea9dc55dcf9aa Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 7 Jul 2022 16:08:59 +0100
Subject: [PATCH 159/560] Add Anthrazz to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 66f33c154de12..191a7aad6eeee 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -620,3 +620,4 @@ put them back in again.` >}}
   * buda 
   * mirekphd <36706320+mirekphd@users.noreply.github.com>
   * vyloy 
+  * Anthrazz <25553648+Anthrazz@users.noreply.github.com>

From 0db50ecb2f3111d0a91284ddcecb4a5d861382f5 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 7 Jul 2022 16:08:59 +0100
Subject: [PATCH 160/560] Add zzr93 to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 191a7aad6eeee..29d29693d81ad 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -621,3 +621,4 @@ put them back in again.` >}}
   * mirekphd <36706320+mirekphd@users.noreply.github.com>
   * vyloy 
   * Anthrazz <25553648+Anthrazz@users.noreply.github.com>
+  * zzr93 <34027824+zzr93@users.noreply.github.com>

From a9c531b9eb3ba2651b466351fd45448fb62f1bbd Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 7 Jul 2022 16:08:59 +0100
Subject: [PATCH 161/560] Add Paul Norman to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 29d29693d81ad..de83c68df95b3 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -622,3 +622,4 @@ put them back in again.` >}}
   * vyloy 
   * Anthrazz <25553648+Anthrazz@users.noreply.github.com>
   * zzr93 <34027824+zzr93@users.noreply.github.com>
+  * Paul Norman 

From 2515039e18d694e9ef02a41a7b2f42ca1cd9bbfd Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 7 Jul 2022 16:08:59 +0100
Subject: [PATCH 162/560] Add Lorenzo Maiorfi to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index de83c68df95b3..d4129e3756f09 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -623,3 +623,4 @@ put them back in again.` >}}
   * Anthrazz <25553648+Anthrazz@users.noreply.github.com>
   * zzr93 <34027824+zzr93@users.noreply.github.com>
   * Paul Norman 
+  * Lorenzo Maiorfi 

From 2e54b56a01b14da2e488ffb3ffde47b399f08866 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 7 Jul 2022 11:31:53 +0100
Subject: [PATCH 163/560] rcat: check checksums by default like copy does #6305

Before this change we were calculating the checksum for an rcat
transfer but never checking it.

See: https://forum.rclone.org/t/optimize-rclone-on-raspberry-pi-4-8gb/31741
---
 fs/operations/operations.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index e00a20a9e066f..0b0f9f75e01d9 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -1391,11 +1391,14 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
 
 	compare := func(dst fs.Object) error {
 		var sums map[hash.Type]string
+		opt := defaultEqualOpt(ctx)
 		if hasher != nil {
+			// force --checksum on if we have hashes
+			opt.checkSum = true
 			sums = hasher.Sums()
 		}
 		src := object.NewStaticObjectInfo(dstFileName, modTime, int64(readCounter.BytesRead()), false, sums, fdst)
-		if !Equal(ctx, src, dst) {
+		if !equal(ctx, src, dst, opt) {
 			err = fmt.Errorf("corrupted on transfer")
 			err = fs.CountError(err)
 			fs.Errorf(dst, "%v", err)

From 62bcc84f6fc437e310599d4e5b4692f45bab0d4a Mon Sep 17 00:00:00 2001
From: Claudio Maradonna 
Date: Tue, 5 Jul 2022 18:29:14 +0200
Subject: [PATCH 164/560] vfs: add --vfs-disk-space-total-size option to
 manually set the total disk space

Now you can specify --vfs-disk-space-total-size to set the total disk
space (default to -1)

fixes #3270
---
 vfs/help.go              |  7 +++
 vfs/vfs.go               |  6 +++
 vfs/vfscommon/options.go | 96 ++++++++++++++++++++--------------------
 vfs/vfsflags/vfsflags.go |  1 +
 4 files changed, 63 insertions(+), 47 deletions(-)

diff --git a/vfs/help.go b/vfs/help.go
index 01382cf5a1770..3bb809ca8b010 100644
--- a/vfs/help.go
+++ b/vfs/help.go
@@ -307,6 +307,13 @@ If the flag is not provided on the command line, then its default value depends
 on the operating system where rclone runs: "true" on Windows and macOS, "false"
 otherwise. If the flag is provided without a value, then it is "true".
 
+### VFS Disk Options
+
+This flag allows you to manually set the statistics about the filing system.
+It can be useful when those statistics cannot be read correctly automatically.
+
+    --vfs-disk-space-total-size    Manually set the total disk space size (example: 256G, default: -1)
+
 ### Alternate report of used bytes
 
 Some backends, most notably S3, do not report the amount of bytes used.
diff --git a/vfs/vfs.go b/vfs/vfs.go
index 304e23756e3ea..b6dc141b3f854 100644
--- a/vfs/vfs.go
+++ b/vfs/vfs.go
@@ -604,6 +604,7 @@ func (vfs *VFS) Statfs() (total, used, free int64) {
 			return
 		}
 	}
+
 	if u := vfs.usage; u != nil {
 		if u.Total != nil {
 			total = *u.Total
@@ -615,6 +616,11 @@ func (vfs *VFS) Statfs() (total, used, free int64) {
 			used = *u.Used
 		}
 	}
+
+	if int64(vfs.Opt.DiskSpaceTotalSize) >= 0 {
+		total = int64(vfs.Opt.DiskSpaceTotalSize)
+	}
+
 	total, used, free = fillInMissingSizes(total, used, free, unknownFreeBytes)
 	return
 }
diff --git a/vfs/vfscommon/options.go b/vfs/vfscommon/options.go
index 5c1ae7900e669..c3631859ad0ae 100644
--- a/vfs/vfscommon/options.go
+++ b/vfs/vfscommon/options.go
@@ -10,57 +10,59 @@ import (
 
 // Options is options for creating the vfs
 type Options struct {
-	NoSeek            bool          // don't allow seeking if set
-	NoChecksum        bool          // don't check checksums if set
-	ReadOnly          bool          // if set VFS is read only
-	NoModTime         bool          // don't read mod times for files
-	DirCacheTime      time.Duration // how long to consider directory listing cache valid
-	PollInterval      time.Duration
-	Umask             int
-	UID               uint32
-	GID               uint32
-	DirPerms          os.FileMode
-	FilePerms         os.FileMode
-	ChunkSize         fs.SizeSuffix // if > 0 read files in chunks
-	ChunkSizeLimit    fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached
-	CacheMode         CacheMode
-	CacheMaxAge       time.Duration
-	CacheMaxSize      fs.SizeSuffix
-	CachePollInterval time.Duration
-	CaseInsensitive   bool
-	WriteWait         time.Duration // time to wait for in-sequence write
-	ReadWait          time.Duration // time to wait for in-sequence read
-	WriteBack         time.Duration // time to wait before writing back dirty files
-	ReadAhead         fs.SizeSuffix // bytes to read ahead in cache mode "full"
-	UsedIsSize        bool          // if true, use the `rclone size` algorithm for Used size
-	FastFingerprint   bool          // if set use fast fingerprints
+	NoSeek             bool          // don't allow seeking if set
+	NoChecksum         bool          // don't check checksums if set
+	ReadOnly           bool          // if set VFS is read only
+	NoModTime          bool          // don't read mod times for files
+	DirCacheTime       time.Duration // how long to consider directory listing cache valid
+	PollInterval       time.Duration
+	Umask              int
+	UID                uint32
+	GID                uint32
+	DirPerms           os.FileMode
+	FilePerms          os.FileMode
+	ChunkSize          fs.SizeSuffix // if > 0 read files in chunks
+	ChunkSizeLimit     fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached
+	CacheMode          CacheMode
+	CacheMaxAge        time.Duration
+	CacheMaxSize       fs.SizeSuffix
+	CachePollInterval  time.Duration
+	CaseInsensitive    bool
+	WriteWait          time.Duration // time to wait for in-sequence write
+	ReadWait           time.Duration // time to wait for in-sequence read
+	WriteBack          time.Duration // time to wait before writing back dirty files
+	ReadAhead          fs.SizeSuffix // bytes to read ahead in cache mode "full"
+	UsedIsSize         bool          // if true, use the `rclone size` algorithm for Used size
+	FastFingerprint    bool          // if set use fast fingerprints
+	DiskSpaceTotalSize fs.SizeSuffix
 }
 
 // DefaultOpt is the default values uses for Opt
 var DefaultOpt = Options{
-	NoModTime:         false,
-	NoChecksum:        false,
-	NoSeek:            false,
-	DirCacheTime:      5 * 60 * time.Second,
-	PollInterval:      time.Minute,
-	ReadOnly:          false,
-	Umask:             0,
-	UID:               ^uint32(0), // these values instruct WinFSP-FUSE to use the current user
-	GID:               ^uint32(0), // overridden for non windows in mount_unix.go
-	DirPerms:          os.FileMode(0777),
-	FilePerms:         os.FileMode(0666),
-	CacheMode:         CacheModeOff,
-	CacheMaxAge:       3600 * time.Second,
-	CachePollInterval: 60 * time.Second,
-	ChunkSize:         128 * fs.Mebi,
-	ChunkSizeLimit:    -1,
-	CacheMaxSize:      -1,
-	CaseInsensitive:   runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
-	WriteWait:         1000 * time.Millisecond,
-	ReadWait:          20 * time.Millisecond,
-	WriteBack:         5 * time.Second,
-	ReadAhead:         0 * fs.Mebi,
-	UsedIsSize:        false,
+	NoModTime:          false,
+	NoChecksum:         false,
+	NoSeek:             false,
+	DirCacheTime:       5 * 60 * time.Second,
+	PollInterval:       time.Minute,
+	ReadOnly:           false,
+	Umask:              0,
+	UID:                ^uint32(0), // these values instruct WinFSP-FUSE to use the current user
+	GID:                ^uint32(0), // overridden for non windows in mount_unix.go
+	DirPerms:           os.FileMode(0777),
+	FilePerms:          os.FileMode(0666),
+	CacheMode:          CacheModeOff,
+	CacheMaxAge:        3600 * time.Second,
+	CachePollInterval:  60 * time.Second,
+	ChunkSize:          128 * fs.Mebi,
+	ChunkSizeLimit:     -1,
+	CacheMaxSize:       -1,
+	CaseInsensitive:    runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
+	WriteWait:          1000 * time.Millisecond,
+	ReadWait:           20 * time.Millisecond,
+	WriteBack:          5 * time.Second,
+	ReadAhead:          0 * fs.Mebi,
+	UsedIsSize:         false,
+	DiskSpaceTotalSize: -1,
 }
 
 // Init the options, making sure everything is withing range
diff --git a/vfs/vfsflags/vfsflags.go b/vfs/vfsflags/vfsflags.go
index 69efab86c8c6c..ccdcb17db7162 100644
--- a/vfs/vfsflags/vfsflags.go
+++ b/vfs/vfsflags/vfsflags.go
@@ -39,5 +39,6 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.FVarP(flagSet, &Opt.ReadAhead, "vfs-read-ahead", "", "Extra read ahead over --buffer-size when using cache-mode full")
 	flags.BoolVarP(flagSet, &Opt.UsedIsSize, "vfs-used-is-size", "", Opt.UsedIsSize, "Use the `rclone size` algorithm for Used size")
 	flags.BoolVarP(flagSet, &Opt.FastFingerprint, "vfs-fast-fingerprint", "", Opt.FastFingerprint, "Use fast (less accurate) fingerprints for change detection")
+	flags.FVarP(flagSet, &Opt.DiskSpaceTotalSize, "vfs-disk-space-total-size", "", "Specify the total space of disk")
 	platformFlags(flagSet)
 }

From 53400d7edcfe0f8b78584e888dd336bc88fdc651 Mon Sep 17 00:00:00 2001
From: Ovidiu Victor Tatar 
Date: Wed, 3 Mar 2021 11:33:29 +0100
Subject: [PATCH 165/560] oauthlib: add method to set a token as expired

This can be used by backends to trigger a refresh of an access token if
they detect an invalid token.
---
 lib/oauthutil/oauthutil.go | 17 +++++++++++++++++
 lib/oauthutil/renew.go     |  5 +++++
 2 files changed, 22 insertions(+)

diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go
index 53a08900c0bcc..2033ac5f55184 100644
--- a/lib/oauthutil/oauthutil.go
+++ b/lib/oauthutil/oauthutil.go
@@ -269,6 +269,23 @@ func (ts *TokenSource) Invalidate() {
 	ts.mu.Unlock()
 }
 
+// Expire marks the token as expired
+//
+// This also marks the token in the config file as expired, if it is the same one
+func (ts *TokenSource) Expire() error {
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	ts.token.Expiry = time.Now().Add(time.Hour * (-1)) // expire token
+	t, err := GetToken(ts.name, ts.m)
+	if err != nil {
+		return err
+	}
+	if t.AccessToken == ts.token.AccessToken {
+		err = PutToken(ts.name, ts.m, ts.token, false)
+	}
+	return err
+}
+
 // timeToExpiry returns how long until the token expires
 //
 // Call with the lock held
diff --git a/lib/oauthutil/renew.go b/lib/oauthutil/renew.go
index 1fb96e77129c2..29786cf69aaf3 100644
--- a/lib/oauthutil/renew.go
+++ b/lib/oauthutil/renew.go
@@ -67,3 +67,8 @@ func (r *Renew) Stop() {
 func (r *Renew) Invalidate() {
 	r.ts.Invalidate()
 }
+
+// Expire expires the token source
+func (r *Renew) Expire() error {
+	return r.ts.Expire()
+}

From 502226bfc8c681e556eabbcd88d5c6e695b6434b Mon Sep 17 00:00:00 2001
From: Ovidiu Victor Tatar 
Date: Thu, 7 Jul 2022 19:58:21 +0200
Subject: [PATCH 166/560] pacer: add ZeroDelayCalculator

---
 lib/pacer/pacers.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/pacer/pacers.go b/lib/pacer/pacers.go
index d510eea6436c2..94ef86ccbcf79 100644
--- a/lib/pacer/pacers.go
+++ b/lib/pacer/pacers.go
@@ -104,6 +104,15 @@ func (c *Default) Calculate(state State) time.Duration {
 	return sleepTime
 }
 
+// ZeroDelayCalculator is a Calculator that never delays.
+type ZeroDelayCalculator struct {
+}
+
+// Calculate takes the current Pacer state and return the wait time until the next try.
+func (c *ZeroDelayCalculator) Calculate(state State) time.Duration {
+	return 0
+}
+
 // AmazonCloudDrive is a specialized pacer for Amazon Drive
 //
 // It implements a truncated exponential backoff strategy with randomization.

From b4d847cadd782b2ea4cc497a88b75ac6c2292e0b Mon Sep 17 00:00:00 2001
From: Ovidiu Victor Tatar 
Date: Thu, 7 Jul 2022 19:58:22 +0200
Subject: [PATCH 167/560] new backend: hidrive - fixes #1069

---
 README.md                                     |    1 +
 backend/all/all.go                            |    1 +
 backend/hidrive/api/queries.go                |   81 ++
 backend/hidrive/api/types.go                  |  135 +++
 backend/hidrive/helpers.go                    |  888 +++++++++++++++
 backend/hidrive/hidrive.go                    | 1002 +++++++++++++++++
 backend/hidrive/hidrive_test.go               |   45 +
 backend/hidrive/hidrivehash/hidrivehash.go    |  410 +++++++
 .../hidrive/hidrivehash/hidrivehash_test.go   |  395 +++++++
 .../hidrive/hidrivehash/internal/internal.go  |   17 +
 bin/make_manual.py                            |    1 +
 docs/content/_index.md                        |    1 +
 docs/content/docs.md                          |    1 +
 docs/content/hidrive.md                       |  461 ++++++++
 docs/content/overview.md                      |    7 +
 docs/layouts/chrome/navbar.html               |    1 +
 fstest/test_all/config.yaml                   |    3 +
 17 files changed, 3450 insertions(+)
 create mode 100644 backend/hidrive/api/queries.go
 create mode 100644 backend/hidrive/api/types.go
 create mode 100644 backend/hidrive/helpers.go
 create mode 100644 backend/hidrive/hidrive.go
 create mode 100644 backend/hidrive/hidrive_test.go
 create mode 100644 backend/hidrive/hidrivehash/hidrivehash.go
 create mode 100644 backend/hidrive/hidrivehash/hidrivehash_test.go
 create mode 100644 backend/hidrive/hidrivehash/internal/internal.go
 create mode 100644 docs/content/hidrive.md

diff --git a/README.md b/README.md
index 01776699e2f1f..7f2302f8998b5 100644
--- a/README.md
+++ b/README.md
@@ -42,6 +42,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
   * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
   * HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
+  * HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
   * HTTP [:page_facing_up:](https://rclone.org/http/)
   * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
   * Hubic [:page_facing_up:](https://rclone.org/hubic/)
diff --git a/backend/all/all.go b/backend/all/all.go
index 44414b2879dd7..dc17911e4c349 100644
--- a/backend/all/all.go
+++ b/backend/all/all.go
@@ -21,6 +21,7 @@ import (
 	_ "github.com/rclone/rclone/backend/googlephotos"
 	_ "github.com/rclone/rclone/backend/hasher"
 	_ "github.com/rclone/rclone/backend/hdfs"
+	_ "github.com/rclone/rclone/backend/hidrive"
 	_ "github.com/rclone/rclone/backend/http"
 	_ "github.com/rclone/rclone/backend/hubic"
 	_ "github.com/rclone/rclone/backend/internetarchive"
diff --git a/backend/hidrive/api/queries.go b/backend/hidrive/api/queries.go
new file mode 100644
index 0000000000000..57a1477c1ebdb
--- /dev/null
+++ b/backend/hidrive/api/queries.go
@@ -0,0 +1,81 @@
+package api
+
+import (
+	"encoding/json"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+)
+
+// Some presets for different amounts of information that can be requested for fields;
+// it is recommended to only request the information that is actually needed.
+var (
+	HiDriveObjectNoMetadataFields            = []string{"name", "type"}
+	HiDriveObjectWithMetadataFields          = append(HiDriveObjectNoMetadataFields, "id", "size", "mtime", "chash")
+	HiDriveObjectWithDirectoryMetadataFields = append(HiDriveObjectWithMetadataFields, "nmembers")
+	DirectoryContentFields                   = []string{"nmembers"}
+)
+
+// QueryParameters represents the parameters passed to an API-call.
+type QueryParameters struct {
+	url.Values
+}
+
+// NewQueryParameters initializes an instance of QueryParameters and
+// returns a pointer to it.
+func NewQueryParameters() *QueryParameters {
+	return &QueryParameters{url.Values{}}
+}
+
+// SetFileInDirectory sets the appropriate parameters
+// to specify a path to a file in a directory.
+// This is used by requests that work with paths for files that do not exist yet.
+// (For example when creating a file).
+// Most requests use the format produced by SetPath(...).
+func (p *QueryParameters) SetFileInDirectory(filePath string) {
+	directory, file := path.Split(path.Clean(filePath))
+	p.Set("dir", path.Clean(directory))
+	p.Set("name", file)
+	// NOTE: It would be possible to switch to pid-based requests
+	// by modifying this function.
+}
+
+// SetPath sets the appropriate parameters to access the given path.
+func (p *QueryParameters) SetPath(objectPath string) {
+	p.Set("path", path.Clean(objectPath))
+	// NOTE: It would be possible to switch to pid-based requests
+	// by modifying this function.
+}
+
+// SetTime sets the key to the time-value. It replaces any existing values.
+func (p *QueryParameters) SetTime(key string, value time.Time) error {
+	valueAPI := Time(value)
+	valueBytes, err := json.Marshal(&valueAPI)
+	if err != nil {
+		return err
+	}
+	p.Set(key, string(valueBytes))
+	return nil
+}
+
+// AddList adds the given values as a list
+// with each value separated by the separator.
+// It appends to any existing values associated with key.
+func (p *QueryParameters) AddList(key string, separator string, values ...string) {
+	original := p.Get(key)
+	p.Set(key, strings.Join(values, separator))
+	if original != "" {
+		p.Set(key, original+separator+p.Get(key))
+	}
+}
+
+// AddFields sets the appropriate parameter to access the given fields.
+// The given fields will be appended to any other existing fields.
+func (p *QueryParameters) AddFields(prefix string, fields ...string) {
+	modifiedFields := make([]string, len(fields))
+	for i, field := range fields {
+		modifiedFields[i] = prefix + field
+	}
+	p.AddList("fields", ",", modifiedFields...)
+}
diff --git a/backend/hidrive/api/types.go b/backend/hidrive/api/types.go
new file mode 100644
index 0000000000000..4cc912a7285bd
--- /dev/null
+++ b/backend/hidrive/api/types.go
@@ -0,0 +1,135 @@
+// Package api has type definitions and code related to API-calls for the HiDrive-API.
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strconv"
+	"time"
+)
+
+// Time represents date and time information for the API.
+type Time time.Time
+
+// MarshalJSON turns Time into JSON (in Unix-time/UTC).
+func (t *Time) MarshalJSON() ([]byte, error) {
+	secs := time.Time(*t).Unix()
+	return []byte(strconv.FormatInt(secs, 10)), nil
+}
+
+// UnmarshalJSON turns JSON into Time.
+func (t *Time) UnmarshalJSON(data []byte) error {
+	secs, err := strconv.ParseInt(string(data), 10, 64)
+	if err != nil {
+		return err
+	}
+	*t = Time(time.Unix(secs, 0))
+	return nil
+}
+
+// Error is returned from the API when things go wrong.
+type Error struct {
+	Code        json.Number `json:"code"`
+	ContextInfo json.RawMessage
+	Message     string `json:"msg"`
+}
+
+// Error returns a string for the error and satisfies the error interface.
+func (e *Error) Error() string {
+	out := fmt.Sprintf("Error %q", e.Code.String())
+	if e.Message != "" {
+		out += ": " + e.Message
+	}
+	if e.ContextInfo != nil {
+		out += fmt.Sprintf(" (%+v)", e.ContextInfo)
+	}
+	return out
+}
+
+// Check Error satisfies the error interface.
+var _ error = (*Error)(nil)
+
+// possible types for HiDriveObject
+const (
+	HiDriveObjectTypeDirectory = "dir"
+	HiDriveObjectTypeFile      = "file"
+	HiDriveObjectTypeSymlink   = "symlink"
+)
+
+// HiDriveObject describes a folder, a symlink or a file.
+// Depending on the type and content, not all fields are present.
+type HiDriveObject struct {
+	Type         string `json:"type"`
+	ID           string `json:"id"`
+	ParentID     string `json:"parent_id"`
+	Name         string `json:"name"`
+	Path         string `json:"path"`
+	Size         int64  `json:"size"`
+	MemberCount  int64  `json:"nmembers"`
+	ModifiedAt   Time   `json:"mtime"`
+	ChangedAt    Time   `json:"ctime"`
+	MetaHash     string `json:"mhash"`
+	MetaOnlyHash string `json:"mohash"`
+	NameHash     string `json:"nhash"`
+	ContentHash  string `json:"chash"`
+	IsTeamfolder bool   `json:"teamfolder"`
+	Readable     bool   `json:"readable"`
+	Writable     bool   `json:"writable"`
+	Shareable    bool   `json:"shareable"`
+	MIMEType     string `json:"mime_type"`
+}
+
+// ModTime returns the modification time of the HiDriveObject.
+func (i *HiDriveObject) ModTime() time.Time {
+	t := time.Time(i.ModifiedAt)
+	if t.IsZero() {
+		t = time.Time(i.ChangedAt)
+	}
+	return t
+}
+
+// UnmarshalJSON turns JSON into HiDriveObject and
+// introduces specific default-values where necessary.
+func (i *HiDriveObject) UnmarshalJSON(data []byte) error {
+	type objectAlias HiDriveObject
+	defaultObject := objectAlias{
+		Size:        -1,
+		MemberCount: -1,
+	}
+
+	err := json.Unmarshal(data, &defaultObject)
+	if err != nil {
+		return err
+	}
+	name, err := url.PathUnescape(defaultObject.Name)
+	if err == nil {
+		defaultObject.Name = name
+	}
+
+	*i = HiDriveObject(defaultObject)
+	return nil
+}
+
+// DirectoryContent describes the content of a directory.
+type DirectoryContent struct {
+	TotalCount int64           `json:"nmembers"`
+	Entries    []HiDriveObject `json:"members"`
+}
+
+// UnmarshalJSON turns JSON into DirectoryContent and
+// introduces specific default-values where necessary.
+func (d *DirectoryContent) UnmarshalJSON(data []byte) error {
+	type directoryContentAlias DirectoryContent
+	defaultDirectoryContent := directoryContentAlias{
+		TotalCount: -1,
+	}
+
+	err := json.Unmarshal(data, &defaultDirectoryContent)
+	if err != nil {
+		return err
+	}
+
+	*d = DirectoryContent(defaultDirectoryContent)
+	return nil
+}
diff --git a/backend/hidrive/helpers.go b/backend/hidrive/helpers.go
new file mode 100644
index 0000000000000..7b925e1b34722
--- /dev/null
+++ b/backend/hidrive/helpers.go
@@ -0,0 +1,888 @@
+package hidrive
+
+// This file is for helper-functions which may provide more general and
+// specialized functionality than the generic interfaces.
+// There are two sections:
+// 1. methods bound to Fs
+// 2. other functions independent from Fs used throughout the package
+
+// NOTE: Functions accessing paths expect any relative paths
+// to be resolved prior to execution with resolvePath(...).
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"path"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/backend/hidrive/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/ranges"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/sync/semaphore"
+)
+
+const (
+	// MaximumUploadBytes represents the maximum amount of bytes
+	// a single upload-operation will support.
+	MaximumUploadBytes = 2147483647 // = 2GiB - 1
+	// iterationChunkSize represents the chunk size used to iterate directory contents.
+	iterationChunkSize = 5000
+)
+
+var (
+	// retryErrorCodes is a slice of error codes that we will always retry.
+	retryErrorCodes = []int{
+		429, // Too Many Requests
+		500, // Internal Server Error
+		502, // Bad Gateway
+		503, // Service Unavailable
+		504, // Gateway Timeout
+		509, // Bandwidth Limit Exceeded
+	}
+	// ErrorFileExists is returned when a query tries to create a file
+	// that already exists.
+	ErrorFileExists = errors.New("destination file already exists")
+)
+
+// MemberType represents the possible types of entries a directory can contain.
+type MemberType string
+
+// possible values for MemberType
+const (
+	AllMembers       MemberType = "all"
+	NoMembers        MemberType = "none"
+	DirectoryMembers MemberType = api.HiDriveObjectTypeDirectory
+	FileMembers      MemberType = api.HiDriveObjectTypeFile
+	SymlinkMembers   MemberType = api.HiDriveObjectTypeSymlink
+)
+
+// SortByField represents possible fields to sort entries of a directory by.
+type SortByField string
+
+// possible values for SortByField
+const (
+	descendingSort             string      = "-"
+	SortByName                 SortByField = "name"
+	SortByModTime              SortByField = "mtime"
+	SortByObjectType           SortByField = "type"
+	SortBySize                 SortByField = "size"
+	SortByNameDescending       SortByField = SortByField(descendingSort) + SortByName
+	SortByModTimeDescending    SortByField = SortByField(descendingSort) + SortByModTime
+	SortByObjectTypeDescending SortByField = SortByField(descendingSort) + SortByObjectType
+	SortBySizeDescending       SortByField = SortByField(descendingSort) + SortBySize
+)
+
+var (
+	// Unsorted disables sorting and can therefore not be combined with other values.
+	Unsorted = []SortByField{"none"}
+	// DefaultSorted does not specify how to sort and
+	// therefore implies the default sort order.
+	DefaultSorted = []SortByField{}
+)
+
+// CopyOrMoveOperationType represents the possible types of copy- and move-operations.
+type CopyOrMoveOperationType int
+
+// possible values for CopyOrMoveOperationType
+const (
+	MoveOriginal CopyOrMoveOperationType = iota
+	CopyOriginal
+	CopyOriginalPreserveModTime
+)
+
+// OnExistAction represents possible actions the API should take,
+// when a request tries to create a path that already exists.
+type OnExistAction string
+
+// possible values for OnExistAction
+const (
+	// IgnoreOnExist instructs the API not to execute
+	// the request in case of a conflict, but to return an error.
+	IgnoreOnExist OnExistAction = "ignore"
+	// AutoNameOnExist instructs the API to automatically rename
+	// any conflicting request-objects.
+	AutoNameOnExist OnExistAction = "autoname"
+	// OverwriteOnExist instructs the API to overwrite any conflicting files.
+	// This can only be used, if the request operates on files directly.
+	// (For example when moving/copying a file.)
+	// For most requests this action will simply be ignored.
+	OverwriteOnExist OnExistAction = "overwrite"
+)
+
+// shouldRetry returns a boolean as to whether this resp and err deserve to be retried.
+// It tries to expire/invalidate the token, if necessary.
+// It returns the err as a convenience.
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+	if resp != nil && (resp.StatusCode == 401 || isHTTPError(err, 401)) && len(resp.Header["Www-Authenticate"]) > 0 {
+		fs.Debugf(f, "Token might be invalid: %v", err)
+		if f.tokenRenewer != nil {
+			iErr := f.tokenRenewer.Expire()
+			if iErr == nil {
+				return true, err
+			}
+		}
+	}
+	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+}
+
+// resolvePath resolves the given (relative) path and
+// returns a path suitable for API-calls.
+// This will consider the root-path of the fs and any needed prefixes.
+//
+// Any relative paths passed to functions that access these paths should
+// be resolved with this first!
+func (f *Fs) resolvePath(objectPath string) string {
+	resolved := path.Join(f.opt.RootPrefix, f.root, f.opt.Enc.FromStandardPath(objectPath))
+	return resolved
+}
+
+// iterateOverDirectory calls the given function callback
+// on each item found in a given directory.
+//
+// If callback ever returns true then this exits early with found = true.
+func (f *Fs) iterateOverDirectory(ctx context.Context, directory string, searchOnly MemberType, callback func(*api.HiDriveObject) bool, fields []string, sortBy []SortByField) (found bool, err error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(directory)
+	parameters.AddFields("members.", fields...)
+	parameters.AddFields("", api.DirectoryContentFields...)
+	parameters.Set("members", string(searchOnly))
+	for _, v := range sortBy {
+		// The explicit conversion is necessary for each element.
+		parameters.AddList("sort", ",", string(v))
+	}
+
+	opts := rest.Opts{
+		Method:     "GET",
+		Path:       "/dir",
+		Parameters: parameters.Values,
+	}
+
+	iterateContent := func(result *api.DirectoryContent, err error) (bool, error) {
+		if err != nil {
+			return false, err
+		}
+		for _, item := range result.Entries {
+			item.Name = f.opt.Enc.ToStandardName(item.Name)
+			if callback(&item) {
+				return true, nil
+			}
+		}
+		return false, nil
+	}
+	return f.paginateDirectoryAccess(ctx, &opts, iterationChunkSize, 0, iterateContent)
+}
+
+// paginateDirectoryAccess executes requests specified via ctx and opts
+// which should produce api.DirectoryContent.
+// This will paginate the requests using limit starting at the given offset.
+//
+// The given function callback is called on each api.DirectoryContent found
+// along with any errors that occurred.
+// If callback ever returns true then this exits early with found = true.
+// If callback ever returns an error then this exits early with that error.
+func (f *Fs) paginateDirectoryAccess(ctx context.Context, opts *rest.Opts, limit int64, offset int64, callback func(*api.DirectoryContent, error) (bool, error)) (found bool, err error) {
+	for {
+		opts.Parameters.Set("limit", strconv.FormatInt(offset, 10)+","+strconv.FormatInt(limit, 10))
+
+		var result api.DirectoryContent
+		var resp *http.Response
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err = f.srv.CallJSON(ctx, opts, nil, &result)
+			return f.shouldRetry(ctx, resp, err)
+		})
+
+		found, err = callback(&result, err)
+		if found || err != nil {
+			return found, err
+		}
+
+		offset += int64(len(result.Entries))
+		if offset >= result.TotalCount || limit > int64(len(result.Entries)) {
+			break
+		}
+	}
+	return false, nil
+}
+
+// fetchMetadataForPath reads the metadata from the path.
+func (f *Fs) fetchMetadataForPath(ctx context.Context, path string, fields []string) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+	parameters.AddFields("", fields...)
+
+	opts := rest.Opts{
+		Method:     "GET",
+		Path:       "/meta",
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+// copyOrMove copies or moves a directory or file
+// from the source-path to the destination-path.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+//
+// NOTE: Use the explicit methods instead of directly invoking this method.
+// (Those are: copyDirectory, moveDirectory, copyFile, moveFile.)
+func (f *Fs) copyOrMove(ctx context.Context, isDirectory bool, operationType CopyOrMoveOperationType, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.Set("src", source)
+	parameters.Set("dst", destination)
+	if onExist == AutoNameOnExist ||
+		(onExist == OverwriteOnExist && !isDirectory) {
+		parameters.Set("on_exist", string(onExist))
+	}
+
+	endpoint := "/"
+	if isDirectory {
+		endpoint += "dir"
+	} else {
+		endpoint += "file"
+	}
+	switch operationType {
+	case MoveOriginal:
+		endpoint += "/move"
+	case CopyOriginalPreserveModTime:
+		parameters.Set("preserve_mtime", strconv.FormatBool(true))
+		fallthrough
+	case CopyOriginal:
+		endpoint += "/copy"
+	}
+
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       endpoint,
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+// copyDirectory moves the directory at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+func (f *Fs) copyDirectory(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, true, CopyOriginalPreserveModTime, source, destination, onExist)
+}
+
+// moveDirectory moves the directory at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+func (f *Fs) moveDirectory(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, true, MoveOriginal, source, destination, onExist)
+}
+
+// copyFile copies the file at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+//
+// NOTE: This operation will expand sparse areas in the content of the source-file
+// to blocks of 0-bytes in the destination-file.
+func (f *Fs) copyFile(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, false, CopyOriginalPreserveModTime, source, destination, onExist)
+}
+
+// moveFile moves the file at the source-path to the destination-path and
+// returns the resulting api-object if successful.
+//
+// The operation will only be successful
+// if the parent-directory of the destination-path exists.
+//
+// NOTE: This operation may expand sparse areas in the content of the source-file
+// to blocks of 0-bytes in the destination-file.
+func (f *Fs) moveFile(ctx context.Context, source string, destination string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	return f.copyOrMove(ctx, false, MoveOriginal, source, destination, onExist)
+}
+
+// createDirectory creates the directory at the given path and
+// returns the resulting api-object if successful.
+//
+// The directory will only be created if its parent-directory exists.
+// This returns fs.ErrorDirNotFound if the parent-directory is not found.
+// This returns fs.ErrorDirExists if the directory already exists.
+func (f *Fs) createDirectory(ctx context.Context, directory string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(directory)
+	if onExist == AutoNameOnExist {
+		parameters.Set("on_exist", string(onExist))
+	}
+
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/dir",
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorDirNotFound
+	case isHTTPError(err, 409):
+		return nil, fs.ErrorDirExists
+	}
+	return nil, err
+}
+
+// createDirectories creates the directory at the given path
+// along with any missing parent directories and
+// returns the resulting api-object (of the created directory) if successful.
+//
+// This returns fs.ErrorDirExists if the directory already exists.
+//
+// If an error occurs while the parent directories are being created,
+// any directories already created will NOT be deleted again.
+func (f *Fs) createDirectories(ctx context.Context, directory string, onExist OnExistAction) (*api.HiDriveObject, error) {
+	result, err := f.createDirectory(ctx, directory, onExist)
+	if err == nil {
+		return result, nil
+	}
+	if err != fs.ErrorDirNotFound {
+		return nil, err
+	}
+	parentDirectory := path.Dir(directory)
+	_, err = f.createDirectories(ctx, parentDirectory, onExist)
+	if err != nil && err != fs.ErrorDirExists {
+		return nil, err
+	}
+	// NOTE: Ignoring fs.ErrorDirExists does no harm,
+	// since it does not mean the child directory cannot be created.
+	return f.createDirectory(ctx, directory, onExist)
+}
+
+// deleteDirectory deletes the directory at the given path.
+//
+// If recursive is false, the directory will only be deleted if it is empty.
+// If recursive is true, the directory will be deleted regardless of its content.
+// This returns fs.ErrorDirNotFound if the directory is not found.
+// This returns fs.ErrorDirectoryNotEmpty if the directory is not empty and
+// recursive is false.
+func (f *Fs) deleteDirectory(ctx context.Context, directory string, recursive bool) error {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(directory)
+	parameters.Set("recursive", strconv.FormatBool(recursive))
+
+	opts := rest.Opts{
+		Method:     "DELETE",
+		Path:       "/dir",
+		Parameters: parameters.Values,
+		NoResponse: true,
+	}
+
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.Call(ctx, &opts)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case isHTTPError(err, 404):
+		return fs.ErrorDirNotFound
+	case isHTTPError(err, 409):
+		return fs.ErrorDirectoryNotEmpty
+	}
+	return err
+}
+
+// deleteObject deletes the object/file at the given path.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) deleteObject(ctx context.Context, path string) error {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+
+	opts := rest.Opts{
+		Method:     "DELETE",
+		Path:       "/file",
+		Parameters: parameters.Values,
+		NoResponse: true,
+	}
+
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.Call(ctx, &opts)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	if isHTTPError(err, 404) {
+		return fs.ErrorObjectNotFound
+	}
+	return err
+}
+
+// createFile creates a file at the given path
+// with the content of the io.ReadSeeker.
+// This guarantees that existing files will not be overwritten.
+// The maximum size of the content is limited by MaximumUploadBytes.
+// The io.ReadSeeker should be resettable by seeking to its start.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// This returns fs.ErrorDirNotFound
+// if the parent directory of the file is not found.
+// This returns ErrorFileExists if a file already exists at the specified path.
+func (f *Fs) createFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetFileInDirectory(path)
+	if onExist == AutoNameOnExist {
+		parameters.Set("on_exist", string(onExist))
+	}
+
+	var err error
+	if !modTime.IsZero() {
+		err = parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:      "POST",
+		Path:        "/file",
+		Body:        content,
+		ContentType: "application/octet-stream",
+		Parameters:  parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		// Reset the reading index (in case this is a retry).
+		if _, err = content.Seek(0, io.SeekStart); err != nil {
+			return false, err
+		}
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorDirNotFound
+	case isHTTPError(err, 409):
+		return nil, ErrorFileExists
+	}
+	return nil, err
+}
+
+// overwriteFile updates the content of the file at the given path
+// with the content of the io.ReadSeeker.
+// If the file does not exist it will be created.
+// The maximum size of the content is limited by MaximumUploadBytes.
+// The io.ReadSeeker should be resettable by seeking to its start.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// This returns fs.ErrorDirNotFound
+// if the parent directory of the file is not found.
+func (f *Fs) overwriteFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetFileInDirectory(path)
+
+	var err error
+	if !modTime.IsZero() {
+		err = parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:      "PUT",
+		Path:        "/file",
+		Body:        content,
+		ContentType: "application/octet-stream",
+		Parameters:  parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		// Reset the reading index (in case this is a retry).
+		if _, err = content.Seek(0, io.SeekStart); err != nil {
+			return false, err
+		}
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorDirNotFound
+	}
+	return nil, err
+}
+
+// uploadFileChunked updates the content of the existing file at the given path
+// with the content of the io.Reader.
+// Returns the position of the last successfully written byte, stopping before the first failed write.
+// If nothing was written this will be 0.
+// Returns the resulting api-object if successful.
+//
+// Replaces the file contents by uploading multiple chunks of the given size in parallel.
+// Therefore this can and be used to upload files of any size efficiently.
+// The number of parallel transfers is limited by transferLimit which should larger than 0.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// NOTE: This method uses updateFileChunked and may create sparse files,
+// if the upload of a chunk fails unexpectedly.
+// See note about sparse files in patchFile.
+// If any of the uploads fail, the process will be aborted and
+// the first error that occurred will be returned.
+// This is not an atomic operation,
+// therefore if the upload fails the file may be partially modified.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) uploadFileChunked(ctx context.Context, path string, content io.Reader, modTime time.Time, chunkSize int, transferLimit int64) (okSize uint64, info *api.HiDriveObject, err error) {
+	okSize, err = f.updateFileChunked(ctx, path, content, 0, chunkSize, transferLimit)
+
+	if err == nil {
+		info, err = f.resizeFile(ctx, path, okSize, modTime)
+	}
+	return okSize, info, err
+}
+
+// updateFileChunked updates the content of the existing file at the given path
+// starting at the given offset.
+// Returns the position of the last successfully written byte, stopping before the first failed write.
+// If nothing was written this will be 0.
+//
+// Replaces the file contents starting from the given byte offset
+// with the content of the io.Reader.
+// If the offset is beyond the file end, the file is extended up to the offset.
+//
+// The upload is done multiple chunks of the given size in parallel.
+// Therefore this can and be used to upload files of any size efficiently.
+// The number of parallel transfers is limited by transferLimit which should larger than 0.
+//
+// NOTE: Because it is inefficient to set the modification time with every chunk,
+// setting it to a specific value must be done in a separate request
+// after this operation finishes.
+//
+// NOTE: This method uses patchFile and may create sparse files,
+// especially if the upload of a chunk fails unexpectedly.
+// See note about sparse files in patchFile.
+// If any of the uploads fail, the process will be aborted and
+// the first error that occurred will be returned.
+// This is not an atomic operation,
+// therefore if the upload fails the file may be partially modified.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Reader, offset uint64, chunkSize int, transferLimit int64) (okSize uint64, err error) {
+	var (
+		okChunksMu sync.Mutex // protects the variables below
+		okChunks   []ranges.Range
+	)
+	g, gCtx := errgroup.WithContext(ctx)
+	transferSemaphore := semaphore.NewWeighted(transferLimit)
+
+	var readErr error
+	startMoreTransfers := true
+	zeroTime := time.Time{}
+	for chunk := uint64(0); startMoreTransfers; chunk++ {
+		// Acquire semaphore to limit number of transfers in parallel.
+		readErr = transferSemaphore.Acquire(gCtx, 1)
+		if readErr != nil {
+			break
+		}
+
+		// Read a chunk of data.
+		chunkReader, bytesRead, readErr := readerForChunk(content, chunkSize)
+		if bytesRead < chunkSize {
+			startMoreTransfers = false
+		}
+		if readErr != nil || bytesRead <= 0 {
+			break
+		}
+
+		// Transfer the chunk.
+		chunkOffset := uint64(chunkSize)*chunk + offset
+		g.Go(func() error {
+			// After this upload is done,
+			// signal that another transfer can be started.
+			defer transferSemaphore.Release(1)
+			uploadErr := f.patchFile(gCtx, path, cachedReader(chunkReader), chunkOffset, zeroTime)
+			if uploadErr == nil {
+				// Remember successfully written chunks.
+				okChunksMu.Lock()
+				okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: int64(bytesRead)})
+				okChunksMu.Unlock()
+				fs.Debugf(f, "Done uploading chunk of size %v at offset %v.", bytesRead, chunkOffset)
+			} else {
+				fs.Infof(f, "Error while uploading chunk at offset %v. Error is %v.", chunkOffset, uploadErr)
+			}
+			return uploadErr
+		})
+	}
+
+	if readErr != nil {
+		// Log the error in case it is later ignored because of an upload-error.
+		fs.Infof(f, "Error while reading/preparing to upload a chunk. Error is %v.", readErr)
+	}
+
+	err = g.Wait()
+
+	// Compute the first continuous range of the file content,
+	// which does not contain any failed chunks.
+	// Do not forget to add the file content up to the starting offset,
+	// which is presumed to be already correct.
+	rs := ranges.Ranges{}
+	rs.Insert(ranges.Range{Pos: 0, Size: int64(offset)})
+	for _, chunkRange := range okChunks {
+		rs.Insert(chunkRange)
+	}
+	if len(rs) > 0 && rs[0].Pos == 0 {
+		okSize = uint64(rs[0].Size)
+	}
+
+	if err != nil {
+		return okSize, err
+	}
+	if readErr != nil {
+		return okSize, readErr
+	}
+
+	return okSize, nil
+}
+
+// patchFile updates the content of the existing file at the given path
+// starting at the given offset.
+//
+// Replaces the file contents starting from the given byte offset
+// with the content of the io.ReadSeeker.
+// If the offset is beyond the file end, the file is extended up to the offset.
+// The maximum size of the update is limited by MaximumUploadBytes.
+// The io.ReadSeeker should be resettable by seeking to its start.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// NOTE: By extending the file up to the offset this may create sparse files,
+// which allocate less space on the file system than their apparent size indicates,
+// since holes between data chunks are "real" holes
+// and not regions made up of consecutive 0-bytes.
+// Subsequent operations (such as copying data)
+// usually expand the holes into regions of 0-bytes.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker, offset uint64, modTime time.Time) error {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+	parameters.Set("offset", strconv.FormatUint(offset, 10))
+
+	if !modTime.IsZero() {
+		err := parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:      "PATCH",
+		Path:        "/file",
+		Body:        content,
+		ContentType: "application/octet-stream",
+		Parameters:  parameters.Values,
+		NoResponse:  true,
+	}
+
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		// Reset the reading index (in case this is a retry).
+		_, err = content.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
+		resp, err = f.srv.Call(ctx, &opts)
+		if isHTTPError(err, 423) {
+			return true, err
+		}
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	if isHTTPError(err, 404) {
+		return fs.ErrorObjectNotFound
+	}
+	return err
+}
+
+// resizeFile updates the existing file at the given path to be of the given size
+// and returns the resulting api-object if successful.
+//
+// If the given size is smaller than the current filesize,
+// the file is cut/truncated at that position.
+// If the given size is larger, the file is extended up to that position.
+// If modTime is not the zero time instant,
+// it will be set as the file's modification time after the operation.
+//
+// NOTE: By extending the file this may create sparse files,
+// which allocate less space on the file system than their apparent size indicates,
+// since holes between data chunks are "real" holes
+// and not regions made up of consecutive 0-bytes.
+// Subsequent operations (such as copying data)
+// usually expand the holes into regions of 0-bytes.
+//
+// This returns fs.ErrorObjectNotFound if the object is not found.
+func (f *Fs) resizeFile(ctx context.Context, path string, size uint64, modTime time.Time) (*api.HiDriveObject, error) {
+	parameters := api.NewQueryParameters()
+	parameters.SetPath(path)
+	parameters.Set("size", strconv.FormatUint(size, 10))
+
+	if !modTime.IsZero() {
+		err := parameters.SetTime("mtime", modTime)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/file/truncate",
+		Parameters: parameters.Values,
+	}
+
+	var result api.HiDriveObject
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return f.shouldRetry(ctx, resp, err)
+	})
+
+	switch {
+	case err == nil:
+		return &result, nil
+	case isHTTPError(err, 404):
+		return nil, fs.ErrorObjectNotFound
+	}
+	return nil, err
+}
+
+// ------------------------------------------------------------
+
+// isHTTPError compares the numerical status code
+// of an api.Error to the given HTTP status.
+//
+// If the given error is not an api.Error or
+// a numerical status code could not be determined, this returns false.
+// Otherwise this returns whether the status code of the error is equal to the given status.
+func isHTTPError(err error, status int64) bool {
+	if apiErr, ok := err.(*api.Error); ok {
+		errStatus, decodeErr := apiErr.Code.Int64()
+		if decodeErr == nil && errStatus == status {
+			return true
+		}
+	}
+	return false
+}
+
+// createHiDriveScopes creates oauth-scopes
+// from the given user-role and access-permissions.
+//
+// If the arguments are empty, they will not be included in the result.
+func createHiDriveScopes(role string, access string) []string {
+	switch {
+	case role != "" && access != "":
+		return []string{access + "," + role}
+	case role != "":
+		return []string{role}
+	case access != "":
+		return []string{access}
+	}
+	return []string{}
+}
+
+// cachedReader returns a version of the reader that caches its contents and
+// can therefore be reset using Seek.
+func cachedReader(reader io.Reader) io.ReadSeeker {
+	bytesReader, ok := reader.(*bytes.Reader)
+	if ok {
+		return bytesReader
+	}
+
+	repeatableReader, ok := reader.(*readers.RepeatableReader)
+	if ok {
+		return repeatableReader
+	}
+
+	return readers.NewRepeatableReader(reader)
+}
+
+// readerForChunk reads a chunk of bytes from reader (after handling any accounting).
+// Returns a new io.Reader (chunkReader) for that chunk
+// and the number of bytes that have been read from reader.
+func readerForChunk(reader io.Reader, length int) (chunkReader io.Reader, bytesRead int, err error) {
+	// Unwrap any accounting from the input if present.
+	reader, wrap := accounting.UnWrap(reader)
+
+	// Read a chunk of data.
+	buffer := make([]byte, length)
+	bytesRead, err = io.ReadFull(reader, buffer)
+	if err == io.EOF || err == io.ErrUnexpectedEOF {
+		err = nil
+	}
+	if err != nil {
+		return nil, bytesRead, err
+	}
+	// Truncate unused capacity.
+	buffer = buffer[:bytesRead]
+
+	// Use wrap to put any accounting back for chunkReader.
+	return wrap(bytes.NewReader(buffer)), bytesRead, nil
+}
diff --git a/backend/hidrive/hidrive.go b/backend/hidrive/hidrive.go
new file mode 100644
index 0000000000000..b0f56844a8767
--- /dev/null
+++ b/backend/hidrive/hidrive.go
@@ -0,0 +1,1002 @@
+// Package hidrive provides an interface to the HiDrive object storage system.
+package hidrive
+
+// FIXME HiDrive only supports file or folder names of 255 characters or less.
+// Operations that create files oder folder with longer names will throw a HTTP error:
+// - 422 Unprocessable Entity
+// A more graceful way for rclone to handle this may be desirable.
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/rclone/rclone/lib/encoder"
+
+	"github.com/rclone/rclone/backend/hidrive/api"
+	"github.com/rclone/rclone/backend/hidrive/hidrivehash"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/oauthutil"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
+)
+
+const (
+	rcloneClientID              = "6b0258fdda630d34db68a3ce3cbf19ae"
+	rcloneEncryptedClientSecret = "GC7UDZ3Ra4jLcmfQSagKCDJ1JEy-mU6pBBhFrS3tDEHILrK7j3TQHUrglkO5SgZ_"
+	minSleep                    = 10 * time.Millisecond
+	maxSleep                    = 2 * time.Second
+	decayConstant               = 2 // bigger for slower decay, exponential
+	defaultUploadChunkSize      = 48 * fs.Mebi
+	defaultUploadCutoff         = 2 * defaultUploadChunkSize
+	defaultUploadConcurrency    = 4
+)
+
+// Globals
+var (
+	// Description of how to auth for this app.
+	oauthConfig = &oauth2.Config{
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  "https://my.hidrive.com/client/authorize",
+			TokenURL: "https://my.hidrive.com/oauth2/token",
+		},
+		ClientID:     rcloneClientID,
+		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
+		RedirectURL:  oauthutil.TitleBarRedirectURL,
+	}
+	// hidrivehashType is the hash.Type for HiDrive hashes.
+	hidrivehashType hash.Type
+)
+
+// Register the backend with Fs.
+func init() {
+	hidrivehashType = hash.RegisterHash("hidrive", "HiDriveHash", 40, hidrivehash.New)
+	fs.Register(&fs.RegInfo{
+		Name:        "hidrive",
+		Description: "HiDrive",
+		NewFs:       NewFs,
+		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
+			// Parse config into Options struct
+			opt := new(Options)
+			err := configstruct.Set(m, opt)
+			if err != nil {
+				return nil, fmt.Errorf("couldn't parse config into struct: %w", err)
+			}
+
+			//fs.Debugf(nil, "hidrive: configuring oauth-token.")
+			oauthConfig.Scopes = createHiDriveScopes(opt.ScopeRole, opt.ScopeAccess)
+			return oauthutil.ConfigOut("", &oauthutil.Options{
+				OAuth2Config: oauthConfig,
+			})
+		},
+		Options: append(oauthutil.SharedOptions, []fs.Option{{
+			Name:    "scope_access",
+			Help:    "Access permissions that rclone should use when requesting access from HiDrive.",
+			Default: "rw",
+			Examples: []fs.OptionExample{{
+				Value: "rw",
+				Help:  "Read and write access to resources.",
+			}, {
+				Value: "ro",
+				Help:  "Read-only access to resources.",
+			}},
+		}, {
+			Name:    "scope_role",
+			Help:    "User-level that rclone should use when requesting access from HiDrive.",
+			Default: "user",
+			Examples: []fs.OptionExample{{
+				Value: "user",
+				Help: `User-level access to management permissions.
+This will be sufficient in most cases.`,
+			}, {
+				Value: "admin",
+				Help:  "Extensive access to management permissions.",
+			}, {
+				Value: "owner",
+				Help:  "Full access to management permissions.",
+			}},
+			Advanced: true,
+		}, {
+			Name: "root_prefix",
+			Help: `The root/parent folder for all paths.
+
+Fill in to use the specified folder as the parent for all paths given to the remote.
+This way rclone can use any folder as its starting point.`,
+			Default: "/",
+			Examples: []fs.OptionExample{{
+				Value: "/",
+				Help: `The topmost directory accessible by rclone.
+This will be equivalent with "root" if rclone uses a regular HiDrive user account.`,
+			}, {
+				Value: "root",
+				Help:  `The topmost directory of the HiDrive user account`,
+			}, {
+				Value: "",
+				Help: `This specifies that there is no root-prefix for your paths.
+When using this you will always need to specify paths to this remote with a valid parent e.g. "remote:/path/to/dir" or "remote:root/path/to/dir".`,
+			}},
+			Advanced: true,
+		}, {
+			Name: "endpoint",
+			Help: `Endpoint for the service.
+
+This is the URL that API-calls will be made to.`,
+			Default:  "https://api.hidrive.strato.com/2.1",
+			Advanced: true,
+		}, {
+			Name: "disable_fetching_member_count",
+			Help: `Do not fetch number of objects in directories unless it is absolutely necessary.
+
+Requests may be faster if the number of objects in subdirectories is not fetched.`,
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name: "chunk_size",
+			Help: fmt.Sprintf(`Chunksize for chunked uploads.
+
+Any files larger than the configured cutoff (or files of unknown size) will be uploaded in chunks of this size.
+
+The upper limit for this is %v bytes (about %v).
+That is the maximum amount of bytes a single upload-operation will support.
+Setting this above the upper limit or to a negative value will cause uploads to fail.
+
+Setting this to larger values may increase the upload speed at the cost of using more memory.
+It can be set to smaller values smaller to save on memory.`, MaximumUploadBytes, fs.SizeSuffix(MaximumUploadBytes)),
+			Default:  defaultUploadChunkSize,
+			Advanced: true,
+		}, {
+			Name: "upload_cutoff",
+			Help: fmt.Sprintf(`Cutoff/Threshold for chunked uploads.
+
+Any files larger than this will be uploaded in chunks of the configured chunksize.
+
+The upper limit for this is %v bytes (about %v).
+That is the maximum amount of bytes a single upload-operation will support.
+Setting this above the upper limit will cause uploads to fail.`, MaximumUploadBytes, fs.SizeSuffix(MaximumUploadBytes)),
+			Default:  defaultUploadCutoff,
+			Advanced: true,
+		}, {
+			Name: "upload_concurrency",
+			Help: `Concurrency for chunked uploads.
+
+This is the upper limit for how many transfers for the same file are running concurrently.
+Setting this above to a value smaller than 1 will cause uploads to deadlock.
+
+If you are uploading small numbers of large files over high-speed links
+and these uploads do not fully utilize your bandwidth, then increasing
+this may help to speed up the transfers.`,
+			Default:  defaultUploadConcurrency,
+			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// HiDrive only supports file or folder names of 255 characters or less.
+			// Names containing "/" are not supported.
+			// The special names "." and ".." are not supported.
+			Default: (encoder.EncodeZero |
+				encoder.EncodeSlash |
+				encoder.EncodeDot),
+		}}...),
+	})
+}
+
+// Options defines the configuration for this backend.
+type Options struct {
+	EndpointAPI                 string               `config:"endpoint"`
+	OptionalMemberCountDisabled bool                 `config:"disable_fetching_member_count"`
+	UploadChunkSize             fs.SizeSuffix        `config:"chunk_size"`
+	UploadCutoff                fs.SizeSuffix        `config:"upload_cutoff"`
+	UploadConcurrency           int64                `config:"upload_concurrency"`
+	Enc                         encoder.MultiEncoder `config:"encoding"`
+	RootPrefix                  string               `config:"root_prefix"`
+	ScopeAccess                 string               `config:"scope_access"`
+	ScopeRole                   string               `config:"scope_role"`
+}
+
+// Fs represents a remote hidrive.
+type Fs struct {
+	name     string       // name of this remote
+	root     string       // the path we are working on
+	opt      Options      // parsed options
+	features *fs.Features // optional features
+	srv      *rest.Client // the connection to the server
+	pacer    *fs.Pacer    // pacer for API calls
+	// retryOnce is NOT intended as a pacer for API calls.
+	// The intended use case is to repeat an action that failed because
+	// some preconditions were not previously fulfilled.
+	// Code using this should then establish these preconditions
+	// and let the pacer retry the operation.
+	retryOnce    *pacer.Pacer     // pacer with no delays to retry certain operations once
+	tokenRenewer *oauthutil.Renew // renew the token on expiry
+}
+
+// Object describes a hidrive object.
+//
+// Will definitely have the remote-path but may lack meta-information.
+type Object struct {
+	fs          *Fs       // what this object is part of
+	remote      string    // The remote path
+	hasMetadata bool      // whether info below has been set
+	size        int64     // size of the object
+	modTime     time.Time // modification time of the object
+	id          string    // ID of the object
+	hash        string    // content-hash of the object
+}
+
+// ------------------------------------------------------------
+
+// Name returns the name of the remote (as passed into NewFs).
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root returns the name of the remote (as passed into NewFs).
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns a string-representation of this Fs.
+func (f *Fs) String() string {
+	return fmt.Sprintf("HiDrive root '%s'", f.root)
+}
+
+// Precision returns the precision of this Fs.
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hidrivehashType)
+}
+
+// Features returns the optional features of this Fs.
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// errorHandler parses a non 2xx error response into an error.
+func errorHandler(resp *http.Response) error {
+	// Decode error response.
+	errResponse := new(api.Error)
+	err := rest.DecodeJSON(resp, &errResponse)
+	if err != nil {
+		fs.Debugf(nil, "Couldn't decode error response: %v", err)
+	}
+	_, err = errResponse.Code.Int64()
+	if err != nil {
+		errResponse.Code = json.Number(strconv.Itoa(resp.StatusCode))
+	}
+	return errResponse
+}
+
+// NewFs creates a new file system from the path.
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+	//fs.Debugf(nil, "hidrive: creating new Fs.")
+	// Parse config into Options struct.
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	// Clean root-prefix and root-path.
+	// NOTE: With the default-encoding "." and ".." will be encoded,
+	// but with custom encodings without encoder.EncodeDot
+	// "." and ".." will be interpreted as paths.
+	if opt.RootPrefix != "" {
+		opt.RootPrefix = path.Clean(opt.Enc.FromStandardPath(opt.RootPrefix))
+	}
+	root = path.Clean(opt.Enc.FromStandardPath(root))
+
+	client, ts, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure HiDrive: %w", err)
+	}
+
+	f := &Fs{
+		name:      name,
+		root:      root,
+		opt:       *opt,
+		srv:       rest.NewClient(client).SetRoot(opt.EndpointAPI),
+		pacer:     fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		retryOnce: pacer.New(pacer.RetriesOption(2), pacer.MaxConnectionsOption(-1), pacer.CalculatorOption(&pacer.ZeroDelayCalculator{})),
+	}
+	f.features = (&fs.Features{
+		CanHaveEmptyDirectories: true,
+	}).Fill(ctx, f)
+	f.srv.SetErrorHandler(errorHandler)
+
+	if ts != nil {
+		transaction := func() error {
+			resolvedRoot := f.resolvePath("")
+			_, err := f.fetchMetadataForPath(ctx, resolvedRoot, api.HiDriveObjectNoMetadataFields)
+			return err
+		}
+		f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, transaction)
+	}
+
+	// Do not allow the root-prefix to be non-existent nor a directory,
+	// but it can be empty.
+	if f.opt.RootPrefix != "" {
+		item, err := f.fetchMetadataForPath(ctx, f.opt.RootPrefix, api.HiDriveObjectNoMetadataFields)
+		if err != nil {
+			return nil, fmt.Errorf("could not access root-prefix: %w", err)
+		}
+		if item.Type != api.HiDriveObjectTypeDirectory {
+			return nil, errors.New("The root-prefix needs to point to a valid directory or be empty")
+		}
+	}
+
+	resolvedRoot := f.resolvePath("")
+	item, err := f.fetchMetadataForPath(ctx, resolvedRoot, api.HiDriveObjectNoMetadataFields)
+	if err != nil {
+		if isHTTPError(err, 404) {
+			// NOTE: NewFs needs to work with paths that do not exist,
+			// in case they will be created later (see mkdir).
+			return f, nil
+		}
+		return nil, fmt.Errorf("could not access root-path: %w", err)
+	}
+	if item.Type != api.HiDriveObjectTypeDirectory {
+		fs.Debugf(f, "The root is not a directory. Setting its parent-directory as the new root.")
+		// NOTE: There is no need to check
+		// if the parent-directory is inside the root-prefix:
+		// If the parent-directory was outside,
+		// then the resolved path would be the root-prefix,
+		// therefore the root-prefix would point to a file,
+		// which has already been checked for.
+		// In case the root-prefix is empty, this needs not be checked,
+		// because top-level files cannot exist.
+		f.root = path.Dir(f.root)
+		return f, fs.ErrorIsFile
+	}
+
+	return f, nil
+}
+
+// newObject constructs an Object by calling the given function metaFiller
+// on an Object with no metadata.
+//
+// metaFiller should set the metadata of the object or
+// return an appropriate error.
+func (f *Fs) newObject(remote string, metaFiller func(*Object) error) (fs.Object, error) {
+	o := &Object{
+		fs:     f,
+		remote: remote,
+	}
+	var err error
+	if metaFiller != nil {
+		err = metaFiller(o)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
+}
+
+// newObjectFromHiDriveObject constructs an Object from the given api.HiDriveObject.
+func (f *Fs) newObjectFromHiDriveObject(remote string, info *api.HiDriveObject) (fs.Object, error) {
+	metaFiller := func(o *Object) error {
+		return o.setMetadata(info)
+	}
+	return f.newObject(remote, metaFiller)
+}
+
+// NewObject finds the Object at remote.
+//
+// If remote points to a directory then it returns fs.ErrorIsDir.
+// If it can not be found it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	//fs.Debugf(f, "executing NewObject(%s).", remote)
+	metaFiller := func(o *Object) error {
+		return o.readMetadata(ctx)
+	}
+	return f.newObject(remote, metaFiller)
+}
+
+// List the objects and directories in dir into entries.
+// The entries can be returned in any order,
+// but should be for a complete directory.
+//
+// dir should be "" to list the root, and should not have trailing slashes.
+//
+// This returns fs.ErrorDirNotFound if the directory is not found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	//fs.Debugf(f, "executing List(%s).", dir)
+	var iErr error
+	addEntry := func(info *api.HiDriveObject) bool {
+		fs.Debugf(f, "found directory-element with name %s", info.Name)
+		remote := path.Join(dir, info.Name)
+		if info.Type == api.HiDriveObjectTypeDirectory {
+			d := fs.NewDir(remote, info.ModTime())
+			d.SetID(info.ID)
+			d.SetSize(info.Size)
+			d.SetItems(info.MemberCount)
+			entries = append(entries, d)
+		} else if info.Type == api.HiDriveObjectTypeFile {
+			o, err := f.newObjectFromHiDriveObject(remote, info)
+			if err != nil {
+				iErr = err
+				return true
+			}
+			entries = append(entries, o)
+		}
+		return false
+	}
+
+	var fields []string
+	if f.opt.OptionalMemberCountDisabled {
+		fields = api.HiDriveObjectWithMetadataFields
+	} else {
+		fields = api.HiDriveObjectWithDirectoryMetadataFields
+	}
+	resolvedDir := f.resolvePath(dir)
+	_, err = f.iterateOverDirectory(ctx, resolvedDir, AllMembers, addEntry, fields, Unsorted)
+
+	if err != nil {
+		if isHTTPError(err, 404) {
+			return nil, fs.ErrorDirNotFound
+		}
+		return nil, err
+	}
+	if iErr != nil {
+		return nil, iErr
+	}
+	return entries, nil
+}
+
+// Put the contents of the io.Reader into the remote path
+// with the modTime given of the given size.
+// The existing or new object is returned.
+//
+// A new object may have been created or
+// an existing one accessed even if an error is returned,
+// in which case both the object and the error will be returned.
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	remote := src.Remote()
+	//fs.Debugf(f, "executing Put(%s, %v).", remote, options)
+
+	existingObj, err := f.NewObject(ctx, remote)
+	switch err {
+	case nil:
+		return existingObj, existingObj.Update(ctx, in, src, options...)
+	case fs.ErrorObjectNotFound:
+		// Object was not found, so create a new one.
+		return f.PutUnchecked(ctx, in, src, options...)
+	}
+	return nil, err
+}
+
+// PutStream uploads the contents of the io.Reader to the remote path
+// with the modTime given of indeterminate size.
+// The existing or new object is returned.
+//
+// A new object may have been created or
+// an existing one accessed even if an error is returned,
+// in which case both the object and the error will be returned.
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	//fs.Debugf(f, "executing PutStream(%s, %v).", src.Remote(), options)
+
+	return f.Put(ctx, in, src, options...)
+}
+
+// PutUnchecked the contents of the io.Reader into the remote path
+// with the modTime given of the given size.
+// This guarantees that existing objects will not be overwritten.
+// The new object is returned.
+//
+// This will produce an error if an object already exists at that path.
+//
+// In case the upload fails and an object has been created,
+// this will try to delete the object at that path.
+// In case the failed upload could not be deleted,
+// both the object and the (upload-)error will be returned.
+func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	remote := src.Remote()
+	modTime := src.ModTime(ctx)
+	//fs.Debugf(f, "executing PutUnchecked(%s, %v).", remote, options)
+	resolvedPath := f.resolvePath(remote)
+
+	// NOTE: The file creation operation is a single atomic operation.
+	// Thus uploading as much content as is reasonable
+	// (i.e. everything up to the cutoff) in the first request,
+	// avoids files being created on upload failure for small files.
+	// (As opposed to creating an empty file and then uploading the content.)
+	tmpReader, bytesRead, err := readerForChunk(in, int(f.opt.UploadCutoff))
+	cutoffReader := cachedReader(tmpReader)
+	if err != nil {
+		return nil, err
+	}
+
+	var info *api.HiDriveObject
+	err = f.retryOnce.Call(func() (bool, error) {
+		var createErr error
+		// Reset the reading index (in case this is a retry).
+		if _, createErr = cutoffReader.Seek(0, io.SeekStart); createErr != nil {
+			return false, createErr
+		}
+		info, createErr = f.createFile(ctx, resolvedPath, cutoffReader, modTime, IgnoreOnExist)
+
+		if createErr == fs.ErrorDirNotFound {
+			// Create the parent-directory for the object and repeat request.
+			_, parentErr := f.createDirectories(ctx, path.Dir(resolvedPath), IgnoreOnExist)
+			if parentErr != nil && parentErr != fs.ErrorDirExists {
+				fs.Errorf(f, "Tried to create parent-directory for '%s', but failed.", resolvedPath)
+				return false, parentErr
+			}
+			return true, createErr
+		}
+		return false, createErr
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	o, err := f.newObjectFromHiDriveObject(remote, info)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if fs.SizeSuffix(bytesRead) < f.opt.UploadCutoff {
+		return o, nil
+	}
+	// If there is more left to write, o.Update needs to skip ahead.
+	// Use a fs.SeekOption with the current offset to do this.
+	options = append(options, &fs.SeekOption{Offset: int64(bytesRead)})
+	err = o.Update(ctx, in, src, options...)
+
+	if err == nil {
+		return o, nil
+	}
+
+	// Try to remove object at path after the its content could not be uploaded.
+	deleteErr := f.pacer.Call(func() (bool, error) {
+		deleteErr := o.Remove(ctx)
+		return deleteErr == fs.ErrorObjectNotFound, deleteErr
+	})
+
+	if deleteErr == nil {
+		return nil, err
+	}
+
+	fs.Errorf(f, "Tried to delete failed upload at path '%s', but failed: %v", resolvedPath, deleteErr)
+	return o, err
+}
+
+// Mkdir creates the directory if it does not exist.
+//
+// This will create any missing parent directories.
+//
+// NOTE: If an error occurs while the parent directories are being created,
+// any directories already created will NOT be deleted again.
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	//fs.Debugf(f, "executing Mkdir(%s).", dir)
+	resolvedDir := f.resolvePath(dir)
+	_, err := f.createDirectories(ctx, resolvedDir, IgnoreOnExist)
+
+	if err == fs.ErrorDirExists {
+		// NOTE: The conflict is caused by the directory already existing,
+		// which should be ignored here.
+		return nil
+	}
+
+	return err
+}
+
+// Rmdir removes the directory if empty.
+//
+// This returns fs.ErrorDirNotFound if the directory is not found.
+// This returns fs.ErrorDirectoryNotEmpty if the directory is not empty.
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	//fs.Debugf(f, "executing Rmdir(%s).", dir)
+	resolvedDir := f.resolvePath(dir)
+	return f.deleteDirectory(ctx, resolvedDir, false)
+}
+
+// Purge removes the directory and all of its contents.
+//
+// This returns fs.ErrorDirectoryNotEmpty if the directory is not empty.
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+	//fs.Debugf(f, "executing Purge(%s).", dir)
+	resolvedDir := f.resolvePath(dir)
+	return f.deleteDirectory(ctx, resolvedDir, true)
+}
+
+// shouldRetryAndCreateParents returns a boolean as to whether the operation
+// should be retried after the parent-directories of the destination have been created.
+// If so, it will create the parent-directories.
+//
+// If any errors arrise while finding the source or
+// creating the parent-directory those will be returned.
+// Otherwise returns the originalError.
+func (f *Fs) shouldRetryAndCreateParents(ctx context.Context, destinationPath string, sourcePath string, originalError error) (bool, error) {
+	if fserrors.ContextError(ctx, &originalError) {
+		return false, originalError
+	}
+	if isHTTPError(originalError, 404) {
+		// Check if source is missing.
+		_, srcErr := f.fetchMetadataForPath(ctx, sourcePath, api.HiDriveObjectNoMetadataFields)
+		if srcErr != nil {
+			return false, srcErr
+		}
+		// Source exists, so the parent of the destination must have been missing.
+		// Create the parent-directory and repeat request.
+		_, parentErr := f.createDirectories(ctx, path.Dir(destinationPath), IgnoreOnExist)
+		if parentErr != nil && parentErr != fs.ErrorDirExists {
+			fs.Errorf(f, "Tried to create parent-directory for '%s', but failed.", destinationPath)
+			return false, parentErr
+		}
+		return true, originalError
+	}
+	return false, originalError
+}
+
+// Copy src to this remote using server-side copy operations.
+//
+// It returns the destination Object and a possible error.
+//
+// This returns fs.ErrorCantCopy if the operation cannot be performed.
+//
+// NOTE: If an error occurs when copying the Object,
+// any parent-directories already created will NOT be deleted again.
+//
+// NOTE: This operation will expand sparse areas in the content of the source-Object
+// to blocks of 0-bytes in the destination-Object.
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+	// Get the absolute path to the source.
+	srcPath := srcObj.fs.resolvePath(srcObj.Remote())
+	//fs.Debugf(f, "executing Copy(%s, %s).", srcPath, remote)
+	dstPath := f.resolvePath(remote)
+
+	var info *api.HiDriveObject
+	err := f.retryOnce.Call(func() (bool, error) {
+		var copyErr error
+		info, copyErr = f.copyFile(ctx, srcPath, dstPath, OverwriteOnExist)
+		return f.shouldRetryAndCreateParents(ctx, dstPath, srcPath, copyErr)
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	dstObj, err := f.newObjectFromHiDriveObject(remote, info)
+	if err != nil {
+		return nil, err
+	}
+	return dstObj, nil
+}
+
+// Move src to this remote using server-side move operations.
+//
+// It returns the destination Object and a possible error.
+//
+// This returns fs.ErrorCantMove if the operation cannot be performed.
+//
+// NOTE: If an error occurs when moving the Object,
+// any parent-directories already created will NOT be deleted again.
+//
+// NOTE: This operation will expand sparse areas in the content of the source-Object
+// to blocks of 0-bytes in the destination-Object.
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't move - not same remote type")
+		return nil, fs.ErrorCantMove
+	}
+	// Get the absolute path to the source.
+	srcPath := srcObj.fs.resolvePath(srcObj.Remote())
+	//fs.Debugf(f, "executing Move(%s, %s).", srcPath, remote)
+	dstPath := f.resolvePath(remote)
+
+	var info *api.HiDriveObject
+	err := f.retryOnce.Call(func() (bool, error) {
+		var moveErr error
+		info, moveErr = f.moveFile(ctx, srcPath, dstPath, OverwriteOnExist)
+		return f.shouldRetryAndCreateParents(ctx, dstPath, srcPath, moveErr)
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	dstObj, err := f.newObjectFromHiDriveObject(remote, info)
+	if err != nil {
+		return nil, err
+	}
+	return dstObj, nil
+
+}
+
+// DirMove moves from src at srcRemote to this remote at dstRemote
+// using server-side move operations.
+//
+// This returns fs.ErrorCantCopy if the operation cannot be performed.
+// This returns fs.ErrorDirExists if the destination already exists.
+//
+// NOTE: If an error occurs when moving the directory,
+// any parent-directories already created will NOT be deleted again.
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		fs.Debugf(srcFs, "Can't move directory - not same remote type")
+		return fs.ErrorCantDirMove
+	}
+
+	// Get the absolute path to the source.
+	srcPath := srcFs.resolvePath(srcRemote)
+	//fs.Debugf(f, "executing DirMove(%s, %s).", srcPath, dstRemote)
+	dstPath := f.resolvePath(dstRemote)
+
+	err := f.retryOnce.Call(func() (bool, error) {
+		var moveErr error
+		_, moveErr = f.moveDirectory(ctx, srcPath, dstPath, IgnoreOnExist)
+		return f.shouldRetryAndCreateParents(ctx, dstPath, srcPath, moveErr)
+	})
+
+	if err != nil {
+		if isHTTPError(err, 409) {
+			return fs.ErrorDirExists
+		}
+		return err
+	}
+	return nil
+}
+
+// ------------------------------------------------------------
+
+// Fs returns the parent Fs.
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// String returns a string-representation of this Object.
+func (o *Object) String() string {
+	if o == nil {
+		return ""
+	}
+	return o.remote
+}
+
+// Remote returns the remote path.
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ID returns the ID of the Object if known, or "" if not.
+func (o *Object) ID() string {
+	err := o.readMetadata(context.TODO())
+	if err != nil {
+		fs.Logf(o, "Failed to read metadata: %v", err)
+		return ""
+	}
+	return o.id
+}
+
+// Hash returns the selected checksum of the file.
+// If no checksum is available it returns "".
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+	err := o.readMetadata(ctx)
+	if err != nil {
+		return "", fmt.Errorf("failed to read hash from metadata: %w", err)
+	}
+	switch t {
+	case hidrivehashType:
+		return o.hash, nil
+	default:
+		return "", hash.ErrUnsupported
+	}
+}
+
+// Size returns the size of an object in bytes.
+func (o *Object) Size() int64 {
+	err := o.readMetadata(context.TODO())
+	if err != nil {
+		fs.Logf(o, "Failed to read metadata: %v", err)
+		return -1
+	}
+	return o.size
+}
+
+// setMetadata sets the metadata from info.
+func (o *Object) setMetadata(info *api.HiDriveObject) error {
+	if info.Type == api.HiDriveObjectTypeDirectory {
+		return fs.ErrorIsDir
+	}
+	if info.Type != api.HiDriveObjectTypeFile {
+		return fmt.Errorf("%q is %q: %w", o.remote, info.Type, fs.ErrorNotAFile)
+	}
+	o.hasMetadata = true
+	o.size = info.Size
+	o.modTime = info.ModTime()
+	o.id = info.ID
+	o.hash = info.ContentHash
+	return nil
+}
+
+// readMetadata fetches the metadata if it has not already been fetched.
+func (o *Object) readMetadata(ctx context.Context) error {
+	if o.hasMetadata {
+		return nil
+	}
+	resolvedPath := o.fs.resolvePath(o.remote)
+	info, err := o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
+	if err != nil {
+		if isHTTPError(err, 404) {
+			return fs.ErrorObjectNotFound
+		}
+		return err
+	}
+	return o.setMetadata(info)
+}
+
+// ModTime returns the modification time of the object.
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	err := o.readMetadata(ctx)
+	if err != nil {
+		fs.Logf(o, "Failed to read metadata: %v", err)
+		return time.Now()
+	}
+	return o.modTime
+}
+
+// SetModTime sets the metadata on the object to set the modification date.
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+	parameters := api.NewQueryParameters()
+	resolvedPath := o.fs.resolvePath(o.remote)
+	parameters.SetPath(resolvedPath)
+	err := parameters.SetTime("mtime", modTime)
+
+	if err != nil {
+		return err
+	}
+
+	opts := rest.Opts{
+		Method:     "PATCH",
+		Path:       "/meta",
+		Parameters: parameters.Values,
+		NoResponse: true,
+	}
+
+	var resp *http.Response
+	err = o.fs.pacer.Call(func() (bool, error) {
+		resp, err = o.fs.srv.Call(ctx, &opts)
+		return o.fs.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return err
+	}
+	o.modTime = modTime
+	return nil
+}
+
+// Storable says whether this object can be stored.
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Open an object for reading.
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+	parameters := api.NewQueryParameters()
+	resolvedPath := o.fs.resolvePath(o.remote)
+	parameters.SetPath(resolvedPath)
+
+	fs.FixRangeOption(options, o.Size())
+	opts := rest.Opts{
+		Method:     "GET",
+		Path:       "/file",
+		Parameters: parameters.Values,
+		Options:    options,
+	}
+	var resp *http.Response
+	var err error
+	err = o.fs.pacer.Call(func() (bool, error) {
+		resp, err = o.fs.srv.Call(ctx, &opts)
+		return o.fs.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return resp.Body, err
+}
+
+// Update the existing object
+// with the contents of the io.Reader, modTime and size.
+//
+// For unknown-sized contents (indicated by src.Size() == -1)
+// this will try to properly upload it in multiple chunks.
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	//fs.Debugf(o.fs, "executing Update(%s, %v).", o.remote, options)
+	modTime := src.ModTime(ctx)
+	resolvedPath := o.fs.resolvePath(o.remote)
+
+	if o.fs.tokenRenewer != nil {
+		o.fs.tokenRenewer.Start()
+		defer o.fs.tokenRenewer.Stop()
+	}
+
+	// PutUnchecked can pass a valid SeekOption to skip ahead.
+	var offset uint64
+	for _, option := range options {
+		if seekoption, ok := option.(*fs.SeekOption); ok {
+			offset = uint64(seekoption.Offset)
+			break
+		}
+	}
+
+	var info *api.HiDriveObject
+	var err, metaErr error
+	if offset > 0 || src.Size() == -1 || src.Size() >= int64(o.fs.opt.UploadCutoff) {
+		fs.Debugf(o.fs, "Uploading with chunks of size %v and %v transfers in parallel at path '%s'.", int(o.fs.opt.UploadChunkSize), o.fs.opt.UploadConcurrency, resolvedPath)
+		// NOTE: o.fs.opt.UploadChunkSize should always
+		// be between 0 and MaximumUploadBytes,
+		// so the conversion to an int does not cause problems for valid inputs.
+		if offset > 0 {
+			// NOTE: The offset is only set
+			// when the file was newly created,
+			// therefore the file does not need truncating.
+			_, err = o.fs.updateFileChunked(ctx, resolvedPath, in, offset, int(o.fs.opt.UploadChunkSize), o.fs.opt.UploadConcurrency)
+			if err == nil {
+				err = o.SetModTime(ctx, modTime)
+			}
+		} else {
+			_, _, err = o.fs.uploadFileChunked(ctx, resolvedPath, in, modTime, int(o.fs.opt.UploadChunkSize), o.fs.opt.UploadConcurrency)
+		}
+		// Try to check if object was updated, eitherway.
+		// Metadata should be updated even if the upload fails.
+		info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
+	} else {
+		info, err = o.fs.overwriteFile(ctx, resolvedPath, cachedReader(in), modTime)
+		metaErr = err
+	}
+
+	// Update metadata of this object,
+	// if there was no error with getting the metadata.
+	if metaErr == nil {
+		metaErr = o.setMetadata(info)
+	}
+
+	// Errors with the upload-process are more relevant, return those first.
+	if err != nil {
+		return err
+	}
+	return metaErr
+}
+
+// Remove an object.
+func (o *Object) Remove(ctx context.Context) error {
+	resolvedPath := o.fs.resolvePath(o.remote)
+	return o.fs.deleteObject(ctx, resolvedPath)
+}
+
+// Check the interfaces are satisfied.
+var (
+	_ fs.Fs             = (*Fs)(nil)
+	_ fs.Purger         = (*Fs)(nil)
+	_ fs.PutStreamer    = (*Fs)(nil)
+	_ fs.PutUncheckeder = (*Fs)(nil)
+	_ fs.Copier         = (*Fs)(nil)
+	_ fs.Mover          = (*Fs)(nil)
+	_ fs.DirMover       = (*Fs)(nil)
+	_ fs.Object         = (*Object)(nil)
+	_ fs.IDer           = (*Object)(nil)
+)
diff --git a/backend/hidrive/hidrive_test.go b/backend/hidrive/hidrive_test.go
new file mode 100644
index 0000000000000..30e1ef2f1c2be
--- /dev/null
+++ b/backend/hidrive/hidrive_test.go
@@ -0,0 +1,45 @@
+// Test HiDrive filesystem interface
+package hidrive
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote.
+func TestIntegration(t *testing.T) {
+	name := "TestHiDrive"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize:       1,
+			MaxChunkSize:       MaximumUploadBytes,
+			CeilChunkSize:      nil,
+			NeedMultipleChunks: false,
+		},
+	})
+}
+
+// Change the configured UploadChunkSize.
+// Will only be called while no transfer is in progress.
+func (f *Fs) SetUploadChunkSize(chunksize fs.SizeSuffix) (fs.SizeSuffix, error) {
+	var old fs.SizeSuffix
+	old, f.opt.UploadChunkSize = f.opt.UploadChunkSize, chunksize
+	return old, nil
+}
+
+// Change the configured UploadCutoff.
+// Will only be called while no transfer is in progress.
+func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
+	var old fs.SizeSuffix
+	old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
+	return old, nil
+}
+
+var (
+	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
+	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
+)
diff --git a/backend/hidrive/hidrivehash/hidrivehash.go b/backend/hidrive/hidrivehash/hidrivehash.go
new file mode 100644
index 0000000000000..092663d4229e5
--- /dev/null
+++ b/backend/hidrive/hidrivehash/hidrivehash.go
@@ -0,0 +1,410 @@
+// Package hidrivehash implements the HiDrive hashing algorithm which combines SHA-1 hashes hierarchically to a single top-level hash.
+//
+// Note: This implementation does not grant access to any partial hashes generated.
+//
+// See: https://developer.hidrive.com/wp-content/uploads/2021/07/HiDrive_Synchronization-v3.3-rev28.pdf
+// (link to newest version: https://static.hidrive.com/dev/0001)
+package hidrivehash
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"encoding"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+
+	"github.com/rclone/rclone/backend/hidrive/hidrivehash/internal"
+)
+
+const (
+	// BlockSize of the checksum in bytes.
+	BlockSize = 4096
+	// Size of the checksum in bytes.
+	Size = sha1.Size
+	// sumsPerLevel is the number of checksums
+	sumsPerLevel = 256
+)
+
+var (
+	// zeroSum is a special hash consisting of 20 null-bytes.
+	// This will be the hash of any empty file (or ones containing only null-bytes).
+	zeroSum = [Size]byte{}
+	// ErrorInvalidEncoding is returned when a hash should be decoded from a binary form that is invalid.
+	ErrorInvalidEncoding = errors.New("encoded binary form is invalid for this hash")
+	// ErrorHashFull is returned when a hash reached its capacity and cannot accept any more input.
+	ErrorHashFull = errors.New("hash reached its capacity")
+)
+
+// writeByBlock writes len(p) bytes from p to the io.Writer in blocks of size blockSize.
+// It returns the number of bytes written from p (0 <= n <= len(p))
+// and any error encountered that caused the write to stop early.
+//
+// A pointer bytesInBlock to a counter needs to be supplied,
+// that is used to keep track how many bytes have been written to the writer already.
+// A pointer onlyNullBytesInBlock to a boolean needs to be supplied,
+// that is used to keep track whether the block so far only consists of null-bytes.
+// The callback onBlockWritten is called whenever a full block has been written to the writer
+// and is given as input the number of bytes that still need to be written.
+func writeByBlock(p []byte, writer io.Writer, blockSize uint32, bytesInBlock *uint32, onlyNullBytesInBlock *bool, onBlockWritten func(remaining int) error) (n int, err error) {
+	total := len(p)
+	nullBytes := make([]byte, blockSize)
+	for len(p) > 0 {
+		toWrite := int(blockSize - *bytesInBlock)
+		if toWrite > len(p) {
+			toWrite = len(p)
+		}
+		c, err := writer.Write(p[:toWrite])
+		*bytesInBlock += uint32(c)
+		*onlyNullBytesInBlock = *onlyNullBytesInBlock && bytes.Equal(nullBytes[:toWrite], p[:toWrite])
+		// Discard data written through a reslice
+		p = p[c:]
+		if err != nil {
+			return total - len(p), err
+		}
+		if *bytesInBlock == blockSize {
+			err = onBlockWritten(len(p))
+			if err != nil {
+				return total - len(p), err
+			}
+			*bytesInBlock = 0
+			*onlyNullBytesInBlock = true
+		}
+	}
+	return total, nil
+}
+
+// level is a hash.Hash that is used to aggregate the checksums produced by the level hierarchically beneath it.
+// It is used to represent any level-n hash, except for level-0.
+type level struct {
+	checksum              [Size]byte // aggregated checksum of this level
+	sumCount              uint32     // number of sums contained in this level so far
+	bytesInHasher         uint32     //  number of bytes written into hasher so far
+	onlyNullBytesInHasher bool       // whether the hasher only contains null-bytes so far
+	hasher                hash.Hash
+}
+
+// NewLevel returns a new hash.Hash computing any level-n hash, except level-0.
+func NewLevel() hash.Hash {
+	l := &level{}
+	l.Reset()
+	return l
+}
+
+// Add takes a position-embedded SHA-1 checksum and adds it to the level.
+func (l *level) Add(sha1sum []byte) {
+	var tmp uint
+	var carry bool
+	for i := Size - 1; i >= 0; i-- {
+		tmp = uint(sha1sum[i]) + uint(l.checksum[i])
+		if carry {
+			tmp++
+		}
+		carry = tmp > 255
+		l.checksum[i] = byte(tmp)
+	}
+}
+
+// IsFull returns whether the number of checksums added to this level reached its capacity.
+func (l *level) IsFull() bool {
+	return l.sumCount >= sumsPerLevel
+}
+
+// Write (via the embedded io.Writer interface) adds more data to the running hash.
+// Contrary to the specification from hash.Hash, this DOES return an error,
+// specifically ErrorHashFull if and only if IsFull() returns true.
+func (l *level) Write(p []byte) (n int, err error) {
+	if l.IsFull() {
+		return 0, ErrorHashFull
+	}
+	onBlockWritten := func(remaining int) error {
+		if !l.onlyNullBytesInHasher {
+			c, err := l.hasher.Write([]byte{byte(l.sumCount)})
+			l.bytesInHasher += uint32(c)
+			if err != nil {
+				return err
+			}
+			l.Add(l.hasher.Sum(nil))
+		}
+		l.sumCount++
+		l.hasher.Reset()
+		if remaining > 0 && l.IsFull() {
+			return ErrorHashFull
+		}
+		return nil
+	}
+	return writeByBlock(p, l.hasher, uint32(l.BlockSize()), &l.bytesInHasher, &l.onlyNullBytesInHasher, onBlockWritten)
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+// It does not change the underlying hash state.
+func (l *level) Sum(b []byte) []byte {
+	return append(b, l.checksum[:]...)
+}
+
+// Reset resets the Hash to its initial state.
+func (l *level) Reset() {
+	l.checksum = zeroSum // clear the current checksum
+	l.sumCount = 0
+	l.bytesInHasher = 0
+	l.onlyNullBytesInHasher = true
+	l.hasher = sha1.New()
+}
+
+// Size returns the number of bytes Sum will return.
+func (l *level) Size() int {
+	return Size
+}
+
+// BlockSize returns the hash's underlying block size.
+// The Write method must be able to accept any amount
+// of data, but it may operate more efficiently if all writes
+// are a multiple of the block size.
+func (l *level) BlockSize() int {
+	return Size
+}
+
+// MarshalBinary encodes the hash into a binary form and returns the result.
+func (l *level) MarshalBinary() ([]byte, error) {
+	b := make([]byte, Size+4+4+1)
+	copy(b, l.checksum[:])
+	binary.BigEndian.PutUint32(b[Size:], l.sumCount)
+	binary.BigEndian.PutUint32(b[Size+4:], l.bytesInHasher)
+	if l.onlyNullBytesInHasher {
+		b[Size+4+4] = 1
+	}
+	encodedHasher, err := l.hasher.(encoding.BinaryMarshaler).MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	b = append(b, encodedHasher...)
+	return b, nil
+}
+
+// UnmarshalBinary decodes the binary form generated by MarshalBinary.
+// The hash will replace its internal state accordingly.
+func (l *level) UnmarshalBinary(b []byte) error {
+	if len(b) < Size+4+4+1 {
+		return ErrorInvalidEncoding
+	}
+	copy(l.checksum[:], b)
+	l.sumCount = binary.BigEndian.Uint32(b[Size:])
+	l.bytesInHasher = binary.BigEndian.Uint32(b[Size+4:])
+	switch b[Size+4+4] {
+	case 0:
+		l.onlyNullBytesInHasher = false
+	case 1:
+		l.onlyNullBytesInHasher = true
+	default:
+		return ErrorInvalidEncoding
+	}
+	err := l.hasher.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[Size+4+4+1:])
+	return err
+}
+
+// hidriveHash is the hash computing the actual checksum used by HiDrive by combining multiple level-hashes.
+type hidriveHash struct {
+	levels               []*level   // collection of level-hashes, one for each level starting at level-1
+	lastSumWritten       [Size]byte // the last checksum written to any of the levels
+	bytesInBlock         uint32     // bytes written into blockHash so far
+	onlyNullBytesInBlock bool       // whether the hasher only contains null-bytes so far
+	blockHash            hash.Hash
+}
+
+// New returns a new hash.Hash computing the HiDrive checksum.
+func New() hash.Hash {
+	h := &hidriveHash{}
+	h.Reset()
+	return h
+}
+
+// aggregateToLevel writes the checksum to the level at the given index
+// and if necessary propagates any changes to levels above.
+func (h *hidriveHash) aggregateToLevel(index int, sum []byte) {
+	for i := index; ; i++ {
+		if i >= len(h.levels) {
+			h.levels = append(h.levels, NewLevel().(*level))
+		}
+		_, err := h.levels[i].Write(sum)
+		copy(h.lastSumWritten[:], sum)
+		if err != nil {
+			panic(fmt.Errorf("level-hash should not have produced an error: %w", err))
+		}
+		if !h.levels[i].IsFull() {
+			break
+		}
+		sum = h.levels[i].Sum(nil)
+		h.levels[i].Reset()
+	}
+}
+
+// Write (via the embedded io.Writer interface) adds more data to the running hash.
+// It never returns an error.
+func (h *hidriveHash) Write(p []byte) (n int, err error) {
+	onBlockWritten := func(remaining int) error {
+		var sum []byte
+		if h.onlyNullBytesInBlock {
+			sum = zeroSum[:]
+		} else {
+			sum = h.blockHash.Sum(nil)
+		}
+		h.blockHash.Reset()
+		h.aggregateToLevel(0, sum)
+		return nil
+	}
+	return writeByBlock(p, h.blockHash, uint32(BlockSize), &h.bytesInBlock, &h.onlyNullBytesInBlock, onBlockWritten)
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+// It does not change the underlying hash state.
+func (h *hidriveHash) Sum(b []byte) []byte {
+	// Save internal state.
+	state, err := h.MarshalBinary()
+	if err != nil {
+		panic(fmt.Errorf("saving the internal state should not have produced an error: %w", err))
+	}
+
+	if h.bytesInBlock > 0 {
+		// Fill remainder of block with null-bytes.
+		filler := make([]byte, h.BlockSize()-int(h.bytesInBlock))
+		_, err = h.Write(filler)
+		if err != nil {
+			panic(fmt.Errorf("filling with null-bytes should not have an error: %w", err))
+		}
+	}
+
+	checksum := zeroSum
+	for i := 0; i < len(h.levels); i++ {
+		level := h.levels[i]
+		if i < len(h.levels)-1 {
+			// Aggregate non-empty non-final levels.
+			if level.sumCount >= 1 {
+				h.aggregateToLevel(i+1, level.Sum(nil))
+				level.Reset()
+			}
+		} else {
+			// Determine sum of final level.
+			if level.sumCount > 1 {
+				copy(checksum[:], level.Sum(nil))
+			} else {
+				// This is needed, otherwise there is no way to return
+				// the non-position-embedded checksum.
+				checksum = h.lastSumWritten
+			}
+		}
+	}
+
+	// Restore internal state.
+	err = h.UnmarshalBinary(state)
+	if err != nil {
+		panic(fmt.Errorf("restoring the internal state should not have produced an error: %w", err))
+	}
+
+	return append(b, checksum[:]...)
+}
+
+// Reset resets the Hash to its initial state.
+func (h *hidriveHash) Reset() {
+	h.levels = nil
+	h.lastSumWritten = zeroSum // clear the last written checksum
+	h.bytesInBlock = 0
+	h.onlyNullBytesInBlock = true
+	h.blockHash = sha1.New()
+}
+
+// Size returns the number of bytes Sum will return.
+func (h *hidriveHash) Size() int {
+	return Size
+}
+
+// BlockSize returns the hash's underlying block size.
+// The Write method must be able to accept any amount
+// of data, but it may operate more efficiently if all writes
+// are a multiple of the block size.
+func (h *hidriveHash) BlockSize() int {
+	return BlockSize
+}
+
+// MarshalBinary encodes the hash into a binary form and returns the result.
+func (h *hidriveHash) MarshalBinary() ([]byte, error) {
+	b := make([]byte, Size+4+1+8)
+	copy(b, h.lastSumWritten[:])
+	binary.BigEndian.PutUint32(b[Size:], h.bytesInBlock)
+	if h.onlyNullBytesInBlock {
+		b[Size+4] = 1
+	}
+
+	binary.BigEndian.PutUint64(b[Size+4+1:], uint64(len(h.levels)))
+	for _, level := range h.levels {
+		encodedLevel, err := level.MarshalBinary()
+		if err != nil {
+			return nil, err
+		}
+		encodedLength := make([]byte, 8)
+		binary.BigEndian.PutUint64(encodedLength, uint64(len(encodedLevel)))
+		b = append(b, encodedLength...)
+		b = append(b, encodedLevel...)
+	}
+	encodedBlockHash, err := h.blockHash.(encoding.BinaryMarshaler).MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	b = append(b, encodedBlockHash...)
+	return b, nil
+}
+
+// UnmarshalBinary decodes the binary form generated by MarshalBinary.
+// The hash will replace its internal state accordingly.
+func (h *hidriveHash) UnmarshalBinary(b []byte) error {
+	if len(b) < Size+4+1+8 {
+		return ErrorInvalidEncoding
+	}
+	copy(h.lastSumWritten[:], b)
+	h.bytesInBlock = binary.BigEndian.Uint32(b[Size:])
+	switch b[Size+4] {
+	case 0:
+		h.onlyNullBytesInBlock = false
+	case 1:
+		h.onlyNullBytesInBlock = true
+	default:
+		return ErrorInvalidEncoding
+	}
+
+	amount := binary.BigEndian.Uint64(b[Size+4+1:])
+	h.levels = make([]*level, int(amount))
+	offset := Size + 4 + 1 + 8
+	for i := range h.levels {
+		length := int(binary.BigEndian.Uint64(b[offset:]))
+		offset += 8
+		h.levels[i] = NewLevel().(*level)
+		err := h.levels[i].UnmarshalBinary(b[offset : offset+length])
+		if err != nil {
+			return err
+		}
+		offset += length
+	}
+	err := h.blockHash.(encoding.BinaryUnmarshaler).UnmarshalBinary(b[offset:])
+	return err
+}
+
+// Sum returns the HiDrive checksum of the data.
+func Sum(data []byte) [Size]byte {
+	h := New().(*hidriveHash)
+	_, _ = h.Write(data)
+	var result [Size]byte
+	copy(result[:], h.Sum(nil))
+	return result
+}
+
+// Check the interfaces are satisfied.
+var (
+	_ hash.Hash                  = (*level)(nil)
+	_ encoding.BinaryMarshaler   = (*level)(nil)
+	_ encoding.BinaryUnmarshaler = (*level)(nil)
+	_ internal.LevelHash         = (*level)(nil)
+	_ hash.Hash                  = (*hidriveHash)(nil)
+	_ encoding.BinaryMarshaler   = (*hidriveHash)(nil)
+	_ encoding.BinaryUnmarshaler = (*hidriveHash)(nil)
+)
diff --git a/backend/hidrive/hidrivehash/hidrivehash_test.go b/backend/hidrive/hidrivehash/hidrivehash_test.go
new file mode 100644
index 0000000000000..d27970c3462e1
--- /dev/null
+++ b/backend/hidrive/hidrivehash/hidrivehash_test.go
@@ -0,0 +1,395 @@
+package hidrivehash_test
+
+import (
+	"crypto/sha1"
+	"encoding"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/rclone/rclone/backend/hidrive/hidrivehash"
+	"github.com/rclone/rclone/backend/hidrive/hidrivehash/internal"
+	"github.com/stretchr/testify/assert"
+)
+
+// helper functions to set up test-tables
+
+func sha1ArrayAsSlice(sum [sha1.Size]byte) []byte {
+	return sum[:]
+}
+
+func mustDecode(hexstring string) []byte {
+	result, err := hex.DecodeString(hexstring)
+	if err != nil {
+		panic(err)
+	}
+	return result
+}
+
+// ------------------------------------------------------------
+
+var testTableLevelPositionEmbedded = []struct {
+	ins  [][]byte
+	outs [][]byte
+	name string
+}{
+	{
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{245, 202, 195, 223, 121, 198, 189, 112, 138, 202, 222, 2, 146, 156, 127, 16, 208, 233, 98, 88}),
+			sha1ArrayAsSlice([20]byte{78, 188, 156, 219, 173, 54, 81, 55, 47, 220, 222, 207, 201, 21, 57, 252, 255, 239, 251, 186}),
+		},
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{245, 202, 195, 223, 121, 198, 189, 112, 138, 202, 222, 2, 146, 156, 127, 16, 208, 233, 98, 88}),
+			sha1ArrayAsSlice([20]byte{68, 135, 96, 187, 38, 253, 14, 167, 186, 167, 188, 210, 91, 177, 185, 13, 208, 217, 94, 18}),
+		},
+		"documentation-v3.2rev27-example L0 (position-embedded)",
+	},
+	{
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{68, 254, 92, 166, 52, 37, 104, 180, 22, 123, 249, 144, 182, 78, 64, 74, 57, 117, 225, 195}),
+			sha1ArrayAsSlice([20]byte{75, 211, 153, 190, 125, 179, 67, 49, 60, 149, 98, 246, 142, 20, 11, 254, 159, 162, 129, 237}),
+			sha1ArrayAsSlice([20]byte{150, 2, 9, 153, 97, 153, 189, 104, 147, 14, 77, 203, 244, 243, 25, 212, 67, 48, 111, 107}),
+		},
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{68, 254, 92, 166, 52, 37, 104, 180, 22, 123, 249, 144, 182, 78, 64, 74, 57, 117, 225, 195}),
+			sha1ArrayAsSlice([20]byte{144, 209, 246, 100, 177, 216, 171, 229, 83, 17, 92, 135, 68, 98, 76, 72, 217, 24, 99, 176}),
+			sha1ArrayAsSlice([20]byte{38, 211, 255, 254, 19, 114, 105, 77, 230, 31, 170, 83, 57, 85, 102, 29, 28, 72, 211, 27}),
+		},
+		"documentation-example L0 (position-embedded)",
+	},
+	{
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{173, 123, 132, 245, 176, 172, 43, 183, 121, 40, 66, 252, 101, 249, 188, 193, 160, 189, 2, 116}),
+			sha1ArrayAsSlice([20]byte{40, 34, 8, 238, 37, 5, 237, 184, 79, 105, 10, 167, 171, 254, 13, 229, 132, 112, 254, 8}),
+			sha1ArrayAsSlice([20]byte{39, 112, 26, 86, 190, 35, 100, 101, 28, 131, 122, 191, 254, 144, 239, 107, 253, 124, 104, 203}),
+		},
+		[][]byte{
+			sha1ArrayAsSlice([20]byte{173, 123, 132, 245, 176, 172, 43, 183, 121, 40, 66, 252, 101, 249, 188, 193, 160, 189, 2, 116}),
+			sha1ArrayAsSlice([20]byte{213, 157, 141, 227, 213, 178, 25, 111, 200, 145, 77, 164, 17, 247, 202, 167, 37, 46, 0, 124}),
+			sha1ArrayAsSlice([20]byte{253, 13, 168, 58, 147, 213, 125, 212, 229, 20, 200, 100, 16, 136, 186, 19, 34, 170, 105, 71}),
+		},
+		"documentation-example L1 (position-embedded)",
+	},
+}
+
+var testTableLevel = []struct {
+	ins  [][]byte
+	outs [][]byte
+	name string
+}{
+	{
+		[][]byte{
+			mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+			mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+			mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+		},
+		[][]byte{
+			mustDecode("44fe5ca6342568b4167bf990b64e404a3975e1c3"),
+			mustDecode("90d1f664b1d8abe553115c8744624c48d91863b0"),
+			mustDecode("26d3fffe1372694de61faa533955661d1c48d31b"),
+		},
+		"documentation-example L0",
+	},
+	{
+		[][]byte{
+			mustDecode("75a9f88fb219ef1dd31adf41c93e2efaac8d0245"),
+			mustDecode("daedc425199501b1e86b5eaba5649cbde205e6ae"),
+			mustDecode("286ac5283f99c4e0f11683900a3e39661c375dd6"),
+		},
+		[][]byte{
+			mustDecode("ad7b84f5b0ac2bb7792842fc65f9bcc1a0bd0274"),
+			mustDecode("d59d8de3d5b2196fc8914da411f7caa7252e007c"),
+			mustDecode("fd0da83a93d57dd4e514c8641088ba1322aa6947"),
+		},
+		"documentation-example L1",
+	},
+	{
+		[][]byte{
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("75a9f88fb219ef1dd31adf41c93e2efaac8d0245"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("daedc425199501b1e86b5eaba5649cbde205e6ae"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("286ac5283f99c4e0f11683900a3e39661c375dd6"),
+			mustDecode("0000000000000000000000000000000000000000"),
+		},
+		[][]byte{
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("0000000000000000000000000000000000000000"),
+			mustDecode("a197464ec19f2b2b2bc6b21f6c939c7e57772843"),
+			mustDecode("a197464ec19f2b2b2bc6b21f6c939c7e57772843"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("b04769357aa4eb4b52cd5bec6935bc8f977fa3a1"),
+			mustDecode("8f56351897b4e1d100646fa122c924347721b2f5"),
+			mustDecode("8f56351897b4e1d100646fa122c924347721b2f5"),
+		},
+		"mixed-with-empties",
+	},
+}
+
+var testTable = []struct {
+	data []byte
+	// pattern describes how to use data to construct the hash-input.
+	// For every entry n at even indices this repeats the data n times.
+	// For every entry m at odd indices this repeats a null-byte m times.
+	// The input-data is constructed by concatinating the results in order.
+	pattern []int64
+	out     []byte
+	name    string
+}{
+	{
+		[]byte("#ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz\n"),
+		[]int64{64},
+		mustDecode("09f077820a8a41f34a639f2172f1133b1eafe4e6"),
+		"documentation-example L0",
+	},
+	{
+		[]byte("#ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz\n"),
+		[]int64{64 * 256},
+		mustDecode("75a9f88fb219ef1dd31adf41c93e2efaac8d0245"),
+		"documentation-example L1",
+	},
+	{
+		[]byte("#ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz\n"),
+		[]int64{64 * 256, 0, 64 * 128, 4096 * 128, 64*2 + 32},
+		mustDecode("fd0da83a93d57dd4e514c8641088ba1322aa6947"),
+		"documentation-example L2",
+	},
+	{
+		[]byte("hello rclone\n"),
+		[]int64{316},
+		mustDecode("72370f9c18a2c20b31d71f3f4cee7a3cd2703737"),
+		"not-block-aligned",
+	},
+	{
+		[]byte("hello rclone\n"),
+		[]int64{13, 4096 * 3, 4},
+		mustDecode("a6990b81791f0d2db750b38f046df321c975aa60"),
+		"not-block-aligned-with-null-bytes",
+	},
+	{
+		[]byte{},
+		[]int64{},
+		mustDecode("0000000000000000000000000000000000000000"),
+		"empty",
+	},
+	{
+		[]byte{},
+		[]int64{0, 4096 * 256 * 256},
+		mustDecode("0000000000000000000000000000000000000000"),
+		"null-bytes",
+	},
+}
+
+// ------------------------------------------------------------
+
+func TestLevelAdd(t *testing.T) {
+	for _, test := range testTableLevelPositionEmbedded {
+		l := hidrivehash.NewLevel().(internal.LevelHash)
+		t.Run(test.name, func(t *testing.T) {
+			for i := range test.ins {
+				l.Add(test.ins[i])
+				assert.Equal(t, test.outs[i], l.Sum(nil))
+			}
+		})
+	}
+}
+
+func TestLevelWrite(t *testing.T) {
+	for _, test := range testTableLevel {
+		l := hidrivehash.NewLevel()
+		t.Run(test.name, func(t *testing.T) {
+			for i := range test.ins {
+				l.Write(test.ins[i])
+				assert.Equal(t, test.outs[i], l.Sum(nil))
+			}
+		})
+	}
+}
+
+func TestLevelIsFull(t *testing.T) {
+	content := [hidrivehash.Size]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
+	l := hidrivehash.NewLevel()
+	for i := 0; i < 256; i++ {
+		assert.False(t, l.(internal.LevelHash).IsFull())
+		written, err := l.Write(content[:])
+		assert.Equal(t, len(content), written)
+		if !assert.NoError(t, err) {
+			t.FailNow()
+		}
+	}
+	assert.True(t, l.(internal.LevelHash).IsFull())
+	written, err := l.Write(content[:])
+	assert.True(t, l.(internal.LevelHash).IsFull())
+	assert.Equal(t, 0, written)
+	assert.ErrorIs(t, err, hidrivehash.ErrorHashFull)
+}
+
+func TestLevelReset(t *testing.T) {
+	l := hidrivehash.NewLevel()
+	zeroHash := l.Sum(nil)
+	_, err := l.Write([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19})
+	if assert.NoError(t, err) {
+		assert.NotEqual(t, zeroHash, l.Sum(nil))
+		l.Reset()
+		assert.Equal(t, zeroHash, l.Sum(nil))
+	}
+}
+
+func TestLevelSize(t *testing.T) {
+	l := hidrivehash.NewLevel()
+	assert.Equal(t, 20, l.Size())
+}
+
+func TestLevelBlockSize(t *testing.T) {
+	l := hidrivehash.NewLevel()
+	assert.Equal(t, 20, l.BlockSize())
+}
+
+func TestLevelBinaryMarshaler(t *testing.T) {
+	content := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}
+	l := hidrivehash.NewLevel().(internal.LevelHash)
+	l.Write(content[:10])
+	encoded, err := l.MarshalBinary()
+	if assert.NoError(t, err) {
+		d := hidrivehash.NewLevel().(internal.LevelHash)
+		err = d.UnmarshalBinary(encoded)
+		if assert.NoError(t, err) {
+			assert.Equal(t, l.Sum(nil), d.Sum(nil))
+			l.Write(content[10:])
+			d.Write(content[10:])
+			assert.Equal(t, l.Sum(nil), d.Sum(nil))
+		}
+	}
+}
+
+func TestLevelInvalidEncoding(t *testing.T) {
+	l := hidrivehash.NewLevel().(internal.LevelHash)
+	err := l.UnmarshalBinary([]byte{})
+	assert.ErrorIs(t, err, hidrivehash.ErrorInvalidEncoding)
+}
+
+// ------------------------------------------------------------
+
+type infiniteReader struct {
+	source []byte
+	offset int
+}
+
+func (m *infiniteReader) Read(b []byte) (int, error) {
+	count := copy(b, m.source[m.offset:])
+	m.offset += count
+	m.offset %= len(m.source)
+	return count, nil
+}
+
+func writeInChunks(writer io.Writer, chunkSize int64, data []byte, pattern []int64) error {
+	readers := make([]io.Reader, len(pattern))
+	nullBytes := [4096]byte{}
+	for i, n := range pattern {
+		if i%2 == 0 {
+			readers[i] = io.LimitReader(&infiniteReader{data, 0}, n*int64(len(data)))
+		} else {
+			readers[i] = io.LimitReader(&infiniteReader{nullBytes[:], 0}, n)
+		}
+	}
+	reader := io.MultiReader(readers...)
+	for {
+		_, err := io.CopyN(writer, reader, chunkSize)
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			return err
+		}
+	}
+}
+
+func TestWrite(t *testing.T) {
+	for _, test := range testTable {
+		t.Run(test.name, func(t *testing.T) {
+			h := hidrivehash.New()
+			err := writeInChunks(h, int64(h.BlockSize()), test.data, test.pattern)
+			if assert.NoError(t, err) {
+				normalSum := h.Sum(nil)
+				assert.Equal(t, test.out, normalSum)
+				// Test if different block-sizes produce differing results.
+				for _, blockSize := range []int64{397, 512, 4091, 8192, 10000} {
+					t.Run(fmt.Sprintf("block-size %v", blockSize), func(t *testing.T) {
+						h := hidrivehash.New()
+						err := writeInChunks(h, blockSize, test.data, test.pattern)
+						if assert.NoError(t, err) {
+							assert.Equal(t, normalSum, h.Sum(nil))
+						}
+					})
+				}
+			}
+		})
+	}
+}
+
+func TestReset(t *testing.T) {
+	h := hidrivehash.New()
+	zeroHash := h.Sum(nil)
+	_, err := h.Write([]byte{1})
+	if assert.NoError(t, err) {
+		assert.NotEqual(t, zeroHash, h.Sum(nil))
+		h.Reset()
+		assert.Equal(t, zeroHash, h.Sum(nil))
+	}
+}
+
+func TestSize(t *testing.T) {
+	h := hidrivehash.New()
+	assert.Equal(t, 20, h.Size())
+}
+
+func TestBlockSize(t *testing.T) {
+	h := hidrivehash.New()
+	assert.Equal(t, 4096, h.BlockSize())
+}
+
+func TestBinaryMarshaler(t *testing.T) {
+	for _, test := range testTable {
+		h := hidrivehash.New()
+		d := hidrivehash.New()
+		half := len(test.pattern) / 2
+		t.Run(test.name, func(t *testing.T) {
+			err := writeInChunks(h, int64(h.BlockSize()), test.data, test.pattern[:half])
+			assert.NoError(t, err)
+			encoded, err := h.(encoding.BinaryMarshaler).MarshalBinary()
+			if assert.NoError(t, err) {
+				err = d.(encoding.BinaryUnmarshaler).UnmarshalBinary(encoded)
+				if assert.NoError(t, err) {
+					assert.Equal(t, h.Sum(nil), d.Sum(nil))
+					err = writeInChunks(h, int64(h.BlockSize()), test.data, test.pattern[half:])
+					assert.NoError(t, err)
+					err = writeInChunks(d, int64(d.BlockSize()), test.data, test.pattern[half:])
+					assert.NoError(t, err)
+					assert.Equal(t, h.Sum(nil), d.Sum(nil))
+				}
+			}
+		})
+	}
+}
+
+func TestInvalidEncoding(t *testing.T) {
+	h := hidrivehash.New()
+	err := h.(encoding.BinaryUnmarshaler).UnmarshalBinary([]byte{})
+	assert.ErrorIs(t, err, hidrivehash.ErrorInvalidEncoding)
+}
+
+func TestSum(t *testing.T) {
+	assert.Equal(t, [hidrivehash.Size]byte{}, hidrivehash.Sum([]byte{}))
+	content := []byte{1}
+	h := hidrivehash.New()
+	h.Write(content)
+	sum := hidrivehash.Sum(content)
+	assert.Equal(t, h.Sum(nil), sum[:])
+}
diff --git a/backend/hidrive/hidrivehash/internal/internal.go b/backend/hidrive/hidrivehash/internal/internal.go
new file mode 100644
index 0000000000000..f1596a9e68d42
--- /dev/null
+++ b/backend/hidrive/hidrivehash/internal/internal.go
@@ -0,0 +1,17 @@
+package internal
+
+import (
+	"encoding"
+	"hash"
+)
+
+// LevelHash is an internal interface for level-hashes.
+type LevelHash interface {
+	encoding.BinaryMarshaler
+	encoding.BinaryUnmarshaler
+	hash.Hash
+	// Add takes a position-embedded checksum and adds it to the level.
+	Add(sum []byte)
+	// IsFull returns whether the number of checksums added to this level reached its capacity.
+	IsFull() bool
+}
diff --git a/bin/make_manual.py b/bin/make_manual.py
index 335f4c8148111..fa14b36a3748c 100755
--- a/bin/make_manual.py
+++ b/bin/make_manual.py
@@ -47,6 +47,7 @@
     "googlephotos.md",
     "hasher.md",
     "hdfs.md",
+    "hidrive.md",
     "http.md",
     "hubic.md",
     "internetarchive.md",
diff --git a/docs/content/_index.md b/docs/content/_index.md
index b6e2ac4d1c0ac..9d0f86bf439d9 100644
--- a/docs/content/_index.md
+++ b/docs/content/_index.md
@@ -128,6 +128,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Google Photos" home="https://www.google.com/photos/about/" config="/googlephotos/" >}}
 {{< provider name="HDFS" home="https://hadoop.apache.org/" config="/hdfs/" >}}
 {{< provider name="Hetzner Storage Box" home="https://www.hetzner.com/storage/storage-box" config="/sftp/#hetzner-storage-box" >}}
+{{< provider name="HiDrive" home="https://www.strato.de/cloud-speicher/" config="/hidrive/" >}}
 {{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}}
 {{< provider name="Hubic" home="https://hubic.com/" config="/hubic/" >}}
 {{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}}
diff --git a/docs/content/docs.md b/docs/content/docs.md
index a2bc38b078249..a71728cc7e98a 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -49,6 +49,7 @@ See the following for detailed instructions for
   * [Google Photos](/googlephotos/)
   * [Hasher](/hasher/) - to handle checksums for other remotes
   * [HDFS](/hdfs/)
+  * [HiDrive](/hidrive/)
   * [HTTP](/http/)
   * [Hubic](/hubic/)
   * [Internet Archive](/internetarchive/)
diff --git a/docs/content/hidrive.md b/docs/content/hidrive.md
new file mode 100644
index 0000000000000..2d667a9e6d255
--- /dev/null
+++ b/docs/content/hidrive.md
@@ -0,0 +1,461 @@
+---
+title: "HiDrive"
+description: "Rclone docs for HiDrive"
+---
+
+# {{< icon "fa fa-cloud" >}} HiDrive
+
+Paths are specified as `remote:path`
+
+Paths may be as deep as required, e.g. `remote:directory/subdirectory`.
+
+The initial setup for hidrive involves getting a token from HiDrive
+which you need to do in your browser.
+`rclone config` walks you through it.
+
+## Configuration
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+[snip]
+XX / HiDrive
+   \ "hidrive"
+[snip]
+Storage> hidrive
+OAuth Client Id - Leave blank normally.
+client_id>
+OAuth Client Secret - Leave blank normally.
+client_secret>
+Access permissions that rclone should use when requesting access from HiDrive.
+Leave blank normally.
+scope_access>
+Edit advanced config?
+y/n> n
+Use auto config?
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+type = hidrive
+token = {"access_token":"xxxxxxxxxxxxxxxxxxxx","token_type":"Bearer","refresh_token":"xxxxxxxxxxxxxxxxxxxxxxx","expiry":"xxxxxxxxxxxxxxxxxxxxxxx"}
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+**You should be aware that OAuth-tokens can be used to access your account
+and hence should not be shared with other persons.**
+See the [below section](#keeping-your-tokens-safe) for more information.
+
+See the [remote setup docs](/remote_setup/) for how to set it up on a
+machine with no Internet browser available.
+
+Note that rclone runs a webserver on your local machine to collect the
+token as returned from HiDrive. This only runs from the moment it opens
+your browser to the moment you get back the verification code.
+The webserver runs on `http://127.0.0.1:53682/`.
+If local port `53682` is protected by a firewall you may need to temporarily
+unblock the firewall to complete authorization.
+
+Once configured you can then use `rclone` like this,
+
+List directories in top level of your HiDrive root folder
+
+    rclone lsd remote:
+
+List all the files in your HiDrive filesystem
+
+    rclone ls remote:
+
+To copy a local directory to a HiDrive directory called backup
+
+    rclone copy /home/source remote:backup
+
+### Keeping your tokens safe
+
+Any OAuth-tokens will be stored by rclone in the remote's configuration file as unencrypted text.
+Anyone can use a valid refresh-token to access your HiDrive filesystem without knowing your password.
+Therefore you should make sure no one else can access your configuration.
+
+It is possible to encrypt rclone's configuration file.
+You can find information on securing your configuration file by viewing the [configuration encryption docs](/docs/#configuration-encryption).
+
+### Invalid refresh token
+
+As can be verified [here](https://developer.hidrive.com/basics-flows/),
+each `refresh_token` (for Native Applications) is valid for 60 days.
+If used to access HiDrivei, its validity will be automatically extended.
+
+This means that if you
+
+  * Don't use the HiDrive remote for 60 days
+
+then rclone will return an error which includes a text
+that implies the refresh token is *invalid* or *expired*.
+
+To fix this you will need to authorize rclone to access your HiDrive account again.
+
+Using
+
+    rclone config reconnect remote:
+
+the process is very similar to the process of initial setup exemplified before.
+
+### Modified time and hashes
+
+HiDrive allows modification times to be set on objects accurate to 1 second.
+
+HiDrive supports [its own hash type](https://static.hidrive.com/dev/0001)
+which is used to verify the integrety of file contents after successful transfers.
+
+### Restricted filename characters
+
+HiDrive cannot store files or folders that include
+`/` (0x2F) or null-bytes (0x00) in their name.
+Any other characters can be used in the names of files or folders.
+Additionally, files or folders cannot be named either of the following: `.` or `..`
+
+Therefore rclone will automatically replace these characters,
+if files or folders are stored or accessed with such names.
+
+You can read about how this filename encoding works in general
+[here](overview/#restricted-filenames).
+
+Keep in mind that HiDrive only supports file or folder names
+with a length of 255 characters or less.
+
+### Transfers
+
+HiDrive limits file sizes per single request to a maximum of 2 GiB.
+To allow storage of larger files and allow for better upload performance,
+the hidrive backend will use a chunked transfer for files larger than 96 MiB.
+Rclone will upload multiple parts/chunks of the file at the same time.
+Chunks in the process of being uploaded are buffered in memory,
+so you may want to restrict this behaviour on systems with limited resources.
+
+You can customize this behaviour using the following options:
+
+* `chunk_size`: size of file parts
+* `upload_cutoff`: files larger or equal to this in size will use a chunked transfer
+* `upload_concurrency`: number of file-parts to upload at the same time
+
+See the below section about configuration options for more details.
+
+### Root folder
+
+You can set the root folder for rclone.
+This is the directory that rclone considers to be the root of your HiDrive.
+
+Usually, you will leave this blank, and rclone will use the root of the account.
+
+However, you can set this to restrict rclone to a specific folder hierarchy.
+
+This works by prepending the contents of the `root_prefix` option
+to any paths accessed by rclone.
+For example, the following two ways to access the home directory are equivalent:
+
+    rclone lsd --hidrive-root-prefix="/users/test/" remote:path
+
+    rclone lsd remote:/users/test/path
+
+See the below section about configuration options for more details.
+
+### Directory member count
+
+By default, rclone will know the number of directory members contained in a directory.
+For example, `rclone lsd` uses this information.
+
+The acquisition of this information will result in additional time costs for HiDrive's API.
+When dealing with large directory structures, it may be desirable to circumvent this time cost,
+especially when this information is not explicitly needed.
+For this, the `disable_fetching_member_count` option can be used.
+
+See the below section about configuration options for more details.
+
+{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/hidrive/hidrive.go then run make backenddocs" >}}
+### Standard options
+
+Here are the standard options specific to hidrive (HiDrive).
+
+#### --hidrive-client-id
+
+OAuth Client Id.
+
+Leave blank normally.
+
+Properties:
+
+- Config:      client_id
+- Env Var:     RCLONE_HIDRIVE_CLIENT_ID
+- Type:        string
+- Required:    false
+
+#### --hidrive-client-secret
+
+OAuth Client Secret.
+
+Leave blank normally.
+
+Properties:
+
+- Config:      client_secret
+- Env Var:     RCLONE_HIDRIVE_CLIENT_SECRET
+- Type:        string
+- Required:    false
+
+#### --hidrive-scope-access
+
+Access permissions that rclone should use when requesting access from HiDrive.
+
+Properties:
+
+- Config:      scope_access
+- Env Var:     RCLONE_HIDRIVE_SCOPE_ACCESS
+- Type:        string
+- Default:     "rw"
+- Examples:
+    - "rw"
+        - Read and write access to resources.
+    - "ro"
+        - Read-only access to resources.
+
+### Advanced options
+
+Here are the advanced options specific to hidrive (HiDrive).
+
+#### --hidrive-token
+
+OAuth Access Token as a JSON blob.
+
+Properties:
+
+- Config:      token
+- Env Var:     RCLONE_HIDRIVE_TOKEN
+- Type:        string
+- Required:    false
+
+#### --hidrive-auth-url
+
+Auth server URL.
+
+Leave blank to use the provider defaults.
+
+Properties:
+
+- Config:      auth_url
+- Env Var:     RCLONE_HIDRIVE_AUTH_URL
+- Type:        string
+- Required:    false
+
+#### --hidrive-token-url
+
+Token server url.
+
+Leave blank to use the provider defaults.
+
+Properties:
+
+- Config:      token_url
+- Env Var:     RCLONE_HIDRIVE_TOKEN_URL
+- Type:        string
+- Required:    false
+
+#### --hidrive-scope-role
+
+User-level that rclone should use when requesting access from HiDrive.
+
+Properties:
+
+- Config:      scope_role
+- Env Var:     RCLONE_HIDRIVE_SCOPE_ROLE
+- Type:        string
+- Default:     "user"
+- Examples:
+    - "user"
+        - User-level access to management permissions.
+        - This will be sufficient in most cases.
+    - "admin"
+        - Extensive access to management permissions.
+    - "owner"
+        - Full access to management permissions.
+
+#### --hidrive-root-prefix
+
+The root/parent folder for all paths.
+
+Fill in to use the specified folder as the parent for all paths given to the remote.
+This way rclone can use any folder as its starting point.
+
+Properties:
+
+- Config:      root_prefix
+- Env Var:     RCLONE_HIDRIVE_ROOT_PREFIX
+- Type:        string
+- Default:     "/"
+- Examples:
+    - "/"
+        - The topmost directory accessible by rclone.
+        - This will be equivalent with "root" if rclone uses a regular HiDrive user account.
+    - "root"
+        - The topmost directory of the HiDrive user account
+    - ""
+        - This specifies that there is no root-prefix for your paths.
+        - When using this you will always need to specify paths to this remote with a valid parent e.g. "remote:/path/to/dir" or "remote:root/path/to/dir".
+
+#### --hidrive-endpoint
+
+Endpoint for the service.
+
+This is the URL that API-calls will be made to.
+
+Properties:
+
+- Config:      endpoint
+- Env Var:     RCLONE_HIDRIVE_ENDPOINT
+- Type:        string
+- Default:     "https://api.hidrive.strato.com/2.1"
+
+#### --hidrive-disable-fetching-member-count
+
+Do not fetch number of objects in directories unless it is absolutely necessary.
+
+Requests may be faster if the number of objects in subdirectories is not fetched.
+
+Properties:
+
+- Config:      disable_fetching_member_count
+- Env Var:     RCLONE_HIDRIVE_DISABLE_FETCHING_MEMBER_COUNT
+- Type:        bool
+- Default:     false
+
+#### --hidrive-disable-unicode-normalization
+
+Do not apply Unicode "Normalization Form C" to remote paths.
+
+In Unicode there are multiple valid representations for the same abstract character.
+They (should) result in the same visual appearance, but are represented by different byte-sequences.
+This is known as canonical equivalence.
+
+In HiDrive paths are always represented as byte-sequences.
+This means that two paths that are canonically equivalent (and therefore look the same) are treated as two distinct paths.
+As this behaviour may be undesired, by default rclone will apply unicode normalization to paths it will access.
+
+Properties:
+
+- Config:      disable_unicode_normalization
+- Env Var:     RCLONE_HIDRIVE_DISABLE_UNICODE_NORMALIZATION
+- Type:        bool
+- Default:     false
+
+#### --hidrive-chunk-size
+
+Chunksize for chunked uploads.
+
+Any files larger than the configured cutoff (or files of unknown size) will be uploaded in chunks of this size.
+
+The upper limit for this is 2147483647 bytes (about 2.000Gi).
+That is the maximum amount of bytes a single upload-operation will support.
+Setting this above the upper limit or to a negative value will cause uploads to fail.
+
+Setting this to larger values may increase the upload speed at the cost of using more memory.
+It can be set to smaller values smaller to save on memory.
+
+Properties:
+
+- Config:      chunk_size
+- Env Var:     RCLONE_HIDRIVE_CHUNK_SIZE
+- Type:        SizeSuffix
+- Default:     48Mi
+
+#### --hidrive-upload-cutoff
+
+Cutoff/Threshold for chunked uploads.
+
+Any files larger than this will be uploaded in chunks of the configured chunksize.
+
+The upper limit for this is 2147483647 bytes (about 2.000Gi).
+That is the maximum amount of bytes a single upload-operation will support.
+Setting this above the upper limit will cause uploads to fail.
+
+Properties:
+
+- Config:      upload_cutoff
+- Env Var:     RCLONE_HIDRIVE_UPLOAD_CUTOFF
+- Type:        SizeSuffix
+- Default:     96Mi
+
+#### --hidrive-upload-concurrency
+
+Concurrency for chunked uploads.
+
+This is the upper limit for how many transfers for the same file are running concurrently.
+Setting this above to a value smaller than 1 will cause uploads to deadlock.
+
+If you are uploading small numbers of large files over high-speed links
+and these uploads do not fully utilize your bandwidth, then increasing
+this may help to speed up the transfers.
+
+Properties:
+
+- Config:      upload_concurrency
+- Env Var:     RCLONE_HIDRIVE_UPLOAD_CONCURRENCY
+- Type:        int
+- Default:     4
+
+#### --hidrive-encoding
+
+The encoding for the backend.
+
+See the [encoding section in the overview](/overview/#encoding) for more info.
+
+Properties:
+
+- Config:      encoding
+- Env Var:     RCLONE_HIDRIVE_ENCODING
+- Type:        MultiEncoder
+- Default:     Slash,Dot
+
+{{< rem autogenerated options stop >}}
+
+## Limitations
+
+### Symbolic links
+
+HiDrive is able to store symbolic links (*symlinks*) by design,
+for example, when unpacked from a zip archive.
+
+There exists no direct mechanism to manage native symlinks in remotes.
+As such this implementation has chosen to ignore any native symlinks present in the remote.
+rclone will not be able to access or show any symlinks stored in the hidrive-remote.
+This means symlinks cannot be individually removed, copied, or moved,
+except when removing, copying, or moving the parent folder.
+
+*This does not affect the `.rclonelink`-files
+that rclone uses to encode and store symbolic links.*
+
+### Sparse files
+
+It is possible to store sparse files in HiDrive.
+
+Note that copying a sparse file will expand the holes
+into null-byte (0x00) regions that will then consume disk space.
+Likewise, when downloading a sparse file,
+the resulting file will have null-byte regions in the place of file holes.
diff --git a/docs/content/overview.md b/docs/content/overview.md
index 74194f28f3f61..3f298c1ba34b6 100644
--- a/docs/content/overview.md
+++ b/docs/content/overview.md
@@ -30,6 +30,7 @@ Here is an overview of the major features of each cloud storage system.
 | Google Drive                 | MD5              | R/W     | No               | Yes             | R/W       | -        |
 | Google Photos                | -                | -       | No               | Yes             | R         | -        |
 | HDFS                         | -                | R/W     | No               | No              | -         | -        |
+| HiDrive                      | HiDrive ¹²       | R/W     | No               | No              | -         | -        |
 | HTTP                         | -                | R       | No               | No              | R         | -        |
 | Hubic                        | MD5              | R/W     | No               | No              | R/W       | -        |
 | Internet Archive             | MD5, SHA1, CRC32 | R/W ¹¹  | No               | No              | -         | -        |
@@ -93,6 +94,11 @@ for more details.
 ¹¹ Internet Archive requires option `wait_archive` to be set to a non-zero value
 for full modtime support.
 
+¹² HiDrive supports [its own custom
+hash](https://static.hidrive.com/dev/0001).
+It combines SHA1 sums for each 4 KiB block hierarchically to a single
+top-level sum.
+
 ### Hash ###
 
 The cloud storage system supports various hash types of the objects.
@@ -475,6 +481,7 @@ upon backend-specific capabilities.
 | Google Drive                 | Yes   | Yes  | Yes  | Yes     | Yes     | Yes   | Yes          | Yes          | Yes   | Yes      |
 | Google Photos                | No    | No   | No   | No      | No      | No    | No           | No           | No    | No       |
 | HDFS                         | Yes   | No   | Yes  | Yes     | No      | No    | Yes          | No           | Yes   | Yes      |
+| HiDrive                      | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes          | No           | No    | Yes      |
 | HTTP                         | No    | No   | No   | No      | No      | No    | No           | No           | No    | Yes      |
 | Hubic                        | Yes † | Yes  | No   | No      | No      | Yes   | Yes          | No           | Yes   | No       |
 | Internet Archive             | No    | Yes  | No   | No      | Yes     | Yes   | No           | Yes          | Yes   | No       |
diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html
index 32c05d3e0c346..f732756e32438 100644
--- a/docs/layouts/chrome/navbar.html
+++ b/docs/layouts/chrome/navbar.html
@@ -72,6 +72,7 @@
            Google Photos
            Hasher (better checksums for others)
            HDFS (Hadoop Distributed Filesystem)
+           HiDrive
            HTTP
            Hubic
            Internet Archive
diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml
index 68f17c2ddd0a5..23cdee6db673c 100644
--- a/fstest/test_all/config.yaml
+++ b/fstest/test_all/config.yaml
@@ -133,6 +133,9 @@ backends:
    remote:   "TestGooglePhotos:"
    tests:
      - backend
+ - backend: "hidrive"
+   remote:   "TestHiDrive:"
+   fastlist: false
  - backend:  "hubic"
    remote:   "TestHubic:"
    fastlist: false

From 42dfadfa1b7c1be6f167b74539757dc9dc7ba62a Mon Sep 17 00:00:00 2001
From: "Lesmiscore (Naoya Ozaki)" 
Date: Sat, 9 Jul 2022 07:47:50 +0900
Subject: [PATCH 168/560] internetarchive: add support for Metadata

---
 backend/internetarchive/internetarchive.go | 196 +++++++++++++++++++--
 docs/content/internetarchive.md            |  27 +++
 docs/content/overview.md                   |   2 +-
 3 files changed, 208 insertions(+), 17 deletions(-)

diff --git a/backend/internetarchive/internetarchive.go b/backend/internetarchive/internetarchive.go
index ce021e35f54a6..9d4dd36dba708 100644
--- a/backend/internetarchive/internetarchive.go
+++ b/backend/internetarchive/internetarchive.go
@@ -38,6 +38,84 @@ func init() {
 		Name:        "internetarchive",
 		Description: "Internet Archive",
 		NewFs:       NewFs,
+
+		MetadataInfo: &fs.MetadataInfo{
+			System: map[string]fs.MetadataHelp{
+				"name": {
+					Help:    "Full file path, without the bucket part",
+					Type:    "filename",
+					Example: "backend/internetarchive/internetarchive.go",
+				},
+				"source": {
+					Help:    "The source of the file",
+					Type:    "string",
+					Example: "original",
+				},
+				"mtime": {
+					Help:    "Time of last modification, managed by Rclone",
+					Type:    "RFC 3339",
+					Example: "2006-01-02T15:04:05.999999999Z",
+				},
+				"size": {
+					Help:    "File size in bytes",
+					Type:    "decimal number",
+					Example: "123456",
+				},
+				"md5": {
+					Help:    "MD5 hash calculated by Internet Archive",
+					Type:    "string",
+					Example: "01234567012345670123456701234567",
+				},
+				"crc32": {
+					Help:    "CRC32 calculated by Internet Archive",
+					Type:    "string",
+					Example: "01234567",
+				},
+				"sha1": {
+					Help:    "SHA1 hash calculated by Internet Archive",
+					Type:    "string",
+					Example: "0123456701234567012345670123456701234567",
+				},
+				"format": {
+					Help:    "Name of format identified by Internet Archive",
+					Type:    "string",
+					Example: "Comma-Separated Values",
+				},
+				"old_version": {
+					Help:    "Whether the file was replaced and moved by keep-old-version flag",
+					Type:    "boolean",
+					Example: "true",
+				},
+				"viruscheck": {
+					Help:    "The last time viruscheck process was run for the file (?)",
+					Type:    "unixtime",
+					Example: "1654191352",
+				},
+
+				"rclone-ia-mtime": {
+					Help:    "Time of last modification, managed by Internet Archive",
+					Type:    "RFC 3339",
+					Example: "2006-01-02T15:04:05.999999999Z",
+				},
+				"rclone-mtime": {
+					Help:    "Time of last modification, managed by Rclone",
+					Type:    "RFC 3339",
+					Example: "2006-01-02T15:04:05.999999999Z",
+				},
+				"rclone-update-track": {
+					Help:    "Random value used by Rclone for tracking changes inside Internet Archive",
+					Type:    "string",
+					Example: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+				},
+			},
+			Help: `Metadata fields provided by Internet Archive.
+If there are multiple values for a key, only the first one is returned.
+This is a limitation of Rclone, that supports one value per one key.
+
+Owner is able to add custom keys. Metadata feature grabs all the keys including them.
+`,
+		},
+
 		Options: []fs.Option{{
 			Name: "access_key_id",
 			Help: "IAS3 Access Key.\n\nLeave blank for anonymous access.\nYou can find one here: https://archive.org/account/s3.php",
@@ -90,6 +168,14 @@ Only enable if you need to be guaranteed to be reflected after write operations.
 // maximum size of an item. this is constant across all items
 const iaItemMaxSize int64 = 1099511627776
 
+// metadata keys that are not writeable
+var roMetadataKey = map[string]interface{}{
+	// do not add mtime here, it's a documented exception
+	"name": nil, "source": nil, "size": nil, "md5": nil,
+	"crc32": nil, "sha1": nil, "format": nil, "old_version": nil,
+	"viruscheck": nil,
+}
+
 // Options defines the configuration for this backend
 type Options struct {
 	AccessKeyID     string               `config:"access_key_id"`
@@ -122,6 +208,7 @@ type Object struct {
 	md5     string    // md5 hash of the file presented by the server
 	sha1    string    // sha1 hash of the file presented by the server
 	crc32   string    // crc32 of the file presented by the server
+	rawData json.RawMessage
 }
 
 // IAFile reprensents a subset of object in MetadataResponse.Files
@@ -135,6 +222,8 @@ type IAFile struct {
 	Md5         string          `json:"md5"`
 	Crc32       string          `json:"crc32"`
 	Sha1        string          `json:"sha1"`
+
+	rawData json.RawMessage
 }
 
 // MetadataResponse reprensents subset of the JSON object returned by (frontend)/metadata/
@@ -143,6 +232,12 @@ type MetadataResponse struct {
 	ItemSize int64    `json:"item_size"`
 }
 
+// MetadataResponseRaw is the form of MetadataResponse to deal with metadata
+type MetadataResponseRaw struct {
+	Files    []json.RawMessage `json:"files"`
+	ItemSize int64             `json:"item_size"`
+}
+
 // ModMetadataResponse represents response for amending metadata
 type ModMetadataResponse struct {
 	// https://archive.org/services/docs/api/md-write.html#example
@@ -226,7 +321,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
-		BucketBased: true,
+		BucketBased:   true,
+		ReadMetadata:  true,
+		WriteMetadata: true,
+		UserMetadata:  true,
 	}).Fill(ctx, f)
 
 	f.srv = rest.NewClient(fshttp.NewClient(ctx))
@@ -307,18 +405,17 @@ func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
 	}
 
 	// https://archive.org/services/docs/api/md-write.html
-	var patch = []interface{}{
+	// the following code might be useful for modifying metadata of an uploaded file
+	patch := []map[string]string{
 		// we should drop it first to clear all rclone-provided mtimes
-		struct {
-			Op   string `json:"op"`
-			Path string `json:"path"`
-		}{"remove", "/rclone-mtime"},
-		struct {
-			Op    string `json:"op"`
-			Path  string `json:"path"`
-			Value string `json:"value"`
-		}{"add", "/rclone-mtime", t.Format(time.RFC3339Nano)},
-	}
+		{
+			"op":   "remove",
+			"path": "/rclone-mtime",
+		}, {
+			"op":    "add",
+			"path":  "/rclone-mtime",
+			"value": t.Format(time.RFC3339Nano),
+		}}
 	res, err := json.Marshal(patch)
 	if err != nil {
 		return err
@@ -685,6 +782,23 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		headers["Content-Length"] = fmt.Sprintf("%d", size)
 		headers["x-archive-size-hint"] = fmt.Sprintf("%d", size)
 	}
+	var mdata fs.Metadata
+	mdata, err = fs.GetMetadataOptions(ctx, src, options)
+	if err == nil && mdata != nil {
+		for mk, mv := range mdata {
+			mk = strings.ToLower(mk)
+			if strings.HasPrefix(mk, "rclone-") {
+				fs.LogPrintf(fs.LogLevelWarning, o, "reserved metadata key %s is about to set", mk)
+			} else if _, ok := roMetadataKey[mk]; ok {
+				fs.LogPrintf(fs.LogLevelWarning, o, "setting or modifying read-only key %s is requested, skipping", mk)
+				continue
+			} else if mk == "mtime" {
+				// redirect to make it work
+				mk = "rclone-mtime"
+			}
+			headers[fmt.Sprintf("x-amz-filemeta-%s", mk)] = mv
+		}
+	}
 
 	// read the md5sum if available
 	var md5sumHex string
@@ -762,6 +876,34 @@ func (o *Object) String() string {
 	return o.remote
 }
 
+// Metadata returns all file metadata provided by Internet Archive
+func (o *Object) Metadata(ctx context.Context) (m fs.Metadata, err error) {
+	if o.rawData == nil {
+		return nil, nil
+	}
+	raw := make(map[string]json.RawMessage)
+	err = json.Unmarshal(o.rawData, &raw)
+	if err != nil {
+		// fatal: json parsing failed
+		return
+	}
+	for k, v := range raw {
+		items, err := listOrString(v)
+		if len(items) == 0 || err != nil {
+			// skip: an entry failed to parse
+			continue
+		}
+		m.Set(k, items[0])
+	}
+	// move the old mtime to an another key
+	if v, ok := m["mtime"]; ok {
+		m["rclone-ia-mtime"] = v
+	}
+	// overwrite with a correct mtime
+	m["mtime"] = o.modTime.Format(time.RFC3339Nano)
+	return
+}
+
 func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
 	if resp != nil {
 		for _, e := range retryErrorCodes {
@@ -788,7 +930,7 @@ func (o *Object) split() (bucket, bucketPath string) {
 	return o.fs.split(o.remote)
 }
 
-func (f *Fs) requestMetadata(ctx context.Context, bucket string) (result MetadataResponse, err error) {
+func (f *Fs) requestMetadata(ctx context.Context, bucket string) (result *MetadataResponse, err error) {
 	var resp *http.Response
 	// make a GET request to (frontend)/metadata/:item/
 	opts := rest.Opts{
@@ -796,12 +938,15 @@ func (f *Fs) requestMetadata(ctx context.Context, bucket string) (result Metadat
 		Path:   path.Join("/metadata/", bucket),
 	}
 
+	var temp MetadataResponseRaw
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.front.CallJSON(ctx, &opts, nil, &result)
+		resp, err = f.front.CallJSON(ctx, &opts, nil, &temp)
 		return f.shouldRetry(resp, err)
 	})
-
-	return result, err
+	if err != nil {
+		return
+	}
+	return temp.unraw()
 }
 
 // list up all files/directories without any filters
@@ -998,6 +1143,7 @@ func makeValidObject(f *Fs, remote string, file IAFile, mtime time.Time, size in
 		md5:     file.Md5,
 		crc32:   file.Crc32,
 		sha1:    file.Sha1,
+		rawData: file.rawData,
 	}
 }
 
@@ -1045,6 +1191,23 @@ func (file IAFile) parseMtime() (mtime time.Time) {
 	return mtime
 }
 
+func (mrr *MetadataResponseRaw) unraw() (_ *MetadataResponse, err error) {
+	var files []IAFile
+	for _, raw := range mrr.Files {
+		var parsed IAFile
+		err = json.Unmarshal(raw, &parsed)
+		if err != nil {
+			return nil, err
+		}
+		parsed.rawData = raw
+		files = append(files, parsed)
+	}
+	return &MetadataResponse{
+		Files:    files,
+		ItemSize: mrr.ItemSize,
+	}, nil
+}
+
 func compareSize(a, b int64) bool {
 	if a < 0 || b < 0 {
 		// we won't compare if any of them is not known
@@ -1106,4 +1269,5 @@ var (
 	_ fs.PublicLinker = &Fs{}
 	_ fs.Abouter      = &Fs{}
 	_ fs.Object       = &Object{}
+	_ fs.Metadataer   = &Object{}
 )
diff --git a/docs/content/internetarchive.md b/docs/content/internetarchive.md
index d0e937fdfaa2e..622db4d608bf3 100644
--- a/docs/content/internetarchive.md
+++ b/docs/content/internetarchive.md
@@ -38,6 +38,33 @@ You can optionally wait for the server's processing to finish, by setting non-ze
 By making it wait, rclone can do normal file comparison.
 Make sure to set a large enough value (e.g. `30m0s` for smaller files) as it can take a long time depending on server's queue.
 
+## About metadata
+This backend supports setting, updating and reading metadata of each file.
+The metadata will appear as file metadata on Internet Archive.
+However, some fields are reserved by both Internet Archive and rclone.
+
+The following are reserved by Internet Archive:
+- `name`
+- `source`
+- `size`
+- `md5`
+- `crc32`
+- `sha1`
+- `format`
+- `old_version`
+- `viruscheck`
+
+Trying to set values to these keys is ignored with a warning.
+Only setting `mtime` is an exception. Doing so make it the identical behavior as setting ModTime.
+
+rclone reserves all the keys starting with `rclone-`. Setting value for these keys will give you warnings, but values are set according to request.
+
+If there are multiple values for a key, only the first one is returned.
+This is a limitation of rclone, that supports one value per one key.
+It can be triggered when you did a server-side copy.
+
+Reading metadata will also provide custom (non-standard nor reserved) ones.
+
 ## Configuration
 
 Here is an example of making an internetarchive configuration.
diff --git a/docs/content/overview.md b/docs/content/overview.md
index 3f298c1ba34b6..2a9301acf45dc 100644
--- a/docs/content/overview.md
+++ b/docs/content/overview.md
@@ -33,7 +33,7 @@ Here is an overview of the major features of each cloud storage system.
 | HiDrive                      | HiDrive ¹²       | R/W     | No               | No              | -         | -        |
 | HTTP                         | -                | R       | No               | No              | R         | -        |
 | Hubic                        | MD5              | R/W     | No               | No              | R/W       | -        |
-| Internet Archive             | MD5, SHA1, CRC32 | R/W ¹¹  | No               | No              | -         | -        |
+| Internet Archive             | MD5, SHA1, CRC32 | R/W ¹¹  | No               | No              | -         | RWU      |
 | Jottacloud                   | MD5              | R/W     | Yes              | No              | R         | -        |
 | Koofr                        | MD5              | -       | Yes              | No              | -         | -        |
 | Mail.ru Cloud                | Mailru ⁶         | R/W     | Yes              | No              | -         | -        |

From 6e9c1eebd96e0c25b4814d58c60da06b4a3cc74b Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Sat, 9 Jul 2022 12:35:59 +0100
Subject: [PATCH 169/560] Add Claudio Maradonna to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index d4129e3756f09..ed7c36d03998c 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -624,3 +624,4 @@ put them back in again.` >}}
   * zzr93 <34027824+zzr93@users.noreply.github.com>
   * Paul Norman 
   * Lorenzo Maiorfi 
+  * Claudio Maradonna 

From 876f12f2c48abf446fbd73a086785c01d2495cd4 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Sat, 9 Jul 2022 12:35:59 +0100
Subject: [PATCH 170/560] Add Ovidiu Victor Tatar to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index ed7c36d03998c..0e8e998691095 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -625,3 +625,4 @@ put them back in again.` >}}
   * Paul Norman 
   * Lorenzo Maiorfi 
   * Claudio Maradonna 
+  * Ovidiu Victor Tatar 

From 1c4ee2feee5c954e5d791f954124cbe26e8fe3f7 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Sat, 9 Jul 2022 17:31:12 +0100
Subject: [PATCH 171/560] gcs: add --gcs-decompress flag to download
 gzip-encoded files

By default these will be downloaded compressed.

This changes the default of the previous commit

2781f8e2f14f146d gcs: Fix download of "Content-Encoding: gzip" compressed objects

But will fit in better with the metadata framework when copying
gzip-encoded objects from backend to backend.
---
 .../googlecloudstorage/googlecloudstorage.go  | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go
index b11771d1b8351..b4d355021b053 100644
--- a/backend/googlecloudstorage/googlecloudstorage.go
+++ b/backend/googlecloudstorage/googlecloudstorage.go
@@ -307,17 +307,15 @@ rclone does if you know the bucket exists already.
 			Default:  false,
 			Advanced: true,
 		}, {
-			Name: "download_compressed",
-			Help: `If set this will download compressed objects as-is.
+			Name: "decompress",
+			Help: `If set this will decompress gzip encoded objects.
 
 It is possible to upload objects to GCS with "Content-Encoding: gzip"
-set. Normally rclone will transparently decompress these files on
-download. This means that rclone can't check the hash or the size of
-the file as both of these refer to the compressed object.
+set. Normally rclone will download these files files as compressed objects.
 
-If this flag is set then rclone will download files with
+If this flag is set then rclone will decompress these files with
 "Content-Encoding: gzip" as they are received. This means that rclone
-can check the size and hash but the file contents will be compressed.
+can't check the size and hash but the file contents will be decompressed.
 `,
 			Advanced: true,
 			Default:  false,
@@ -344,7 +342,7 @@ type Options struct {
 	Location                  string               `config:"location"`
 	StorageClass              string               `config:"storage_class"`
 	NoCheckBucket             bool                 `config:"no_check_bucket"`
-	DownloadCompressed        bool                 `config:"download_compressed"`
+	Decompress                bool                 `config:"decompress"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -1036,12 +1034,9 @@ func (o *Object) setMetaData(info *storage.Object) {
 	}
 
 	// If gunzipping then size and md5sum are unknown
-	if o.gzipped && !o.fs.opt.DownloadCompressed {
+	if o.gzipped && o.fs.opt.Decompress {
 		o.bytes = -1
 		o.md5sum = ""
-		o.fs.warnCompressed.Do(func() {
-			fs.Logf(o.fs, "Decompressing 'Content-Encoding: gzip' compressed file. Use --gcs-download-compressed to override")
-		})
 	}
 }
 
@@ -1143,7 +1138,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, err
 	}
 	fs.FixRangeOption(options, o.bytes)
-	if o.gzipped && o.fs.opt.DownloadCompressed {
+	if o.gzipped && !o.fs.opt.Decompress {
 		// Allow files which are stored on the cloud storage system
 		// compressed to be downloaded without being decompressed.  Note
 		// that setting this here overrides the automatic decompression
@@ -1151,6 +1146,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		//
 		// See: https://cloud.google.com/storage/docs/transcoding
 		req.Header.Set("Accept-Encoding", "gzip")
+		o.fs.warnCompressed.Do(func() {
+			fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --gcs-decompress to override")
+		})
 	}
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
 	var res *http.Response

From 00a684d877548b47990a96cef8eeb1462a590c2a Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Sat, 9 Jul 2022 18:08:20 +0100
Subject: [PATCH 172/560] Version v1.59.0

---
 MANUAL.html                                   | 5392 ++++++++---
 MANUAL.md                                     | 5225 +++++++++--
 MANUAL.txt                                    | 5276 +++++++++--
 docs/content/alias.md                         |    2 +-
 docs/content/amazonclouddrive.md              |    4 +-
 docs/content/azureblob.md                     |    4 +-
 docs/content/b2.md                            |   18 +-
 docs/content/box.md                           |    4 +-
 docs/content/cache.md                         |    6 +-
 docs/content/changelog.md                     |  159 +
 docs/content/chunker.md                       |    4 +-
 docs/content/combine.md                       |    8 +-
 docs/content/commands/rclone.md               |    2 +-
 docs/content/commands/rclone_check.md         |    4 +
 docs/content/commands/rclone_completion.md    |   13 +-
 .../commands/rclone_completion_bash.md        |   24 +-
 .../commands/rclone_completion_fish.md        |   13 +-
 .../commands/rclone_completion_powershell.md  |   10 +-
 .../content/commands/rclone_completion_zsh.md |   21 +-
 docs/content/commands/rclone_copy.md          |    9 +-
 docs/content/commands/rclone_copyto.md        |    4 +-
 docs/content/commands/rclone_copyurl.md       |   20 +-
 docs/content/commands/rclone_cryptcheck.md    |    6 +-
 docs/content/commands/rclone_cryptdecode.md   |    6 +-
 docs/content/commands/rclone_dedupe.md        |    2 +-
 docs/content/commands/rclone_delete.md        |   10 +-
 .../commands/rclone_genautocomplete.md        |    2 +-
 docs/content/commands/rclone_hashsum.md       |    4 +
 docs/content/commands/rclone_listremotes.md   |    2 +-
 docs/content/commands/rclone_lsd.md           |    4 +-
 docs/content/commands/rclone_lsf.md           |   13 +-
 docs/content/commands/rclone_lsjson.md        |   29 +-
 docs/content/commands/rclone_md5sum.md        |    4 +
 docs/content/commands/rclone_mount.md         |   67 +-
 docs/content/commands/rclone_move.md          |    6 +-
 docs/content/commands/rclone_moveto.md        |    2 +-
 docs/content/commands/rclone_ncdu.md          |   31 +-
 docs/content/commands/rclone_obscure.md       |    2 +-
 docs/content/commands/rclone_purge.md         |    7 +-
 docs/content/commands/rclone_rc.md            |   20 +-
 docs/content/commands/rclone_rcat.md          |    6 +-
 docs/content/commands/rclone_rmdir.md         |    6 +-
 docs/content/commands/rclone_rmdirs.md        |   13 +-
 docs/content/commands/rclone_serve.md         |    6 +-
 docs/content/commands/rclone_serve_dlna.md    |   77 +-
 docs/content/commands/rclone_serve_docker.md  |   61 +-
 docs/content/commands/rclone_serve_ftp.md     |   67 +-
 docs/content/commands/rclone_serve_http.md    |  115 +-
 docs/content/commands/rclone_serve_restic.md  |   58 +-
 docs/content/commands/rclone_serve_sftp.md    |   95 +-
 docs/content/commands/rclone_serve_webdav.md  |  124 +-
 docs/content/commands/rclone_sha1sum.md       |    4 +
 docs/content/commands/rclone_size.md          |   22 +
 docs/content/commands/rclone_sync.md          |    6 +-
 docs/content/commands/rclone_test.md          |    1 +
 docs/content/commands/rclone_test_makefile.md |   33 +
 .../content/commands/rclone_test_makefiles.md |    5 +
 docs/content/commands/rclone_tree.md          |    8 +-
 docs/content/compress.md                      |   10 +-
 docs/content/crypt.md                         |   12 +-
 docs/content/drive.md                         |   91 +-
 docs/content/dropbox.md                       |    4 +-
 docs/content/fichier.md                       |    4 +-
 docs/content/filefabric.md                    |    4 +-
 docs/content/flags.md                         |   51 +-
 docs/content/ftp.md                           |    4 +-
 docs/content/googlecloudstorage.md            |   38 +-
 docs/content/googlephotos.md                  |    4 +-
 docs/content/hasher.md                        |   12 +-
 docs/content/hdfs.md                          |    4 +-
 docs/content/hidrive.md                       |   23 +-
 docs/content/http.md                          |    6 +-
 docs/content/hubic.md                         |    4 +-
 docs/content/internetarchive.md               |   32 +-
 docs/content/jottacloud.md                    |    2 +-
 docs/content/koofr.md                         |    4 +-
 docs/content/local.md                         |    9 +-
 docs/content/mailru.md                        |    4 +-
 docs/content/mega.md                          |    4 +-
 docs/content/netstorage.md                    |   12 +-
 docs/content/onedrive.md                      |   26 +-
 docs/content/opendrive.md                     |    4 +-
 docs/content/pcloud.md                        |    4 +-
 docs/content/premiumizeme.md                  |    4 +-
 docs/content/putio.md                         |    2 +-
 docs/content/qingstor.md                      |    4 +-
 docs/content/rc.md                            |  152 +-
 docs/content/s3.md                            |  310 +-
 docs/content/seafile.md                       |    4 +-
 docs/content/sftp.md                          |  103 +-
 docs/content/sharefile.md                     |    4 +-
 docs/content/sia.md                           |    4 +-
 docs/content/storj.md                         |    2 +-
 docs/content/sugarsync.md                     |    4 +-
 docs/content/swift.md                         |    4 +-
 docs/content/union.md                         |   26 +-
 docs/content/uptobox.md                       |    4 +-
 docs/content/webdav.md                        |    6 +-
 docs/content/yandex.md                        |    4 +-
 docs/content/zoho.md                          |    8 +-
 rclone.1                                      | 8019 ++++++++++++++---
 101 files changed, 21980 insertions(+), 4202 deletions(-)
 create mode 100644 docs/content/commands/rclone_test_makefile.md

diff --git a/MANUAL.html b/MANUAL.html
index eb2a7d1970566..4c0d99e6195f2 100644
--- a/MANUAL.html
+++ b/MANUAL.html
@@ -13,75 +13,13 @@
     div.column{display: inline-block; vertical-align: top; width: 50%;}
     div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
     ul.task-list{list-style: none;}
-    pre > code.sourceCode { white-space: pre; position: relative; }
-    pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
-    pre > code.sourceCode > span:empty { height: 1.2em; }
-    code.sourceCode > span { color: inherit; text-decoration: inherit; }
-    div.sourceCode { margin: 1em 0; }
-    pre.sourceCode { margin: 0; }
-    @media screen {
-    div.sourceCode { overflow: auto; }
-    }
-    @media print {
-    pre > code.sourceCode { white-space: pre-wrap; }
-    pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
-    }
-    pre.numberSource code
-      { counter-reset: source-line 0; }
-    pre.numberSource code > span
-      { position: relative; left: -4em; counter-increment: source-line; }
-    pre.numberSource code > span > a:first-child::before
-      { content: counter(source-line);
-        position: relative; left: -1em; text-align: right; vertical-align: baseline;
-        border: none; display: inline-block;
-        -webkit-touch-callout: none; -webkit-user-select: none;
-        -khtml-user-select: none; -moz-user-select: none;
-        -ms-user-select: none; user-select: none;
-        padding: 0 4px; width: 4em;
-        color: #aaaaaa;
-      }
-    pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa;  padding-left: 4px; }
-    div.sourceCode
-      {   }
-    @media screen {
-    pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
-    }
-    code span.al { color: #ff0000; font-weight: bold; } /* Alert */
-    code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */
-    code span.at { color: #7d9029; } /* Attribute */
-    code span.bn { color: #40a070; } /* BaseN */
-    code span.bu { } /* BuiltIn */
-    code span.cf { color: #007020; font-weight: bold; } /* ControlFlow */
-    code span.ch { color: #4070a0; } /* Char */
-    code span.cn { color: #880000; } /* Constant */
-    code span.co { color: #60a0b0; font-style: italic; } /* Comment */
-    code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */
-    code span.do { color: #ba2121; font-style: italic; } /* Documentation */
-    code span.dt { color: #902000; } /* DataType */
-    code span.dv { color: #40a070; } /* DecVal */
-    code span.er { color: #ff0000; font-weight: bold; } /* Error */
-    code span.ex { } /* Extension */
-    code span.fl { color: #40a070; } /* Float */
-    code span.fu { color: #06287e; } /* Function */
-    code span.im { } /* Import */
-    code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */
-    code span.kw { color: #007020; font-weight: bold; } /* Keyword */
-    code span.op { color: #666666; } /* Operator */
-    code span.ot { color: #007020; } /* Other */
-    code span.pp { color: #bc7a00; } /* Preprocessor */
-    code span.sc { color: #4070a0; } /* SpecialChar */
-    code span.ss { color: #bb6688; } /* SpecialString */
-    code span.st { color: #4070a0; } /* String */
-    code span.va { color: #19177c; } /* Variable */
-    code span.vs { color: #4070a0; } /* VerbatimString */
-    code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */
   
 
 
 

 
rclone(1) User Manual

 
Nick Craig-Wood

-
Mar 18, 2022

+
Jul 09, 2022

 

 
Rclone syncs your files to cloud storage

 
rclone logo

@@ -130,7 +68,7 @@ 
Features

 
  • Move files to cloud storage deleting the local after verification
    • 
 
  • Check hashes and for missing/extra files
    • 
 
  • Mount your cloud storage as a network disk
    • 
-
  • Serve local or remote files over HTTP/WebDav/FTP/SFTP/dlna
    • 
+
  • Serve local or remote files over HTTP/WebDav/FTP/SFTP/DLNA
    • 
 
  • Experimental Web based GUI
    • 
 
 
    Supported providers
    
@@ -144,8 +82,11 @@ 
    Supported providers
    
 
  • Backblaze B2
    • 
 
  • Box
    • 
 
  • Ceph
    • 
+
  • China Mobile Ecloud Elastic Object Storage (EOS)
    • 
+
  • Arvan Cloud Object Storage (AOS)
    • 
 
  • Citrix ShareFile
    • 
 
  • C14
    • 
+
  • Cloudflare R2
    • 
 
  • DigitalOcean Spaces
    • 
 
  • Digi Storage
    • 
 
  • Dreamhost
    • 
@@ -156,10 +97,14 @@ 
    Supported providers
    
 
  • Google Drive
    • 
 
  • Google Photos
    • 
 
  • HDFS
    • 
+
  • Hetzner Storage Box
    • 
+
  • HiDrive
    • 
 
  • HTTP
    • 
 
  • Hubic
    • 
+
  • Internet Archive
    • 
 
  • Jottacloud
    • 
 
  • IBM COS S3
    • 
+
  • IDrive e2
    • 
 
  • Koofr
    • 
 
  • Mail.ru Cloud
    • 
 
  • Memset Memstore
    • 
@@ -197,7 +142,19 @@ 
    Supported providers
    
 
  • Zoho WorkDrive
    • 
 
  • The local filesystem
    • 
 
-
    Links
    
+
    Virtual providers
    
+
    These backends adapt or modify other storage providers:
    
+
      
+
    • Alias: Rename existing remotes
      • 
+
    • Cache: Cache remotes (DEPRECATED)
      • 
+
    • Chunker: Split large files
      • 
+
    • Combine: Combine multiple remotes into a directory tree
      • 
+
    • Compress: Compress files
      • 
+
    • Crypt: Encrypt files
      • 
+
    • Hasher: Hash files
      • 
+
    • Union: Join multiple remotes to work together
      • 
+
    
+
 
      
 
    • Home page
      • 
 
    • GitHub project page for source and bug tracker
      • 
@@ -218,9 +175,9 @@ 
      Quickstart
      
 
      Already installed rclone can be easily updated to the latest version using the rclone selfupdate command.
      
 
      Script installation
      
 
      To install rclone on Linux/macOS/BSD systems, run:
      
-
      curl https://rclone.org/install.sh | sudo bash
      
+
      sudo -v ; curl https://rclone.org/install.sh | sudo bash
      
 
      For beta installation, run:
      
-
      curl https://rclone.org/install.sh | sudo bash -s beta
      
+
      sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta
      
 
      Note that this script checks the version of rclone installed first and won't re-download if not needed.
      
 
      Linux installation from precompiled binary
      
 
      Fetch and unpack
      
@@ -256,7 +213,7 @@ 
      macOS installatio
 
      rclone config
      
 
      macOS installation from precompiled binary, using a web browser
      
 
      When downloading a binary with a web browser, the browser will set the macOS gatekeeper quarantine attribute. Starting from Catalina, when attempting to run rclone, a pop-up will appear saying:
      
-
      “rclone” cannot be opened because the developer cannot be verified.
+
      "rclone" cannot be opened because the developer cannot be verified.
 macOS cannot verify that this app is free from malware.
      
 
      The simplest fix is to run
      
 
      xattr -d com.apple.quarantine rclone
      
@@ -308,18 +265,26 @@ 
      Install with docker
      
 ls ~/data/mount
 kill %1
      
 
      Install from source
      
-
      Make sure you have at least Go go1.15 installed. Download go if necessary. The latest release is recommended. Then
      
-
      git clone https://github.com/rclone/rclone.git
-cd rclone
-go build
-# If on macOS and mount is wanted, instead run: make GOTAGS=cmount
-./rclone version
      
-
      This will leave you a checked out version of rclone you can modify and send pull requests with. If you use make instead of go build then the rclone build will have the correct version information in it.
      
-
      You can also build the latest stable rclone with:
      
+
      Make sure you have git and Go installed. Go version 1.16 or newer is required, latest release is recommended. You can get it from your package manager, or download it from golang.org/dl. Then you can run the following:
      
+
      git clone https://github.com/rclone/rclone.git
+cd rclone
+go build
      
+
      This will check out the rclone source in subfolder rclone, which you can later modify and send pull requests with. Then it will build the rclone executable in the same folder. As an initial check you can now run ./rclone version (.\rclone version on Windows).
      
+
      Note that on macOS and Windows the mount command will not be available unless you specify additional build tag cmount.
      
+
      go build -tags cmount
      
+
      This assumes you have a GCC compatible C compiler (GCC or Clang) in your PATH, as it uses cgo. But on Windows, the cgofuse library that the cmount implementation is based on, also supports building without cgo, i.e. by setting environment variable CGO_ENABLED to value 0 (static linking). This is how the official Windows release of rclone is being built, starting with version 1.59. It is still possible to build with cgo on Windows as well, by using the MinGW port of GCC, e.g. by installing it in a MSYS2 distribution (make sure you install it in the classic mingw64 subsystem, the ucrt64 version is not compatible).
      
+
      Additionally, on Windows, you must install the third party utility WinFsp, with the "Developer" feature selected. If building with cgo, you must also set environment variable CPATH pointing to the fuse include directory within the WinFsp installation (normally C:\Program Files (x86)\WinFsp\inc\fuse).
      
+
      You may also add arguments -ldflags -s (with or without -tags cmount), to omit symbol table and debug information, making the executable file smaller, and -trimpath to remove references to local file system paths. This is how the official rclone releases are built.
      
+
      go build -trimpath -ldflags -s -tags cmount
      
+
      Instead of executing the go build command directly, you can run it via the Makefile, which also sets version information and copies the resulting rclone executable into your GOPATH bin folder ($(go env GOPATH)/bin, which corresponds to ~/go/bin/rclone by default).
      
+
      make
      
+
      To include mount command on macOS and Windows with Makefile build:
      
+
      make GOTAGS=cmount
      
+
      As an alternative you can download the source, build and install rclone in one operation, as a regular Go package. The source will be stored it in the Go module cache, and the resulting executable will be in your GOPATH bin folder ($(go env GOPATH)/bin, which corresponds to ~/go/bin/rclone by default).
      
+
      With Go version 1.17 or newer:
      
+
      go install github.com/rclone/rclone@latest
      
+
      With Go versions older than 1.17 (do not use the -u flag, it causes Go to try to update the dependencies that rclone uses and sometimes these don't work with the current version):
      
 
      go get github.com/rclone/rclone
      
-
      or the latest version (equivalent to the beta) with
      
-
      go get github.com/rclone/rclone@master
      
-
      These will build the binary in $(go env GOPATH)/bin (~/go/bin/rclone by default) after downloading the source to the go module cache. Note - do not use the -u flag here. This causes go to try to update the dependencies that rclone uses and sometimes these don't work with the current version of rclone.
      
 
      Installation with Ansible
      
 
      This can be done with Stefan Weichinger's ansible role.
      
 
      Instructions
      
@@ -354,7 +319,7 @@ 
      Start from Task Scheduler
      
 
      Run as service
      
 
      For running rclone at system startup, you can create a Windows service that executes your rclone command, as an alternative to scheduled task configured to run at startup.
      
 
      Mount command built-in service integration
      
-
      For mount commands, Rclone has a built-in Windows service integration via the third-party WinFsp library it uses. Registering as a regular Windows service easy, as you just have to execute the built-in PowerShell command New-Service (requires administrative privileges).
      
+
      For mount commands, rclone has a built-in Windows service integration via the third-party WinFsp library it uses. Registering as a regular Windows service easy, as you just have to execute the built-in PowerShell command New-Service (requires administrative privileges).
      
 
      Example of a PowerShell command that creates a Windows service for mounting some remote:/files as drive letter X:, for all users (service will be running as the local system account):
      
 
      New-Service -Name Rclone -BinaryPathName 'c:\rclone\rclone.exe mount remote:/files X: --config c:\rclone\config\rclone.conf --log-file c:\rclone\logs\mount.txt'
      
 
      The WinFsp service infrastructure supports incorporating services for file system implementations, such as rclone, into its own launcher service, as kind of "child services". This has the additional advantage that it also implements a network provider that integrates into Windows standard methods for managing network drives. This is currently not officially supported by Rclone, but with WinFsp version 2019.3 B2 / v1.5B2 or later it should be possible through path rewriting as described here.
      
@@ -384,6 +349,7 @@ 
      Configure
      
 
    • Chunker - transparently splits large files for other remotes
      • 
 
    • Citrix ShareFile
      • 
 
    • Compress
      • 
+
    • Combine
      • 
 
    • Crypt - to encrypt other remotes
      • 
 
    • DigitalOcean Spaces
      • 
 
    • Digi Storage
      • 
@@ -395,8 +361,10 @@ 
      Configure
      
 
    • Google Photos
      • 
 
    • Hasher - to handle checksums for other remotes
      • 
 
    • HDFS
      • 
+
    • HiDrive
      • 
 
    • HTTP
      • 
 
    • Hubic
      • 
+
    • Internet Archive
      • 
 
    • Jottacloud
      • 
 
    • Koofr
      • 
 
    • Mail.ru Cloud
      • 
@@ -462,8 +430,9 @@ 
      SEE ALSO
      
 
      rclone copy
      
 
      Copy files from source to dest, skipping identical files.
      
 
      Synopsis
      
-
      Copy the source to the destination. Does not transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Doesn't delete files from the destination.
      
-
      Note that it is always the contents of the directory that is synced, not the directory so when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents.
      
+
      Copy the source to the destination. Does not transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Doesn't delete files from the destination. If you want to also delete files from destination, to make it match source, use the sync command instead.
      
+
      Note that it is always the contents of the directory that is synced, not the directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents.
      
+
      To copy single files, use the copyto command instead.
      
 
      If dest:path doesn't exist, it is created and the source:path contents go there.
      
 
      For example
      
 
      rclone copy source:sourcepath dest:destpath
      
@@ -494,11 +463,11 @@ 
      SEE ALSO
      
 
      rclone sync
      
 
      Make source and dest identical, modifying destination only.
      
 
      Synopsis
      
-
      Sync the source to the destination, changing the destination only. Doesn't transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Destination is updated to match source, including deleting files if necessary (except duplicate objects, see below).
      
+
      Sync the source to the destination, changing the destination only. Doesn't transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Destination is updated to match source, including deleting files if necessary (except duplicate objects, see below). If you don't want to delete files from destination, use the copy command instead.
      
 
      Important: Since this can cause data loss, test first with the --dry-run or the --interactive/-i flag.
      
 
      rclone sync -i SOURCE remote:DESTINATION
      
 
      Note that files in the destination won't be deleted if there were any errors at any point. Duplicate objects (files with the same name, on those providers that support it) are also not yet handled.
      
-
      It is always the contents of the directory that is synced, not the directory so when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. See extended explanation in the copy command above if unsure.
      
+
      It is always the contents of the directory that is synced, not the directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. See extended explanation in the copy command if unsure.
      
 
      If dest:path doesn't exist, it is created and the source:path contents go there.
      
 
      Note: Use the -P/--progress flag to view real-time transfer statistics
      
 
      Note: Use the rclone dedupe command to deal with "Duplicate object/directory found in source/destination - ignoring" errors. See this forum post for more info.
      
@@ -515,9 +484,10 @@ 
      rclone move
      
 
      Move files from source to dest.
      
 
      Synopsis
      
 
      Moves the contents of the source directory to the destination directory. Rclone will error if the source and destination overlap and the remote does not support a server-side directory move operation.
      
+
      To move single files, use the moveto command instead.
      
 
      If no filters are in use and if possible this will server-side move source:path into dest:path. After this source:path will no longer exist.
      
 
      Otherwise for each file in source:path selected by the filters (if any) this will move it into dest:path. If possible a server-side move will be used, otherwise it will copy it (server-side if possible) into dest:path then delete the original (if no errors on copy) in source:path.
      
-
      If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag.
      
+
      If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag.
      
 
      See the --no-traverse option for controlling whether rclone lists the destination directory or not. Supplying this option when moving a small number of files into a large destination can speed transfers up greatly.
      
 
      Important: Since this can cause data loss, test first with the --dry-run or the --interactive/-i flag.
      
 
      Note: Use the -P/--progress flag to view real-time transfer statistics.
      
@@ -534,9 +504,9 @@ 
      SEE ALSO
      
 
      rclone delete
      
 
      Remove the files in path.
      
 
      Synopsis
      
-
      Remove the files in path. Unlike purge it obeys include/exclude filters so can be used to selectively delete files.
      
-
      rclone delete only deletes files but leaves the directory structure alone. If you want to delete a directory and all of its contents use the purge command.
      
-
      If you supply the --rmdirs flag, it will remove all empty directories along with it. You can also use the separate command rmdir or rmdirs to delete empty directories only.
      
+
      Remove the files in path. Unlike purge it obeys include/exclude filters so can be used to selectively delete files.
      
+
      rclone delete only deletes files but leaves the directory structure alone. If you want to delete a directory and all of its contents use the purge command.
      
+
      If you supply the --rmdirs flag, it will remove all empty directories along with it. You can also use the separate command rmdir or rmdirs to delete empty directories only.
      
 
      For example, to delete all files bigger than 100 MiB, you may first want to check what would be deleted (use either):
      
 
      rclone --min-size 100M lsl remote:path
 rclone --dry-run --min-size 100M delete remote:path
      
@@ -556,7 +526,7 @@ 
      SEE ALSO
      
 
      rclone purge
      
 
      Remove the path and all of its contents.
      
 
      Synopsis
      
-
      Remove the path and all of its contents. Note that this does not obey include/exclude filters - everything will be removed. Use the delete command if you want to selectively delete files. To delete empty directories only, use command rmdir or rmdirs.
      
+
      Remove the path and all of its contents. Note that this does not obey include/exclude filters - everything will be removed. Use the delete command if you want to selectively delete files. To delete empty directories only, use command rmdir or rmdirs.
      
 
      Important: Since this can cause data loss, test first with the --dry-run or the --interactive/-i flag.
      
 
      rclone purge remote:path [flags]
      
 
      Options
      
@@ -579,8 +549,8 @@ 
      SEE ALSO
      
 
      rclone rmdir
      
 
      Remove the empty directory at path.
      
 
      Synopsis
      
-
      This removes empty directory given by path. Will not remove the path if it has any objects in it, not even empty subdirectories. Use command rmdirs (or delete with option --rmdirs) to do that.
      
-
      To delete a path and any objects in it, use purge command.
      
+
      This removes empty directory given by path. Will not remove the path if it has any objects in it, not even empty subdirectories. Use command rmdirs (or delete with option --rmdirs) to do that.
      
+
      To delete a path and any objects in it, use purge command.
      
 
      rclone rmdir remote:path [flags]
      
 
      Options
      
 
        -h, --help   help for rmdir
      
@@ -593,6 +563,7 @@ 
      rclone check
      
 
      Checks the files in the source and destination match.
      
 
      Synopsis
      
 
      Checks the files in the source and destination match. It compares sizes and hashes (MD5 or SHA1) and logs a report of files that don't match. It doesn't alter the source or destination.
      
+
      For the crypt remote there is a dedicated command, cryptcheck, that are able to check the checksums of the crypted files.
      
 
      If you supply the --size-only flag, it will only compare the sizes not the hashes as well. Use this for a quick check.
      
 
      If you supply the --download flag, it will download the data from both remotes and check them against each other on the fly. This can be useful for remotes that don't support hashes or if you really want to check all the data.
      
 
      If you supply the --checkfile HASH flag with a valid hash name, the source:path must point to a text file in the SUM format.
      
@@ -657,7 +628,7 @@ 
      SEE ALSO
      
 
      rclone lsd
      
 
      List all directories/containers/buckets in the path.
      
 
      Synopsis
      
-
      Lists the directories in the source path to standard output. Does not recurse by default. Use the -R flag to recurse.
      
+
      Lists the directories in the source path to standard output. Does not recurse by default. Use the -R flag to recurse.
      
 
      This command lists the total size of the directory (if known, -1 if not), the modification time (if known, the current time if not), the number of objects in the directory (if known, -1 if not) and the name of the directory, Eg
      
 
      $ rclone lsd swift:
       494000 2018-04-26 08:43:20     10000 10000files
@@ -667,7 +638,7 @@ 
      Synopsis
      
           -1 2016-10-17 17:41:53        -1 1000files
           -1 2017-01-03 14:40:54        -1 2500files
           -1 2017-07-08 14:39:28        -1 4000files
      
-
      If you just want the directory names use "rclone lsf --dirs-only".
      
+
      If you just want the directory names use rclone lsf --dirs-only.
      
 
      Any of the filtering options can be applied to this command.
      
 
      There are several related list commands
      
 
        
@@ -726,6 +697,7 @@ 
        rclone md5sum
        
 
        Synopsis
        
 
        Produces an md5sum file for all the objects in the path. This is in the same format as the standard md5sum tool produces.
        
 
        By default, the hash is requested from the remote. If MD5 is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling MD5 for any remote.
        
+
        For other algorithms, see the hashsum command. Running rclone md5sum remote:path is equivalent to running rclone hashsum MD5 remote:path.
        
 
        This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a relative path).
        
 
        rclone md5sum remote:path [flags]
        
 
        Options
        
@@ -744,6 +716,7 @@ 
        rclone sha1sum
        
 
        Synopsis
        
 
        Produces an sha1sum file for all the objects in the path. This is in the same format as the standard sha1sum tool produces.
        
 
        By default, the hash is requested from the remote. If SHA-1 is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling SHA-1 for any remote.
        
+
        For other algorithms, see the hashsum command. Running rclone sha1sum remote:path is equivalent to running rclone hashsum SHA1 remote:path.
        
 
        This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a relative path).
        
 
        This command can also hash data received on STDIN, if not passing a remote:path.
        
 
        rclone sha1sum remote:path [flags]
        
@@ -760,6 +733,11 @@ 
        SEE ALSO
        
 
      
 
      rclone size
      
 
      Prints the total size and number of objects in remote:path.
      
+
      Synopsis
      
+
      Counts objects in the path and calculates the total size. Prints the result to standard output.
      
+
      By default the output is in human-readable format, but shows values in both human-readable format as well as the raw numbers (global option --human-readable is not considered). Use option --json to format output as JSON instead.
      
+
      Recurses by default, use --max-depth 1 to stop the recursion.
      
+
      Some backends do not always provide file sizes, see for example Google Photos and Google Drive. Rclone will then show a notice in the log indicating how many such files were encountered, and count them in as empty files in the output of the size command.
      
 
      rclone size remote:path [flags]
      
 
      Options
      
 
        -h, --help   help for size
@@ -771,7 +749,7 @@ 
      SEE ALSO
      
 
    
 
    rclone version
    
 
    Show the version number.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Show the rclone version number, the go version, the build target OS and architecture, the runtime OS and kernel version and bitness, build tags and the type of executable (static or dynamic).
    
 
    For example:
    
 
    $ rclone version
@@ -807,7 +785,7 @@ 
    SEE ALSO
    
 
 
    rclone cleanup
    
 
    Clean up the remote if possible.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Clean up the remote if possible. Empty the trash or delete old file versions. Not supported by all remotes.
    
 
    rclone cleanup remote:path [flags]
    
 
    Options
    
@@ -819,10 +797,10 @@ 
    SEE ALSO
    
 
 
    rclone dedupe
    
 
    Interactively find duplicate filenames and delete/rename them.
    
-
    Synopsis
    
+
    Synopsis
    
 
    By default dedupe interactively finds files with duplicate names and offers to delete all but one or rename them to be different. This is known as deduping by name.
    
 
    Deduping by name is only useful with a small group of backends (e.g. Google Drive, Opendrive) that can have duplicate file names. It can be run on wrapping backends (e.g. crypt) if they wrap a backend which supports duplicate file names.
    
-
    However if --by-hash is passed in then dedupe will find files with duplicate hashes instead which will work on any backend which supports at least one hash. This can be used to find files with duplicate content. This is known as deduping by hash.
    
+
    However if --by-hash is passed in then dedupe will find files with duplicate hashes instead which will work on any backend which supports at least one hash. This can be used to find files with duplicate content. This is known as deduping by hash.
    
 
    If deduping by name, first rclone will merge directories with the same name. It will do this iteratively until all the identically named directories have been merged.
    
 
    Next, if deduping by name, for every group of duplicate file names / hashes, it will delete all but one identical file it finds without confirmation. This means that for most duplicated files the dedupe command will not be interactive.
    
 
    dedupe considers files to be identical if they have the same file path and the same hash. If the backend does not support hashes (e.g. crypt wrapping Google Drive) then they will never be found to be identical. If you use the --size-only flag then files will be considered identical if they have the same size (any hash will be ignored). This can be useful on crypt backends which do not support hashes.
    
@@ -898,7 +876,7 @@ 
    SEE ALSO
    
 
 
    rclone about
    
 
    Get quota information from the remote.
    
-
    Synopsis
    
+
    Synopsis
    
 
    rclone about prints quota information about a remote to standard output. The output is typically used, free, quota and trash contents.
    
 
    E.g. Typical output from rclone about remote: is:
    
 
    Total:   17 GiB
@@ -944,7 +922,7 @@ 
    SEE ALSO
    
 
 
    rclone authorize
    
 
    Remote authorization.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Remote authorization. Used to authorize a remote or headless rclone from a machine with a browser - use as instructed by rclone config.
    
 
    Use the --auth-no-open-browser to prevent rclone to open auth link in default browser automatically.
    
 
    rclone authorize [flags]
    
@@ -958,7 +936,7 @@ 
    SEE ALSO
    
 
 
    rclone backend
    
 
    Run a backend-specific command.
    
-
    Synopsis
    
+
    Synopsis
    
 
    This runs a backend-specific command. The commands themselves (except for "help" and "features") are defined by the backends and you should see the backend docs for definitions.
    
 
    You can discover what commands a backend implements by using
    
 
    rclone backend help remote:
@@ -982,7 +960,7 @@ 
    SEE ALSO
    
 
 
    rclone bisync
    
 
    Perform bidirectonal synchronization between two paths.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Perform bidirectonal synchronization between two paths.
    
 
    Bisync provides a bidirectional cloud sync solution in rclone. It retains the Path1 and Path2 filesystem listings from the prior run. On each successive run it will: - list files on Path1 and Path2, and check for changes on each side. Changes include New, Newer, Older, and Deleted files. - Propagate changes on Path1 to Path2, and vice-versa.
    
 
    See full bisync description for details.
    
@@ -1006,7 +984,7 @@ 
    SEE ALSO
    
 
 
    rclone cat
    
 
    Concatenates any files and sends them to stdout.
    
-
    Synopsis
    
+
    Synopsis
    
 
    rclone cat sends any files to standard output.
    
 
    You can use it like this to output a single file
    
 
    rclone cat remote:path/to/file
    
@@ -1030,7 +1008,7 @@ 
    SEE ALSO
    
 
 
    rclone checksum
    
 
    Checks the files in the source against a SUM file.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Checks that hashsums of source files match the SUM file. It compares hashes (MD5, SHA1, etc) and logs a report of files which don't match. It doesn't alter the file system.
    
 
    If you supply the --download flag, it will download the data from remote and calculate the contents hash on the fly. This can be useful for remotes that don't support hashes or if you really want to check all the data.
    
 
    Note that hash values in the SUM file are treated as case insensitive.
    
@@ -1061,8 +1039,8 @@ 
    SEE ALSO
    
 
  • rclone - Show help for rclone commands, flags and backends.
    • 
 
 
    rclone completion
    
-
    generate the autocompletion script for the specified shell
    
-
    Synopsis
    
+
    Generate the autocompletion script for the specified shell
    
+
    Synopsis
    
 
    Generate the autocompletion script for rclone for the specified shell. See each sub-command's help for details on how to use the generated script.
    
 
    Options
    
 
      -h, --help   help for completion
    
@@ -1070,18 +1048,23 @@ 
    Options
    
 
    SEE ALSO
    
 
 
    rclone completion bash
    
-
    generate the autocompletion script for bash
    
-
    Synopsis
    
+
    Generate the autocompletion script for bash
    
+
    Synopsis
    
 
    Generate the autocompletion script for the bash shell.
    
 
    This script depends on the 'bash-completion' package. If it is not installed already, you can install it via your OS's package manager.
    
-
    To load completions in your current shell session: $ source <(rclone completion bash)
    
-
    To load completions for every new session, execute once: Linux: $ rclone completion bash > /etc/bash_completion.d/rclone MacOS: $ rclone completion bash > /usr/local/etc/bash_completion.d/rclone
    
+
    To load completions in your current shell session:
    
+
    source <(rclone completion bash)
    
+
    To load completions for every new session, execute once:
    
+
    Linux:
    
+
    rclone completion bash > /etc/bash_completion.d/rclone
    
+
    macOS:
    
+
    rclone completion bash > /usr/local/etc/bash_completion.d/rclone
    
 
    You will need to start a new shell for this setup to take effect.
    
 
    rclone completion bash
    
 
    Options
    
@@ -1090,14 +1073,16 @@ 
    Options
    
 
    See the global flags page for global options not listed here.
    
 
    SEE ALSO
    
 
 
    rclone completion fish
    
-
    generate the autocompletion script for fish
    
-
    Synopsis
    
+
    Generate the autocompletion script for fish
    
+
    Synopsis
    
 
    Generate the autocompletion script for the fish shell.
    
-
    To load completions in your current shell session: $ rclone completion fish | source
    
-
    To load completions for every new session, execute once: $ rclone completion fish > ~/.config/fish/completions/rclone.fish
    
+
    To load completions in your current shell session:
    
+
    rclone completion fish | source
    
+
    To load completions for every new session, execute once:
    
+
    rclone completion fish > ~/.config/fish/completions/rclone.fish
    
 
    You will need to start a new shell for this setup to take effect.
    
 
    rclone completion fish [flags]
    
 
    Options
    
@@ -1106,13 +1091,14 @@ 
    Options
    
 
    See the global flags page for global options not listed here.
    
 
    SEE ALSO
    
 
 
    rclone completion powershell
    
-
    generate the autocompletion script for powershell
    
-
    Synopsis
    
+
    Generate the autocompletion script for powershell
    
+
    Synopsis
    
 
    Generate the autocompletion script for powershell.
    
-
    To load completions in your current shell session: PS C:> rclone completion powershell | Out-String | Invoke-Expression
    
+
    To load completions in your current shell session:
    
+
    rclone completion powershell | Out-String | Invoke-Expression
    
 
    To load completions for every new session, add the output of the above command to your powershell profile.
    
 
    rclone completion powershell [flags]
    
 
    Options
    
@@ -1121,15 +1107,19 @@ 
    Options
    
 
    See the global flags page for global options not listed here.
    
 
    SEE ALSO
    
 
 
    rclone completion zsh
    
-
    generate the autocompletion script for zsh
    
-
    Synopsis
    
+
    Generate the autocompletion script for zsh
    
+
    Synopsis
    
 
    Generate the autocompletion script for the zsh shell.
    
 
    If shell completion is not already enabled in your environment you will need to enable it. You can execute the following once:
    
-
    $ echo "autoload -U compinit; compinit" >> ~/.zshrc
    
-
    To load completions for every new session, execute once: # Linux: $ rclone completion zsh > "${fpath[1]}/_rclone" # macOS: $ rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone
    
+
    echo "autoload -U compinit; compinit" >> ~/.zshrc
    
+
    To load completions for every new session, execute once:
    
+
    Linux:
    
+
    rclone completion zsh > "${fpath[1]}/_rclone"
    
+
    macOS:
    
+
    rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone
    
 
    You will need to start a new shell for this setup to take effect.
    
 
    rclone completion zsh [flags]
    
 
    Options
    
@@ -1138,11 +1128,11 @@ 
    Options
    
 
    See the global flags page for global options not listed here.
    
 
    SEE ALSO
    
 
 
    rclone config create
    
 
    Create a new remote with name, type and options.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Create a new remote of name with type and options. The options should be passed in pairs of key value or as key=value.
    
 
    For example, to make a swift remote of name myremote using auto config you would do:
    
 
    rclone config create myremote swift env_auth true
@@ -1223,7 +1213,7 @@ 
    SEE ALSO
    
 
 
    rclone config disconnect
    
 
    Disconnects user from remote
    
-
    Synopsis
    
+
    Synopsis
    
 
    This disconnects the remote: passed in to the cloud storage system.
    
 
    This normally means revoking the oauth token.
    
 
    To reconnect use "rclone config reconnect".
    
@@ -1247,7 +1237,7 @@ 
    SEE ALSO
    
 
 
    rclone config edit
    
 
    Enter an interactive configuration session.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Enter an interactive configuration session where you can setup new remotes and manage existing ones. You may also set or remove a password to protect your configuration.
    
 
    rclone config edit [flags]
    
 
    Options
    
@@ -1269,7 +1259,7 @@ 
    SEE ALSO
    
 
 
    rclone config password
    
 
    Update password in an existing remote.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Update an existing remote's password. The password should be passed in pairs of key password or as key=password. The password should be passed in in clear (unobscured).
    
 
    For example, to set password of a remote of name myremote you would do:
    
 
    rclone config password myremote fieldname mypassword
@@ -1305,7 +1295,7 @@ 
    SEE ALSO
    
 
 
    rclone config reconnect
    
 
    Re-authenticates user with remote.
    
-
    Synopsis
    
+
    Synopsis
    
 
    This reconnects remote: passed in to the cloud storage system.
    
 
    To disconnect the remote use "rclone config disconnect".
    
 
    This normally means going through the interactive oauth flow again.
    
@@ -1339,7 +1329,7 @@ 
    SEE ALSO
    
 
 
    rclone config update
    
 
    Update options in an existing remote.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Update an existing remote's options. The options should be passed in pairs of key value or as key=value.
    
 
    For example, to update the env_auth field of a remote of name myremote you would do:
    
 
    rclone config update myremote env_auth true
@@ -1410,7 +1400,7 @@ 
    SEE ALSO
    
 
 
    rclone config userinfo
    
 
    Prints info about logged in user of remote.
    
-
    Synopsis
    
+
    Synopsis
    
 
    This prints the details of the person logged in to the cloud storage system.
    
 
    rclone config userinfo remote: [flags]
    
 
    Options
    
@@ -1423,9 +1413,9 @@ 
    SEE ALSO
    
 
 
    rclone copyto
    
 
    Copy files from source to dest, skipping identical files.
    
-
    Synopsis
    
+
    Synopsis
    
 
    If source:path is a file or directory then it copies it to a file or directory named dest:path.
    
-
    This can be used to upload single files to other than their current name. If the source is a directory then it acts exactly like the copy command.
    
+
    This can be used to upload single files to other than their current name. If the source is a directory then it acts exactly like the copy command.
    
 
    So
    
 
    rclone copyto src dst
    
 
    where src and dst are rclone paths, either remote:path or /path/to/local or C:.
    
@@ -1447,18 +1437,19 @@ 
    SEE ALSO
    
 
 
    rclone copyurl
    
 
    Copy url content to dest.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Download a URL's content and copy it to the destination without saving it in temporary storage.
    
-
    Setting --auto-filename will cause the file name to be retrieved from the URL (after any redirections) and used in the destination path. With --print-filename in addition, the resulting file name will be printed.
    
+
    Setting --auto-filename will attempt to automatically determine the filename from the URL (after any redirections) and used in the destination path. With --auto-filename-header in addition, if a specific filename is set in HTTP headers, it will be used instead of the name from the URL. With --print-filename in addition, the resulting file name will be printed.
    
 
    Setting --no-clobber will prevent overwriting file on the destination if there is one with the same name.
    
 
    Setting --stdout or making the output file name - will cause the output to be written to standard output.
    
 
    rclone copyurl https://example.com dest:path [flags]
    
 
    Options
    
-
      -a, --auto-filename    Get the file name from the URL and use it for destination file path
-  -h, --help             help for copyurl
-      --no-clobber       Prevent overwriting file with same name
-  -p, --print-filename   Print the resulting name from --auto-filename
-      --stdout           Write the output to stdout rather than a file
    
+
      -a, --auto-filename     Get the file name from the URL and use it for destination file path
+      --header-filename   Get the file name from the Content-Disposition header
+  -h, --help              help for copyurl
+      --no-clobber        Prevent overwriting file with same name
+  -p, --print-filename    Print the resulting name from --auto-filename
+      --stdout            Write the output to stdout rather than a file
    
 
    See the global flags page for global options not listed here.
    
 
    SEE ALSO
    
 
      
@@ -1466,8 +1457,8 @@ 
      SEE ALSO
      
 
    
 
    rclone cryptcheck
    
 
    Cryptcheck checks the integrity of a crypted remote.
    
-
    Synopsis
    
-
    rclone cryptcheck checks a remote against a crypted remote. This is the equivalent of running rclone check, but able to check the checksums of the crypted remote.
    
+
    Synopsis
    
+
    rclone cryptcheck checks a remote against a crypted remote. This is the equivalent of running rclone check, but able to check the checksums of the crypted remote.
    
 
    For it to work the underlying remote of the cryptedremote must support some kind of checksum.
    
 
    It works by reading the nonce from each file on the cryptedremote: and using that to encrypt each file on the remote:. It then checks the checksum of the underlying file on the cryptedremote: against the checksum of the file it has just encrypted.
    
 
    Use it like this
    
@@ -1502,14 +1493,14 @@ 
    SEE ALSO
    
 
 
    rclone cryptdecode
    
 
    Cryptdecode returns unencrypted file names.
    
-
    Synopsis
    
+
    Synopsis
    
 
    rclone cryptdecode returns unencrypted file names when provided with a list of encrypted file names. List limit is 10 items.
    
-
    If you supply the --reverse flag, it will return encrypted file names.
    
+
    If you supply the --reverse flag, it will return encrypted file names.
    
 
    use it like this
    
 
    rclone cryptdecode encryptedremote: encryptedfilename1 encryptedfilename2
 
 rclone cryptdecode --reverse encryptedremote: filename1 filename2
    
-
    Another way to accomplish this is by using the rclone backend encode (or decode)command. See the documentation on the crypt overlay for more info.
    
+
    Another way to accomplish this is by using the rclone backend encode (or decode) command. See the documentation on the crypt overlay for more info.
    
 
    rclone cryptdecode encryptedremote: encryptedfilename [flags]
    
 
    Options
    
 
      -h, --help      help for cryptdecode
@@ -1521,7 +1512,7 @@ 
    SEE ALSO
    
 
 
    rclone deletefile
    
 
    Remove a single file from remote.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Remove a single file from remote. Unlike delete it cannot be used to remove a directory and it doesn't obey include/exclude filters - if the specified file exists, it will always be removed.
    
 
    rclone deletefile remote:path [flags]
    
 
    Options
    
@@ -1533,8 +1524,8 @@ 
    SEE ALSO
    
 
 
    rclone genautocomplete
    
 
    Output completion script for a given shell.
    
-
    Synopsis
    
-
    Generates a shell completion script for rclone. Run with --help to list the supported shells.
    
+
    Synopsis
    
+
    Generates a shell completion script for rclone. Run with --help to list the supported shells.
    
 
    Options
    
 
      -h, --help   help for genautocomplete
    
 
    See the global flags page for global options not listed here.
    
@@ -1547,7 +1538,7 @@ 
    SEE ALSO
    
 
 
    rclone genautocomplete bash
    
 
    Output bash completion script for rclone.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Generates a bash shell autocompletion script for rclone.
    
 
    This writes to /etc/bash_completion.d/rclone by default so will probably need to be run with sudo or as root, e.g.
    
 
    sudo rclone genautocomplete bash
    
@@ -1565,7 +1556,7 @@ 
    SEE ALSO
    
 
 
    rclone genautocomplete fish
    
 
    Output fish completion script for rclone.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Generates a fish autocompletion script for rclone.
    
 
    This writes to /etc/fish/completions/rclone.fish by default so will probably need to be run with sudo or as root, e.g.
    
 
    sudo rclone genautocomplete fish
    
@@ -1583,7 +1574,7 @@ 
    SEE ALSO
    
 
 
    rclone genautocomplete zsh
    
 
    Output zsh completion script for rclone.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Generates a zsh autocompletion script for rclone.
    
 
    This writes to /usr/share/zsh/vendor-completions/_rclone by default so will probably need to be run with sudo or as root, e.g.
    
 
    sudo rclone genautocomplete zsh
    
@@ -1601,7 +1592,7 @@ 
    SEE ALSO
    
 
 
    rclone gendocs
    
 
    Output markdown docs for rclone to the directory supplied.
    
-
    Synopsis
    
+
    Synopsis
    
 
    This produces markdown docs for the rclone commands to the directory supplied. These are in a format suitable for hugo to render into the rclone.org website.
    
 
    rclone gendocs output_directory [flags]
    
 
    Options
    
@@ -1613,9 +1604,10 @@ 
    SEE ALSO
    
 
 
    rclone hashsum
    
 
    Produces a hashsum file for all the objects in the path.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Produces a hash file for all the objects in the path using the hash named. The output is in the same format as the standard md5sum/sha1sum tool.
    
 
    By default, the hash is requested from the remote. If the hash is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling any hash for any remote.
    
+
    For the MD5 and SHA1 algorithms there are also dedicated commands, md5sum and sha1sum.
    
 
    This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a relative path).
    
 
    Run without a hash to see the list of all supported hashes, e.g.
    
 
    $ rclone hashsum
@@ -1626,6 +1618,7 @@ 
    Synopsis
    
   * crc32
   * sha256
   * dropbox
+  * hidrive
   * mailru
   * quickxor
    
 
    Then
    
@@ -1645,7 +1638,7 @@ 
    SEE ALSO
    
 
 
    rclone link
    
 
    Generate public link to file/folder.
    
-
    Synopsis
    
+
    Synopsis
    
 
    rclone link will create, retrieve or remove a public link to the given file or folder.
    
 
    rclone link remote:path/to/file
 rclone link remote:path/to/folder/
@@ -1666,9 +1659,9 @@ 
    SEE ALSO
    
 
 
    rclone listremotes
    
 
    List all the remotes in the config file.
    
-
    Synopsis
    
+
    Synopsis
    
 
    rclone listremotes lists all the available remotes from the config file.
    
-
    When uses with the -l flag it lists the types too.
    
+
    When used with the --long flag it lists the types too.
    
 
    rclone listremotes [flags]
    
 
    Options
    
 
      -h, --help   help for listremotes
@@ -1680,7 +1673,7 @@ 
    SEE ALSO
    
 
 
    rclone lsf
    
 
    List directories and objects in remote:path formatted for parsing.
    
-
    Synopsis
    
+
    Synopsis
    
 
    List the contents of the source path (directories and objects) to standard output in a form which is easy to parse by scripts. By default this will just be the names of the objects and directories, one per line. The directories will have a / suffix.
    
 
    Eg
    
 
    $ rclone lsf swift:bucket
@@ -1689,7 +1682,7 @@ 
    Synopsis
    
 diwogej7
 ferejej3gux/
 fubuwic
    
-
    Use the --format option to control what gets listed. By default this is just the path, but you can use these parameters to control the output:
    
+
    Use the --format option to control what gets listed. By default this is just the path, but you can use these parameters to control the output:
    
 
    p - path
 s - size
 t - modification time
@@ -1698,8 +1691,9 @@ 
    Synopsis
    
 o - Original ID of underlying object
 m - MimeType of object if known
 e - encrypted name
-T - tier of storage if known, e.g. "Hot" or "Cool"
    
-
    So if you wanted the path, size and modification time, you would use --format "pst", or maybe --format "tsp" to put the path last.
    
+T - tier of storage if known, e.g. "Hot" or "Cool"
+M - Metadata of object in JSON blob format, eg {"key":"value"}
    
+
    So if you wanted the path, size and modification time, you would use --format "pst", or maybe --format "tsp" to put the path last.
    
 
    Eg
    
 
    $ rclone lsf  --format "tsp" swift:bucket
 2016-06-25 18:55:41;60295;bevajer5jef
@@ -1707,7 +1701,7 @@ 
    Synopsis
    
 2016-06-25 18:55:43;94467;diwogej7
 2018-04-26 08:50:45;0;ferejej3gux/
 2016-06-25 18:55:40;37600;fubuwic
    
-
    If you specify "h" in the format you will get the MD5 hash by default, use the "--hash" flag to change which hash you want. Note that this can be returned as an empty string if it isn't available on the object (and for directories), "ERROR" if there was an error reading it from the object and "UNSUPPORTED" if that object does not support that hash type.
    
+
    If you specify "h" in the format you will get the MD5 hash by default, use the --hash flag to change which hash you want. Note that this can be returned as an empty string if it isn't available on the object (and for directories), "ERROR" if there was an error reading it from the object and "UNSUPPORTED" if that object does not support that hash type.
    
 
    For example, to emulate the md5sum command you can use
    
 
    rclone lsf -R --hash MD5 --format hp --separator "  " --files-only .
    
 
    Eg
    
@@ -1718,7 +1712,7 @@ 
    Synopsis
    
 8fd37c3810dd660778137ac3a66cc06d  fubuwic
 99713e14a4c4ff553acaf1930fad985b  gixacuh7ku
    
 
    (Though "rclone md5sum ." is an easier way of typing this.)
    
-
    By default the separator is ";" this can be changed with the --separator flag. Note that separators aren't escaped in the path so putting it last is a good strategy.
    
+
    By default the separator is ";" this can be changed with the --separator flag. Note that separators aren't escaped in the path so putting it last is a good strategy.
    
 
    Eg
    
 
    $ rclone lsf  --separator "," --format "tshp" swift:bucket
 2016-06-25 18:55:41,60295,7908e352297f0f530b84a756f188baa3,bevajer5jef
@@ -1732,7 +1726,7 @@ 
    Synopsis
    
 test.log,22355
 test.sh,449
 "this file contains a comma, in the file name.txt",6
    
-
    Note that the --absolute parameter is useful for making lists of files to pass to an rclone copy with the --files-from-raw flag.
    
+
    Note that the --absolute parameter is useful for making lists of files to pass to an rclone copy with the --files-from-raw flag.
    
 
    For example, to find all the files modified within one day and copy those only (without traversing the whole directory structure):
    
 
    rclone lsf --absolute --files-only --max-age 1d /path/to/local > new_files
 rclone copy --files-from-raw new_files /path/to/local remote:path
    
@@ -1768,18 +1762,37 @@ 
    SEE ALSO
    
 
 
    rclone lsjson
    
 
    List directories and objects in the path in JSON format.
    
-
    Synopsis
    
+
    Synopsis
    
 
    List directories and objects in the path in JSON format.
    
 
    The output is an array of Items, where each Item looks like this
    
-
    { "Hashes" : { "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f", "MD5" : "b1946ac92492d2347c6235b4d2611184", "DropboxHash" : "ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc" }, "ID": "y2djkhiujf83u33", "OrigID": "UYOJVTUW00Q1RzTDA", "IsBucket" : false, "IsDir" : false, "MimeType" : "application/octet-stream", "ModTime" : "2017-05-31T16:15:57.034468261+01:00", "Name" : "file.txt", "Encrypted" : "v0qpsdq8anpci8n929v3uu9338", "EncryptedPath" : "kja9098349023498/v0qpsdq8anpci8n929v3uu9338", "Path" : "full/path/goes/here/file.txt", "Size" : 6, "Tier" : "hot", }
    
-
    If --hash is not specified the Hashes property won't be emitted. The types of hash can be specified with the --hash-type parameter (which may be repeated). If --hash-type is set then it implies --hash.
    
-
    If --no-modtime is specified then ModTime will be blank. This can speed things up on remotes where reading the ModTime takes an extra request (e.g. s3, swift).
    
-
    If --no-mimetype is specified then MimeType will be blank. This can speed things up on remotes where reading the MimeType takes an extra request (e.g. s3, swift).
    
-
    If --encrypted is not specified the Encrypted won't be emitted.
    
-
    If --dirs-only is not specified files in addition to directories are returned
    
-
    If --files-only is not specified directories in addition to the files will be returned.
    
-
    if --stat is set then a single JSON blob will be returned about the item pointed to. This will return an error if the item isn't found. However on bucket based backends (like s3, gcs, b2, azureblob etc) if the item isn't found it will return an empty directory as it isn't possible to tell empty directories from missing directories there.
    
-
    The Path field will only show folders below the remote path being listed. If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt" will be "subfolder/file.txt", not "remote:path/subfolder/file.txt". When used without --recursive the Path will always be the same as Name.
    
+
    {
+  "Hashes" : {
+     "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f",
+     "MD5" : "b1946ac92492d2347c6235b4d2611184",
+     "DropboxHash" : "ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc"
+  },
+  "ID": "y2djkhiujf83u33",
+  "OrigID": "UYOJVTUW00Q1RzTDA",
+  "IsBucket" : false,
+  "IsDir" : false,
+  "MimeType" : "application/octet-stream",
+  "ModTime" : "2017-05-31T16:15:57.034468261+01:00",
+  "Name" : "file.txt",
+  "Encrypted" : "v0qpsdq8anpci8n929v3uu9338",
+  "EncryptedPath" : "kja9098349023498/v0qpsdq8anpci8n929v3uu9338",
+  "Path" : "full/path/goes/here/file.txt",
+  "Size" : 6,
+  "Tier" : "hot",
+}
    
+
    If --hash is not specified the Hashes property won't be emitted. The types of hash can be specified with the --hash-type parameter (which may be repeated). If --hash-type is set then it implies --hash.
    
+
    If --no-modtime is specified then ModTime will be blank. This can speed things up on remotes where reading the ModTime takes an extra request (e.g. s3, swift).
    
+
    If --no-mimetype is specified then MimeType will be blank. This can speed things up on remotes where reading the MimeType takes an extra request (e.g. s3, swift).
    
+
    If --encrypted is not specified the Encrypted won't be emitted.
    
+
    If --dirs-only is not specified files in addition to directories are returned
    
+
    If --files-only is not specified directories in addition to the files will be returned.
    
+
    If --metadata is set then an additional Metadata key will be returned. This will have metdata in rclone standard format as a JSON object.
    
+
    if --stat is set then a single JSON blob will be returned about the item pointed to. This will return an error if the item isn't found. However on bucket based backends (like s3, gcs, b2, azureblob etc) if the item isn't found it will return an empty directory as it isn't possible to tell empty directories from missing directories there.
    
+
    The Path field will only show folders below the remote path being listed. If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt" will be "subfolder/file.txt", not "remote:path/subfolder/file.txt". When used without --recursive the Path will always be the same as Name.
    
 
    If the directory is a bucket in a bucket-based backend, then "IsBucket" will be set to true. This key won't be present unless it is "true".
    
 
    The time is in RFC3339 format with up to nanosecond precision. The number of decimal digits in the seconds will depend on the precision that the remote can hold the times, so if times are accurate to the nearest millisecond (e.g. Google Drive) then 3 digits will always be shown ("2017-05-31T16:15:57.034+01:00") whereas if the times are accurate to the nearest second (Dropbox, Box, WebDav, etc.) no digits will be shown ("2017-05-31T16:15:57+01:00").
    
 
    The whole output can be processed as a JSON blob, or alternatively it can be processed line by line as each item is written one to a line.
    
@@ -1799,7 +1812,7 @@ 
    Synopsis
    
 
    rclone lsjson remote:path [flags]
    
 
    Options
    
 
          --dirs-only               Show only directories in the listing
-  -M, --encrypted               Show the encrypted names
+      --encrypted               Show the encrypted names
       --files-only              Show only files in the listing
       --hash                    Include hashes in the output (may take longer)
       --hash-type stringArray   Show only this hash type (may be repeated)
@@ -1816,7 +1829,7 @@ 
    SEE ALSO
    
 
 
    rclone mount
    
 
    Mount the remote as file system on a mountpoint.
    
-
    Synopsis
    
+
    Synopsis
    
 
    rclone mount allows Linux, FreeBSD, macOS and Windows to mount any of Rclone's cloud storage systems as a file system with FUSE.
    
 
    First set up your remote using rclone config. Check it works with rclone ls etc.
    
 
    On Linux and macOS, you can run mount in either foreground or background (aka daemon) mode. Mount runs in foreground mode by default. Use the --daemon flag to force background mode. On Windows you can run mount in foreground only, the flag is ignored.
    
@@ -1839,7 +1852,7 @@ 
    Synopsis
    
 
    The size of the mounted file system will be set according to information retrieved from the remote, the same as returned by the rclone about command. Remotes with unlimited storage may report the used size only, then an additional 1 PiB of free space is assumed. If the remote does not support the about feature at all, then 1 PiB is set as both the total and the free size.
    
 
    Installing on Windows
    
 
    To run rclone mount on Windows, you will need to download and install WinFsp.
    
-
    WinFsp is an open-source Windows File System Proxy which makes it easy to write user space file systems for Windows. It provides a FUSE emulation layer which rclone uses combination with cgofuse. Both of these packages are by Bill Zissimopoulos who was very helpful during the implementation of rclone mount for Windows.
    
+
    WinFsp is an open-source Windows File System Proxy which makes it easy to write user space file systems for Windows. It provides a FUSE emulation layer which rclone uses combination with cgofuse. Both of these packages are by Bill Zissimopoulos who was very helpful during the implementation of rclone mount for Windows.
    
 
    Mounting modes on windows
    
 
    Unlike other operating systems, Microsoft Windows provides a different filesystem type for network and fixed drives. It optimises access on the assumption fixed disk drives are fast and reliable, while network drives have relatively high latency and less reliability. Some settings can also be differentiated between the two types, for example that Windows Explorer should just display icons and not create preview thumbnails for image and video files on network drives.
    
 
    In most cases, rclone will mount the remote as a normal, fixed disk drive by default. However, you can also choose to mount it as a remote network drive, often described as a network share. If you mount an rclone remote using the default, fixed drive mode and experience unexpected program errors, freezes or other issues, consider mounting as a network drive instead.
    
@@ -1873,7 +1886,7 @@ 
    Windows caveats
    
 
    Drives created as Administrator are not visible to other accounts, not even an account that was elevated to Administrator with the User Account Control (UAC) feature. A result of this is that if you mount to a drive letter from a Command Prompt run as Administrator, and then try to access the same drive from Windows Explorer (which does not run as Administrator), you will not be able to see the mounted drive.
    
 
    If you don't need to access the drive from applications running with administrative privileges, the easiest way around this is to always create the mount from a non-elevated command prompt.
    
 
    To make mapped drives available to the user account that created them regardless if elevated or not, there is a special Windows setting called linked connections that can be enabled.
    
-
    It is also possible to make a drive mount available to everyone on the system, by running the process creating it as the built-in SYSTEM account. There are several ways to do this: One is to use the command-line utility PsExec, from Microsoft's Sysinternals suite, which has option -s to start processes as the SYSTEM account. Another alternative is to run the mount command from a Windows Scheduled Task, or a Windows Service, configured to run as the SYSTEM account. A third alternative is to use the WinFsp.Launcher infrastructure). Note that when running rclone as another user, it will not use the configuration file from your profile unless you tell it to with the --config option. Read more in the install documentation.
    
+
    It is also possible to make a drive mount available to everyone on the system, by running the process creating it as the built-in SYSTEM account. There are several ways to do this: One is to use the command-line utility PsExec, from Microsoft's Sysinternals suite, which has option -s to start processes as the SYSTEM account. Another alternative is to run the mount command from a Windows Scheduled Task, or a Windows Service, configured to run as the SYSTEM account. A third alternative is to use the WinFsp.Launcher infrastructure). Note that when running rclone as another user, it will not use the configuration file from your profile unless you tell it to with the --config option. Read more in the install documentation.
    
 
    Note that mapping to a directory path, instead of a drive letter, does not suffer from the same limitations.
    
 
    Limitations
    
 
    Without the use of --vfs-cache-mode this can only write files sequentially, it can only seek when reading. This means that many applications won't work with their files on an rclone mount without --vfs-cache-mode writes or --vfs-cache-mode full. See the VFS File Caching section for more info.
    
@@ -1936,7 +1949,7 @@ 
    VFS - Virtual File System
    
 
    Cloud storage objects have lots of properties which aren't like disk files - you can't extend them or write to the middle of them, so the VFS layer has to deal with that. Because there is no one right way of doing this there are various options explained below.
    
 
    The VFS layer also implements a directory cache - this caches info about files and directories (but not the data) in memory.
    
 
    VFS Directory Cache
    
-
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the mount will appear immediately or invalidate the cache.
    
+
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the VFS will appear immediately or invalidate the cache.
    
 
    --dir-cache-time duration   Time to cache directory entries for (default 5m0s)
 --poll-interval duration    Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable (default 1m0s)
    
 
    However, changes made directly on the cloud storage by the web interface or a different copy of rclone will only be picked up once the directory cache expires if the backend configured does not support polling for changes. If the backend supports polling, changes will be picked up within the polling interval.
    
@@ -1999,6 +2012,19 @@ 
    --vfs-cache-mode full
    
 
    When reading a file rclone will read --buffer-size plus --vfs-read-ahead bytes ahead. The --buffer-size is buffered in memory whereas the --vfs-read-ahead is buffered on disk.
    
 
    When using this mode it is recommended that --buffer-size is not set too large and --vfs-read-ahead is set large if required.
    
 
    IMPORTANT not all file systems support sparse files. In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected.
    
+
    Fingerprinting
    
+
    Various parts of the VFS use fingerprinting to see if a local file copy has changed relative to a remote file. Fingerprints are made from:
    
+
      
+
    • size
      • 
+
    • modification time
      • 
+
    • hash
      • 
+
    
+
    where available on an object.
    
+
    On some backends some of these attributes are slow to read (they take an extra API call per object, or extra work per object).
    
+
    For example hash is slow with the local and sftp backends as they have to read the entire file and hash it, and modtime is slow with the s3, swift, ftp and qinqstor backends because they need to do an extra API call to fetch it.
    
+
    If you use the --vfs-fast-fingerprint flag then rclone will not include the slow operations in the fingerprint. This makes the fingerprinting less accurate but much faster and will improve the opening time of cached files.
    
+
    If you are running a vfs cache over local, s3 or swift backends then using this flag is recommended.
    
+
    Note that if you change the value of this flag, the fingerprints of the files in the cache may be invalidated and the files will need to be downloaded again.
    
 
    VFS Chunked Reading
    
 
    When rclone reads files from a remote it reads them in chunks. This means that rather than requesting the whole file rclone reads the chunk specified. This can reduce the used download quota for some remotes by requesting only chunks from the remote that are actually read, at the cost of an increased number of requests.
    
 
    These flags control the chunking:
    
@@ -2013,20 +2039,23 @@ 
    VFS Performance
    
 
    --no-checksum     Don't compare checksums on up/download.
 --no-modtime      Don't read/write the modification time (can speed things up).
 --no-seek         Don't allow seeking in files.
---read-only       Mount read-only.
    
+--read-only       Only allow read-only access.
    
 
    Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write to come in. These flags only come into effect when not using an on disk cache file.
    
 
    --vfs-read-wait duration   Time to wait for in-sequence read before seeking (default 20ms)
 --vfs-write-wait duration  Time to wait for in-sequence write before giving error (default 1s)
    
-
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from cache (the related global flag --checkers have no effect on mount).
    
+
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from the cache (the related global flag --checkers has no effect on the VFS).
    
 
    --transfers int  Number of file transfers to run in parallel (default 4)
    
 
    VFS Case Sensitivity
    
 
    Linux file systems are case-sensitive: two files can differ only by case, and the exact case must be used when opening a file.
    
 
    File systems in modern Windows are case-insensitive but case-preserving: although existing files can be opened using any case, the exact case used to create the file is preserved and available for programs to query. It is not allowed for two files in the same directory to differ only by case.
    
 
    Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default.
    
-
    The --vfs-case-insensitive mount flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the mounted file system as-is. If the flag is "true" (or appears without a value on command line), rclone may perform a "fixup" as explained below.
    
-
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on mounted file system. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by an underlying mounted file system.
    
-
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system mounted by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
+
    The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the remote as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below.
    
+
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by the underlying remote.
    
+
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
 
    If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true".
    
+
    VFS Disk Options
    
+
    This flag allows you to manually set the statistics about the filing system. It can be useful when those statistics cannot be read correctly automatically.
    
+
    --vfs-disk-space-total-size    Manually set the total disk space size (example: 256G, default: -1)
    
 
    Alternate report of used bytes
    
 
    Some backends, most notably S3, do not report the amount of bytes used. If you need this information to be available when running df on the filesystem, then pass the flag --vfs-used-is-size to rclone. With this flag set, instead of relying on the backend to report this information, rclone will scan the whole remote similar to rclone size and compute the total used space itself.
    
 
    WARNING. Contrary to rclone size, this flag ignores filters so that the result is accurate. However, this is very inefficient and may cost lots of API calls resulting in extra charges. Use it as a last resort and only with caching.
    
@@ -2058,7 +2087,7 @@ 
    Options
    
       --noapplexattr                           Ignore all "com.apple.*" extended attributes (supported on OSX only)
   -o, --option stringArray                     Option for libfuse/WinFsp (repeat if required)
       --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
-      --read-only                              Mount read-only
+      --read-only                              Only allow read-only access
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
       --vfs-cache-max-age duration             Max age of objects in the cache (default 1h0m0s)
@@ -2066,6 +2095,8 @@ 
    Options
    
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
       --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
+      --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
+      --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
@@ -2082,9 +2113,9 @@ 
    SEE ALSO
    
 
 
    rclone moveto
    
 
    Move file or directory from source to dest.
    
-
    Synopsis
    
+
    Synopsis
    
 
    If source:path is a file or directory then it moves it to a file or directory named dest:path.
    
-
    This can be used to rename files or upload single files to other than their existing name. If the source is a directory then it acts exactly like the move command.
    
+
    This can be used to rename files or upload single files to other than their existing name. If the source is a directory then it acts exactly like the move command.
    
 
    So
    
 
    rclone moveto src dst
    
 
    where src and dst are rclone paths, either remote:path or /path/to/local or C:.
    
@@ -2107,10 +2138,10 @@ 
    SEE ALSO
    
 
 
    rclone ncdu
    
 
    Explore a remote with a text based user interface.
    
-
    Synopsis
    
+
    Synopsis
    
 
    This displays a text based user interface allowing the navigation of a remote. It is most useful for answering the question - "What is using all my disk space?".
    
 
    To make the user interface it first scans the entire remote given and builds an in memory representation. rclone ncdu can be used during this scanning phase and you will see it building up the directory structure as it goes along.
    
-
    Here are the keys - press '?' to toggle the help on and off
    
+
    You can interact with the user interface using key presses, press '?' to toggle the help on and off. The supported keys are:
    
 
     ↑,↓ or k,j to Move
  →,l to enter
  ←,h to return
@@ -2120,13 +2151,28 @@ 
    Synopsis
    
  u toggle human-readable format
  n,s,C,A sort by name,size,count,average size
  d delete file/directory
+ v select file/directory
+ V enter visual select mode
+ D delete selected files/directories
  y copy current path to clipboard
  Y display current path
- ^L refresh screen
+ ^L refresh screen (fix screen corruption)
  ? to toggle help on and off
- q/ESC/c-C to quit
    
+ q/ESC/^c to quit
    
+
    Listed files/directories may be prefixed by a one-character flag, some of them combined with a description in brackes at end of line. These flags have the following meaning:
    
+
    e means this is an empty directory, i.e. contains no files (but
+  may contain empty subdirectories)
+~ means this is a directory where some of the files (possibly in
+  subdirectories) have unknown size, and therefore the directory
+  size may be underestimated (and average size inaccurate, as it
+  is average of the files with known sizes).
+. means an error occurred while reading a subdirectory, and
+  therefore the directory size may be underestimated (and average
+  size inaccurate)
+! means an error occurred while reading this directory
    
 
    This an homage to the ncdu tool but for rclone remotes. It is missing lots of features at the moment but is useful as it stands.
    
-
    Note that it might take some time to delete big files/folders. The UI won't respond in the meantime since the deletion is done synchronously.
    
+
    Note that it might take some time to delete big files/directories. The UI won't respond in the meantime since the deletion is done synchronously.
    
+
    For a non-interactive listing of the remote, see the tree command. To just get the total size of the remote you can also use the size command.
    
 
    rclone ncdu remote:path [flags]
    
 
    Options
    
 
      -h, --help   help for ncdu
    
@@ -2137,11 +2183,11 @@ 
    SEE ALSO
    
 
 
    rclone obscure
    
 
    Obscure password for use in the rclone config file.
    
-
    Synopsis
    
+
    Synopsis
    
 
    In the rclone config file, human-readable passwords are obscured. Obscuring them is done by encrypting them and writing them out in base64. This is not a secure way of encrypting these passwords as rclone can decrypt them - it is to prevent "eyedropping" - namely someone seeing a password in the rclone config file by accident.
    
 
    Many equally important things (like access tokens) are not obscured in the config file. However it is very hard to shoulder surf a 64 character hex token.
    
 
    This command can also accept a password through STDIN instead of an argument by passing a hyphen as an argument. This will use the first line of STDIN as the password not including the trailing newline.
    
-
    echo "secretpassword" | rclone obscure -
    
+
    echo "secretpassword" | rclone obscure -
    
 
    If there is no data on STDIN to read, rclone obscure will default to obfuscating the hyphen itself.
    
 
    If you want to encrypt the config file then please use config file encryption - see rclone config for more info.
    
 
    rclone obscure password [flags]
    
@@ -2154,24 +2200,24 @@ 
    SEE ALSO
    
 
 
    rclone rc
    
 
    Run a command against a running rclone.
    
-
    Synopsis
    
-
    This runs a command against a running rclone. Use the --url flag to specify an non default URL to connect on. This can be either a ":port" which is taken to mean "http://localhost:port" or a "host:port" which is taken to mean "http://host:port"
    
-
    A username and password can be passed in with --user and --pass.
    
-
    Note that --rc-addr, --rc-user, --rc-pass will be read also for --url, --user, --pass.
    
+
    Synopsis
    
+
    This runs a command against a running rclone. Use the --url flag to specify an non default URL to connect on. This can be either a ":port" which is taken to mean "http://localhost:port" or a "host:port" which is taken to mean "http://host:port"
    
+
    A username and password can be passed in with --user and --pass.
    
+
    Note that --rc-addr, --rc-user, --rc-pass will be read also for --url, --user, --pass.
    
 
    Arguments should be passed in as parameter=value.
    
 
    The result will be returned as a JSON object by default.
    
-
    The --json parameter can be used to pass in a JSON blob as an input instead of key=value arguments. This is the only way of passing in more complicated values.
    
-
    The -o/--opt option can be used to set a key "opt" with key, value options in the form "-o key=value" or "-o key". It can be repeated as many times as required. This is useful for rc commands which take the "opt" parameter which by convention is a dictionary of strings.
    
+
    The --json parameter can be used to pass in a JSON blob as an input instead of key=value arguments. This is the only way of passing in more complicated values.
    
+
    The -o/--opt option can be used to set a key "opt" with key, value options in the form -o key=value or -o key. It can be repeated as many times as required. This is useful for rc commands which take the "opt" parameter which by convention is a dictionary of strings.
    
 
    -o key=value -o key2
    
 
    Will place this in the "opt" value
    
 
    {"key":"value", "key2","")
    
-
    The -a/--arg option can be used to set strings in the "arg" value. It can be repeated as many times as required. This is useful for rc commands which take the "arg" parameter which by convention is a list of strings.
    
+
    The -a/--arg option can be used to set strings in the "arg" value. It can be repeated as many times as required. This is useful for rc commands which take the "arg" parameter which by convention is a list of strings.
    
 
    -a value -a value2
    
 
    Will place this in the "arg" value
    
 
    ["value", "value2"]
    
-
    Use --loopback to connect to the rclone instance running "rclone rc". This is very useful for testing commands without having to run an rclone rc server, e.g.:
    
+
    Use --loopback to connect to the rclone instance running rclone rc. This is very useful for testing commands without having to run an rclone rc server, e.g.:
    
 
    rclone rc --loopback operations/about fs=/
    
-
    Use "rclone rc" to see a list of all possible commands.
    
+
    Use rclone rc to see a list of all possible commands.
    
 
    rclone rc commands parameter [flags]
    
 
    Options
    
 
      -a, --arg stringArray   Argument placed in the "arg" array
@@ -2190,14 +2236,14 @@ 
    SEE ALSO
    
 
 
    rclone rcat
    
 
    Copies standard input to file on remote.
    
-
    Synopsis
    
+
    Synopsis
    
 
    rclone rcat reads from standard input (stdin) and copies it to a single remote file.
    
 
    echo "hello world" | rclone rcat remote:path/to/file
 ffmpeg - | rclone rcat remote:path/to/file
    
 
    If the remote file already exists, it will be overwritten.
    
 
    rcat will try to upload small files in a single request, which is usually more efficient than the streaming/chunked upload endpoints, which use multiple requests. Exact behaviour depends on the remote. What is considered a small file may be set through --streaming-upload-cutoff. Uploading only starts after the cutoff is reached or if the file ends before that. The data must fit into RAM. The cutoff needs to be small enough to adhere the limits of your remote, please see there. Generally speaking, setting this cutoff too high will decrease your performance.
    
-
    Use the |--size| flag to preallocate the file in advance at the remote end and actually stream it, even if remote backend doesn't support streaming.
    
-
    |--size| should be the exact size of the input stream in bytes. If the size of the stream is different in length to the |--size| passed in then the transfer will likely fail.
    
+
    Use the --size flag to preallocate the file in advance at the remote end and actually stream it, even if remote backend doesn't support streaming.
    
+
    --size should be the exact size of the input stream in bytes. If the size of the stream is different in length to the --size passed in then the transfer will likely fail.
    
 
    Note that the upload can also not be retried because the data is not kept around until the upload succeeds. If you need to transfer a lot of data, you're better off caching locally and then rclone move it to the destination.
    
 
    rclone rcat remote:path [flags]
    
 
    Options
    
@@ -2210,7 +2256,7 @@ 
    SEE ALSO
    
 
 
    rclone rcd
    
 
    Run rclone listening to remote control commands only.
    
-
    Synopsis
    
+
    Synopsis
    
 
    This runs rclone so that it only listens to remote control commands.
    
 
    This is useful if you are controlling rclone via the rc API.
    
 
    If you pass in a path to a directory, rclone will serve that directory for GET requests on the URL passed in. It will also open the URL in the browser when rclone is run.
    
@@ -2225,11 +2271,11 @@ 
    SEE ALSO
    
 
 
    rclone rmdirs
    
 
    Remove empty directories under the path.
    
-
    Synopsis
    
+
    Synopsis
    
 
    This recursively removes any empty directories (including directories that only contain empty directories), that it finds under the path. The root path itself will also be removed if it is empty, unless you supply the --leave-root flag.
    
-
    Use command rmdir to delete just the empty directory given by path, not recurse.
    
-
    This is useful for tidying up remotes that rclone has left a lot of empty directories in. For example the delete command will delete files but leave the directory structure (unless used with option --rmdirs).
    
-
    To delete a path and any objects in it, use purge command.
    
+
    Use command rmdir to delete just the empty directory given by path, not recurse.
    
+
    This is useful for tidying up remotes that rclone has left a lot of empty directories in. For example the delete command will delete files but leave the directory structure (unless used with option --rmdirs).
    
+
    To delete a path and any objects in it, use purge command.
    
 
    rclone rmdirs remote:path [flags]
    
 
    Options
    
 
      -h, --help         help for rmdirs
@@ -2241,7 +2287,7 @@ 
    SEE ALSO
    
 
 
    rclone selfupdate
    
 
    Update the rclone binary.
    
-
    Synopsis
    
+
    Synopsis
    
 
    This command downloads the latest release of rclone and replaces the currently running binary. The download is verified with a hashsum and cryptographically signed signature.
    
 
    If used without flags (or with implied --stable flag), this command will install the latest stable release. However, some issues may be fixed (or features added) only in the latest beta release. In such cases you should run the command with the --beta flag, i.e. rclone selfupdate --beta. You can check in advance what version would be installed by adding the --check flag, then repeat the command without it when you are satisfied.
    
 
    Sometimes the rclone team may recommend you a concrete beta or stable rclone release to troubleshoot your issue or add a bleeding edge feature. The --version VER flag, if given, will update to the concrete version instead of the latest one. If you omit micro version from VER (for example 1.53), the latest matching micro version will be used.
    
@@ -2266,8 +2312,8 @@ 
    SEE ALSO
    
 
 
    rclone serve
    
 
    Serve a remote over a protocol.
    
-
    Synopsis
    
-
    rclone serve is used to serve a remote over a given protocol. This command requires the use of a subcommand to specify the protocol, e.g.
    
+
    Synopsis
    
+
    Serve a remote over a given protocol. Requires the use of a subcommand to specify the protocol, e.g.
    
 
    rclone serve http remote:
    
 
    Each subcommand has its own options which you can see in their help.
    
 
    rclone serve <protocol> [opts] <remote> [flags]
    
@@ -2283,12 +2329,12 @@ 
    SEE ALSO
    
 
  • rclone serve http - Serve the remote over HTTP.
    • 
 
  • rclone serve restic - Serve the remote for restic's REST API.
    • 
 
  • rclone serve sftp - Serve the remote over SFTP.
    • 
-
  • rclone serve webdav - Serve remote:path over webdav.
    • 
+
  • rclone serve webdav - Serve remote:path over WebDAV.
    • 
 
 
    rclone serve dlna
    
 
    Serve remote:path over DLNA
    
-
    Synopsis
    
-
    rclone serve dlna is a DLNA media server for media stored in an rclone remote. Many devices, such as the Xbox and PlayStation, can automatically discover this server in the LAN and play audio/video from it. VLC is also supported. Service discovery uses UDP multicast packets (SSDP) and will thus only work on LANs.
    
+
    Synopsis
    
+
    Run a DLNA media server for media stored in an rclone remote. Many devices, such as the Xbox and PlayStation, can automatically discover this server in the LAN and play audio/video from it. VLC is also supported. Service discovery uses UDP multicast packets (SSDP) and will thus only work on LANs.
    
 
    Rclone will list all files present in the remote, without filtering based on media formats or file extensions. Additionally, there is no media transcoding support. This means that some players might show files that they are not able to play back correctly.
    
 
    Server options
    
 
    Use --addr to specify which IP address and port the server should listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs.
    
@@ -2299,7 +2345,7 @@ 
    VFS - Virtual File System
    
 
    Cloud storage objects have lots of properties which aren't like disk files - you can't extend them or write to the middle of them, so the VFS layer has to deal with that. Because there is no one right way of doing this there are various options explained below.
    
 
    The VFS layer also implements a directory cache - this caches info about files and directories (but not the data) in memory.
    
 
    VFS Directory Cache
    
-
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the mount will appear immediately or invalidate the cache.
    
+
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the VFS will appear immediately or invalidate the cache.
    
 
    --dir-cache-time duration   Time to cache directory entries for (default 5m0s)
 --poll-interval duration    Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable (default 1m0s)
    
 
    However, changes made directly on the cloud storage by the web interface or a different copy of rclone will only be picked up once the directory cache expires if the backend configured does not support polling for changes. If the backend supports polling, changes will be picked up within the polling interval.
    
@@ -2362,6 +2408,19 @@ 
    --vfs-cache-mode full
    
 
    When reading a file rclone will read --buffer-size plus --vfs-read-ahead bytes ahead. The --buffer-size is buffered in memory whereas the --vfs-read-ahead is buffered on disk.
    
 
    When using this mode it is recommended that --buffer-size is not set too large and --vfs-read-ahead is set large if required.
    
 
    IMPORTANT not all file systems support sparse files. In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected.
    
+
    Fingerprinting
    
+
    Various parts of the VFS use fingerprinting to see if a local file copy has changed relative to a remote file. Fingerprints are made from:
    
+
      
+
    • size
      • 
+
    • modification time
      • 
+
    • hash
      • 
+
    
+
    where available on an object.
    
+
    On some backends some of these attributes are slow to read (they take an extra API call per object, or extra work per object).
    
+
    For example hash is slow with the local and sftp backends as they have to read the entire file and hash it, and modtime is slow with the s3, swift, ftp and qinqstor backends because they need to do an extra API call to fetch it.
    
+
    If you use the --vfs-fast-fingerprint flag then rclone will not include the slow operations in the fingerprint. This makes the fingerprinting less accurate but much faster and will improve the opening time of cached files.
    
+
    If you are running a vfs cache over local, s3 or swift backends then using this flag is recommended.
    
+
    Note that if you change the value of this flag, the fingerprints of the files in the cache may be invalidated and the files will need to be downloaded again.
    
 
    VFS Chunked Reading
    
 
    When rclone reads files from a remote it reads them in chunks. This means that rather than requesting the whole file rclone reads the chunk specified. This can reduce the used download quota for some remotes by requesting only chunks from the remote that are actually read, at the cost of an increased number of requests.
    
 
    These flags control the chunking:
    
@@ -2376,20 +2435,23 @@ 
    VFS Performance
    
 
    --no-checksum     Don't compare checksums on up/download.
 --no-modtime      Don't read/write the modification time (can speed things up).
 --no-seek         Don't allow seeking in files.
---read-only       Mount read-only.
    
+--read-only       Only allow read-only access.
    
 
    Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write to come in. These flags only come into effect when not using an on disk cache file.
    
 
    --vfs-read-wait duration   Time to wait for in-sequence read before seeking (default 20ms)
 --vfs-write-wait duration  Time to wait for in-sequence write before giving error (default 1s)
    
-
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from cache (the related global flag --checkers have no effect on mount).
    
+
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from the cache (the related global flag --checkers has no effect on the VFS).
    
 
    --transfers int  Number of file transfers to run in parallel (default 4)
    
 
    VFS Case Sensitivity
    
 
    Linux file systems are case-sensitive: two files can differ only by case, and the exact case must be used when opening a file.
    
 
    File systems in modern Windows are case-insensitive but case-preserving: although existing files can be opened using any case, the exact case used to create the file is preserved and available for programs to query. It is not allowed for two files in the same directory to differ only by case.
    
 
    Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default.
    
-
    The --vfs-case-insensitive mount flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the mounted file system as-is. If the flag is "true" (or appears without a value on command line), rclone may perform a "fixup" as explained below.
    
-
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on mounted file system. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by an underlying mounted file system.
    
-
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system mounted by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
+
    The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the remote as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below.
    
+
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by the underlying remote.
    
+
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
 
    If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true".
    
+
    VFS Disk Options
    
+
    This flag allows you to manually set the statistics about the filing system. It can be useful when those statistics cannot be read correctly automatically.
    
+
    --vfs-disk-space-total-size    Manually set the total disk space size (example: 256G, default: -1)
    
 
    Alternate report of used bytes
    
 
    Some backends, most notably S3, do not report the amount of bytes used. If you need this information to be available when running df on the filesystem, then pass the flag --vfs-used-is-size to rclone. With this flag set, instead of relying on the backend to report this information, rclone will scan the whole remote similar to rclone size and compute the total used space itself.
    
 
    WARNING. Contrary to rclone size, this flag ignores filters so that the result is accurate. However, this is very inefficient and may cost lots of API calls resulting in extra charges. Use it as a last resort and only with caching.
    
@@ -2407,7 +2469,7 @@ 
    Options
    
       --no-modtime                             Don't read/write the modification time (can speed things up)
       --no-seek                                Don't allow seeking in files
       --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
-      --read-only                              Mount read-only
+      --read-only                              Only allow read-only access
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
       --vfs-cache-max-age duration             Max age of objects in the cache (default 1h0m0s)
@@ -2415,6 +2477,8 @@ 
    Options
    
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
       --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
+      --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
+      --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
@@ -2429,7 +2493,7 @@ 
    SEE ALSO
    
 
 
    rclone serve docker
    
 
    Serve any remote on docker's volume plugin API.
    
-
    Synopsis
    
+
    Synopsis
    
 
    This command implements the Docker volume plugin API allowing docker to use rclone as a data storage mechanism for various cloud providers. rclone provides docker volume plugin based on it.
    
 
    To create a docker plugin, one must create a Unix or TCP socket that Docker will look for when you use the plugin and then it listens for commands from docker daemon and runs the corresponding code when necessary. Docker plugins can run as a managed plugin under control of the docker daemon or as an independent native service. For testing, you can just run it directly from the command line, for example:
    
 
    sudo rclone serve docker --base-dir /tmp/rclone-volumes --socket-addr localhost:8787 -vv
    
@@ -2442,7 +2506,7 @@ 
    VFS - Virtual File System
    
 
    Cloud storage objects have lots of properties which aren't like disk files - you can't extend them or write to the middle of them, so the VFS layer has to deal with that. Because there is no one right way of doing this there are various options explained below.
    
 
    The VFS layer also implements a directory cache - this caches info about files and directories (but not the data) in memory.
    
 
    VFS Directory Cache
    
-
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the mount will appear immediately or invalidate the cache.
    
+
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the VFS will appear immediately or invalidate the cache.
    
 
    --dir-cache-time duration   Time to cache directory entries for (default 5m0s)
 --poll-interval duration    Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable (default 1m0s)
    
 
    However, changes made directly on the cloud storage by the web interface or a different copy of rclone will only be picked up once the directory cache expires if the backend configured does not support polling for changes. If the backend supports polling, changes will be picked up within the polling interval.
    
@@ -2505,6 +2569,19 @@ 
    --vfs-cache-mode full
    
 
    When reading a file rclone will read --buffer-size plus --vfs-read-ahead bytes ahead. The --buffer-size is buffered in memory whereas the --vfs-read-ahead is buffered on disk.
    
 
    When using this mode it is recommended that --buffer-size is not set too large and --vfs-read-ahead is set large if required.
    
 
    IMPORTANT not all file systems support sparse files. In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected.
    
+
    Fingerprinting
    
+
    Various parts of the VFS use fingerprinting to see if a local file copy has changed relative to a remote file. Fingerprints are made from:
    
+
      
+
    • size
      • 
+
    • modification time
      • 
+
    • hash
      • 
+
    
+
    where available on an object.
    
+
    On some backends some of these attributes are slow to read (they take an extra API call per object, or extra work per object).
    
+
    For example hash is slow with the local and sftp backends as they have to read the entire file and hash it, and modtime is slow with the s3, swift, ftp and qinqstor backends because they need to do an extra API call to fetch it.
    
+
    If you use the --vfs-fast-fingerprint flag then rclone will not include the slow operations in the fingerprint. This makes the fingerprinting less accurate but much faster and will improve the opening time of cached files.
    
+
    If you are running a vfs cache over local, s3 or swift backends then using this flag is recommended.
    
+
    Note that if you change the value of this flag, the fingerprints of the files in the cache may be invalidated and the files will need to be downloaded again.
    
 
    VFS Chunked Reading
    
 
    When rclone reads files from a remote it reads them in chunks. This means that rather than requesting the whole file rclone reads the chunk specified. This can reduce the used download quota for some remotes by requesting only chunks from the remote that are actually read, at the cost of an increased number of requests.
    
 
    These flags control the chunking:
    
@@ -2519,20 +2596,23 @@ 
    VFS Performance
    
 
    --no-checksum     Don't compare checksums on up/download.
 --no-modtime      Don't read/write the modification time (can speed things up).
 --no-seek         Don't allow seeking in files.
---read-only       Mount read-only.
    
+--read-only       Only allow read-only access.
    
 
    Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write to come in. These flags only come into effect when not using an on disk cache file.
    
 
    --vfs-read-wait duration   Time to wait for in-sequence read before seeking (default 20ms)
 --vfs-write-wait duration  Time to wait for in-sequence write before giving error (default 1s)
    
-
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from cache (the related global flag --checkers have no effect on mount).
    
+
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from the cache (the related global flag --checkers has no effect on the VFS).
    
 
    --transfers int  Number of file transfers to run in parallel (default 4)
    
 
    VFS Case Sensitivity
    
 
    Linux file systems are case-sensitive: two files can differ only by case, and the exact case must be used when opening a file.
    
 
    File systems in modern Windows are case-insensitive but case-preserving: although existing files can be opened using any case, the exact case used to create the file is preserved and available for programs to query. It is not allowed for two files in the same directory to differ only by case.
    
 
    Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default.
    
-
    The --vfs-case-insensitive mount flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the mounted file system as-is. If the flag is "true" (or appears without a value on command line), rclone may perform a "fixup" as explained below.
    
-
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on mounted file system. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by an underlying mounted file system.
    
-
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system mounted by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
+
    The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the remote as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below.
    
+
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by the underlying remote.
    
+
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
 
    If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true".
    
+
    VFS Disk Options
    
+
    This flag allows you to manually set the statistics about the filing system. It can be useful when those statistics cannot be read correctly automatically.
    
+
    --vfs-disk-space-total-size    Manually set the total disk space size (example: 256G, default: -1)
    
 
    Alternate report of used bytes
    
 
    Some backends, most notably S3, do not report the amount of bytes used. If you need this information to be available when running df on the filesystem, then pass the flag --vfs-used-is-size to rclone. With this flag set, instead of relying on the backend to report this information, rclone will scan the whole remote similar to rclone size and compute the total used space itself.
    
 
    WARNING. Contrary to rclone size, this flag ignores filters so that the result is accurate. However, this is very inefficient and may cost lots of API calls resulting in extra charges. Use it as a last resort and only with caching.
    
@@ -2567,7 +2647,7 @@ 
    Options
    
       --noapplexattr                           Ignore all "com.apple.*" extended attributes (supported on OSX only)
   -o, --option stringArray                     Option for libfuse/WinFsp (repeat if required)
       --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
-      --read-only                              Mount read-only
+      --read-only                              Only allow read-only access
       --socket-addr string                     Address <host:port> or absolute path (default: /run/docker/plugins/rclone.sock)
       --socket-gid int                         GID for unix socket (default: current process GID) (default 1000)
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
@@ -2577,6 +2657,8 @@ 
    Options
    
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
       --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
+      --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
+      --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
@@ -2593,8 +2675,8 @@ 
    SEE ALSO
    
 
 
    rclone serve ftp
    
 
    Serve remote:path over FTP.
    
-
    Synopsis
    
-
    rclone serve ftp implements a basic ftp server to serve the remote over FTP protocol. This can be viewed with a ftp client or you can make a remote of type ftp to read and write it.
    
+
    Synopsis
    
+
    Run a basic FTP server to serve a remote over FTP protocol. This can be viewed with a FTP client or you can make a remote of type FTP to read and write it.
    
 
    Server options
    
 
    Use --addr to specify which IP address and port the server should listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.
    
 
    If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.
    
@@ -2606,7 +2688,7 @@ 
    VFS - Virtual File System
    
 
    Cloud storage objects have lots of properties which aren't like disk files - you can't extend them or write to the middle of them, so the VFS layer has to deal with that. Because there is no one right way of doing this there are various options explained below.
    
 
    The VFS layer also implements a directory cache - this caches info about files and directories (but not the data) in memory.
    
 
    VFS Directory Cache
    
-
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the mount will appear immediately or invalidate the cache.
    
+
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the VFS will appear immediately or invalidate the cache.
    
 
    --dir-cache-time duration   Time to cache directory entries for (default 5m0s)
 --poll-interval duration    Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable (default 1m0s)
    
 
    However, changes made directly on the cloud storage by the web interface or a different copy of rclone will only be picked up once the directory cache expires if the backend configured does not support polling for changes. If the backend supports polling, changes will be picked up within the polling interval.
    
@@ -2669,6 +2751,19 @@ 
    --vfs-cache-mode full
    
 
    When reading a file rclone will read --buffer-size plus --vfs-read-ahead bytes ahead. The --buffer-size is buffered in memory whereas the --vfs-read-ahead is buffered on disk.
    
 
    When using this mode it is recommended that --buffer-size is not set too large and --vfs-read-ahead is set large if required.
    
 
    IMPORTANT not all file systems support sparse files. In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected.
    
+
    Fingerprinting
    
+
    Various parts of the VFS use fingerprinting to see if a local file copy has changed relative to a remote file. Fingerprints are made from:
    
+
      
+
    • size
      • 
+
    • modification time
      • 
+
    • hash
      • 
+
    
+
    where available on an object.
    
+
    On some backends some of these attributes are slow to read (they take an extra API call per object, or extra work per object).
    
+
    For example hash is slow with the local and sftp backends as they have to read the entire file and hash it, and modtime is slow with the s3, swift, ftp and qinqstor backends because they need to do an extra API call to fetch it.
    
+
    If you use the --vfs-fast-fingerprint flag then rclone will not include the slow operations in the fingerprint. This makes the fingerprinting less accurate but much faster and will improve the opening time of cached files.
    
+
    If you are running a vfs cache over local, s3 or swift backends then using this flag is recommended.
    
+
    Note that if you change the value of this flag, the fingerprints of the files in the cache may be invalidated and the files will need to be downloaded again.
    
 
    VFS Chunked Reading
    
 
    When rclone reads files from a remote it reads them in chunks. This means that rather than requesting the whole file rclone reads the chunk specified. This can reduce the used download quota for some remotes by requesting only chunks from the remote that are actually read, at the cost of an increased number of requests.
    
 
    These flags control the chunking:
    
@@ -2683,20 +2778,23 @@ 
    VFS Performance
    
 
    --no-checksum     Don't compare checksums on up/download.
 --no-modtime      Don't read/write the modification time (can speed things up).
 --no-seek         Don't allow seeking in files.
---read-only       Mount read-only.
    
+--read-only       Only allow read-only access.
    
 
    Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write to come in. These flags only come into effect when not using an on disk cache file.
    
 
    --vfs-read-wait duration   Time to wait for in-sequence read before seeking (default 20ms)
 --vfs-write-wait duration  Time to wait for in-sequence write before giving error (default 1s)
    
-
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from cache (the related global flag --checkers have no effect on mount).
    
+
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from the cache (the related global flag --checkers has no effect on the VFS).
    
 
    --transfers int  Number of file transfers to run in parallel (default 4)
    
 
    VFS Case Sensitivity
    
 
    Linux file systems are case-sensitive: two files can differ only by case, and the exact case must be used when opening a file.
    
 
    File systems in modern Windows are case-insensitive but case-preserving: although existing files can be opened using any case, the exact case used to create the file is preserved and available for programs to query. It is not allowed for two files in the same directory to differ only by case.
    
 
    Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default.
    
-
    The --vfs-case-insensitive mount flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the mounted file system as-is. If the flag is "true" (or appears without a value on command line), rclone may perform a "fixup" as explained below.
    
-
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on mounted file system. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by an underlying mounted file system.
    
-
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system mounted by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
+
    The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the remote as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below.
    
+
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by the underlying remote.
    
+
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
 
    If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true".
    
+
    VFS Disk Options
    
+
    This flag allows you to manually set the statistics about the filing system. It can be useful when those statistics cannot be read correctly automatically.
    
+
    --vfs-disk-space-total-size    Manually set the total disk space size (example: 256G, default: -1)
    
 
    Alternate report of used bytes
    
 
    Some backends, most notably S3, do not report the amount of bytes used. If you need this information to be available when running df on the filesystem, then pass the flag --vfs-used-is-size to rclone. With this flag set, instead of relying on the backend to report this information, rclone will scan the whole remote similar to rclone size and compute the total used space itself.
    
 
    WARNING. Contrary to rclone size, this flag ignores filters so that the result is accurate. However, this is very inefficient and may cost lots of API calls resulting in extra charges. Use it as a last resort and only with caching.
    
@@ -2748,7 +2846,7 @@ 
    Options
    
       --passive-port string                    Passive port range to use (default "30000-32000")
       --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --public-ip string                       Public IP address to advertise for passive connections
-      --read-only                              Mount read-only
+      --read-only                              Only allow read-only access
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
       --user string                            User name for authentication (default "anonymous")
@@ -2757,6 +2855,8 @@ 
    Options
    
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
       --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
+      --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
+      --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
@@ -2771,22 +2871,22 @@ 
    SEE ALSO
    
 
 
    rclone serve http
    
 
    Serve the remote over HTTP.
    
-
    Synopsis
    
-
    rclone serve http implements a basic web server to serve the remote over HTTP. This can be viewed in a web browser or you can make a remote of type http read from it.
    
-
    You can use the filter flags (e.g. --include, --exclude) to control what is served.
    
-
    The server will log errors. Use -v to see access logs.
    
-
    --bwlimit will be respected for file transfers. Use --stats to control the stats printing.
    
+
    Synopsis
    
+
    Run a basic web server to serve a remote over HTTP. This can be viewed in a web browser or you can make a remote of type http read from it.
    
+
    You can use the filter flags (e.g. --include, --exclude) to control what is served.
    
+
    The server will log errors. Use -v to see access logs.
    
+
    --bwlimit will be respected for file transfers. Use --stats to control the stats printing.
    
 
    Server options
    
-
    Use --addr to specify which IP address and port the server should listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.
    
-
    If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.
    
-
    --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.
    
-
    --max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.
    
-
    --baseurl controls the URL prefix that rclone serves from. By default rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically.
    
+
    Use --addr to specify which IP address and port the server should listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.
    
+
    If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.
    
+
    --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.
    
+
    --max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.
    
+
    --baseurl controls the URL prefix that rclone serves from. By default rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically.
    
 
    SSL/TLS
    
-
    By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.
    
-
    --cert should be a either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.
    
+
    By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.
    
+
    --cert should be a either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.
    
 
    Template
    
-
    --template allows a user to specify a custom markup template for http and webdav serve functions. The server exports the following markup to be used within the template to server pages:
    
+
    --template allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages:
    
 @@ -2867,21 +2967,21 @@ 
    Template
    
 
 

 
    
 
    Authentication
    
 
    By default this will serve files without needing a login.
    
-
    You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.
    
-
    Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.
    
+
    You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.
    
+
    Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.
    
 
    To create an htpasswd file:
    
 
    touch htpasswd
 htpasswd -B htpasswd user
 htpasswd -B htpasswd anotherUser
    
 
    The password file can be updated while rclone is running.
    
-
    Use --realm to set the authentication realm.
    
-
    Use --salt to change the password hashing salt from the default.
    
+
    Use --realm to set the authentication realm.
    
+
    Use --salt to change the password hashing salt from the default.
    
 
    VFS - Virtual File System
    
 
    This command uses the VFS layer. This adapts the cloud storage objects that rclone uses into something which looks much more like a disk filing system.
    
 
    Cloud storage objects have lots of properties which aren't like disk files - you can't extend them or write to the middle of them, so the VFS layer has to deal with that. Because there is no one right way of doing this there are various options explained below.
    
 
    The VFS layer also implements a directory cache - this caches info about files and directories (but not the data) in memory.
    
 
    VFS Directory Cache
    
-
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the mount will appear immediately or invalidate the cache.
    
+
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the VFS will appear immediately or invalidate the cache.
    
 
    --dir-cache-time duration   Time to cache directory entries for (default 5m0s)
 --poll-interval duration    Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable (default 1m0s)
    
 
    However, changes made directly on the cloud storage by the web interface or a different copy of rclone will only be picked up once the directory cache expires if the backend configured does not support polling for changes. If the backend supports polling, changes will be picked up within the polling interval.
    
@@ -2944,6 +3044,19 @@ 
    --vfs-cache-mode full
    
 
    When reading a file rclone will read --buffer-size plus --vfs-read-ahead bytes ahead. The --buffer-size is buffered in memory whereas the --vfs-read-ahead is buffered on disk.
    
 
    When using this mode it is recommended that --buffer-size is not set too large and --vfs-read-ahead is set large if required.
    
 
    IMPORTANT not all file systems support sparse files. In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected.
    
+
    Fingerprinting
    
+
    Various parts of the VFS use fingerprinting to see if a local file copy has changed relative to a remote file. Fingerprints are made from:
    
+
      
+
    • size
      • 
+
    • modification time
      • 
+
    • hash
      • 
+
    
+
    where available on an object.
    
+
    On some backends some of these attributes are slow to read (they take an extra API call per object, or extra work per object).
    
+
    For example hash is slow with the local and sftp backends as they have to read the entire file and hash it, and modtime is slow with the s3, swift, ftp and qinqstor backends because they need to do an extra API call to fetch it.
    
+
    If you use the --vfs-fast-fingerprint flag then rclone will not include the slow operations in the fingerprint. This makes the fingerprinting less accurate but much faster and will improve the opening time of cached files.
    
+
    If you are running a vfs cache over local, s3 or swift backends then using this flag is recommended.
    
+
    Note that if you change the value of this flag, the fingerprints of the files in the cache may be invalidated and the files will need to be downloaded again.
    
 
    VFS Chunked Reading
    
 
    When rclone reads files from a remote it reads them in chunks. This means that rather than requesting the whole file rclone reads the chunk specified. This can reduce the used download quota for some remotes by requesting only chunks from the remote that are actually read, at the cost of an increased number of requests.
    
 
    These flags control the chunking:
    
@@ -2958,20 +3071,23 @@ 
    VFS Performance
    
 
    --no-checksum     Don't compare checksums on up/download.
 --no-modtime      Don't read/write the modification time (can speed things up).
 --no-seek         Don't allow seeking in files.
---read-only       Mount read-only.
    
+--read-only       Only allow read-only access.
    
 
    Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write to come in. These flags only come into effect when not using an on disk cache file.
    
 
    --vfs-read-wait duration   Time to wait for in-sequence read before seeking (default 20ms)
 --vfs-write-wait duration  Time to wait for in-sequence write before giving error (default 1s)
    
-
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from cache (the related global flag --checkers have no effect on mount).
    
+
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from the cache (the related global flag --checkers has no effect on the VFS).
    
 
    --transfers int  Number of file transfers to run in parallel (default 4)
    
 
    VFS Case Sensitivity
    
 
    Linux file systems are case-sensitive: two files can differ only by case, and the exact case must be used when opening a file.
    
 
    File systems in modern Windows are case-insensitive but case-preserving: although existing files can be opened using any case, the exact case used to create the file is preserved and available for programs to query. It is not allowed for two files in the same directory to differ only by case.
    
 
    Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default.
    
-
    The --vfs-case-insensitive mount flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the mounted file system as-is. If the flag is "true" (or appears without a value on command line), rclone may perform a "fixup" as explained below.
    
-
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on mounted file system. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by an underlying mounted file system.
    
-
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system mounted by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
+
    The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the remote as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below.
    
+
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by the underlying remote.
    
+
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
 
    If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true".
    
+
    VFS Disk Options
    
+
    This flag allows you to manually set the statistics about the filing system. It can be useful when those statistics cannot be read correctly automatically.
    
+
    --vfs-disk-space-total-size    Manually set the total disk space size (example: 256G, default: -1)
    
 
    Alternate report of used bytes
    
 
    Some backends, most notably S3, do not report the amount of bytes used. If you need this information to be available when running df on the filesystem, then pass the flag --vfs-used-is-size to rclone. With this flag set, instead of relying on the backend to report this information, rclone will scan the whole remote similar to rclone size and compute the total used space itself.
    
 
    WARNING. Contrary to rclone size, this flag ignores filters so that the result is accurate. However, this is very inefficient and may cost lots of API calls resulting in extra charges. Use it as a last resort and only with caching.
    
@@ -2994,7 +3110,7 @@ 
    Options
    
       --no-seek                                Don't allow seeking in files
       --pass string                            Password for authentication
       --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
-      --read-only                              Mount read-only
+      --read-only                              Only allow read-only access
       --realm string                           Realm for authentication
       --salt string                            Password hashing salt (default "dlPL2MqE")
       --server-read-timeout duration           Timeout for server reading data (default 1h0m0s)
@@ -3008,6 +3124,8 @@ 
    Options
    
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
       --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
+      --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
+      --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
@@ -3022,20 +3140,20 @@ 
    SEE ALSO
    
 
 
    rclone serve restic
    
 
    Serve the remote for restic's REST API.
    
-
    Synopsis
    
-
    rclone serve restic implements restic's REST backend API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly.
    
+
    Synopsis
    
+
    Run a basic web server to serve a remove over restic's REST backend API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly.
    
 
    Restic is a command-line program for doing backups.
    
 
    The server will log errors. Use -v to see access logs.
    
-
    --bwlimit will be respected for file transfers. Use --stats to control the stats printing.
    
+
    --bwlimit will be respected for file transfers. Use --stats to control the stats printing.
    
 
    Setting up rclone for use by restic
    
 
    First set up a remote for your chosen cloud provider.
    
 
    Once you have set up the remote, check it is working with, for example "rclone lsd remote:". You may have called the remote something other than "remote:" - just substitute whatever you called it in the following instructions.
    
 
    Now start the rclone restic server
    
 
    rclone serve restic -v remote:backup
    
 
    Where you can replace "backup" in the above by whatever path in the remote you wish to use.
    
-
    By default this will serve on "localhost:8080" you can change this with use of the "--addr" flag.
    
+
    By default this will serve on "localhost:8080" you can change this with use of the --addr flag.
    
 
    You might wish to start this server on boot.
    
-
    Adding --cache-objects=false will cause rclone to stop caching objects returned from the List call. Caching is normally desirable as it speeds up downloading objects, saves transactions and uses very little memory.
    
+
    Adding --cache-objects=false will cause rclone to stop caching objects returned from the List call. Caching is normally desirable as it speeds up downloading objects, saves transactions and uses very little memory.
    
 
    Setting up restic to use rclone
    
 
    Now you can follow the restic instructions on setting up restic.
    
 
    Note that you will need restic 0.8.2 or later to interoperate with rclone.
    
@@ -3062,14 +3180,14 @@ 
    Multiple repositories
    
 $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user2repo/
 # backup user2 stuff
    
 
    Private repositories
    
-
    The "--private-repos" flag can be used to limit users to repositories starting with a path of /<username>/.
    
+
    The--private-repos flag can be used to limit users to repositories starting with a path of /<username>/.
    
 
    Server options
    
-
    Use --addr to specify which IP address and port the server should listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.
    
-
    If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.
    
-
    --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.
    
-
    --max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.
    
-
    --baseurl controls the URL prefix that rclone serves from. By default rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically.
    
-
    --template allows a user to specify a custom markup template for http and webdav serve functions. The server exports the following markup to be used within the template to server pages:
    
+
    Use --addr to specify which IP address and port the server should listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.
    
+
    If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.
    
+
    --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.
    
+
    --max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.
    
+
    --baseurl controls the URL prefix that rclone serves from. By default rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically.
    
+
    --template allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages:
    
 @@ -3150,17 +3268,17 @@ 
    Server options
    
 
 

 
    
 
    Authentication
    
 
    By default this will serve files without needing a login.
    
-
    You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.
    
-
    Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.
    
+
    You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.
    
+
    Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.
    
 
    To create an htpasswd file:
    
 
    touch htpasswd
 htpasswd -B htpasswd user
 htpasswd -B htpasswd anotherUser
    
 
    The password file can be updated while rclone is running.
    
-
    Use --realm to set the authentication realm.
    
+
    Use --realm to set the authentication realm.
    
 
    SSL/TLS
    
-
    By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.
    
-
    --cert should be either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.
    
+
    By default this will serve over HTTP. If you want you can serve over HTTPS. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.
    
+
    --cert should be either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.
    
 
    rclone serve restic remote:path [flags]
    
 
    Options
    
 
          --addr string                     IPaddress:Port or :Port to bind server to (default "localhost:8080")
@@ -3188,26 +3306,26 @@ 
    SEE ALSO
    
 
 
    rclone serve sftp
    
 
    Serve the remote over SFTP.
    
-
    Synopsis
    
-
    rclone serve sftp implements an SFTP server to serve the remote over SFTP. This can be used with an SFTP client or you can make a remote of type sftp to use with it.
    
-
    You can use the filter flags (e.g. --include, --exclude) to control what is served.
    
-
    The server will log errors. Use -v to see access logs.
    
-
    --bwlimit will be respected for file transfers. Use --stats to control the stats printing.
    
-
    You must provide some means of authentication, either with --user/--pass, an authorized keys file (specify location with --authorized-keys - the default is the same as ssh), an --auth-proxy, or set the --no-auth flag for no authentication when logging in.
    
+
    Synopsis
    
+
    Run a SFTP server to serve a remote over SFTP. This can be used with an SFTP client or you can make a remote of type sftp to use with it.
    
+
    You can use the filter flags (e.g. --include, --exclude) to control what is served.
    
+
    The server will log errors. Use -v to see access logs.
    
+
    --bwlimit will be respected for file transfers. Use --stats to control the stats printing.
    
+
    You must provide some means of authentication, either with --user/--pass, an authorized keys file (specify location with --authorized-keys - the default is the same as ssh), an --auth-proxy, or set the --no-auth flag for no authentication when logging in.
    
 
    Note that this also implements a small number of shell commands so that it can provide md5sum/sha1sum/df information for the rclone sftp backend. This means that is can support SHA1SUMs, MD5SUMs and the about command when paired with the rclone sftp backend.
    
-
    If you don't supply a host --key then rclone will generate rsa, ecdsa and ed25519 variants, and cache them for later use in rclone's cache directory (see "rclone help flags cache-dir") in the "serve-sftp" directory.
    
-
    By default the server binds to localhost:2022 - if you want it to be reachable externally then supply "--addr :2022" for example.
    
-
    Note that the default of "--vfs-cache-mode off" is fine for the rclone sftp backend, but it may not be with other SFTP clients.
    
-
    If --stdio is specified, rclone will serve SFTP over stdio, which can be used with sshd via ~/.ssh/authorized_keys, for example:
    
+
    If you don't supply a host --key then rclone will generate rsa, ecdsa and ed25519 variants, and cache them for later use in rclone's cache directory (see rclone help flags cache-dir) in the "serve-sftp" directory.
    
+
    By default the server binds to localhost:2022 - if you want it to be reachable externally then supply --addr :2022 for example.
    
+
    Note that the default of --vfs-cache-mode off is fine for the rclone sftp backend, but it may not be with other SFTP clients.
    
+
    If --stdio is specified, rclone will serve SFTP over stdio, which can be used with sshd via ~/.ssh/authorized_keys, for example:
    
 
    restrict,command="rclone serve sftp --stdio ./photos" ssh-rsa ...
    
-
    On the client you need to set "--transfers 1" when using --stdio. Otherwise multiple instances of the rclone server are started by OpenSSH which can lead to "corrupted on transfer" errors. This is the case because the client chooses indiscriminately which server to send commands to while the servers all have different views of the state of the filing system.
    
-
    The "restrict" in authorized_keys prevents SHA1SUMs and MD5SUMs from beeing used. Omitting "restrict" and using --sftp-path-override to enable checksumming is possible but less secure and you could use the SFTP server provided by OpenSSH in this case.
    
+
    On the client you need to set --transfers 1 when using --stdio. Otherwise multiple instances of the rclone server are started by OpenSSH which can lead to "corrupted on transfer" errors. This is the case because the client chooses indiscriminately which server to send commands to while the servers all have different views of the state of the filing system.
    
+
    The "restrict" in authorized_keys prevents SHA1SUMs and MD5SUMs from beeing used. Omitting "restrict" and using --sftp-path-override to enable checksumming is possible but less secure and you could use the SFTP server provided by OpenSSH in this case.
    
 
    VFS - Virtual File System
    
 
    This command uses the VFS layer. This adapts the cloud storage objects that rclone uses into something which looks much more like a disk filing system.
    
 
    Cloud storage objects have lots of properties which aren't like disk files - you can't extend them or write to the middle of them, so the VFS layer has to deal with that. Because there is no one right way of doing this there are various options explained below.
    
 
    The VFS layer also implements a directory cache - this caches info about files and directories (but not the data) in memory.
    
 
    VFS Directory Cache
    
-
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the mount will appear immediately or invalidate the cache.
    
+
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the VFS will appear immediately or invalidate the cache.
    
 
    --dir-cache-time duration   Time to cache directory entries for (default 5m0s)
 --poll-interval duration    Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable (default 1m0s)
    
 
    However, changes made directly on the cloud storage by the web interface or a different copy of rclone will only be picked up once the directory cache expires if the backend configured does not support polling for changes. If the backend supports polling, changes will be picked up within the polling interval.
    
@@ -3270,6 +3388,19 @@ 
    --vfs-cache-mode full
    
 
    When reading a file rclone will read --buffer-size plus --vfs-read-ahead bytes ahead. The --buffer-size is buffered in memory whereas the --vfs-read-ahead is buffered on disk.
    
 
    When using this mode it is recommended that --buffer-size is not set too large and --vfs-read-ahead is set large if required.
    
 
    IMPORTANT not all file systems support sparse files. In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected.
    
+
    Fingerprinting
    
+
    Various parts of the VFS use fingerprinting to see if a local file copy has changed relative to a remote file. Fingerprints are made from:
    
+
      
+
    • size
      • 
+
    • modification time
      • 
+
    • hash
      • 
+
    
+
    where available on an object.
    
+
    On some backends some of these attributes are slow to read (they take an extra API call per object, or extra work per object).
    
+
    For example hash is slow with the local and sftp backends as they have to read the entire file and hash it, and modtime is slow with the s3, swift, ftp and qinqstor backends because they need to do an extra API call to fetch it.
    
+
    If you use the --vfs-fast-fingerprint flag then rclone will not include the slow operations in the fingerprint. This makes the fingerprinting less accurate but much faster and will improve the opening time of cached files.
    
+
    If you are running a vfs cache over local, s3 or swift backends then using this flag is recommended.
    
+
    Note that if you change the value of this flag, the fingerprints of the files in the cache may be invalidated and the files will need to be downloaded again.
    
 
    VFS Chunked Reading
    
 
    When rclone reads files from a remote it reads them in chunks. This means that rather than requesting the whole file rclone reads the chunk specified. This can reduce the used download quota for some remotes by requesting only chunks from the remote that are actually read, at the cost of an increased number of requests.
    
 
    These flags control the chunking:
    
@@ -3284,20 +3415,23 @@ 
    VFS Performance
    
 
    --no-checksum     Don't compare checksums on up/download.
 --no-modtime      Don't read/write the modification time (can speed things up).
 --no-seek         Don't allow seeking in files.
---read-only       Mount read-only.
    
+--read-only       Only allow read-only access.
    
 
    Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write to come in. These flags only come into effect when not using an on disk cache file.
    
 
    --vfs-read-wait duration   Time to wait for in-sequence read before seeking (default 20ms)
 --vfs-write-wait duration  Time to wait for in-sequence write before giving error (default 1s)
    
-
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from cache (the related global flag --checkers have no effect on mount).
    
+
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from the cache (the related global flag --checkers has no effect on the VFS).
    
 
    --transfers int  Number of file transfers to run in parallel (default 4)
    
 
    VFS Case Sensitivity
    
 
    Linux file systems are case-sensitive: two files can differ only by case, and the exact case must be used when opening a file.
    
 
    File systems in modern Windows are case-insensitive but case-preserving: although existing files can be opened using any case, the exact case used to create the file is preserved and available for programs to query. It is not allowed for two files in the same directory to differ only by case.
    
 
    Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default.
    
-
    The --vfs-case-insensitive mount flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the mounted file system as-is. If the flag is "true" (or appears without a value on command line), rclone may perform a "fixup" as explained below.
    
-
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on mounted file system. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by an underlying mounted file system.
    
-
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system mounted by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
+
    The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the remote as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below.
    
+
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by the underlying remote.
    
+
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
 
    If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true".
    
+
    VFS Disk Options
    
+
    This flag allows you to manually set the statistics about the filing system. It can be useful when those statistics cannot be read correctly automatically.
    
+
    --vfs-disk-space-total-size    Manually set the total disk space size (example: 256G, default: -1)
    
 
    Alternate report of used bytes
    
 
    Some backends, most notably S3, do not report the amount of bytes used. If you need this information to be available when running df on the filesystem, then pass the flag --vfs-used-is-size to rclone. With this flag set, instead of relying on the backend to report this information, rclone will scan the whole remote similar to rclone size and compute the total used space itself.
    
 
    WARNING. Contrary to rclone size, this flag ignores filters so that the result is accurate. However, this is very inefficient and may cost lots of API calls resulting in extra charges. Use it as a last resort and only with caching.
    
@@ -3348,7 +3482,7 @@ 
    Options
    
       --no-seek                                Don't allow seeking in files
       --pass string                            Password for authentication
       --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
-      --read-only                              Mount read-only
+      --read-only                              Only allow read-only access
       --stdio                                  Run an sftp server on run stdin/stdout
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
@@ -3358,6 +3492,8 @@ 
    Options
    
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
       --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
+      --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
+      --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
@@ -3371,21 +3507,20 @@ 
    SEE ALSO
    
 
  • rclone serve - Serve a remote over a protocol.
    • 
 
 
    rclone serve webdav
    
-
    Serve remote:path over webdav.
    
-
    Synopsis
    
-
    rclone serve webdav implements a basic webdav server to serve the remote over HTTP via the webdav protocol. This can be viewed with a webdav client, through a web browser, or you can make a remote of type webdav to read and write it.
    
-
    Webdav options
    
+
    Serve remote:path over WebDAV.
    
+
    Synopsis
    
+
    Run a basic WebDAV server to serve a remote over HTTP via the WebDAV protocol. This can be viewed with a WebDAV client, through a web browser, or you can make a remote of type WebDAV to read and write it.
    
+
    WebDAV options
    
 
    --etag-hash
    
 
    This controls the ETag header. Without this flag the ETag will be based on the ModTime and Size of the object.
    
-
    If this flag is set to "auto" then rclone will choose the first supported hash on the backend or you can use a named hash such as "MD5" or "SHA-1".
    
-
    Use "rclone hashsum" to see the full list.
    
+
    If this flag is set to "auto" then rclone will choose the first supported hash on the backend or you can use a named hash such as "MD5" or "SHA-1". Use the hashsum command to see the full list.
    
 
    Server options
    
-
    Use --addr to specify which IP address and port the server should listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.
    
-
    If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.
    
-
    --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.
    
-
    --max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.
    
-
    --baseurl controls the URL prefix that rclone serves from. By default rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically.
    
-
    --template allows a user to specify a custom markup template for http and webdav serve functions. The server exports the following markup to be used within the template to server pages:
    
+
    Use --addr to specify which IP address and port the server should listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.
    
+
    If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.
    
+
    --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.
    
+
    --max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.
    
+
    --baseurl controls the URL prefix that rclone serves from. By default rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically.
    
+
    --template allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages:
    
 @@ -3466,23 +3601,23 @@ 
    Server options
    
 
 

 
    
 
    Authentication
    
 
    By default this will serve files without needing a login.
    
-
    You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.
    
-
    Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.
    
+
    You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.
    
+
    Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.
    
 
    To create an htpasswd file:
    
 
    touch htpasswd
 htpasswd -B htpasswd user
 htpasswd -B htpasswd anotherUser
    
 
    The password file can be updated while rclone is running.
    
-
    Use --realm to set the authentication realm.
    
+
    Use --realm to set the authentication realm.
    
 
    SSL/TLS
    
-
    By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.
    
-
    --cert should be either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.
    
+
    By default this will serve over HTTP. If you want you can serve over HTTPS. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.
    
+
    --cert should be either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.
    
 
    VFS - Virtual File System
    
 
    This command uses the VFS layer. This adapts the cloud storage objects that rclone uses into something which looks much more like a disk filing system.
    
 
    Cloud storage objects have lots of properties which aren't like disk files - you can't extend them or write to the middle of them, so the VFS layer has to deal with that. Because there is no one right way of doing this there are various options explained below.
    
 
    The VFS layer also implements a directory cache - this caches info about files and directories (but not the data) in memory.
    
 
    VFS Directory Cache
    
-
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the mount will appear immediately or invalidate the cache.
    
+
    Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. Changes made through the VFS will appear immediately or invalidate the cache.
    
 
    --dir-cache-time duration   Time to cache directory entries for (default 5m0s)
 --poll-interval duration    Time to wait between polling for changes. Must be smaller than dir-cache-time. Only on supported remotes. Set to 0 to disable (default 1m0s)
    
 
    However, changes made directly on the cloud storage by the web interface or a different copy of rclone will only be picked up once the directory cache expires if the backend configured does not support polling for changes. If the backend supports polling, changes will be picked up within the polling interval.
    
@@ -3545,6 +3680,19 @@ 
    --vfs-cache-mode full
    
 
    When reading a file rclone will read --buffer-size plus --vfs-read-ahead bytes ahead. The --buffer-size is buffered in memory whereas the --vfs-read-ahead is buffered on disk.
    
 
    When using this mode it is recommended that --buffer-size is not set too large and --vfs-read-ahead is set large if required.
    
 
    IMPORTANT not all file systems support sparse files. In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected.
    
+
    Fingerprinting
    
+
    Various parts of the VFS use fingerprinting to see if a local file copy has changed relative to a remote file. Fingerprints are made from:
    
+
      
+
    • size
      • 
+
    • modification time
      • 
+
    • hash
      • 
+
    
+
    where available on an object.
    
+
    On some backends some of these attributes are slow to read (they take an extra API call per object, or extra work per object).
    
+
    For example hash is slow with the local and sftp backends as they have to read the entire file and hash it, and modtime is slow with the s3, swift, ftp and qinqstor backends because they need to do an extra API call to fetch it.
    
+
    If you use the --vfs-fast-fingerprint flag then rclone will not include the slow operations in the fingerprint. This makes the fingerprinting less accurate but much faster and will improve the opening time of cached files.
    
+
    If you are running a vfs cache over local, s3 or swift backends then using this flag is recommended.
    
+
    Note that if you change the value of this flag, the fingerprints of the files in the cache may be invalidated and the files will need to be downloaded again.
    
 
    VFS Chunked Reading
    
 
    When rclone reads files from a remote it reads them in chunks. This means that rather than requesting the whole file rclone reads the chunk specified. This can reduce the used download quota for some remotes by requesting only chunks from the remote that are actually read, at the cost of an increased number of requests.
    
 
    These flags control the chunking:
    
@@ -3559,20 +3707,23 @@ 
    VFS Performance
    
 
    --no-checksum     Don't compare checksums on up/download.
 --no-modtime      Don't read/write the modification time (can speed things up).
 --no-seek         Don't allow seeking in files.
---read-only       Mount read-only.
    
+--read-only       Only allow read-only access.
    
 
    Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write to come in. These flags only come into effect when not using an on disk cache file.
    
 
    --vfs-read-wait duration   Time to wait for in-sequence read before seeking (default 20ms)
 --vfs-write-wait duration  Time to wait for in-sequence write before giving error (default 1s)
    
-
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from cache (the related global flag --checkers have no effect on mount).
    
+
    When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of parallel uploads of modified files from the cache (the related global flag --checkers has no effect on the VFS).
    
 
    --transfers int  Number of file transfers to run in parallel (default 4)
    
 
    VFS Case Sensitivity
    
 
    Linux file systems are case-sensitive: two files can differ only by case, and the exact case must be used when opening a file.
    
 
    File systems in modern Windows are case-insensitive but case-preserving: although existing files can be opened using any case, the exact case used to create the file is preserved and available for programs to query. It is not allowed for two files in the same directory to differ only by case.
    
 
    Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default.
    
-
    The --vfs-case-insensitive mount flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the mounted file system as-is. If the flag is "true" (or appears without a value on command line), rclone may perform a "fixup" as explained below.
    
-
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on mounted file system. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by an underlying mounted file system.
    
-
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system mounted by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
+
    The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the remote as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below.
    
+
    The user may specify a file name to open/delete/rename/etc with a case different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by the underlying remote.
    
+
    Note that case sensitivity of the operating system running rclone (the target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target.
    
 
    If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true".
    
+
    VFS Disk Options
    
+
    This flag allows you to manually set the statistics about the filing system. It can be useful when those statistics cannot be read correctly automatically.
    
+
    --vfs-disk-space-total-size    Manually set the total disk space size (example: 256G, default: -1)
    
 
    Alternate report of used bytes
    
 
    Some backends, most notably S3, do not report the amount of bytes used. If you need this information to be available when running df on the filesystem, then pass the flag --vfs-used-is-size to rclone. With this flag set, instead of relying on the backend to report this information, rclone will scan the whole remote similar to rclone size and compute the total used space itself.
    
 
    WARNING. Contrary to rclone size, this flag ignores filters so that the result is accurate. However, this is very inefficient and may cost lots of API calls resulting in extra charges. Use it as a last resort and only with caching.
    
@@ -3628,7 +3779,7 @@ 
    Options
    
       --no-seek                                Don't allow seeking in files
       --pass string                            Password for authentication
       --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
-      --read-only                              Mount read-only
+      --read-only                              Only allow read-only access
       --realm string                           Realm for authentication (default "rclone")
       --server-read-timeout duration           Timeout for server reading data (default 1h0m0s)
       --server-write-timeout duration          Timeout for server writing data (default 1h0m0s)
@@ -3641,6 +3792,8 @@ 
    Options
    
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
       --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
+      --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
+      --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
@@ -3655,7 +3808,7 @@ 
    SEE ALSO
    
 
 
    rclone settier
    
 
    Changes storage class/tier of objects in remote.
    
-
    Synopsis
    
+
    Synopsis
    
 
    rclone settier changes storage tier or class at remote if supported. Few cloud storage services provides different storage classes on objects, for example AWS S3 and Glacier, Azure Blob storage - Hot, Cool and Archive, Google Cloud Storage, Regional Storage, Nearline, Coldline etc.
    
 
    Note that, certain tier changes make objects not available to access immediately. For example tiering to archive in azure blob storage makes objects in frozen state, user can restore by setting tier to Hot/Cool, similarly S3 to Glacier makes object inaccessible.true
    
 
    You can use it to tier single object
    
@@ -3674,7 +3827,7 @@ 
    SEE ALSO
    
 
 
    rclone test
    
 
    Run a test command
    
-
    Synopsis
    
+
    Synopsis
    
 
    Rclone test is used to run test commands.
    
 
    Select which test comand you want with the subcommand, eg
    
 
    rclone test memory remote:
    
@@ -3689,6 +3842,7 @@ 
    SEE ALSO
    
 
  • rclone test changenotify - Log any change notify requests for the remote passed in.
    • 
 
  • rclone test histogram - Makes a histogram of file name characters.
    • 
 
  • rclone test info - Discovers file name or other limitations for paths.
    • 
+
  • rclone test makefile - Make files with random contents of the size given
    • 
 
  • rclone test makefiles - Make a random file hierarchy in a directory
    • 
 
  • rclone test memory - Load all the objects at remote:path into memory and report memory stats.
    • 
 
@@ -3705,7 +3859,7 @@ 
    SEE ALSO
    
 
 
    rclone test histogram
    
 
    Makes a histogram of file name characters.
    
-
    Synopsis
    
+
    Synopsis
    
 
    This command outputs JSON which shows the histogram of characters used in filenames in the remote:path specified.
    
 
    The data doesn't contain any identifying information but is useful for the rclone developers when developing filename compression.
    
 
    rclone test histogram [remote:path] [flags]
    
@@ -3718,7 +3872,7 @@ 
    SEE ALSO
    
 
 
    rclone test info
    
 
    Discovers file name or other limitations for paths.
    
-
    Synopsis
    
+
    Synopsis
    
 
    rclone info discovers what filenames and upload methods are possible to write to the paths passed in and how long they can be. It can take some time. It will write test files into the remote:path passed in. It outputs a bit of go code for each one.
    
 
    NB this can create undeletable files and other hazards - use with care
    
 
    rclone test info [remote:path]+ [flags]
    
@@ -3736,36 +3890,57 @@ 
    SEE ALSO
    
 
+
    rclone test makefile
    
+
    Make files with random contents of the size given
    
+
    rclone test makefile <size> [<file>]+ [flags]
    
+
    Options
    
+
          --ascii      Fill files with random ASCII printable bytes only
+      --chargen    Fill files with a ASCII chargen pattern
+  -h, --help       help for makefile
+      --pattern    Fill files with a periodic pattern
+      --seed int   Seed for the random number generator (0 for random) (default 1)
+      --sparse     Make the files sparse (appear to be filled with ASCII 0x00)
+      --zero       Fill files with ASCII 0x00
    
+
    See the global flags page for global options not listed here.
    
+
    SEE ALSO
    
+
 
    rclone test makefiles
    
 
    Make a random file hierarchy in a directory
    
 
    rclone test makefiles <dir> [flags]
    
-
    Options
    
-
          --files int                  Number of files to create (default 1000)
+
    Options
    
+
          --ascii                      Fill files with random ASCII printable bytes only
+      --chargen                    Fill files with a ASCII chargen pattern
+      --files int                  Number of files to create (default 1000)
       --files-per-directory int    Average number of files per directory (default 10)
   -h, --help                       help for makefiles
       --max-file-size SizeSuffix   Maximum size of files to create (default 100)
       --max-name-length int        Maximum size of file names (default 12)
       --min-file-size SizeSuffix   Minimum size of file to create
       --min-name-length int        Minimum size of file names (default 4)
-      --seed int                   Seed for the random number generator (0 for random) (default 1)
    
+      --pattern                    Fill files with a periodic pattern
+      --seed int                   Seed for the random number generator (0 for random) (default 1)
+      --sparse                     Make the files sparse (appear to be filled with ASCII 0x00)
+      --zero                       Fill files with ASCII 0x00
    
 
    See the global flags page for global options not listed here.
    
-
    SEE ALSO
    
+
    SEE ALSO
    
 
 
    rclone test memory
    
 
    Load all the objects at remote:path into memory and report memory stats.
    
 
    rclone test memory remote:path [flags]
    
-
    Options
    
+
    Options
    
 
      -h, --help   help for memory
    
 
    See the global flags page for global options not listed here.
    
-
    SEE ALSO
    
+
    SEE ALSO
    
 
 
    rclone touch
    
 
    Create new file or change file modification time.
    
-
    Synopsis
    
+
    Synopsis
    
 
    Set the modification time on file(s) as specified by remote:path to have the current time.
    
 
    If remote:path does not exist then a zero sized file will be created, unless --no-create or --recursive is provided.
    
 
    If --recursive is used then recursively sets the modification time on all existing files that is found under the path. Filters are supported, and you can test with the --dry-run or the --interactive flag.
    
@@ -3777,20 +3952,20 @@ 
    Synopsis
    
 
 
    Note that value of --timestamp is in UTC. If you want local time then add the --localtime flag.
    
 
    rclone touch remote:path [flags]
    
-
    Options
    
+
    Options
    
 
      -h, --help               help for touch
       --localtime          Use localtime for timestamp, not UTC
   -C, --no-create          Do not create the file if it does not exist (implied with --recursive)
   -R, --recursive          Recursively touch all files
   -t, --timestamp string   Use specified time instead of the current time of day
    
 
    See the global flags page for global options not listed here.
    
-
    SEE ALSO
    
+
    SEE ALSO
    
 
      
 
    • rclone - Show help for rclone commands, flags and backends.
      • 
 
    
 
    rclone tree
    
 
    List the contents of the remote in a tree like fashion.
    
-
    Synopsis
    
+
    Synopsis
    
 
    rclone tree lists the contents of a remote in a similar way to the unix tree command.
    
 
    For example
    
 
    $ rclone tree remote:path
@@ -3803,10 +3978,11 @@ 
    Synopsis
    
     └── file5
 
 1 directories, 5 files
    
-
    You can use any of the filtering options with the tree command (e.g. --include and --exclude). You can also use --fast-list.
    
-
    The tree command has many options for controlling the listing which are compatible with the tree command. Note that not all of them have short options as they conflict with rclone's short options.
    
+
    You can use any of the filtering options with the tree command (e.g. --include and --exclude. You can also use --fast-list.
    
+
    The tree command has many options for controlling the listing which are compatible with the tree command, for example you can include file sizes with --size. Note that not all of them have short options as they conflict with rclone's short options.
    
+
    For a more interactive navigation of the remote see the ncdu command.
    
 
    rclone tree remote:path [flags]
    
-
    Options
    
+
    Options
    
 
      -a, --all             All files are listed (list . files too)
   -C, --color           Turn colorization on always
   -d, --dirs-only       List directories only
@@ -3828,7 +4004,7 @@ 
    Options
    
   -U, --unsorted        Leave files unsorted
       --version         Sort files alphanumerically by version
    
 
    See the global flags page for global options not listed here.
    
-
    SEE ALSO
    
+
    SEE ALSO
    
 
      
 
    • rclone - Show help for rclone commands, flags and backends.
      • 
 
    
@@ -3940,7 +4116,119 @@ 
    Server Side Copy
    
 
    This can be used when scripting to make aged backups efficiently, e.g.
    
 
    rclone sync -i remote:current-backup remote:previous-backup
 rclone sync -i /path/to/files remote:current-backup
    
-
    Options
    
+
    Metadata support
    
+
    Metadata is data about a file which isn't the contents of the file. Normally rclone only preserves the modification time and the content (MIME) type where possible.
    
+
    Rclone supports preserving all the available metadata on files (not directories) when using the --metadata or -M flag.
    
+
    Exactly what metadata is supported and what that support means depends on the backend. Backends that support metadata have a metadata section in their docs and are listed in the features table (Eg local, s3)
    
+
    Rclone only supports a one-time sync of metadata. This means that metadata will be synced from the source object to the destination object only when the source object has changed and needs to be re-uploaded. If the metadata subsequently changes on the source object without changing the object itself then it won't be synced to the destination object. This is in line with the way rclone syncs Content-Type without the --metadata flag.
    
+
    Using --metadata when syncing from local to local will preserve file attributes such as file mode, owner, extended attributes (not Windows).
    
+
    Note that arbitrary metadata may be added to objects using the --metadata-set key=value flag when the object is first uploaded. This flag can be repeated as many times as necessary.
    
+
    Types of metadata
    
+
    Metadata is divided into two type. System metadata and User metadata.
    
+
    Metadata which the backend uses itself is called system metadata. For example on the local backend the system metadata uid will store the user ID of the file when used on a unix based platform.
    
+
    Arbitrary metadata is called user metadata and this can be set however is desired.
    
+
    When objects are copied from backend to backend, they will attempt to interpret system metadata if it is supplied. Metadata may change from being user metadata to system metadata as objects are copied between different backends. For example copying an object from s3 sets the content-type metadata. In a backend which understands this (like azureblob) this will become the Content-Type of the object. In a backend which doesn't understand this (like the local backend) this will become user metadata. However should the local object be copied back to s3, the Content-Type will be set correctly.
    
+
    Metadata framework
    
+
    Rclone implements a metadata framework which can read metadata from an object and write it to the object when (and only when) it is being uploaded.
    
+
    This metadata is stored as a dictionary with string keys and string values.
    
+
    There are some limits on the names of the keys (these may be clarified further in the future).
    
+
      
+
    • must be lower case
      • 
+
    • may be a-z 0-9 containing . - or _
      • 
+
    • length is backend dependent
      • 
+
    
+
    Each backend can provide system metadata that it understands. Some backends can also store arbitrary user metadata.
    
+
    Where possible the key names are standardized, so, for example, it is possible to copy object metadata from s3 to azureblob for example and metadata will be translated apropriately.
    
+
    Some backends have limits on the size of the metadata and rclone will give errors on upload if they are exceeded.
    
+
    Metadata preservation
    
+
    The goal of the implementation is to
    
+
      
+
    1. Preserve metadata if at all possible
      2. 
+
    2. Interpret metadata if at all possible
      3. 
+
    
+
    The consequences of 1 is that you can copy an S3 object to a local disk then back to S3 losslessly. Likewise you can copy a local file with file attributes and xattrs from local disk to s3 and back again losslessly.
    
+
    The consequence of 2 is that you can copy an S3 object with metadata to Azureblob (say) and have the metadata appear on the Azureblob object also.
    
+
    Standard system metadata
    
+
    Here is a table of standard system metadata which, if appropriate, a backend may implement.
    
+
+++++
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
    




    keydescriptionexample
    modeFile type and mode: octal, unix style0100664
    uidUser ID of owner: decimal number500
    gidGroup ID of owner: decimal number500
    rdevDevice ID (if special file) => hexadecimal0
    atimeTime of last access: RFC 33392006-01-02T15:04:05.999999999Z07:00
    mtimeTime of last modification: RFC 33392006-01-02T15:04:05.999999999Z07:00
    btimeTime of file creation (birth): RFC 33392006-01-02T15:04:05.999999999Z07:00
    cache-controlCache-Control headerno-cache
    content-dispositionContent-Disposition headerinline
    content-encodingContent-Encoding headergzip
    content-languageContent-Language headeren-US
    content-typeContent-Type headertext/plain
    
+
    The metadata keys mtime and content-type will take precedence if supplied in the metadata over reading the Content-Type or modification time of the source object.
    
+
    Hashes are not included in system metadata as there is a well defined way of reading those already.
    
+
    Options
    
 
    Rclone has a number of options to control its behaviour.
    
 
    Options that take parameters can have the values passed in two ways, --option=value or --option value. However boolean (true/false) options behave slightly differently to the other options in that --boolean sets the option to true and the absence of the flag sets it to false. It is also possible to specify --boolean=false or --boolean=true. Note that --boolean false is not valid - this is parsed as --boolean and the false is parsed as an extra command line argument for rclone.
    
 
    Options which use TIME use the go time parser. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
    
@@ -4010,8 +4298,9 @@ 
    --check-first
    
 
    It can also be useful to ensure perfect ordering when using --order-by.
    
 
    Using this flag can use more memory as it effectively sets --max-backlog to infinite. This means that all the info on the objects to transfer is held in memory before the transfers start.
    
 
    --checkers=N
    
-
    The number of checkers to run in parallel. Checkers do the equality checking of files during a sync. For some storage systems (e.g. S3, Swift, Dropbox) this can take a significant amount of time so they are run in parallel.
    
-
    The default is to run 8 checkers in parallel.
    
+
    Originally controlling just the number of file checkers to run in parallel, e.g. by rclone copy. Now a fairly universal parallelism control used by rclone in several places.
    
+
    Note: checkers do the equality checking of files during a sync. For some storage systems (e.g. S3, Swift, Dropbox) this can take a significant amount of time so they are run in parallel.
    
+
    The default is to run 8 checkers in parallel. However, in case of slow-reacting backends you may need to lower (rather than increase) this default by setting --checkers to 4 or less threads. This is especially advised if you are experiencing backend server crashes during file checking phase (e.g. on subsequent or top-up backups where little or no file copying is done and checking takes up most of the time). Increase this setting only with utmost care, while monitoring your server health and file checking throughput.
    
 
    -c, --checksum
    
 
    Normally rclone will look at modification time and size of files to see if they are equal. If you set this flag then rclone will check the file hash and size to determine if files are equal.
    
 
    This is useful when the remote doesn't support setting modified time and a more accurate sync is desired than just checking the file size.
    
@@ -4068,7 +4357,8 @@ 
    --copy-dest=DIR
    
 
    The remote in use must support server-side copy and you must use the same remote as the destination of the sync. The compare directory must not overlap the destination directory.
    
 
    See --compare-dest and --backup-dir.
    
 
    --dedupe-mode MODE
    
-
    Mode to run dedupe command in. One of interactive, skip, first, newest, oldest, rename. The default is interactive. See the dedupe command for more information as to what these options mean.
    
+
    Mode to run dedupe command in. One of interactive, skip, first, newest, oldest, rename. The default is interactive.
    
+See the dedupe command for more information as to what these options mean.
    
 
    --disable FEATURE,FEATURE,...
    
 
    This disables a comma separated list of optional features. For example to disable server-side move and server-side copy use:
    
 
    --disable move,copy
    
@@ -4118,11 +4408,11 @@ 
    --header-upload
    
 
    --human-readable
    
 
    Rclone commands output values for sizes (e.g. number of bytes) and counts (e.g. number of files) either as raw numbers, or in human-readable format.
    
 
    In human-readable format the values are scaled to larger units, indicated with a suffix shown after the value, and rounded to three decimals. Rclone consistently uses binary units (powers of 2) for sizes and decimal units (powers of 10) for counts. The unit prefix for size is according to IEC standard notation, e.g. Ki for kibi. Used with byte unit, 1 KiB means 1024 Byte. In list type of output, only the unit prefix appended to the value (e.g. 9.762Ki), while in more textual output the full unit is shown (e.g. 9.762 KiB). For counts the SI standard notation is used, e.g. prefix k for kilo. Used with file counts, 1k means 1000 files.
    
-
    The various list commands output raw numbers by default. Option --human-readable will make them output values in human-readable format instead (with the short unit prefix).
    
-
    The about command outputs human-readable by default, with a command-specific option --full to output the raw numbers instead.
    
-
    Command size outputs both human-readable and raw numbers in the same output.
    
-
    The tree command also considers --human-readable, but it will not use the exact same notation as the other commands: It rounds to one decimal, and uses single letter suffix, e.g. K instead of Ki. The reason for this is that it relies on an external library.
    
-
    The interactive command ncdu shows human-readable by default, and responds to key u for toggling human-readable format.
    
+
    The various list commands output raw numbers by default. Option --human-readable will make them output values in human-readable format instead (with the short unit prefix).
    
+
    The about command outputs human-readable by default, with a command-specific option --full to output the raw numbers instead.
    
+
    Command size outputs both human-readable and raw numbers in the same output.
    
+
    The tree command also considers --human-readable, but it will not use the exact same notation as the other commands: It rounds to one decimal, and uses single letter suffix, e.g. K instead of Ki. The reason for this is that it relies on an external library.
    
+
    The interactive command ncdu shows human-readable by default, and responds to key u for toggling human-readable format.
    
 
    --ignore-case-sync
    
 
    Using this option will cause rclone to ignore the case of the files when synchronizing so files will not be copied/synced when the existing filenames are the same, even if the casing is different.
    
 
    --ignore-checksum
    
@@ -4208,6 +4498,10 @@ 
    --max-transfer=SIZE
    
 
    Rclone will stop transferring when it has reached the size specified. Defaults to off.
    
 
    When the limit is reached all transfers will stop immediately.
    
 
    Rclone will exit with exit code 8 if the transfer limit is reached.
    
+
    --metadata / -M
    
+
    Setting this flag enables rclone to copy the metadata from the source to the destination. For local backends this is ownership, permissions, xattr etc. See the #metadata for more info.
    
+
    --metadata-set key=value
    
+
    Add metadata key = value when uploading. This can be repeated as many times as required. See the #metadata for more info.
    
 
    --cutoff-mode=hard|soft|cautious
    
 
    This modifies the behavior of --max-transfer Defaults to --cutoff-mode=hard.
    
 
    Specifying --cutoff-mode=hard will stop transferring immediately when Rclone reaches the limit.
    
@@ -4434,6 +4728,7 @@ 
    --timeout=TIME
    
 
    --transfers=N
    
 
    The number of file transfers to run in parallel. It can sometimes be useful to set this to a smaller number if the remote is giving a lot of timeouts or bigger if you have lots of bandwidth and a fast remote.
    
 
    The default is to run 4 file transfers in parallel.
    
+
    Look at --multi-thread-streams if you would like to control single file transfers.
    
 
    -u, --update
    
 
    This forces rclone to skip any files which exist on the destination and have a modified time that is newer than the source file.
    
 
    This can be useful in avoiding needless transfers when transferring to a remote which doesn't support modification times directly (or when using --use-server-modtime to avoid extra API calls) as it is more accurate than a --size-only check and faster than using --checksum. On such remotes (or when using --use-server-modtime) the time checked will be the uploaded time.
    
@@ -4452,6 +4747,7 @@ 
    --use-server-modtime
    
 
    -v, -vv, --verbose
    
 
    With -v rclone will tell you about each file that is transferred and a small number of significant events.
    
 
    With -vv rclone will become very verbose telling you about every file it considers and transfers. Please send bug reports with a log with this setting.
    
+
    When setting verbosity as an environment variable, use RCLONE_VERBOSE=1 or RCLONE_VERBOSE=2 for -v and -vv respectively.
    
 
    -V, --version
    
 
    Prints the version number
    
 
    SSL/TLS options
    
@@ -4552,6 +4848,7 @@ 
    Filtering
    
 
  • --filter-from
    • 
 
  • --exclude
    • 
 
  • --exclude-from
    • 
+
  • --exclude-if-present
    • 
 
  • --include
    • 
 
  • --include-from
    • 
 
  • --files-from
    • 
@@ -4600,11 +4897,12 @@ 
    List of exit codes
    
 
 
    Environment Variables
    
 
    Rclone can be configured entirely using environment variables. These can be used to set defaults for options or config file entries.
    
-
    Options
    
+
    Options
    
 
    Every option in rclone can have its default set by environment variable.
    
 
    To find the name of the environment variable, first, take the long option name, strip the leading --, change - to _, make upper case and prepend RCLONE_.
    
 
    For example, to always set --stats 5s, set the environment variable RCLONE_STATS=5s. If you set stats on the command line this will override the environment variable setting.
    
 
    Or to always use the trash in drive --drive-use-trash, set RCLONE_DRIVE_USE_TRASH=true.
    
+
    Verbosity is slightly different, the environment variable equivalent of --verbose or -v is RCLONE_VERBOSE=1, or for -vv, RCLONE_VERBOSE=2.
    
 
    The same parser is used for the options and the environment variables so they take exactly the same form.
    
 
    The options set by environment variables can be seen with the -vv flag, e.g. rclone version -vv.
    
 
    Config file
    
@@ -4714,6 +5012,19 @@ 
    Configuring by copying the confi
 Configuration file is stored at:
 /home/user/.rclone.conf
    
 
    Now transfer it to the remote box (scp, cut paste, ftp, sftp, etc.) and place it in the correct place (use rclone config file on the remote box to find out where).
    
+
    Configuring using SSH Tunnel
    
+
    Linux and MacOS users can utilize SSH Tunnel to redirect the headless box port 53682 to local machine by using the following command:
    
+
    ssh -L localhost:53682:localhost:53682 username@remote_server
    
+
    Then on the headless box run rclone config and answer Y to the Use auto config? question.
    
+
    ...
+Remote config
+Use auto config?
+ * Say Y if not sure
+ * Say N if you are working on a remote or headless machine
+y) Yes (default)
+n) No
+y/n> y
    
+
    Then copy and paste the auth url http://127.0.0.1:53682/auth?state=xxxxxxxxxxxx to the browser on your local machine, complete the auth and it is done.
    
 
    Filtering, includes and excludes
    
 
    Filter flags determine which files rclone sync, move, ls, lsl, md5sum, sha1sum, size, delete, check and similar commands apply to.
    
 
    They are specified in terms of path/file name patterns; path/file lists; file age and size, or presence of a file in a directory. Bucket based remotes without the concept of directory apply filters to object key, age and size in an analogous way.
    
@@ -4780,7 +5091,7 @@ 
    Pattern syntax
    
        - matches "POTATO"
    
 
    Using regular expressions in filter patterns
    
 
    The syntax of filter patterns is glob style matching (like bash uses) to make things easy for users. However this does not provide absolute control over the matching, so for advanced users rclone also provides a regular expression syntax.
    
-
    The regular expressions used are as defined in the Go regular expression reference. Regular expressions should be enclosed in {{ }}. They will match only the last path segment if the glob doesn't start with / or the whole path name if it does.
    
+
    The regular expressions used are as defined in the Go regular expression reference. Regular expressions should be enclosed in {{ }}. They will match only the last path segment if the glob doesn't start with / or the whole path name if it does. Note that rclone does not attempt to parse the supplied regular expression, meaning that using any regular expression filter will prevent rclone from using directory filter rules, as it will instead check every path against the supplied regular expression(s).
    
 
    Here is how the {{regexp}} is transformed into an full regular expression to match the entire path:
    
 
    {{regexp}}  becomes (^|/)(regexp)$
 /{{regexp}} becomes ^(regexp)$
    
@@ -4949,14 +5260,15 @@ 
    How filter rules are applied to f
 
    Any path/file included at that stage is processed by the rclone command.
    
 
    --files-from and --files-from-raw flags over-ride and cannot be combined with other filter options.
    
 
    To see the internal combined rule list, in regular expression form, for a command add the --dump filters flag. Running an rclone command with --dump filters and -vv flags lists the internal filter elements and shows how they are applied to each source path/file. There is not currently a means provided to pass regular expression filter options into rclone directly though character class filter rules contain character classes. Go regular expression reference
    
-
    How filter rules are applied to directories
    
+
    How filter rules are applied to directories
    
 
    Rclone commands are applied to path/file names not directories. The entire contents of a directory can be matched to a filter by the pattern directory/* or recursively by directory/**.
    
 
    Directory filter rules are defined with a closing / separator.
    
 
    E.g. /directory/subdirectory/ is an rclone directory filter rule.
    
 
    Rclone commands can use directory filter rules to determine whether they recurse into subdirectories. This potentially optimises access to a remote by avoiding listing unnecessary directories. Whether optimisation is desirable depends on the specific filter rules and source remote content.
    
+
    If any regular expression filters are in use, then no directory recursion optimisation is possible, as rclone must check every path against the supplied regular expression(s).
    
 
    Directory recursion optimisation occurs if either:
    
 
      
-
    • A source remote does not support the rclone ListR primitive. local, sftp, Microsoft OneDrive and WebDav do not support ListR. Google Drive and most bucket type storage do. Full list
      • 
+
    • A source remote does not support the rclone ListR primitive. local, sftp, Microsoft OneDrive and WebDAV do not support ListR. Google Drive and most bucket type storage do. Full list
      • 
 
    • On other remotes (those that support ListR), if the rclone command is not naturally recursive, and provided it is not run with the --fast-list flag. ls, lsf -R and size are naturally recursive but sync, copy and move are not.
      • 
 
    • Whenever the --disable ListR flag is applied to an rclone command.
      • 
 
    
@@ -5157,7 +5469,7 @@ 
    --dump filtersDumps the defined filters to standard output in regular expression format.
    
 
    Useful for debugging.
    
 
    Exclude directory based on a file
    
-
    The --exclude-if-present flag controls whether a directory is within the scope of an rclone command based on the presence of a named file within it.
    
+
    The --exclude-if-present flag controls whether a directory is within the scope of an rclone command based on the presence of a named file within it. The flag can be repeated to check for multiple file names, presence of any of them will exclude the directory.
    
 
    This flag has a priority over other filter flags.
    
 
    E.g. for the following directory structure:
    
 
    dir1/file1
@@ -5165,7 +5477,6 @@ 
    Exclude directory based on a file
    
 
    The command rclone ls --exclude-if-present .ignore dir1 does not list dir3, file3 or .ignore.
    
-
    --exclude-if-present can only be used once in an rclone command.
    
 
    Common pitfalls
    
 
    The most frequent filter support issues on the rclone forum are:
    
 
      
@@ -5245,8 +5556,8 @@ 
      Project
      
 
      If you have questions then please ask them on the rclone forum.
      
 
      Remote controlling rclone with its API
      
 
      If rclone is run with the --rc flag then it starts an HTTP server which can be used to remote control rclone using its API.
      
-
      You can either use the rclone rc command to access the API or use HTTP directly.
      
-
      If you just want to run a remote control then see the rcd command.
      
+
      You can either use the rc command to access the API or use HTTP directly.
      
+
      If you just want to run a remote control then see the rcd command.
      
 
      Supported parameters
      
 
      --rc
      
 
      Flag to start the http server listen on remote requests
      
@@ -5310,6 +5621,11 @@ 
      --rc-no-auth
      
 
      By default rclone will require authorisation to have been set up on the rc interface in order to use any methods which access any rclone remotes. Eg operations/list is denied as it involved creating a remote as is sync/copy.
      
 
      If this is set then no authorisation will be required on the server to use these methods. The alternative is to use --rc-user and --rc-pass and use these credentials in the request.
      
 
      Default Off.
      
+
      --rc-baseurl
      
+
      Prefix for URLs.
      
+
      Default is root
      
+
      --rc-template
      
+
      User-specified template.
      
 
      Accessing the remote control via the rclone rc command
      
 
      Rclone itself implements the remote control protocol in its rclone rc command.
      
 
      You can use it like this
      
@@ -5516,30 +5832,30 @@ 
      config/create: create the config for a remote.
      
 
    • result - result to restart with - used with continue
      • 
 
    
 
-
    See the config create command command for more information on the above.
    
+
    See the config create command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    config/delete: Delete a remote in the config file.
    
 
    Parameters:
    
 
      
 
    • name - name of remote to delete
      • 
 
    
-
    See the config delete command command for more information on the above.
    
+
    See the config delete command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    config/dump: Dumps the config file.
    
 
    Returns a JSON object: - key: value
    
 
    Where keys are remote names and values are the config parameters.
    
-
    See the config dump command command for more information on the above.
    
+
    See the config dump command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    config/get: Get a remote in the config file.
    
 
    Parameters:
    
 
      
 
    • name - name of remote to get
      • 
 
    
-
    See the config dump command command for more information on the above.
    
+
    See the config dump command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    config/listremotes: Lists the remotes in the config file.
    
 
    Returns - remotes - array of remote names
    
-
    See the listremotes command command for more information on the above.
    
+
    See the listremotes command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    config/password: password the config for a remote.
    
 
    This takes the following parameters:
    
@@ -5547,11 +5863,11 @@ 
    config/password: password the config for a remote.
    
 
  • name - name of remote
    • 
 
  • parameters - a map of { "key": "value" } pairs
    • 
 
-
    See the config password command command for more information on the above.
    
+
    See the config password command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    config/providers: Shows how providers are configured in the config file.
    
 
    Returns a JSON object: - providers - array of objects
    
-
    See the config providers command command for more information on the above.
    
+
    See the config providers command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    config/update: update the config for a remote.
    
 
    This takes the following parameters:
    
@@ -5569,7 +5885,7 @@ 
    config/update: update the config for a remote.
    
 
  • result - result to restart with - used with continue
    • 
 
 
-
    See the config update command command for more information on the above.
    
+
    See the config update command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    core/bwlimit: Set the bandwidth limit.
    
 
    This sets the bandwidth limit to the string passed in. This should be a single bandwidth limit entry or a pair of upload:download bandwidth.
    
@@ -5882,14 +6198,14 @@ 
    operations/about: Return the space used on the remote<
 
  • fs - a remote name string e.g. "drive:"
    • 
 
 
    The result is as returned from rclone about --json
    
-
    See the about command command for more information on the above.
    
+
    See the about command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    operations/cleanup: Remove trashed files in the remote or path
    
 
    This takes the following parameters:
    
 
      
 
    • fs - a remote name string e.g. "drive:"
      • 
 
    
-
    See the cleanup command command for more information on the above.
    
+
    See the cleanup command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    operations/copyfile: Copy a file from source remote to destination remote
    
 
    This takes the following parameters:
    
@@ -5906,15 +6222,16 @@ 
    operations/copyurl: Copy the URL to the object
    
 
  • fs - a remote name string e.g. "drive:"
    • 
 
  • remote - a path within that remote e.g. "dir"
    • 
 
  • url - string, URL to read from
    • 
-
  • autoFilename - boolean, set to true to retrieve destination file name from url See the copyurl command command for more information on the above.
    • 
+
  • autoFilename - boolean, set to true to retrieve destination file name from url
    • 
 
+
    See the copyurl command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    operations/delete: Remove files in the path
    
 
    This takes the following parameters:
    
 
      
 
    • fs - a remote name string e.g. "drive:"
      • 
 
    
-
    See the delete command command for more information on the above.
    
+
    See the delete command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    operations/deletefile: Remove the single file pointed to
    
 
    This takes the following parameters:
    
@@ -5922,7 +6239,7 @@ 
    operations/deletefile: Remove the single file poi
 
  • fs - a remote name string e.g. "drive:"
    • 
 
  • remote - a path within that remote e.g. "dir"
    • 
 
-
    See the deletefile command command for more information on the above.
    
+
    See the deletefile command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    operations/fsinfo: Return information about the remote
    
 
    This takes the following parameters:
    
@@ -5931,46 +6248,103 @@ 
    operations/fsinfo: Return information about the remot
 
 
    This returns info about the remote passed in;
    
 
    {
-    // optional features and whether they are available or not
-    "Features": {
-        "About": true,
-        "BucketBased": false,
-        "CanHaveEmptyDirectories": true,
-        "CaseInsensitive": false,
-        "ChangeNotify": false,
-        "CleanUp": false,
-        "Copy": false,
-        "DirCacheFlush": false,
-        "DirMove": true,
-        "DuplicateFiles": false,
-        "GetTier": false,
-        "ListR": false,
-        "MergeDirs": false,
-        "Move": true,
-        "OpenWriterAt": true,
-        "PublicLink": false,
-        "Purge": true,
-        "PutStream": true,
-        "PutUnchecked": false,
-        "ReadMimeType": false,
-        "ServerSideAcrossConfigs": false,
-        "SetTier": false,
-        "SetWrapper": false,
-        "UnWrap": false,
-        "WrapFs": false,
-        "WriteMimeType": false
-    },
-    // Names of hashes available
-    "Hashes": [
-        "MD5",
-        "SHA-1",
-        "DropboxHash",
-        "QuickXorHash"
-    ],
-    "Name": "local",    // Name as created
-    "Precision": 1,     // Precision of timestamps in ns
-    "Root": "/",        // Path as created
-    "String": "Local file system at /" // how the remote will appear in logs
+        // optional features and whether they are available or not
+        "Features": {
+                "About": true,
+                "BucketBased": false,
+                "BucketBasedRootOK": false,
+                "CanHaveEmptyDirectories": true,
+                "CaseInsensitive": false,
+                "ChangeNotify": false,
+                "CleanUp": false,
+                "Command": true,
+                "Copy": false,
+                "DirCacheFlush": false,
+                "DirMove": true,
+                "Disconnect": false,
+                "DuplicateFiles": false,
+                "GetTier": false,
+                "IsLocal": true,
+                "ListR": false,
+                "MergeDirs": false,
+                "MetadataInfo": true,
+                "Move": true,
+                "OpenWriterAt": true,
+                "PublicLink": false,
+                "Purge": true,
+                "PutStream": true,
+                "PutUnchecked": false,
+                "ReadMetadata": true,
+                "ReadMimeType": false,
+                "ServerSideAcrossConfigs": false,
+                "SetTier": false,
+                "SetWrapper": false,
+                "Shutdown": false,
+                "SlowHash": true,
+                "SlowModTime": false,
+                "UnWrap": false,
+                "UserInfo": false,
+                "UserMetadata": true,
+                "WrapFs": false,
+                "WriteMetadata": true,
+                "WriteMimeType": false
+        },
+        // Names of hashes available
+        "Hashes": [
+                "md5",
+                "sha1",
+                "whirlpool",
+                "crc32",
+                "sha256",
+                "dropbox",
+                "mailru",
+                "quickxor"
+        ],
+        "Name": "local",        // Name as created
+        "Precision": 1,         // Precision of timestamps in ns
+        "Root": "/",            // Path as created
+        "String": "Local file system at /", // how the remote will appear in logs
+        // Information about the system metadata for this backend
+        "MetadataInfo": {
+                "System": {
+                        "atime": {
+                                "Help": "Time of last access",
+                                "Type": "RFC 3339",
+                                "Example": "2006-01-02T15:04:05.999999999Z07:00"
+                        },
+                        "btime": {
+                                "Help": "Time of file birth (creation)",
+                                "Type": "RFC 3339",
+                                "Example": "2006-01-02T15:04:05.999999999Z07:00"
+                        },
+                        "gid": {
+                                "Help": "Group ID of owner",
+                                "Type": "decimal number",
+                                "Example": "500"
+                        },
+                        "mode": {
+                                "Help": "File type and mode",
+                                "Type": "octal, unix style",
+                                "Example": "0100664"
+                        },
+                        "mtime": {
+                                "Help": "Time of last modification",
+                                "Type": "RFC 3339",
+                                "Example": "2006-01-02T15:04:05.999999999Z07:00"
+                        },
+                        "rdev": {
+                                "Help": "Device ID (if special file)",
+                                "Type": "hexadecimal",
+                                "Example": "1abc"
+                        },
+                        "uid": {
+                                "Help": "User ID of owner",
+                                "Type": "decimal number",
+                                "Example": "500"
+                        }
+                },
+                "Help": "Textual help string\n"
+        }
 }
    
 
    This command does not have a command line equivalent so use this instead:
    
 
    rclone rc --loopback operations/fsinfo fs=remote:
    
@@ -5989,6 +6363,7 @@ 
    operations/list: List the given remote and path in JSON
 
  • noMimeType - If set don't show mime types
    • 
 
  • dirsOnly - If set only show directories
    • 
 
  • filesOnly - If set only show files
    • 
+
  • metadata - If set return metadata of objects also
    • 
 
  • hashTypes - array of strings of hash types to show if showHash set
    • 
 
 
@@ -5999,7 +6374,7 @@ 
    operations/list: List the given remote and path in JSON
 
  • This is an array of objects as described in the lsjson command
    • 
 
 
-
    See the lsjson command for more information on the above and examples.
    
+
    See the lsjson command for more information on the above and examples.
    
 
    Authentication is required for this call.
    
 
    operations/mkdir: Make a destination directory or container
    
 
    This takes the following parameters:
    
@@ -6007,7 +6382,7 @@ 
    operations/mkdir: Make a destination directory or cont
 
  • fs - a remote name string e.g. "drive:"
    • 
 
  • remote - a path within that remote e.g. "dir"
    • 
 
-
    See the mkdir command command for more information on the above.
    
+
    See the mkdir command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    operations/movefile: Move a file from source remote to destination remote
    
 
    This takes the following parameters:
    
@@ -6030,7 +6405,7 @@ 
    operations/purge: Remove a directory or container and all of its contents
    
 
    This takes the following parameters:
    
@@ -6038,7 +6413,7 @@ 
    operations/purge: Remove a directory or container and
 
  • fs - a remote name string e.g. "drive:"
    • 
 
  • remote - a path within that remote e.g. "dir"
    • 
 
-
    See the purge command command for more information on the above.
    
+
    See the purge command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    operations/rmdir: Remove an empty directory or container
    
 
    This takes the following parameters:
    
@@ -6046,15 +6421,16 @@ 
    operations/rmdir: Remove an empty directory or contain
 
  • fs - a remote name string e.g. "drive:"
    • 
 
  • remote - a path within that remote e.g. "dir"
    • 
 
-
    See the rmdir command command for more information on the above.
    
+
    See the rmdir command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    operations/rmdirs: Remove all the empty directories in the path
    
 
    This takes the following parameters:
    
 
      
 
    • fs - a remote name string e.g. "drive:"
      • 
 
    • remote - a path within that remote e.g. "dir"
      • 
-
    • leaveRoot - boolean, set to true not to delete the root See the rmdirs command command for more information on the above.
      • 
+
    • leaveRoot - boolean, set to true not to delete the root
      • 
 
    
+
    See the rmdirs command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    operations/size: Count the number of bytes and files in remote
    
 
    This takes the following parameters:
    
@@ -6066,7 +6442,7 @@ 
    operations/size: Count the number of bytes and files in
 
  • count - number of files
    • 
 
  • bytes - number of bytes in those files
    • 
 
-
    See the size command command for more information on the above.
    
+
    See the size command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    operations/stat: Give information about the supplied file or directory
    
 
    This takes the following parameters
    
@@ -6083,15 +6459,16 @@ 
    operations/stat: Give information about the supplied fi
 
  • item - an object as described in the lsjson command. Will be null if not found.
    • 
 
 
    Note that if you are only interested in files then it is much more efficient to set the filesOnly flag in the options.
    
-
    See the lsjson command for more information on the above and examples.
    
+
    See the lsjson command for more information on the above and examples.
    
 
    Authentication is required for this call.
    
 
    operations/uploadfile: Upload file using multiform/form-data
    
 
    This takes the following parameters:
    
 
      
 
    • fs - a remote name string e.g. "drive:"
      • 
 
    • remote - a path within that remote e.g. "dir"
      • 
-
    • each part in body represents a file to be uploaded See the uploadfile command command for more information on the above.
      • 
+
    • each part in body represents a file to be uploaded
      • 
 
    
+
    See the uploadfile command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    options/blocks: List all the option blocks
    
 
    Returns: - options - a list of the options block names
    
@@ -6218,7 +6595,7 @@ 
    sync/copy: copy a directory from source remote to destination
 
  • dstFs - a remote name string e.g. "drive:dst" for the destination
    • 
 
  • createEmptySrcDirs - create empty src directories on destination if set
    • 
 
-
    See the copy command command for more information on the above.
    
+
    See the copy command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    sync/move: move a directory from source remote to destination remote
    
 
    This takes the following parameters:
    
@@ -6228,7 +6605,7 @@ 
    sync/move: move a directory from source remote to destination
 
  • createEmptySrcDirs - create empty src directories on destination if set
    • 
 
  • deleteEmptySrcDirs - delete empty src directories if set
    • 
 
-
    See the move command command for more information on the above.
    
+
    See the move command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    sync/sync: sync a directory from source remote to destination remote
    
 
    This takes the following parameters:
    
@@ -6237,7 +6614,7 @@ 
    sync/sync: sync a directory from source remote to destination
 
  • dstFs - a remote name string e.g. "drive:dst" for the destination
    • 
 
  • createEmptySrcDirs - create empty src directories on destination if set
    • 
 
-
    See the sync command command for more information on the above.
    
+
    See the sync command for more information on the above.
    
 
    Authentication is required for this call.
    
 
    vfs/forget: Forget files or directories in the directory cache.
    
 
    This forgets the paths in the directory cache causing them to be re-read from the remote when needed.
    
@@ -6428,320 +6805,378 @@ 
    Features
    
 Case Insensitive
 Duplicate Files
 MIME Type
+Metadata
 
 
 
 
 1Fichier
 Whirlpool
-No
+-
 No
 Yes
 R
+-
 
 
 Akamai Netstorage
 MD5, SHA256
-Yes
+R/W
 No
 No
 R
+-
 
 
 Amazon Drive
 MD5
-No
+-
 Yes
 No
 R
+-
 
 
 Amazon S3 (or S3 compatible)
 MD5
-Yes
+R/W
 No
 No
 R/W
+RWU
 
 
 Backblaze B2
 SHA1
-Yes
+R/W
 No
 No
 R/W
+-
 
 
 Box
 SHA1
-Yes
+R/W
 Yes
 No
 -
+-
 
 
 Citrix ShareFile
 MD5
-Yes
+R/W
 Yes
 No
 -
+-
 
 
 Dropbox
 DBHASH ¹
-Yes
+R
 Yes
 No
 -
+-
 
 
 Enterprise File Fabric
 -
-Yes
+R/W
 Yes
 No
 R/W
+-
 
 
 FTP
 -
+R/W ¹⁰
 No
 No
-No
+-
 -
 
 
 Google Cloud Storage
 MD5
-Yes
+R/W
 No
 No
 R/W
+-
 
 
 Google Drive
 MD5
-Yes
+R/W
 No
 Yes
 R/W
+-
 
 
 Google Photos
 -
-No
+-
 No
 Yes
 R
+-
 
 
 HDFS
 -
-Yes
+R/W
 No
 No
 -
+-
 
 
+HiDrive
+HiDrive ¹²
+R/W
+No
+No
+-
+-
+
+
 HTTP
 -
-No
+R
 No
 No
 R
+-
 
-
+
 Hubic
 MD5
-Yes
+R/W
 No
 No
 R/W
+-
+
+
+Internet Archive
+MD5, SHA1, CRC32
+R/W ¹¹
+No
+No
+-
+RWU
 
 
 Jottacloud
 MD5
-Yes
+R/W
 Yes
 No
 R
+-
 
 
 Koofr
 MD5
-No
+-
 Yes
 No
 -
+-
 
 
 Mail.ru Cloud
 Mailru ⁶
-Yes
+R/W
 Yes
 No
 -
+-
 
 
 Mega
 -
-No
+-
 No
 Yes
 -
+-
 
 
 Memory
 MD5
-Yes
+R/W
 No
 No
 -
+-
 
 
 Microsoft Azure Blob Storage
 MD5
-Yes
+R/W
 No
 No
 R/W
+-
 
 
 Microsoft OneDrive
 SHA1 ⁵
-Yes
+R/W
 Yes
 No
 R
+-
 
 
 OpenDrive
 MD5
-Yes
+R/W
 Yes
 Partial ⁸
 -
+-
 
 
 OpenStack Swift
 MD5
-Yes
+R/W
 No
 No
 R/W
+-
 
 
 pCloud
 MD5, SHA1 ⁷
-Yes
+R
 No
 No
 W
+-
 
 
 premiumize.me
 -
-No
+-
 Yes
 No
 R
+-
 
 
 put.io
 CRC-32
-Yes
+R/W
 No
 Yes
 R
+-
 
 
 QingStor
 MD5
-No
+- ⁹
 No
 No
 R/W
+-
 
 
 Seafile
 -
-No
+-
 No
 No
 -
+-
 
 
 SFTP
 MD5, SHA1 ²
-Yes
+R/W
 Depends
 No
 -
+-
 
 
 Sia
 -
-No
+-
 No
 No
 -
+-
 
 
 SugarSync
 -
+-
 No
 No
-No
+-
 -
 
 
 Storj
 -
-Yes
+R
 No
 No
 -
+-
 
 
 Uptobox
 -
-No
+-
 No
 Yes
 -
+-
 
 
 WebDAV
 MD5, SHA1 ³
-Yes ⁴
+R ⁴
 Depends
 No
 -
+-
 
 
 Yandex Disk
 MD5
-Yes
+R/W
 No
 No
 R
+-
 
 
 Zoho WorkDrive
 -
-No
+-
 No
 No
 -
+-
 
 
 The local filesystem
 All
-Yes
+R/W
 Depends
 No
 -
+RWU
 
 
 
@@ -6754,12 +7189,18 @@ 
    Notes
    
 
    ⁶ Mail.ru uses its own modified SHA1 hash
    
 
    ⁷ pCloud only supports SHA1 (not MD5) in its EU region
    
 
    ⁸ Opendrive does not support creation of duplicate files using their web client interface or other stock clients, but the underlying storage platform has been determined to allow duplicate files, and it is possible to create them with rclone. It may be that this is a mistake or an unsupported feature.
    
+
    ⁹ QingStor does not support SetModTime for objects bigger than 5 GiB.
    
+
    ¹⁰ FTP supports modtimes for the major FTP servers, and also others if they advertised required protocol extensions. See this for more details.
    
+
    ¹¹ Internet Archive requires option wait_archive to be set to a non-zero value for full modtime support.
    
+
    ¹² HiDrive supports its own custom hash. It combines SHA1 sums for each 4 KiB block hierarchically to a single top-level sum.
    
 
    Hash
    
 
    The cloud storage system supports various hash types of the objects. The hashes are used when transferring data as an integrity check and can be specifically used with the --checksum flag in syncs and in the check command.
    
 
    To use the verify checksums when transferring between cloud storage systems they must support a common hash type.
    
 
    ModTime
    
-
    The cloud storage system supports setting modification times on objects. If it does then this enables a using the modification times as part of the sync. If not then only the size will be checked by default, though the MD5SUM can be checked with the --checksum flag.
    
-
    All cloud storage systems support some kind of date on the object and these will be set when transferring from the cloud storage system.
    
+
    Allmost all cloud storage systems store some sort of timestamp on objects, but several of them not something that is appropriate to use for syncing. E.g. some backends will only write a timestamp that represent the time of the upload. To be relevant for syncing it should be able to store the modification time of the source object. If this is not the case, rclone will only check the file size by default, though can be configured to check the file hash (with the --checksum flag). Ideally it should also be possible to change the timestamp of an existing file without having to re-upload it.
    
+
    Storage systems with a - in the ModTime column, means the modification read on objects is not the modification time of the file when uploaded. It is most likely the time the file was uploaded, or possibly something else (like the time the picture was taken in Google Photos).
    
+
    Storage systems with a R (for read-only) in the ModTime column, means the it keeps modification times on objects, and updates them when uploading objects, but it does not support changing only the modification time (SetModTime operation) without re-uploading, possibly not even without deleting existing first. Some operations in rclone, such as copy and sync commands, will automatically check for SetModTime support and re-upload if necessary to keep the modification times in sync. Other commands will not work without SetModTime support, e.g. touch command on an existing file will fail, and changes to modification time only on a files in a mount will be silently ignored.
    
+
    Storage systems with R/W (for read/write) in the ModTime column, means they do also support modtime-only operations.
    
 
    Case Insensitive
    
 
    If a cloud storage systems is case sensitive then it is possible to have two files which differ only in case, e.g. file.txt and FILE.txt. If a cloud storage system is case insensitive then that isn't possible.
    
 
    This can cause problems when syncing between a case insensitive system and a case sensitive system. The symptom of this is that no matter how many times you run the sync it never completes fully.
    
@@ -7000,120 +7441,158 @@ 
    Encoding option
    
 
    The --backend-encoding flags allow you to change that. You can disable the encoding completely with --backend-encoding None or set encoding = None in the config file.
    
 
    Encoding takes a comma separated list of encodings. You can see the list of all possible values by passing an invalid value to this flag, e.g. --local-encoding "help". The command rclone help flags encoding will show you the defaults for the backends.
    
 
+++++
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
-
+
+
    




 
 
    
 Encoding
 CharactersEncoded as
 
    
     
 
 
    
 Asterisk
 *＊
 
    
 
    
 BackQuote
 `｀
 
    
 
    
 BackSlash
 \＼
 
    
 
    
 Colon
 :：
 
    
 
    
 CrLf
 CR 0x0D, LF 0x0A␍, ␊
 
    
 
    
 Ctl
 All control characters 0x00-0x1F␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟
 
    
 
    
 Del
 DEL 0x7F␡
 
    
 
    
 Dollar
 $＄
 
    
 
    
 Dot
 . or .. as entire string．, ．．
 
    
 
    
 DoubleQuote
 "＂
 
    
 
    
 Hash
 #＃
 
    
 
    
 InvalidUtf8
 An invalid UTF-8 character (e.g. latin1)�
 
    
 
    
 LeftCrLfHtVtCR 0x0D, LF 0x0A,HT 0x09, VT 0x0B on the left of a stringCR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the left of a string␍, ␊, ␉, ␋
 
    
 
    
 LeftPeriod
 . on the left of a string.
 
    
 
    
 LeftSpace
 SPACE on the left of a string␠
 
    
 
    
 LeftTilde
 ~ on the left of a string～
 
    
 
    
 LtGt
 <, >＜, ＞
 
    
 
    
 None
 No characters are encoded
 
    
 
    
 Percent
 %％
 
    
 
    
 Pipe
 |｜
 
    
 
    
 Question
 ?？
 
    
 
    
 RightCrLfHtVt
 CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string␍, ␊, ␉, ␋
 
    
 
    
 RightPeriod
 . on the right of a string.
 
    
 
    
 RightSpace
 SPACE on the right of a string␠
 
    
 
    Semicolon;；
    
 SingleQuote
 '＇
 
    
 Slash
 /／
 
    
 SquareBracket
 [, ]［, ］
 
    
     
 
    
@@ -7139,6 +7618,32 @@ 
    MIME Type
    
 
    Some cloud storage systems support reading (R) the MIME type of objects and some support writing (W) the MIME type of objects.
    
 
    The MIME type can be important if you are serving files directly to HTTP from the storage system.
    
 
    If you are copying from a remote which supports reading (R) to a remote which supports writing (W) then rclone will preserve the MIME types. Otherwise they will be guessed from the extension, or the remote itself may assign the MIME type.
    
+
    Metadata
    
+
    Backends may or may support reading or writing metadata. They may support reading and writing system metadata (metadata intrinsic to that backend) and/or user metadata (general purpose metadata).
    
+
    The levels of metadata support are
    
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
    KeyExplanation
    RRead only System Metadata
    RWRead and write System Metadata
    RWURead and write System Metadata and read and write User Metadata
    
+
    See the metadata docs for more info.
    
 
    Optional Features
    
 
    All rclone remotes support a base command set. Other features depend upon backend-specific capabilities.
    
 
@@ -7172,6 +7677,19 @@ 
    Optional Features
    
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -7184,8 +7702,8 @@ 
    Optional Features
    
-
-
+
+
@@ -7197,7 +7715,7 @@ 
    Optional Features
    
-
+
@@ -7210,7 +7728,7 @@ 
    Optional Features
    
-
+
@@ -7223,7 +7741,7 @@ 
    Optional Features
    
-
+
@@ -7236,7 +7754,7 @@ 
    Optional Features
    
-
+
@@ -7249,7 +7767,7 @@ 
    Optional Features
    
-
+
@@ -7262,7 +7780,7 @@ 
    Optional Features
    
-
+
@@ -7275,7 +7793,7 @@ 
    Optional Features
    
-
+
@@ -7288,7 +7806,7 @@ 
    Optional Features
    
-
+
@@ -7301,7 +7819,7 @@ 
    Optional Features
    
-
+
@@ -7314,7 +7832,7 @@ 
    Optional Features
    
-
+
@@ -7327,6 +7845,19 @@ 
    Optional Features
    
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -7354,6 +7885,19 @@ 
    Optional Features
    
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -7366,6 +7910,19 @@ 
    Optional Features
    
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -7536,6 +8093,19 @@ 
    Optional Features
    
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -7548,7 +8118,7 @@ 
    Optional Features
    
-
+
@@ -7561,7 +8131,7 @@ 
    Optional Features
    
-
+
@@ -7574,7 +8144,7 @@ 
    Optional Features
    
-
+
@@ -7587,7 +8157,7 @@ 
    Optional Features
    
-
+
@@ -7600,7 +8170,7 @@ 
    Optional Features
    
-
+
@@ -7613,7 +8183,7 @@ 
    Optional Features
    
-
+
@@ -7686,6 +8256,7 @@ 
    Non Backend Flags
    
       --delete-during                        When synchronizing, delete files during transfer
       --delete-excluded                      Delete files on dest excluded from sync
       --disable string                       Disable a comma separated list of features (use --disable help to see a list)
+      --disable-http-keep-alives             Disable HTTP keep-alives and use each connection once.
       --disable-http2                        Disable HTTP/2 in the global transport
   -n, --dry-run                              Do a trial run with no permanent changes
       --dscp string                          Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21
@@ -7695,7 +8266,7 @@ 
    Non Backend Flags
    
       --error-on-no-transfer                 Sets exit code 9 if no files are transferred, useful in scripts
       --exclude stringArray                  Exclude files matching pattern
       --exclude-from stringArray             Read exclude patterns from file (use - to read from stdin)
-      --exclude-if-present string            Exclude directories if filename is present
+      --exclude-if-present stringArray       Exclude directories if filename is present
       --expect-continue-timeout duration     Timeout when using expect / 100-continue in HTTP (default 1s)
       --fast-list                            Use recursive list if available; uses more memory but fewer transactions
       --files-from stringArray               Read list of source-file names from file (use - to read from stdin)
@@ -7734,6 +8305,8 @@ 
    Non Backend Flags
    
       --max-stats-groups int                 Maximum number of stats groups to keep in memory, on max oldest is discarded (default 1000)
       --max-transfer SizeSuffix              Maximum size of data to transfer (default off)
       --memprofile string                    Write memory profile to file
+  -M, --metadata                             If set, preserve metadata when copying objects
+      --metadata-set stringArray             Add metadata key=value when uploading
       --min-age Duration                     Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off)
       --min-size SizeSuffix                  Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off)
       --modify-window duration               Max time diff to be considered the same (default 1ns)
@@ -7805,7 +8378,7 @@ 
    Non Backend Flags
    
       --use-json-log                         Use json log format
       --use-mmap                             Use mmap allocator (see docs)
       --use-server-modtime                   Use server modified time instead of object metadata
-      --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.58.0")
+      --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.59.0")
   -v, --verbose count                        Print lots more stuff (repeat for more)
    Backend Flags
    These flags are available for every command. They control the backends and may be set in the config file.
    
@@ -7854,6 +8427,7 @@ 
    Backend Flags
    
       --b2-memory-pool-use-mmap                      Whether to use mmap buffers in internal memory pool
       --b2-test-mode string                          A flag string for X-Bz-Test-Mode header for debugging
       --b2-upload-cutoff SizeSuffix                  Cutoff for switching to chunked upload (default 200Mi)
+      --b2-version-at Time                           Show file versions as they were at the specified time (default off)
       --b2-versions                                  Include old versions in directory listings
       --box-access-token string                      Box App Primary Access Token
       --box-auth-url string                          Auth server URL
@@ -7893,6 +8467,7 @@ 
    Backend Flags
    
       --chunker-fail-hard                            Choose how chunker should handle files with missing or invalid chunks
       --chunker-hash-type string                     Choose how chunker handles hash sums (default "md5")
       --chunker-remote string                        Remote to chunk/unchunk
+      --combine-upstreams SpaceSepList               Upstreams for combining
       --compress-level int                           GZIP compression level (-2 to 9) (default -1)
       --compress-mode string                         Compression mode (default "gzip")
       --compress-ram-cache-limit SizeSuffix          Some remotes don't allow the upload of files with unknown size (default 20Mi)
@@ -7925,6 +8500,7 @@ 
    Backend Flags
    
       --drive-list-chunk int                         Size of listing chunk 100-1000, 0 to disable (default 1000)
       --drive-pacer-burst int                        Number of API calls to allow without sleeping (default 100)
       --drive-pacer-min-sleep Duration               Minimum time to sleep between API calls (default 100ms)
+      --drive-resource-key string                    Resource key for accessing a link-shared file
       --drive-root-folder-id string                  ID of the root folder
       --drive-scope string                           Scope that rclone should use when requesting access from drive
       --drive-server-side-across-configs             Allow server-side operations (e.g. copy) to work across different drive configs
@@ -7980,6 +8556,7 @@ 
    Backend Flags
    
       --ftp-disable-epsv                             Disable using EPSV even if server advertises support
       --ftp-disable-mlsd                             Disable using MLSD even if server advertises support
       --ftp-disable-tls13                            Disable TLS 1.3 (workaround for FTP servers with buggy TLS)
+      --ftp-disable-utf8                             Disable using UTF-8 even if server advertises support
       --ftp-encoding MultiEncoder                    The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
       --ftp-explicit-tls                             Use Explicit FTPS (FTP over TLS)
       --ftp-host string                              FTP host to connect to
@@ -7998,8 +8575,10 @@ 
    Backend Flags
    
       --gcs-bucket-policy-only                       Access checks should use bucket-level IAM policies
       --gcs-client-id string                         OAuth Client Id
       --gcs-client-secret string                     OAuth Client Secret
+      --gcs-decompress                               If set this will decompress gzip encoded objects
       --gcs-encoding MultiEncoder                    The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot)
       --gcs-location string                          Location for the newly created buckets
+      --gcs-no-check-bucket                          If set, don't attempt to check the bucket exists or create it
       --gcs-object-acl string                        Access Control List for new objects
       --gcs-project-number string                    Project number
       --gcs-service-account-file string              Service Account Credentials JSON file path
@@ -8025,10 +8604,24 @@ 
    Backend Flags
    
       --hdfs-namenode string                         Hadoop name node and port
       --hdfs-service-principal-name string           Kerberos service principal name for the namenode
       --hdfs-username string                         Hadoop user name
+      --hidrive-auth-url string                      Auth server URL
+      --hidrive-chunk-size SizeSuffix                Chunksize for chunked uploads (default 48Mi)
+      --hidrive-client-id string                     OAuth Client Id
+      --hidrive-client-secret string                 OAuth Client Secret
+      --hidrive-disable-fetching-member-count        Do not fetch number of objects in directories unless it is absolutely necessary
+      --hidrive-encoding MultiEncoder                The encoding for the backend (default Slash,Dot)
+      --hidrive-endpoint string                      Endpoint for the service (default "https://api.hidrive.strato.com/2.1")
+      --hidrive-root-prefix string                   The root/parent folder for all paths (default "/")
+      --hidrive-scope-access string                  Access permissions that rclone should use when requesting access from HiDrive (default "rw")
+      --hidrive-scope-role string                    User-level that rclone should use when requesting access from HiDrive (default "user")
+      --hidrive-token string                         OAuth Access Token as a JSON blob
+      --hidrive-token-url string                     Token server url
+      --hidrive-upload-concurrency int               Concurrency for chunked uploads (default 4)
+      --hidrive-upload-cutoff SizeSuffix             Cutoff/Threshold for chunked uploads (default 96Mi)
       --http-headers CommaSepList                    Set HTTP headers for all transactions
       --http-no-head                                 Don't use HEAD requests
       --http-no-slash                                Set this if the site doesn't end directories with /
-      --http-url string                              URL of http host to connect to
+      --http-url string                              URL of HTTP host to connect to
       --hubic-auth-url string                        Auth server URL
       --hubic-chunk-size SizeSuffix                  Above this size files will be chunked into a _segments container (default 5Gi)
       --hubic-client-id string                       OAuth Client Id
@@ -8037,6 +8630,13 @@ 
    Backend Flags
    
       --hubic-no-chunk                               Don't chunk files during streaming upload
       --hubic-token string                           OAuth Access Token as a JSON blob
       --hubic-token-url string                       Token server url
+      --internetarchive-access-key-id string         IAS3 Access Key
+      --internetarchive-disable-checksum             Don't ask the server to test against MD5 checksum calculated by rclone (default true)
+      --internetarchive-encoding MultiEncoder        The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot)
+      --internetarchive-endpoint string              IAS3 Endpoint (default "https://s3.us.archive.org")
+      --internetarchive-front-endpoint string        Host of InternetArchive Frontend (default "https://archive.org")
+      --internetarchive-secret-access-key string     IAS3 Secret Key (password)
+      --internetarchive-wait-archive Duration        Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s)
       --jottacloud-encoding MultiEncoder             The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot)
       --jottacloud-hard-delete                       Delete files permanently rather than putting them into the trash
       --jottacloud-md5-memory-limit SizeSuffix       Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi)
@@ -8058,7 +8658,7 @@ 
    Backend Flags
    
       --local-no-preallocate                         Disable preallocation of disk space for transferred files
       --local-no-set-modtime                         Disable setting modtime
       --local-no-sparse                              Disable sparse files for multi-thread downloads
-      --local-nounc string                           Disable UNC (long path names) conversion on Windows
+      --local-nounc                                  Disable UNC (long path names) conversion on Windows
       --local-unicode-normalization                  Apply unicode NFC normalization to paths and filenames
       --local-zero-size-links                        Assume the Stat size of links is zero (and read them instead) (deprecated)
       --mailru-check-hash                            What should copy do if file checksum is mismatched or invalid (default true)
@@ -8079,11 +8679,11 @@ 
    Backend Flags
    
       --netstorage-protocol string                   Select between HTTP or HTTPS protocol (default "https")
       --netstorage-secret string                     Set the NetStorage account secret/G2O key for authentication (obscured)
   -x, --one-file-system                              Don't cross filesystem boundaries (unix/macOS only)
+      --onedrive-access-scopes SpaceSepList          Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access)
       --onedrive-auth-url string                     Auth server URL
       --onedrive-chunk-size SizeSuffix               Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi)
       --onedrive-client-id string                    OAuth Client Id
       --onedrive-client-secret string                OAuth Client Secret
-      --onedrive-disable-site-permission             Disable the request for Sites.Read.All permission
       --onedrive-drive-id string                     The ID of the drive to use
       --onedrive-drive-type string                   The type of the drive (personal | business | documentLibrary)
       --onedrive-encoding MultiEncoder               The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot)
@@ -8107,9 +8707,11 @@ 
    Backend Flags
    
       --pcloud-client-secret string                  OAuth Client Secret
       --pcloud-encoding MultiEncoder                 The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
       --pcloud-hostname string                       Hostname to connect to (default "api.pcloud.com")
+      --pcloud-password string                       Your pcloud password (obscured)
       --pcloud-root-folder-id string                 Fill in for rclone to use a non root folder as its starting point (default "d0")
       --pcloud-token string                          OAuth Access Token as a JSON blob
       --pcloud-token-url string                      Token server url
+      --pcloud-username string                       Your pcloud username
       --premiumizeme-encoding MultiEncoder           The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot)
       --putio-encoding MultiEncoder                  The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
       --qingstor-access-key-id string                QingStor Access Key ID
@@ -8162,6 +8764,7 @@ 
    Backend Flags
    
       --s3-upload-cutoff SizeSuffix                  Cutoff for switching to chunked upload (default 200Mi)
       --s3-use-accelerate-endpoint                   If true use the AWS S3 accelerated endpoint
       --s3-use-multipart-etag Tristate               Whether to use ETag in multipart uploads for verification (default unset)
+      --s3-use-presigned-request                     Whether to use a presigned request or PutObject for single part uploads
       --s3-v2-auth                                   If true use v2 authentication
       --seafile-2fa                                  Two-factor authentication ('true' if the account has 2FA enabled)
       --seafile-create-library                       Should rclone create a library if it doesn't exist
@@ -8172,6 +8775,8 @@ 
    Backend Flags
    
       --seafile-url string                           URL of seafile host to connect to
       --seafile-user string                          User name (usually email address)
       --sftp-ask-password                            Allow asking for SFTP password when needed
+      --sftp-chunk-size SizeSuffix                   Upload and download chunk size (default 32Ki)
+      --sftp-concurrency int                         The maximum number of outstanding requests for one file (default 64)
       --sftp-disable-concurrent-reads                If set don't use concurrent reads
       --sftp-disable-concurrent-writes               If set don't use concurrent writes
       --sftp-disable-hashcheck                       Disable the execution of SSH commands to determine if remote file hashing is available
@@ -8184,12 +8789,14 @@ 
    Backend Flags
    
       --sftp-known-hosts-file string                 Optional path to known_hosts file
       --sftp-md5sum-command string                   The command used to read md5 hashes
       --sftp-pass string                             SSH password, leave blank to use ssh-agent (obscured)
-      --sftp-path-override string                    Override path used by SSH connection
+      --sftp-path-override string                    Override path used by SSH shell commands
       --sftp-port int                                SSH port number (default 22)
       --sftp-pubkey-file string                      Optional path to public key file
       --sftp-server-command string                   Specifies the path or command to run a sftp server on the remote host
+      --sftp-set-env SpaceSepList                    Environment variables to pass to sftp and commands
       --sftp-set-modtime                             Set the modified time on the remote if set (default true)
       --sftp-sha1sum-command string                  The command used to read sha1 hashes
+      --sftp-shell-type string                       The type of SSH shell on remote server, if any
       --sftp-skip-links                              Set to skip any symlinks and any other non regular files
       --sftp-subsystem string                        Specifies the SSH2 subsystem on the remote host (default "sftp")
       --sftp-use-fstat                               If set use fstat instead of stat
@@ -8246,6 +8853,7 @@ 
    Backend Flags
    
       --union-action-policy string                   Policy to choose upstream on ACTION category (default "epall")
       --union-cache-time int                         Cache time of usage and free space (in seconds) (default 120)
       --union-create-policy string                   Policy to choose upstream on CREATE category (default "epmfs")
+      --union-min-free-space SizeSuffix              Minimum viable free space for lfs/eplfs policies (default 1Gi)
       --union-search-policy string                   Policy to choose upstream on SEARCH category (default "ff")
       --union-upstreams string                       List of space separated upstreams
       --uptobox-access-token string                  Your access token
@@ -8257,7 +8865,7 @@ 
    Backend Flags
    
       --webdav-pass string                           Password (obscured)
       --webdav-url string                            URL of http host to connect to
       --webdav-user string                           User name
-      --webdav-vendor string                         Name of the Webdav site/service/software you are using
+      --webdav-vendor string                         Name of the WebDAV site/service/software you are using
       --yandex-auth-url string                       Auth server URL
       --yandex-client-id string                      OAuth Client Id
       --yandex-client-secret string                  OAuth Client Secret
@@ -8539,7 +9147,7 @@ 
    Command line syntax
    
   -v, --verbose                 Increases logging verbosity.
                                 May be specified more than once for more details.
   -h, --help                    help for bisync
-
    Arbitrary rclone flags may be specified on the bisync command line, for example rclone bsync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s Note that interactions of various rclone flags with bisync process flow has not been fully tested yet.
    
+
    Arbitrary rclone flags may be specified on the bisync command line, for example rclone bisync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s Note that interactions of various rclone flags with bisync process flow has not been fully tested yet.
    Paths
    Path1 and Path2 arguments may be references to any mix of local directory paths (absolute or relative), UNC paths (//server/share/path), Windows drive paths (with a drive letter and :) or configured remotes with optional subdirectory paths. Cloud references are distinguished by having a : in the argument (see Windows support below).
    Path1 and Path2 are treated equally, in that neither has priority for file changes, and access efficiency does not change whether a remote is on Path1 or Path2.
    
@@ -8715,7 +9323,7 @@ 
    Return codes
    rclone bisync returns the following codes to calling program: - 0 on a successful run, - 1 for a non-critical failing run (a rerun may be successful), - 2 for a critically aborted run (requires a --resync to recover).
    Limitations
    Supported backends
    
-
    Bisync is considered BETA and has been tested with the following backends: - Local filesystem - Google Drive - Dropbox - OneDrive - S3 - SFTP
    
+
    Bisync is considered BETA and has been tested with the following backends: - Local filesystem - Google Drive - Dropbox - OneDrive - S3 - SFTP - Yandex Disk
    It has not been fully tested with other services yet. If it works, or sorta works, please let us know and we'll update the list. Run the test suite to check for proper operation as described below.
    First release of rclone bisync requires that underlying backend supported the modification time feature and will refuse to run otherwise. This limitation will be lifted in a future rclone bisync release.
    Concurrent modifications
    
@@ -8883,7 +9491,7 @@ 
    Retries
    Denied downloads of "infected" or "abusive" files
    Google Drive has a filter for certain file types (.exe, .apk, et cetera) that by default cannot be copied from Google Drive to the local filesystem. If you are having problems, run with --verbose to see specifically which files are generating complaints. If the error is This file has been identified as malware or spam and cannot be downloaded, consider using the flag --drive-acknowledge-abuse.
    Google Doc files
    
-
    Google docs exist as virtual files on Google Drive and cannot be transferred to other filesystems natively. While it is possible to export a Google doc to a normal file (with .xlsx extension, for example), it's not possible to import a normal file back into a Google document.
    
+
    Google docs exist as virtual files on Google Drive and cannot be transferred to other filesystems natively. While it is possible to export a Google doc to a normal file (with .xlsx extension, for example), it is not possible to import a normal file back into a Google document.
    Bisync's handling of Google Doc files is to flag them in the run log output for user's attention and ignore them for any file transfers, deletes, or syncs. They will show up with a length of -1 in the listings. This bisync run is otherwise successful:
    2021/05/11 08:23:15 INFO  : Synching Path1 "/path/to/local/tree/base/" with Path2 "GDrive:"
 2021/05/11 08:23:15 INFO  : ...path2.lst-new: Ignoring incorrect line: "- -1 - - 2018-07-29T08:49:30.136000000+0000 GoogleDoc.docx"
@@ -9223,7 +9831,7 @@ 
    Restricted filename characters
    
 
    
 
    Yes
 
    
 
    Akamai NetstorageYesNoNoNoNoYesYesNoNoYes
    
 Amazon Drive
 Yes
 No
 No
 Yes
 
    Amazon S3
    Amazon S3 (or S3 compatible)
 No
 Yes
 No
 No
 No
 
    
 Backblaze B2
 No
 Yes
 No
 No
 
    
 Box
 Yes
 Yes
 Yes
 Yes
 
    
 Citrix ShareFile
 Yes
 Yes
 No
 Yes
 
    
 Dropbox
 Yes
 Yes
 Yes
 Yes
 
    
 Enterprise File Fabric
 Yes
 Yes
 No
 Yes
 
    
 FTP
 No
 No
 No
 Yes
 
    
 Google Cloud Storage
 Yes
 Yes
 No
 No
 
    
 Google Drive
 Yes
 Yes
 Yes
 Yes
 
    
 Google Photos
 No
 No
 No
 No
 
    
 HDFS
 Yes
 No
 Yes
 Yes
 
    HiDriveYesYesYesYesNoNoYesNoNoYes
    
 
    
 HTTP
 No
 No
 
    
 
    Internet ArchiveNoYesNoNoYesYesNoYesYesNo
    
 Jottacloud
 Yes
 Yes
 Yes
 Yes
 
    KoofrYesYesYesYesNoNoYesYesYesYes
    
 
    
 Mail.ru Cloud
 Yes
 Yes
 
    
 
    SiaNoNoNoNoNoNoYesNoNoYes
    
 SugarSync
 Yes
 Yes
 No
 Yes
 
    
 Storj
 Yes †
 No
 No
 No
 
    
 Uptobox
 No
 Yes
 No
 No
 
    
 WebDAV
 Yes
 Yes
 Yes
 Yes
 
    
 Yandex Disk
 Yes
 Yes
 Yes
 Yes
 
    
 Zoho WorkDrive
 Yes
 Yes
 Yes
 Yes
 
    
 The local filesystem
 Yes
 No
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
    
 
    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.
    
 
    Standard options
    
-
    Here are the standard options specific to fichier (1Fichier).
    
+
    Here are the Standard options specific to fichier (1Fichier).
    
 
    --fichier-api-key
    
 
    Your API Key, get it from https://1fichier.com/console/params.pl.
    
 
    Properties:
    
@@ -9234,7 +9842,7 @@ 
    --fichier-api-key
    
 
  • Required: false
    • 
 
 
    Advanced options
    
-
    Here are the advanced options specific to fichier (1Fichier).
    
+
    Here are the Advanced options specific to fichier (1Fichier).
    
 
    --fichier-shared-folder
    
 
    If you want to download a shared folder, add this parameter.
    
 
    Properties:
    
@@ -9276,7 +9884,7 @@ 
    --fichier-encoding
    
 
 
    Limitations
    
 
    rclone about is not supported by the 1Fichier backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.
    
-
    See List of backends that do not support rclone about See rclone about
    
+
    See List of backends that do not support rclone about and rclone about
    
 
    Alias
    
 
    The alias remote provides a new name for another remote.
    
 
    Paths may be as deep as required or a local path, e.g. remote:directory/subdirectory or /directory/subdirectory.
    
@@ -9334,7 +9942,7 @@ 
    Configuration
    
 
    Copy another local directory to the alias directory called source
    
 
    rclone copy /home/source remote:source
    
 
    Standard options
    
-
    Here are the standard options specific to alias (Alias for an existing remote).
    
+
    Here are the Standard options specific to alias (Alias for an existing remote).
    
 
    --alias-remote
    
 
    Remote or path to alias.
    
 
    Can be "myremote:path/to/dir", "myremote:bucket", "myremote:" or "/local/path".
    
@@ -9446,7 +10054,7 @@ 
    Deleting files
    
 
    Using with non .com Amazon accounts
    
 
    Let's say you usually use amazon.co.uk. When you authenticate with rclone it will take you to an amazon.com page to log in. Your amazon.co.uk email and password should work here just fine.
    
 
    Standard options
    
-
    Here are the standard options specific to amazon cloud drive (Amazon Drive).
    
+
    Here are the Standard options specific to amazon cloud drive (Amazon Drive).
    
 
    --acd-client-id
    
 
    OAuth Client Id.
    
 
    Leave blank normally.
    
@@ -9468,7 +10076,7 @@ 
    --acd-client-secret
    
 
  • Required: false
    • 
 
 
    Advanced options
    
-
    Here are the advanced options specific to amazon cloud drive (Amazon Drive).
    
+
    Here are the Advanced options specific to amazon cloud drive (Amazon Drive).
    
 
    --acd-token
    
 
    OAuth Access Token as a JSON blob.
    
 
    Properties:
    
@@ -9549,16 +10157,21 @@ 
    Limitations
    
 
    At the time of writing (Jan 2016) is in the area of 50 GiB per file. This means that larger files are likely to fail.
    
 
    Unfortunately there is no way for rclone to see that this failure is because of file size, so it will retry the operation, as any other failure. To avoid this problem, use --max-size 50000M option to limit the maximum size of uploaded files. Note that --max-size does not split files into segments, it only ignores files over this size.
    
 
    rclone about is not supported by the Amazon Drive backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.
    
-
    See List of backends that do not support rclone about See rclone about
    
+
    See List of backends that do not support rclone about and rclone about
    
 
    Amazon S3 Storage Providers
    
 
    The S3 backend can be used with a number of different providers:
    
 
      
 
    • AWS S3
      • 
 
    • Alibaba Cloud (Aliyun) Object Storage System (OSS)
      • 
 
    • Ceph
      • 
+
    • China Mobile Ecloud Elastic Object Storage (EOS)
      • 
+
    • Cloudflare R2
      • 
+
    • Arvan Cloud Object Storage (AOS)
      • 
 
    • DigitalOcean Spaces
      • 
 
    • Dreamhost
      • 
+
    • Huawei OBS
      • 
 
    • IBM COS S3
      • 
+
    • IDrive e2
      • 
 
    • Minio
      • 
 
    • RackCorp Object Storage
      • 
 
    • Scaleway
      • 
@@ -9593,7 +10206,7 @@ 
      Configuration
      
 Type of storage to configure.
 Choose a number from below, or type in your own value
 [snip]
-XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, Dreamhost, IBM COS, Minio, and Tencent COS
+XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS
    \ "s3"
 [snip]
 Storage> s3
@@ -9979,7 +10592,7 @@ 
      Object-lock enabled S3 bucket
      
 
 
      As mentioned in the Hashes section, small files that are not uploaded as multipart, use a different tag, causing the upload to fail. A simple solution is to set the --s3-upload-cutoff 0 and force all the files to be uploaded as multipart.
      
 
      Standard options
      
-
      Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
      
+
      Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi).
      
 
      --s3-provider
      
 
      Choose your S3 provider.
      
 
      Properties:
      
@@ -10002,6 +10615,18 @@ 
      --s3-provider
      
 
        
 
      • Ceph Object Storage
        • 
 
      
+
    • "ChinaMobile"
+
        
+
      • China Mobile Ecloud Elastic Object Storage (EOS)
        • 
+
      • 
+
    • "Cloudflare"
+
        
+
      • Cloudflare R2 Storage
        • 
+
      • 
+
    • "ArvanCloud"
+
        
+
      • Arvan Cloud Object Storage (AOS)
        • 
+
      • 
 
    • "DigitalOcean"
 
        
 
      • Digital Ocean Spaces
        • 
@@ -10010,10 +10635,18 @@ 
        --s3-provider
        
 
          
 
        • Dreamhost DreamObjects
          • 
 
        
+
      • "HuaweiOBS"
+
          
+
        • Huawei Object Storage Service
          • 
+
        • 
 
      • "IBMCOS"
 
          
 
        • IBM COS S3
          • 
 
        • 
+
      • "IDrive"
+
          
+
        • IDrive e2
          • 
+
        • 
 
      • "LyveCloud"
 
          
 
        • Seagate Lyve Cloud
          • 
@@ -10348,76 +10981,326 @@ 
          --s3-region
          
 
            
 
          • Paris, France
            • 
 
          
+
        • "pl-waw"
+
            
+
          • Warsaw, Poland
            • 
+
          • 
 
        • 
 
      
 
      --s3-region
      
-
      Region to connect to.
      
-
      Leave blank if you are using an S3 clone and you don't have a region.
      
+
      Region to connect to. - the location where your bucket will be created and your data stored. Need bo be same with your endpoint.
      
 
      Properties:
      
 
        
 
      • Config: region
        • 
 
      • Env Var: RCLONE_S3_REGION
        • 
-
      • Provider: !AWS,Alibaba,RackCorp,Scaleway,Storj,TencentCOS
        • 
+
      • Provider: HuaweiOBS
        • 
 
      • Type: string
        • 
 
      • Required: false
        • 
 
      • Examples:
 
          
-
        • ""
+
        • "af-south-1"
 
            
-
          • Use this if unsure.
            • 
-
          • Will use v4 signatures and an empty region.
            • 
+
          • AF-Johannesburg
            • 
 
          • 
-
        • "other-v2-signature"
+
        • "ap-southeast-2"
 
            
-
          • Use this only if v4 signatures don't work.
            • 
-
          • E.g. pre Jewel/v10 CEPH.
            • 
+
          • AP-Bangkok
            • 
 
          • 
+
        • "ap-southeast-3"
+
            
+
          • AP-Singapore
            • 
 
          • 
-
        
-
        --s3-endpoint
        
-
        Endpoint for S3 API.
        
-
        Leave blank if using AWS to use the default endpoint for the region.
        
-
        Properties:
        
+
      • "cn-east-3"
 
          
-
        • Config: endpoint
          • 
-
        • Env Var: RCLONE_S3_ENDPOINT
          • 
-
        • Provider: AWS
          • 
-
        • Type: string
          • 
-
        • Required: false
          • 
-
        
-
        --s3-endpoint
        
-
        Endpoint for IBM COS S3 API.
        
-
        Specify if using an IBM COS On Premise.
        
-
        Properties:
        
+
      • CN East-Shanghai1
        • 
+
      • 
+
    • "cn-east-2"
 
        
-
      • Config: endpoint
        • 
-
      • Env Var: RCLONE_S3_ENDPOINT
        • 
-
      • Provider: IBMCOS
        • 
-
      • Type: string
        • 
-
      • Required: false
        • 
-
      • Examples:
+
      • CN East-Shanghai2
        • 
+
      • 
+
    • "cn-north-1"
 
        
-
      • "s3.us.cloud-object-storage.appdomain.cloud"
+
      • CN North-Beijing1
        • 
+
      • 
+
    • "cn-north-4"
 
        
-
      • US Cross Region Endpoint
        • 
+
      • CN North-Beijing4
        • 
 
      • 
-
    • "s3.dal.us.cloud-object-storage.appdomain.cloud"
+
    • "cn-south-1"
 
        
-
      • US Cross Region Dallas Endpoint
        • 
+
      • CN South-Guangzhou
        • 
 
      • 
-
    • "s3.wdc.us.cloud-object-storage.appdomain.cloud"
+
    • "ap-southeast-1"
 
        
-
      • US Cross Region Washington DC Endpoint
        • 
+
      • CN-Hong Kong
        • 
 
      • 
-
    • "s3.sjc.us.cloud-object-storage.appdomain.cloud"
+
    • "sa-argentina-1"
 
        
-
      • US Cross Region San Jose Endpoint
        • 
+
      • LA-Buenos Aires1
        • 
 
      • 
-
    • "s3.private.us.cloud-object-storage.appdomain.cloud"
+
    • "sa-peru-1"
 
        
-
      • US Cross Region Private Endpoint
        • 
+
      • LA-Lima1
        • 
 
      • 
-
    • "s3.private.dal.us.cloud-object-storage.appdomain.cloud"
+
    • "na-mexico-1"
+
        
+
      • LA-Mexico City1
        • 
+
      • 
+
    • "sa-chile-1"
+
        
+
      • LA-Santiago2
        • 
+
      • 
+
    • "sa-brazil-1"
+
        
+
      • LA-Sao Paulo1
        • 
+
      • 
+
    • "ru-northwest-2"
+
        
+
      • RU-Moscow2
        • 
+
      • 
+
    
+
+
    --s3-region
    
+
    Region to connect to.
    
+
    Properties:
    
+
      
+
    • Config: region
      • 
+
    • Env Var: RCLONE_S3_REGION
      • 
+
    • Provider: Cloudflare
      • 
+
    • Type: string
      • 
+
    • Required: false
      • 
+
    • Examples:
+
        
+
      • "auto"
+
          
+
        • R2 buckets are automatically distributed across Cloudflare's data centers for low latency.
          • 
+
        • 
+
      • 
+
    
+
    --s3-region
    
+
    Region to connect to.
    
+
    Leave blank if you are using an S3 clone and you don't have a region.
    
+
    Properties:
    
+
      
+
    • Config: region
      • 
+
    • Env Var: RCLONE_S3_REGION
      • 
+
    • Provider: !AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
      • 
+
    • Type: string
      • 
+
    • Required: false
      • 
+
    • Examples:
+
        
+
      • ""
+
          
+
        • Use this if unsure.
          • 
+
        • Will use v4 signatures and an empty region.
          • 
+
        • 
+
      • "other-v2-signature"
+
          
+
        • Use this only if v4 signatures don't work.
          • 
+
        • E.g. pre Jewel/v10 CEPH.
          • 
+
        • 
+
      • 
+
    
+
    --s3-endpoint
    
+
    Endpoint for S3 API.
    
+
    Leave blank if using AWS to use the default endpoint for the region.
    
+
    Properties:
    
+
      
+
    • Config: endpoint
      • 
+
    • Env Var: RCLONE_S3_ENDPOINT
      • 
+
    • Provider: AWS
      • 
+
    • Type: string
      • 
+
    • Required: false
      • 
+
    
+
    --s3-endpoint
    
+
    Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API.
    
+
    Properties:
    
+
      
+
    • Config: endpoint
      • 
+
    • Env Var: RCLONE_S3_ENDPOINT
      • 
+
    • Provider: ChinaMobile
      • 
+
    • Type: string
      • 
+
    • Required: false
      • 
+
    • Examples:
+
        
+
      • "eos-wuxi-1.cmecloud.cn"
+
          
+
        • The default endpoint - a good choice if you are unsure.
          • 
+
        • East China (Suzhou)
          • 
+
        • 
+
      • "eos-jinan-1.cmecloud.cn"
+
          
+
        • East China (Jinan)
          • 
+
        • 
+
      • "eos-ningbo-1.cmecloud.cn"
+
          
+
        • East China (Hangzhou)
          • 
+
        • 
+
      • "eos-shanghai-1.cmecloud.cn"
+
          
+
        • East China (Shanghai-1)
          • 
+
        • 
+
      • "eos-zhengzhou-1.cmecloud.cn"
+
          
+
        • Central China (Zhengzhou)
          • 
+
        • 
+
      • "eos-hunan-1.cmecloud.cn"
+
          
+
        • Central China (Changsha-1)
          • 
+
        • 
+
      • "eos-zhuzhou-1.cmecloud.cn"
+
          
+
        • Central China (Changsha-2)
          • 
+
        • 
+
      • "eos-guangzhou-1.cmecloud.cn"
+
          
+
        • South China (Guangzhou-2)
          • 
+
        • 
+
      • "eos-dongguan-1.cmecloud.cn"
+
          
+
        • South China (Guangzhou-3)
          • 
+
        • 
+
      • "eos-beijing-1.cmecloud.cn"
+
          
+
        • North China (Beijing-1)
          • 
+
        • 
+
      • "eos-beijing-2.cmecloud.cn"
+
          
+
        • North China (Beijing-2)
          • 
+
        • 
+
      • "eos-beijing-4.cmecloud.cn"
+
          
+
        • North China (Beijing-3)
          • 
+
        • 
+
      • "eos-huhehaote-1.cmecloud.cn"
+
          
+
        • North China (Huhehaote)
          • 
+
        • 
+
      • "eos-chengdu-1.cmecloud.cn"
+
          
+
        • Southwest China (Chengdu)
          • 
+
        • 
+
      • "eos-chongqing-1.cmecloud.cn"
+
          
+
        • Southwest China (Chongqing)
          • 
+
        • 
+
      • "eos-guiyang-1.cmecloud.cn"
+
          
+
        • Southwest China (Guiyang)
          • 
+
        • 
+
      • "eos-xian-1.cmecloud.cn"
+
          
+
        • Nouthwest China (Xian)
          • 
+
        • 
+
      • "eos-yunnan.cmecloud.cn"
+
          
+
        • Yunnan China (Kunming)
          • 
+
        • 
+
      • "eos-yunnan-2.cmecloud.cn"
+
          
+
        • Yunnan China (Kunming-2)
          • 
+
        • 
+
      • "eos-tianjin-1.cmecloud.cn"
+
          
+
        • Tianjin China (Tianjin)
          • 
+
        • 
+
      • "eos-jilin-1.cmecloud.cn"
+
          
+
        • Jilin China (Changchun)
          • 
+
        • 
+
      • "eos-hubei-1.cmecloud.cn"
+
          
+
        • Hubei China (Xiangyan)
          • 
+
        • 
+
      • "eos-jiangxi-1.cmecloud.cn"
+
          
+
        • Jiangxi China (Nanchang)
          • 
+
        • 
+
      • "eos-gansu-1.cmecloud.cn"
+
          
+
        • Gansu China (Lanzhou)
          • 
+
        • 
+
      • "eos-shanxi-1.cmecloud.cn"
+
          
+
        • Shanxi China (Taiyuan)
          • 
+
        • 
+
      • "eos-liaoning-1.cmecloud.cn"
+
          
+
        • Liaoning China (Shenyang)
          • 
+
        • 
+
      • "eos-hebei-1.cmecloud.cn"
+
          
+
        • Hebei China (Shijiazhuang)
          • 
+
        • 
+
      • "eos-fujian-1.cmecloud.cn"
+
          
+
        • Fujian China (Xiamen)
          • 
+
        • 
+
      • "eos-guangxi-1.cmecloud.cn"
+
          
+
        • Guangxi China (Nanning)
          • 
+
        • 
+
      • "eos-anhui-1.cmecloud.cn"
+
          
+
        • Anhui China (Huainan)
          • 
+
        • 
+
      • 
+
    
+
    --s3-endpoint
    
+
    Endpoint for Arvan Cloud Object Storage (AOS) API.
    
+
    Properties:
    
+
      
+
    • Config: endpoint
      • 
+
    • Env Var: RCLONE_S3_ENDPOINT
      • 
+
    • Provider: ArvanCloud
      • 
+
    • Type: string
      • 
+
    • Required: false
      • 
+
    • Examples:
+
        
+
      • "s3.ir-thr-at1.arvanstorage.com"
+
          
+
        • The default endpoint - a good choice if you are unsure.
          • 
+
        • Tehran Iran (Asiatech)
          • 
+
        • 
+
      • "s3.ir-tbz-sh1.arvanstorage.com"
+
          
+
        • Tabriz Iran (Shahriar)
          • 
+
        • 
+
      • 
+
    
+
    --s3-endpoint
    
+
    Endpoint for IBM COS S3 API.
    
+
    Specify if using an IBM COS On Premise.
    
+
    Properties:
    
+
      
+
    • Config: endpoint
      • 
+
    • Env Var: RCLONE_S3_ENDPOINT
      • 
+
    • Provider: IBMCOS
      • 
+
    • Type: string
      • 
+
    • Required: false
      • 
+
    • Examples:
+
        
+
      • "s3.us.cloud-object-storage.appdomain.cloud"
+
          
+
        • US Cross Region Endpoint
          • 
+
        • 
+
      • "s3.dal.us.cloud-object-storage.appdomain.cloud"
+
          
+
        • US Cross Region Dallas Endpoint
          • 
+
        • 
+
      • "s3.wdc.us.cloud-object-storage.appdomain.cloud"
+
          
+
        • US Cross Region Washington DC Endpoint
          • 
+
        • 
+
      • "s3.sjc.us.cloud-object-storage.appdomain.cloud"
+
          
+
        • US Cross Region San Jose Endpoint
          • 
+
        • 
+
      • "s3.private.us.cloud-object-storage.appdomain.cloud"
+
          
+
        • US Cross Region Private Endpoint
          • 
+
        • 
+
      • "s3.private.dal.us.cloud-object-storage.appdomain.cloud"
 
          
 
        • US Cross Region Dallas Private Endpoint
          • 
 
        • 
@@ -10647,7 +11530,7 @@ 
        --s3-endpoint
        
 
      • 
 
    
 
-
    --s3-endpoint
    
+
    --s3-endpoint
    
 
    Endpoint for OSS API.
    
 
    Properties:
    
 
      
@@ -10760,7 +11643,80 @@ 
      --s3-endpoint
      
 
    
 
 
-
    --s3-endpoint
    
+
    --s3-endpoint
    
+
    Endpoint for OBS API.
    
+
    Properties:
    
+
      
+
    • Config: endpoint
      • 
+
    • Env Var: RCLONE_S3_ENDPOINT
      • 
+
    • Provider: HuaweiOBS
      • 
+
    • Type: string
      • 
+
    • Required: false
      • 
+
    • Examples:
+
        
+
      • "obs.af-south-1.myhuaweicloud.com"
+
          
+
        • AF-Johannesburg
          • 
+
        • 
+
      • "obs.ap-southeast-2.myhuaweicloud.com"
+
          
+
        • AP-Bangkok
          • 
+
        • 
+
      • "obs.ap-southeast-3.myhuaweicloud.com"
+
          
+
        • AP-Singapore
          • 
+
        • 
+
      • "obs.cn-east-3.myhuaweicloud.com"
+
          
+
        • CN East-Shanghai1
          • 
+
        • 
+
      • "obs.cn-east-2.myhuaweicloud.com"
+
          
+
        • CN East-Shanghai2
          • 
+
        • 
+
      • "obs.cn-north-1.myhuaweicloud.com"
+
          
+
        • CN North-Beijing1
          • 
+
        • 
+
      • "obs.cn-north-4.myhuaweicloud.com"
+
          
+
        • CN North-Beijing4
          • 
+
        • 
+
      • "obs.cn-south-1.myhuaweicloud.com"
+
          
+
        • CN South-Guangzhou
          • 
+
        • 
+
      • "obs.ap-southeast-1.myhuaweicloud.com"
+
          
+
        • CN-Hong Kong
          • 
+
        • 
+
      • "obs.sa-argentina-1.myhuaweicloud.com"
+
          
+
        • LA-Buenos Aires1
          • 
+
        • 
+
      • "obs.sa-peru-1.myhuaweicloud.com"
+
          
+
        • LA-Lima1
          • 
+
        • 
+
      • "obs.na-mexico-1.myhuaweicloud.com"
+
          
+
        • LA-Mexico City1
          • 
+
        • 
+
      • "obs.sa-chile-1.myhuaweicloud.com"
+
          
+
        • LA-Santiago2
          • 
+
        • 
+
      • "obs.sa-brazil-1.myhuaweicloud.com"
+
          
+
        • LA-Sao Paulo1
          • 
+
        • 
+
      • "obs.ru-northwest-2.myhuaweicloud.com"
+
          
+
        • RU-Moscow2
          • 
+
        • 
+
      • 
+
    
+
    --s3-endpoint
    
 
    Endpoint for Scaleway Object Storage.
    
 
    Properties:
    
 
      
@@ -10779,9 +11735,13 @@ 
      --s3-endpoint
      
 
        
 
      • Paris Endpoint
        • 
 
      
+
    • "s3.pl-waw.scw.cloud"
+
        
+
      • Warsaw Endpoint
        • 
+
      • 
 
    
 
-
    --s3-endpoint
    
+
    --s3-endpoint
    
 
    Endpoint for StackPath Object Storage.
    
 
    Properties:
    
 
      
@@ -10806,7 +11766,7 @@ 
      --s3-endpoint
      
 
    
 
 
-
    --s3-endpoint
    
+
    --s3-endpoint
    
 
    Endpoint of the Shared Gateway.
    
 
    Properties:
    
 
      
@@ -10831,7 +11791,7 @@ 
      --s3-endpoint
      
 
    
 
 
-
    --s3-endpoint
    
+
    --s3-endpoint
    
 
    Endpoint for Tencent COS API.
    
 
    Properties:
    
 
      
@@ -10920,7 +11880,7 @@ 
      --s3-endpoint
      
 
    
 
 
-
    --s3-endpoint
    
+
    --s3-endpoint
    
 
    Endpoint for RackCorp Object Storage.
    
 
    Properties:
    
 
      
@@ -11009,14 +11969,14 @@ 
      --s3-endpoint
      
 
    
 
 
-
    --s3-endpoint
    
+
    --s3-endpoint
    
 
    Endpoint for S3 API.
    
 
    Required when using an S3 clone.
    
 
    Properties:
    
 
      
 
    • Config: endpoint
      • 
 
    • Env Var: RCLONE_S3_ENDPOINT
      • 
-
    • Provider: !AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,Storj,RackCorp
      • 
+
    • Provider: !AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp
      • 
 
    • Type: string
      • 
 
    • Required: false
      • 
 
    • Examples:
@@ -11073,6 +12033,10 @@ 
      --s3-endpoint
      
 
        
 
      • Wasabi AP Northeast 2 (Osaka) endpoint
        • 
 
      • 
+
    • "s3.ir-thr-at1.arvanstorage.com"
+
        
+
      • ArvanCloud Tehran Iran (Asiatech) endpoint
        • 
+
      • 
 
    
 
 
    --s3-location-constraint
    
@@ -11190,52 +12154,208 @@ 
    --s3-location-constraint
    
 
 
 
    --s3-location-constraint
    
-
    Location constraint - must match endpoint when using IBM Cloud Public.
    
-
    For on-prem COS, do not make a selection from this list, hit enter.
    
+
    Location constraint - must match endpoint.
    
+
    Used when creating buckets only.
    
 
    Properties:
    
 
      
 
    • Config: location_constraint
      • 
 
    • Env Var: RCLONE_S3_LOCATION_CONSTRAINT
      • 
-
    • Provider: IBMCOS
      • 
+
    • Provider: ChinaMobile
      • 
 
    • Type: string
      • 
 
    • Required: false
      • 
 
    • Examples:
 
        
-
      • "us-standard"
+
      • "wuxi1"
 
          
-
        • US Cross Region Standard
          • 
+
        • East China (Suzhou)
          • 
 
        • 
-
      • "us-vault"
+
      • "jinan1"
 
          
-
        • US Cross Region Vault
          • 
+
        • East China (Jinan)
          • 
 
        • 
-
      • "us-cold"
+
      • "ningbo1"
 
          
-
        • US Cross Region Cold
          • 
+
        • East China (Hangzhou)
          • 
 
        • 
-
      • "us-flex"
+
      • "shanghai1"
 
          
-
        • US Cross Region Flex
          • 
+
        • East China (Shanghai-1)
          • 
 
        • 
-
      • "us-east-standard"
+
      • "zhengzhou1"
 
          
-
        • US East Region Standard
          • 
+
        • Central China (Zhengzhou)
          • 
 
        • 
-
      • "us-east-vault"
+
      • "hunan1"
 
          
-
        • US East Region Vault
          • 
+
        • Central China (Changsha-1)
          • 
 
        • 
-
      • "us-east-cold"
+
      • "zhuzhou1"
 
          
-
        • US East Region Cold
          • 
+
        • Central China (Changsha-2)
          • 
 
        • 
-
      • "us-east-flex"
+
      • "guangzhou1"
 
          
-
        • US East Region Flex
          • 
+
        • South China (Guangzhou-2)
          • 
 
        • 
-
      • "us-south-standard"
+
      • "dongguan1"
 
          
-
        • US South Region Standard
          • 
+
        • South China (Guangzhou-3)
          • 
+
        • 
+
      • "beijing1"
+
          
+
        • North China (Beijing-1)
          • 
+
        • 
+
      • "beijing2"
+
          
+
        • North China (Beijing-2)
          • 
+
        • 
+
      • "beijing4"
+
          
+
        • North China (Beijing-3)
          • 
+
        • 
+
      • "huhehaote1"
+
          
+
        • North China (Huhehaote)
          • 
+
        • 
+
      • "chengdu1"
+
          
+
        • Southwest China (Chengdu)
          • 
+
        • 
+
      • "chongqing1"
+
          
+
        • Southwest China (Chongqing)
          • 
+
        • 
+
      • "guiyang1"
+
          
+
        • Southwest China (Guiyang)
          • 
+
        • 
+
      • "xian1"
+
          
+
        • Nouthwest China (Xian)
          • 
+
        • 
+
      • "yunnan"
+
          
+
        • Yunnan China (Kunming)
          • 
+
        • 
+
      • "yunnan2"
+
          
+
        • Yunnan China (Kunming-2)
          • 
+
        • 
+
      • "tianjin1"
+
          
+
        • Tianjin China (Tianjin)
          • 
+
        • 
+
      • "jilin1"
+
          
+
        • Jilin China (Changchun)
          • 
+
        • 
+
      • "hubei1"
+
          
+
        • Hubei China (Xiangyan)
          • 
+
        • 
+
      • "jiangxi1"
+
          
+
        • Jiangxi China (Nanchang)
          • 
+
        • 
+
      • "gansu1"
+
          
+
        • Gansu China (Lanzhou)
          • 
+
        • 
+
      • "shanxi1"
+
          
+
        • Shanxi China (Taiyuan)
          • 
+
        • 
+
      • "liaoning1"
+
          
+
        • Liaoning China (Shenyang)
          • 
+
        • 
+
      • "hebei1"
+
          
+
        • Hebei China (Shijiazhuang)
          • 
+
        • 
+
      • "fujian1"
+
          
+
        • Fujian China (Xiamen)
          • 
+
        • 
+
      • "guangxi1"
+
          
+
        • Guangxi China (Nanning)
          • 
+
        • 
+
      • "anhui1"
+
          
+
        • Anhui China (Huainan)
          • 
+
        • 
+
      • 
+
    
+
    --s3-location-constraint
    
+
    Location constraint - must match endpoint.
    
+
    Used when creating buckets only.
    
+
    Properties:
    
+
      
+
    • Config: location_constraint
      • 
+
    • Env Var: RCLONE_S3_LOCATION_CONSTRAINT
      • 
+
    • Provider: ArvanCloud
      • 
+
    • Type: string
      • 
+
    • Required: false
      • 
+
    • Examples:
+
        
+
      • "ir-thr-at1"
+
          
+
        • Tehran Iran (Asiatech)
          • 
+
        • 
+
      • "ir-tbz-sh1"
+
          
+
        • Tabriz Iran (Shahriar)
          • 
+
        • 
+
      • 
+
    
+
    --s3-location-constraint
    
+
    Location constraint - must match endpoint when using IBM Cloud Public.
    
+
    For on-prem COS, do not make a selection from this list, hit enter.
    
+
    Properties:
    
+
      
+
    • Config: location_constraint
      • 
+
    • Env Var: RCLONE_S3_LOCATION_CONSTRAINT
      • 
+
    • Provider: IBMCOS
      • 
+
    • Type: string
      • 
+
    • Required: false
      • 
+
    • Examples:
+
        
+
      • "us-standard"
+
          
+
        • US Cross Region Standard
          • 
+
        • 
+
      • "us-vault"
+
          
+
        • US Cross Region Vault
          • 
+
        • 
+
      • "us-cold"
+
          
+
        • US Cross Region Cold
          • 
+
        • 
+
      • "us-flex"
+
          
+
        • US Cross Region Flex
          • 
+
        • 
+
      • "us-east-standard"
+
          
+
        • US East Region Standard
          • 
+
        • 
+
      • "us-east-vault"
+
          
+
        • US East Region Vault
          • 
+
        • 
+
      • "us-east-cold"
+
          
+
        • US East Region Cold
          • 
+
        • 
+
      • "us-east-flex"
+
          
+
        • US East Region Flex
          • 
+
        • 
+
      • "us-south-standard"
+
          
+
        • US South Region Standard
          • 
 
        • 
 
      • "us-south-vault"
 
          
@@ -11331,7 +12451,7 @@ 
          --s3-location-constraint
          
 
        • 
 
      • 
 
    
-
    --s3-location-constraint
    
+
    --s3-location-constraint
    
 
    Location constraint - the location where your bucket will be located and your data stored.
    
 
    Properties:
    
 
      
@@ -11420,14 +12540,14 @@ 
      --s3-location-constraint
      
 
    
 
 
-
    --s3-location-constraint
    
+
    --s3-location-constraint
    
 
    Location constraint - must be set to match the Region.
    
 
    Leave blank if not sure. Used when creating buckets only.
    
 
    Properties:
    
 
      
 
    • Config: location_constraint
      • 
 
    • Env Var: RCLONE_S3_LOCATION_CONSTRAINT
      • 
-
    • Provider: !AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,Storj,TencentCOS
      • 
+
    • Provider: !AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS
      • 
 
    • Type: string
      • 
 
    • Required: false
      • 
 
    
@@ -11440,7 +12560,7 @@ 
    --s3-acl
    
 
      
 
    • Config: acl
      • 
 
    • Env Var: RCLONE_S3_ACL
      • 
-
    • Provider: !Storj
      • 
+
    • Provider: !Storj,Cloudflare
      • 
 
    • Type: string
      • 
 
    • Required: false
      • 
 
    • Examples:
@@ -11515,7 +12635,7 @@ 
      --s3-server-side-encryption
      
 
        
 
      • Config: server_side_encryption
        • 
 
      • Env Var: RCLONE_S3_SERVER_SIDE_ENCRYPTION
        • 
-
      • Provider: AWS,Ceph,Minio
        • 
+
      • Provider: AWS,Ceph,ChinaMobile,Minio
        • 
 
      • Type: string
        • 
 
      • Required: false
        • 
 
      • Examples:
@@ -11634,6 +12754,52 @@ 
        --s3-storage-class
        
 
      • 
 
    
 
    --s3-storage-class
    
+
    The storage class to use when storing new objects in ChinaMobile.
    
+
    Properties:
    
+
      
+
    • Config: storage_class
      • 
+
    • Env Var: RCLONE_S3_STORAGE_CLASS
      • 
+
    • Provider: ChinaMobile
      • 
+
    • Type: string
      • 
+
    • Required: false
      • 
+
    • Examples:
+
        
+
      • ""
+
          
+
        • Default
          • 
+
        • 
+
      • "STANDARD"
+
          
+
        • Standard storage class
          • 
+
        • 
+
      • "GLACIER"
+
          
+
        • Archive storage mode
          • 
+
        • 
+
      • "STANDARD_IA"
+
          
+
        • Infrequent access storage mode
          • 
+
        • 
+
      • 
+
    
+
    --s3-storage-class
    
+
    The storage class to use when storing new objects in ArvanCloud.
    
+
    Properties:
    
+
      
+
    • Config: storage_class
      • 
+
    • Env Var: RCLONE_S3_STORAGE_CLASS
      • 
+
    • Provider: ArvanCloud
      • 
+
    • Type: string
      • 
+
    • Required: false
      • 
+
    • Examples:
+
        
+
      • "STANDARD"
+
          
+
        • Standard storage class
          • 
+
        • 
+
      • 
+
    
+
    --s3-storage-class
    
 
    The storage class to use when storing new objects in Tencent COS.
    
 
    Properties:
    
 
      
@@ -11662,7 +12828,7 @@ 
      --s3-storage-class
      
 
    
 
 
-
    --s3-storage-class
    
+
    --s3-storage-class
    
 
    The storage class to use when storing new objects in S3.
    
 
    Properties:
    
 
      
@@ -11690,7 +12856,7 @@ 
      --s3-storage-class
      
 
    
 
 
    Advanced options
    
-
    Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
    
+
    Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi).
    
 
    --s3-bucket-acl
    
 
    Canned ACL used when creating buckets.
    
 
    For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
    
@@ -11742,7 +12908,7 @@ 
    --s3-sse-customer-algorithm
    
 
      
 
    • Config: sse_customer_algorithm
      • 
 
    • Env Var: RCLONE_S3_SSE_CUSTOMER_ALGORITHM
      • 
-
    • Provider: AWS,Ceph,Minio
      • 
+
    • Provider: AWS,Ceph,ChinaMobile,Minio
      • 
 
    • Type: string
      • 
 
    • Required: false
      • 
 
    • Examples:
@@ -11763,7 +12929,7 @@ 
      --s3-sse-customer-key
      
 
        
 
      • Config: sse_customer_key
        • 
 
      • Env Var: RCLONE_S3_SSE_CUSTOMER_KEY
        • 
-
      • Provider: AWS,Ceph,Minio
        • 
+
      • Provider: AWS,Ceph,ChinaMobile,Minio
        • 
 
      • Type: string
        • 
 
      • Required: false
        • 
 
      • Examples:
@@ -11781,7 +12947,7 @@ 
        --s3-sse-customer-key-md5
        
 
          
 
        • Config: sse_customer_key_md5
          • 
 
        • Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_MD5
          • 
-
        • Provider: AWS,Ceph,Minio
          • 
+
        • Provider: AWS,Ceph,ChinaMobile,Minio
          • 
 
        • Type: string
          • 
 
        • Required: false
          • 
 
        • Examples:
@@ -11809,6 +12975,7 @@ 
          --s3-chunk-size
          
 
          If you are transferring large files over high-speed links and you have enough memory, then increasing this will speed up the transfers.
          
 
          Rclone will automatically increase the chunk size when uploading a large file of known size to stay below the 10,000 chunks limit.
          
 
          Files of unknown size are uploaded with the configured chunk_size. Since the default chunk size is 5 MiB and there can be at most 10,000 chunks, this means that by default the maximum size of a file you can stream upload is 48 GiB. If you wish to stream upload larger files then you will need to increase chunk_size.
          
+
          Increasing the chunk size decreases the accuracy of the progress statistics displayed with "-P" flag. Rclone treats chunk as sent when it's buffered by the AWS SDK, when in fact it may still be uploading. A bigger chunk size means a bigger AWS SDK buffer and progress reporting more deviating from the truth.
          
 
          Properties:
          
 
            
 
          • Config: chunk_size
            • 
@@ -12073,12 +13240,103 @@ 
            --s3-use-multipart-etag
            
 
          • Type: Tristate
            • 
 
          • Default: unset
            • 
 
          
+
          --s3-use-presigned-request
          
+
          Whether to use a presigned request or PutObject for single part uploads
          
+
          If this is false rclone will use PutObject from the AWS SDK to upload an object.
          
+
          Versions of rclone < 1.59 use presigned requests to upload a single part object and setting this flag to true will re-enable that functionality. This shouldn't be necessary except in exceptional circumstances or for testing.
          
+
          Properties:
          
+
            
+
          • Config: use_presigned_request
            • 
+
          • Env Var: RCLONE_S3_USE_PRESIGNED_REQUEST
            • 
+
          • Type: bool
            • 
+
          • Default: false
            • 
+
          
+
          Metadata
          
+
          User metadata is stored as x-amz-meta- keys. S3 metadata keys are case insensitive and are always returned in lower case.
          
+
          Here are the possible system metadata items for the s3 backend.
          
+
+++++++
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
          






          NameHelpTypeExampleRead Only
          btimeTime of file birth (creation) read from Last-Modified headerRFC 33392006-01-02T15:04:05.999999999Z07:00Y
          cache-controlCache-Control headerstringno-cacheN
          content-dispositionContent-Disposition headerstringinlineN
          content-encodingContent-Encoding headerstringgzipN
          content-languageContent-Language headerstringen-USN
          content-typeContent-Type headerstringtext/plainN
          mtimeTime of last modification, read from rclone metadataRFC 33392006-01-02T15:04:05.999999999Z07:00N
          tierTier of the objectstringGLACIERY
          
+
          See the metadata docs for more info.
          
 
          Backend commands
          
 
          Here are the commands specific to the s3 backend.
          
 
          Run them with
          
 
          rclone backend COMMAND remote:
          
 
          The help below will explain what arguments each command takes.
          
-
          See the "rclone backend" command for more info on how to pass options and arguments.
          
+
          See the backend command for more info on how to pass options and arguments.
          
 
          These can be run on a running backend using the rc command backend/command.
          
 
          restore
          
 
          Restore objects from GLACIER to normal storage
          
@@ -12170,7 +13428,9 @@ 
          AWS S3
          
 
          This is the provider used as main example and described in the configuration section above.
          
 
          AWS Snowball Edge
          
 
          AWS Snowball is a hardware appliance used for transferring bulk data back to AWS. Its main software interface is S3 object storage.
          
-
          To use rclone with AWS Snowball Edge devices, configure as standard for an 'S3 Compatible Service' be sure to set upload_cutoff = 0 otherwise you will run into authentication header issues as the snowball device does not support query parameter based authentication.
          
+
          To use rclone with AWS Snowball Edge devices, configure as standard for an 'S3 Compatible Service'.
          
+
          If using rclone pre v1.59 be sure to set upload_cutoff = 0 otherwise you will run into authentication header issues as the snowball device does not support query parameter based authentication.
          
+
          With rclone v1.59 or later setting upload_cutoff should not be necessary.
          
 
          eg.
          
 
          [snowball]
 type = s3
@@ -12194,7 +13454,7 @@ 
          Ceph
          
 acl =
 server_side_encryption =
 storage_class =
          
-
          If you are using an older version of CEPH, e.g. 10.2.x Jewel, then you may need to supply the parameter --s3-upload-cutoff 0 or put this in the config file as upload_cutoff 0 to work around a bug which causes uploading of small files to fail.
          
+
          If you are using an older version of CEPH (e.g. 10.2.x Jewel) and a version of rclone before v1.59 then you may need to supply the parameter --s3-upload-cutoff 0 or put this in the config file as upload_cutoff 0 to work around a bug which causes uploading of small files to fail.
          
 
          Note also that Ceph sometimes puts / in the passwords it gives users. If you read the secret access key using the command line tools you will get a JSON blob with the / escaped as \/. Make sure you only write / in the secret access key.
          
 
          Eg the dump from Ceph looks something like this (irrelevant keys removed).
          
 
          {
@@ -12209,6 +13469,86 @@ 
          Ceph
          
     ],
 }
          
 
          Because this is a json dump, it is encoding the / as \/, so if you use the secret key as xxxxxx/xxxx it will work fine.
          
+
          Cloudflare R2
          
+
          Cloudflare R2 Storage allows developers to store large amounts of unstructured data without the costly egress bandwidth fees associated with typical cloud storage services.
          
+
          Here is an example of making a Cloudflare R2 configuration. First run:
          
+
          rclone config
          
+
          This will guide you through an interactive setup process.
          
+
          Note that all buckets are private, and all are stored in the same "auto" region. It is necessary to use Cloudflare workers to share the content of a bucket publicly.
          
+
          No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> r2
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+...
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+   \ (s3)
+...
+Storage> s3
+Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+...
+XX / Cloudflare R2 Storage
+   \ (Cloudflare)
+...
+provider> Cloudflare
+Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth> 1
+Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> ACCESS_KEY
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> SECRET_ACCESS_KEY
+Option region.
+Region to connect to.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / R2 buckets are automatically distributed across Cloudflare's data centers for low latency.
+   \ (auto)
+region> 1
+Option endpoint.
+Endpoint for S3 API.
+Required when using an S3 clone.
+Enter a value. Press Enter to leave empty.
+endpoint> https://ACCOUNT_ID.r2.cloudflarestorage.com
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
          
+
          This will leave your config looking something like:
          
+
          [r2]
+type = s3
+provider = Cloudflare
+access_key_id = ACCESS_KEY
+secret_access_key = SECRET_ACCESS_KEY
+region = auto
+endpoint = https://ACCOUNT_ID.r2.cloudflarestorage.com
+acl = private
          
+
          Now run rclone lsf r2: to see your buckets and rclone lsf r2:bucket to look within a bucket.
          
 
          Dreamhost
          
 
          Dreamhost DreamObjects is an object storage system based on CEPH.
          
 
          To use rclone with Dreamhost, configure as above but leave the region blank and set the endpoint. You should end up with something like this in your config:
          
@@ -12254,15 +13594,135 @@ 
          DigitalOcean Spaces
          
 
          Once configured, you can create a new Space and begin copying files. For example:
          
 
          rclone mkdir spaces:my-new-space
 rclone copy /path/to/files spaces:my-new-space
          
-
          IBM COS (S3)
          
-
          Information stored with IBM Cloud Object Storage is encrypted and dispersed across multiple geographic locations, and accessed through an implementation of the S3 API. This service makes use of the distributed storage technologies provided by IBM’s Cloud Object Storage System (formerly Cleversafe). For more information visit: (http://www.ibm.com/cloud/object-storage)
          
-
          To configure access to IBM COS S3, follow the steps below:
          
-
            
-
          1. Run rclone config and select n for a new remote.
            2. 
-
          
-
              2018/02/14 14:13:11 NOTICE: Config file "C:\\Users\\a\\.config\\rclone\\rclone.conf" not found - using defaults
-    No remotes found, make a new one?
-    n) New remote
+
          Huawei OBS
          
+
          Object Storage Service (OBS) provides stable, secure, efficient, and easy-to-use cloud storage that lets you store virtually any volume of unstructured data in any format and access it from anywhere.
          
+
          OBS provides an S3 interface, you can copy and modify the following configuration and add it to your rclone configuration file.
          
+
          [obs]
+type = s3
+provider = HuaweiOBS
+access_key_id = your-access-key-id
+secret_access_key = your-secret-access-key
+region = af-south-1
+endpoint = obs.af-south-1.myhuaweicloud.com
+acl = private
          
+
          Or you can also configure via the interactive command line:
          
+
          No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> obs
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+   \ (s3)
+[snip]
+Storage> 5
+Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+[snip]
+ 9 / Huawei Object Storage Service
+   \ (HuaweiOBS)
+[snip]
+provider> 9
+Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth> 1
+Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> your-access-key-id
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> your-secret-access-key
+Option region.
+Region to connect to.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / AF-Johannesburg
+   \ (af-south-1)
+ 2 / AP-Bangkok
+   \ (ap-southeast-2)
+[snip]
+region> 1
+Option endpoint.
+Endpoint for OBS API.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / AF-Johannesburg
+   \ (obs.af-south-1.myhuaweicloud.com)
+ 2 / AP-Bangkok
+   \ (obs.ap-southeast-2.myhuaweicloud.com)
+[snip]
+endpoint> 1
+Option acl.
+Canned ACL used when creating buckets and storing or copying objects.
+This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+[snip]
+acl> 1
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n>
+--------------------
+[obs]
+type = s3
+provider = HuaweiOBS
+access_key_id = your-access-key-id
+secret_access_key = your-secret-access-key
+region = af-south-1
+endpoint = obs.af-south-1.myhuaweicloud.com
+acl = private
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+Current remotes:
+
+Name                 Type
+====                 ====
+obs                  s3
+
+e) Edit existing remote
+n) New remote
+d) Delete remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+e/n/d/r/c/s/q> q
          
+
          IBM COS (S3)
          
+
          Information stored with IBM Cloud Object Storage is encrypted and dispersed across multiple geographic locations, and accessed through an implementation of the S3 API. This service makes use of the distributed storage technologies provided by IBM’s Cloud Object Storage System (formerly Cleversafe). For more information visit: (http://www.ibm.com/cloud/object-storage)
          
+
          To configure access to IBM COS S3, follow the steps below:
          
+
            
+
          1. Run rclone config and select n for a new remote.
            2. 
+
          
+
              2018/02/14 14:13:11 NOTICE: Config file "C:\\Users\\a\\.config\\rclone\\rclone.conf" not found - using defaults
+    No remotes found, make a new one?
+    n) New remote
     s) Set configuration password
     q) Quit config
     n/s/q> n
          
@@ -12278,12 +13738,12 @@ 
          IBM COS (S3)
          
     \ "alias"
     2 / Amazon Drive
     \ "amazon cloud drive"
-    3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, Minio, IBM COS)
+    3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS)
     \ "s3"
     4 / Backblaze B2
     \ "b2"
 [snip]
-    23 / http Connection
+    23 / HTTP
     \ "http"
 Storage> 3
          
 
            
@@ -12411,6 +13871,108 @@ 
            IBM COS (S3)
            
         rclone copy IBM-COS-XREGION:newbucket/file.txt .
     6)  Delete a file on remote.
         rclone delete IBM-COS-XREGION:newbucket/file.txt
    +

    IDrive e2

    +

    Here is an example of making an IDrive e2 configuration. First run:

    +
    rclone config
    +

    This will guide you through an interactive setup process.

    +
    No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+
+Enter name for new remote.
+name> e2
+
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+   \ (s3)
+[snip]
+Storage> s3
+
+Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+[snip]
+XX / IDrive e2
+   \ (IDrive)
+[snip]
+provider> IDrive
+
+Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth> 
+
+Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> YOUR_ACCESS_KEY
+
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> YOUR_SECRET_KEY
+
+Option acl.
+Canned ACL used when creating buckets and storing or copying objects.
+This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+   / Owner gets FULL_CONTROL.
+ 2 | The AllUsers group gets READ access.
+   \ (public-read)
+   / Owner gets FULL_CONTROL.
+ 3 | The AllUsers group gets READ and WRITE access.
+   | Granting this on a bucket is generally not recommended.
+   \ (public-read-write)
+   / Owner gets FULL_CONTROL.
+ 4 | The AuthenticatedUsers group gets READ access.
+   \ (authenticated-read)
+   / Object owner gets FULL_CONTROL.
+ 5 | Bucket owner gets READ access.
+   | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
+   \ (bucket-owner-read)
+   / Both the object owner and the bucket owner get FULL_CONTROL over the object.
+ 6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it.
+   \ (bucket-owner-full-control)
+acl> 
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> 
+
+Configuration complete.
+Options:
+- type: s3
+- provider: IDrive
+- access_key_id: YOUR_ACCESS_KEY
+- secret_access_key: YOUR_SECRET_KEY
+- endpoint: q9d9.la12.idrivee2-5.com
+Keep this "e2" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y

    Minio

    Minio is an object storage server built for cloud application developers and devops.

    It is very easy to install and provides an S3 compatible server which can be used by rclone.

    @@ -12485,6 +14047,7 @@

    Scaleway

    acl = private server_side_encryption = storage_class = +

    C14 Cold Storage is the low-cost S3 Glacier alternative from Scaleway and it works the same way as on S3 by accepting the "GLACIER" storage_class. So you can configure your remote with the storage_class = GLACIER option to upload directly to C14. Don't forget that in this state you can't read files back after, you will need to restore them to "STANDARD" storage_class first before being able to read them (see "restore" section above)

    Seagate Lyve Cloud

    Seagate Lyve Cloud is an S3 compatible object storage platform from Seagate intended for enterprise use.

    Here is a config run through for a remote called remote - you may choose a different name of course. Note that to create an access key and secret key you will need to create a service account first.

    @@ -12499,7 +14062,7 @@

    Seagate Lyve Cloud

    Type of storage to configure.
 Choose a number from below, or type in your own value.
 [snip]
-XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
    \ (s3)
 [snip]
 Storage> s3
    @@ -12624,7 +14187,7 @@

    Wasabi

    Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 (also Dreamhost, Ceph, Minio) +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) \ "s3" [snip] Storage> s3 @@ -12726,7 +14289,7 @@

    Alibaba OSS

    Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [snip] - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -12814,83 +14377,420 @@

    Alibaba OSS

    e) Edit this remote d) Delete this remote y/e/d> y -

    Tencent COS

    -

    Tencent Cloud Object Storage (COS) is a distributed storage service offered by Tencent Cloud for unstructured data. It is secure, stable, massive, convenient, low-delay and low-cost.

    -

    To configure access to Tencent COS, follow the steps below:

    -
      -
    1. Run rclone config and select n for a new remote.
      2. -
    -
    rclone config
-No remotes found, make a new one?
+
    China Mobile Ecloud Elastic Object Storage (EOS)
    
+
    Here is an example of making an China Mobile Ecloud Elastic Object Storage (EOS) configuration. First run:
    
+
    rclone config
    
+
    This will guide you through an interactive setup process.
    
+
    No remotes found, make a new one?
 n) New remote
 s) Set configuration password
 q) Quit config
-n/s/q> n
    
-
      
-
    1. Give the name of the configuration. For example, name it 'cos'.
      2. 
-
    
-
    name> cos
    
-
      
-
    1. Select s3 storage.
      2. 
-
    
-
    Choose a number from below, or type in your own value
-1 / 1Fichier
-   \ "fichier"
- 2 / Alias for an existing remote
-   \ "alias"
- 3 / Amazon Drive
-   \ "amazon cloud drive"
- 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
-   \ "s3"
-[snip]
-Storage> s3
    
-
      
-
    1. Select TencentCOS provider.
      2. 
-
    
-
    Choose a number from below, or type in your own value
-1 / Amazon Web Services (AWS) S3
-   \ "AWS"
-[snip]
-11 / Tencent Cloud Object Storage (COS)
-   \ "TencentCOS"
-[snip]
-provider> TencentCOS
    
-
      
-
    1. Enter your SecretId and SecretKey of Tencent Cloud.
      2. 
-
    
-
    Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+n/s/q> n
+name> ChinaMobile
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+ ...
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
+   \ (s3)
+ ...
+Storage> s3
+Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ ...
+ 4 / China Mobile Ecloud Elastic Object Storage (EOS)
+   \ (ChinaMobile)
+ ...
+provider> ChinaMobile
+Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
 Only applies if access_key_id and secret_access_key is blank.
-Enter a boolean value (true or false). Press Enter for the default ("false").
-Choose a number from below, or type in your own value
- 1 / Enter AWS credentials in the next step
-   \ "false"
- 2 / Get AWS credentials from the environment (env vars or IAM)
-   \ "true"
-env_auth> 1
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth>
+Option access_key_id.
 AWS Access Key ID.
 Leave blank for anonymous access or runtime credentials.
-Enter a string value. Press Enter for the default ("").
-access_key_id> AKIDxxxxxxxxxx
-AWS Secret Access Key (password)
+Enter a value. Press Enter to leave empty.
+access_key_id> accesskeyid
+Option secret_access_key.
+AWS Secret Access Key (password).
 Leave blank for anonymous access or runtime credentials.
-Enter a string value. Press Enter for the default ("").
-secret_access_key> xxxxxxxxxxx
    
-
      
-
    1. Select endpoint for Tencent COS. This is the standard endpoint for different region.
      2. 
-
    
-
     1 / Beijing Region.
-   \ "cos.ap-beijing.myqcloud.com"
- 2 / Nanjing Region.
-   \ "cos.ap-nanjing.myqcloud.com"
- 3 / Shanghai Region.
-   \ "cos.ap-shanghai.myqcloud.com"
- 4 / Guangzhou Region.
-   \ "cos.ap-guangzhou.myqcloud.com"
-[snip]
-endpoint> 4
    
-
      
-
    1. Choose acl and storage class.
      2. 
-
    
+Enter a value. Press Enter to leave empty.
+secret_access_key> secretaccesskey
+Option endpoint.
+Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+   / The default endpoint - a good choice if you are unsure.
+ 1 | East China (Suzhou)
+   \ (eos-wuxi-1.cmecloud.cn)
+ 2 / East China (Jinan)
+   \ (eos-jinan-1.cmecloud.cn)
+ 3 / East China (Hangzhou)
+   \ (eos-ningbo-1.cmecloud.cn)
+ 4 / East China (Shanghai-1)
+   \ (eos-shanghai-1.cmecloud.cn)
+ 5 / Central China (Zhengzhou)
+   \ (eos-zhengzhou-1.cmecloud.cn)
+ 6 / Central China (Changsha-1)
+   \ (eos-hunan-1.cmecloud.cn)
+ 7 / Central China (Changsha-2)
+   \ (eos-zhuzhou-1.cmecloud.cn)
+ 8 / South China (Guangzhou-2)
+   \ (eos-guangzhou-1.cmecloud.cn)
+ 9 / South China (Guangzhou-3)
+   \ (eos-dongguan-1.cmecloud.cn)
+10 / North China (Beijing-1)
+   \ (eos-beijing-1.cmecloud.cn)
+11 / North China (Beijing-2)
+   \ (eos-beijing-2.cmecloud.cn)
+12 / North China (Beijing-3)
+   \ (eos-beijing-4.cmecloud.cn)
+13 / North China (Huhehaote)
+   \ (eos-huhehaote-1.cmecloud.cn)
+14 / Southwest China (Chengdu)
+   \ (eos-chengdu-1.cmecloud.cn)
+15 / Southwest China (Chongqing)
+   \ (eos-chongqing-1.cmecloud.cn)
+16 / Southwest China (Guiyang)
+   \ (eos-guiyang-1.cmecloud.cn)
+17 / Nouthwest China (Xian)
+   \ (eos-xian-1.cmecloud.cn)
+18 / Yunnan China (Kunming)
+   \ (eos-yunnan.cmecloud.cn)
+19 / Yunnan China (Kunming-2)
+   \ (eos-yunnan-2.cmecloud.cn)
+20 / Tianjin China (Tianjin)
+   \ (eos-tianjin-1.cmecloud.cn)
+21 / Jilin China (Changchun)
+   \ (eos-jilin-1.cmecloud.cn)
+22 / Hubei China (Xiangyan)
+   \ (eos-hubei-1.cmecloud.cn)
+23 / Jiangxi China (Nanchang)
+   \ (eos-jiangxi-1.cmecloud.cn)
+24 / Gansu China (Lanzhou)
+   \ (eos-gansu-1.cmecloud.cn)
+25 / Shanxi China (Taiyuan)
+   \ (eos-shanxi-1.cmecloud.cn)
+26 / Liaoning China (Shenyang)
+   \ (eos-liaoning-1.cmecloud.cn)
+27 / Hebei China (Shijiazhuang)
+   \ (eos-hebei-1.cmecloud.cn)
+28 / Fujian China (Xiamen)
+   \ (eos-fujian-1.cmecloud.cn)
+29 / Guangxi China (Nanning)
+   \ (eos-guangxi-1.cmecloud.cn)
+30 / Anhui China (Huainan)
+   \ (eos-anhui-1.cmecloud.cn)
+endpoint> 1
+Option location_constraint.
+Location constraint - must match endpoint.
+Used when creating buckets only.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / East China (Suzhou)
+   \ (wuxi1)
+ 2 / East China (Jinan)
+   \ (jinan1)
+ 3 / East China (Hangzhou)
+   \ (ningbo1)
+ 4 / East China (Shanghai-1)
+   \ (shanghai1)
+ 5 / Central China (Zhengzhou)
+   \ (zhengzhou1)
+ 6 / Central China (Changsha-1)
+   \ (hunan1)
+ 7 / Central China (Changsha-2)
+   \ (zhuzhou1)
+ 8 / South China (Guangzhou-2)
+   \ (guangzhou1)
+ 9 / South China (Guangzhou-3)
+   \ (dongguan1)
+10 / North China (Beijing-1)
+   \ (beijing1)
+11 / North China (Beijing-2)
+   \ (beijing2)
+12 / North China (Beijing-3)
+   \ (beijing4)
+13 / North China (Huhehaote)
+   \ (huhehaote1)
+14 / Southwest China (Chengdu)
+   \ (chengdu1)
+15 / Southwest China (Chongqing)
+   \ (chongqing1)
+16 / Southwest China (Guiyang)
+   \ (guiyang1)
+17 / Nouthwest China (Xian)
+   \ (xian1)
+18 / Yunnan China (Kunming)
+   \ (yunnan)
+19 / Yunnan China (Kunming-2)
+   \ (yunnan2)
+20 / Tianjin China (Tianjin)
+   \ (tianjin1)
+21 / Jilin China (Changchun)
+   \ (jilin1)
+22 / Hubei China (Xiangyan)
+   \ (hubei1)
+23 / Jiangxi China (Nanchang)
+   \ (jiangxi1)
+24 / Gansu China (Lanzhou)
+   \ (gansu1)
+25 / Shanxi China (Taiyuan)
+   \ (shanxi1)
+26 / Liaoning China (Shenyang)
+   \ (liaoning1)
+27 / Hebei China (Shijiazhuang)
+   \ (hebei1)
+28 / Fujian China (Xiamen)
+   \ (fujian1)
+29 / Guangxi China (Nanning)
+   \ (guangxi1)
+30 / Anhui China (Huainan)
+   \ (anhui1)
+location_constraint> 1
+Option acl.
+Canned ACL used when creating buckets and storing or copying objects.
+This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+   / Owner gets FULL_CONTROL.
+ 2 | The AllUsers group gets READ access.
+   \ (public-read)
+   / Owner gets FULL_CONTROL.
+ 3 | The AllUsers group gets READ and WRITE access.
+   | Granting this on a bucket is generally not recommended.
+   \ (public-read-write)
+   / Owner gets FULL_CONTROL.
+ 4 | The AuthenticatedUsers group gets READ access.
+   \ (authenticated-read)
+   / Object owner gets FULL_CONTROL.
+acl> private
+Option server_side_encryption.
+The server-side encryption algorithm used when storing this object in S3.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / None
+   \ ()
+ 2 / AES256
+   \ (AES256)
+server_side_encryption>
+Option storage_class.
+The storage class to use when storing new objects in ChinaMobile.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / Default
+   \ ()
+ 2 / Standard storage class
+   \ (STANDARD)
+ 3 / Archive storage mode
+   \ (GLACIER)
+ 4 / Infrequent access storage mode
+   \ (STANDARD_IA)
+storage_class>
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+--------------------
+[ChinaMobile]
+type = s3
+provider = ChinaMobile
+access_key_id = accesskeyid
+secret_access_key = secretaccesskey
+endpoint = eos-wuxi-1.cmecloud.cn
+location_constraint = wuxi1
+acl = private
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
    +

    ArvanCloud

    +

    ArvanCloud ArvanCloud Object Storage goes beyond the limited traditional file storage. It gives you access to backup and archived files and allows sharing. Files like profile image in the app, images sent by users or scanned documents can be stored securely and easily in our Object Storage service.

    +

    ArvanCloud provides an S3 interface which can be configured for use with rclone like this.

    +
    No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+n/s> n
+name> ArvanCloud
+Type of storage to configure.
+Choose a number from below, or type in your own value
+[snip]
+XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio)
+   \ "s3"
+[snip]
+Storage> s3
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+AWS Access Key ID - leave blank for anonymous access or runtime credentials.
+access_key_id> YOURACCESSKEY
+AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
+secret_access_key> YOURSECRETACCESSKEY
+Region to connect to.
+Choose a number from below, or type in your own value
+   / The default endpoint - a good choice if you are unsure.
+ 1 | US Region, Northern Virginia, or Pacific Northwest.
+   | Leave location constraint empty.
+   \ "us-east-1"
+[snip]
+region> 
+Endpoint for S3 API.
+Leave blank if using ArvanCloud to use the default endpoint for the region.
+Specify if using an S3 clone such as Ceph.
+endpoint> s3.arvanstorage.com
+Location constraint - must be set to match the Region. Used when creating buckets only.
+Choose a number from below, or type in your own value
+ 1 / Empty for Iran-Tehran Region.
+   \ ""
+[snip]
+location_constraint>
+Canned ACL used when creating buckets and/or storing objects in S3.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Choose a number from below, or type in your own value
+ 1 / Owner gets FULL_CONTROL. No one else has access rights (default).
+   \ "private"
+[snip]
+acl>
+The server-side encryption algorithm used when storing this object in S3.
+Choose a number from below, or type in your own value
+ 1 / None
+   \ ""
+ 2 / AES256
+   \ "AES256"
+server_side_encryption>
+The storage class to use when storing objects in S3.
+Choose a number from below, or type in your own value
+ 1 / Default
+   \ ""
+ 2 / Standard storage class
+   \ "STANDARD"
+storage_class>
+Remote config
+--------------------
+[ArvanCloud]
+env_auth = false
+access_key_id = YOURACCESSKEY
+secret_access_key = YOURSECRETACCESSKEY
+region = ir-thr-at1
+endpoint = s3.arvanstorage.com
+location_constraint =
+acl =
+server_side_encryption =
+storage_class =
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
    +

    This will leave the config file looking like this.

    +
    [ArvanCloud]
+type = s3
+provider = ArvanCloud
+env_auth = false
+access_key_id = YOURACCESSKEY
+secret_access_key = YOURSECRETACCESSKEY
+region =
+endpoint = s3.arvanstorage.com
+location_constraint =
+acl =
+server_side_encryption =
+storage_class =
    +

    Tencent COS

    +

    Tencent Cloud Object Storage (COS) is a distributed storage service offered by Tencent Cloud for unstructured data. It is secure, stable, massive, convenient, low-delay and low-cost.

    +

    To configure access to Tencent COS, follow the steps below:

    +
      +
    1. Run rclone config and select n for a new remote.
      2. +
    +
    rclone config
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
    +
      +
    1. Give the name of the configuration. For example, name it 'cos'.
      2. +
    +
    name> cos
    +
      +
    1. Select s3 storage.
      2. +
    +
    Choose a number from below, or type in your own value
+1 / 1Fichier
+   \ "fichier"
+ 2 / Alias for an existing remote
+   \ "alias"
+ 3 / Amazon Drive
+   \ "amazon cloud drive"
+ 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS
+   \ "s3"
+[snip]
+Storage> s3
    +
      +
    1. Select TencentCOS provider.
      2. +
    +
    Choose a number from below, or type in your own value
+1 / Amazon Web Services (AWS) S3
+   \ "AWS"
+[snip]
+11 / Tencent Cloud Object Storage (COS)
+   \ "TencentCOS"
+[snip]
+provider> TencentCOS
    +
      +
    1. Enter your SecretId and SecretKey of Tencent Cloud.
      2. +
    +
    Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Enter a boolean value (true or false). Press Enter for the default ("false").
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+access_key_id> AKIDxxxxxxxxxx
+AWS Secret Access Key (password)
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+secret_access_key> xxxxxxxxxxx
    +
      +
    1. Select endpoint for Tencent COS. This is the standard endpoint for different region.
      2. +
    +
     1 / Beijing Region.
+   \ "cos.ap-beijing.myqcloud.com"
+ 2 / Nanjing Region.
+   \ "cos.ap-nanjing.myqcloud.com"
+ 3 / Shanghai Region.
+   \ "cos.ap-shanghai.myqcloud.com"
+ 4 / Guangzhou Region.
+   \ "cos.ap-guangzhou.myqcloud.com"
+[snip]
+endpoint> 4
    +
      +
    1. Choose acl and storage class.
      2. +
    Note that this ACL is applied when server-side copying objects as S3
 doesn't copy the ACL from the source but rather writes a fresh one.
 Enter a string value. Press Enter for the default ("").
@@ -13001,7 +14901,7 @@ 
    Comparison with the native protocol
 
    For more detailed comparison please check the documentation of the storj backend.
    
 
    Limitations
    
 
    rclone about is not supported by the S3 backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.
    
-
    See List of backends that do not support rclone about See rclone about
    
+
    See List of backends that do not support rclone about and rclone about
    
 
    Backblaze B2
    
 
    B2 is Backblaze's cloud storage system.
    
 
    Paths are specified as remote:bucket (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:bucket/path/to/dir.
    
@@ -13089,6 +14989,7 @@ 
    Transfers
    
 
    Versions
    
 
    When rclone uploads a new version of a file it creates a new version of it. Likewise when you delete a file, the old version will be marked hidden and still be available. Conversely, you may opt in to a "hard delete" of files with the --b2-hard-delete flag which would permanently remove the file instead of hiding it.
    
 
    Old versions of files, where available, are visible using the --b2-versions flag.
    
+
    It is also possible to view a bucket as it was at a certain point in time, using the --b2-version-at flag. This will show the file versions as they were at that time, showing files that have been deleted afterwards, and hiding files that were created since.
    
 
    If you wish to remove all the old versions then you can use the rclone cleanup remote:bucket command which will delete all the old versions of files, leaving the current ones intact. You can also supply a path and only old versions under that path will be deleted, e.g. rclone cleanup remote:bucket/path/to/stuff.
    
 
    Note that cleanup will remove partially uploaded files from the bucket if they are more than a day old.
    
 
    When you purge a bucket, the current and the old versions will be deleted then the bucket will be deleted.
    
@@ -13159,7 +15060,7 @@ 
 https://f002.backblazeb2.com/file/bucket/path/folder/file3?Authorization=xxxxxxxx

    Standard options

    -

    Here are the standard options specific to b2 (Backblaze B2).

    +

    Here are the Standard options specific to b2 (Backblaze B2).

    --b2-account

    Account ID or Application Key ID.

    Properties:

    @@ -13188,7 +15089,7 @@

    --b2-hard-delete

  • Default: false

    • Advanced options

    -

    Here are the advanced options specific to b2 (Backblaze B2).

    +

    Here are the Advanced options specific to b2 (Backblaze B2).

    --b2-endpoint

    Endpoint for the service.

    Leave blank normally.

    @@ -13225,6 +15126,16 @@

    --b2-versions

  • Type: bool
  • Default: false
    • +

    --b2-version-at

    +

    Show file versions as they were at the specified time.

    +

    Note that when using this no file write operations are permitted, so you can't upload files or delete them.

    +

    Properties:

    +
      +
    • Config: version_at
      • +
    • Env Var: RCLONE_B2_VERSION_AT
      • +
    • Type: Time
      • +
    • Default: off
      • +

    --b2-upload-cutoff

    Cutoff for switching to chunked upload.

    Files above this size will be uploaded in chunks of "--b2-chunk-size".

    @@ -13321,7 +15232,7 @@

    --b2-encoding

    Limitations

    rclone about is not supported by the B2 backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

    -

    See List of backends that do not support rclone about See rclone about

    +

    See List of backends that do not support rclone about and rclone about

    Box

    Paths are specified as remote:path

    Paths may be as deep as required, e.g. remote:directory/subdirectory.

    @@ -13519,7 +15430,7 @@

    Root folder ID

    In order to do this you will have to find the Folder ID of the directory you wish rclone to display. This will be the last segment of the URL when you open the relevant folder in the Box web interface.

    So if the folder you want rclone to use has a URL which looks like https://app.box.com/folder/11xxxxxxxxx8 in the browser, then you use 11xxxxxxxxx8 as the root_folder_id in the config.

    Standard options

    -

    Here are the standard options specific to box (Box).

    +

    Here are the Standard options specific to box (Box).

    --box-client-id

    OAuth Client Id.

    Leave blank normally.

    @@ -13581,7 +15492,7 @@

    --box-box-sub-type

    Advanced options

    -

    Here are the advanced options specific to box (Box).

    +

    Here are the Advanced options specific to box (Box).

    --box-token

    OAuth Access Token as a JSON blob.

    Properties:

    @@ -13671,7 +15582,7 @@

    Limitations

    Box file names can't have the \ character in. rclone maps this to and from an identical looking unicode equivalent ＼ (U+FF3C Fullwidth Reverse Solidus).

    Box only supports filenames up to 255 characters in length.

    rclone about is not supported by the Box backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

    -

    See List of backends that do not support rclone about See rclone about

    +

    See List of backends that do not support rclone about and rclone about

    Cache (DEPRECATED)

    The cache remote wraps another existing remote and stores file structure and its data for long running tasks like rclone mount.

    Status

    @@ -13833,7 +15744,7 @@

    rc cache/expire

    Purge a remote from the cache backend. Supports either a directory or a file. It supports both encrypted and unencrypted file names if cache is wrapped by crypt.

    Params: - remote = path to remote (required) - withData = true/false to delete cached data (chunks) as well (optional, false by default)

    Standard options

    -

    Here are the standard options specific to cache (Cache a remote).

    +

    Here are the Standard options specific to cache (Cache a remote).

    --cache-remote

    Remote to cache.

    Normally should contain a ':' and a path, e.g. "myremote:path/to/dir", "myremote:bucket" or maybe "myremote:" (not recommended).

    @@ -13947,7 +15858,7 @@

    --cache-chunk-total-size

    Advanced options

    -

    Here are the advanced options specific to cache (Cache a remote).

    +

    Here are the Advanced options specific to cache (Cache a remote).

    --cache-plex-token

    The plex token for authentication - auto set normally.

    Properties:

    @@ -14101,7 +16012,7 @@

    Backend commands

    Run them with

    rclone backend COMMAND remote:

    The help below will explain what arguments each command takes.

    -

    See the "rclone backend" command for more info on how to pass options and arguments.

    +

    See the backend command for more info on how to pass options and arguments.

    These can be run on a running backend using the rc command backend/command.

    stats

    Print stats on the cache backend in JSON format.

    @@ -14180,7 +16091,7 @@

    Chunk names

    For example, if name format is big_*-##.part and original file name is data.txt and numbering starts from 0, then the first chunk will be named big_data.txt-00.part, the 99th chunk will be big_data.txt-98.part and the 302nd chunk will become big_data.txt-301.part.

    Note that list assembles composite directory entries only when chunk names match the configured format and treats non-conforming file names as normal non-chunked files.

    When using norename transactions, chunk names will additionally have a unique file version suffix. For example, BIG_FILE_NAME.rclone_chunk.001_bp562k.

    -

    Metadata

    +

    Metadata

    Besides data chunks chunker will by default create metadata object for a composite file. The object is named after the original file. Chunker allows user to disable metadata completely (the none format). Note that metadata is normally not created for files smaller than the configured chunk size. This may change in future rclone releases.

    Simple JSON metadata format

    This is the default format. It supports hash sums and chunk validation for composite files. Meta objects carry the following fields:

    @@ -14221,7 +16132,7 @@

    Caveats and Limitations

    Chunker included in rclone releases up to v1.54 can sometimes fail to detect metadata produced by recent versions of rclone. We recommend users to keep rclone up-to-date to avoid data corruption.

    Changing transactions is dangerous and requires explicit migration.

    Standard options

    -

    Here are the standard options specific to chunker (Transparently chunk/split large files).

    +

    Here are the Standard options specific to chunker (Transparently chunk/split large files).

    --chunker-remote

    Remote to chunk/unchunk.

    Normally should contain a ':' and a path, e.g. "myremote:path/to/dir", "myremote:bucket" or maybe "myremote:" (not recommended).

    @@ -14285,7 +16196,7 @@

    --chunker-hash-type

    Advanced options

    -

    Here are the advanced options specific to chunker (Transparently chunk/split large files).

    +

    Here are the Advanced options specific to chunker (Transparently chunk/split large files).

    --chunker-name-format

    String format of chunk file names.

    The two placeholders are: base file name (*) and chunk number (#...). There must be one and only one asterisk and one or more consecutive hash characters. If chunk number has less digits than the number of hashes, it is left-padded by zeros. If there are more digits in the number, they are left as is. Possible chunk files are ignored if their name does not match given format.

    @@ -14536,7 +16447,7 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.

    Standard options

    -

    Here are the standard options specific to sharefile (Citrix Sharefile).

    +

    Here are the Standard options specific to sharefile (Citrix Sharefile).

    --sharefile-root-folder-id

    ID of the root folder.

    Leave blank to access "Personal Folders". You can use one of the standard values here or any folder ID (long hex number ID).

    @@ -14571,7 +16482,7 @@

    --sharefile-root-folder-id

    Advanced options

    -

    Here are the advanced options specific to sharefile (Citrix Sharefile).

    +

    Here are the Advanced options specific to sharefile (Citrix Sharefile).

    --sharefile-upload-cutoff

    Cutoff for switching to multipart upload.

    Properties:

    @@ -14617,7 +16528,7 @@

    Limitations

    Note that ShareFile is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

    ShareFile only supports filenames up to 256 characters in length.

    rclone about is not supported by the Citrix ShareFile backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

    -

    See List of backends that do not support rclone about See rclone about

    +

    See List of backends that do not support rclone about and rclone about

    Crypt

    Rclone crypt remotes encrypt and decrypt other remotes.

    A remote of type crypt does not access a storage system directly, but instead wraps another remote, which in turn accesses the storage system. This is similar to how alias, union, chunker and a few others work. It makes the usage very flexible, as you can add a layer, in this case an encryption layer, on top of any other backend, even in multiple layers. Rclone's functionality can be used as with any other remote, for example you can mount a crypt remote.

    @@ -14816,7 +16727,7 @@

    Modified time and hashes

    Hashes are not stored for crypt. However the data integrity is protected by an extremely strong crypto authenticator.

    Use the rclone cryptcheck command to check the integrity of a crypted remote instead of rclone check which can't check the checksums properly.

    Standard options

    -

    Here are the standard options specific to crypt (Encrypt/Decrypt a remote).

    +

    Here are the Standard options specific to crypt (Encrypt/Decrypt a remote).

    --crypt-remote

    Remote to encrypt/decrypt.

    Normally should contain a ':' and a path, e.g. "myremote:path/to/dir", "myremote:bucket" or maybe "myremote:" (not recommended).

    @@ -14896,7 +16807,7 @@

    --crypt-password2

  • Required: false

    • Advanced options

    -

    Here are the advanced options specific to crypt (Encrypt/Decrypt a remote).

    +

    Here are the Advanced options specific to crypt (Encrypt/Decrypt a remote).

    --crypt-server-side-across-configs

    Allow server-side operations (e.g. copy) to work across different crypt configs.

    Normally this option is not what you want, but if you have two crypts pointing to the same backend you can use it.

    @@ -14965,12 +16876,15 @@

    --crypt-filename-encoding

    +

    Metadata

    +

    Any metadata supported by the underlying remote is read and written.

    +

    See the metadata docs for more info.

    Backend commands

    Here are the commands specific to the crypt backend.

    Run them with

    rclone backend COMMAND remote:

    The help below will explain what arguments each command takes.

    -

    See the "rclone backend" command for more info on how to pass options and arguments.

    +

    See the backend command for more info on how to pass options and arguments.

    These can be run on a running backend using the rc command backend/command.

    encode

    Encode the given filename(s)

    @@ -15050,7 +16964,7 @@

    Name encryption

    Key derivation

    Rclone uses scrypt with parameters N=16384, r=8, p=1 with an optional user supplied salt (password2) to derive the 32+32+16 = 80 bytes of key material required. If the user doesn't supply a salt then rclone uses an internal one.

    scrypt makes it impractical to mount a dictionary attack on rclone encrypted data. For full protection against this you should always use a salt.

    -

    SEE ALSO

    +

    SEE ALSO

    @@ -15114,7 +17028,7 @@

    File types

    File names

    The compressed files will be named *.###########.gz where * is the base file and the # part is base64 encoded size of the uncompressed file. The file names should not be changed by anything other than the rclone compression backend.

    Standard options

    -

    Here are the standard options specific to compress (Compress a remote).

    +

    Here are the Standard options specific to compress (Compress a remote).

    --compress-remote

    Remote to compress.

    Properties:

    @@ -15141,7 +17055,7 @@

    --compress-mode

    Advanced options

    -

    Here are the advanced options specific to compress (Compress a remote).

    +

    Here are the Advanced options specific to compress (Compress a remote).

    --compress-level

    GZIP compression level (-2 to 9).

    Generally -1 (default, equivalent to 5) is recommended. Levels 1 to 9 increase compression at the cost of speed. Going past 6 generally offers very little return.

    @@ -15163,41 +17077,142 @@

    --compress-ram-cache-limit

  • Type: SizeSuffix
  • Default: 20Mi
    • -

    Dropbox

    -

    Paths are specified as remote:path

    -

    Dropbox paths may be as deep as required, e.g. remote:directory/subdirectory.

    +

    Metadata

    +

    Any metadata supported by the underlying remote is read and written.

    +

    See the metadata docs for more info.

    +

    Combine

    +

    The combine backend joins remotes together into a single directory tree.

    +

    For example you might have a remote for images on one provider:

    +
    $ rclone tree s3:imagesbucket
+/
+├── image1.jpg
+└── image2.jpg
    +

    And a remote for files on another:

    +
    $ rclone tree drive:important/files
+/
+├── file1.txt
+└── file2.txt
    +

    The combine backend can join these together into a synthetic directory structure like this:

    +
    $ rclone tree combined:
+/
+├── files
+│   ├── file1.txt
+│   └── file2.txt
+└── images
+    ├── image1.jpg
+    └── image2.jpg
    +

    You'd do this by specifying an upstreams parameter in the config like this

    +
    upstreams = images=s3:imagesbucket files=drive:important/files
    +

    During the initial setup with rclone config you will specify the upstreams remotes as a space separated list. The upstream remotes can either be a local paths or other remotes.

    Configuration

    -

    The initial setup for dropbox involves getting a token from Dropbox which you need to do in your browser. rclone config walks you through it.

    -

    Here is an example of how to make a remote called remote. First run:

    +

    Here is an example of how to make a combine called remote for the example above. First run:

     rclone config

    This will guide you through an interactive setup process:

    -
    n) New remote
-d) Delete remote
+
    No remotes found, make a new one?
+n) New remote
+s) Set configuration password
 q) Quit config
-e/n/d/q> n
+n/s/q> n
 name> remote
+Option Storage.
 Type of storage to configure.
-Choose a number from below, or type in your own value
-[snip]
-XX / Dropbox
-   \ "dropbox"
-[snip]
-Storage> dropbox
-Dropbox App Key - leave blank normally.
-app_key>
-Dropbox App Secret - leave blank normally.
-app_secret>
-Remote config
-Please visit:
-https://www.dropbox.com/1/oauth2/authorize?client_id=XXXXXXXXXXXXXXX&response_type=code
-Enter the code: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXXXXXXXX
+Choose a number from below, or type in your own value.
+...
+XX / Combine several remotes into one
+   \ (combine)
+...
+Storage> combine
+Option upstreams.
+Upstreams for combining
+These should be in the form
+    dir=remote:path dir2=remote2:path
+Where before the = is specified the root directory and after is the remote to
+put there.
+Embedded spaces can be added using quotes
+    "dir=remote:path with space" "dir2=remote2:path with space"
+Enter a fs.SpaceSepList value.
+upstreams> images=s3:imagesbucket files=drive:important/files
 --------------------
 [remote]
-app_key =
-app_secret =
-token = XXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXX_XXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+type = combine
+upstreams = images=s3:imagesbucket files=drive:important/files
 --------------------
-y) Yes this is OK
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
    
+
    Configuring for Google Drive Shared Drives
    
+
    Rclone has a convenience feature for making a combine backend for all the shared drives you have access to.
    
+
    Assuming your main (non shared drive) Google drive remote is called drive: you would run
    
+
    rclone backend -o config drives drive:
    
+
    This would produce something like this:
    
+
    [My Drive]
+type = alias
+remote = drive,team_drive=0ABCDEF-01234567890,root_folder_id=:
+
+[Test Drive]
+type = alias
+remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=:
+
+[AllDrives]
+type = combine
+remote = "My Drive=My Drive:" "Test Drive=Test Drive:"
    
+
    If you then add that config to your config file (find it with rclone config file) then you can access all the shared drives in one place with the AllDrives: remote.
    
+
    See the Google Drive docs for full info.
    
+
    Standard options
    
+
    Here are the Standard options specific to combine (Combine several remotes into one).
    
+
    --combine-upstreams
    
+
    Upstreams for combining
    
+
    These should be in the form
    
+
    dir=remote:path dir2=remote2:path
    
+
    Where before the = is specified the root directory and after is the remote to put there.
    
+
    Embedded spaces can be added using quotes
    
+
    "dir=remote:path with space" "dir2=remote2:path with space"
    
+
    Properties:
    
+
      
+
    • Config: upstreams
      • 
+
    • Env Var: RCLONE_COMBINE_UPSTREAMS
      • 
+
    • Type: SpaceSepList
      • 
+
    • Default:
      • 
+
    
+
    Metadata
    
+
    Any metadata supported by the underlying remote is read and written.
    
+
    See the metadata docs for more info.
    
+
    Dropbox
    
+
    Paths are specified as remote:path
    
+
    Dropbox paths may be as deep as required, e.g. remote:directory/subdirectory.
    
+
    Configuration
    
+
    The initial setup for dropbox involves getting a token from Dropbox which you need to do in your browser. rclone config walks you through it.
    
+
    Here is an example of how to make a remote called remote. First run:
    
+
     rclone config
    
+
    This will guide you through an interactive setup process:
    
+
    n) New remote
+d) Delete remote
+q) Quit config
+e/n/d/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+[snip]
+XX / Dropbox
+   \ "dropbox"
+[snip]
+Storage> dropbox
+Dropbox App Key - leave blank normally.
+app_key>
+Dropbox App Secret - leave blank normally.
+app_secret>
+Remote config
+Please visit:
+https://www.dropbox.com/1/oauth2/authorize?client_id=XXXXXXXXXXXXXXX&response_type=code
+Enter the code: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXXXXXXXX
+--------------------
+[remote]
+app_key =
+app_secret =
+token = XXXXXXXXXXXXXXXXXXXXXXXXXXXXX_XXXX_XXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+--------------------
+y) Yes this is OK
 e) Edit this remote
 d) Delete this remote
 y/e/d> y
    
@@ -15287,8 +17302,8 @@ 
    --dropbox-batch-mode async
    
 
    This provides the maximum possible upload speed especially with lots of small files, however rclone can't check the file got uploaded properly using this mode.
    
 
    If you are using this mode then using "rclone check" after the transfer completes is recommended. Or you could do an initial transfer with --dropbox-batch-mode async then do a final transfer with --dropbox-batch-mode sync (the default).
    
 
    Note that there may be a pause when quitting rclone while rclone finishes up the last batch using this mode.
    
-
    Standard options
    
-
    Here are the standard options specific to dropbox (Dropbox).
    
+
    Standard options
    
+
    Here are the Standard options specific to dropbox (Dropbox).
    
 
    --dropbox-client-id
    
 
    OAuth Client Id.
    
 
    Leave blank normally.
    
@@ -15310,7 +17325,7 @@ 
    --dropbox-client-secret
    
 
  • Required: false
    • 
 
 
    Advanced options
    
-
    Here are the advanced options specific to dropbox (Dropbox).
    
+
    Here are the Advanced options specific to dropbox (Dropbox).
    
 
    --dropbox-token
    
 
    OAuth Access Token as a JSON blob.
    
 
    Properties:
    
@@ -15476,7 +17491,7 @@ 
    Get your own Dropbox App ID
    
 
 
    Enterprise File Fabric
    
 
    This backend supports Storage Made Easy's Enterprise File Fabric™ which provides a software solution to integrate and unify File and Object Storage accessible through a global file system.
    
-
    Configuration
    
+
    Configuration
    
 
    The initial setup for the Enterprise File Fabric backend involves getting a token from the the Enterprise File Fabric which you need to do in your browser. rclone config walks you through it.
    
 
    Here is an example of how to make a remote called remote. First run:
    
 
     rclone config
    
@@ -15570,8 +17585,8 @@ 
    Root folder ID
    
 120673757,My contacts/
 120673761,S3 Storage/

    The ID for "S3 Storage" would be 120673761.

    -

    Standard options

    -

    Here are the standard options specific to filefabric (Enterprise File Fabric).

    +

    Standard options

    +

    Here are the Standard options specific to filefabric (Enterprise File Fabric).

    --filefabric-url

    URL of the Enterprise File Fabric to connect to.

    Properties:

    @@ -15620,7 +17635,7 @@

    --filefabric-permanent-token

  • Required: false

    • Advanced options

    -

    Here are the advanced options specific to filefabric (Enterprise File Fabric).

    +

    Here are the Advanced options specific to filefabric (Enterprise File Fabric).

    --filefabric-token

    Session Token.

    This is a session token which rclone caches in the config file. It is usually valid for 1 hour.

    @@ -15666,7 +17681,7 @@

    FTP

    FTP is the File Transfer Protocol. Rclone FTP support is provided using the github.com/jlaffaye/ftp package.

    Limitations of Rclone's FTP backend

    Paths are specified as remote:path. If the path does not begin with a / it is relative to the home directory of the user. An empty path remote: refers to the user's home directory.

    -

    Configuration

    +

    Configuration

    To create an FTP configuration named remote, run

    rclone config

    Rclone config guides you through an interactive setup process. A minimal rclone FTP remote definition only requires host, username and password. For an anonymous FTP server, use anonymous as username and your email address as password.

    @@ -15682,7 +17697,7 @@

    Configuration

    Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [snip] -XX / FTP Connection +XX / FTP \ "ftp" [snip] Storage> ftp @@ -15776,8 +17791,8 @@

    Restricted filename characters

    This backend's interactive configuration wizard provides a selection of sensible encoding settings for major FTP servers: ProFTPd, PureFTPd, VsFTPd. Just hit a selection number when prompted.

    -

    Standard options

    -

    Here are the standard options specific to ftp (FTP Connection).

    +

    Standard options

    +

    Here are the Standard options specific to ftp (FTP).

    --ftp-host

    FTP host to connect to.

    E.g. "ftp.example.com".

    @@ -15837,7 +17852,7 @@

    --ftp-explicit-tls

  • Default: false

    • Advanced options

    -

    Here are the advanced options specific to ftp (FTP Connection).

    +

    Here are the Advanced options specific to ftp (FTP).

    --ftp-concurrency

    Maximum number of FTP simultaneous connections, 0 for unlimited.

    Properties:

    @@ -15874,6 +17889,15 @@

    --ftp-disable-mlsd

  • Type: bool
  • Default: false
    • +

    --ftp-disable-utf8

    +

    Disable using UTF-8 even if server advertises support.

    +

    Properties:

    +
      +
    • Config: disable_utf8
      • +
    • Env Var: RCLONE_FTP_DISABLE_UTF8
      • +
    • Type: bool
      • +
    • Default: false
      • +

    --ftp-writing-mdtm

    Use MDTM to set modification time (VsFtpd quirk)

    Properties:

    @@ -15970,7 +17994,7 @@

    Limitations

    FTP servers acting as rclone remotes must support passive mode. The mode cannot be configured as passive is the only supported one. Rclone's FTP implementation is not compatible with active mode as the library it uses doesn't support it. This will likely never be supported due to security concerns.

    Rclone's FTP backend does not support any checksums but can compare file sizes.

    rclone about is not supported by the FTP backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

    -

    See List of backends that do not support rclone about See rclone about

    +

    See List of backends that do not support rclone about and rclone about

    The implementation of : --dump headers, --dump bodies, --dump auth for debugging isn't the same as for rclone HTTP based backends - it has less fine grained control.

    --timeout isn't supported (but --contimeout is).

    --bind isn't supported.

    @@ -15982,7 +18006,7 @@

    Modified time

    You can use the following command to check whether rclone can use precise time with your FTP server: rclone backend features your_ftp_remote: (the trailing colon is important). Look for the number in the line tagged by Precision designating the remote time precision expressed as nanoseconds. A value of 1000000000 means that file time precision of 1 second is available. A value of 3153600000000000000 (or another large number) means "unsupported".

    Google Cloud Storage

    Paths are specified as remote:bucket (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:bucket/path/to/dir.

    -

    Configuration

    +

    Configuration

    The initial setup for google cloud storage involves getting a token from Google Cloud Storage which you need to do in your browser. rclone config walks you through it.

    Here is an example of how to make a remote called remote. First run:

     rclone config
    @@ -16177,8 +18201,8 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.

    -

    Standard options

    -

    Here are the standard options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)).

    +

    Standard options

    +

    Here are the Standard options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)).

    --gcs-client-id

    OAuth Client Id.

    Leave blank normally.

    @@ -16532,7 +18556,7 @@

    --gcs-storage-class

    Advanced options

    -

    Here are the advanced options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)).

    +

    Here are the Advanced options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)).

    --gcs-token

    OAuth Access Token as a JSON blob.

    Properties:

    @@ -16562,6 +18586,27 @@

    --gcs-token-url

  • Type: string
  • Required: false
    • +

    --gcs-no-check-bucket

    +

    If set, don't attempt to check the bucket exists or create it.

    +

    This can be useful when trying to minimise the number of transactions rclone does if you know the bucket exists already.

    +

    Properties:

    +
      +
    • Config: no_check_bucket
      • +
    • Env Var: RCLONE_GCS_NO_CHECK_BUCKET
      • +
    • Type: bool
      • +
    • Default: false
      • +
    +

    --gcs-decompress

    +

    If set this will decompress gzip encoded objects.

    +

    It is possible to upload objects to GCS with "Content-Encoding: gzip" set. Normally rclone will download these files files as compressed objects.

    +

    If this flag is set then rclone will decompress these files with "Content-Encoding: gzip" as they are received. This means that rclone can't check the size and hash but the file contents will be decompressed.

    +

    Properties:

    +
      +
    • Config: decompress
      • +
    • Env Var: RCLONE_GCS_DECOMPRESS
      • +
    • Type: bool
      • +
    • Default: false
      • +

    --gcs-encoding

    The encoding for the backend.

    See the encoding section in the overview for more info.

    @@ -16574,11 +18619,11 @@

    --gcs-encoding

    Limitations

    rclone about is not supported by the Google Cloud Storage backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

    -

    See List of backends that do not support rclone about See rclone about

    +

    See List of backends that do not support rclone about and rclone about

    Google Drive

    Paths are specified as drive:path

    Drive paths may be as deep as required, e.g. drive:directory/subdirectory.

    -

    Configuration

    +

    Configuration

    The initial setup for drive involves getting a token from Google drive which you need to do in your browser. rclone config walks you through it.

    Here is an example of how to make a remote called remote. First run:

     rclone config
    @@ -16619,8 +18664,6 @@

    Configuration

    5 | does not allow any access to read or download file content. \ "drive.metadata.readonly" scope> 1 -ID of the root folder - leave blank normally. Fill in to access "Computers" folders. (see docs). -root_folder_id> Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. service_account_file> Remote config @@ -16677,7 +18720,7 @@

    drive.appfolder

    drive.metadata.readonly

    This allows read only access to file names only. It does not allow rclone to download or upload data, or rename or delete files or directories.

    Root folder ID

    -

    You can set the root_folder_id for rclone. This is the directory (identified by its Folder ID) that rclone considers to be the root of your drive.

    +

    This option has been moved to the advanced section. You can set the root_folder_id for rclone. This is the directory (identified by its Folder ID) that rclone considers to be the root of your drive.

    Normally you will leave this blank and rclone will determine the correct root to use itself.

    However you can set this to restrict rclone to a specific folder hierarchy or to access data within the "Computers" tab on the drive web interface (where files from Google's Backup and Sync desktop program go).

    In order to do this you will have to find the Folder ID of the directory you wish rclone to display. This will be the last segment of the URL when you open the relevant folder in the drive web interface.

    @@ -16919,10 +18962,20 @@

    Import/Export of google documents

    +bmp +image/bmp +Windows Bitmap format + + csv text/csv Standard CSV format for Spreadsheets + +doc +application/msword +Classic Word file + docx application/vnd.openxmlformats-officedocument.wordprocessingml.document @@ -16946,7 +18999,7 @@

    Import/Export of google documents

    json application/vnd.google-apps.script+json -JSON Text Format +JSON Text Format for Google Apps scripts odp @@ -16974,41 +19027,56 @@

    Import/Export of google documents

    Adobe PDF Format +pjpeg +image/pjpeg +Progressive JPEG Image + + png image/png PNG Image Format - + pptx application/vnd.openxmlformats-officedocument.presentationml.presentation Microsoft Office Powerpoint - + rtf application/rtf Rich Text Format - + svg image/svg+xml Scalable Vector Graphics Format - + tsv text/tab-separated-values Standard TSV format for spreadsheets - + txt text/plain Plain Text + +wmf +application/x-msmetafile +Windows Meta File + +xls +application/vnd.ms-excel +Classic Excel file + + xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet Microsoft Office Spreadsheet - + zip application/zip A ZIP file of HTML, Images CSS @@ -17047,8 +19115,8 @@

    Import/Export of google documents

    -

    Standard options

    -

    Here are the standard options specific to drive (Google Drive).

    +

    Standard options

    +

    Here are the Standard options specific to drive (Google Drive).

    --drive-client-id

    Google Application Client Id Setting your own is recommended. See https://rclone.org/drive/#making-your-own-client-id for how to create your own. If you leave this blank, it will use an internal key which is low performance.

    Properties:

    @@ -17104,16 +19172,6 @@

    --drive-scope

    -

    --drive-root-folder-id

    -

    ID of the root folder. Leave blank normally.

    -

    Fill in to access "Computers" folders (see docs), or for rclone to use a non root folder as its starting point.

    -

    Properties:

    -
      -
    • Config: root_folder_id
      • -
    • Env Var: RCLONE_DRIVE_ROOT_FOLDER_ID
      • -
    • Type: string
      • -
    • Required: false
      • -

    --drive-service-account-file

    Service Account Credentials JSON file path.

    Leave blank normally. Needed only if you want use SA instead of interactive login.

    @@ -17135,7 +19193,7 @@

    --drive-alternate-export

  • Default: false

    • Advanced options

    -

    Here are the advanced options specific to drive (Google Drive).

    +

    Here are the Advanced options specific to drive (Google Drive).

    --drive-token

    OAuth Access Token as a JSON blob.

    Properties:

    @@ -17165,6 +19223,16 @@

    --drive-token-url

  • Type: string
  • Required: false
    • +

    --drive-root-folder-id

    +

    ID of the root folder. Leave blank normally.

    +

    Fill in to access "Computers" folders (see docs), or for rclone to use a non root folder as its starting point.

    +

    Properties:

    +
      +
    • Config: root_folder_id
      • +
    • Env Var: RCLONE_DRIVE_ROOT_FOLDER_ID
      • +
    • Type: string
      • +
    • Required: false
      • +

    --drive-service-account-credentials

    Service Account Credentials JSON blob.

    Leave blank normally. Needed only if you want use SA instead of interactive login.

    @@ -17490,6 +19558,21 @@

    --drive-skip-dangling-shortcuts

  • Type: bool
  • Default: false
    • +

    --drive-resource-key

    +

    Resource key for accessing a link-shared file.

    +

    If you need to access files shared with a link like this

    +
    https://drive.google.com/drive/folders/XXX?resourcekey=YYY&usp=sharing
    +

    Then you will need to use the first part "XXX" as the "root_folder_id" and the second part "YYY" as the "resource_key" otherwise you will get 404 not found errors when trying to access the directory.

    +

    See: https://developers.google.com/drive/api/guides/resource-keys

    +

    This resource key requirement only applies to a subset of old files.

    +

    Note also that opening the folder once in the web interface (with the user you've authenticated rclone with) seems to be enough so that the resource key is no needed.

    +

    Properties:

    +
      +
    • Config: resource_key
      • +
    • Env Var: RCLONE_DRIVE_RESOURCE_KEY
      • +
    • Type: string
      • +
    • Required: false
      • +

    --drive-encoding

    The encoding for the backend.

    See the encoding section in the overview for more info.

    @@ -17505,7 +19588,7 @@

    Backend commands

    Run them with

    rclone backend COMMAND remote:

    The help below will explain what arguments each command takes.

    -

    See the "rclone backend" command for more info on how to pass options and arguments.

    +

    See the backend command for more info on how to pass options and arguments.

    These can be run on a running backend using the rc command backend/command.

    get

    Get command for fetching the drive config parameters

    @@ -17563,15 +19646,19 @@

    drives

    "name": "Test Drive" } ] -

    With the -o config parameter it will output the list in a format suitable for adding to a config file to make aliases for all the drives found.

    +

    With the -o config parameter it will output the list in a format suitable for adding to a config file to make aliases for all the drives found and a combined drive.

    [My Drive]
 type = alias
 remote = drive,team_drive=0ABCDEF-01234567890,root_folder_id=:
 
 [Test Drive]
 type = alias
-remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=:
    -

    Adding this to the rclone config file will cause those team drives to be accessible with the aliases shown. This may require manual editing of the names.

    +remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=: + +[AllDrives] +type = combine +remote = "My Drive=My Drive:" "Test Drive=Test Drive:" +

    Adding this to the rclone config file will cause those team drives to be accessible with the aliases shown. Any illegal charactes will be substituted with "_" and duplicate names will have numbers suffixed. It will also add a remote called AllDrives which shows all the shared drives combined into one directory tree.

    untrash

    Untrash files and directories

    rclone backend untrash remote: [options] [<arguments>+]
    @@ -17597,11 +19684,17 @@

    copyid

    The path should end with a / to indicate copy the file as named to this directory. If it doesn't end with a / then the last path component will be used as the file name.

    If the destination is a drive backend then server-side copying will be attempted if possible.

    Use the -i flag to see what would be copied before copying.

    +

    exportformats

    +

    Dump the export formats for debug purposes

    +
    rclone backend exportformats remote: [options] [<arguments>+]
    +

    importformats

    +

    Dump the import formats for debug purposes

    +
    rclone backend importformats remote: [options] [<arguments>+]

    Limitations

    Drive has quite a lot of rate limiting. This causes rclone to be limited to transferring about 2 files per second only. Individual files may be transferred much faster at 100s of MiB/s but lots of small files can take a long time.

    Server side copies are also subject to a separate rate limit. If you see User rate limit exceeded errors, wait at least 24 hours and retry. You can disable server-side copies with --disable copy to download and upload the files if you prefer.

    Limitations of Google Docs

    -

    Google docs will appear as size -1 in rclone ls and as size 0 in anything which uses the VFS layer, e.g. rclone mount, rclone serve.

    +

    Google docs will appear as size -1 in rclone ls, rclone ncdu etc, and as size 0 in anything which uses the VFS layer, e.g. rclone mount and rclone serve. When calculating directory totals, e.g. in rclone size and rclone ncdu, they will be counted in as empty files.

    This is because rclone can't find out the size of the Google docs without downloading them.

    Google docs will transfer correctly with rclone sync, rclone copy etc as rclone knows to ignore the size when doing the transfer.

    However an unfortunate consequence of this is that you may not be able to download Google docs using rclone mount. If it doesn't work you will get a 0 sized file. If you try again the doc may gain its correct size and be downloadable. Whether it will work on not depends on the application accessing the mount and the OS you are running - experiment to find out if it does work for you!

    @@ -17623,16 +19716,15 @@

    Making your own client_id

  • Select a project or create a new project.

  • Under "ENABLE APIS AND SERVICES" search for "Drive", and enable the "Google Drive API".

  • Click "Credentials" in the left-side panel (not "Create credentials", which opens the wizard), then "Create credentials"

    • -

  • If you already configured an "Oauth Consent Screen", then skip to the next step; if not, click on "CONFIGURE CONSENT SCREEN" button (near the top right corner of the right panel), then select "External" and click on "CREATE"; on the next screen, enter an "Application name" ("rclone" is OK); enter "User Support Email" (your own email is OK); enter "Developer Contact Email" (your own email is OK); then click on "Save" (all other data is optional). Click again on "Credentials" on the left panel to go back to the "Credentials" screen.

    • - -

    (PS: if you are a GSuite user, you could also select "Internal" instead of "External" above, but this has not been tested/documented so far).

    -
      +

    1. If you already configured an "Oauth Consent Screen", then skip to the next step; if not, click on "CONFIGURE CONSENT SCREEN" button (near the top right corner of the right panel), then select "External" and click on "CREATE"; on the next screen, enter an "Application name" ("rclone" is OK); enter "User Support Email" (your own email is OK); enter "Developer Contact Email" (your own email is OK); then click on "Save" (all other data is optional). Click again on "Credentials" on the left panel to go back to the "Credentials" screen.

      +

      (PS: if you are a GSuite user, you could also select "Internal" instead of "External" above, but this will restrict API use to Google Workspace users in your organisation).

    2. Click on the "+ CREATE CREDENTIALS" button at the top of the screen, then select "OAuth client ID".

    3. Choose an application type of "Desktop app" and click "Create". (the default name is fine)

      4. -

    4. It will show you a client ID and client secret. Make a note of these.

      5. +

    5. It will show you a client ID and client secret. Make a note of these.

      +

      (If you selected "External" at Step 5 continue to "Publish App" in the Steps 9 and 10. If you chose "Internal" you don't need to publish and can skip straight to Step 11.)

    6. Go to "Oauth consent screen" and press "Publish App"

      7. -

    7. Provide the noted client ID and client secret to rclone.

    8. Click "OAuth consent screen", then click "PUBLISH APP" button and confirm, or add your account under "Test users".

      9. +

    9. Provide the noted client ID and client secret to rclone.

    Be aware that, due to the "enhanced security" recently introduced by Google, you are theoretically expected to "submit your app for verification" and then wait a few weeks(!) for their response; in practice, you can go right ahead and use the client ID and client secret with rclone, the only issue will be a very scary confirmation screen shown when you connect via your browser for rclone to be able to get its token-id (but as this only happens during the remote configuration, it's not such a big deal).

    (Thanks to @balazer on github for these instructions.)

    @@ -17640,7 +19732,7 @@

    Making your own client_id

    Google Photos

    The rclone backend for Google Photos is a specialized backend for transferring photos and videos to and from Google Photos.

    NB The Google Photos API which rclone uses has quite a few limitations, so please read the limitations section carefully to make sure it is suitable for your use.

    -

    Configuration

    +

    Configuration

    The initial setup for google cloud storage involves getting a token from Google Photos which you need to do in your browser. rclone config walks you through it.

    Here is an example of how to make a remote called remote. First run:

     rclone config
    @@ -17793,8 +19885,8 @@

    Layout

    This means that you can use the album path pretty much like a normal filesystem and it is a good target for repeated syncing.

    The shared-album directory shows albums shared with you or by you. This is similar to the Sharing tab in the Google Photos web interface.

    -

    Standard options

    -

    Here are the standard options specific to google photos (Google Photos).

    +

    Standard options

    +

    Here are the Standard options specific to google photos (Google Photos).

    --gphotos-client-id

    OAuth Client Id.

    Leave blank normally.

    @@ -17826,7 +19918,7 @@

    --gphotos-read-only

  • Default: false

    • Advanced options

    -

    Here are the advanced options specific to google photos (Google Photos).

    +

    Here are the Advanced options specific to google photos (Google Photos).

    --gphotos-token

    OAuth Access Token as a JSON blob.

    Properties:

    @@ -18007,8 +20099,8 @@

    Pre-Seed from a SUM File

    rclone backend stickyimport hasher:path/to/data sha1 remote:/path/to/sum.sha1

    stickyimport is similar to import but works much faster because it does not need to stat existing files and skips initial tree walk. Instead of binding cache entries to file fingerprints it creates sticky entries bound to the file name alone ignoring size, modification time etc. Such hash entries can be replaced only by purge, delete, backend drop or by full re-read/re-write of the files.

    Configuration reference

    -

    Standard options

    -

    Here are the standard options specific to hasher (Better checksums for other remotes).

    +

    Standard options

    +

    Here are the Standard options specific to hasher (Better checksums for other remotes).

    --hasher-remote

    Remote to cache checksums for (e.g. myRemote:path).

    Properties:

    @@ -18037,7 +20129,7 @@

    --hasher-max-age

  • Default: off

    • Advanced options

    -

    Here are the advanced options specific to hasher (Better checksums for other remotes).

    +

    Here are the Advanced options specific to hasher (Better checksums for other remotes).

    --hasher-auto-size

    Auto-update checksum for files smaller than this size (disabled by default).

    Properties:

    @@ -18047,12 +20139,15 @@

    --hasher-auto-size

  • Type: SizeSuffix
  • Default: 0
    • +

    Metadata

    +

    Any metadata supported by the underlying remote is read and written.

    +

    See the metadata docs for more info.

    Backend commands

    Here are the commands specific to the hasher backend.

    Run them with

    rclone backend COMMAND remote:

    The help below will explain what arguments each command takes.

    -

    See the "rclone backend" command for more info on how to pass options and arguments.

    +

    See the backend command for more info on how to pass options and arguments.

    These can be run on a running backend using the rc command backend/command.

    drop

    Drop cache

    @@ -18101,7 +20196,7 @@

    Cache storage

    HDFS

    HDFS is a distributed file-system, part of the Apache Hadoop framework.

    Paths are specified as remote: or remote:path/to/dir.

    -

    Configuration

    +

    Configuration

    Here is an example of how to make a remote called remote. First run:

     rclone config

    This will guide you through an interactive setup process:

    @@ -18209,8 +20304,8 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced.

    -

    Standard options

    -

    Here are the standard options specific to hdfs (Hadoop distributed file system).

    +

    Standard options

    +

    Here are the Standard options specific to hdfs (Hadoop distributed file system).

    --hdfs-namenode

    Hadoop name node and port.

    E.g. "namenode:8020" to connect to host namenode at port 8020.

    @@ -18238,7 +20333,7 @@

    --hdfs-username

    Advanced options

    -

    Here are the advanced options specific to hdfs (Hadoop distributed file system).

    +

    Here are the Advanced options specific to hdfs (Hadoop distributed file system).

    --hdfs-service-principal-name

    Kerberos service principal name for the namenode.

    Enables KERBEROS authentication. Specifies the Service Principal Name (SERVICE/FQDN) for the namenode. E.g. "hdfs/namenode.hadoop.docker" for namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'.

    @@ -18281,13 +20376,309 @@

    Limitations

  • No server-side Move or DirMove.
  • Checksums not implemented.
    • +

    HiDrive

    +

    Paths are specified as remote:path

    +

    Paths may be as deep as required, e.g. remote:directory/subdirectory.

    +

    The initial setup for hidrive involves getting a token from HiDrive which you need to do in your browser. rclone config walks you through it.

    +

    Configuration

    +

    Here is an example of how to make a remote called remote. First run:

    +
     rclone config
    +

    This will guide you through an interactive setup process:

    +
    No remotes found - make a new one
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+[snip]
+XX / HiDrive
+   \ "hidrive"
+[snip]
+Storage> hidrive
+OAuth Client Id - Leave blank normally.
+client_id>
+OAuth Client Secret - Leave blank normally.
+client_secret>
+Access permissions that rclone should use when requesting access from HiDrive.
+Leave blank normally.
+scope_access>
+Edit advanced config?
+y/n> n
+Use auto config?
+y/n> y
+If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx
+Log in and authorize rclone for access
+Waiting for code...
+Got code
+--------------------
+[remote]
+type = hidrive
+token = {"access_token":"xxxxxxxxxxxxxxxxxxxx","token_type":"Bearer","refresh_token":"xxxxxxxxxxxxxxxxxxxxxxx","expiry":"xxxxxxxxxxxxxxxxxxxxxxx"}
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
    +

    You should be aware that OAuth-tokens can be used to access your account and hence should not be shared with other persons. See the below section for more information.

    +

    See the remote setup docs for how to set it up on a machine with no Internet browser available.

    +

    Note that rclone runs a webserver on your local machine to collect the token as returned from HiDrive. This only runs from the moment it opens your browser to the moment you get back the verification code. The webserver runs on http://127.0.0.1:53682/. If local port 53682 is protected by a firewall you may need to temporarily unblock the firewall to complete authorization.

    +

    Once configured you can then use rclone like this,

    +

    List directories in top level of your HiDrive root folder

    +
    rclone lsd remote:
    +

    List all the files in your HiDrive filesystem

    +
    rclone ls remote:
    +

    To copy a local directory to a HiDrive directory called backup

    +
    rclone copy /home/source remote:backup
    +

    Keeping your tokens safe

    +

    Any OAuth-tokens will be stored by rclone in the remote's configuration file as unencrypted text. Anyone can use a valid refresh-token to access your HiDrive filesystem without knowing your password. Therefore you should make sure no one else can access your configuration.

    +

    It is possible to encrypt rclone's configuration file. You can find information on securing your configuration file by viewing the configuration encryption docs.

    +

    Invalid refresh token

    +

    As can be verified here, each refresh_token (for Native Applications) is valid for 60 days. If used to access HiDrivei, its validity will be automatically extended.

    +

    This means that if you

    +
      +
    • Don't use the HiDrive remote for 60 days
      • +
    +

    then rclone will return an error which includes a text that implies the refresh token is invalid or expired.

    +

    To fix this you will need to authorize rclone to access your HiDrive account again.

    +

    Using

    +
    rclone config reconnect remote:
    +

    the process is very similar to the process of initial setup exemplified before.

    +

    Modified time and hashes

    +

    HiDrive allows modification times to be set on objects accurate to 1 second.

    +

    HiDrive supports its own hash type which is used to verify the integrety of file contents after successful transfers.

    +

    Restricted filename characters

    +

    HiDrive cannot store files or folders that include / (0x2F) or null-bytes (0x00) in their name. Any other characters can be used in the names of files or folders. Additionally, files or folders cannot be named either of the following: . or ..

    +

    Therefore rclone will automatically replace these characters, if files or folders are stored or accessed with such names.

    +

    You can read about how this filename encoding works in general here.

    +

    Keep in mind that HiDrive only supports file or folder names with a length of 255 characters or less.

    +

    Transfers

    +

    HiDrive limits file sizes per single request to a maximum of 2 GiB. To allow storage of larger files and allow for better upload performance, the hidrive backend will use a chunked transfer for files larger than 96 MiB. Rclone will upload multiple parts/chunks of the file at the same time. Chunks in the process of being uploaded are buffered in memory, so you may want to restrict this behaviour on systems with limited resources.

    +

    You can customize this behaviour using the following options:

    +
      +
    • chunk_size: size of file parts
      • +
    • upload_cutoff: files larger or equal to this in size will use a chunked transfer
      • +
    • upload_concurrency: number of file-parts to upload at the same time
      • +
    +

    See the below section about configuration options for more details.

    +

    Root folder

    +

    You can set the root folder for rclone. This is the directory that rclone considers to be the root of your HiDrive.

    +

    Usually, you will leave this blank, and rclone will use the root of the account.

    +

    However, you can set this to restrict rclone to a specific folder hierarchy.

    +

    This works by prepending the contents of the root_prefix option to any paths accessed by rclone. For example, the following two ways to access the home directory are equivalent:

    +
    rclone lsd --hidrive-root-prefix="/users/test/" remote:path
+
+rclone lsd remote:/users/test/path
    +

    See the below section about configuration options for more details.

    +

    Directory member count

    +

    By default, rclone will know the number of directory members contained in a directory. For example, rclone lsd uses this information.

    +

    The acquisition of this information will result in additional time costs for HiDrive's API. When dealing with large directory structures, it may be desirable to circumvent this time cost, especially when this information is not explicitly needed. For this, the disable_fetching_member_count option can be used.

    +

    See the below section about configuration options for more details.

    +

    Standard options

    +

    Here are the Standard options specific to hidrive (HiDrive).

    +

    --hidrive-client-id

    +

    OAuth Client Id.

    +

    Leave blank normally.

    +

    Properties:

    +
      +
    • Config: client_id
      • +
    • Env Var: RCLONE_HIDRIVE_CLIENT_ID
      • +
    • Type: string
      • +
    • Required: false
      • +
    +

    --hidrive-client-secret

    +

    OAuth Client Secret.

    +

    Leave blank normally.

    +

    Properties:

    +
      +
    • Config: client_secret
      • +
    • Env Var: RCLONE_HIDRIVE_CLIENT_SECRET
      • +
    • Type: string
      • +
    • Required: false
      • +
    +

    --hidrive-scope-access

    +

    Access permissions that rclone should use when requesting access from HiDrive.

    +

    Properties:

    +
      +
    • Config: scope_access
      • +
    • Env Var: RCLONE_HIDRIVE_SCOPE_ACCESS
      • +
    • Type: string
      • +
    • Default: "rw"
      • +
    • Examples: +
        +
      • "rw" +
          +
        • Read and write access to resources.
          • +
        • +
      • "ro" +
          +
        • Read-only access to resources.
          • +
        • +
      • +
    +

    Advanced options

    +

    Here are the Advanced options specific to hidrive (HiDrive).

    +

    --hidrive-token

    +

    OAuth Access Token as a JSON blob.

    +

    Properties:

    +
      +
    • Config: token
      • +
    • Env Var: RCLONE_HIDRIVE_TOKEN
      • +
    • Type: string
      • +
    • Required: false
      • +
    +

    --hidrive-auth-url

    +

    Auth server URL.

    +

    Leave blank to use the provider defaults.

    +

    Properties:

    +
      +
    • Config: auth_url
      • +
    • Env Var: RCLONE_HIDRIVE_AUTH_URL
      • +
    • Type: string
      • +
    • Required: false
      • +
    +

    --hidrive-token-url

    +

    Token server url.

    +

    Leave blank to use the provider defaults.

    +

    Properties:

    +
      +
    • Config: token_url
      • +
    • Env Var: RCLONE_HIDRIVE_TOKEN_URL
      • +
    • Type: string
      • +
    • Required: false
      • +
    +

    --hidrive-scope-role

    +

    User-level that rclone should use when requesting access from HiDrive.

    +

    Properties:

    +
      +
    • Config: scope_role
      • +
    • Env Var: RCLONE_HIDRIVE_SCOPE_ROLE
      • +
    • Type: string
      • +
    • Default: "user"
      • +
    • Examples: +
        +
      • "user" +
          +
        • User-level access to management permissions.
          • +
        • This will be sufficient in most cases.
          • +
        • +
      • "admin" +
          +
        • Extensive access to management permissions.
          • +
        • +
      • "owner" +
          +
        • Full access to management permissions.
          • +
        • +
      • +
    +

    --hidrive-root-prefix

    +

    The root/parent folder for all paths.

    +

    Fill in to use the specified folder as the parent for all paths given to the remote. This way rclone can use any folder as its starting point.

    +

    Properties:

    +
      +
    • Config: root_prefix
      • +
    • Env Var: RCLONE_HIDRIVE_ROOT_PREFIX
      • +
    • Type: string
      • +
    • Default: "/"
      • +
    • Examples: +
        +
      • "/" +
          +
        • The topmost directory accessible by rclone.
          • +
        • This will be equivalent with "root" if rclone uses a regular HiDrive user account.
          • +
        • +
      • "root" +
          +
        • The topmost directory of the HiDrive user account
          • +
        • +
      • "" +
          +
        • This specifies that there is no root-prefix for your paths.
          • +
        • When using this you will always need to specify paths to this remote with a valid parent e.g. "remote:/path/to/dir" or "remote:root/path/to/dir".
          • +
        • +
      • +
    +

    --hidrive-endpoint

    +

    Endpoint for the service.

    +

    This is the URL that API-calls will be made to.

    +

    Properties:

    +
      +
    • Config: endpoint
      • +
    • Env Var: RCLONE_HIDRIVE_ENDPOINT
      • +
    • Type: string
      • +
    • Default: "https://api.hidrive.strato.com/2.1"
      • +
    +

    --hidrive-disable-fetching-member-count

    +

    Do not fetch number of objects in directories unless it is absolutely necessary.

    +

    Requests may be faster if the number of objects in subdirectories is not fetched.

    +

    Properties:

    +
      +
    • Config: disable_fetching_member_count
      • +
    • Env Var: RCLONE_HIDRIVE_DISABLE_FETCHING_MEMBER_COUNT
      • +
    • Type: bool
      • +
    • Default: false
      • +
    +

    --hidrive-chunk-size

    +

    Chunksize for chunked uploads.

    +

    Any files larger than the configured cutoff (or files of unknown size) will be uploaded in chunks of this size.

    +

    The upper limit for this is 2147483647 bytes (about 2.000Gi). That is the maximum amount of bytes a single upload-operation will support. Setting this above the upper limit or to a negative value will cause uploads to fail.

    +

    Setting this to larger values may increase the upload speed at the cost of using more memory. It can be set to smaller values smaller to save on memory.

    +

    Properties:

    +
      +
    • Config: chunk_size
      • +
    • Env Var: RCLONE_HIDRIVE_CHUNK_SIZE
      • +
    • Type: SizeSuffix
      • +
    • Default: 48Mi
      • +
    +

    --hidrive-upload-cutoff

    +

    Cutoff/Threshold for chunked uploads.

    +

    Any files larger than this will be uploaded in chunks of the configured chunksize.

    +

    The upper limit for this is 2147483647 bytes (about 2.000Gi). That is the maximum amount of bytes a single upload-operation will support. Setting this above the upper limit will cause uploads to fail.

    +

    Properties:

    +
      +
    • Config: upload_cutoff
      • +
    • Env Var: RCLONE_HIDRIVE_UPLOAD_CUTOFF
      • +
    • Type: SizeSuffix
      • +
    • Default: 96Mi
      • +
    +

    --hidrive-upload-concurrency

    +

    Concurrency for chunked uploads.

    +

    This is the upper limit for how many transfers for the same file are running concurrently. Setting this above to a value smaller than 1 will cause uploads to deadlock.

    +

    If you are uploading small numbers of large files over high-speed links and these uploads do not fully utilize your bandwidth, then increasing this may help to speed up the transfers.

    +

    Properties:

    +
      +
    • Config: upload_concurrency
      • +
    • Env Var: RCLONE_HIDRIVE_UPLOAD_CONCURRENCY
      • +
    • Type: int
      • +
    • Default: 4
      • +
    +

    --hidrive-encoding

    +

    The encoding for the backend.

    +

    See the encoding section in the overview for more info.

    +

    Properties:

    +
      +
    • Config: encoding
      • +
    • Env Var: RCLONE_HIDRIVE_ENCODING
      • +
    • Type: MultiEncoder
      • +
    • Default: Slash,Dot
      • +
    +

    Limitations

    + +

    HiDrive is able to store symbolic links (symlinks) by design, for example, when unpacked from a zip archive.

    +

    There exists no direct mechanism to manage native symlinks in remotes. As such this implementation has chosen to ignore any native symlinks present in the remote. rclone will not be able to access or show any symlinks stored in the hidrive-remote. This means symlinks cannot be individually removed, copied, or moved, except when removing, copying, or moving the parent folder.

    +

    This does not affect the .rclonelink-files that rclone uses to encode and store symbolic links.

    +

    Sparse files

    +

    It is possible to store sparse files in HiDrive.

    +

    Note that copying a sparse file will expand the holes into null-byte (0x00) regions that will then consume disk space. Likewise, when downloading a sparse file, the resulting file will have null-byte regions in the place of file holes.

    HTTP

    The HTTP remote is a read only remote for reading files of a webserver. The webserver should provide file listings which rclone will read and turn into a remote. This has been tested with common webservers such as Apache/Nginx/Caddy and will likely work with file listings from most web servers. (If it doesn't then please file an issue, or send a pull request!)

    Paths are specified as remote: or remote:path.

    The remote: represents the configured url, and any path following it will be resolved relative to this url, according to the URL standard. This means with remote url https://beta.rclone.org/branch and path fix, the resolved URL will be https://beta.rclone.org/branch/fix, while with path /fix the resolved URL will be https://beta.rclone.org/fix as the absolute path is resolved from the root of the domain.

    If the path following the remote: ends with / it will be assumed to point to a directory. If the path does not end with /, then a HEAD request is sent and the response used to decide if it it is treated as a file or a directory (run with -vv to see details). When --http-no-head is specified, a path without ending / is always assumed to be a file. If rclone incorrectly assumes the path is a file, the solution is to specify the path with ending /. When you know the path is a directory, ending it with / is always better as it avoids the initial HEAD request.

    To just download a single file it is easier to use copyurl.

    -

    Configuration

    +

    Configuration

    Here is an example of how to make a remote called remote. First run:

     rclone config

    This will guide you through an interactive setup process:

    @@ -18300,7 +20691,7 @@

    Configuration

    Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / http Connection +XX / HTTP \ "http" [snip] Storage> http @@ -18350,10 +20741,10 @@

    Usage without a config file

    rclone lsd --http-url https://beta.rclone.org :http:

    or:

    rclone lsd :http,url='https://beta.rclone.org':
    -

    Standard options

    -

    Here are the standard options specific to http (http Connection).

    +

    Standard options

    +

    Here are the Standard options specific to http (HTTP).

    --http-url

    -

    URL of http host to connect to.

    +

    URL of HTTP host to connect to.

    E.g. "https://example.com", or "https://user:pass@example.com" to use a username and password.

    Properties:

      @@ -18362,8 +20753,8 @@

      --http-url

    • Type: string
    • Required: true
    -

    Advanced options

    -

    Here are the advanced options specific to http (http Connection).

    +

    Advanced options

    +

    Here are the Advanced options specific to http (HTTP).

    --http-headers

    Set HTTP headers for all transactions.

    Use this to set additional HTTP headers for all transactions.

    @@ -18405,13 +20796,13 @@

    --http-no-head

  • Type: bool
  • Default: false
    • -

    Limitations

    +

    Limitations

    rclone about is not supported by the HTTP backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

    -

    See List of backends that do not support rclone about See rclone about

    +

    See List of backends that do not support rclone about and rclone about

    Hubic

    Paths are specified as remote:path

    Paths are specified as remote:container (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:container/path/to/dir.

    -

    Configuration

    +

    Configuration

    The initial setup for Hubic involves getting a token from Hubic which you need to do in your browser. rclone config walks you through it.

    Here is an example of how to make a remote called remote. First run:

     rclone config
    @@ -18469,8 +20860,8 @@

    Modified time

    The modified time is stored as metadata on the object as X-Object-Meta-Mtime as floating point since the epoch accurate to 1 ns.

    This is a de facto standard (used in the official python-swiftclient amongst others) for storing the modification time for an object.

    Note that Hubic wraps the Swift backend, so most of the properties of are the same.

    -

    Standard options

    -

    Here are the standard options specific to hubic (Hubic).

    +

    Standard options

    +

    Here are the Standard options specific to hubic (Hubic).

    --hubic-client-id

    OAuth Client Id.

    Leave blank normally.

    @@ -18491,8 +20882,8 @@

    --hubic-client-secret

  • Type: string
  • Required: false
    • -

    Advanced options

    -

    Here are the advanced options specific to hubic (Hubic).

    +

    Advanced options

    +

    Here are the Advanced options specific to hubic (Hubic).

    --hubic-token

    OAuth Access Token as a JSON blob.

    Properties:

    @@ -18554,9 +20945,288 @@

    --hubic-encoding

  • Type: MultiEncoder
  • Default: Slash,InvalidUtf8
    • -

    Limitations

    +

    Limitations

    This uses the normal OpenStack Swift mechanism to refresh the Swift API credentials and ignores the expires field returned by the Hubic API.

    The Swift API doesn't return a correct MD5SUM for segmented files (Dynamic or Static Large Objects) so rclone won't check or use the MD5SUM for these.

    +

    Internet Archive

    +

    The Internet Archive backend utilizes Items on archive.org

    +

    Refer to IAS3 API documentation for the API this backend uses.

    +

    Paths are specified as remote:bucket (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:item/path/to/dir.

    +

    Once you have made a remote (see the provider specific section above) you can use it like this:

    +

    Unlike S3, listing up all items uploaded by you isn't supported.

    +

    Make a new item

    +
    rclone mkdir remote:item
    +

    List the contents of a item

    +
    rclone ls remote:item
    +

    Sync /home/local/directory to the remote item, deleting any excess files in the item.

    +
    rclone sync -i /home/local/directory remote:item
    +

    Notes

    +

    Because of Internet Archive's architecture, it enqueues write operations (and extra post-processings) in a per-item queue. You can check item's queue at https://catalogd.archive.org/history/item-name-here . Because of that, all uploads/deletes will not show up immediately and takes some time to be available. The per-item queue is enqueued to an another queue, Item Deriver Queue. You can check the status of Item Deriver Queue here. This queue has a limit, and it may block you from uploading, or even deleting. You should avoid uploading a lot of small files for better behavior.

    +

    You can optionally wait for the server's processing to finish, by setting non-zero value to wait_archive key. By making it wait, rclone can do normal file comparison. Make sure to set a large enough value (e.g. 30m0s for smaller files) as it can take a long time depending on server's queue.

    +

    About metadata

    +

    This backend supports setting, updating and reading metadata of each file. The metadata will appear as file metadata on Internet Archive. However, some fields are reserved by both Internet Archive and rclone.

    +

    The following are reserved by Internet Archive: - name - source - size - md5 - crc32 - sha1 - format - old_version - viruscheck

    +

    Trying to set values to these keys is ignored with a warning. Only setting mtime is an exception. Doing so make it the identical behavior as setting ModTime.

    +

    rclone reserves all the keys starting with rclone-. Setting value for these keys will give you warnings, but values are set according to request.

    +

    If there are multiple values for a key, only the first one is returned. This is a limitation of rclone, that supports one value per one key. It can be triggered when you did a server-side copy.

    +

    Reading metadata will also provide custom (non-standard nor reserved) ones.

    +

    Configuration

    +

    Here is an example of making an internetarchive configuration. Most applies to the other providers as well, any differences are described below.

    +

    First run

    +
    rclone config
    +

    This will guide you through an interactive setup process.

    +
    No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+XX / InternetArchive Items
+   \ (internetarchive)
+Storage> internetarchive
+Option access_key_id.
+IAS3 Access Key.
+Leave blank for anonymous access.
+You can find one here: https://archive.org/account/s3.php
+Enter a value. Press Enter to leave empty.
+access_key_id> XXXX
+Option secret_access_key.
+IAS3 Secret Key (password).
+Leave blank for anonymous access.
+Enter a value. Press Enter to leave empty.
+secret_access_key> XXXX
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> y
+Option endpoint.
+IAS3 Endpoint.
+Leave blank for default value.
+Enter a string value. Press Enter for the default (https://s3.us.archive.org).
+endpoint> 
+Option front_endpoint.
+Host of InternetArchive Frontend.
+Leave blank for default value.
+Enter a string value. Press Enter for the default (https://archive.org).
+front_endpoint> 
+Option disable_checksum.
+Don't store MD5 checksum with object metadata.
+Normally rclone will calculate the MD5 checksum of the input before
+uploading it so it can ask the server to check the object against checksum.
+This is great for data integrity checking but can cause long delays for
+large files to start uploading.
+Enter a boolean value (true or false). Press Enter for the default (true).
+disable_checksum> true
+Option encoding.
+The encoding for the backend.
+See the [encoding section in the overview](https://rclone.org/overview/#encoding) for more info.
+Enter a encoder.MultiEncoder value. Press Enter for the default (Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot).
+encoding> 
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+--------------------
+[remote]
+type = internetarchive
+access_key_id = XXXX
+secret_access_key = XXXX
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
    +

    Standard options

    +

    Here are the Standard options specific to internetarchive (Internet Archive).

    +

    --internetarchive-access-key-id

    +

    IAS3 Access Key.

    +

    Leave blank for anonymous access. You can find one here: https://archive.org/account/s3.php

    +

    Properties:

    +
      +
    • Config: access_key_id
      • +
    • Env Var: RCLONE_INTERNETARCHIVE_ACCESS_KEY_ID
      • +
    • Type: string
      • +
    • Required: false
      • +
    +

    --internetarchive-secret-access-key

    +

    IAS3 Secret Key (password).

    +

    Leave blank for anonymous access.

    +

    Properties:

    +
      +
    • Config: secret_access_key
      • +
    • Env Var: RCLONE_INTERNETARCHIVE_SECRET_ACCESS_KEY
      • +
    • Type: string
      • +
    • Required: false
      • +
    +

    Advanced options

    +

    Here are the Advanced options specific to internetarchive (Internet Archive).

    +

    --internetarchive-endpoint

    +

    IAS3 Endpoint.

    +

    Leave blank for default value.

    +

    Properties:

    +
      +
    • Config: endpoint
      • +
    • Env Var: RCLONE_INTERNETARCHIVE_ENDPOINT
      • +
    • Type: string
      • +
    • Default: "https://s3.us.archive.org"
      • +
    +

    --internetarchive-front-endpoint

    +

    Host of InternetArchive Frontend.

    +

    Leave blank for default value.

    +

    Properties:

    +
      +
    • Config: front_endpoint
      • +
    • Env Var: RCLONE_INTERNETARCHIVE_FRONT_ENDPOINT
      • +
    • Type: string
      • +
    • Default: "https://archive.org"
      • +
    +

    --internetarchive-disable-checksum

    +

    Don't ask the server to test against MD5 checksum calculated by rclone. Normally rclone will calculate the MD5 checksum of the input before uploading it so it can ask the server to check the object against checksum. This is great for data integrity checking but can cause long delays for large files to start uploading.

    +

    Properties:

    +
      +
    • Config: disable_checksum
      • +
    • Env Var: RCLONE_INTERNETARCHIVE_DISABLE_CHECKSUM
      • +
    • Type: bool
      • +
    • Default: true
      • +
    +

    --internetarchive-wait-archive

    +

    Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish. Only enable if you need to be guaranteed to be reflected after write operations. 0 to disable waiting. No errors to be thrown in case of timeout.

    +

    Properties:

    +
      +
    • Config: wait_archive
      • +
    • Env Var: RCLONE_INTERNETARCHIVE_WAIT_ARCHIVE
      • +
    • Type: Duration
      • +
    • Default: 0s
      • +
    +

    --internetarchive-encoding

    +

    The encoding for the backend.

    +

    See the encoding section in the overview for more info.

    +

    Properties:

    +
      +
    • Config: encoding
      • +
    • Env Var: RCLONE_INTERNETARCHIVE_ENCODING
      • +
    • Type: MultiEncoder
      • +
    • Default: Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot
      • +
    +

    Metadata

    +

    Metadata fields provided by Internet Archive. If there are multiple values for a key, only the first one is returned. This is a limitation of Rclone, that supports one value per one key.

    +

    Owner is able to add custom keys. Metadata feature grabs all the keys including them.

    +

    Here are the possible system metadata items for the internetarchive backend.

    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameHelpTypeExampleRead Only
    crc32CRC32 calculated by Internet Archivestring01234567N
    formatName of format identified by Internet ArchivestringComma-Separated ValuesN
    md5MD5 hash calculated by Internet Archivestring01234567012345670123456701234567N
    mtimeTime of last modification, managed by RcloneRFC 33392006-01-02T15:04:05.999999999ZN
    nameFull file path, without the bucket partfilenamebackend/internetarchive/internetarchive.goN
    old_versionWhether the file was replaced and moved by keep-old-version flagbooleantrueN
    rclone-ia-mtimeTime of last modification, managed by Internet ArchiveRFC 33392006-01-02T15:04:05.999999999ZN
    rclone-mtimeTime of last modification, managed by RcloneRFC 33392006-01-02T15:04:05.999999999ZN
    rclone-update-trackRandom value used by Rclone for tracking changes inside Internet ArchivestringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaN
    sha1SHA1 hash calculated by Internet Archivestring0123456701234567012345670123456701234567N
    sizeFile size in bytesdecimal number123456N
    sourceThe source of the filestringoriginalN
    viruscheckThe last time viruscheck process was run for the file (?)unixtime1654191352N
    +

    See the metadata docs for more info.

    Jottacloud

    Jottacloud is a cloud storage service provider from a Norwegian company, using its own datacenters in Norway. In addition to the official service at jottacloud.com, it also provides white-label solutions to different companies, such as: * Telia * Telia Cloud (cloud.telia.se) * Telia Sky (sky.telia.no) * Tele2 * Tele2 Cloud (mittcloud.tele2.se) * Elkjøp (with subsidiaries): * Elkjøp Cloud (cloud.elkjop.no) * Elgiganten Sweden (cloud.elgiganten.se) * Elgiganten Denmark (cloud.elgiganten.dk) * Giganti Cloud (cloud.gigantti.fi) * ELKO Clouud (cloud.elko.is)

    Most of the white-label versions are supported by this backend, although may require different authentication setup - described below.

    @@ -18572,7 +21242,7 @@

    Telia Cloud authentication

    Similar to other whitelabel versions Telia Cloud doesn't offer the option of creating a CLI token, and additionally uses a separate authentication flow where the username is generated internally. To setup rclone to use Telia Cloud, choose Telia Cloud authentication in the setup. The rest of the setup is identical to the default setup.

    Tele2 Cloud authentication

    As Tele2-Com Hem merger was completed this authentication can be used for former Com Hem Cloud and Tele2 Cloud customers as no support for creating a CLI token exists, and additionally uses a separate authentication flow where the username is generated internally. To setup rclone to use Tele2 Cloud, choose Tele2 Cloud authentication in the setup. The rest of the setup is identical to the default setup.

    -

    Configuration

    +

    Configuration

    Here is an example of how to make a remote called remote with the default setup. First run:

    rclone config

    This will guide you through an interactive setup process:

    @@ -18582,56 +21252,78 @@

    Configuration

    q) Quit config n/s/q> n name> remote +Option Storage. Type of storage to configure. -Enter a string value. Press Enter for the default (""). -Choose a number from below, or type in your own value +Choose a number from below, or type in your own value. [snip] XX / Jottacloud - \ "jottacloud" + \ (jottacloud) [snip] Storage> jottacloud -** See help for jottacloud backend at: https://rclone.org/jottacloud/ ** - -Edit advanced config? (y/n) -y) Yes -n) No -y/n> n -Remote config -Use legacy authentication?. -This is only required for certain whitelabel versions of Jottacloud and not recommended for normal users. +Edit advanced config? y) Yes n) No (default) y/n> n - -Generate a personal login token here: https://www.jottacloud.com/web/secure +Option config_type. +Select authentication type. +Choose a number from below, or type in an existing string value. +Press Enter for the default (standard). + / Standard authentication. + 1 | Use this if you're a normal Jottacloud user. + \ (standard) + / Legacy authentication. + 2 | This is only required for certain whitelabel versions of Jottacloud and not recommended for normal users. + \ (legacy) + / Telia Cloud authentication. + 3 | Use this if you are using Telia Cloud. + \ (telia) + / Tele2 Cloud authentication. + 4 | Use this if you are using Tele2 Cloud. + \ (tele2) +config_type> 1 +Personal login token. +Generate here: https://www.jottacloud.com/web/secure Login Token> <your token here> - -Do you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client? - +Use a non-standard device/mountpoint? +Choosing no, the default, will let you access the storage used for the archive +section of the official Jottacloud client. If you instead want to access the +sync or the backup section, for example, you must choose yes. y) Yes -n) No +n) No (default) y/n> y -Please select the device to use. Normally this will be Jotta -Choose a number from below, or type in an existing value +Option config_device. +The device to use. In standard setup the built-in Jotta device is used, +which contains predefined mountpoints for archive, sync etc. All other devices +are treated as backup devices by the official Jottacloud client. You may create +a new by entering a unique name. +Choose a number from below, or type in your own string value. +Press Enter for the default (DESKTOP-3H31129). 1 > DESKTOP-3H31129 2 > Jotta -Devices> 2 -Please select the mountpoint to user. Normally this will be Archive -Choose a number from below, or type in an existing value +config_device> 2 +Option config_mountpoint. +The mountpoint to use for the built-in device Jotta. +The standard setup is to use the Archive mountpoint. Most other mountpoints +have very limited support in rclone and should generally be avoided. +Choose a number from below, or type in an existing string value. +Press Enter for the default (Archive). 1 > Archive - 2 > Links + 2 > Shared 3 > Sync - -Mountpoints> 1 +config_mountpoint> 1 -------------------- -[jotta] +[remote] type = jottacloud +configVersion = 1 +client_id = jottacli +client_secret = +tokenURL = https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token token = {........} +username = 2940e57271a93d987d6f8a21 device = Jotta mountpoint = Archive -configVersion = 1 -------------------- -y) Yes this is OK +y) Yes this is OK (default) e) Edit this remote d) Delete this remote y/e/d> y @@ -18643,18 +21335,19 @@

    Configuration

    To copy a local directory to an Jottacloud directory called backup

    rclone copy /home/source remote:backup

    Devices and Mountpoints

    -

    The official Jottacloud client registers a device for each computer you install it on, and then creates a mountpoint for each folder you select for Backup. The web interface uses a special device called Jotta for the Archive and Sync mountpoints.

    -

    With rclone you'll want to use the Jotta/Archive device/mountpoint in most cases, however if you want to access files uploaded by any of the official clients rclone provides the option to select other devices and mountpoints during config. Note that uploading files is currently not supported to other devices than Jotta.

    -

    The built-in Jotta device may also contain several other mountpoints, such as: Latest, Links, Shared and Trash. These are special mountpoints with a different internal representation than the "regular" mountpoints. Rclone will only to a very limited degree support them. Generally you should avoid these, unless you know what you are doing.

    +

    The official Jottacloud client registers a device for each computer you install it on, and shows them in the backup section of the user interface. For each folder you select for backup it will create a mountpoint within this device. A built-in device called Jotta is special, and contains mountpoints Archive, Sync and some others, used for corresponding features in official clients.

    +

    With rclone you'll want to use the standard Jotta/Archive device/mountpoint in most cases. However, you may for example want to access files from the sync or backup functionality provided by the official clients, and rclone therefore provides the option to select other devices and mountpoints during config.

    +

    You are allowed to create new devices and mountpoints. All devices except the built-in Jotta device are treated as backup devices by official Jottacloud clients, and the mountpoints on them are individual backup sets.

    +

    With the built-in Jotta device, only existing, built-in, mountpoints can be selected. In addition to the mentioned Archive and Sync, it may contain several other mountpoints such as: Latest, Links, Shared and Trash. All of these are special mountpoints with a different internal representation than the "regular" mountpoints. Rclone will only to a very limited degree support them. Generally you should avoid these, unless you know what you are doing.

    --fast-list

    This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.

    Note that the implementation in Jottacloud always uses only a single API request to get the entire list, so for large folders this could lead to long wait time before the first results are shown.

    Note also that with rclone version 1.58 and newer information about MIME types are not available when using --fast-list.

    -

    Modified time and hashes

    +

    Modified time and hashes

    Jottacloud allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not.

    Jottacloud supports MD5 type hashes, so you can use the --checksum flag.

    Note that Jottacloud requires the MD5 hash before upload so if the source does not have an MD5 checksum then the file will be cached temporarily on disk (in location given by --temp-dir) before it is uploaded. Small files will be cached in memory - see the --jottacloud-md5-memory-limit flag. When uploading from local disk the source checksum is always available, so this does not apply. Starting with rclone version 1.52 the same is true for crypted remotes (in older versions the crypt backend would not calculate hashes for uploads from local disk, so the Jottacloud backend had to do it as described above).

    -

    Restricted filename characters

    +

    Restricted filename characters

    In addition to the default restricted characters set the following characters are also replaced:

    @@ -18710,8 +21403,8 @@

    Versions

    Versioning can be disabled by --jottacloud-no-versions option. This is achieved by deleting the remote file prior to uploading a new version. If the upload the fails no version of the file will be available in the remote.

    Quota information

    To view your current quota you can use the rclone about remote: command which will display your usage limit (unless it is unlimited) and the current usage.

    -

    Advanced options

    -

    Here are the advanced options specific to jottacloud (Jottacloud).

    +

    Advanced options

    +

    Here are the Advanced options specific to jottacloud (Jottacloud).

    --jottacloud-md5-memory-limit

    Files bigger than this will be cached on disk to calculate the MD5 if required.

    Properties:

    @@ -18768,7 +21461,7 @@

    --jottacloud-encoding

  • Type: MultiEncoder
  • Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot
    • -

    Limitations

    +

    Limitations

    Note that Jottacloud is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

    There are quite a few characters that can't be in Jottacloud file names. Rclone will map these names to and from an identical looking unicode equivalent. For example if a file has a ? in it will be mapped to ？ instead.

    Jottacloud only supports filenames up to 255 characters in length.

    @@ -18777,7 +21470,7 @@

    Troubleshooting

    Koofr

    Paths are specified as remote:path

    Paths may be as deep as required, e.g. remote:directory/subdirectory.

    -

    Configuration

    +

    Configuration

    The initial setup for Koofr involves creating an application password for rclone. You can do that by opening the Koofr web application, giving the password a nice name like rclone and clicking on generate.

    Here is an example of how to make a remote called koofr. First run:

     rclone config
    @@ -18845,7 +21538,7 @@

    Configuration

    rclone ls koofr:

    To copy a local directory to an Koofr directory called backup

    rclone copy /home/source koofr:backup
    -

    Restricted filename characters

    +

    Restricted filename characters

    In addition to the default restricted characters set the following characters are also replaced:

    @@ -18864,8 +21557,8 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced, as they can't be used in XML strings.

    -

    Standard options

    -

    Here are the standard options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers).

    +

    Standard options

    +

    Here are the Standard options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers).

    --koofr-provider

    Choose your storage provider.

    Properties:

    @@ -18942,8 +21635,8 @@

    --koofr-password

  • Type: string
  • Required: true
    • -

    Advanced options

    -

    Here are the advanced options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers).

    +

    Advanced options

    +

    Here are the Advanced options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers).

    --koofr-mountid

    Mount ID of the mount to use.

    If omitted, the primary mount is used.

    @@ -18974,7 +21667,7 @@

    --koofr-encoding

  • Type: MultiEncoder
  • Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot
    • -

    Limitations

    +

    Limitations

    Note that Koofr is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

    Providers

    Koofr

    @@ -19116,7 +21809,7 @@

    Features highlights

  • Storage keeps hash for all files and performs transparent deduplication, the hash algorithm is a modified SHA1
  • If a particular file is already present in storage, one can quickly submit file hash instead of long file upload (this optimization is supported by rclone)
    • -

    Configuration

    +

    Configuration

    Here is an example of making a mailru configuration. First create a Mail.ru Cloud account and choose a tariff, then run

    rclone config

    This will guide you through an interactive setup process:

    @@ -19190,7 +21883,7 @@

    Emptying Trash

    Removing a file or directory actually moves it to the trash, which is not visible to rclone but can be seen in a web browser. The trashed file still occupies part of total quota. If you wish to empty your trash and free some quota, you can use the rclone cleanup remote: command, which will permanently delete all your trashed files. This command does not take any path arguments.

    Quota information

    To view your current quota you can use the rclone about remote: command which will display your usage limit (quota) and the current usage.

    -

    Restricted filename characters

    +

    Restricted filename characters

    In addition to the default restricted characters set the following characters are also replaced:

    @@ -19244,8 +21937,8 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.

    -

    Standard options

    -

    Here are the standard options specific to mailru (Mail.ru Cloud).

    +

    Standard options

    +

    Here are the Standard options specific to mailru (Mail.ru Cloud).

    --mailru-user

    User name (usually email).

    Properties:

    @@ -19286,8 +21979,8 @@

    --mailru-speedup-enable

    -

    Advanced options

    -

    Here are the advanced options specific to mailru (Mail.ru Cloud).

    +

    Advanced options

    +

    Here are the Advanced options specific to mailru (Mail.ru Cloud).

    --mailru-speedup-file-patterns

    Comma separated list of file name patterns eligible for speedup (put by hash).

    Patterns are case insensitive and can contain '*' or '?' meta characters.

    @@ -19416,7 +22109,7 @@

    --mailru-encoding

  • Type: MultiEncoder
  • Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot
    • -

    Limitations

    +

    Limitations

    File size limits depend on your account. A single file size is limited by 2G for a free account and unlimited for paid tariffs. Please refer to the Mail.ru site for the total uploaded size limits.

    Note that Mailru is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

    Mega

    @@ -19424,7 +22117,7 @@

    Mega

    This is an rclone backend for Mega which supports the file transfer features of Mega using the same client side encryption.

    Paths are specified as remote:path

    Paths may be as deep as required, e.g. remote:directory/subdirectory.

    -

    Configuration

    +

    Configuration

    Here is an example of how to make a remote called remote. First run:

     rclone config

    This will guide you through an interactive setup process:

    @@ -19471,9 +22164,9 @@

    Configuration

    rclone ls remote:

    To copy a local directory to an Mega directory called backup

    rclone copy /home/source remote:backup
    -

    Modified time and hashes

    +

    Modified time and hashes

    Mega does not support modification times or hashes yet.

    -

    Restricted filename characters

    +

    Restricted filename characters

    @@ -19501,6 +22194,20 @@

    Duplicated files

    Duplicated files cause problems with the syncing and you will see messages in the log about duplicates.

    Use rclone dedupe to fix duplicated files.

    Failure to log-in

    +

    Object not found

    +

    If you are connecting to your Mega remote for the first time, to test access and syncronisation, you may receive an error such as

    +
    Failed to create file system for "my-mega-remote:": 
+couldn't login: Object (typically, node or user) not found
    +

    The diagnostic steps often recommended in the rclone forum start with the MEGAcmd utility. Note that this refers to the official C++ command from https://github.com/meganz/MEGAcmd and not the go language built command from t3rm1n4l/megacmd that is no longer maintained.

    +

    Follow the instructions for installing MEGAcmd and try accessing your remote as they recommend. You can establish whether or not you can log in using MEGAcmd, and obtain diagnostic information to help you, and search or work with others in the forum.

    +
    MEGA CMD> login me@example.com
+Password:
+Fetching nodes ...
+Loading transfers from local cache
+Login complete as me@example.com
+me@example.com:/$
    +

    Note that some have found issues with passwords containing special characters. If you can not log on with rclone, but MEGAcmd logs on just fine, then consider changing your password temporarily to pure alphanumeric characters, in case that helps.

    +

    Repeated commands blocks access

    Mega remotes seem to get blocked (reject logins) under "heavy use". We haven't worked out the exact blocking rules but it seems to be related to fast paced, successive rclone commands.

    For example, executing this command 90 times in a row rclone link remote:file will cause the remote to become "blocked". This is not an abnormal situation, for example if you wish to get the public links of a directory with hundred of files... After more or less a week, the remote will remote accept rclone logins normally again.

    You can mitigate this issue by mounting the remote it with rclone mount. This will log-in when mounting and a log-out when unmounting only. You can also run rclone rcd and then use rclone rc to run the commands over the API to avoid logging in each time.

    @@ -19511,8 +22218,8 @@

    Failure to log-in

    Note that once blocked, the use of other tools (such as megacmd) is not a sure workaround: following megacmd login times have been observed in succession for blocked remote: 7 minutes, 20 min, 30min, 30 min, 30min. Web access looks unaffected though.

    Investigation is continuing in relation to workarounds based on timeouts, pacers, retrials and tpslimits - if you discover something relevant, please post on the forum.

    So, if rclone was working nicely and suddenly you are unable to log-in and you are sure the user and the password are correct, likely you have got the remote blocked for a while.

    -

    Standard options

    -

    Here are the standard options specific to mega (Mega).

    +

    Standard options

    +

    Here are the Standard options specific to mega (Mega).

    --mega-user

    User name.

    Properties:

    @@ -19532,8 +22239,8 @@

    --mega-pass

  • Type: string
  • Required: true
    • -

    Advanced options

    -

    Here are the advanced options specific to mega (Mega).

    +

    Advanced options

    +

    Here are the Advanced options specific to mega (Mega).

    --mega-debug

    Output more debug from Mega.

    If this flag is set (along with -vv) it will print further debugging information from the mega backend.

    @@ -19564,13 +22271,13 @@

    --mega-encoding

  • Type: MultiEncoder
  • Default: Slash,InvalidUtf8,Dot
    • -

    Limitations

    +

    Limitations

    This backend uses the go-mega go library which is an opensource go library implementing the Mega API. There doesn't appear to be any documentation for the mega protocol beyond the mega C++ SDK source code so there are likely quite a few errors still remaining in this library.

    Mega allows duplicate files which may confuse rclone.

    Memory

    The memory backend is an in RAM backend. It does not persist its data - use the local backend for that.

    The memory backend behaves like a bucket-based remote (e.g. like s3). Because it has no parameters you can just use it with the :memory: remote name.

    -

    Configuration

    +

    Configuration

    You can configure it as a remote like this with rclone config too if you want to:

    No remotes found, make a new one?
 n) New remote
@@ -19602,14 +22309,15 @@ 
    Configuration
    
 
    rclone mount :memory: /mnt/tmp
 rclone serve webdav :memory:
 rclone serve sftp :memory:
    
-
    Modified time and hashes
    
+
    Modified time and hashes
    
 
    The memory backend supports MD5 hashes and modification times accurate to 1 nS.
    
-
    Restricted filename characters
    
+
    Restricted filename characters
    
 
    The memory backend replaces the default restricted characters set.
    
-
    Akamai NetStorage
    
+
    Akamai NetStorage
    
 
    Paths are specified as remote: You may put subdirectories in too, e.g. remote:/path/to/dir. If you have a CP code you can use that as the folder after the domain such as <domain>/<cpcode>/<internal directories within cpcode>.
    
 
    For example, this is commonly configured with or without a CP code: * With a CP code. [your-domain-prefix]-nsu.akamaihd.net/123456/subdirectory/ * Without a CP code. [your-domain-prefix]-nsu.akamaihd.net
    
 
    See all buckets rclone lsd remote: The initial setup for Netstorage involves getting an account and secret. Use rclone config to walk you through the setup process.
    
+
    Configuration
    
 
    Here's an example of how to make a remote called ns1.
    
 
      
 
    1. To begin the interactive configuration process, enter this command:
      2. 
@@ -19680,19 +22388,20 @@ 
      Akamai NetStorage
      
 d) Delete this remote
 y/e/d> y

    This remote is called ns1 and can now be used.

    -

    Example operations

    +

    Example operations

    Get started with rclone and NetStorage with these examples. For additional rclone commands, visit https://rclone.org/commands/.

    -
    See contents of a directory in your project
    +

    See contents of a directory in your project

    rclone lsd ns1:/974012/testing/
    -
    Sync the contents local with remote
    +

    Sync the contents local with remote

    rclone sync . ns1:/974012/testing/
    -
    Upload local content to remote
    +

    Upload local content to remote

    rclone copy notes.txt ns1:/974012/testing/
    -
    Delete content on remote
    +

    Delete content on remote

    rclone delete ns1:/974012/testing/notes.txt
    -
    Move or copy content between CP codes.
    +

    Move or copy content between CP codes.

    Your credentials must have access to two CP codes on the same remote. You can't perform operations between different remotes.

    rclone move ns1:/974012/testing/notes.txt ns1:/974450/testing2/
    +

    Features

    The Netstorage backend changes the rclone --links, -l behavior. When uploading, instead of creating the .rclonelink file, use the "symlink" API in order to create the corresponding symlink on the remote. The .rclonelink file will not be created, the upload will be intercepted and only the symlink file that matches the source file name with no suffix will be created on the remote.

    This will effectively allow commands like copy/copyto, move/moveto and sync to upload from local to remote and download from remote to local directories with symlinks. Due to internal rclone limitations, it is not possible to upload an individual symlink file to any remote backend. You can always use the "backend symlink" command to create a symlink on the NetStorage server, refer to "symlink" section below.

    @@ -19705,7 +22414,7 @@

    Implicit vs. Explicit Directories

    Implicit Directory. This refers to a directory within a path that has not been physically created. For example, during upload of a file, non-existent subdirectories can be specified in the target path. NetStorage creates these as "implicit." While the directories aren't physically created, they exist implicitly and the noted path is connected with the uploaded file.

    Rclone will intercept all file uploads and mkdir commands for the NetStorage remote and will explicitly issue the mkdir command for each directory in the uploading path. This will help with the interoperability with the other Akamai services such as SFTP and the Content Management Shell (CMShell). Rclone will not guarantee correctness of operations with implicit directories which might have been created as a result of using an upload API directly.

    -

    ListR Feature

    +

    --fast-list / ListR support

    NetStorage remote supports the ListR feature by using the "list" NetStorage API action to return a lexicographical list of all objects within the specified CP code, recursing into subdirectories as they're encountered.

    • Rclone will use the ListR method for some commands by default. Commands such as lsf -R will use ListR by default. To disable this, include the --disable listR option to use the non-recursive method of listing objects.

      • @@ -19713,11 +22422,11 @@

      ListR Feature

    There are pros and cons of using the ListR method, refer to rclone documentation. In general, the sync command over an existing deep tree on the remote will run faster with the "--fast-list" flag but with extra memory usage as a side effect. It might also result in higher CPU utilization but the whole task can be completed faster.

    Note: There is a known limitation that "lsf -R" will display number of files in the directory and directory size as -1 when ListR method is used. The workaround is to pass "--disable listR" flag if these numbers are important in the output.

    -

    Purge Feature

    +

    Purge

    NetStorage remote supports the purge feature by using the "quick-delete" NetStorage API action. The quick-delete action is disabled by default for security reasons and can be enabled for the account through the Akamai portal. Rclone will first try to use quick-delete action for the purge command and if this functionality is disabled then will fall back to a standard delete method.

    Note: Read the NetStorage Usage API for considerations when using "quick-delete". In general, using quick-delete method will not delete the tree immediately and objects targeted for quick-delete may still be accessible.

    -

    Standard options

    -

    Here are the standard options specific to netstorage (Akamai NetStorage).

    +

    Standard options

    +

    Here are the Standard options specific to netstorage (Akamai NetStorage).

    --netstorage-host

    Domain+path of NetStorage host to connect to.

    Format should be <domain>/<internal folders>

    @@ -19748,8 +22457,8 @@

    --netstorage-secret

  • Type: string
  • Required: true
    • -

    Advanced options

    -

    Here are the advanced options specific to netstorage (Akamai NetStorage).

    +

    Advanced options

    +

    Here are the Advanced options specific to netstorage (Akamai NetStorage).

    --netstorage-protocol

    Select between HTTP or HTTPS protocol.

    Most users should choose HTTPS, which is the default. HTTP is provided primarily for debugging purposes.

    @@ -19776,7 +22485,7 @@

    Backend commands

    Run them with

    rclone backend COMMAND remote:

    The help below will explain what arguments each command takes.

    -

    See the "rclone backend" command for more info on how to pass options and arguments.

    +

    See the backend command for more info on how to pass options and arguments.

    These can be run on a running backend using the rc command backend/command.

    du

    Return disk usage information for a specified directory

    @@ -19788,7 +22497,7 @@

    The desired path location (including applicable sub-directories) ending in the object that will be the target of the symlink (for example, /links/mylink). Include the file extension for the object, if applicable. rclone backend symlink <src> <path>

    Microsoft Azure Blob Storage

    Paths are specified as remote:container (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:container/path/to/dir.

    -

    Configuration

    +

    Configuration

    Here is an example of making a Microsoft Azure Blob Storage configuration. For a remote called remote. First run:

     rclone config

    This will guide you through an interactive setup process:

    @@ -19836,7 +22545,7 @@

    Modified time

    The modified time is stored as metadata on the object with the mtime key. It is stored using RFC3339 Format time with nanosecond precision. The metadata is supplied during directory listings so there is no overhead to using it.

    Performance

    When uploading large files, increasing the value of --azureblob-upload-concurrency will increase performance at the cost of using more memory. The default of 16 is set quite conservatively to use less memory. It maybe be necessary raise it to 64 or higher to fully utilize a 1 GBit/s link with a single file transfer.

    -

    Restricted filename characters

    +

    Restricted filename characters

    In addition to the default restricted characters set the following characters are also replaced:

    @@ -19895,8 +22604,8 @@

    SAS URL

    Note that you can't see or access any other containers - this will fail

    rclone ls azureblob:othercontainer

    Container level SAS URLs are useful for temporarily allowing third parties access to a single container or putting credentials into an untrusted environment such as a CI build server.

    -

    Standard options

    -

    Here are the standard options specific to azureblob (Microsoft Azure Blob Storage).

    +

    Standard options

    +

    Here are the Standard options specific to azureblob (Microsoft Azure Blob Storage).

    --azureblob-account

    Storage Account Name.

    Leave blank to use SAS URL or Emulator.

    @@ -19963,8 +22672,8 @@

    --azureblob-use-emulator

  • Type: bool
  • Default: false
    • -

    Advanced options

    -

    Here are the advanced options specific to azureblob (Microsoft Azure Blob Storage).

    +

    Advanced options

    +

    Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage).

    --azureblob-msi-object-id

    Object ID of the user-assigned MSI to use, if any.

    Leave blank if msi_client_id or msi_mi_res_id specified.

    @@ -20143,16 +22852,18 @@

    --azureblob-no-head-object

  • Type: bool
  • Default: false
    • -

    Limitations

    +

    Limitations

    MD5 sums are only uploaded with chunked files if the source has an MD5 sum. This will always be the case for a local to azure copy.

    rclone about is not supported by the Microsoft Azure Blob storage backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

    -

    See List of backends that do not support rclone about See rclone about

    +

    See List of backends that do not support rclone about and rclone about

    Azure Storage Emulator Support

    -

    You can test rclone with storage emulator locally, to do this make sure azure storage emulator installed locally and set up a new remote with rclone config follow instructions described in introduction, set use_emulator config as true, you do not need to provide default account name or key if using emulator.

    +

    You can run rclone with storage emulator (usually azurite).

    +

    To do this, just set up a new remote with rclone config following instructions described in introduction and set use_emulator config as true. You do not need to provide default account name neither an account key.

    +

    Also, if you want to access a storage emulator instance running on a different machine, you can override Endpoint parameter in advanced settings, setting it to http(s)://<host>:<port>/devstoreaccount1 (e.g. http://10.254.2.5:10000/devstoreaccount1).

    Microsoft OneDrive

    Paths are specified as remote:path

    Paths may be as deep as required, e.g. remote:directory/subdirectory.

    -

    Configuration

    +

    Configuration

    The initial setup for OneDrive involves getting a token from Microsoft which you need to do in your browser. rclone config walks you through it.

    Here is an example of how to make a remote called remote. First run:

     rclone config
    @@ -20238,22 +22949,36 @@

    Configuration

    To copy a local directory to an OneDrive directory called backup

    rclone copy /home/source remote:backup

    Getting your own Client ID and Key

    -

    You can use your own Client ID if the default (client_id left blank) one doesn't work for you or you see lots of throttling. The default Client ID and Key is shared by all rclone users when performing requests.

    -

    If you are having problems with them (E.g., seeing a lot of throttling), you can get your own Client ID and Key by following the steps below:

    +

    rclone uses a default Client ID when talking to OneDrive, unless a custom client_id is specified in the config. The default Client ID and Key are shared by all rclone users when performing requests.

    +

    You may choose to create and use your own Client ID, in case the default one does not work well for you. For example, you might see throtting.

    +

    Creating Client ID for OneDrive Personal

    +

    To create your own Client ID, please follow these steps:

    1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade and then click New registration.
    2. Enter a name for your app, choose account type Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox), select Web in Redirect URI, then type (do not copy and paste) http://localhost:53682/ and click Register. Copy and keep the Application (client) ID under the app name for later use.
    3. Under manage select Certificates & secrets, click New client secret. Enter a description (can be anything) and set Expires to 24 months. Copy and keep that secret Value for later use (you won't be able to see this value afterwards).
    4. Under manage select API permissions, click Add a permission and select Microsoft Graph then select delegated permissions.
      5. -
    5. Search and select the following permissions: Files.Read, Files.ReadWrite, Files.Read.All, Files.ReadWrite.All, offline_access, User.Read, and optionally Sites.Read.All (see below). Once selected click Add permissions at the bottom.
      6. +
    6. Search and select the following permissions: Files.Read, Files.ReadWrite, Files.Read.All, Files.ReadWrite.All, offline_access, User.Read and Sites.Read.All (if custom access scopes are configured, select the permissions accordingly). Once selected click Add permissions at the bottom.

    Now the application is complete. Run rclone config to create or edit a OneDrive remote. Supply the app ID and password as Client ID and Secret, respectively. rclone will walk you through the remaining steps.

    -

    The Sites.Read.All permission is required if you need to search SharePoint sites when configuring the remote. However, if that permission is not assigned, you need to set disable_site_permission option to true in the advanced options.

    +

    The access_scopes option allows you to configure the permissions requested by rclone. See Microsoft Docs for more information about the different scopes.

    +

    The Sites.Read.All permission is required if you need to search SharePoint sites when configuring the remote. However, if that permission is not assigned, you need to exclude Sites.Read.All from your access scopes or set disable_site_permission option to true in the advanced options.

    +

    Creating Client ID for OneDrive Business

    +

    The steps for OneDrive Personal may or may not work for OneDrive Business, depending on the security settings of the organization. A common error is that the publisher of the App is not verified.

    +

    You may try to verify you account, or try to limit the App to your organization only, as shown below.

    +
      +
    1. Make sure to create the App with your business account.
      2. +
    2. Follow the steps above to create an App. However, we need a different account type here: Accounts in this organizational directory only (*** - Single tenant). Note that you can also change the account type aftering creating the App.
      3. +
    3. Find the tenant ID of your organization.
      4. +
    4. In the rclone config, set auth_url to https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/authorize.
      5. +
    5. In the rclone config, set token_url to https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/token.
      6. +
    +

    Note: If you have a special region, you may need a different host in step 4 and 5. Here are some hints.

    Modification time and hashes

    OneDrive allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not.

    OneDrive personal supports SHA1 type hashes. OneDrive for business and Sharepoint Server support QuickXorHash.

    For all types of OneDrive you can use the --checksum flag.

    -

    Restricted filename characters

    +

    Restricted filename characters

    In addition to the default restricted characters set the following characters are also replaced:

    @@ -20353,8 +23078,8 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.

    Deleting files

    Any files you delete with rclone will end up in the trash. Microsoft doesn't provide an API to permanently delete files, nor to empty the trash, so you will have to do that with one of Microsoft's apps or via the OneDrive website.

    -

    Standard options

    -

    Here are the standard options specific to onedrive (Microsoft OneDrive).

    +

    Standard options

    +

    Here are the Standard options specific to onedrive (Microsoft OneDrive).

    --onedrive-client-id

    OAuth Client Id.

    Leave blank normally.

    @@ -20403,8 +23128,8 @@

    --onedrive-region

    -

    Advanced options

    -

    Here are the advanced options specific to onedrive (Microsoft OneDrive).

    +

    Advanced options

    +

    Here are the Advanced options specific to onedrive (Microsoft OneDrive).

    --onedrive-token

    OAuth Access Token as a JSON blob.

    Properties:

    @@ -20472,6 +23197,32 @@

    --onedrive-root-folder-id

  • Type: string
  • Required: false
    • +

    --onedrive-access-scopes

    +

    Set scopes to be requested by rclone.

    +

    Choose or manually enter a custom space separated list with all scopes, that rclone should request.

    +

    Properties:

    +
      +
    • Config: access_scopes
      • +
    • Env Var: RCLONE_ONEDRIVE_ACCESS_SCOPES
      • +
    • Type: SpaceSepList
      • +
    • Default: Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access
      • +
    • Examples: +
        +
      • "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access" +
          +
        • Read and write access to all resources
          • +
        • +
      • "Files.Read Files.Read.All Sites.Read.All offline_access" +
          +
        • Read only access to all resources
          • +
        • +
      • "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All offline_access" +
          +
        • Read and write access to all resources, without the ability to browse SharePoint sites.
          • +
        • Same as if disable_site_permission was set to true
          • +
        • +
      • +

    --onedrive-disable-site-permission

    Disable the request for Sites.Read.All permission.

    If set to true, you will no longer be able to search for a SharePoint site when configuring drive ID, because rclone will not request Sites.Read.All permission. Set it to true if your organization didn't assign Sites.Read.All permission to the application, and your organization disallows users to consent app permission request on their own.

    @@ -20591,7 +23342,7 @@

    --onedrive-encoding

  • Type: MultiEncoder
  • Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot
    • -

    Limitations

    +

    Limitations

    If you don't use rclone for 90 days the refresh token will expire. This will result in authorization problems. This is easy to fix by running the rclone config reconnect remote: command to get a new token and refresh token.

    Naming

    Note that OneDrive is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

    @@ -20645,14 +23396,14 @@

    Unexpected file size 
    --ignore-checksum --ignore-size

    Alternatively, if you have write access to the OneDrive files, it may be possible to fix this problem for certain files, by attempting the steps below. Open the web interface for OneDrive and find the affected files (which will be in the error messages/log for rclone). Simply click on each of these files, causing OneDrive to open them on the web. This will cause each file to be converted in place to a format that is functionally equivalent but which will no longer trigger the size discrepancy. Once all problematic files are converted you will no longer need the ignore options above.

    Replacing/deleting existing files on Sharepoint gets "item not found"

    -

    It is a known issue that Sharepoint (not OneDrive or OneDrive for Business) may return "item not found" errors when users try to replace or delete uploaded files; this seems to mainly affect Office files (.docx, .xlsx, etc.). As a workaround, you may use the --backup-dir <BACKUP_DIR> command line argument so rclone moves the files to be replaced/deleted into a given backup directory (instead of directly replacing/deleting them). For example, to instruct rclone to move the files into the directory rclone-backup-dir on backend mysharepoint, you may use:

    +

    It is a known issue that Sharepoint (not OneDrive or OneDrive for Business) may return "item not found" errors when users try to replace or delete uploaded files; this seems to mainly affect Office files (.docx, .xlsx, etc.) and web files (.html, .aspx, etc.). As a workaround, you may use the --backup-dir <BACKUP_DIR> command line argument so rclone moves the files to be replaced/deleted into a given backup directory (instead of directly replacing/deleting them). For example, to instruct rclone to move the files into the directory rclone-backup-dir on backend mysharepoint, you may use:

    --backup-dir mysharepoint:rclone-backup-dir

    access_denied (AADSTS65005)

    Error: access_denied
 Code: AADSTS65005
 Description: Using application 'rclone' is currently not supported for your organization [YOUR_ORGANIZATION] because it is in an unmanaged state. An administrator needs to claim ownership of the company by DNS validation of [YOUR_ORGANIZATION] before the application rclone can be provisioned.

    This means that rclone can't use the OneDrive for Business API with your account. You can't do much about it, maybe write an email to your admins.

    -

    However, there are other ways to interact with your OneDrive account. Have a look at the webdav backend: https://rclone.org/webdav/#sharepoint

    +

    However, there are other ways to interact with your OneDrive account. Have a look at the WebDAV backend: https://rclone.org/webdav/#sharepoint

    invalid_grant (AADSTS50076)

    Error: invalid_grant
 Code: AADSTS50076
@@ -20663,7 +23414,7 @@ 
    OpenDrive
    
 
    Paths are specified as remote:path
    
 
    Paths may be as deep as required, e.g. remote:directory/subdirectory.
    
-
    Configuration
    
+
    Configuration
    
 
    Here is an example of how to make a remote called remote. First run:
    
 
     rclone config
    
 
    This will guide you through an interactive setup process:
    
@@ -20706,7 +23457,7 @@ 
    Configuration
    
 
    rclone copy /home/source remote:backup
    
 
    Modified time and MD5SUMs
    
 
    OpenDrive allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not.
    
-
    Restricted filename characters
    
+
    Restricted filename characters
    @@ -20806,8 +23557,8 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.

    -

    Standard options

    -

    Here are the standard options specific to opendrive (OpenDrive).

    +

    Standard options

    +

    Here are the Standard options specific to opendrive (OpenDrive).

    --opendrive-username

    Username.

    Properties:

    @@ -20827,8 +23578,8 @@

    --opendrive-password

  • Type: string
  • Required: true
    • -

    Advanced options

    -

    Here are the advanced options specific to opendrive (OpenDrive).

    +

    Advanced options

    +

    Here are the Advanced options specific to opendrive (OpenDrive).

    --opendrive-encoding

    The encoding for the backend.

    See the encoding section in the overview for more info.

    @@ -20849,14 +23600,14 @@

    --opendrive-chunk-size

  • Type: SizeSuffix
  • Default: 10Mi
    • -

    Limitations

    +

    Limitations

    Note that OpenDrive is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

    There are quite a few characters that can't be in OpenDrive file names. These can't occur on Windows platforms, but on non-Windows platforms they are common. Rclone will map these names to and from an identical looking unicode equivalent. For example if a file has a ? in it will be mapped to ？ instead.

    rclone about is not supported by the OpenDrive backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

    -

    See List of backends that do not support rclone about See rclone about

    +

    See List of backends that do not support rclone about and rclone about

    QingStor

    Paths are specified as remote:bucket (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:bucket/path/to/dir.

    -

    Configuration

    +

    Configuration

    Here is an example of making an QingStor configuration. First run

    rclone config

    This will guide you through an interactive setup process.

    @@ -20948,11 +23699,11 @@

    Authentication

    -

    Restricted filename characters

    +

    Restricted filename characters

    The control characters 0x00-0x1F and / are replaced as in the default restricted characters set. Note that 0x7F is not replaced.

    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.

    -

    Standard options

    -

    Here are the standard options specific to qingstor (QingCloud Object Storage).

    +

    Standard options

    +

    Here are the Standard options specific to qingstor (QingCloud Object Storage).

    --qingstor-env-auth

    Get QingStor credentials from runtime.

    Only applies if access_key_id and secret_access_key is blank.

    @@ -21032,8 +23783,8 @@

    --qingstor-zone

    -

    Advanced options

    -

    Here are the advanced options specific to qingstor (QingCloud Object Storage).

    +

    Advanced options

    +

    Here are the Advanced options specific to qingstor (QingCloud Object Storage).

    --qingstor-connection-retries

    Number of connection retries.

    Properties:

    @@ -21087,9 +23838,9 @@

    --qingstor-encoding

  • Type: MultiEncoder
  • Default: Slash,Ctl,InvalidUtf8
    • -

    Limitations

    +

    Limitations

    rclone about is not supported by the qingstor backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

    -

    See List of backends that do not support rclone about See rclone about

    +

    See List of backends that do not support rclone about and rclone about

    Sia

    Sia (sia.tech) is a decentralized cloud storage platform based on the blockchain technology. With rclone you can use it like any other remote filesystem or mount Sia folders locally. The technology behind it involves a number of new concepts such as Siacoins and Wallet, Blockchain and Consensus, Renting and Hosting, and so on. If you are new to it, you'd better first familiarize yourself using their excellent support documentation.

    Introduction

    @@ -21097,7 +23848,7 @@

    Introduction

    rclone interacts with Sia network by talking to the Sia daemon via HTTP API which is usually available on port 9980. By default you will run the daemon locally on the same computer so it's safe to leave the API password blank (the API URL will be http://127.0.0.1:9980 making external access impossible).

    However, if you want to access Sia daemon running on another node, for example due to memory constraints or because you want to share single daemon between several rclone and Sia-UI instances, you'll need to make a few more provisions: - Ensure you have Sia daemon installed directly or in a docker container because Sia-UI does not support this mode natively. - Run it on externally accessible port, for example provide --api-addr :9980 and --disable-api-security arguments on the daemon command line. - Enforce API password for the siad daemon via environment variable SIA_API_PASSWORD or text file named apipassword in the daemon directory. - Set rclone backend option api_password taking it from above locations.

    Notes: 1. If your wallet is locked, rclone cannot unlock it automatically. You should either unlock it in advance by using Sia-UI or via command line siac wallet unlock. Alternatively you can make siad unlock your wallet automatically upon startup by running it with environment variable SIA_WALLET_PASSWORD. 2. If siad cannot find the SIA_API_PASSWORD variable or the apipassword file in the SIA_DIR directory, it will generate a random password and store in the text file named apipassword under YOUR_HOME/.sia/ directory on Unix or C:\Users\YOUR_HOME\AppData\Local\Sia\apipassword on Windows. Remember this when you configure password in rclone. 3. The only way to use siad without API password is to run it on localhost with command line argument --authorize-api=false, but this is insecure and strongly discouraged.

    -

    Configuration

    +

    Configuration

    Here is an example of how to make a sia remote called mySia. First, run:

     rclone config

    This will guide you through an interactive setup process:

    @@ -21157,8 +23908,8 @@

    Configuration

  • Upload a local directory to the Sia directory called backup
    • rclone copy /home/source mySia:backup
    -

    Standard options

    -

    Here are the standard options specific to sia (Sia Decentralized Cloud).

    +

    Standard options

    +

    Here are the Standard options specific to sia (Sia Decentralized Cloud).

    --sia-api-url

    Sia daemon API URL, like http://sia.daemon.host:9980.

    Note that siad must run with --disable-api-security to open API port for other hosts (not recommended). Keep default if Sia daemon runs on localhost.

    @@ -21180,8 +23931,8 @@

    --sia-api-password

  • Type: string
  • Required: false
    • -

    Advanced options

    -

    Here are the advanced options specific to sia (Sia Decentralized Cloud).

    +

    Advanced options

    +

    Here are the Advanced options specific to sia (Sia Decentralized Cloud).

    --sia-user-agent

    Siad User Agent

    Sia daemon requires the 'Sia-Agent' user agent by default for security

    @@ -21202,7 +23953,7 @@

    --sia-encoding

  • Type: MultiEncoder
  • Default: Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot
    • -

    Limitations

    +

    Limitations

    Paths are specified as remote:container (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:container/path/to/dir.

    -

    Configuration

    +

    Configuration

    Here is an example of making a swift configuration. First run

    rclone config

    This will guide you through an interactive setup process.

    @@ -21360,7 +24111,7 @@

    --update and --use-server-modtime

    Modified time

    The modified time is stored as metadata on the object as X-Object-Meta-Mtime as floating point since the epoch accurate to 1 ns.

    This is a de facto standard (used in the official python-swiftclient amongst others) for storing the modification time for an object.

    -

    Restricted filename characters

    +

    Restricted filename characters

    @@ -21383,8 +24134,8 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.

    -

    Standard options

    -

    Here are the standard options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)).

    +

    Standard options

    +

    Here are the Standard options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)).

    --swift-env-auth

    Get swift credentials from environment variables in standard OpenStack form.

    Properties:

    @@ -21617,8 +24368,8 @@

    --swift-storage-policy

    -

    Advanced options

    -

    Here are the advanced options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)).

    +

    Advanced options

    +

    Here are the Advanced options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)).

    --swift-leave-parts-on-error

    If true avoid calling abort upload on a failure.

    It should be set to true for resuming uploads across different sessions.

    @@ -21661,7 +24412,7 @@

    --swift-encoding

  • Type: MultiEncoder
  • Default: Slash,InvalidUtf8
    • -

    Limitations

    +

    Limitations

    The Swift API doesn't return a correct MD5SUM for segmented files (Dynamic or Static Large Objects) so rclone won't check or use the MD5SUM for these.

    Troubleshooting

    Rclone gives Failed to create file system for "remote:": Bad Request

    @@ -21681,7 +24432,7 @@

    Retrieving Objects

    pCloud

    Paths are specified as remote:path

    Paths may be as deep as required, e.g. remote:directory/subdirectory.

    -

    Configuration

    +

    Configuration

    The initial setup for pCloud involves getting a token from pCloud which you need to do in your browser. rclone config walks you through it.

    Here is an example of how to make a remote called remote. First run:

     rclone config
    @@ -21733,10 +24484,10 @@

    Configuration

    rclone ls remote:

    To copy a local directory to a pCloud directory called backup

    rclone copy /home/source remote:backup
    -

    Modified time and hashes

    +

    Modified time and hashes

    pCloud allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not. In order to set a Modification time pCloud requires the object be re-uploaded.

    pCloud supports MD5 and SHA1 hashes in the US region, and SHA1 and SHA256 hashes in the EU region, so you can use the --checksum flag.

    -

    Restricted filename characters

    +

    Restricted filename characters

    In addition to the default restricted characters set the following characters are also replaced:

    @@ -21757,14 +24508,16 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.

    Deleting files

    Deleted files will be moved to the trash. Your subscription level will determine how long items stay in the trash. rclone cleanup can be used to empty the trash.

    +

    Emptying the trash

    +

    Due to an API limitation, the rclone cleanup command will only work if you set your username and password in the advanced options for this backend. Since we generally want to avoid storing user passwords in the rclone config file, we advise you to only set this up if you need the rclone cleanup command to work.

    Root folder ID

    You can set the root_folder_id for rclone. This is the directory (identified by its Folder ID) that rclone considers to be the root of your pCloud drive.

    Normally you will leave this blank and rclone will determine the correct root to use itself.

    However you can set this to restrict rclone to a specific folder hierarchy.

    In order to do this you will have to find the Folder ID of the directory you wish rclone to display. This will be the folder field of the URL when you open the relevant folder in the pCloud web interface.

    So if the folder you want rclone to use has a URL which looks like https://my.pcloud.com/#page=filemanager&folder=5xxxxxxxx8&tpl=foldergrid in the browser, then you use 5xxxxxxxx8 as the root_folder_id in the config.

    -

    Standard options

    -

    Here are the standard options specific to pcloud (Pcloud).

    +

    Standard options

    +

    Here are the Standard options specific to pcloud (Pcloud).

    --pcloud-client-id

    OAuth Client Id.

    Leave blank normally.

    @@ -21785,8 +24538,8 @@

    --pcloud-client-secret

  • Type: string
  • Required: false
    • -

    Advanced options

    -

    Here are the advanced options specific to pcloud (Pcloud).

    +

    Advanced options

    +

    Here are the Advanced options specific to pcloud (Pcloud).

    --pcloud-token

    OAuth Access Token as a JSON blob.

    Properties:

    @@ -21856,10 +24609,30 @@

    --pcloud-hostname

    +

    --pcloud-username

    +

    Your pcloud username.

    +

    This is only required when you want to use the cleanup command. Due to a bug in the pcloud API the required API does not support OAuth authentication so we have to rely on user password authentication for it.

    +

    Properties:

    +
      +
    • Config: username
      • +
    • Env Var: RCLONE_PCLOUD_USERNAME
      • +
    • Type: string
      • +
    • Required: false
      • +
    +

    --pcloud-password

    +

    Your pcloud password.

    +

    NB Input to this must be obscured - see rclone obscure.

    +

    Properties:

    +
      +
    • Config: password
      • +
    • Env Var: RCLONE_PCLOUD_PASSWORD
      • +
    • Type: string
      • +
    • Required: false
      • +

    premiumize.me

    Paths are specified as remote:path

    Paths may be as deep as required, e.g. remote:directory/subdirectory.

    -

    Configuration

    +

    Configuration

    The initial setup for premiumize.me involves getting a token from premiumize.me which you need to do in your browser. rclone config walks you through it.

    Here is an example of how to make a remote called remote. First run:

     rclone config
    @@ -21909,9 +24682,9 @@

    Configuration

    rclone ls remote:

    To copy a local directory to an premiumize.me directory called backup

    rclone copy /home/source remote:backup
    -

    Modified time and hashes

    +

    Modified time and hashes

    premiumize.me does not support modification times or hashes, therefore syncing will default to --size-only checking. Note that using --update will work.

    -

    Restricted filename characters

    +

    Restricted filename characters

    In addition to the default restricted characters set the following characters are also replaced:

    @@ -21935,8 +24708,8 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.

    -

    Standard options

    -

    Here are the standard options specific to premiumizeme (premiumize.me).

    +

    Standard options

    +

    Here are the Standard options specific to premiumizeme (premiumize.me).

    --premiumizeme-api-key

    API Key.

    This is not normally used - use oauth instead.

    @@ -21947,8 +24720,8 @@

    --premiumizeme-api-key

  • Type: string
  • Required: false
    • -

    Advanced options

    -

    Here are the advanced options specific to premiumizeme (premiumize.me).

    +

    Advanced options

    +

    Here are the Advanced options specific to premiumizeme (premiumize.me).

    --premiumizeme-encoding

    The encoding for the backend.

    See the encoding section in the overview for more info.

    @@ -21959,14 +24732,14 @@

    --premiumizeme-encoding

  • Type: MultiEncoder
  • Default: Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot
    • -

    Limitations

    +

    Limitations

    Note that premiumize.me is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

    premiumize.me file names can't have the \ or " characters in. rclone maps these to and from an identical looking unicode equivalents ＼ and ＂

    premiumize.me only supports filenames up to 255 characters in length.

    put.io

    Paths are specified as remote:path

    put.io paths may be as deep as required, e.g. remote:directory/subdirectory.

    -

    Configuration

    +

    Configuration

    The initial setup for put.io involves getting a token from put.io which you need to do in your browser. rclone config walks you through it.

    Here is an example of how to make a remote called remote. First run:

     rclone config
    @@ -22029,7 +24802,7 @@

    Configuration

    rclone ls remote:

    To copy a local directory to a put.io directory called backup

    rclone copy /home/source remote:backup
    -

    Restricted filename characters

    +

    Restricted filename characters

    In addition to the default restricted characters set the following characters are also replaced:

    @@ -22048,8 +24821,8 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.

    -

    Advanced options

    -

    Here are the advanced options specific to putio (Put.io).

    +

    Advanced options

    +

    Here are the Advanced options specific to putio (Put.io).

    --putio-encoding

    The encoding for the backend.

    See the encoding section in the overview for more info.

    @@ -22060,9 +24833,12 @@

    --putio-encoding

  • Type: MultiEncoder
  • Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot
    • +

    Limitations

    +

    put.io has rate limiting. When you hit a limit, rclone automatically retries after waiting the amount of time requested by the server.

    +

    If you want to avoid ever hitting these limits, you may use the --tpslimit flag with a low number. Note that the imposed limits may be different for different operations, and may change over time.

    Seafile

    This is a backend for the Seafile storage service: - It works with both the free community edition or the professional edition. - Seafile versions 6.x and 7.x are all supported. - Encrypted libraries are also supported. - It supports 2FA enabled users

    -

    Configuration

    +

    Configuration

    There are two distinct modes you can setup your remote: - you point your remote to the root of the server, meaning you don't specify a library during the configuration: Paths are specified as remote:library. You may put subdirectories in too, e.g. remote:library/path/to/dir. - you point your remote to a specific library during the configuration: Paths are specified as remote:path/to/dir. This is the recommended mode when using encrypted libraries. (This mode is possibly slightly faster than the root mode)

    Configuration in root mode

    Here is an example of making a seafile configuration for a user with no two-factor authentication. First run

    @@ -22221,7 +24997,7 @@

    Configuration in library mode

    rclone sync -i /home/local/directory seafile:

    --fast-list

    Seafile version 7+ supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details. Please note this is not supported on seafile server version 6.x

    -

    Restricted filename characters

    +

    Restricted filename characters

    In addition to the default restricted characters set the following characters are also replaced:

    @@ -22262,8 +25038,8 @@

    Compatibility

    It has been actively tested using the seafile docker image of these versions: - 6.3.4 community edition - 7.0.5 community edition - 7.1.3 community edition

    Versions below 6.0 are not supported. Versions between 6.0 and 6.3 haven't been tested and might not work properly.

    -

    Standard options

    -

    Here are the standard options specific to seafile (seafile).

    +

    Standard options

    +

    Here are the Standard options specific to seafile (seafile).

    --seafile-url

    URL of seafile host to connect to.

    Properties:

    @@ -22338,8 +25114,8 @@

    --seafile-auth-token

  • Type: string
  • Required: false
    • -

    Advanced options

    -

    Here are the advanced options specific to seafile (seafile).

    +

    Advanced options

    +

    Here are the Advanced options specific to seafile (seafile).

    --seafile-create-library

    Should rclone create a library if it doesn't exist.

    Properties:

    @@ -22363,13 +25139,14 @@

    SFTP

    SFTP is the Secure (or SSH) File Transfer Protocol.

    The SFTP backend can be used with a number of different providers:

      -
    • C14
      • +
    • Hetzner Storage Box
    • rsync.net

    SFTP runs over SSH v2 and is installed as standard with most modern SSH installations.

    Paths are specified as remote:path. If the path does not begin with a / it is relative to the home directory of the user. An empty path remote: refers to the user's home directory. For example, rclone lsd remote: would list the home directory of the user cofigured in the rclone remote config (i.e /home/sftpuser). However, rclone lsd remote:/ would list the root directory for remote machine (i.e. /)

    -

    Note that some SFTP servers will need the leading / - Synology is a good example of this. rsync.net, on the other hand, requires users to OMIT the leading /.

    -

    Configuration

    +

    Note that some SFTP servers will need the leading / - Synology is a good example of this. rsync.net and Hetzner, on the other hand, requires users to OMIT the leading /.

    +

    Note that by default rclone will try to execute shell commands on the server, see shell access considerations.

    +

    Configuration

    Here is an example of making an SFTP configuration. First run

    rclone config

    This will guide you through an interactive setup process.

    @@ -22382,7 +25159,7 @@

    Configuration

    Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / SSH/SFTP Connection +XX / SSH/SFTP \ "sftp" [snip] Storage> sftp @@ -22490,12 +25267,30 @@

    ssh-agent on macOS

    And then at the end of the session

    eval `ssh-agent -k`

    These commands can be used in scripts of course.

    +

    Shell access

    +

    Some functionality of the SFTP backend relies on remote shell access, and the possibility to execute commands. This includes checksum, and in some cases also about. The shell commands that must be executed may be different on different type of shells, and also quoting/escaping of file path arguments containing special characters may be different. Rclone therefore needs to know what type of shell it is, and if shell access is available at all.

    +

    Most servers run on some version of Unix, and then a basic Unix shell can be assumed, without further distinction. Windows 10, Server 2019, and later can also run a SSH server, which is a port of OpenSSH (see official installation guide). On a Windows server the shell handling is different: Although it can also be set up to use a Unix type shell, e.g. Cygwin bash, the default is to use Windows Command Prompt (cmd.exe), and PowerShell is a recommended alternative. All of these have bahave differently, which rclone must handle.

    +

    Rclone tries to auto-detect what type of shell is used on the server, first time you access the SFTP remote. If a remote shell session is successfully created, it will look for indications that it is CMD or PowerShell, with fall-back to Unix if not something else is detected. If unable to even create a remote shell session, then shell command execution will be disabled entirely. The result is stored in the SFTP remote configuration, in option shell_type, so that the auto-detection only have to be performed once. If you manually set a value for this option before first run, the auto-detection will be skipped, and if you set a different value later this will override any existing. Value none can be set to avoid any attempts at executing shell commands, e.g. if this is not allowed on the server.

    +

    When the server is rclone serve sftp, the rclone SFTP remote will detect this as a Unix type shell - even if it is running on Windows. This server does not actually have a shell, but it accepts input commands matching the specific ones that the SFTP backend relies on for Unix shells, e.g. md5sum and df. Also it handles the string escape rules used for Unix shell. Treating it as a Unix type shell from a SFTP remote will therefore always be correct, and support all features.

    +

    Shell access considerations

    +

    The shell type auto-detection logic, described above, means that by default rclone will try to run a shell command the first time a new sftp remote is accessed. If you configure a sftp remote without a config file, e.g. an on the fly remote, rclone will have nowhere to store the result, and it will re-run the command on every access. To avoid this you should explicitely set the shell_type option to the correct value, or to none if you want to prevent rclone from executing any remote shell commands.

    +

    It is also important to note that, since the shell type decides how quoting and escaping of file paths used as command-line arguments are performed, configuring the wrong shell type may leave you exposed to command injection exploits. Make sure to confirm the auto-detected shell type, or explicitely set the shell type you know is correct, or disable shell access until you know.

    +

    Checksum

    +

    SFTP does not natively support checksums (file hash), but rclone is able to use checksumming if the same login has shell access, and can execute remote commands. If there is a command that can calculate compatible checksums on the remote system, Rclone can then be configured to execute this whenever a checksum is needed, and read back the results. Currently MD5 and SHA-1 are supported.

    +

    Normally this requires an external utility being available on the server. By default rclone will try commands md5sum, md5 and rclone md5sum for MD5 checksums, and the first one found usable will be picked. Same with sha1sum, sha1 and rclone sha1sum commands for SHA-1 checksums. These utilities normally need to be in the remote's PATH to be found.

    +

    In some cases the shell itself is capable of calculating checksums. PowerShell is an example of such a shell. If rclone detects that the remote shell is PowerShell, which means it most probably is a Windows OpenSSH server, rclone will use a predefined script block to produce the checksums when no external checksum commands are found (see shell access). This assumes PowerShell version 4.0 or newer.

    +

    The options md5sum_command and sha1_command can be used to customize the command to be executed for calculation of checksums. You can for example set a specific path to where md5sum and sha1sum executables are located, or use them to specify some other tools that print checksums in compatible format. The value can include command-line arguments, or even shell script blocks as with PowerShell. Rclone has subcommands md5sum and sha1sum that use compatible format, which means if you have an rclone executable on the server it can be used. As mentioned above, they will be automatically picked up if found in PATH, but if not you can set something like /path/to/rclone md5sum as the value of option md5sum_command to make sure a specific executable is used.

    +

    Remote checksumming is recommended and enabled by default. First time rclone is using a SFTP remote, if options md5sum_command or sha1_command are not set, it will check if any of the default commands for each of them, as described above, can be used. The result will be saved in the remote configuration, so next time it will use the same. Value none will be set if none of the default commands could be used for a specific algorithm, and this algorithm will not be supported by the remote.

    +

    Disabling the checksumming may be required if you are connecting to SFTP servers which are not under your control, and to which the execution of remote shell commands is prohibited. Set the configuration option disable_hashcheck to true to disable checksumming entirely, or set shell_type to none to disable all functionality based on remote shell command execution.

    Modified time

    Modified times are stored on the server to 1 second precision.

    Modified times are used in syncing and are fully supported.

    Some SFTP servers disable setting/modifying the file modification time after upload (for example, certain configurations of ProFTPd with mod_sftp). If you are using one of these servers, you can set the option set_modtime = false in your RClone backend configuration to disable this behaviour.

    -

    Standard options

    -

    Here are the standard options specific to sftp (SSH/SFTP Connection).

    +

    About command

    +

    The about command returns the total space, free space, and used space on the remote for the disk of the specified path on the remote or, if not set, the disk of the root on the remote.

    +

    SFTP usually supports the about command, but it depends on the server. If the server implements the vendor-specific VFS statistics extension, which is normally the case with OpenSSH instances, it will be used. If not, but the same login has access to a Unix shell, where the df command is available (e.g. in the remote's PATH), then this will be used instead. If the server shell is PowerShell, probably with a Windows OpenSSH server, rclone will use a built-in shell command (see shell access). If none of the above is applicable, about will fail.

    +

    Standard options

    +

    Here are the Standard options specific to sftp (SSH/SFTP).

    --sftp-host

    SSH host to connect to.

    E.g. "example.com".

    @@ -22627,8 +25422,8 @@

    --sftp-disable-hashcheck

  • Type: bool
  • Default: false
    • -

    Advanced options

    -

    Here are the advanced options specific to sftp (SSH/SFTP Connection).

    +

    Advanced options

    +

    Here are the Advanced options specific to sftp (SSH/SFTP).

    --sftp-known-hosts-file

    Optional path to known_hosts file.

    Set this value to enable server host key validation.

    @@ -22658,11 +25453,11 @@

    --sftp-ask-password

  • Default: false

    • --sftp-path-override

    -

    Override path used by SSH connection.

    +

    Override path used by SSH shell commands.

    This allows checksum calculation when SFTP and SSH paths are different. This issue affects among others Synology NAS boxes.

    -

    Shared folders can be found in directories representing volumes

    +

    E.g. if shared folders can be found in directories representing volumes:

    rclone sync /home/local/directory remote:/directory --sftp-path-override /volume2/directory
    -

    Home directory can be found in a shared folder called "home"

    +

    E.g. if home directory can be found in a shared folder called "home":

    rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory

    Properties:

      @@ -22680,6 +25475,35 @@

      --sftp-set-modtime

    • Type: bool
    • Default: true
    +

    --sftp-shell-type

    +

    The type of SSH shell on remote server, if any.

    +

    Leave blank for autodetect.

    +

    Properties:

    +
      +
    • Config: shell_type
      • +
    • Env Var: RCLONE_SFTP_SHELL_TYPE
      • +
    • Type: string
      • +
    • Required: false
      • +
    • Examples: +
        +
      • "none" +
          +
        • No shell access
          • +
        • +
      • "unix" +
          +
        • Unix shell
          • +
        • +
      • "powershell" +
          +
        • PowerShell
          • +
        • +
      • "cmd" +
          +
        • Windows Command Prompt
          • +
        • +
      • +

    --sftp-md5sum-command

    The command used to read md5 hashes.

    Leave blank for autodetect.

    @@ -22775,21 +25599,58 @@

    --sftp-idle-timeout

  • Type: Duration
  • Default: 1m0s
    • -

    Limitations

    -

    SFTP supports checksums if the same login has shell access and md5sum or sha1sum as well as echo are in the remote's PATH. This remote checksumming (file hashing) is recommended and enabled by default. Disabling the checksumming may be required if you are connecting to SFTP servers which are not under your control, and to which the execution of remote commands is prohibited. Set the configuration option disable_hashcheck to true to disable checksumming.

    -

    SFTP also supports about if the same login has shell access and df are in the remote's PATH. about will return the total space, free space, and used space on the remote for the disk of the specified path on the remote or, if not set, the disk of the root on the remote. about will fail if it does not have shell access or if df is not in the remote's PATH.

    -

    Note that some SFTP servers (e.g. Synology) the paths are different for SSH and SFTP so the hashes can't be calculated properly. For them using disable_hashcheck is a good idea.

    +

    --sftp-chunk-size

    +

    Upload and download chunk size.

    +

    This controls the maximum packet size used in the SFTP protocol. The RFC limits this to 32768 bytes (32k), however a lot of servers support larger sizes and setting it larger will increase transfer speed dramatically on high latency links.

    +

    Only use a setting higher than 32k if you always connect to the same server or after sufficiently broad testing.

    +

    For example using the value of 252k with OpenSSH works well with its maximum packet size of 256k.

    +

    If you get the error "failed to send packet header: EOF" when copying a large file, try lowering this number.

    +

    Properties:

    +
      +
    • Config: chunk_size
      • +
    • Env Var: RCLONE_SFTP_CHUNK_SIZE
      • +
    • Type: SizeSuffix
      • +
    • Default: 32Ki
      • +
    +

    --sftp-concurrency

    +

    The maximum number of outstanding requests for one file

    +

    This controls the maximum number of outstanding requests for one file. Increasing it will increase throughput on high latency links at the cost of using more memory.

    +

    Properties:

    +
      +
    • Config: concurrency
      • +
    • Env Var: RCLONE_SFTP_CONCURRENCY
      • +
    • Type: int
      • +
    • Default: 64
      • +
    +

    --sftp-set-env

    +

    Environment variables to pass to sftp and commands

    +

    Set environment variables in the form:

    +
    VAR=value
    +

    to be passed to the sftp client and to any commands run (eg md5sum).

    +

    Pass multiple variables space separated, eg

    +
    VAR1=value VAR2=value
    +

    and pass variables with spaces in in quotes, eg

    +
    "VAR3=value with space" "VAR4=value with space" VAR5=nospacehere
    +

    Properties:

    +
      +
    • Config: set_env
      • +
    • Env Var: RCLONE_SFTP_SET_ENV
      • +
    • Type: SpaceSepList
      • +
    • Default:
      • +
    +

    Limitations

    +

    On some SFTP servers (e.g. Synology) the paths are different for SSH and SFTP so the hashes can't be calculated properly. For them using disable_hashcheck is a good idea.

    The only ssh agent supported under Windows is Putty's pageant.

    The Go SSH library disables the use of the aes128-cbc cipher by default, due to security concerns. This can be re-enabled on a per-connection basis by setting the use_insecure_cipher setting in the configuration file to true. Further details on the insecurity of this cipher can be found in this paper.

    SFTP isn't supported under plan9 until this issue is fixed.

    -

    Note that since SFTP isn't HTTP based the following flags don't work with it: --dump-headers, --dump-bodies, --dump-auth

    +

    Note that since SFTP isn't HTTP based the following flags don't work with it: --dump-headers, --dump-bodies, --dump-auth.

    Note that --timeout and --contimeout are both supported.

    -

    C14

    -

    C14 is supported through the SFTP backend.

    -

    See C14's documentation

    rsync.net

    rsync.net is supported through the SFTP backend.

    See rsync.net's documentation of rclone examples.

    +

    Hetzner Storage Box

    +

    Hetzner Storage Boxes are supported through the SFTP backend on port 23.

    +

    See Hetzner's documentation for details

    Storj

    Storj is an encrypted, secure, and cost-effective object storage service that enables you to store, back up, and archive large amounts of data in a decentralized manner.

    Backend options

    @@ -22849,7 +25710,7 @@

    Backend options

  • S3 backend: secret encryption key is shared with the gateway
    • -

    Configuration

    +

    Configuration

    To make a new Storj configuration you need one of the following: * Access Grant that someone else shared with you. * API Key of a Storj project you are a member of.

    Here is an example of how to make a remote called remote. First run:

     rclone config
    @@ -22946,8 +25807,8 @@

    Setup with API key and passphrase

    -

    Standard options

    -

    Here are the standard options specific to storj (Storj Decentralized Cloud Storage).

    +

    Standard options

    +

    Here are the Standard options specific to storj (Storj Decentralized Cloud Storage).

    --storj-provider

    Choose an authentication method.

    Properties:

    @@ -23079,15 +25940,15 @@

    Sync two Locations

    rclone sync -i --progress remote-us:bucket/path/to/dir/ remote-europe:bucket/path/to/dir/

    Or even between another cloud storage and Storj.

    rclone sync -i --progress s3:bucket/path/to/dir/ storj:bucket/path/to/dir/
    -

    Limitations

    +

    Limitations

    rclone about is not supported by the rclone Storj backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

    -

    See List of backends that do not support rclone about See rclone about

    +

    See List of backends that do not support rclone about and rclone about

    Known issues

    If you get errors like too many open files this usually happens when the default ulimit for system max open files is exceeded. Native Storj protocol opens a large number of TCP connections (each of which is counted as an open file). For a single upload stream you can expect 110 TCP connections to be opened. For a single download stream you can expect 35. This batch of connections will be opened for every 64 MiB segment and you should also expect TCP connections to be reused. If you do many transfers you eventually open a connection to most storage nodes (thousands of nodes).

    To fix these, please raise your system limits. You can do this issuing a ulimit -n 65536 just before you run rclone. To change the limits more permanently you can add this to your shell startup script, e.g. $HOME/.bashrc, or change the system-wide configuration, usually /etc/sysctl.conf and/or /etc/security/limits.conf, but please refer to your operating system manual.

    SugarSync

    SugarSync is a cloud service that enables active synchronization of files across computers and other devices for file backup, access, syncing, and sharing.

    -

    Configuration

    +

    Configuration

    The initial setup for SugarSync involves getting a token from SugarSync which you can do with rclone. rclone config walks you through it.

    Here is an example of how to make a remote called remote. First run:

     rclone config
    @@ -23152,16 +26013,16 @@

    Configuration

    Paths are specified as remote:path

    Paths may be as deep as required, e.g. remote:directory/subdirectory.

    NB you can't create files in the top level folder you have to create a folder, which rclone will create as a "Sync Folder" with SugarSync.

    -

    Modified time and hashes

    +

    Modified time and hashes

    SugarSync does not support modification times or hashes, therefore syncing will default to --size-only checking. Note that using --update will work as rclone can read the time files were uploaded.

    -

    Restricted filename characters

    +

    Restricted filename characters

    SugarSync replaces the default restricted characters set except for DEL.

    Invalid UTF-8 bytes will also be replaced, as they can't be used in XML strings.

    Deleting files

    Deleted files will be moved to the "Deleted items" folder by default.

    However you can supply the flag --sugarsync-hard-delete or set the config parameter hard_delete = true if you would like files to be deleted straight away.

    -

    Standard options

    -

    Here are the standard options specific to sugarsync (Sugarsync).

    +

    Standard options

    +

    Here are the Standard options specific to sugarsync (Sugarsync).

    --sugarsync-app-id

    Sugarsync App ID.

    Leave blank to use rclone's.

    @@ -23201,8 +26062,8 @@

    --sugarsync-hard-delete

  • Type: bool
  • Default: false
    • -

    Advanced options

    -

    Here are the advanced options specific to sugarsync (Sugarsync).

    +

    Advanced options

    +

    Here are the Advanced options specific to sugarsync (Sugarsync).

    --sugarsync-refresh-token

    Sugarsync refresh token.

    Leave blank normally, will be auto configured by rclone.

    @@ -23273,16 +26134,16 @@

    --sugarsync-encoding

  • Type: MultiEncoder
  • Default: Slash,Ctl,InvalidUtf8,Dot
    • -

    Limitations

    +

    Limitations

    rclone about is not supported by the SugarSync backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

    -

    See List of backends that do not support rclone about See rclone about

    +

    See List of backends that do not support rclone about and rclone about

    Tardigrade

    The Tardigrade backend has been renamed to be the Storj backend. Old configuration files will continue to work.

    Uptobox

    This is a Backend for Uptobox file storage service. Uptobox is closer to a one-click hoster than a traditional cloud storage provider and therefore not suitable for long term storage.

    Paths are specified as remote:path

    Paths may be as deep as required, e.g. remote:directory/subdirectory.

    -

    Configuration

    +

    Configuration

    To configure an Uptobox backend you'll need your personal api token. You'll find it in your account settings

    Here is an example of how to make a remote called remote with the default setup. First run:

    rclone config
    @@ -23336,9 +26197,9 @@

    Configuration

    rclone ls remote:

    To copy a local directory to an Uptobox directory called backup

    rclone copy /home/source remote:backup
    -

    Modified time and hashes

    +

    Modified time and hashes

    Uptobox supports neither modified times nor checksums.

    -

    Restricted filename characters

    +

    Restricted filename characters

    In addition to the default restricted characters set the following characters are also replaced:

    @@ -23362,8 +26223,8 @@

    Restricted filename characters

    Invalid UTF-8 bytes will also be replaced, as they can't be used in XML strings.

    -

    Standard options

    -

    Here are the standard options specific to uptobox (Uptobox).

    +

    Standard options

    +

    Here are the Standard options specific to uptobox (Uptobox).

    --uptobox-access-token

    Your access token.

    Get it from https://uptobox.com/my_account.

    @@ -23374,8 +26235,8 @@

    --uptobox-access-token

  • Type: string
  • Required: false
    • -

    Advanced options

    -

    Here are the advanced options specific to uptobox (Uptobox).

    +

    Advanced options

    +

    Here are the Advanced options specific to uptobox (Uptobox).

    --uptobox-encoding

    The encoding for the backend.

    See the encoding section in the overview for more info.

    @@ -23386,7 +26247,7 @@

    --uptobox-encoding

  • Type: MultiEncoder
  • Default: Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot
    • -

    Limitations

    +

    Limitations

    Uptobox will delete inactive files that have not been accessed in 60 days.

    rclone about is not supported by this backend an overview of used space can however been seen in the uptobox web interface.

    Union

    @@ -23396,7 +26257,7 @@

    Union

    Attribute :ro and :nc can be attach to the end of path to tag the remote as read only or no create, e.g. remote:directory/subdirectory:ro or remote:directory/subdirectory:nc.

    Subfolders can be used in upstream remotes. Assume a union remote named backup with the remotes mydrive:private/backup. Invoking rclone mkdir backup:desktop is exactly the same as invoking rclone mkdir mydrive:private/backup/desktop.

    There will be no special handling of paths containing .. segments. Invoking rclone mkdir backup:../desktop is exactly the same as invoking rclone mkdir mydrive:private/backup/../desktop.

    -

    Configuration

    +

    Configuration

    Here is an example of how to make a union called remote for local folders. First run:

     rclone config

    This will guide you through an interactive setup process:

    @@ -23617,8 +26478,8 @@

    Policy descriptions

    -

    Standard options

    -

    Here are the standard options specific to union (Union merges the contents of several upstream fs).

    +

    Standard options

    +

    Here are the Standard options specific to union (Union merges the contents of several upstream fs).

    --union-upstreams

    List of space separated upstreams.

    Can be 'upstreama:test/dir upstreamb:', '"upstreama:test/space:ro dir" upstreamb:', etc.

    @@ -23666,10 +26527,25 @@

    --union-cache-time

  • Type: int
  • Default: 120
    • +

    Advanced options

    +

    Here are the Advanced options specific to union (Union merges the contents of several upstream fs).

    +

    --union-min-free-space

    +

    Minimum viable free space for lfs/eplfs policies.

    +

    If a remote has less than this much free space then it won't be considered for use in lfs or eplfs policies.

    +

    Properties:

    +
      +
    • Config: min_free_space
      • +
    • Env Var: RCLONE_UNION_MIN_FREE_SPACE
      • +
    • Type: SizeSuffix
      • +
    • Default: 1Gi
      • +
    +

    Metadata

    +

    Any metadata supported by the underlying remote is read and written.

    +

    See the metadata docs for more info.

    WebDAV

    Paths are specified as remote:path

    Paths may be as deep as required, e.g. remote:directory/subdirectory.

    -

    Configuration

    +

    Configuration

    To configure the WebDAV remote you will need to have a URL for it, and a username and password. If you know what kind of system you are connecting to then rclone can enable extra features.

    Here is an example of how to make a remote called remote. First run:

     rclone config
    @@ -23683,7 +26559,7 @@

    Configuration

    Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Webdav +XX / WebDAV \ "webdav" [snip] Storage> webdav @@ -23692,7 +26568,7 @@

    Configuration

    1 / Connect to example.com \ "https://example.com" url> https://example.com/remote.php/webdav/ -Name of the Webdav site/service/software you are using +Name of the WebDAV site/service/software you are using Choose a number from below, or type in your own value 1 / Nextcloud \ "nextcloud" @@ -23739,11 +26615,11 @@

    Configuration

    rclone ls remote:

    To copy a local directory to an WebDAV directory called backup

    rclone copy /home/source remote:backup
    -

    Modified time and hashes

    +

    Modified time and hashes

    Plain WebDAV does not support modified times. However when used with Owncloud or Nextcloud rclone will support modified times.

    Likewise plain WebDAV does not support hashes, however when used with Owncloud or Nextcloud rclone will support SHA1 and MD5 hashes. Depending on the exact version of Owncloud or Nextcloud hashes may appear on all objects, or only on objects which had a hash uploaded with them.

    -

    Standard options

    -

    Here are the standard options specific to webdav (Webdav).

    +

    Standard options

    +

    Here are the Standard options specific to webdav (WebDAV).

    --webdav-url

    URL of http host to connect to.

    E.g. https://example.com.

    @@ -23755,7 +26631,7 @@

    --webdav-url

  • Required: true

    • --webdav-vendor

    -

    Name of the Webdav site/service/software you are using.

    +

    Name of the WebDAV site/service/software you are using.

    Properties:

    • Config: vendor
      • @@ -23815,8 +26691,8 @@

      --webdav-bearer-token

    • Type: string
    • Required: false
    -

    Advanced options

    -

    Here are the advanced options specific to webdav (Webdav).

    +

    Advanced options

    +

    Here are the Advanced options specific to webdav (WebDAV).

    --webdav-bearer-token-command

    Command to run to get a bearer token.

    Properties:

    @@ -23922,7 +26798,7 @@

    OpenID-Connect

    bearer_token_command = oidc-token XDC

    Yandex Disk

    Yandex Disk is a cloud storage solution created by Yandex.

    -

    Configuration

    +

    Configuration

    Here is an example of making a yandex configuration. First run

    rclone config

    This will guide you through an interactive setup process:

    @@ -23983,11 +26859,11 @@

    Emptying Trash

    If you wish to empty your trash you can use the rclone cleanup remote: command which will permanently delete all your trashed files. This command does not take any path arguments.

    Quota information

    To view your current quota you can use the rclone about remote: command which will display your usage limit (quota) and the current usage.

    -

    Restricted filename characters

    +

    Restricted filename characters

    The default restricted characters set are replaced.

    Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.

    -

    Standard options

    -

    Here are the standard options specific to yandex (Yandex Disk).

    +

    Standard options

    +

    Here are the Standard options specific to yandex (Yandex Disk).

    --yandex-client-id

    OAuth Client Id.

    Leave blank normally.

    @@ -24008,8 +26884,8 @@

    --yandex-client-secret

  • Type: string
  • Required: false
    • -

    Advanced options

    -

    Here are the advanced options specific to yandex (Yandex Disk).

    +

    Advanced options

    +

    Here are the Advanced options specific to yandex (Yandex Disk).

    --yandex-token

    OAuth Access Token as a JSON blob.

    Properties:

    @@ -24058,13 +26934,13 @@

    --yandex-encoding

  • Type: MultiEncoder
  • Default: Slash,Del,Ctl,InvalidUtf8,Dot
    • -

    Limitations

    +

    Limitations

    When uploading very large files (bigger than about 5 GiB) you will need to increase the --timeout parameter. This is because Yandex pauses (perhaps to calculate the MD5SUM for the entire file) before returning confirmation that the file has been uploaded. The default handling of timeouts in rclone is to assume a 5 minute pause is an error and close the connection - you'll see net/http: timeout awaiting response headers errors in the logs if this is happening. Setting the timeout to twice the max size of file in GiB should be enough, so if you want to upload a 30 GiB file set a timeout of 2 * 30 = 60m, that is --timeout 60m.

    Having a Yandex Mail account is mandatory to use the Yandex.Disk subscription. Token generation will work without a mail account, but Rclone won't be able to complete any actions.

    [403 - DiskUnsupportedUserAccountTypeError] User account type is not supported.

    Zoho Workdrive

    Zoho WorkDrive is a cloud storage solution created by Zoho.

    -

    Configuration

    +

    Configuration

    Here is an example of making a zoho configuration. First run

    rclone config

    This will guide you through an interactive setup process:

    @@ -24142,10 +27018,10 @@

    Checksums

    No checksums are supported.

    Usage information

    To view your current quota you can use the rclone about remote: command which will display your current usage.

    -

    Restricted filename characters

    +

    Restricted filename characters

    Only control characters and invalid UTF-8 are replaced. In addition most Unicode full-width characters are not supported at all and will be removed from filenames during upload.

    -

    Standard options

    -

    Here are the standard options specific to zoho (Zoho).

    +

    Standard options

    +

    Here are the Standard options specific to zoho (Zoho).

    --zoho-client-id

    OAuth Client Id.

    Leave blank normally.

    @@ -24189,14 +27065,22 @@

    --zoho-region

    • India
    +
  • "jp" +
      +
    • Japan
      • +
    • +
  • "com.cn" +
      +
    • China
      • +
  • "com.au"
    • Australia
    • -

    Advanced options

    -

    Here are the advanced options specific to zoho (Zoho).

    +

    Advanced options

    +

    Here are the Advanced options specific to zoho (Zoho).

    --zoho-token

    OAuth Access Token as a JSON blob.

    Properties:

    @@ -24236,11 +27120,19 @@

    --zoho-encoding

  • Type: MultiEncoder
  • Default: Del,Ctl,InvalidUtf8
    • +

    Setting up your own client_id

    +

    For Zoho we advise you to set up your own client_id. To do so you have to complete the following steps.

    +
      +

    1. Log in to the Zoho API Console

      2. +

    2. Create a new client of type "Server-based Application". The name and website don't matter, but you must add the redirect URL http://localhost:53682/.

      3. +

    3. Once the client is created, you can go to the settings tab and enable it in other regions.

      4. +
    +

    The client id and client secret can now be used with rclone.

    Local Filesystem

    Local paths are specified as normal filesystem paths, e.g. /path/to/wherever, so

    rclone sync -i /home/source /tmp/destination

    Will sync /home/source to /tmp/destination.

    -

    Configuration

    +

    Configuration

    For consistencies sake one can also configure a remote of type local in the config file, and access the local filesystem using rclone remote paths, e.g. remote:path/to/wherever, but it is probably easier not to.

    Modified time

    Rclone reads and writes the modified time using an accuracy determined by the OS. Typically this is 1ns on Linux, 10 ns on Windows and 1 Second on OS X.

    @@ -24611,16 +27503,16 @@

    Restricting filesystems 0 file2

    NB Rclone (like most unix tools such as du, rsync and tar) treats a bind mount to the same device as being on the same filesystem.

    NB This flag is only available on Unix based systems. On systems where it isn't supported (e.g. Windows) it will be ignored.

    -

    Advanced options

    -

    Here are the advanced options specific to local (Local Disk).

    +

    Advanced options

    +

    Here are the Advanced options specific to local (Local Disk).

    --local-nounc

    Disable UNC (long path names) conversion on Windows.

    Properties:

    • Config: nounc
    • Env Var: RCLONE_LOCAL_NOUNC
      • -
    • Type: string
      • -
    • Required: false
      • +
    • Type: bool
      • +
    • Default: false
    • Examples:
      • "true" @@ -24774,12 +27666,86 @@

        --local-encoding

      • Type: MultiEncoder
      • Default: Slash,Dot
      +

      Metadata

      +

      Depending on which OS is in use the local backend may return only some of the system metadata. Setting system metadata is supported on all OSes but setting user metadata is only supported on linux, freebsd, netbsd, macOS and Solaris. It is not supported on Windows yet (see pkg/attrs#47).

      +

      User metadata is stored as extended attributes (which may not be supported by all file systems) under the "user.*" prefix.

      +

      Here are the possible system metadata items for the local backend.

      + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      NameHelpTypeExampleRead Only
      atimeTime of last accessRFC 33392006-01-02T15:04:05.999999999Z07:00N
      btimeTime of file birth (creation)RFC 33392006-01-02T15:04:05.999999999Z07:00N
      gidGroup ID of ownerdecimal number500N
      modeFile type and modeoctal, unix style0100664N
      mtimeTime of last modificationRFC 33392006-01-02T15:04:05.999999999Z07:00N
      rdevDevice ID (if special file)hexadecimal1abcN
      uidUser ID of ownerdecimal number500N
      +

      See the metadata docs for more info.

      Backend commands

      Here are the commands specific to the local backend.

      Run them with

      rclone backend COMMAND remote:

      The help below will explain what arguments each command takes.

      -

      See the "rclone backend" command for more info on how to pass options and arguments.

      +

      See the backend command for more info on how to pass options and arguments.

      These can be run on a running backend using the rc command backend/command.

      noop

      A null operation for testing backend commands

      @@ -24791,6 +27757,315 @@

      noop

    • "error": return an error based on option value

    Changelog

    +

    v1.59.0 - 2022-07-09

    +

    See commits

    +
      +
    • New backends +
      • +
    • New commands +
        +
      • test makefile: Create a single file for testing (Nick Craig-Wood)
        • +
      • +
    • New Features +
        +
      • Metadata framework to read and write system and user metadata on backends (Nick Craig-Wood) +
          +
        • Implemented initially for local, s3 and internetarchive backends
          • +
        • --metadata/-M flag to control whether metadata is copied
          • +
        • --metadata-set flag to specify metadata for uploads
          • +
        • Thanks to Manz Solutions for sponsoring this work.
          • +
        • +
      • build +
          +
        • Update to go1.18 and make go1.16 the minimum required version (Nick Craig-Wood)
          • +
        • Update android go build to 1.18.x and NDK to 23.1.7779620 (Nick Craig-Wood)
          • +
        • All windows binaries now no longer CGO (Nick Craig-Wood)
          • +
        • Add linux/arm/v6 to docker images (Nick Craig-Wood)
          • +
        • A huge number of fixes found with staticcheck (albertony)
          • +
        • Configurable version suffix independent of version number (albertony)
          • +
        • +
      • check: Implement --no-traverse and --no-unicode-normalization (Nick Craig-Wood)
        • +
      • config: Readability improvements (albertony)
        • +
      • copyurl: Add --header-filename to honor the HTTP header filename directive (J-P Treen)
        • +
      • filter: Allow multiple --exclude-if-present flags (albertony)
        • +
      • fshttp: Add --disable-http-keep-alives to disable HTTP Keep Alives (Nick Craig-Wood)
        • +
      • install.sh +
          +
        • Set the modes on the files and/or directories on macOS (Michael C Tiernan - MIT-Research Computing Project)
          • +
        • Pre verify sudo authorization -v before calling curl. (Michael C Tiernan - MIT-Research Computing Project)
          • +
        • +
      • lib/encoder: Add Semicolon encoding (Nick Craig-Wood)
        • +
      • lsf: Add metadata support with M flag (Nick Craig-Wood)
        • +
      • lsjson: Add --metadata/-M flag (Nick Craig-Wood)
        • +
      • ncdu +
          +
        • Implement multi selection (CrossR)
          • +
        • Replace termbox with tcell's termbox wrapper (eNV25)
          • +
        • Display correct path in delete confirmation dialog (Roberto Ricci)
          • +
        • +
      • operations +
          +
        • Speed up hash checking by aborting the other hash if first returns nothing (Nick Craig-Wood)
          • +
        • Use correct src/dst in some log messages (zzr93)
          • +
        • +
      • rcat: Check checksums by default like copy does (Nick Craig-Wood)
        • +
      • selfupdate: Replace deprecated x/crypto/openpgp package with ProtonMail/go-crypto (albertony)
        • +
      • serve ftp: Check --passive-port arguments are correct (Nick Craig-Wood)
        • +
      • size: Warn about inaccurate results when objects with unknown size (albertony)
        • +
      • sync: Overlap check is now filter-sensitive so --backup-dir can be in the root provided it is filtered (Nick)
        • +
      • test info: Check file name lengths using 1,2,3,4 byte unicode characters (Nick Craig-Wood)
        • +
      • test makefile(s): --sparse, --zero, --pattern, --ascii, --chargen flags to control file contents (Nick Craig-Wood)
        • +
      • Make sure we call the Shutdown method on backends (Martin Czygan)
        • +
      • +
    • Bug Fixes +
        +
      • accounting: Fix unknown length file transfers counting 3 transfers each (buda)
        • +
      • ncdu: Fix issue where dir size is summed when file sizes are -1 (albertony)
        • +
      • sync/copy/move +
          +
        • Fix --fast-list --create-empty-src-dirs and --exclude (Nick Craig-Wood)
          • +
        • Fix --max-duration and --cutoff-mode soft (Nick Craig-Wood)
          • +
        • +
      • Fix fs cache unpin (Martin Czygan)
        • +
      • Set proper exit code for errors that are not low-level retried (e.g. size/timestamp changing) (albertony)
        • +
      • +
    • Mount +
        +
      • Support windows/arm64 (may still be problems - see #5828) (Nick Craig-Wood)
        • +
      • Log IO errors at ERROR level (Nick Craig-Wood)
        • +
      • Ignore _netdev mount argument (Hugal31)
        • +
      • +
    • VFS +
        +
      • Add --vfs-fast-fingerprint for less accurate but faster fingerprints (Nick Craig-Wood)
        • +
      • Add --vfs-disk-space-total-size option to manually set the total disk space (Claudio Maradonna)
        • +
      • vfscache: Fix fatal error: sync: unlock of unlocked mutex error (Nick Craig-Wood)
        • +
      • +
    • Local +
        +
      • Fix parsing of --local-nounc flag (Nick Craig-Wood)
        • +
      • Add Metadata support (Nick Craig-Wood)
        • +
      • +
    • Crypt +
        +
      • Support metadata (Nick Craig-Wood)
        • +
      • +
    • Azure Blob +
        +
      • Calculate Chunksize/blocksize to stay below maxUploadParts (Leroy van Logchem)
        • +
      • Use chunksize lib to determine chunksize dynamically (Derek Battams)
        • +
      • Case insensitive access tier (Rob Pickerill)
        • +
      • Allow remote emulator (azurite) (Lorenzo Maiorfi)
        • +
      • +
    • B2 +
        +
      • Add --b2-version-at flag to show file versions at time specified (SwazRGB)
        • +
      • Use chunksize lib to determine chunksize dynamically (Derek Battams)
        • +
      • +
    • Chunker +
        +
      • Mark as not supporting metadata (Nick Craig-Wood)
        • +
      • +
    • Compress +
        +
      • Support metadata (Nick Craig-Wood)
        • +
      • +
    • Drive +
        +
      • Make backend config -o config add a combined AllDrives: remote (Nick Craig-Wood)
        • +
      • Make --drive-shared-with-me work with shared drives (Nick Craig-Wood)
        • +
      • Add --drive-resource-key for accessing link-shared files (Nick Craig-Wood)
        • +
      • Add backend commands exportformats and importformats for debugging (Nick Craig-Wood)
        • +
      • Fix 404 errors on copy/server side copy objects from public folder (Nick Craig-Wood)
        • +
      • Update Internal OAuth consent screen docs (Phil Shackleton)
        • +
      • Moved root_folder_id to advanced section (Abhiraj)
        • +
      • +
    • Dropbox +
        +
      • Migrate from deprecated api (m8rge)
        • +
      • Add logs to show when poll interval limits are exceeded (Nick Craig-Wood)
        • +
      • Fix nil pointer exception on dropbox impersonate user not found (Nick Craig-Wood)
        • +
      • +
    • Fichier +
        +
      • Parse api error codes and them accordingly (buengese)
        • +
      • +
    • FTP +
        +
      • Add support for disable_utf8 option (Jason Zheng)
        • +
      • Revert to upstream github.com/jlaffaye/ftp from our fork (Nick Craig-Wood)
        • +
      • +
    • Google Cloud Storage +
        +
      • Add --gcs-no-check-bucket to minimise transactions and perms (Nick Gooding)
        • +
      • Add --gcs-decompress flag to decompress gzip-encoded files (Nick Craig-Wood) +
          +
        • by default these will be downloaded compressed (which previously failed)
          • +
        • +
      • +
    • Hasher +
        +
      • Support metadata (Nick Craig-Wood)
        • +
      • +
    • HTTP +
        +
      • Fix missing response when using custom auth handler (albertony)
        • +
      • +
    • Jottacloud +
        +
      • Add support for upload to custom device and mountpoint (albertony)
        • +
      • Always store username in config and use it to avoid initial API request (albertony)
        • +
      • Fix issue with server-side copy when destination is in trash (albertony)
        • +
      • Fix listing output of remote with special characters (albertony)
        • +
      • +
    • Mailru +
        +
      • Fix timeout by using int instead of time.Duration for keeping number of seconds (albertony)
        • +
      • +
    • Mega +
        +
      • Document using MEGAcmd to help with login failures (Art M. Gallagher)
        • +
      • +
    • Onedrive +
        +
      • Implement --poll-interval for onedrive (Hugo Laloge)
        • +
      • Add access scopes option (Sven Gerber)
        • +
      • +
    • Opendrive +
        +
      • Resolve lag and truncate bugs (Scott Grimes)
        • +
      • +
    • Pcloud +
        +
      • Fix about with no free space left (buengese)
        • +
      • Fix cleanup (buengese)
        • +
      • +
    • S3 +
        +
      • Use PUT Object instead of presigned URLs to upload single part objects (Nick Craig-Wood)
        • +
      • Backend restore command to skip non-GLACIER objects (Vincent Murphy)
        • +
      • Use chunksize lib to determine chunksize dynamically (Derek Battams)
        • +
      • Retry RequestTimeout errors (Nick Craig-Wood)
        • +
      • Implement reading and writing of metadata (Nick Craig-Wood)
        • +
      • +
    • SFTP +
        +
      • Add support for about and hashsum on windows server (albertony)
        • +
      • Use vendor-specific VFS statistics extension for about if available (albertony)
        • +
      • Add --sftp-chunk-size to control packets sizes for high latency links (Nick Craig-Wood)
        • +
      • Add --sftp-concurrency to improve high latency transfers (Nick Craig-Wood)
        • +
      • Add --sftp-set-env option to set environment variables (Nick Craig-Wood)
        • +
      • Add Hetzner Storage Boxes to supported sftp backends (Anthrazz)
        • +
      • +
    • Storj +
        +
      • Fix put which lead to the file being unreadable when using mount (Erik van Velzen)
        • +
      • +
    • Union +
        +
      • Add min_free_space option for lfs/eplfs policies (Nick Craig-Wood)
        • +
      • Fix uploading files to union of all bucket based remotes (Nick Craig-Wood)
        • +
      • Fix get free space for remotes which don't support it (Nick Craig-Wood)
        • +
      • Fix eplus policy to select correct entry for existing files (Nick Craig-Wood)
        • +
      • Support metadata (Nick Craig-Wood)
        • +
      • +
    • Uptobox +
        +
      • Fix root path handling (buengese)
        • +
      • +
    • WebDAV +
        +
      • Add SharePoint in other specific regions support (Noah Hsu)
        • +
      • +
    • Yandex +
        +
      • Handle api error on server-side move (albertony)
        • +
      • +
    • Zoho +
        +
      • Add Japan and China regions (buengese)
        • +
      • +
    +

    v1.58.1 - 2022-04-29

    +

    See commits

    +
      +
    • Bug Fixes +
        +
      • build: Update github.com/billziss-gh to github.com/winfsp (Nick Craig-Wood)
        • +
      • filter: Fix timezone of --min-age/-max-age from UTC to local as documented (Nick Craig-Wood)
        • +
      • rc/js: Correct RC method names (Sơn Trần-Nguyễn)
        • +
      • docs +
          +
        • Fix some links to command pages (albertony)
          • +
        • Add --multi-thread-streams note to --transfers. (Zsolt Ero)
          • +
        • +
      • +
    • Mount +
        +
      • Fix --devname and fusermount: unknown option 'fsname' when mounting via rc (Nick Craig-Wood)
        • +
      • +
    • VFS +
        +
      • Remove wording which suggests VFS is only for mounting (Nick Craig-Wood)
        • +
      • +
    • Dropbox +
        +
      • Fix retries of multipart uploads with incorrect_offset error (Nick Craig-Wood)
        • +
      • +
    • Google Cloud Storage +
        +
      • Use the s3 pacer to speed up transactions (Nick Craig-Wood)
        • +
      • pacer: Default the Google pacer to a burst of 100 to fix gcs pacing (Nick Craig-Wood)
        • +
      • +
    • Jottacloud +
        +
      • Fix scope in token request (albertony)
        • +
      • +
    • Netstorage +
        +
      • Fix unescaped HTML in documentation (Nick Craig-Wood)
        • +
      • Make levels of headings consistent (Nick Craig-Wood)
        • +
      • Add support contacts to netstorage doc (Nil Alexandrov)
        • +
      • +
    • Onedrive +
        +
      • Note that sharepoint also changes web files (.html, .aspx) (GH)
        • +
      • +
    • Putio +
        +
      • Handle rate limit errors (Berkan Teber)
        • +
      • Fix multithread download and other ranged requests (rafma0)
        • +
      • +
    • S3 +
        +
      • Add ChinaMobile EOS to provider list (GuoXingbin)
        • +
      • Sync providers in config description with providers (Nick Craig-Wood)
        • +
      • +
    • SFTP +
        +
      • Fix OpenSSH 8.8+ RSA keys incompatibility (KARBOWSKI Piotr)
        • +
      • Note that Scaleway C14 is deprecating SFTP in favor of S3 (Adrien Rey-Jarthon)
        • +
      • +
    • Storj +
        +
      • Fix bucket creation on Move (Nick Craig-Wood)
        • +
      • +
    • WebDAV +
        +
      • Don't override Referer if user sets it (Nick Craig-Wood)
        • +
      • +

    v1.58.0 - 2022-03-18

    See commits

      @@ -30447,7 +33722,7 @@

      v0.00 - 2012-11-18

    • Project started

    Bugs and Limitations

    -

    Limitations

    +

    Limitations

    Directory timestamps aren't preserved

    Rclone doesn't currently preserve the timestamps of directories. This is because rclone only really considers objects when syncing.

    Rclone struggles with millions of files in a directory/bucket

    @@ -30474,7 +33749,7 @@

    Can rclone sync directly from

    Rclone can sync between two remote cloud storage systems just fine.

    Note that it effectively downloads the file and uploads it again, so the node running rclone would need to have lots of bandwidth.

    The syncs would be incremental (on a file by file basis).

    -

    Eg

    +

    e.g.

    rclone sync -i drive:Folder s3:bucket

    Using rclone from multiple locations at the same time

    You can use rclone from multiple places at the same time if you choose different subdirectory for the output, e.g.

    @@ -30504,7 +33779,7 @@

    Can I use rclone with an HTTP proxy

    e.g.

    export no_proxy=localhost,127.0.0.0/8,my.host.name
 export NO_PROXY=$no_proxy
    -

    Note that the ftp backend does not support ftp_proxy yet.

    +

    Note that the FTP backend does not support ftp_proxy yet.

    Rclone gives x509: failed to load system roots and no roots provided error

    This means that rclone can't find the SSL root certificates. Likely you are running rclone on a NAS with a cut-down Linux OS, or possibly on Solaris.

    Rclone (via the Go runtime) tries to load the root certificates from these places on Linux.

    @@ -31127,6 +34402,57 @@

    Contributors

  • Vincent Murphy
  • ctrl-q
  • Nil Alexandrov
    • +
  • GuoXingbin
    • +
  • Berkan Teber
    • +
  • Tobias Klauser
    • +
  • KARBOWSKI Piotr
    • +
  • GH
    • +
  • rafma0
    • +
  • Adrien Rey-Jarthon
    • +
  • Nick Gooding
    • +
  • Leroy van Logchem
    • +
  • Zsolt Ero
    • +
  • Lesmiscore
    • +
  • ehsantdy
    • +
  • SwazRGB
    • +
  • Mateusz Puczyński
    • +
  • Michael C Tiernan - MIT-Research Computing Project
    • +
  • Kaspian
    • +
  • Werner
    • +
  • Hugal31
    • +
  • Christian Galo
    • +
  • Erik van Velzen
    • +
  • Derek Battams
    • +
  • SimonLiu
    • +
  • Hugo Laloge
    • +
  • Mr-Kanister
    • +
  • Rob Pickerill
    • +
  • Andrey
    • +
  • Eric Wolf
    • +
  • Nick
    • +
  • Jason Zheng
    • +
  • Matthew Vernon
    • +
  • Noah Hsu
    • +
  • m00594701
    • +
  • Art M. Gallagher
    • +
  • Sven Gerber
    • +
  • CrossR
    • +
  • Maciej Radzikowski
    • +
  • Scott Grimes
    • +
  • Phil Shackleton
    • +
  • eNV25
    • +
  • Caleb
    • +
  • J-P Treen
    • +
  • Martin Czygan
    • +
  • buda
    • +
  • mirekphd
    • +
  • vyloy
    • +
  • Anthrazz
    • +
  • zzr93
    • +
  • Paul Norman
    • +
  • Lorenzo Maiorfi
    • +
  • Claudio Maradonna
    • +
  • Ovidiu Victor Tatar

    • Contact the rclone project

    Forum

    diff --git a/MANUAL.md b/MANUAL.md index 58e4ddad6a03d..6cd4e96044ec6 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -1,6 +1,6 @@ % rclone(1) User Manual % Nick Craig-Wood -% Mar 18, 2022 +% Jul 09, 2022 # Rclone syncs your files to cloud storage @@ -92,7 +92,7 @@ Rclone helps you: - [Move](https://rclone.org/commands/rclone_move/) files to cloud storage deleting the local after verification - [Check](https://rclone.org/commands/rclone_check/) hashes and for missing/extra files - [Mount](https://rclone.org/commands/rclone_mount/) your cloud storage as a network disk -- [Serve](https://rclone.org/commands/rclone_serve/) local or remote files over [HTTP](https://rclone.org/commands/rclone_serve_http/)/[WebDav](https://rclone.org/commands/rclone_serve_webdav/)/[FTP](https://rclone.org/commands/rclone_serve_ftp/)/[SFTP](https://rclone.org/commands/rclone_serve_sftp/)/[dlna](https://rclone.org/commands/rclone_serve_dlna/) +- [Serve](https://rclone.org/commands/rclone_serve/) local or remote files over [HTTP](https://rclone.org/commands/rclone_serve_http/)/[WebDav](https://rclone.org/commands/rclone_serve_webdav/)/[FTP](https://rclone.org/commands/rclone_serve_ftp/)/[SFTP](https://rclone.org/commands/rclone_serve_sftp/)/[DLNA](https://rclone.org/commands/rclone_serve_dlna/) - Experimental [Web based GUI](https://rclone.org/gui/) ## Supported providers {#providers} @@ -109,8 +109,11 @@ WebDAV or S3, that work out of the box.) - Backblaze B2 - Box - Ceph +- China Mobile Ecloud Elastic Object Storage (EOS) +- Arvan Cloud Object Storage (AOS) - Citrix ShareFile - C14 +- Cloudflare R2 - DigitalOcean Spaces - Digi Storage - Dreamhost @@ -121,10 +124,14 @@ WebDAV or S3, that work out of the box.) - Google Drive - Google Photos - HDFS +- Hetzner Storage Box +- HiDrive - HTTP - Hubic +- Internet Archive - Jottacloud - IBM COS S3 +- IDrive e2 - Koofr - Mail.ru Cloud - Memset Memstore @@ -163,18 +170,32 @@ WebDAV or S3, that work out of the box.) - The local filesystem -Links +## Virtual providers + +These backends adapt or modify other storage providers: + +- Alias: Rename existing remotes +- Cache: Cache remotes (DEPRECATED) +- Chunker: Split large files +- Combine: Combine multiple remotes into a directory tree +- Compress: Compress files +- Crypt: Encrypt files +- Hasher: Hash files +- Union: Join multiple remotes to work together + + +## Links * [Home page](https://rclone.org/) * [GitHub project page for source and bug tracker](https://github.com/rclone/rclone) * [Rclone Forum](https://forum.rclone.org) * [Downloads](https://rclone.org/downloads/) -# Install # +# Install Rclone is a Go program and comes as a single binary file. -## Quickstart ## +## Quickstart * [Download](https://rclone.org/downloads/) the relevant binary. * Extract the `rclone` executable, `rclone.exe` on Windows, from the archive. @@ -189,20 +210,20 @@ run `rclone -h`. Already installed rclone can be easily updated to the latest version using the [rclone selfupdate](https://rclone.org/commands/rclone_selfupdate/) command. -## Script installation ## +## Script installation To install rclone on Linux/macOS/BSD systems, run: - curl https://rclone.org/install.sh | sudo bash + sudo -v ; curl https://rclone.org/install.sh | sudo bash For beta installation, run: - curl https://rclone.org/install.sh | sudo bash -s beta + sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta Note that this script checks the version of rclone installed first and won't re-download if not needed. -## Linux installation from precompiled binary ## +## Linux installation from precompiled binary Fetch and unpack @@ -226,7 +247,7 @@ Run `rclone config` to setup. See [rclone config docs](https://rclone.org/docs/) rclone config -## macOS installation with brew ## +## macOS installation with brew brew install rclone @@ -235,7 +256,7 @@ NOTE: This version of rclone will not support `mount` any more (see on macOS, either install a precompiled binary or enable the relevant option when [installing from source](#install-from-source). -## macOS installation from precompiled binary, using curl ## +## macOS installation from precompiled binary, using curl To avoid problems with macOS gatekeeper enforcing the binary to be signed and notarized it is enough to download with `curl`. @@ -263,20 +284,20 @@ Run `rclone config` to setup. See [rclone config docs](https://rclone.org/docs/) rclone config -## macOS installation from precompiled binary, using a web browser ## +## macOS installation from precompiled binary, using a web browser When downloading a binary with a web browser, the browser will set the macOS gatekeeper quarantine attribute. Starting from Catalina, when attempting to run `rclone`, a pop-up will appear saying: - “rclone” cannot be opened because the developer cannot be verified. + "rclone" cannot be opened because the developer cannot be verified. macOS cannot verify that this app is free from malware. The simplest fix is to run xattr -d com.apple.quarantine rclone -## Install with docker ## +## Install with docker The rclone maintains a [docker image for rclone](https://hub.docker.com/r/rclone/rclone). These images are autobuilt by docker hub from the rclone source based @@ -355,39 +376,93 @@ ls ~/data/mount kill %1 ``` -## Install from source ## +## Install from source -Make sure you have at least [Go](https://golang.org/) go1.15 -installed. [Download go](https://golang.org/dl/) if necessary. The -latest release is recommended. Then +Make sure you have git and [Go](https://golang.org/) installed. +Go version 1.16 or newer is required, latest release is recommended. +You can get it from your package manager, or download it from +[golang.org/dl](https://golang.org/dl/). Then you can run the following: -```sh +``` git clone https://github.com/rclone/rclone.git cd rclone go build -# If on macOS and mount is wanted, instead run: make GOTAGS=cmount -./rclone version ``` -This will leave you a checked out version of rclone you can modify and -send pull requests with. If you use `make` instead of `go build` then -the rclone build will have the correct version information in it. +This will check out the rclone source in subfolder rclone, which you can later +modify and send pull requests with. Then it will build the rclone executable +in the same folder. As an initial check you can now run `./rclone version` +(`.\rclone version` on Windows). -You can also build the latest stable rclone with: +Note that on macOS and Windows the [mount](https://rclone.org/commands/rclone_mount/) +command will not be available unless you specify additional build tag `cmount`. - go get github.com/rclone/rclone +``` +go build -tags cmount +``` -or the latest version (equivalent to the beta) with +This assumes you have a GCC compatible C compiler (GCC or Clang) in your PATH, +as it uses [cgo](https://pkg.go.dev/cmd/cgo). But on Windows, the +[cgofuse](https://github.com/winfsp/cgofuse) library that the cmount +implementation is based on, also supports building +[without cgo](https://github.com/golang/go/wiki/WindowsDLLs), i.e. by setting +environment variable CGO_ENABLED to value 0 (static linking). This is how the +official Windows release of rclone is being built, starting with version 1.59. +It is still possible to build with cgo on Windows as well, by using the MinGW +port of GCC, e.g. by installing it in a [MSYS2](https://www.msys2.org) +distribution (make sure you install it in the classic mingw64 subsystem, the +ucrt64 version is not compatible). - go get github.com/rclone/rclone@master +Additionally, on Windows, you must install the third party utility +[WinFsp](http://www.secfs.net/winfsp/), with the "Developer" feature selected. +If building with cgo, you must also set environment variable CPATH pointing to +the fuse include directory within the WinFsp installation +(normally `C:\Program Files (x86)\WinFsp\inc\fuse`). -These will build the binary in `$(go env GOPATH)/bin` -(`~/go/bin/rclone` by default) after downloading the source to the go -module cache. Note - do **not** use the `-u` flag here. This causes go -to try to update the dependencies that rclone uses and sometimes these -don't work with the current version of rclone. +You may also add arguments `-ldflags -s` (with or without `-tags cmount`), +to omit symbol table and debug information, making the executable file smaller, +and `-trimpath` to remove references to local file system paths. This is how +the official rclone releases are built. -## Installation with Ansible ## +``` +go build -trimpath -ldflags -s -tags cmount +``` + +Instead of executing the `go build` command directly, you can run it via the +Makefile, which also sets version information and copies the resulting rclone +executable into your GOPATH bin folder (`$(go env GOPATH)/bin`, which +corresponds to `~/go/bin/rclone` by default). + +``` +make +``` + +To include mount command on macOS and Windows with Makefile build: + +``` +make GOTAGS=cmount +``` + +As an alternative you can download the source, build and install rclone in one +operation, as a regular Go package. The source will be stored it in the Go +module cache, and the resulting executable will be in your GOPATH bin folder +(`$(go env GOPATH)/bin`, which corresponds to `~/go/bin/rclone` by default). + +With Go version 1.17 or newer: + +``` +go install github.com/rclone/rclone@latest +``` + +With Go versions older than 1.17 (do **not** use the `-u` flag, it causes Go to +try to update the dependencies that rclone uses and sometimes these don't work +with the current version): + +``` +go get github.com/rclone/rclone +``` + +## Installation with Ansible This can be done with [Stefan Weichinger's ansible role](https://github.com/stefangweichinger/ansible-rclone). @@ -403,7 +478,7 @@ Instructions - rclone ``` -## Portable installation ## +## Portable installation As mentioned [above](https://rclone.org/install/#quickstart), rclone is single executable (`rclone`, or `rclone.exe` on Windows) that you can download as a @@ -481,7 +556,7 @@ the [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec) utility from Microsoft's Sysinternals suite, which takes option `-s` to execute commands as the `SYSTEM` user. -#### Start from Startup folder ### +#### Start from Startup folder To quickly execute an rclone command you can simply create a standard Windows Explorer shortcut for the complete rclone command you want to run. If you @@ -496,7 +571,7 @@ functionality to set it to run as different user, or to set conditions or actions on certain events. Setting up a scheduled task as described below will often give you better results. -#### Start from Task Scheduler ### +#### Start from Task Scheduler Task Scheduler is an administrative tool built into Windows, and it can be used to configure rclone to be started automatically in a highly configurable way, e.g. @@ -506,14 +581,14 @@ be available to all users it can run as the `SYSTEM` user. For technical information, see https://docs.microsoft.com/windows/win32/taskschd/task-scheduler-start-page. -#### Run as service ### +#### Run as service For running rclone at system startup, you can create a Windows service that executes your rclone command, as an alternative to scheduled task configured to run at startup. -##### Mount command built-in service integration #### +##### Mount command built-in service integration -For mount commands, Rclone has a built-in Windows service integration via the third-party +For mount commands, rclone has a built-in Windows service integration via the third-party WinFsp library it uses. Registering as a regular Windows service easy, as you just have to execute the built-in PowerShell command `New-Service` (requires administrative privileges). @@ -533,7 +608,7 @@ Windows standard methods for managing network drives. This is currently not officially supported by Rclone, but with WinFsp version 2019.3 B2 / v1.5B2 or later it should be possible through path rewriting as described [here](https://github.com/rclone/rclone/issues/3340). -##### Third-party service integration ##### +##### Third-party service integration To Windows service running any rclone command, the excellent third-party utility [NSSM](http://nssm.cc), the "Non-Sucking Service Manager", can be used. @@ -601,6 +676,7 @@ See the following for detailed instructions for * [Chunker](https://rclone.org/chunker/) - transparently splits large files for other remotes * [Citrix ShareFile](https://rclone.org/sharefile/) * [Compress](https://rclone.org/compress/) + * [Combine](https://rclone.org/combine/) * [Crypt](https://rclone.org/crypt/) - to encrypt other remotes * [DigitalOcean Spaces](https://rclone.org/s3/#digitalocean-spaces) * [Digi Storage](https://rclone.org/koofr/#digi-storage) @@ -612,8 +688,10 @@ See the following for detailed instructions for * [Google Photos](https://rclone.org/googlephotos/) * [Hasher](https://rclone.org/hasher/) - to handle checksums for other remotes * [HDFS](https://rclone.org/hdfs/) + * [HiDrive](https://rclone.org/hidrive/) * [HTTP](https://rclone.org/http/) * [Hubic](https://rclone.org/hubic/) + * [Internet Archive](https://rclone.org/internetarchive/) * [Jottacloud](https://rclone.org/jottacloud/) * [Koofr](https://rclone.org/koofr/) * [Mail.ru Cloud](https://rclone.org/mailru/) @@ -715,13 +793,18 @@ Copy files from source to dest, skipping identical files. Copy the source to the destination. Does not transfer files that are identical on source and destination, testing by size and modification -time or MD5SUM. Doesn't delete files from the destination. +time or MD5SUM. Doesn't delete files from the destination. If you +want to also delete files from destination, to make it match source, +use the [sync](https://rclone.org/commands/rclone_sync/) command instead. Note that it is always the contents of the directory that is synced, -not the directory so when source:path is a directory, it's the +not the directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. +To copy single files, use the [copyto](https://rclone.org/commands/rclone_copyto/) +command instead. + If dest:path doesn't exist, it is created and the source:path contents go there. @@ -793,7 +876,9 @@ Sync the source to the destination, changing the destination only. Doesn't transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Destination is updated to match source, including deleting files -if necessary (except duplicate objects, see below). +if necessary (except duplicate objects, see below). If you don't +want to delete files from destination, use the +[copy](https://rclone.org/commands/rclone_copy/) command instead. **Important**: Since this can cause data loss, test first with the `--dry-run` or the `--interactive`/`-i` flag. @@ -805,9 +890,9 @@ errors at any point. Duplicate objects (files with the same name, on those providers that support it) are also not yet handled. It is always the contents of the directory that is synced, not the -directory so when source:path is a directory, it's the contents of +directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. See -extended explanation in the `copy` command above if unsure. +extended explanation in the [copy](https://rclone.org/commands/rclone_copy/) command if unsure. If dest:path doesn't exist, it is created and the source:path contents go there. @@ -846,6 +931,9 @@ Moves the contents of the source directory to the destination directory. Rclone will error if the source and destination overlap and the remote does not support a server-side directory move operation. +To move single files, use the [moveto](https://rclone.org/commands/rclone_moveto/) +command instead. + If no filters are in use and if possible this will server-side move `source:path` into `dest:path`. After this `source:path` will no longer exist. @@ -856,7 +944,8 @@ move will be used, otherwise it will copy it (server-side if possible) into `dest:path` then delete the original (if no errors on copy) in `source:path`. -If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag. +If you want to delete empty source directories after move, use the +`--delete-empty-src-dirs` flag. See the [--no-traverse](https://rclone.org/docs/#no-traverse) option for controlling whether rclone lists the destination directory or not. Supplying this @@ -894,16 +983,16 @@ Remove the files in path. ## Synopsis -Remove the files in path. Unlike `purge` it obeys include/exclude -filters so can be used to selectively delete files. +Remove the files in path. Unlike [purge](https://rclone.org/commands/rclone_purge/) it +obeys include/exclude filters so can be used to selectively delete files. `rclone delete` only deletes files but leaves the directory structure alone. If you want to delete a directory and all of its contents use -the `purge` command. +the [purge](https://rclone.org/commands/rclone_purge/) command. If you supply the `--rmdirs` flag, it will remove all empty directories along with it. -You can also use the separate command `rmdir` or `rmdirs` to -delete empty directories only. +You can also use the separate command [rmdir](https://rclone.org/commands/rclone_rmdir/) or +[rmdirs](https://rclone.org/commands/rclone_rmdirs/) to delete empty directories only. For example, to delete all files bigger than 100 MiB, you may first want to check what would be deleted (use either): @@ -947,9 +1036,10 @@ Remove the path and all of its contents. Remove the path and all of its contents. Note that this does not obey -include/exclude filters - everything will be removed. Use the `delete` -command if you want to selectively delete files. To delete empty directories only, -use command `rmdir` or `rmdirs`. +include/exclude filters - everything will be removed. Use the +[delete](https://rclone.org/commands/rclone_delete/) command if you want to selectively +delete files. To delete empty directories only, use command +[rmdir](https://rclone.org/commands/rclone_rmdir/) or [rmdirs](https://rclone.org/commands/rclone_rmdirs/). **Important**: Since this can cause data loss, test first with the `--dry-run` or the `--interactive`/`-i` flag. @@ -1000,10 +1090,10 @@ Remove the empty directory at path. This removes empty directory given by path. Will not remove the path if it has any objects in it, not even empty subdirectories. Use -command `rmdirs` (or `delete` with option `--rmdirs`) -to do that. +command [rmdirs](https://rclone.org/commands/rclone_rmdirs/) (or [delete](https://rclone.org/commands/rclone_delete/) +with option `--rmdirs`) to do that. -To delete a path and any objects in it, use `purge` command. +To delete a path and any objects in it, use [purge](https://rclone.org/commands/rclone_purge/) command. ``` @@ -1033,6 +1123,10 @@ Checks the files in the source and destination match. It compares sizes and hashes (MD5 or SHA1) and logs a report of files that don't match. It doesn't alter the source or destination. +For the [crypt](https://rclone.org/crypt/) remote there is a dedicated command, +[cryptcheck](https://rclone.org/commands/rclone_cryptcheck/), that are able to check +the checksums of the crypted files. + If you supply the `--size-only` flag, it will only compare the sizes not the hashes as well. Use this for a quick check. @@ -1157,7 +1251,7 @@ List all directories/containers/buckets in the path. Lists the directories in the source path to standard output. Does not -recurse by default. Use the -R flag to recurse. +recurse by default. Use the `-R` flag to recurse. This command lists the total size of the directory (if known, -1 if not), the modification time (if known, the current time if not), the @@ -1175,7 +1269,7 @@ Or -1 2017-01-03 14:40:54 -1 2500files -1 2017-07-08 14:39:28 -1 4000files -If you just want the directory names use "rclone lsf --dirs-only". +If you just want the directory names use `rclone lsf --dirs-only`. Any of the filtering options can be applied to this command. @@ -1291,6 +1385,10 @@ not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling MD5 for any remote. +For other algorithms, see the [hashsum](https://rclone.org/commands/rclone_hashsum/) +command. Running `rclone md5sum remote:path` is equivalent +to running `rclone hashsum MD5 remote:path`. + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, @@ -1332,6 +1430,10 @@ not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling SHA-1 for any remote. +For other algorithms, see the [hashsum](https://rclone.org/commands/rclone_hashsum/) +command. Running `rclone sha1sum remote:path` is equivalent +to running `rclone hashsum SHA1 remote:path`. + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, @@ -1365,6 +1467,28 @@ See the [global flags page](https://rclone.org/flags/) for global options not li Prints the total size and number of objects in remote:path. +## Synopsis + + +Counts objects in the path and calculates the total size. Prints the +result to standard output. + +By default the output is in human-readable format, but shows values in +both human-readable format as well as the raw numbers (global option +`--human-readable` is not considered). Use option `--json` +to format output as JSON instead. + +Recurses by default, use `--max-depth 1` to stop the +recursion. + +Some backends do not always provide file sizes, see for example +[Google Photos](https://rclone.org/googlephotos/#size) and +[Google Drive](https://rclone.org/drive/#limitations-of-google-docs). +Rclone will then show a notice in the log indicating how many such +files were encountered, and count them in as empty files in the output +of the size command. + + ``` rclone size remote:path [flags] ``` @@ -1488,7 +1612,7 @@ Opendrive) that can have duplicate file names. It can be run on wrapping backend (e.g. crypt) if they wrap a backend which supports duplicate file names. -However if --by-hash is passed in then dedupe will find files with +However if `--by-hash` is passed in then dedupe will find files with duplicate hashes instead which will work on any backend which supports at least one hash. This can be used to find files with duplicate content. This is known as deduping by hash. @@ -1916,11 +2040,10 @@ See the [global flags page](https://rclone.org/flags/) for global options not li # rclone completion -generate the autocompletion script for the specified shell +Generate the autocompletion script for the specified shell ## Synopsis - Generate the autocompletion script for rclone for the specified shell. See each sub-command's help for details on how to use the generated script. @@ -1936,34 +2059,38 @@ See the [global flags page](https://rclone.org/flags/) for global options not li ## SEE ALSO * [rclone](https://rclone.org/commands/rclone/) - Show help for rclone commands, flags and backends. -* [rclone completion bash](https://rclone.org/commands/rclone_completion_bash/) - generate the autocompletion script for bash -* [rclone completion fish](https://rclone.org/commands/rclone_completion_fish/) - generate the autocompletion script for fish -* [rclone completion powershell](https://rclone.org/commands/rclone_completion_powershell/) - generate the autocompletion script for powershell -* [rclone completion zsh](https://rclone.org/commands/rclone_completion_zsh/) - generate the autocompletion script for zsh +* [rclone completion bash](https://rclone.org/commands/rclone_completion_bash/) - Generate the autocompletion script for bash +* [rclone completion fish](https://rclone.org/commands/rclone_completion_fish/) - Generate the autocompletion script for fish +* [rclone completion powershell](https://rclone.org/commands/rclone_completion_powershell/) - Generate the autocompletion script for powershell +* [rclone completion zsh](https://rclone.org/commands/rclone_completion_zsh/) - Generate the autocompletion script for zsh # rclone completion bash -generate the autocompletion script for bash +Generate the autocompletion script for bash ## Synopsis - Generate the autocompletion script for the bash shell. This script depends on the 'bash-completion' package. If it is not installed already, you can install it via your OS's package manager. To load completions in your current shell session: -$ source <(rclone completion bash) + + source <(rclone completion bash) To load completions for every new session, execute once: -Linux: - $ rclone completion bash > /etc/bash_completion.d/rclone -MacOS: - $ rclone completion bash > /usr/local/etc/bash_completion.d/rclone + +### Linux: + + rclone completion bash > /etc/bash_completion.d/rclone + +### macOS: + + rclone completion bash > /usr/local/etc/bash_completion.d/rclone You will need to start a new shell for this setup to take effect. - + ``` rclone completion bash @@ -1980,22 +2107,23 @@ See the [global flags page](https://rclone.org/flags/) for global options not li ## SEE ALSO -* [rclone completion](https://rclone.org/commands/rclone_completion/) - generate the autocompletion script for the specified shell +* [rclone completion](https://rclone.org/commands/rclone_completion/) - Generate the autocompletion script for the specified shell # rclone completion fish -generate the autocompletion script for fish +Generate the autocompletion script for fish ## Synopsis - Generate the autocompletion script for the fish shell. To load completions in your current shell session: -$ rclone completion fish | source + + rclone completion fish | source To load completions for every new session, execute once: -$ rclone completion fish > ~/.config/fish/completions/rclone.fish + + rclone completion fish > ~/.config/fish/completions/rclone.fish You will need to start a new shell for this setup to take effect. @@ -2015,19 +2143,19 @@ See the [global flags page](https://rclone.org/flags/) for global options not li ## SEE ALSO -* [rclone completion](https://rclone.org/commands/rclone_completion/) - generate the autocompletion script for the specified shell +* [rclone completion](https://rclone.org/commands/rclone_completion/) - Generate the autocompletion script for the specified shell # rclone completion powershell -generate the autocompletion script for powershell +Generate the autocompletion script for powershell ## Synopsis - Generate the autocompletion script for powershell. To load completions in your current shell session: -PS C:\> rclone completion powershell | Out-String | Invoke-Expression + + rclone completion powershell | Out-String | Invoke-Expression To load completions for every new session, add the output of the above command to your powershell profile. @@ -2048,27 +2176,30 @@ See the [global flags page](https://rclone.org/flags/) for global options not li ## SEE ALSO -* [rclone completion](https://rclone.org/commands/rclone_completion/) - generate the autocompletion script for the specified shell +* [rclone completion](https://rclone.org/commands/rclone_completion/) - Generate the autocompletion script for the specified shell # rclone completion zsh -generate the autocompletion script for zsh +Generate the autocompletion script for zsh ## Synopsis - Generate the autocompletion script for the zsh shell. If shell completion is not already enabled in your environment you will need to enable it. You can execute the following once: -$ echo "autoload -U compinit; compinit" >> ~/.zshrc + echo "autoload -U compinit; compinit" >> ~/.zshrc To load completions for every new session, execute once: -# Linux: -$ rclone completion zsh > "${fpath[1]}/_rclone" -# macOS: -$ rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone + +### Linux: + + rclone completion zsh > "${fpath[1]}/_rclone" + +### macOS: + + rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone You will need to start a new shell for this setup to take effect. @@ -2088,7 +2219,7 @@ See the [global flags page](https://rclone.org/flags/) for global options not li ## SEE ALSO -* [rclone completion](https://rclone.org/commands/rclone_completion/) - generate the autocompletion script for the specified shell +* [rclone completion](https://rclone.org/commands/rclone_completion/) - Generate the autocompletion script for the specified shell # rclone config create @@ -2656,8 +2787,8 @@ If source:path is a file or directory then it copies it to a file or directory named dest:path. This can be used to upload single files to other than their current -name. If the source is a directory then it acts exactly like the copy -command. +name. If the source is a directory then it acts exactly like the +[copy](https://rclone.org/commands/rclone_copy/) command. So @@ -2707,10 +2838,11 @@ Copy url content to dest. Download a URL's content and copy it to the destination without saving it in temporary storage. -Setting `--auto-filename` will cause the file name to be retrieved from -the URL (after any redirections) and used in the destination -path. With `--print-filename` in addition, the resulting file name will -be printed. +Setting `--auto-filename` will attempt to automatically determine the filename from the URL +(after any redirections) and used in the destination path. +With `--auto-filename-header` in +addition, if a specific filename is set in HTTP headers, it will be used instead of the name from the URL. +With `--print-filename` in addition, the resulting file name will be printed. Setting `--no-clobber` will prevent overwriting file on the destination if there is one with the same name. @@ -2726,11 +2858,12 @@ rclone copyurl https://example.com dest:path [flags] ## Options ``` - -a, --auto-filename Get the file name from the URL and use it for destination file path - -h, --help help for copyurl - --no-clobber Prevent overwriting file with same name - -p, --print-filename Print the resulting name from --auto-filename - --stdout Write the output to stdout rather than a file + -a, --auto-filename Get the file name from the URL and use it for destination file path + --header-filename Get the file name from the Content-Disposition header + -h, --help help for copyurl + --no-clobber Prevent overwriting file with same name + -p, --print-filename Print the resulting name from --auto-filename + --stdout Write the output to stdout rather than a file ``` See the [global flags page](https://rclone.org/flags/) for global options not listed here. @@ -2746,9 +2879,9 @@ Cryptcheck checks the integrity of a crypted remote. ## Synopsis -rclone cryptcheck checks a remote against a crypted remote. This is -the equivalent of running rclone check, but able to check the -checksums of the crypted remote. +rclone cryptcheck checks a remote against a [crypted](https://rclone.org/crypt/) remote. +This is the equivalent of running rclone [check](https://rclone.org/commands/rclone_check/), +but able to check the checksums of the crypted remote. For it to work the underlying remote of the cryptedremote must support some kind of checksum. @@ -2824,7 +2957,7 @@ Cryptdecode returns unencrypted file names. rclone cryptdecode returns unencrypted file names when provided with a list of encrypted file names. List limit is 10 items. -If you supply the --reverse flag, it will return encrypted file names. +If you supply the `--reverse` flag, it will return encrypted file names. use it like this @@ -2832,8 +2965,8 @@ use it like this rclone cryptdecode --reverse encryptedremote: filename1 filename2 -Another way to accomplish this is by using the `rclone backend encode` (or `decode`)command. -See the documentation on the `crypt` overlay for more info. +Another way to accomplish this is by using the `rclone backend encode` (or `decode`) command. +See the documentation on the [crypt](https://rclone.org/crypt/) overlay for more info. ``` @@ -2889,7 +3022,7 @@ Output completion script for a given shell. Generates a shell completion script for rclone. -Run with --help to list the supported shells. +Run with `--help` to list the supported shells. ## Options @@ -3073,6 +3206,9 @@ not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling any hash for any remote. +For the MD5 and SHA1 algorithms there are also dedicated commands, +[md5sum](https://rclone.org/commands/rclone_md5sum/) and [sha1sum](https://rclone.org/commands/rclone_sha1sum/). + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, @@ -3088,6 +3224,7 @@ Run without a hash to see the list of all supported hashes, e.g. * crc32 * sha256 * dropbox + * hidrive * mailru * quickxor @@ -3174,7 +3311,7 @@ List all the remotes in the config file. rclone listremotes lists all the available remotes from the config file. -When uses with the -l flag it lists the types too. +When used with the `--long` flag it lists the types too. ``` @@ -3215,7 +3352,7 @@ Eg ferejej3gux/ fubuwic -Use the --format option to control what gets listed. By default this +Use the `--format` option to control what gets listed. By default this is just the path, but you can use these parameters to control the output: @@ -3228,9 +3365,10 @@ output: m - MimeType of object if known e - encrypted name T - tier of storage if known, e.g. "Hot" or "Cool" + M - Metadata of object in JSON blob format, eg {"key":"value"} So if you wanted the path, size and modification time, you would use ---format "pst", or maybe --format "tsp" to put the path last. +`--format "pst"`, or maybe `--format "tsp"` to put the path last. Eg @@ -3242,7 +3380,7 @@ Eg 2016-06-25 18:55:40;37600;fubuwic If you specify "h" in the format you will get the MD5 hash by default, -use the "--hash" flag to change which hash you want. Note that this +use the `--hash` flag to change which hash you want. Note that this can be returned as an empty string if it isn't available on the object (and for directories), "ERROR" if there was an error reading it from the object and "UNSUPPORTED" if that object does not support that hash @@ -3264,7 +3402,7 @@ Eg (Though "rclone md5sum ." is an easier way of typing this.) By default the separator is ";" this can be changed with the ---separator flag. Note that separators aren't escaped in the path so +`--separator` flag. Note that separators aren't escaped in the path so putting it last is a good strategy. Eg @@ -3286,8 +3424,8 @@ Eg test.sh,449 "this file contains a comma, in the file name.txt",6 -Note that the --absolute parameter is useful for making lists of files -to pass to an rclone copy with the --files-from-raw flag. +Note that the `--absolute` parameter is useful for making lists of files +to pass to an rclone copy with the `--files-from-raw` flag. For example, to find all the files modified within one day and copy those only (without traversing the whole directory structure): @@ -3354,7 +3492,7 @@ List directories and objects in the path in JSON format. The output is an array of Items, where each Item looks like this - { + { "Hashes" : { "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f", "MD5" : "b1946ac92492d2347c6235b4d2611184", @@ -3372,29 +3510,32 @@ The output is an array of Items, where each Item looks like this "Path" : "full/path/goes/here/file.txt", "Size" : 6, "Tier" : "hot", - } + } -If --hash is not specified the Hashes property won't be emitted. The -types of hash can be specified with the --hash-type parameter (which -may be repeated). If --hash-type is set then it implies --hash. +If `--hash` is not specified the Hashes property won't be emitted. The +types of hash can be specified with the `--hash-type` parameter (which +may be repeated). If `--hash-type` is set then it implies `--hash`. -If --no-modtime is specified then ModTime will be blank. This can +If `--no-modtime` is specified then ModTime will be blank. This can speed things up on remotes where reading the ModTime takes an extra request (e.g. s3, swift). -If --no-mimetype is specified then MimeType will be blank. This can +If `--no-mimetype` is specified then MimeType will be blank. This can speed things up on remotes where reading the MimeType takes an extra request (e.g. s3, swift). -If --encrypted is not specified the Encrypted won't be emitted. +If `--encrypted` is not specified the Encrypted won't be emitted. -If --dirs-only is not specified files in addition to directories are +If `--dirs-only` is not specified files in addition to directories are returned -If --files-only is not specified directories in addition to the files +If `--files-only` is not specified directories in addition to the files will be returned. -if --stat is set then a single JSON blob will be returned about the +If `--metadata` is set then an additional Metadata key will be returned. +This will have metdata in rclone standard format as a JSON object. + +if `--stat` is set then a single JSON blob will be returned about the item pointed to. This will return an error if the item isn't found. However on bucket based backends (like s3, gcs, b2, azureblob etc) if the item isn't found it will return an empty directory as it isn't @@ -3403,7 +3544,7 @@ possible to tell empty directories from missing directories there. The Path field will only show folders below the remote path being listed. If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt" will be "subfolder/file.txt", not "remote:path/subfolder/file.txt". -When used without --recursive the Path will always be the same as Name. +When used without `--recursive` the Path will always be the same as Name. If the directory is a bucket in a bucket-based backend, then "IsBucket" will be set to true. This key won't be present unless it is @@ -3451,7 +3592,7 @@ rclone lsjson remote:path [flags] ``` --dirs-only Show only directories in the listing - -M, --encrypted Show the encrypted names + --encrypted Show the encrypted names --files-only Show only files in the listing --hash Include hashes in the output (may take longer) --hash-type stringArray Show only this hash type (may be repeated) @@ -3539,10 +3680,10 @@ at all, then 1 PiB is set as both the total and the free size. To run rclone mount on Windows, you will need to download and install [WinFsp](http://www.secfs.net/winfsp/). -[WinFsp](https://github.com/billziss-gh/winfsp) is an open-source +[WinFsp](https://github.com/winfsp/winfsp) is an open-source Windows File System Proxy which makes it easy to write user space file systems for Windows. It provides a FUSE emulation layer which rclone -uses combination with [cgofuse](https://github.com/billziss-gh/cgofuse). +uses combination with [cgofuse](https://github.com/winfsp/cgofuse). Both of these packages are by Bill Zissimopoulos who was very helpful during the implementation of rclone mount for Windows. @@ -3692,7 +3833,7 @@ from Microsoft's Sysinternals suite, which has option `-s` to start processes as the SYSTEM account. Another alternative is to run the mount command from a Windows Scheduled Task, or a Windows Service, configured to run as the SYSTEM account. A third alternative is to use the -[WinFsp.Launcher infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)). +[WinFsp.Launcher infrastructure](https://github.com/winfsp/winfsp/wiki/WinFsp-Service-Architecture)). Note that when running rclone as another user, it will not use the configuration file from your profile unless you tell it to with the [`--config`](https://rclone.org/docs/#config-config-file) option. @@ -3874,7 +4015,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -4031,6 +4172,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -4071,7 +4244,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -4083,7 +4256,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -4100,28 +4273,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -4169,7 +4349,7 @@ rclone mount remote:path /path/to/mountpoint [flags] --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) @@ -4177,6 +4357,8 @@ rclone mount remote:path /path/to/mountpoint [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -4206,7 +4388,7 @@ directory named dest:path. This can be used to rename files or upload single files to other than their existing name. If the source is a directory then it acts exactly -like the move command. +like the [move](https://rclone.org/commands/rclone_move/) command. So @@ -4267,7 +4449,8 @@ builds an in memory representation. rclone ncdu can be used during this scanning phase and you will see it building up the directory structure as it goes along. -Here are the keys - press '?' to toggle the help on and off +You can interact with the user interface using key presses, +press '?' to toggle the help on and off. The supported keys are: ↑,↓ or k,j to Move →,l to enter @@ -4278,19 +4461,41 @@ Here are the keys - press '?' to toggle the help on and off u toggle human-readable format n,s,C,A sort by name,size,count,average size d delete file/directory + v select file/directory + V enter visual select mode + D delete selected files/directories y copy current path to clipboard Y display current path - ^L refresh screen + ^L refresh screen (fix screen corruption) ? to toggle help on and off - q/ESC/c-C to quit + q/ESC/^c to quit + +Listed files/directories may be prefixed by a one-character flag, +some of them combined with a description in brackes at end of line. +These flags have the following meaning: + + e means this is an empty directory, i.e. contains no files (but + may contain empty subdirectories) + ~ means this is a directory where some of the files (possibly in + subdirectories) have unknown size, and therefore the directory + size may be underestimated (and average size inaccurate, as it + is average of the files with known sizes). + . means an error occurred while reading a subdirectory, and + therefore the directory size may be underestimated (and average + size inaccurate) + ! means an error occurred while reading this directory This an homage to the [ncdu tool](https://dev.yorhel.nl/ncdu) but for rclone remotes. It is missing lots of features at the moment but is useful as it stands. -Note that it might take some time to delete big files/folders. The +Note that it might take some time to delete big files/directories. The UI won't respond in the meantime since the deletion is done synchronously. +For a non-interactive listing of the remote, see the +[tree](https://rclone.org/commands/rclone_tree/) command. To just get the total size of +the remote you can also use the [size](https://rclone.org/commands/rclone_size/) command. + ``` rclone ncdu remote:path [flags] @@ -4329,7 +4534,7 @@ This command can also accept a password through STDIN instead of an argument by passing a hyphen as an argument. This will use the first line of STDIN as the password not including the trailing newline. -echo "secretpassword" | rclone obscure - + echo "secretpassword" | rclone obscure - If there is no data on STDIN to read, rclone obscure will default to obfuscating the hyphen itself. @@ -4362,26 +4567,26 @@ Run a command against a running rclone. -This runs a command against a running rclone. Use the --url flag to +This runs a command against a running rclone. Use the `--url` flag to specify an non default URL to connect on. This can be either a ":port" which is taken to mean "http://localhost:port" or a "host:port" which is taken to mean "http://host:port" -A username and password can be passed in with --user and --pass. +A username and password can be passed in with `--user` and `--pass`. -Note that --rc-addr, --rc-user, --rc-pass will be read also for --url, ---user, --pass. +Note that `--rc-addr`, `--rc-user`, `--rc-pass` will be read also for +`--url`, `--user`, `--pass`. Arguments should be passed in as parameter=value. The result will be returned as a JSON object by default. -The --json parameter can be used to pass in a JSON blob as an input +The `--json` parameter can be used to pass in a JSON blob as an input instead of key=value arguments. This is the only way of passing in more complicated values. -The -o/--opt option can be used to set a key "opt" with key, value -options in the form "-o key=value" or "-o key". It can be repeated as +The `-o`/`--opt` option can be used to set a key "opt" with key, value +options in the form `-o key=value` or `-o key`. It can be repeated as many times as required. This is useful for rc commands which take the "opt" parameter which by convention is a dictionary of strings. @@ -4392,7 +4597,7 @@ Will place this in the "opt" value {"key":"value", "key2","") -The -a/--arg option can be used to set strings in the "arg" value. It +The `-a`/`--arg` option can be used to set strings in the "arg" value. It can be repeated as many times as required. This is useful for rc commands which take the "arg" parameter which by convention is a list of strings. @@ -4403,13 +4608,13 @@ Will place this in the "arg" value ["value", "value2"] -Use --loopback to connect to the rclone instance running "rclone rc". +Use `--loopback` to connect to the rclone instance running `rclone rc`. This is very useful for testing commands without having to run an rclone rc server, e.g.: rclone rc --loopback operations/about fs=/ -Use "rclone rc" to see a list of all possible commands. +Use `rclone rc` to see a list of all possible commands. ``` rclone rc commands parameter [flags] @@ -4460,11 +4665,11 @@ must fit into RAM. The cutoff needs to be small enough to adhere the limits of your remote, please see there. Generally speaking, setting this cutoff too high will decrease your performance. -Use the |--size| flag to preallocate the file in advance at the remote end +Use the `--size` flag to preallocate the file in advance at the remote end and actually stream it, even if remote backend doesn't support streaming. -|--size| should be the exact size of the input stream in bytes. If the -size of the stream is different in length to the |--size| passed in +`--size` should be the exact size of the input stream in bytes. If the +size of the stream is different in length to the `--size` passed in then the transfer will likely fail. Note that the upload can also not be retried because the data is @@ -4535,15 +4740,16 @@ that only contain empty directories), that it finds under the path. The root path itself will also be removed if it is empty, unless you supply the `--leave-root` flag. -Use command `rmdir` to delete just the empty directory -given by path, not recurse. +Use command [rmdir](https://rclone.org/commands/rclone_rmdir/) to delete just the empty +directory given by path, not recurse. This is useful for tidying up remotes that rclone has left a lot of -empty directories in. For example the `delete` command will -delete files but leave the directory structure (unless used with -option `--rmdirs`). +empty directories in. For example the [delete](https://rclone.org/commands/rclone_delete/) +command will delete files but leave the directory structure (unless +used with option `--rmdirs`). -To delete a path and any objects in it, use `purge` command. +To delete a path and any objects in it, use [purge](https://rclone.org/commands/rclone_purge/) +command. ``` @@ -4646,8 +4852,8 @@ Serve a remote over a protocol. ## Synopsis -rclone serve is used to serve a remote over a given protocol. This -command requires the use of a subcommand to specify the protocol, e.g. +Serve a remote over a given protocol. Requires the use of a +subcommand to specify the protocol, e.g. rclone serve http remote: @@ -4675,7 +4881,7 @@ See the [global flags page](https://rclone.org/flags/) for global options not li * [rclone serve http](https://rclone.org/commands/rclone_serve_http/) - Serve the remote over HTTP. * [rclone serve restic](https://rclone.org/commands/rclone_serve_restic/) - Serve the remote for restic's REST API. * [rclone serve sftp](https://rclone.org/commands/rclone_serve_sftp/) - Serve the remote over SFTP. -* [rclone serve webdav](https://rclone.org/commands/rclone_serve_webdav/) - Serve remote:path over webdav. +* [rclone serve webdav](https://rclone.org/commands/rclone_serve_webdav/) - Serve remote:path over WebDAV. # rclone serve dlna @@ -4683,14 +4889,16 @@ Serve remote:path over DLNA ## Synopsis -rclone serve dlna is a DLNA media server for media stored in an rclone remote. Many -devices, such as the Xbox and PlayStation, can automatically discover this server in the LAN -and play audio/video from it. VLC is also supported. Service discovery uses UDP multicast -packets (SSDP) and will thus only work on LANs. +Run a DLNA media server for media stored in an rclone remote. Many +devices, such as the Xbox and PlayStation, can automatically discover +this server in the LAN and play audio/video from it. VLC is also +supported. Service discovery uses UDP multicast packets (SSDP) and +will thus only work on LANs. -Rclone will list all files present in the remote, without filtering based on media formats or -file extensions. Additionally, there is no media transcoding support. This means that some -players might show files that they are not able to play back correctly. +Rclone will list all files present in the remote, without filtering +based on media formats or file extensions. Additionally, there is no +media transcoding support. This means that some players might show +files that they are not able to play back correctly. ## Server options @@ -4723,7 +4931,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -4880,6 +5088,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -4920,7 +5160,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -4932,7 +5172,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -4949,28 +5189,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -5004,7 +5251,7 @@ rclone serve dlna remote:path [flags] --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) @@ -5012,6 +5259,8 @@ rclone serve dlna remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -5091,7 +5340,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -5248,6 +5497,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -5288,7 +5569,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -5300,7 +5581,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -5317,28 +5598,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -5389,7 +5677,7 @@ rclone serve docker [flags] --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --socket-addr string Address or absolute path (default: /run/docker/plugins/rclone.sock) --socket-gid int GID for unix socket (default: current process GID) (default 1000) --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) @@ -5399,6 +5687,8 @@ rclone serve docker [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -5423,9 +5713,9 @@ Serve remote:path over FTP. ## Synopsis -rclone serve ftp implements a basic ftp server to serve the -remote over FTP protocol. This can be viewed with a ftp client -or you can make a remote of type ftp to read and write it. +Run a basic FTP server to serve a remote over FTP protocol. +This can be viewed with a FTP client or you can make a remote of +type FTP to read and write it. ## Server options @@ -5461,7 +5751,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -5618,6 +5908,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -5658,7 +5980,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -5670,7 +5992,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -5687,28 +6009,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -5827,7 +6156,7 @@ rclone serve ftp remote:path [flags] --passive-port string Passive port range to use (default "30000-32000") --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --public-ip string Public IP address to advertise for passive connections - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication (default "anonymous") @@ -5836,6 +6165,8 @@ rclone serve ftp remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -5857,59 +6188,59 @@ Serve the remote over HTTP. ## Synopsis -rclone serve http implements a basic web server to serve the remote -over HTTP. This can be viewed in a web browser or you can make a -remote of type http read from it. +Run a basic web server to serve a remote over HTTP. +This can be viewed in a web browser or you can make a remote of type +http read from it. -You can use the filter flags (e.g. --include, --exclude) to control what +You can use the filter flags (e.g. `--include`, `--exclude`) to control what is served. -The server will log errors. Use -v to see access logs. +The server will log errors. Use `-v` to see access logs. ---bwlimit will be respected for file transfers. Use --stats to +`--bwlimit` will be respected for file transfers. Use `--stats` to control the stats printing. ## Server options -Use --addr to specify which IP address and port the server should -listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all +Use `--addr` to specify which IP address and port the server should +listen on, eg `--addr 1.2.3.4:8000` or `--addr :8080` to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. -If you set --addr to listen on a public or LAN accessible IP address +If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. ---server-read-timeout and --server-write-timeout can be used to +`--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. ---max-header-bytes controls the maximum number of bytes the server will +`--max-header-bytes` controls the maximum number of bytes the server will accept in the HTTP header. ---baseurl controls the URL prefix that rclone serves from. By default -rclone will serve from the root. If you used --baseurl "/rclone" then +`--baseurl` controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used `--baseurl "/rclone"` then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically -inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", ---baseurl "/rclone" and --baseurl "/rclone/" are all treated +inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, +`--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. ### SSL/TLS By default this will serve over http. If you want you can serve over -https. You will need to supply the --cert and --key flags. If you -wish to do client side certificate validation then you will need to -supply --client-ca also. +https. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. ---cert should be a either a PEM encoded certificate or a concatenation -of that with the CA certificate. --key should be the PEM encoded -private key and --client-ca should be the PEM encoded client +`--cert` should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client certificate authority certificate. ### Template ---template allows a user to specify a custom markup template for http -and webdav serve functions. The server exports the following markup +`--template` allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: | Parameter | Description | @@ -5936,9 +6267,9 @@ to be used within the template to server pages: By default this will serve files without needing a login. You can either use an htpasswd file which can take lots of users, or -set a single username and password with the --user and --pass flags. +set a single username and password with the `--user` and `--pass` flags. -Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is +Use `--htpasswd /path/to/htpasswd` to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. @@ -5950,9 +6281,9 @@ To create an htpasswd file: The password file can be updated while rclone is running. -Use --realm to set the authentication realm. +Use `--realm` to set the authentication realm. -Use --salt to change the password hashing salt from the default. +Use `--salt` to change the password hashing salt from the default. ## VFS - Virtual File System @@ -5972,7 +6303,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -6129,6 +6460,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -6169,7 +6532,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -6181,7 +6544,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -6198,28 +6561,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -6258,7 +6628,7 @@ rclone serve http remote:path [flags] --no-seek Don't allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --realm string Realm for authentication --salt string Password hashing salt (default "dlPL2MqE") --server-read-timeout duration Timeout for server reading data (default 1h0m0s) @@ -6272,6 +6642,8 @@ rclone serve http remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -6293,8 +6665,8 @@ Serve the remote for restic's REST API. ## Synopsis -rclone serve restic implements restic's REST backend API -over HTTP. This allows restic to use rclone as a data storage +Run a basic web server to serve a remove over restic's REST backend +API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly. [Restic](https://restic.net/) is a command-line program for doing @@ -6302,8 +6674,8 @@ backups. The server will log errors. Use -v to see access logs. ---bwlimit will be respected for file transfers. Use --stats to -control the stats printing. +`--bwlimit` will be respected for file transfers. +Use `--stats` to control the stats printing. ## Setting up rclone for use by restic ### @@ -6322,11 +6694,11 @@ Where you can replace "backup" in the above by whatever path in the remote you wish to use. By default this will serve on "localhost:8080" you can change this -with use of the "--addr" flag. +with use of the `--addr` flag. You might wish to start this server on boot. -Adding --cache-objects=false will cause rclone to stop caching objects +Adding `--cache-objects=false` will cause rclone to stop caching objects returned from the List call. Caching is normally desirable as it speeds up downloading objects, saves transactions and uses very little memory. @@ -6372,36 +6744,36 @@ these **must** end with /. Eg ### Private repositories #### -The "--private-repos" flag can be used to limit users to repositories starting +The`--private-repos` flag can be used to limit users to repositories starting with a path of `//`. ## Server options -Use --addr to specify which IP address and port the server should -listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all -IPs. By default it only listens on localhost. You can use port +Use `--addr` to specify which IP address and port the server should +listen on, e.g. `--addr 1.2.3.4:8000` or `--addr :8080` to +listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. -If you set --addr to listen on a public or LAN accessible IP address +If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. ---server-read-timeout and --server-write-timeout can be used to +`--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. ---max-header-bytes controls the maximum number of bytes the server will +`--max-header-bytes` controls the maximum number of bytes the server will accept in the HTTP header. ---baseurl controls the URL prefix that rclone serves from. By default -rclone will serve from the root. If you used --baseurl "/rclone" then +`--baseurl` controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used `--baseurl "/rclone"` then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically -inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", ---baseurl "/rclone" and --baseurl "/rclone/" are all treated +inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, +`--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. ---template allows a user to specify a custom markup template for http -and webdav serve functions. The server exports the following markup +`--template` allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: | Parameter | Description | @@ -6428,9 +6800,9 @@ to be used within the template to server pages: By default this will serve files without needing a login. You can either use an htpasswd file which can take lots of users, or -set a single username and password with the --user and --pass flags. +set a single username and password with the `--user` and `--pass` flags. -Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is +Use `--htpasswd /path/to/htpasswd` to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. @@ -6442,18 +6814,18 @@ To create an htpasswd file: The password file can be updated while rclone is running. -Use --realm to set the authentication realm. +Use `--realm` to set the authentication realm. ### SSL/TLS -By default this will serve over http. If you want you can serve over -https. You will need to supply the --cert and --key flags. If you -wish to do client side certificate validation then you will need to -supply --client-ca also. +By default this will serve over HTTP. If you want you can serve over +HTTPS. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. ---cert should be either a PEM encoded certificate or a concatenation -of that with the CA certificate. --key should be the PEM encoded -private key and --client-ca should be the PEM encoded client +`--cert` should be either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client certificate authority certificate. @@ -6496,21 +6868,21 @@ Serve the remote over SFTP. ## Synopsis -rclone serve sftp implements an SFTP server to serve the remote -over SFTP. This can be used with an SFTP client or you can make a -remote of type sftp to use with it. +Run a SFTP server to serve a remote over SFTP. This can be used +with an SFTP client or you can make a remote of type sftp to use with it. -You can use the filter flags (e.g. --include, --exclude) to control what +You can use the filter flags (e.g. `--include`, `--exclude`) to control what is served. -The server will log errors. Use -v to see access logs. +The server will log errors. Use `-v` to see access logs. ---bwlimit will be respected for file transfers. Use --stats to -control the stats printing. +`--bwlimit` will be respected for file transfers. +Use `--stats` to control the stats printing. -You must provide some means of authentication, either with --user/--pass, -an authorized keys file (specify location with --authorized-keys - the -default is the same as ssh), an --auth-proxy, or set the --no-auth flag for no +You must provide some means of authentication, either with +`--user`/`--pass`, an authorized keys file (specify location with +`--authorized-keys` - the default is the same as ssh), an +`--auth-proxy`, or set the `--no-auth` flag for no authentication when logging in. Note that this also implements a small number of shell commands so @@ -6518,30 +6890,30 @@ that it can provide md5sum/sha1sum/df information for the rclone sftp backend. This means that is can support SHA1SUMs, MD5SUMs and the about command when paired with the rclone sftp backend. -If you don't supply a host --key then rclone will generate rsa, ecdsa +If you don't supply a host `--key` then rclone will generate rsa, ecdsa and ed25519 variants, and cache them for later use in rclone's cache -directory (see "rclone help flags cache-dir") in the "serve-sftp" +directory (see `rclone help flags cache-dir`) in the "serve-sftp" directory. By default the server binds to localhost:2022 - if you want it to be -reachable externally then supply "--addr :2022" for example. +reachable externally then supply `--addr :2022` for example. -Note that the default of "--vfs-cache-mode off" is fine for the rclone +Note that the default of `--vfs-cache-mode off` is fine for the rclone sftp backend, but it may not be with other SFTP clients. -If --stdio is specified, rclone will serve SFTP over stdio, which can +If `--stdio` is specified, rclone will serve SFTP over stdio, which can be used with sshd via ~/.ssh/authorized_keys, for example: restrict,command="rclone serve sftp --stdio ./photos" ssh-rsa ... -On the client you need to set "--transfers 1" when using --stdio. +On the client you need to set `--transfers 1` when using `--stdio`. Otherwise multiple instances of the rclone server are started by OpenSSH which can lead to "corrupted on transfer" errors. This is the case because the client chooses indiscriminately which server to send commands to while the servers all have different views of the state of the filing system. The "restrict" in authorized_keys prevents SHA1SUMs and MD5SUMs from beeing -used. Omitting "restrict" and using --sftp-path-override to enable +used. Omitting "restrict" and using `--sftp-path-override` to enable checksumming is possible but less secure and you could use the SFTP server provided by OpenSSH in this case. @@ -6564,7 +6936,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -6721,6 +7093,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -6761,7 +7165,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -6773,7 +7177,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -6790,28 +7194,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -6929,7 +7340,7 @@ rclone serve sftp remote:path [flags] --no-seek Don't allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --stdio Run an sftp server on run stdin/stdout --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) @@ -6939,6 +7350,8 @@ rclone serve sftp remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -6956,17 +7369,15 @@ See the [global flags page](https://rclone.org/flags/) for global options not li # rclone serve webdav -Serve remote:path over webdav. +Serve remote:path over WebDAV. ## Synopsis +Run a basic WebDAV server to serve a remote over HTTP via the +WebDAV protocol. This can be viewed with a WebDAV client, through a web +browser, or you can make a remote of type WebDAV to read and write it. -rclone serve webdav implements a basic webdav server to serve the -remote over HTTP via the webdav protocol. This can be viewed with a -webdav client, through a web browser, or you can make a remote of -type webdav to read and write it. - -## Webdav options +## WebDAV options ### --etag-hash @@ -6975,38 +7386,37 @@ based on the ModTime and Size of the object. If this flag is set to "auto" then rclone will choose the first supported hash on the backend or you can use a named hash such as -"MD5" or "SHA-1". - -Use "rclone hashsum" to see the full list. +"MD5" or "SHA-1". Use the [hashsum](https://rclone.org/commands/rclone_hashsum/) command +to see the full list. ## Server options -Use --addr to specify which IP address and port the server should -listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all -IPs. By default it only listens on localhost. You can use port +Use `--addr` to specify which IP address and port the server should +listen on, e.g. `--addr 1.2.3.4:8000` or `--addr :8080` to +listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. -If you set --addr to listen on a public or LAN accessible IP address +If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. ---server-read-timeout and --server-write-timeout can be used to +`--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. ---max-header-bytes controls the maximum number of bytes the server will +`--max-header-bytes` controls the maximum number of bytes the server will accept in the HTTP header. ---baseurl controls the URL prefix that rclone serves from. By default -rclone will serve from the root. If you used --baseurl "/rclone" then +`--baseurl` controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used `--baseurl "/rclone"` then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically -inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", ---baseurl "/rclone" and --baseurl "/rclone/" are all treated +inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, +`--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. ---template allows a user to specify a custom markup template for http -and webdav serve functions. The server exports the following markup +`--template` allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: | Parameter | Description | @@ -7033,9 +7443,9 @@ to be used within the template to server pages: By default this will serve files without needing a login. You can either use an htpasswd file which can take lots of users, or -set a single username and password with the --user and --pass flags. +set a single username and password with the `--user` and `--pass` flags. -Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is +Use `--htpasswd /path/to/htpasswd` to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. @@ -7047,18 +7457,18 @@ To create an htpasswd file: The password file can be updated while rclone is running. -Use --realm to set the authentication realm. +Use `--realm` to set the authentication realm. ### SSL/TLS -By default this will serve over http. If you want you can serve over -https. You will need to supply the --cert and --key flags. If you -wish to do client side certificate validation then you will need to -supply --client-ca also. +By default this will serve over HTTP. If you want you can serve over +HTTPS. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. ---cert should be either a PEM encoded certificate or a concatenation -of that with the CA certificate. --key should be the PEM encoded -private key and --client-ca should be the PEM encoded client +`--cert` should be either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client certificate authority certificate. ## VFS - Virtual File System @@ -7079,7 +7489,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -7236,6 +7646,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -7276,7 +7718,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -7288,7 +7730,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -7305,28 +7747,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -7449,7 +7898,7 @@ rclone serve webdav remote:path [flags] --no-seek Don't allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --realm string Realm for authentication (default "rclone") --server-read-timeout duration Timeout for server reading data (default 1h0m0s) --server-write-timeout duration Timeout for server writing data (default 1h0m0s) @@ -7462,6 +7911,8 @@ rclone serve webdav remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -7555,6 +8006,7 @@ See the [global flags page](https://rclone.org/flags/) for global options not li * [rclone test changenotify](https://rclone.org/commands/rclone_test_changenotify/) - Log any change notify requests for the remote passed in. * [rclone test histogram](https://rclone.org/commands/rclone_test_histogram/) - Makes a histogram of file name characters. * [rclone test info](https://rclone.org/commands/rclone_test_info/) - Discovers file name or other limitations for paths. +* [rclone test makefile](https://rclone.org/commands/rclone_test_makefile/) - Make files with random contents of the size given * [rclone test makefiles](https://rclone.org/commands/rclone_test_makefiles/) - Make a random file hierarchy in a directory * [rclone test memory](https://rclone.org/commands/rclone_test_memory/) - Load all the objects at remote:path into memory and report memory stats. @@ -7645,6 +8097,32 @@ See the [global flags page](https://rclone.org/flags/) for global options not li * [rclone test](https://rclone.org/commands/rclone_test/) - Run a test command +# rclone test makefile + +Make files with random contents of the size given + +``` +rclone test makefile []+ [flags] +``` + +## Options + +``` + --ascii Fill files with random ASCII printable bytes only + --chargen Fill files with a ASCII chargen pattern + -h, --help help for makefile + --pattern Fill files with a periodic pattern + --seed int Seed for the random number generator (0 for random) (default 1) + --sparse Make the files sparse (appear to be filled with ASCII 0x00) + --zero Fill files with ASCII 0x00 +``` + +See the [global flags page](https://rclone.org/flags/) for global options not listed here. + +## SEE ALSO + +* [rclone test](https://rclone.org/commands/rclone_test/) - Run a test command + # rclone test makefiles Make a random file hierarchy in a directory @@ -7656,6 +8134,8 @@ rclone test makefiles [flags] ## Options ``` + --ascii Fill files with random ASCII printable bytes only + --chargen Fill files with a ASCII chargen pattern --files int Number of files to create (default 1000) --files-per-directory int Average number of files per directory (default 10) -h, --help help for makefiles @@ -7663,7 +8143,10 @@ rclone test makefiles [flags] --max-name-length int Maximum size of file names (default 12) --min-file-size SizeSuffix Minimum size of file to create --min-name-length int Minimum size of file names (default 4) + --pattern Fill files with a periodic pattern --seed int Seed for the random number generator (0 for random) (default 1) + --sparse Make the files sparse (appear to be filled with ASCII 0x00) + --zero Fill files with ASCII 0x00 ``` See the [global flags page](https://rclone.org/flags/) for global options not listed here. @@ -7764,12 +8247,16 @@ For example 1 directories, 5 files You can use any of the filtering options with the tree command (e.g. ---include and --exclude). You can also use --fast-list. +`--include` and `--exclude`. You can also use `--fast-list`. The tree command has many options for controlling the listing which -are compatible with the tree command. Note that not all of them have +are compatible with the tree command, for example you can include file +sizes with `--size`. Note that not all of them have short options as they conflict with rclone's short options. +For a more interactive navigation of the remote see the +[ncdu](https://rclone.org/commands/rclone_ncdu/) command. + ``` rclone tree remote:path [flags] @@ -8092,6 +8579,126 @@ This can be used when scripting to make aged backups efficiently, e.g. rclone sync -i remote:current-backup remote:previous-backup rclone sync -i /path/to/files remote:current-backup +## Metadata support {#metadata} + +Metadata is data about a file which isn't the contents of the file. +Normally rclone only preserves the modification time and the content +(MIME) type where possible. + +Rclone supports preserving all the available metadata on files (not +directories) when using the `--metadata` or `-M` flag. + +Exactly what metadata is supported and what that support means depends +on the backend. Backends that support metadata have a metadata section +in their docs and are listed in the [features table](https://rclone.org/overview/#features) +(Eg [local](https://rclone.org/local/#metadata), [s3](/s3/#metadata)) + +Rclone only supports a one-time sync of metadata. This means that +metadata will be synced from the source object to the destination +object only when the source object has changed and needs to be +re-uploaded. If the metadata subsequently changes on the source object +without changing the object itself then it won't be synced to the +destination object. This is in line with the way rclone syncs +`Content-Type` without the `--metadata` flag. + +Using `--metadata` when syncing from local to local will preserve file +attributes such as file mode, owner, extended attributes (not +Windows). + +Note that arbitrary metadata may be added to objects using the +`--metadata-set key=value` flag when the object is first uploaded. +This flag can be repeated as many times as necessary. + +### Types of metadata + +Metadata is divided into two type. System metadata and User metadata. + +Metadata which the backend uses itself is called system metadata. For +example on the local backend the system metadata `uid` will store the +user ID of the file when used on a unix based platform. + +Arbitrary metadata is called user metadata and this can be set however +is desired. + +When objects are copied from backend to backend, they will attempt to +interpret system metadata if it is supplied. Metadata may change from +being user metadata to system metadata as objects are copied between +different backends. For example copying an object from s3 sets the +`content-type` metadata. In a backend which understands this (like +`azureblob`) this will become the Content-Type of the object. In a +backend which doesn't understand this (like the `local` backend) this +will become user metadata. However should the local object be copied +back to s3, the Content-Type will be set correctly. + +### Metadata framework + +Rclone implements a metadata framework which can read metadata from an +object and write it to the object when (and only when) it is being +uploaded. + +This metadata is stored as a dictionary with string keys and string +values. + +There are some limits on the names of the keys (these may be clarified +further in the future). + +- must be lower case +- may be `a-z` `0-9` containing `.` `-` or `_` +- length is backend dependent + +Each backend can provide system metadata that it understands. Some +backends can also store arbitrary user metadata. + +Where possible the key names are standardized, so, for example, it is +possible to copy object metadata from s3 to azureblob for example and +metadata will be translated apropriately. + +Some backends have limits on the size of the metadata and rclone will +give errors on upload if they are exceeded. + +### Metadata preservation + +The goal of the implementation is to + +1. Preserve metadata if at all possible +2. Interpret metadata if at all possible + +The consequences of 1 is that you can copy an S3 object to a local +disk then back to S3 losslessly. Likewise you can copy a local file +with file attributes and xattrs from local disk to s3 and back again +losslessly. + +The consequence of 2 is that you can copy an S3 object with metadata +to Azureblob (say) and have the metadata appear on the Azureblob +object also. + +### Standard system metadata + +Here is a table of standard system metadata which, if appropriate, a +backend may implement. + +| key | description | example | +|---------------------|-------------|---------| +| mode | File type and mode: octal, unix style | 0100664 | +| uid | User ID of owner: decimal number | 500 | +| gid | Group ID of owner: decimal number | 500 | +| rdev | Device ID (if special file) => hexadecimal | 0 | +| atime | Time of last access: RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | +| mtime | Time of last modification: RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | +| btime | Time of file creation (birth): RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | +| cache-control | Cache-Control header | no-cache | +| content-disposition | Content-Disposition header | inline | +| content-encoding | Content-Encoding header | gzip | +| content-language | Content-Language header | en-US | +| content-type | Content-Type header | text/plain | + +The metadata keys `mtime` and `content-type` will take precedence if +supplied in the metadata over reading the `Content-Type` or +modification time of the source object. + +Hashes are not included in system metadata as there is a well defined +way of reading those already. + Options ------- @@ -8306,12 +8913,22 @@ objects to transfer is held in memory before the transfers start. ### --checkers=N ### -The number of checkers to run in parallel. Checkers do the equality -checking of files during a sync. For some storage systems (e.g. S3, -Swift, Dropbox) this can take a significant amount of time so they are -run in parallel. +Originally controlling just the number of file checkers to run in parallel, +e.g. by `rclone copy`. Now a fairly universal parallelism control +used by `rclone` in several places. + +Note: checkers do the equality checking of files during a sync. +For some storage systems (e.g. S3, Swift, Dropbox) this can take +a significant amount of time so they are run in parallel. -The default is to run 8 checkers in parallel. +The default is to run 8 checkers in parallel. However, in case +of slow-reacting backends you may need to lower (rather than increase) +this default by setting `--checkers` to 4 or less threads. This is +especially advised if you are experiencing backend server crashes +during file checking phase (e.g. on subsequent or top-up backups +where little or no file copying is done and checking takes up +most of the time). Increase this setting only with utmost care, +while monitoring your server health and file checking throughput. ### -c, --checksum ### @@ -8456,7 +9073,9 @@ See `--compare-dest` and `--backup-dir`. ### --dedupe-mode MODE ### -Mode to run dedupe command in. One of `interactive`, `skip`, `first`, `newest`, `oldest`, `rename`. The default is `interactive`. See the dedupe command for more information as to what these options mean. +Mode to run dedupe command in. One of `interactive`, `skip`, `first`, +`newest`, `oldest`, `rename`. The default is `interactive`. +See the dedupe command for more information as to what these options mean. ### --disable FEATURE,FEATURE,... ### @@ -8608,22 +9227,22 @@ unit prefix appended to the value (e.g. `9.762Ki`), while in more textual output the full unit is shown (e.g. `9.762 KiB`). For counts the SI standard notation is used, e.g. prefix `k` for kilo. Used with file counts, `1k` means 1000 files. -The various [list](commands/rclone_ls/) commands output raw numbers by default. +The various [list](https://rclone.org/commands/rclone_ls/) commands output raw numbers by default. Option `--human-readable` will make them output values in human-readable format instead (with the short unit prefix). -The [about](commands/rclone_about/) command outputs human-readable by default, +The [about](https://rclone.org/commands/rclone_about/) command outputs human-readable by default, with a command-specific option `--full` to output the raw numbers instead. -Command [size](commands/rclone_size/) outputs both human-readable and raw numbers +Command [size](https://rclone.org/commands/rclone_size/) outputs both human-readable and raw numbers in the same output. -The [tree](commands/rclone_tree/) command also considers `--human-readable`, but +The [tree](https://rclone.org/commands/rclone_tree/) command also considers `--human-readable`, but it will not use the exact same notation as the other commands: It rounds to one decimal, and uses single letter suffix, e.g. `K` instead of `Ki`. The reason for this is that it relies on an external library. -The interactive command [ncdu](commands/rclone_ncdu/) shows human-readable by +The interactive command [ncdu](https://rclone.org/commands/rclone_ncdu/) shows human-readable by default, and responds to key `u` for toggling human-readable format. ### --ignore-case-sync ### @@ -8754,7 +9373,12 @@ have a signal to rotate logs. ### --log-format LIST ### -Comma separated list of log format options. Accepted options are `date`, `time`, `microseconds`, `pid`, `longfile`, `shortfile`, `UTC`. Any other keywords will be silently ignored. `pid` will tag log messages with process identifier which useful with `rclone mount --daemon`. Other accepted options are explained in the [go documentation](https://pkg.go.dev/log#pkg-constants). The default log format is "`date`,`time`". +Comma separated list of log format options. Accepted options are `date`, +`time`, `microseconds`, `pid`, `longfile`, `shortfile`, `UTC`. Any other +keywords will be silently ignored. `pid` will tag log messages with process +identifier which useful with `rclone mount --daemon`. Other accepted +options are explained in the [go documentation](https://pkg.go.dev/log#pkg-constants). +The default log format is "`date`,`time`". ### --log-level LEVEL ### @@ -8856,6 +9480,18 @@ When the limit is reached all transfers will stop immediately. Rclone will exit with exit code 8 if the transfer limit is reached. +## --metadata / -M + +Setting this flag enables rclone to copy the metadata from the source +to the destination. For local backends this is ownership, permissions, +xattr etc. See the [#metadata](metadata section) for more info. + +### --metadata-set key=value + +Add metadata `key` = `value` when uploading. This can be repeated as +many times as required. See the [#metadata](metadata section) for more +info. + ### --cutoff-mode=hard|soft|cautious ### This modifies the behavior of `--max-transfer` @@ -9462,6 +10098,8 @@ of timeouts or bigger if you have lots of bandwidth and a fast remote. The default is to run 4 file transfers in parallel. +Look at --multi-thread-streams if you would like to control single file transfers. + ### -u, --update ### This forces rclone to skip any files which exist on the destination @@ -9533,6 +10171,9 @@ With `-vv` rclone will become very verbose telling you about every file it considers and transfers. Please send bug reports with a log with this setting. +When setting verbosity as an environment variable, use +`RCLONE_VERBOSE=1` or `RCLONE_VERBOSE=2` for `-v` and `-vv` respectively. + ### -V, --version ### Prints the version number @@ -9791,6 +10432,7 @@ For the filtering options * `--filter-from` * `--exclude` * `--exclude-from` + * `--exclude-if-present` * `--include` * `--include-from` * `--files-from` @@ -9898,6 +10540,10 @@ override the environment variable setting. Or to always use the trash in drive `--drive-use-trash`, set `RCLONE_DRIVE_USE_TRASH=true`. +Verbosity is slightly different, the environment variable +equivalent of `--verbose` or `-v` is `RCLONE_VERBOSE=1`, +or for `-vv`, `RCLONE_VERBOSE=2`. + The same parser is used for the options and the environment variables so they take exactly the same form. @@ -10079,6 +10725,27 @@ Now transfer it to the remote box (scp, cut paste, ftp, sftp, etc.) and place it in the correct place (use `rclone config file` on the remote box to find out where). +## Configuring using SSH Tunnel ## + +Linux and MacOS users can utilize SSH Tunnel to redirect the headless box port 53682 to local machine by using the following command: +``` +ssh -L localhost:53682:localhost:53682 username@remote_server +``` +Then on the headless box run `rclone` config and answer `Y` to the `Use +auto config?` question. + +``` +... +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes (default) +n) No +y/n> y +``` +Then copy and paste the auth url `http://127.0.0.1:53682/auth?state=xxxxxxxxxxxx` to the browser on your local machine, complete the auth and it is done. + # Filtering, includes and excludes Filter flags determine which files rclone `sync`, `move`, `ls`, `lsl`, @@ -10207,7 +10874,11 @@ The regular expressions used are as defined in the [Go regular expression reference](https://golang.org/pkg/regexp/syntax/). Regular expressions should be enclosed in `{{` `}}`. They will match only the last path segment if the glob doesn't start with `/` or the whole path -name if it does. +name if it does. Note that rclone does not attempt to parse the +supplied regular expression, meaning that using any regular expression +filter will prevent rclone from using [directory filter rules](#directory_filter), +as it will instead check every path against +the supplied regular expression(s). Here is how the `{{regexp}}` is transformed into an full regular expression to match the entire path: @@ -10323,7 +10994,7 @@ currently a means provided to pass regular expression filter options into rclone directly though character class filter rules contain character classes. [Go regular expression reference](https://golang.org/pkg/regexp/syntax/) -### How filter rules are applied to directories +### How filter rules are applied to directories {#directory_filter} Rclone commands are applied to path/file names not directories. The entire contents of a directory can be matched @@ -10339,10 +11010,14 @@ recurse into subdirectories. This potentially optimises access to a remote by avoiding listing unnecessary directories. Whether optimisation is desirable depends on the specific filter rules and source remote content. +If any [regular expression filters](#regexp) are in use, then no +directory recursion optimisation is possible, as rclone must check +every path against the supplied regular expression(s). + Directory recursion optimisation occurs if either: * A source remote does not support the rclone `ListR` primitive. local, -sftp, Microsoft OneDrive and WebDav do not support `ListR`. Google +sftp, Microsoft OneDrive and WebDAV do not support `ListR`. Google Drive and most bucket type storage do. [Full list](https://rclone.org/overview/#optional-features) * On other remotes (those that support `ListR`), if the rclone command is not naturally recursive, and @@ -10818,7 +11493,9 @@ Useful for debugging. The `--exclude-if-present` flag controls whether a directory is within the scope of an rclone command based on the presence of a -named file within it. +named file within it. The flag can be repeated to check for +multiple file names, presence of any of them will exclude the +directory. This flag has a priority over other filter flags. @@ -10832,8 +11509,6 @@ E.g. for the following directory structure: The command `rclone ls --exclude-if-present .ignore dir1` does not list `dir3`, `file3` or `.ignore`. -`--exclude-if-present` can only be used once in an rclone command. - ## Common pitfalls The most frequent filter support issues on @@ -10952,10 +11627,10 @@ If you have questions then please ask them on the [rclone forum](https://forum.r If rclone is run with the `--rc` flag then it starts an HTTP server which can be used to remote control rclone using its API. -You can either use the [rclone rc](#api-rc) command to access the API +You can either use the [rc](#api-rc) command to access the API or [use HTTP directly](#api-http). -If you just want to run a remote control then see the [rcd command](https://rclone.org/commands/rclone_rcd/). +If you just want to run a remote control then see the [rcd](https://rclone.org/commands/rclone_rcd/) command. ## Supported parameters @@ -11094,6 +11769,16 @@ use these methods. The alternative is to use `--rc-user` and Default Off. +### --rc-baseurl + +Prefix for URLs. + +Default is root + +### --rc-template + +User-specified template. + ## Accessing the remote control via the rclone rc command {#api-rc} Rclone itself implements the remote control protocol in its `rclone @@ -11479,7 +12164,7 @@ This takes the following parameters: - result - result to restart with - used with continue -See the [config create command](https://rclone.org/commands/rclone_config_create/) command for more information on the above. +See the [config create](https://rclone.org/commands/rclone_config_create/) command for more information on the above. **Authentication is required for this call.** @@ -11489,7 +12174,7 @@ Parameters: - name - name of remote to delete -See the [config delete command](https://rclone.org/commands/rclone_config_delete/) command for more information on the above. +See the [config delete](https://rclone.org/commands/rclone_config_delete/) command for more information on the above. **Authentication is required for this call.** @@ -11500,7 +12185,7 @@ Returns a JSON object: Where keys are remote names and values are the config parameters. -See the [config dump command](https://rclone.org/commands/rclone_config_dump/) command for more information on the above. +See the [config dump](https://rclone.org/commands/rclone_config_dump/) command for more information on the above. **Authentication is required for this call.** @@ -11510,7 +12195,7 @@ Parameters: - name - name of remote to get -See the [config dump command](https://rclone.org/commands/rclone_config_dump/) command for more information on the above. +See the [config dump](https://rclone.org/commands/rclone_config_dump/) command for more information on the above. **Authentication is required for this call.** @@ -11519,7 +12204,7 @@ See the [config dump command](https://rclone.org/commands/rclone_config_dump/) c Returns - remotes - array of remote names -See the [listremotes command](https://rclone.org/commands/rclone_listremotes/) command for more information on the above. +See the [listremotes](https://rclone.org/commands/rclone_listremotes/) command for more information on the above. **Authentication is required for this call.** @@ -11531,7 +12216,7 @@ This takes the following parameters: - parameters - a map of \{ "key": "value" \} pairs -See the [config password command](https://rclone.org/commands/rclone_config_password/) command for more information on the above. +See the [config password](https://rclone.org/commands/rclone_config_password/) command for more information on the above. **Authentication is required for this call.** @@ -11540,7 +12225,7 @@ See the [config password command](https://rclone.org/commands/rclone_config_pass Returns a JSON object: - providers - array of objects -See the [config providers command](https://rclone.org/commands/rclone_config_providers/) command for more information on the above. +See the [config providers](https://rclone.org/commands/rclone_config_providers/) command for more information on the above. **Authentication is required for this call.** @@ -11560,7 +12245,7 @@ This takes the following parameters: - result - result to restart with - used with continue -See the [config update command](https://rclone.org/commands/rclone_config_update/) command for more information on the above. +See the [config update](https://rclone.org/commands/rclone_config_update/) command for more information on the above. **Authentication is required for this call.** @@ -12006,7 +12691,7 @@ This takes the following parameters: The result is as returned from rclone about --json -See the [about command](https://rclone.org/commands/rclone_size/) command for more information on the above. +See the [about](https://rclone.org/commands/rclone_about/) command for more information on the above. **Authentication is required for this call.** @@ -12016,7 +12701,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" -See the [cleanup command](https://rclone.org/commands/rclone_cleanup/) command for more information on the above. +See the [cleanup](https://rclone.org/commands/rclone_cleanup/) command for more information on the above. **Authentication is required for this call.** @@ -12039,7 +12724,8 @@ This takes the following parameters: - remote - a path within that remote e.g. "dir" - url - string, URL to read from - autoFilename - boolean, set to true to retrieve destination file name from url -See the [copyurl command](https://rclone.org/commands/rclone_copyurl/) command for more information on the above. + +See the [copyurl](https://rclone.org/commands/rclone_copyurl/) command for more information on the above. **Authentication is required for this call.** @@ -12049,7 +12735,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" -See the [delete command](https://rclone.org/commands/rclone_delete/) command for more information on the above. +See the [delete](https://rclone.org/commands/rclone_delete/) command for more information on the above. **Authentication is required for this call.** @@ -12060,7 +12746,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the [deletefile command](https://rclone.org/commands/rclone_deletefile/) command for more information on the above. +See the [deletefile](https://rclone.org/commands/rclone_deletefile/) command for more information on the above. **Authentication is required for this call.** @@ -12074,46 +12760,103 @@ This returns info about the remote passed in; ``` { - // optional features and whether they are available or not - "Features": { - "About": true, - "BucketBased": false, - "CanHaveEmptyDirectories": true, - "CaseInsensitive": false, - "ChangeNotify": false, - "CleanUp": false, - "Copy": false, - "DirCacheFlush": false, - "DirMove": true, - "DuplicateFiles": false, - "GetTier": false, - "ListR": false, - "MergeDirs": false, - "Move": true, - "OpenWriterAt": true, - "PublicLink": false, - "Purge": true, - "PutStream": true, - "PutUnchecked": false, - "ReadMimeType": false, - "ServerSideAcrossConfigs": false, - "SetTier": false, - "SetWrapper": false, - "UnWrap": false, - "WrapFs": false, - "WriteMimeType": false - }, - // Names of hashes available - "Hashes": [ - "MD5", - "SHA-1", - "DropboxHash", - "QuickXorHash" - ], - "Name": "local", // Name as created - "Precision": 1, // Precision of timestamps in ns - "Root": "/", // Path as created - "String": "Local file system at /" // how the remote will appear in logs + // optional features and whether they are available or not + "Features": { + "About": true, + "BucketBased": false, + "BucketBasedRootOK": false, + "CanHaveEmptyDirectories": true, + "CaseInsensitive": false, + "ChangeNotify": false, + "CleanUp": false, + "Command": true, + "Copy": false, + "DirCacheFlush": false, + "DirMove": true, + "Disconnect": false, + "DuplicateFiles": false, + "GetTier": false, + "IsLocal": true, + "ListR": false, + "MergeDirs": false, + "MetadataInfo": true, + "Move": true, + "OpenWriterAt": true, + "PublicLink": false, + "Purge": true, + "PutStream": true, + "PutUnchecked": false, + "ReadMetadata": true, + "ReadMimeType": false, + "ServerSideAcrossConfigs": false, + "SetTier": false, + "SetWrapper": false, + "Shutdown": false, + "SlowHash": true, + "SlowModTime": false, + "UnWrap": false, + "UserInfo": false, + "UserMetadata": true, + "WrapFs": false, + "WriteMetadata": true, + "WriteMimeType": false + }, + // Names of hashes available + "Hashes": [ + "md5", + "sha1", + "whirlpool", + "crc32", + "sha256", + "dropbox", + "mailru", + "quickxor" + ], + "Name": "local", // Name as created + "Precision": 1, // Precision of timestamps in ns + "Root": "/", // Path as created + "String": "Local file system at /", // how the remote will appear in logs + // Information about the system metadata for this backend + "MetadataInfo": { + "System": { + "atime": { + "Help": "Time of last access", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "btime": { + "Help": "Time of file birth (creation)", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "gid": { + "Help": "Group ID of owner", + "Type": "decimal number", + "Example": "500" + }, + "mode": { + "Help": "File type and mode", + "Type": "octal, unix style", + "Example": "0100664" + }, + "mtime": { + "Help": "Time of last modification", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "rdev": { + "Help": "Device ID (if special file)", + "Type": "hexadecimal", + "Example": "1abc" + }, + "uid": { + "Help": "User ID of owner", + "Type": "decimal number", + "Example": "500" + } + }, + "Help": "Textual help string\n" + } } ``` @@ -12136,6 +12879,7 @@ This takes the following parameters: - noMimeType - If set don't show mime types - dirsOnly - If set only show directories - filesOnly - If set only show files + - metadata - If set return metadata of objects also - hashTypes - array of strings of hash types to show if showHash set Returns: @@ -12143,7 +12887,7 @@ Returns: - list - This is an array of objects as described in the lsjson command -See the [lsjson command](https://rclone.org/commands/rclone_lsjson/) for more information on the above and examples. +See the [lsjson](https://rclone.org/commands/rclone_lsjson/) command for more information on the above and examples. **Authentication is required for this call.** @@ -12154,7 +12898,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the [mkdir command](https://rclone.org/commands/rclone_mkdir/) command for more information on the above. +See the [mkdir](https://rclone.org/commands/rclone_mkdir/) command for more information on the above. **Authentication is required for this call.** @@ -12182,7 +12926,7 @@ Returns: - url - URL of the resource -See the [link command](https://rclone.org/commands/rclone_link/) command for more information on the above. +See the [link](https://rclone.org/commands/rclone_link/) command for more information on the above. **Authentication is required for this call.** @@ -12193,7 +12937,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the [purge command](https://rclone.org/commands/rclone_purge/) command for more information on the above. +See the [purge](https://rclone.org/commands/rclone_purge/) command for more information on the above. **Authentication is required for this call.** @@ -12204,7 +12948,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the [rmdir command](https://rclone.org/commands/rclone_rmdir/) command for more information on the above. +See the [rmdir](https://rclone.org/commands/rclone_rmdir/) command for more information on the above. **Authentication is required for this call.** @@ -12215,7 +12959,8 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" - leaveRoot - boolean, set to true not to delete the root -See the [rmdirs command](https://rclone.org/commands/rclone_rmdirs/) command for more information on the above. + +See the [rmdirs](https://rclone.org/commands/rclone_rmdirs/) command for more information on the above. **Authentication is required for this call.** @@ -12230,7 +12975,7 @@ Returns: - count - number of files - bytes - number of bytes in those files -See the [size command](https://rclone.org/commands/rclone_size/) command for more information on the above. +See the [size](https://rclone.org/commands/rclone_size/) command for more information on the above. **Authentication is required for this call.** @@ -12250,7 +12995,7 @@ The result is Note that if you are only interested in files then it is much more efficient to set the filesOnly flag in the options. -See the [lsjson command](https://rclone.org/commands/rclone_lsjson/) for more information on the above and examples. +See the [lsjson](https://rclone.org/commands/rclone_lsjson/) command for more information on the above and examples. **Authentication is required for this call.** @@ -12261,7 +13006,8 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" - each part in body represents a file to be uploaded -See the [uploadfile command](https://rclone.org/commands/rclone_uploadfile/) command for more information on the above. + +See the [uploadfile](https://rclone.org/commands/rclone_uploadfile/) command for more information on the above. **Authentication is required for this call.** @@ -12476,7 +13222,7 @@ This takes the following parameters: - createEmptySrcDirs - create empty src directories on destination if set -See the [copy command](https://rclone.org/commands/rclone_copy/) command for more information on the above. +See the [copy](https://rclone.org/commands/rclone_copy/) command for more information on the above. **Authentication is required for this call.** @@ -12490,7 +13236,7 @@ This takes the following parameters: - deleteEmptySrcDirs - delete empty src directories if set -See the [move command](https://rclone.org/commands/rclone_move/) command for more information on the above. +See the [move](https://rclone.org/commands/rclone_move/) command for more information on the above. **Authentication is required for this call.** @@ -12503,7 +13249,7 @@ This takes the following parameters: - createEmptySrcDirs - create empty src directories on destination if set -See the [sync command](https://rclone.org/commands/rclone_sync/) command for more information on the above. +See the [sync](https://rclone.org/commands/rclone_sync/) command for more information on the above. **Authentication is required for this call.** @@ -12854,47 +13600,49 @@ show through. Here is an overview of the major features of each cloud storage system. -| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | -| ---------------------------- |:-----------:|:-------:|:----------------:|:---------------:|:---------:| -| 1Fichier | Whirlpool | No | No | Yes | R | -| Akamai Netstorage | MD5, SHA256 | Yes | No | No | R | -| Amazon Drive | MD5 | No | Yes | No | R | -| Amazon S3 (or S3 compatible) | MD5 | Yes | No | No | R/W | -| Backblaze B2 | SHA1 | Yes | No | No | R/W | -| Box | SHA1 | Yes | Yes | No | - | -| Citrix ShareFile | MD5 | Yes | Yes | No | - | -| Dropbox | DBHASH ¹ | Yes | Yes | No | - | -| Enterprise File Fabric | - | Yes | Yes | No | R/W | -| FTP | - | No | No | No | - | -| Google Cloud Storage | MD5 | Yes | No | No | R/W | -| Google Drive | MD5 | Yes | No | Yes | R/W | -| Google Photos | - | No | No | Yes | R | -| HDFS | - | Yes | No | No | - | -| HTTP | - | No | No | No | R | -| Hubic | MD5 | Yes | No | No | R/W | -| Jottacloud | MD5 | Yes | Yes | No | R | -| Koofr | MD5 | No | Yes | No | - | -| Mail.ru Cloud | Mailru ⁶ | Yes | Yes | No | - | -| Mega | - | No | No | Yes | - | -| Memory | MD5 | Yes | No | No | - | -| Microsoft Azure Blob Storage | MD5 | Yes | No | No | R/W | -| Microsoft OneDrive | SHA1 ⁵ | Yes | Yes | No | R | -| OpenDrive | MD5 | Yes | Yes | Partial ⁸ | - | -| OpenStack Swift | MD5 | Yes | No | No | R/W | -| pCloud | MD5, SHA1 ⁷ | Yes | No | No | W | -| premiumize.me | - | No | Yes | No | R | -| put.io | CRC-32 | Yes | No | Yes | R | -| QingStor | MD5 | No | No | No | R/W | -| Seafile | - | No | No | No | - | -| SFTP | MD5, SHA1 ² | Yes | Depends | No | - | -| Sia | - | No | No | No | - | -| SugarSync | - | No | No | No | - | -| Storj | - | Yes | No | No | - | -| Uptobox | - | No | No | Yes | - | -| WebDAV | MD5, SHA1 ³ | Yes ⁴ | Depends | No | - | -| Yandex Disk | MD5 | Yes | No | No | R | -| Zoho WorkDrive | - | No | No | No | - | -| The local filesystem | All | Yes | Depends | No | - | +| Name | Hash | ModTime | Case Insensitive | Duplicate Files | MIME Type | Metadata | +| ---------------------------- |:----------------:|:-------:|:----------------:|:---------------:|:---------:|:--------:| +| 1Fichier | Whirlpool | - | No | Yes | R | - | +| Akamai Netstorage | MD5, SHA256 | R/W | No | No | R | - | +| Amazon Drive | MD5 | - | Yes | No | R | - | +| Amazon S3 (or S3 compatible) | MD5 | R/W | No | No | R/W | RWU | +| Backblaze B2 | SHA1 | R/W | No | No | R/W | - | +| Box | SHA1 | R/W | Yes | No | - | - | +| Citrix ShareFile | MD5 | R/W | Yes | No | - | - | +| Dropbox | DBHASH ¹ | R | Yes | No | - | - | +| Enterprise File Fabric | - | R/W | Yes | No | R/W | - | +| FTP | - | R/W ¹⁰ | No | No | - | - | +| Google Cloud Storage | MD5 | R/W | No | No | R/W | - | +| Google Drive | MD5 | R/W | No | Yes | R/W | - | +| Google Photos | - | - | No | Yes | R | - | +| HDFS | - | R/W | No | No | - | - | +| HiDrive | HiDrive ¹² | R/W | No | No | - | - | +| HTTP | - | R | No | No | R | - | +| Hubic | MD5 | R/W | No | No | R/W | - | +| Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU | +| Jottacloud | MD5 | R/W | Yes | No | R | - | +| Koofr | MD5 | - | Yes | No | - | - | +| Mail.ru Cloud | Mailru ⁶ | R/W | Yes | No | - | - | +| Mega | - | - | No | Yes | - | - | +| Memory | MD5 | R/W | No | No | - | - | +| Microsoft Azure Blob Storage | MD5 | R/W | No | No | R/W | - | +| Microsoft OneDrive | SHA1 ⁵ | R/W | Yes | No | R | - | +| OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | - | +| OpenStack Swift | MD5 | R/W | No | No | R/W | - | +| pCloud | MD5, SHA1 ⁷ | R | No | No | W | - | +| premiumize.me | - | - | Yes | No | R | - | +| put.io | CRC-32 | R/W | No | Yes | R | - | +| QingStor | MD5 | - ⁹ | No | No | R/W | - | +| Seafile | - | - | No | No | - | - | +| SFTP | MD5, SHA1 ² | R/W | Depends | No | - | - | +| Sia | - | - | No | No | - | - | +| SugarSync | - | - | No | No | - | - | +| Storj | - | R | No | No | - | - | +| Uptobox | - | - | No | Yes | - | - | +| WebDAV | MD5, SHA1 ³ | R ⁴ | Depends | No | - | - | +| Yandex Disk | MD5 | R/W | No | No | R | - | +| Zoho WorkDrive | - | - | No | No | - | - | +| The local filesystem | All | R/W | Depends | No | - | RWU | ### Notes @@ -12923,6 +13671,20 @@ storage platform has been determined to allow duplicate files, and it is possible to create them with `rclone`. It may be that this is a mistake or an unsupported feature. +⁹ QingStor does not support SetModTime for objects bigger than 5 GiB. + +¹⁰ FTP supports modtimes for the major FTP servers, and also others +if they advertised required protocol extensions. See [this](https://rclone.org/ftp/#modified-time) +for more details. + +¹¹ Internet Archive requires option `wait_archive` to be set to a non-zero value +for full modtime support. + +¹² HiDrive supports [its own custom +hash](https://static.hidrive.com/dev/0001). +It combines SHA1 sums for each 4 KiB block hierarchically to a single +top-level sum. + ### Hash ### The cloud storage system supports various hash types of the objects. @@ -12935,13 +13697,36 @@ systems they must support a common hash type. ### ModTime ### -The cloud storage system supports setting modification times on -objects. If it does then this enables a using the modification times -as part of the sync. If not then only the size will be checked by -default, though the MD5SUM can be checked with the `--checksum` flag. - -All cloud storage systems support some kind of date on the object and -these will be set when transferring from the cloud storage system. +Allmost all cloud storage systems store some sort of timestamp +on objects, but several of them not something that is appropriate +to use for syncing. E.g. some backends will only write a timestamp +that represent the time of the upload. To be relevant for syncing +it should be able to store the modification time of the source +object. If this is not the case, rclone will only check the file +size by default, though can be configured to check the file hash +(with the `--checksum` flag). Ideally it should also be possible to +change the timestamp of an existing file without having to re-upload it. + +Storage systems with a `-` in the ModTime column, means the +modification read on objects is not the modification time of the +file when uploaded. It is most likely the time the file was uploaded, +or possibly something else (like the time the picture was taken in +Google Photos). + +Storage systems with a `R` (for read-only) in the ModTime column, +means the it keeps modification times on objects, and updates them +when uploading objects, but it does not support changing only the +modification time (`SetModTime` operation) without re-uploading, +possibly not even without deleting existing first. Some operations +in rclone, such as `copy` and `sync` commands, will automatically +check for `SetModTime` support and re-upload if necessary to keep +the modification times in sync. Other commands will not work +without `SetModTime` support, e.g. `touch` command on an existing +file will fail, and changes to modification time only on a files +in a `mount` will be silently ignored. + +Storage systems with `R/W` (for read/write) in the ModTime column, +means they do also support modtime-only operations. ### Case Insensitive ### @@ -13141,35 +13926,36 @@ list of all possible values by passing an invalid value to this flag, e.g. `--local-encoding "help"`. The command `rclone help flags encoding` will show you the defaults for the backends. -| Encoding | Characters | -| --------- | ---------- | -| Asterisk | `*` | -| BackQuote | `` ` `` | -| BackSlash | `\` | -| Colon | `:` | -| CrLf | CR 0x0D, LF 0x0A | -| Ctl | All control characters 0x00-0x1F | -| Del | DEL 0x7F | -| Dollar | `$` | -| Dot | `.` or `..` as entire string | -| DoubleQuote | `"` | -| Hash | `#` | -| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) | -| LeftCrLfHtVt | CR 0x0D, LF 0x0A,HT 0x09, VT 0x0B on the left of a string | -| LeftPeriod | `.` on the left of a string | -| LeftSpace | SPACE on the left of a string | -| LeftTilde | `~` on the left of a string | -| LtGt | `<`, `>` | -| None | No characters are encoded | -| Percent | `%` | -| Pipe | \| | -| Question | `?` | -| RightCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string | -| RightPeriod | `.` on the right of a string | -| RightSpace | SPACE on the right of a string | -| SingleQuote | `'` | -| Slash | `/` | -| SquareBracket | `[`, `]` | +| Encoding | Characters | Encoded as | +| --------- | ---------- | ---------- | +| Asterisk | `*` | `＊` | +| BackQuote | `` ` `` | `｀` | +| BackSlash | `\` | `＼` | +| Colon | `:` | `：` | +| CrLf | CR 0x0D, LF 0x0A | `␍`, `␊` | +| Ctl | All control characters 0x00-0x1F | `␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟` | +| Del | DEL 0x7F | `␡` | +| Dollar | `$` | `＄` | +| Dot | `.` or `..` as entire string | `．`, `．．` | +| DoubleQuote | `"` | `＂` | +| Hash | `#` | `＃` | +| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) | `�` | +| LeftCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the left of a string | `␍`, `␊`, `␉`, `␋` | +| LeftPeriod | `.` on the left of a string | `.` | +| LeftSpace | SPACE on the left of a string | `␠` | +| LeftTilde | `~` on the left of a string | `～` | +| LtGt | `<`, `>` | `＜`, `＞` | +| None | No characters are encoded | | +| Percent | `%` | `％` | +| Pipe | \| | `｜` | +| Question | `?` | `？` | +| RightCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string | `␍`, `␊`, `␉`, `␋` | +| RightPeriod | `.` on the right of a string | `.` | +| RightSpace | SPACE on the right of a string | `␠` | +| Semicolon | `;` | `；` | +| SingleQuote | `'` | `＇` | +| Slash | `/` | `／` | +| SquareBracket | `[`, `]` | `［`, `］` | ##### Encoding example: FTP @@ -13244,6 +14030,22 @@ remote which supports writing (`W`) then rclone will preserve the MIME types. Otherwise they will be guessed from the extension, or the remote itself may assign the MIME type. +### Metadata + +Backends may or may support reading or writing metadata. They may +support reading and writing system metadata (metadata intrinsic to +that backend) and/or user metadata (general purpose metadata). + +The levels of metadata support are + +| Key | Explanation | +|-----|-------------| +| `R` | Read only System Metadata | +| `RW` | Read and write System Metadata | +| `RWU` | Read and write System Metadata and read and write User Metadata | + +See [the metadata docs](https://rclone.org/docs/#metadata) for more info. + ## Optional Features ## All rclone remotes support a base command set. Other features depend @@ -13252,8 +14054,9 @@ upon backend-specific capabilities. | Name | Purge | Copy | Move | DirMove | CleanUp | ListR | StreamUpload | LinkSharing | About | EmptyDir | | ---------------------------- |:-----:|:----:|:----:|:-------:|:-------:|:-----:|:------------:|:------------:|:-----:|:--------:| | 1Fichier | No | Yes | Yes | No | No | No | No | Yes | No | Yes | +| Akamai Netstorage | Yes | No | No | No | No | Yes | Yes | No | No | Yes | | Amazon Drive | Yes | No | Yes | Yes | No | No | No | No | No | Yes | -| Amazon S3 | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | +| Amazon S3 (or S3 compatible) | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | | Backblaze B2 | No | Yes | No | No | Yes | Yes | Yes | Yes | No | No | | Box | Yes | Yes | Yes | Yes | Yes ‡‡ | No | Yes | Yes | Yes | Yes | | Citrix ShareFile | Yes | Yes | Yes | Yes | No | No | Yes | No | No | Yes | @@ -13264,9 +14067,12 @@ upon backend-specific capabilities. | Google Drive | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | | Google Photos | No | No | No | No | No | No | No | No | No | No | | HDFS | Yes | No | Yes | Yes | No | No | Yes | No | Yes | Yes | +| HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | Yes | | HTTP | No | No | No | No | No | No | No | No | No | Yes | | Hubic | Yes † | Yes | No | No | No | Yes | Yes | No | Yes | No | +| Internet Archive | No | Yes | No | No | Yes | Yes | No | Yes | Yes | No | | Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | +| Koofr | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | Yes | | Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | Mega | Yes | No | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | @@ -13280,6 +14086,7 @@ upon backend-specific capabilities. | QingStor | No | Yes | No | No | Yes | Yes | No | No | No | No | | Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | | SFTP | No | No | Yes | Yes | No | No | Yes | No | Yes | Yes | +| Sia | No | No | No | No | No | No | Yes | No | No | Yes | | SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes | | Storj | Yes † | No | Yes | No | No | Yes | Yes | No | No | No | | Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | @@ -13407,6 +14214,7 @@ These flags are available for every command. --delete-during When synchronizing, delete files during transfer --delete-excluded Delete files on dest excluded from sync --disable string Disable a comma separated list of features (use --disable help to see a list) + --disable-http-keep-alives Disable HTTP keep-alives and use each connection once. --disable-http2 Disable HTTP/2 in the global transport -n, --dry-run Do a trial run with no permanent changes --dscp string Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21 @@ -13416,7 +14224,7 @@ These flags are available for every command. --error-on-no-transfer Sets exit code 9 if no files are transferred, useful in scripts --exclude stringArray Exclude files matching pattern --exclude-from stringArray Read exclude patterns from file (use - to read from stdin) - --exclude-if-present string Exclude directories if filename is present + --exclude-if-present stringArray Exclude directories if filename is present --expect-continue-timeout duration Timeout when using expect / 100-continue in HTTP (default 1s) --fast-list Use recursive list if available; uses more memory but fewer transactions --files-from stringArray Read list of source-file names from file (use - to read from stdin) @@ -13455,6 +14263,8 @@ These flags are available for every command. --max-stats-groups int Maximum number of stats groups to keep in memory, on max oldest is discarded (default 1000) --max-transfer SizeSuffix Maximum size of data to transfer (default off) --memprofile string Write memory profile to file + -M, --metadata If set, preserve metadata when copying objects + --metadata-set stringArray Add metadata key=value when uploading --min-age Duration Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off) --min-size SizeSuffix Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off) --modify-window duration Max time diff to be considered the same (default 1ns) @@ -13526,7 +14336,7 @@ These flags are available for every command. --use-json-log Use json log format --use-mmap Use mmap allocator (see docs) --use-server-modtime Use server modified time instead of object metadata - --user-agent string Set the user-agent to a specified string (default "rclone/v1.58.0") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.59.0") -v, --verbose count Print lots more stuff (repeat for more) ``` @@ -13581,6 +14391,7 @@ and may be set in the config file. --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --b2-version-at Time Show file versions as they were at the specified time (default off) --b2-versions Include old versions in directory listings --box-access-token string Box App Primary Access Token --box-auth-url string Auth server URL @@ -13620,6 +14431,7 @@ and may be set in the config file. --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks --chunker-hash-type string Choose how chunker handles hash sums (default "md5") --chunker-remote string Remote to chunk/unchunk + --combine-upstreams SpaceSepList Upstreams for combining --compress-level int GZIP compression level (-2 to 9) (default -1) --compress-mode string Compression mode (default "gzip") --compress-ram-cache-limit SizeSuffix Some remotes don't allow the upload of files with unknown size (default 20Mi) @@ -13652,6 +14464,7 @@ and may be set in the config file. --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) + --drive-resource-key string Resource key for accessing a link-shared file --drive-root-folder-id string ID of the root folder --drive-scope string Scope that rclone should use when requesting access from drive --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs @@ -13707,6 +14520,7 @@ and may be set in the config file. --ftp-disable-epsv Disable using EPSV even if server advertises support --ftp-disable-mlsd Disable using MLSD even if server advertises support --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) + --ftp-disable-utf8 Disable using UTF-8 even if server advertises support --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) --ftp-host string FTP host to connect to @@ -13725,8 +14539,10 @@ and may be set in the config file. --gcs-bucket-policy-only Access checks should use bucket-level IAM policies --gcs-client-id string OAuth Client Id --gcs-client-secret string OAuth Client Secret + --gcs-decompress If set this will decompress gzip encoded objects --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) --gcs-location string Location for the newly created buckets + --gcs-no-check-bucket If set, don't attempt to check the bucket exists or create it --gcs-object-acl string Access Control List for new objects --gcs-project-number string Project number --gcs-service-account-file string Service Account Credentials JSON file path @@ -13752,10 +14568,24 @@ and may be set in the config file. --hdfs-namenode string Hadoop name node and port --hdfs-service-principal-name string Kerberos service principal name for the namenode --hdfs-username string Hadoop user name + --hidrive-auth-url string Auth server URL + --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) + --hidrive-client-id string OAuth Client Id + --hidrive-client-secret string OAuth Client Secret + --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary + --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --hidrive-endpoint string Endpoint for the service (default "https://api.hidrive.strato.com/2.1") + --hidrive-root-prefix string The root/parent folder for all paths (default "/") + --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default "rw") + --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default "user") + --hidrive-token string OAuth Access Token as a JSON blob + --hidrive-token-url string Token server url + --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) + --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) --http-headers CommaSepList Set HTTP headers for all transactions --http-no-head Don't use HEAD requests --http-no-slash Set this if the site doesn't end directories with / - --http-url string URL of http host to connect to + --http-url string URL of HTTP host to connect to --hubic-auth-url string Auth server URL --hubic-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) --hubic-client-id string OAuth Client Id @@ -13764,6 +14594,13 @@ and may be set in the config file. --hubic-no-chunk Don't chunk files during streaming upload --hubic-token string OAuth Access Token as a JSON blob --hubic-token-url string Token server url + --internetarchive-access-key-id string IAS3 Access Key + --internetarchive-disable-checksum Don't ask the server to test against MD5 checksum calculated by rclone (default true) + --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) + --internetarchive-endpoint string IAS3 Endpoint (default "https://s3.us.archive.org") + --internetarchive-front-endpoint string Host of InternetArchive Frontend (default "https://archive.org") + --internetarchive-secret-access-key string IAS3 Secret Key (password) + --internetarchive-wait-archive Duration Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s) --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) --jottacloud-hard-delete Delete files permanently rather than putting them into the trash --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) @@ -13785,7 +14622,7 @@ and may be set in the config file. --local-no-preallocate Disable preallocation of disk space for transferred files --local-no-set-modtime Disable setting modtime --local-no-sparse Disable sparse files for multi-thread downloads - --local-nounc string Disable UNC (long path names) conversion on Windows + --local-nounc Disable UNC (long path names) conversion on Windows --local-unicode-normalization Apply unicode NFC normalization to paths and filenames --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) @@ -13806,11 +14643,11 @@ and may be set in the config file. --netstorage-protocol string Select between HTTP or HTTPS protocol (default "https") --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) -x, --one-file-system Don't cross filesystem boundaries (unix/macOS only) + --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) --onedrive-auth-url string Auth server URL --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) --onedrive-client-id string OAuth Client Id --onedrive-client-secret string OAuth Client Secret - --onedrive-disable-site-permission Disable the request for Sites.Read.All permission --onedrive-drive-id string The ID of the drive to use --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) @@ -13834,9 +14671,11 @@ and may be set in the config file. --pcloud-client-secret string OAuth Client Secret --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) --pcloud-hostname string Hostname to connect to (default "api.pcloud.com") + --pcloud-password string Your pcloud password (obscured) --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default "d0") --pcloud-token string OAuth Access Token as a JSON blob --pcloud-token-url string Token server url + --pcloud-username string Your pcloud username --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) --qingstor-access-key-id string QingStor Access Key ID @@ -13889,6 +14728,7 @@ and may be set in the config file. --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) + --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads --s3-v2-auth If true use v2 authentication --seafile-2fa Two-factor authentication ('true' if the account has 2FA enabled) --seafile-create-library Should rclone create a library if it doesn't exist @@ -13899,6 +14739,8 @@ and may be set in the config file. --seafile-url string URL of seafile host to connect to --seafile-user string User name (usually email address) --sftp-ask-password Allow asking for SFTP password when needed + --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) + --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) --sftp-disable-concurrent-reads If set don't use concurrent reads --sftp-disable-concurrent-writes If set don't use concurrent writes --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available @@ -13911,12 +14753,14 @@ and may be set in the config file. --sftp-known-hosts-file string Optional path to known_hosts file --sftp-md5sum-command string The command used to read md5 hashes --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) - --sftp-path-override string Override path used by SSH connection + --sftp-path-override string Override path used by SSH shell commands --sftp-port int SSH port number (default 22) --sftp-pubkey-file string Optional path to public key file --sftp-server-command string Specifies the path or command to run a sftp server on the remote host + --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands --sftp-set-modtime Set the modified time on the remote if set (default true) --sftp-sha1sum-command string The command used to read sha1 hashes + --sftp-shell-type string The type of SSH shell on remote server, if any --sftp-skip-links Set to skip any symlinks and any other non regular files --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default "sftp") --sftp-use-fstat If set use fstat instead of stat @@ -13973,6 +14817,7 @@ and may be set in the config file. --union-action-policy string Policy to choose upstream on ACTION category (default "epall") --union-cache-time int Cache time of usage and free space (in seconds) (default 120) --union-create-policy string Policy to choose upstream on CREATE category (default "epmfs") + --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) --union-search-policy string Policy to choose upstream on SEARCH category (default "ff") --union-upstreams string List of space separated upstreams --uptobox-access-token string Your access token @@ -13984,7 +14829,7 @@ and may be set in the config file. --webdav-pass string Password (obscured) --webdav-url string URL of http host to connect to --webdav-user string User name - --webdav-vendor string Name of the Webdav site/service/software you are using + --webdav-vendor string Name of the WebDAV site/service/software you are using --yandex-auth-url string Auth server URL --yandex-client-id string OAuth Client Id --yandex-client-secret string OAuth Client Secret @@ -14660,7 +15505,7 @@ Optional Flags: Arbitrary rclone flags may be specified on the [bisync command line](https://rclone.org/commands/rclone_bisync/), for example -`rclone bsync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s` +`rclone bisync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s` Note that interactions of various rclone flags with bisync process flow has not been fully tested yet. @@ -14917,6 +15762,7 @@ Bisync is considered _BETA_ and has been tested with the following backends: - OneDrive - S3 - SFTP +- Yandex Disk It has not been fully tested with other services yet. If it works, or sorta works, please let us know and we'll update the list. @@ -15254,7 +16100,7 @@ consider using the flag Google docs exist as virtual files on Google Drive and cannot be transferred to other filesystems natively. While it is possible to export a Google doc to -a normal file (with `.xlsx` extension, for example), it's not possible +a normal file (with `.xlsx` extension, for example), it is not possible to import a normal file back into a Google document. Bisync's handling of Google Doc files is to flag them in the run log output @@ -15755,7 +16601,7 @@ as they can't be used in JSON strings. ### Standard options -Here are the standard options specific to fichier (1Fichier). +Here are the Standard options specific to fichier (1Fichier). #### --fichier-api-key @@ -15770,7 +16616,7 @@ Properties: ### Advanced options -Here are the advanced options specific to fichier (1Fichier). +Here are the Advanced options specific to fichier (1Fichier). #### --fichier-shared-folder @@ -15831,8 +16677,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) # Alias @@ -15922,7 +16767,7 @@ Copy another local directory to the alias directory called source ### Standard options -Here are the standard options specific to alias (Alias for an existing remote). +Here are the Standard options specific to alias (Alias for an existing remote). #### --alias-remote @@ -16096,7 +16941,7 @@ rclone it will take you to an `amazon.com` page to log in. Your ### Standard options -Here are the standard options specific to amazon cloud drive (Amazon Drive). +Here are the Standard options specific to amazon cloud drive (Amazon Drive). #### --acd-client-id @@ -16126,7 +16971,7 @@ Properties: ### Advanced options -Here are the advanced options specific to amazon cloud drive (Amazon Drive). +Here are the Advanced options specific to amazon cloud drive (Amazon Drive). #### --acd-token @@ -16270,8 +17115,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) # Amazon S3 Storage Providers @@ -16281,9 +17125,14 @@ The S3 backend can be used with a number of different providers: - AWS S3 - Alibaba Cloud (Aliyun) Object Storage System (OSS) - Ceph +- China Mobile Ecloud Elastic Object Storage (EOS) +- Cloudflare R2 +- Arvan Cloud Object Storage (AOS) - DigitalOcean Spaces - Dreamhost +- Huawei OBS - IBM COS S3 +- IDrive e2 - Minio - RackCorp Object Storage - Scaleway @@ -16339,7 +17188,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, Dreamhost, IBM COS, Minio, and Tencent COS +XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -16836,7 +17685,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t ### Standard options -Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS). +Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi). #### --s3-provider @@ -16855,12 +17704,22 @@ Properties: - Alibaba Cloud Object Storage System (OSS) formerly Aliyun - "Ceph" - Ceph Object Storage + - "ChinaMobile" + - China Mobile Ecloud Elastic Object Storage (EOS) + - "Cloudflare" + - Cloudflare R2 Storage + - "ArvanCloud" + - Arvan Cloud Object Storage (AOS) - "DigitalOcean" - Digital Ocean Spaces - "Dreamhost" - Dreamhost DreamObjects + - "HuaweiOBS" + - Huawei Object Storage Service - "IBMCOS" - IBM COS S3 + - "IDrive" + - IDrive e2 - "LyveCloud" - Seagate Lyve Cloud - "Minio" @@ -17085,6 +17944,67 @@ Properties: - Amsterdam, The Netherlands - "fr-par" - Paris, France + - "pl-waw" + - Warsaw, Poland + +#### --s3-region + +Region to connect to. - the location where your bucket will be created and your data stored. Need bo be same with your endpoint. + + +Properties: + +- Config: region +- Env Var: RCLONE_S3_REGION +- Provider: HuaweiOBS +- Type: string +- Required: false +- Examples: + - "af-south-1" + - AF-Johannesburg + - "ap-southeast-2" + - AP-Bangkok + - "ap-southeast-3" + - AP-Singapore + - "cn-east-3" + - CN East-Shanghai1 + - "cn-east-2" + - CN East-Shanghai2 + - "cn-north-1" + - CN North-Beijing1 + - "cn-north-4" + - CN North-Beijing4 + - "cn-south-1" + - CN South-Guangzhou + - "ap-southeast-1" + - CN-Hong Kong + - "sa-argentina-1" + - LA-Buenos Aires1 + - "sa-peru-1" + - LA-Lima1 + - "na-mexico-1" + - LA-Mexico City1 + - "sa-chile-1" + - LA-Santiago2 + - "sa-brazil-1" + - LA-Sao Paulo1 + - "ru-northwest-2" + - RU-Moscow2 + +#### --s3-region + +Region to connect to. + +Properties: + +- Config: region +- Env Var: RCLONE_S3_REGION +- Provider: Cloudflare +- Type: string +- Required: false +- Examples: + - "auto" + - R2 buckets are automatically distributed across Cloudflare's data centers for low latency. #### --s3-region @@ -17096,7 +18016,7 @@ Properties: - Config: region - Env Var: RCLONE_S3_REGION -- Provider: !AWS,Alibaba,RackCorp,Scaleway,Storj,TencentCOS +- Provider: !AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive - Type: string - Required: false - Examples: @@ -17123,6 +18043,98 @@ Properties: #### --s3-endpoint +Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Provider: ChinaMobile +- Type: string +- Required: false +- Examples: + - "eos-wuxi-1.cmecloud.cn" + - The default endpoint - a good choice if you are unsure. + - East China (Suzhou) + - "eos-jinan-1.cmecloud.cn" + - East China (Jinan) + - "eos-ningbo-1.cmecloud.cn" + - East China (Hangzhou) + - "eos-shanghai-1.cmecloud.cn" + - East China (Shanghai-1) + - "eos-zhengzhou-1.cmecloud.cn" + - Central China (Zhengzhou) + - "eos-hunan-1.cmecloud.cn" + - Central China (Changsha-1) + - "eos-zhuzhou-1.cmecloud.cn" + - Central China (Changsha-2) + - "eos-guangzhou-1.cmecloud.cn" + - South China (Guangzhou-2) + - "eos-dongguan-1.cmecloud.cn" + - South China (Guangzhou-3) + - "eos-beijing-1.cmecloud.cn" + - North China (Beijing-1) + - "eos-beijing-2.cmecloud.cn" + - North China (Beijing-2) + - "eos-beijing-4.cmecloud.cn" + - North China (Beijing-3) + - "eos-huhehaote-1.cmecloud.cn" + - North China (Huhehaote) + - "eos-chengdu-1.cmecloud.cn" + - Southwest China (Chengdu) + - "eos-chongqing-1.cmecloud.cn" + - Southwest China (Chongqing) + - "eos-guiyang-1.cmecloud.cn" + - Southwest China (Guiyang) + - "eos-xian-1.cmecloud.cn" + - Nouthwest China (Xian) + - "eos-yunnan.cmecloud.cn" + - Yunnan China (Kunming) + - "eos-yunnan-2.cmecloud.cn" + - Yunnan China (Kunming-2) + - "eos-tianjin-1.cmecloud.cn" + - Tianjin China (Tianjin) + - "eos-jilin-1.cmecloud.cn" + - Jilin China (Changchun) + - "eos-hubei-1.cmecloud.cn" + - Hubei China (Xiangyan) + - "eos-jiangxi-1.cmecloud.cn" + - Jiangxi China (Nanchang) + - "eos-gansu-1.cmecloud.cn" + - Gansu China (Lanzhou) + - "eos-shanxi-1.cmecloud.cn" + - Shanxi China (Taiyuan) + - "eos-liaoning-1.cmecloud.cn" + - Liaoning China (Shenyang) + - "eos-hebei-1.cmecloud.cn" + - Hebei China (Shijiazhuang) + - "eos-fujian-1.cmecloud.cn" + - Fujian China (Xiamen) + - "eos-guangxi-1.cmecloud.cn" + - Guangxi China (Nanning) + - "eos-anhui-1.cmecloud.cn" + - Anhui China (Huainan) + +#### --s3-endpoint + +Endpoint for Arvan Cloud Object Storage (AOS) API. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Provider: ArvanCloud +- Type: string +- Required: false +- Examples: + - "s3.ir-thr-at1.arvanstorage.com" + - The default endpoint - a good choice if you are unsure. + - Tehran Iran (Asiatech) + - "s3.ir-tbz-sh1.arvanstorage.com" + - Tabriz Iran (Shahriar) + +#### --s3-endpoint + Endpoint for IBM COS S3 API. Specify if using an IBM COS On Premise. @@ -17325,6 +18337,49 @@ Properties: #### --s3-endpoint +Endpoint for OBS API. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Provider: HuaweiOBS +- Type: string +- Required: false +- Examples: + - "obs.af-south-1.myhuaweicloud.com" + - AF-Johannesburg + - "obs.ap-southeast-2.myhuaweicloud.com" + - AP-Bangkok + - "obs.ap-southeast-3.myhuaweicloud.com" + - AP-Singapore + - "obs.cn-east-3.myhuaweicloud.com" + - CN East-Shanghai1 + - "obs.cn-east-2.myhuaweicloud.com" + - CN East-Shanghai2 + - "obs.cn-north-1.myhuaweicloud.com" + - CN North-Beijing1 + - "obs.cn-north-4.myhuaweicloud.com" + - CN North-Beijing4 + - "obs.cn-south-1.myhuaweicloud.com" + - CN South-Guangzhou + - "obs.ap-southeast-1.myhuaweicloud.com" + - CN-Hong Kong + - "obs.sa-argentina-1.myhuaweicloud.com" + - LA-Buenos Aires1 + - "obs.sa-peru-1.myhuaweicloud.com" + - LA-Lima1 + - "obs.na-mexico-1.myhuaweicloud.com" + - LA-Mexico City1 + - "obs.sa-chile-1.myhuaweicloud.com" + - LA-Santiago2 + - "obs.sa-brazil-1.myhuaweicloud.com" + - LA-Sao Paulo1 + - "obs.ru-northwest-2.myhuaweicloud.com" + - RU-Moscow2 + +#### --s3-endpoint + Endpoint for Scaleway Object Storage. Properties: @@ -17339,6 +18394,8 @@ Properties: - Amsterdam Endpoint - "s3.fr-par.scw.cloud" - Paris Endpoint + - "s3.pl-waw.scw.cloud" + - Warsaw Endpoint #### --s3-endpoint @@ -17490,7 +18547,7 @@ Properties: - Config: endpoint - Env Var: RCLONE_S3_ENDPOINT -- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,Storj,RackCorp +- Provider: !AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp - Type: string - Required: false - Examples: @@ -17520,6 +18577,8 @@ Properties: - Wasabi AP Northeast 1 (Tokyo) endpoint - "s3.ap-northeast-2.wasabisys.com" - Wasabi AP Northeast 2 (Osaka) endpoint + - "s3.ir-thr-at1.arvanstorage.com" + - ArvanCloud Tehran Iran (Asiatech) endpoint #### --s3-location-constraint @@ -17588,6 +18647,100 @@ Properties: #### --s3-location-constraint +Location constraint - must match endpoint. + +Used when creating buckets only. + +Properties: + +- Config: location_constraint +- Env Var: RCLONE_S3_LOCATION_CONSTRAINT +- Provider: ChinaMobile +- Type: string +- Required: false +- Examples: + - "wuxi1" + - East China (Suzhou) + - "jinan1" + - East China (Jinan) + - "ningbo1" + - East China (Hangzhou) + - "shanghai1" + - East China (Shanghai-1) + - "zhengzhou1" + - Central China (Zhengzhou) + - "hunan1" + - Central China (Changsha-1) + - "zhuzhou1" + - Central China (Changsha-2) + - "guangzhou1" + - South China (Guangzhou-2) + - "dongguan1" + - South China (Guangzhou-3) + - "beijing1" + - North China (Beijing-1) + - "beijing2" + - North China (Beijing-2) + - "beijing4" + - North China (Beijing-3) + - "huhehaote1" + - North China (Huhehaote) + - "chengdu1" + - Southwest China (Chengdu) + - "chongqing1" + - Southwest China (Chongqing) + - "guiyang1" + - Southwest China (Guiyang) + - "xian1" + - Nouthwest China (Xian) + - "yunnan" + - Yunnan China (Kunming) + - "yunnan2" + - Yunnan China (Kunming-2) + - "tianjin1" + - Tianjin China (Tianjin) + - "jilin1" + - Jilin China (Changchun) + - "hubei1" + - Hubei China (Xiangyan) + - "jiangxi1" + - Jiangxi China (Nanchang) + - "gansu1" + - Gansu China (Lanzhou) + - "shanxi1" + - Shanxi China (Taiyuan) + - "liaoning1" + - Liaoning China (Shenyang) + - "hebei1" + - Hebei China (Shijiazhuang) + - "fujian1" + - Fujian China (Xiamen) + - "guangxi1" + - Guangxi China (Nanning) + - "anhui1" + - Anhui China (Huainan) + +#### --s3-location-constraint + +Location constraint - must match endpoint. + +Used when creating buckets only. + +Properties: + +- Config: location_constraint +- Env Var: RCLONE_S3_LOCATION_CONSTRAINT +- Provider: ArvanCloud +- Type: string +- Required: false +- Examples: + - "ir-thr-at1" + - Tehran Iran (Asiatech) + - "ir-tbz-sh1" + - Tabriz Iran (Shahriar) + +#### --s3-location-constraint + Location constraint - must match endpoint when using IBM Cloud Public. For on-prem COS, do not make a selection from this list, hit enter. @@ -17727,7 +18880,7 @@ Properties: - Config: location_constraint - Env Var: RCLONE_S3_LOCATION_CONSTRAINT -- Provider: !AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,Storj,TencentCOS +- Provider: !AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS - Type: string - Required: false @@ -17746,7 +18899,7 @@ Properties: - Config: acl - Env Var: RCLONE_S3_ACL -- Provider: !Storj +- Provider: !Storj,Cloudflare - Type: string - Required: false - Examples: @@ -17799,7 +18952,7 @@ Properties: - Config: server_side_encryption - Env Var: RCLONE_S3_SERVER_SIDE_ENCRYPTION -- Provider: AWS,Ceph,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -17881,6 +19034,42 @@ Properties: #### --s3-storage-class +The storage class to use when storing new objects in ChinaMobile. + +Properties: + +- Config: storage_class +- Env Var: RCLONE_S3_STORAGE_CLASS +- Provider: ChinaMobile +- Type: string +- Required: false +- Examples: + - "" + - Default + - "STANDARD" + - Standard storage class + - "GLACIER" + - Archive storage mode + - "STANDARD_IA" + - Infrequent access storage mode + +#### --s3-storage-class + +The storage class to use when storing new objects in ArvanCloud. + +Properties: + +- Config: storage_class +- Env Var: RCLONE_S3_STORAGE_CLASS +- Provider: ArvanCloud +- Type: string +- Required: false +- Examples: + - "STANDARD" + - Standard storage class + +#### --s3-storage-class + The storage class to use when storing new objects in Tencent COS. Properties: @@ -17923,7 +19112,7 @@ Properties: ### Advanced options -Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS). +Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi). #### --s3-bucket-acl @@ -17975,7 +19164,7 @@ Properties: - Config: sse_customer_algorithm - Env Var: RCLONE_S3_SSE_CUSTOMER_ALGORITHM -- Provider: AWS,Ceph,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -17992,7 +19181,7 @@ Properties: - Config: sse_customer_key - Env Var: RCLONE_S3_SSE_CUSTOMER_KEY -- Provider: AWS,Ceph,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -18010,7 +19199,7 @@ Properties: - Config: sse_customer_key_md5 - Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_MD5 -- Provider: AWS,Ceph,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -18055,6 +19244,13 @@ most 10,000 chunks, this means that by default the maximum size of a file you can stream upload is 48 GiB. If you wish to stream upload larger files then you will need to increase chunk_size. +Increasing the chunk size decreases the accuracy of the progress +statistics displayed with "-P" flag. Rclone treats chunk as sent when +it's buffered by the AWS SDK, when in fact it may still be uploading. +A bigger chunk size means a bigger AWS SDK buffer and progress +reporting more deviating from the truth. + + Properties: - Config: chunk_size @@ -18460,6 +19656,45 @@ Properties: - Type: Tristate - Default: unset +#### --s3-use-presigned-request + +Whether to use a presigned request or PutObject for single part uploads + +If this is false rclone will use PutObject from the AWS SDK to upload +an object. + +Versions of rclone < 1.59 use presigned requests to upload a single +part object and setting this flag to true will re-enable that +functionality. This shouldn't be necessary except in exceptional +circumstances or for testing. + + +Properties: + +- Config: use_presigned_request +- Env Var: RCLONE_S3_USE_PRESIGNED_REQUEST +- Type: bool +- Default: false + +### Metadata + +User metadata is stored as x-amz-meta- keys. S3 metadata keys are case insensitive and are always returned in lower case. + +Here are the possible system metadata items for the s3 backend. + +| Name | Help | Type | Example | Read Only | +|------|------|------|---------|-----------| +| btime | Time of file birth (creation) read from Last-Modified header | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | **Y** | +| cache-control | Cache-Control header | string | no-cache | N | +| content-disposition | Content-Disposition header | string | inline | N | +| content-encoding | Content-Encoding header | string | gzip | N | +| content-language | Content-Language header | string | en-US | N | +| content-type | Content-Type header | string | text/plain | N | +| mtime | Time of last modification, read from rclone metadata | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N | +| tier | Tier of the object | string | GLACIER | **Y** | + +See the [metadata](https://rclone.org/docs/#metadata) docs for more info. + ## Backend commands Here are the commands specific to the s3 backend. @@ -18470,7 +19705,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](https://rclone.org/commands/rclone_backend/) for more +See the [backend](https://rclone.org/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command @@ -18585,7 +19820,7 @@ Options: -### Anonymous access to public buckets ### +### Anonymous access to public buckets If you want to use rclone to access a public bucket, configure with a blank `access_key_id` and `secret_access_key`. Your config should end @@ -18619,12 +19854,19 @@ You will be able to list and copy data but not upload it. This is the provider used as main example and described in the [configuration](#configuration) section above. ### AWS Snowball Edge -[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware appliance used for transferring -bulk data back to AWS. Its main software interface is S3 object storage. -To use rclone with AWS Snowball Edge devices, configure as standard for an 'S3 Compatible Service' -be sure to set `upload_cutoff = 0` otherwise you will run into authentication header issues as -the snowball device does not support query parameter based authentication. +[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware +appliance used for transferring bulk data back to AWS. Its main +software interface is S3 object storage. + +To use rclone with AWS Snowball Edge devices, configure as standard +for an 'S3 Compatible Service'. + +If using rclone pre v1.59 be sure to set `upload_cutoff = 0` otherwise +you will run into authentication header issues as the snowball device +does not support query parameter based authentication. + +With rclone v1.59 or later setting `upload_cutoff` should not be necessary. eg. ``` @@ -18663,10 +19905,11 @@ server_side_encryption = storage_class = ``` -If you are using an older version of CEPH, e.g. 10.2.x Jewel, then you -may need to supply the parameter `--s3-upload-cutoff 0` or put this in -the config file as `upload_cutoff 0` to work around a bug which causes -uploading of small files to fail. +If you are using an older version of CEPH (e.g. 10.2.x Jewel) and a +version of rclone before v1.59 then you may need to supply the +parameter `--s3-upload-cutoff 0` or put this in the config file as +`upload_cutoff 0` to work around a bug which causes uploading of small +files to fail. Note also that Ceph sometimes puts `/` in the passwords it gives users. If you read the secret access key using the command line tools @@ -18693,6 +19936,106 @@ removed). Because this is a json dump, it is encoding the `/` as `\/`, so if you use the secret key as `xxxxxx/xxxx` it will work fine. +### Cloudflare R2 {#cloudflare-r2} + +[Cloudflare R2](https://blog.cloudflare.com/r2-open-beta/) Storage +allows developers to store large amounts of unstructured data without +the costly egress bandwidth fees associated with typical cloud storage +services. + +Here is an example of making a Cloudflare R2 configuration. First run: + + rclone config + +This will guide you through an interactive setup process. + +Note that all buckets are private, and all are stored in the same +"auto" region. It is necessary to use Cloudflare workers to share the +content of a bucket publicly. + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> r2 +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +... +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \ (s3) +... +Storage> s3 +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. +... +XX / Cloudflare R2 Storage + \ (Cloudflare) +... +provider> Cloudflare +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) +env_auth> 1 +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> ACCESS_KEY +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> SECRET_ACCESS_KEY +Option region. +Region to connect to. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / R2 buckets are automatically distributed across Cloudflare's data centers for low latency. + \ (auto) +region> 1 +Option endpoint. +Endpoint for S3 API. +Required when using an S3 clone. +Enter a value. Press Enter to leave empty. +endpoint> https://ACCOUNT_ID.r2.cloudflarestorage.com +Edit advanced config? +y) Yes +n) No (default) +y/n> n +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This will leave your config looking something like: + +``` +[r2] +type = s3 +provider = Cloudflare +access_key_id = ACCESS_KEY +secret_access_key = SECRET_ACCESS_KEY +region = auto +endpoint = https://ACCOUNT_ID.r2.cloudflarestorage.com +acl = private +``` + +Now run `rclone lsf r2:` to see your buckets and `rclone lsf +r2:bucket` to look within a bucket. + ### Dreamhost Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is @@ -18762,6 +20105,133 @@ Once configured, you can create a new Space and begin copying files. For example rclone mkdir spaces:my-new-space rclone copy /path/to/files spaces:my-new-space ``` +### Huawei OBS {#huawei-obs} + +Object Storage Service (OBS) provides stable, secure, efficient, and easy-to-use cloud storage that lets you store virtually any volume of unstructured data in any format and access it from anywhere. + +OBS provides an S3 interface, you can copy and modify the following configuration and add it to your rclone configuration file. +``` +[obs] +type = s3 +provider = HuaweiOBS +access_key_id = your-access-key-id +secret_access_key = your-secret-access-key +region = af-south-1 +endpoint = obs.af-south-1.myhuaweicloud.com +acl = private +``` + +Or you can also configure via the interactive command line: +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> obs +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +[snip] + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \ (s3) +[snip] +Storage> 5 +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. +[snip] + 9 / Huawei Object Storage Service + \ (HuaweiOBS) +[snip] +provider> 9 +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) +env_auth> 1 +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> your-access-key-id +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> your-secret-access-key +Option region. +Region to connect to. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / AF-Johannesburg + \ (af-south-1) + 2 / AP-Bangkok + \ (ap-southeast-2) +[snip] +region> 1 +Option endpoint. +Endpoint for OBS API. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / AF-Johannesburg + \ (obs.af-south-1.myhuaweicloud.com) + 2 / AP-Bangkok + \ (obs.ap-southeast-2.myhuaweicloud.com) +[snip] +endpoint> 1 +Option acl. +Canned ACL used when creating buckets and storing or copying objects. +This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Note that this ACL is applied when server-side copying objects as S3 +doesn't copy the ACL from the source but rather writes a fresh one. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \ (private) +[snip] +acl> 1 +Edit advanced config? +y) Yes +n) No (default) +y/n> +-------------------- +[obs] +type = s3 +provider = HuaweiOBS +access_key_id = your-access-key-id +secret_access_key = your-secret-access-key +region = af-south-1 +endpoint = obs.af-south-1.myhuaweicloud.com +acl = private +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +Current remotes: + +Name Type +==== ==== +obs s3 + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> q +``` ### IBM COS (S3) @@ -18791,12 +20261,12 @@ Choose a number from below, or type in your own value \ "alias" 2 / Amazon Drive \ "amazon cloud drive" - 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, Minio, IBM COS) + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS) \ "s3" 4 / Backblaze B2 \ "b2" [snip] - 23 / http Connection + 23 / HTTP \ "http" Storage> 3 ``` @@ -18935,6 +20405,116 @@ acl> 1 rclone delete IBM-COS-XREGION:newbucket/file.txt ``` +### IDrive e2 {#idrive-e2} + +Here is an example of making an [IDrive e2](https://www.idrive.com/e2/) +configuration. First run: + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n + +Enter name for new remote. +name> e2 + +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +[snip] +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \ (s3) +[snip] +Storage> s3 + +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. +[snip] +XX / IDrive e2 + \ (IDrive) +[snip] +provider> IDrive + +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) +env_auth> + +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> YOUR_ACCESS_KEY + +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> YOUR_SECRET_KEY + +Option acl. +Canned ACL used when creating buckets and storing or copying objects. +This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Note that this ACL is applied when server-side copying objects as S3 +doesn't copy the ACL from the source but rather writes a fresh one. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \ (private) + / Owner gets FULL_CONTROL. + 2 | The AllUsers group gets READ access. + \ (public-read) + / Owner gets FULL_CONTROL. + 3 | The AllUsers group gets READ and WRITE access. + | Granting this on a bucket is generally not recommended. + \ (public-read-write) + / Owner gets FULL_CONTROL. + 4 | The AuthenticatedUsers group gets READ access. + \ (authenticated-read) + / Object owner gets FULL_CONTROL. + 5 | Bucket owner gets READ access. + | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ (bucket-owner-read) + / Both the object owner and the bucket owner get FULL_CONTROL over the object. + 6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ (bucket-owner-full-control) +acl> + +Edit advanced config? +y) Yes +n) No (default) +y/n> + +Configuration complete. +Options: +- type: s3 +- provider: IDrive +- access_key_id: YOUR_ACCESS_KEY +- secret_access_key: YOUR_SECRET_KEY +- endpoint: q9d9.la12.idrivee2-5.com +Keep this "e2" remote? +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + ### Minio [Minio](https://minio.io/) is an object storage server built for cloud application developers and devops. @@ -19048,6 +20628,9 @@ server_side_encryption = storage_class = ``` +[C14 Cold Storage](https://www.online.net/en/storage/c14-cold-storage) is the low-cost S3 Glacier alternative from Scaleway and it works the same way as on S3 by accepting the "GLACIER" `storage_class`. +So you can configure your remote with the `storage_class = GLACIER` option to upload directly to C14. Don't forget that in this state you can't read files back after, you will need to restore them to "STANDARD" storage_class first before being able to read them (see "restore" section above) + ### Seagate Lyve Cloud {#lyve} [Seagate Lyve Cloud](https://www.seagate.com/gb/en/services/cloud/storage/) is an S3 @@ -19073,7 +20656,7 @@ Choose `s3` backend Type of storage to configure. Choose a number from below, or type in your own value. [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \ (s3) [snip] Storage> s3 @@ -19260,7 +20843,7 @@ name> wasabi Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 (also Dreamhost, Ceph, Minio) +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) \ "s3" [snip] Storage> s3 @@ -19374,7 +20957,7 @@ Type of storage to configure. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [snip] - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -19464,6 +21047,364 @@ d) Delete this remote y/e/d> y ``` +### China Mobile Ecloud Elastic Object Storage (EOS) {#china-mobile-ecloud-eos} + +Here is an example of making an [China Mobile Ecloud Elastic Object Storage (EOS)](https:///ecloud.10086.cn/home/product-introduction/eos/) +configuration. First run: + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> ChinaMobile +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. + ... + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS + \ (s3) + ... +Storage> s3 +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + ... + 4 / China Mobile Ecloud Elastic Object Storage (EOS) + \ (ChinaMobile) + ... +provider> ChinaMobile +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) +env_auth> +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> accesskeyid +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> secretaccesskey +Option endpoint. +Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + / The default endpoint - a good choice if you are unsure. + 1 | East China (Suzhou) + \ (eos-wuxi-1.cmecloud.cn) + 2 / East China (Jinan) + \ (eos-jinan-1.cmecloud.cn) + 3 / East China (Hangzhou) + \ (eos-ningbo-1.cmecloud.cn) + 4 / East China (Shanghai-1) + \ (eos-shanghai-1.cmecloud.cn) + 5 / Central China (Zhengzhou) + \ (eos-zhengzhou-1.cmecloud.cn) + 6 / Central China (Changsha-1) + \ (eos-hunan-1.cmecloud.cn) + 7 / Central China (Changsha-2) + \ (eos-zhuzhou-1.cmecloud.cn) + 8 / South China (Guangzhou-2) + \ (eos-guangzhou-1.cmecloud.cn) + 9 / South China (Guangzhou-3) + \ (eos-dongguan-1.cmecloud.cn) +10 / North China (Beijing-1) + \ (eos-beijing-1.cmecloud.cn) +11 / North China (Beijing-2) + \ (eos-beijing-2.cmecloud.cn) +12 / North China (Beijing-3) + \ (eos-beijing-4.cmecloud.cn) +13 / North China (Huhehaote) + \ (eos-huhehaote-1.cmecloud.cn) +14 / Southwest China (Chengdu) + \ (eos-chengdu-1.cmecloud.cn) +15 / Southwest China (Chongqing) + \ (eos-chongqing-1.cmecloud.cn) +16 / Southwest China (Guiyang) + \ (eos-guiyang-1.cmecloud.cn) +17 / Nouthwest China (Xian) + \ (eos-xian-1.cmecloud.cn) +18 / Yunnan China (Kunming) + \ (eos-yunnan.cmecloud.cn) +19 / Yunnan China (Kunming-2) + \ (eos-yunnan-2.cmecloud.cn) +20 / Tianjin China (Tianjin) + \ (eos-tianjin-1.cmecloud.cn) +21 / Jilin China (Changchun) + \ (eos-jilin-1.cmecloud.cn) +22 / Hubei China (Xiangyan) + \ (eos-hubei-1.cmecloud.cn) +23 / Jiangxi China (Nanchang) + \ (eos-jiangxi-1.cmecloud.cn) +24 / Gansu China (Lanzhou) + \ (eos-gansu-1.cmecloud.cn) +25 / Shanxi China (Taiyuan) + \ (eos-shanxi-1.cmecloud.cn) +26 / Liaoning China (Shenyang) + \ (eos-liaoning-1.cmecloud.cn) +27 / Hebei China (Shijiazhuang) + \ (eos-hebei-1.cmecloud.cn) +28 / Fujian China (Xiamen) + \ (eos-fujian-1.cmecloud.cn) +29 / Guangxi China (Nanning) + \ (eos-guangxi-1.cmecloud.cn) +30 / Anhui China (Huainan) + \ (eos-anhui-1.cmecloud.cn) +endpoint> 1 +Option location_constraint. +Location constraint - must match endpoint. +Used when creating buckets only. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / East China (Suzhou) + \ (wuxi1) + 2 / East China (Jinan) + \ (jinan1) + 3 / East China (Hangzhou) + \ (ningbo1) + 4 / East China (Shanghai-1) + \ (shanghai1) + 5 / Central China (Zhengzhou) + \ (zhengzhou1) + 6 / Central China (Changsha-1) + \ (hunan1) + 7 / Central China (Changsha-2) + \ (zhuzhou1) + 8 / South China (Guangzhou-2) + \ (guangzhou1) + 9 / South China (Guangzhou-3) + \ (dongguan1) +10 / North China (Beijing-1) + \ (beijing1) +11 / North China (Beijing-2) + \ (beijing2) +12 / North China (Beijing-3) + \ (beijing4) +13 / North China (Huhehaote) + \ (huhehaote1) +14 / Southwest China (Chengdu) + \ (chengdu1) +15 / Southwest China (Chongqing) + \ (chongqing1) +16 / Southwest China (Guiyang) + \ (guiyang1) +17 / Nouthwest China (Xian) + \ (xian1) +18 / Yunnan China (Kunming) + \ (yunnan) +19 / Yunnan China (Kunming-2) + \ (yunnan2) +20 / Tianjin China (Tianjin) + \ (tianjin1) +21 / Jilin China (Changchun) + \ (jilin1) +22 / Hubei China (Xiangyan) + \ (hubei1) +23 / Jiangxi China (Nanchang) + \ (jiangxi1) +24 / Gansu China (Lanzhou) + \ (gansu1) +25 / Shanxi China (Taiyuan) + \ (shanxi1) +26 / Liaoning China (Shenyang) + \ (liaoning1) +27 / Hebei China (Shijiazhuang) + \ (hebei1) +28 / Fujian China (Xiamen) + \ (fujian1) +29 / Guangxi China (Nanning) + \ (guangxi1) +30 / Anhui China (Huainan) + \ (anhui1) +location_constraint> 1 +Option acl. +Canned ACL used when creating buckets and storing or copying objects. +This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Note that this ACL is applied when server-side copying objects as S3 +doesn't copy the ACL from the source but rather writes a fresh one. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \ (private) + / Owner gets FULL_CONTROL. + 2 | The AllUsers group gets READ access. + \ (public-read) + / Owner gets FULL_CONTROL. + 3 | The AllUsers group gets READ and WRITE access. + | Granting this on a bucket is generally not recommended. + \ (public-read-write) + / Owner gets FULL_CONTROL. + 4 | The AuthenticatedUsers group gets READ access. + \ (authenticated-read) + / Object owner gets FULL_CONTROL. +acl> private +Option server_side_encryption. +The server-side encryption algorithm used when storing this object in S3. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / None + \ () + 2 / AES256 + \ (AES256) +server_side_encryption> +Option storage_class. +The storage class to use when storing new objects in ChinaMobile. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / Default + \ () + 2 / Standard storage class + \ (STANDARD) + 3 / Archive storage mode + \ (GLACIER) + 4 / Infrequent access storage mode + \ (STANDARD_IA) +storage_class> +Edit advanced config? +y) Yes +n) No (default) +y/n> n +-------------------- +[ChinaMobile] +type = s3 +provider = ChinaMobile +access_key_id = accesskeyid +secret_access_key = secretaccesskey +endpoint = eos-wuxi-1.cmecloud.cn +location_constraint = wuxi1 +acl = private +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +### ArvanCloud {#arvan-cloud} + +[ArvanCloud](https://www.arvancloud.com/en/products/cloud-storage) ArvanCloud Object Storage goes beyond the limited traditional file storage. +It gives you access to backup and archived files and allows sharing. +Files like profile image in the app, images sent by users or scanned documents can be stored securely and easily in our Object Storage service. + +ArvanCloud provides an S3 interface which can be configured for use with +rclone like this. + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +n/s> n +name> ArvanCloud +Type of storage to configure. +Choose a number from below, or type in your own value +[snip] +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) + \ "s3" +[snip] +Storage> s3 +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" +env_auth> 1 +AWS Access Key ID - leave blank for anonymous access or runtime credentials. +access_key_id> YOURACCESSKEY +AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. +secret_access_key> YOURSECRETACCESSKEY +Region to connect to. +Choose a number from below, or type in your own value + / The default endpoint - a good choice if you are unsure. + 1 | US Region, Northern Virginia, or Pacific Northwest. + | Leave location constraint empty. + \ "us-east-1" +[snip] +region> +Endpoint for S3 API. +Leave blank if using ArvanCloud to use the default endpoint for the region. +Specify if using an S3 clone such as Ceph. +endpoint> s3.arvanstorage.com +Location constraint - must be set to match the Region. Used when creating buckets only. +Choose a number from below, or type in your own value + 1 / Empty for Iran-Tehran Region. + \ "" +[snip] +location_constraint> +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" +[snip] +acl> +The server-side encryption algorithm used when storing this object in S3. +Choose a number from below, or type in your own value + 1 / None + \ "" + 2 / AES256 + \ "AES256" +server_side_encryption> +The storage class to use when storing objects in S3. +Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Standard storage class + \ "STANDARD" +storage_class> +Remote config +-------------------- +[ArvanCloud] +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +region = ir-thr-at1 +endpoint = s3.arvanstorage.com +location_constraint = +acl = +server_side_encryption = +storage_class = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This will leave the config file looking like this. + +``` +[ArvanCloud] +type = s3 +provider = ArvanCloud +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +region = +endpoint = s3.arvanstorage.com +location_constraint = +acl = +server_side_encryption = +storage_class = +``` + ### Tencent COS {#tencent-cos} [Tencent Cloud Object Storage (COS)](https://intl.cloud.tencent.com/product/cos) is a distributed storage service offered by Tencent Cloud for unstructured data. It is secure, stable, massive, convenient, low-delay and low-cost. @@ -19497,7 +21438,7 @@ Choose a number from below, or type in your own value \ "alias" 3 / Amazon Drive \ "amazon cloud drive" - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -19710,8 +21651,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) # Backblaze B2 @@ -19883,6 +21823,11 @@ the file instead of hiding it. Old versions of files, where available, are visible using the `--b2-versions` flag. +It is also possible to view a bucket as it was at a certain point in time, +using the `--b2-version-at` flag. This will show the file versions as they +were at that time, showing files that have been deleted afterwards, and +hiding files that were created since. + If you wish to remove all the old versions then you can use the `rclone cleanup remote:bucket` command which will delete all the old versions of files, leaving the current ones intact. You can also @@ -20033,7 +21978,7 @@ https://f002.backblazeb2.com/file/bucket/path/folder/file3?Authorization=xxxxxxx ### Standard options -Here are the standard options specific to b2 (Backblaze B2). +Here are the Standard options specific to b2 (Backblaze B2). #### --b2-account @@ -20070,7 +22015,7 @@ Properties: ### Advanced options -Here are the advanced options specific to b2 (Backblaze B2). +Here are the Advanced options specific to b2 (Backblaze B2). #### --b2-endpoint @@ -20120,6 +22065,20 @@ Properties: - Type: bool - Default: false +#### --b2-version-at + +Show file versions as they were at the specified time. + +Note that when using this no file write operations are permitted, +so you can't upload files or delete them. + +Properties: + +- Config: version_at +- Env Var: RCLONE_B2_VERSION_AT +- Type: Time +- Default: off + #### --b2-upload-cutoff Cutoff for switching to chunked upload. @@ -20271,8 +22230,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) # Box @@ -20538,7 +22496,7 @@ the `root_folder_id` in the config. ### Standard options -Here are the standard options specific to box (Box). +Here are the Standard options specific to box (Box). #### --box-client-id @@ -20612,7 +22570,7 @@ Properties: ### Advanced options -Here are the advanced options specific to box (Box). +Here are the Advanced options specific to box (Box). #### --box-token @@ -20737,8 +22695,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) # Cache (DEPRECATED) @@ -21044,7 +23001,7 @@ Params: ### Standard options -Here are the standard options specific to cache (Cache a remote). +Here are the Standard options specific to cache (Cache a remote). #### --cache-remote @@ -21160,7 +23117,7 @@ Properties: ### Advanced options -Here are the advanced options specific to cache (Cache a remote). +Here are the Advanced options specific to cache (Cache a remote). #### --cache-plex-token @@ -21409,7 +23366,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](https://rclone.org/commands/rclone_backend/) for more +See the [backend](https://rclone.org/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command @@ -21733,7 +23690,7 @@ Changing `transactions` is dangerous and requires explicit migration. ### Standard options -Here are the standard options specific to chunker (Transparently chunk/split large files). +Here are the Standard options specific to chunker (Transparently chunk/split large files). #### --chunker-remote @@ -21792,7 +23749,7 @@ Properties: ### Advanced options -Here are the advanced options specific to chunker (Transparently chunk/split large files). +Here are the Advanced options specific to chunker (Transparently chunk/split large files). #### --chunker-name-format @@ -22036,7 +23993,7 @@ as they can't be used in JSON strings. ### Standard options -Here are the standard options specific to sharefile (Citrix Sharefile). +Here are the Standard options specific to sharefile (Citrix Sharefile). #### --sharefile-root-folder-id @@ -22065,7 +24022,7 @@ Properties: ### Advanced options -Here are the advanced options specific to sharefile (Citrix Sharefile). +Here are the Advanced options specific to sharefile (Citrix Sharefile). #### --sharefile-upload-cutoff @@ -22137,8 +24094,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) # Crypt @@ -22556,7 +24512,7 @@ check the checksums properly. ### Standard options -Here are the standard options specific to crypt (Encrypt/Decrypt a remote). +Here are the Standard options specific to crypt (Encrypt/Decrypt a remote). #### --crypt-remote @@ -22641,7 +24597,7 @@ Properties: ### Advanced options -Here are the advanced options specific to crypt (Encrypt/Decrypt a remote). +Here are the Advanced options specific to crypt (Encrypt/Decrypt a remote). #### --crypt-server-side-across-configs @@ -22721,6 +24677,12 @@ Properties: - Encode using base32768. Suitable if your remote counts UTF-16 or - Unicode codepoint instead of UTF-8 byte length. (Eg. Onedrive) +### Metadata + +Any metadata supported by the underlying remote is read and written. + +See the [metadata](https://rclone.org/docs/#metadata) docs for more info. + ## Backend commands Here are the commands specific to the crypt backend. @@ -22731,7 +24693,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](https://rclone.org/commands/rclone_backend/) for more +See the [backend](https://rclone.org/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command @@ -22987,7 +24949,7 @@ size of the uncompressed file. The file names should not be changed by anything ### Standard options -Here are the standard options specific to compress (Compress a remote). +Here are the Standard options specific to compress (Compress a remote). #### --compress-remote @@ -23016,7 +24978,7 @@ Properties: ### Advanced options -Here are the advanced options specific to compress (Compress a remote). +Here are the Advanced options specific to compress (Compress a remote). #### --compress-level @@ -23053,6 +25015,170 @@ Properties: - Type: SizeSuffix - Default: 20Mi +### Metadata + +Any metadata supported by the underlying remote is read and written. + +See the [metadata](https://rclone.org/docs/#metadata) docs for more info. + + + +# Combine + +The `combine` backend joins remotes together into a single directory +tree. + +For example you might have a remote for images on one provider: + +``` +$ rclone tree s3:imagesbucket +/ +├── image1.jpg +└── image2.jpg +``` + +And a remote for files on another: + +``` +$ rclone tree drive:important/files +/ +├── file1.txt +└── file2.txt +``` + +The `combine` backend can join these together into a synthetic +directory structure like this: + +``` +$ rclone tree combined: +/ +├── files +│ ├── file1.txt +│ └── file2.txt +└── images + ├── image1.jpg + └── image2.jpg +``` + +You'd do this by specifying an `upstreams` parameter in the config +like this + + upstreams = images=s3:imagesbucket files=drive:important/files + +During the initial setup with `rclone config` you will specify the +upstreams remotes as a space separated list. The upstream remotes can +either be a local paths or other remotes. + +## Configuration + +Here is an example of how to make a combine called `remote` for the +example above. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +... +XX / Combine several remotes into one + \ (combine) +... +Storage> combine +Option upstreams. +Upstreams for combining +These should be in the form + dir=remote:path dir2=remote2:path +Where before the = is specified the root directory and after is the remote to +put there. +Embedded spaces can be added using quotes + "dir=remote:path with space" "dir2=remote2:path with space" +Enter a fs.SpaceSepList value. +upstreams> images=s3:imagesbucket files=drive:important/files +-------------------- +[remote] +type = combine +upstreams = images=s3:imagesbucket files=drive:important/files +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +### Configuring for Google Drive Shared Drives + +Rclone has a convenience feature for making a combine backend for all +the shared drives you have access to. + +Assuming your main (non shared drive) Google drive remote is called +`drive:` you would run + + rclone backend -o config drives drive: + +This would produce something like this: + + [My Drive] + type = alias + remote = drive,team_drive=0ABCDEF-01234567890,root_folder_id=: + + [Test Drive] + type = alias + remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=: + + [AllDrives] + type = combine + remote = "My Drive=My Drive:" "Test Drive=Test Drive:" + +If you then add that config to your config file (find it with `rclone +config file`) then you can access all the shared drives in one place +with the `AllDrives:` remote. + +See [the Google Drive docs](https://rclone.org/drive/#drives) for full info. + + +### Standard options + +Here are the Standard options specific to combine (Combine several remotes into one). + +#### --combine-upstreams + +Upstreams for combining + +These should be in the form + + dir=remote:path dir2=remote2:path + +Where before the = is specified the root directory and after is the remote to +put there. + +Embedded spaces can be added using quotes + + "dir=remote:path with space" "dir2=remote2:path with space" + + + +Properties: + +- Config: upstreams +- Env Var: RCLONE_COMBINE_UPSTREAMS +- Type: SpaceSepList +- Default: + +### Metadata + +Any metadata supported by the underlying remote is read and written. + +See the [metadata](https://rclone.org/docs/#metadata) docs for more info. + # Dropbox @@ -23234,7 +25360,7 @@ finishes up the last batch using this mode. ### Standard options -Here are the standard options specific to dropbox (Dropbox). +Here are the Standard options specific to dropbox (Dropbox). #### --dropbox-client-id @@ -23264,7 +25390,7 @@ Properties: ### Advanced options -Here are the advanced options specific to dropbox (Dropbox). +Here are the Advanced options specific to dropbox (Dropbox). #### --dropbox-token @@ -23687,7 +25813,7 @@ The ID for "S3 Storage" would be `120673761`. ### Standard options -Here are the standard options specific to filefabric (Enterprise File Fabric). +Here are the Standard options specific to filefabric (Enterprise File Fabric). #### --filefabric-url @@ -23746,7 +25872,7 @@ Properties: ### Advanced options -Here are the advanced options specific to filefabric (Enterprise File Fabric). +Here are the Advanced options specific to filefabric (Enterprise File Fabric). #### --filefabric-token @@ -23844,7 +25970,7 @@ Type of storage to configure. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [snip] -XX / FTP Connection +XX / FTP \ "ftp" [snip] Storage> ftp @@ -23943,7 +26069,7 @@ Just hit a selection number when prompted. ### Standard options -Here are the standard options specific to ftp (FTP Connection). +Here are the Standard options specific to ftp (FTP). #### --ftp-host @@ -24026,7 +26152,7 @@ Properties: ### Advanced options -Here are the advanced options specific to ftp (FTP Connection). +Here are the Advanced options specific to ftp (FTP). #### --ftp-concurrency @@ -24072,6 +26198,17 @@ Properties: - Type: bool - Default: false +#### --ftp-disable-utf8 + +Disable using UTF-8 even if server advertises support. + +Properties: + +- Config: disable_utf8 +- Env Var: RCLONE_FTP_DISABLE_UTF8 +- Type: bool +- Default: false + #### --ftp-writing-mdtm Use MDTM to set modification time (VsFtpd quirk) @@ -24200,8 +26337,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) The implementation of : `--dump headers`, `--dump bodies`, `--dump auth` for debugging isn't the same as @@ -24506,7 +26642,7 @@ as they can't be used in JSON strings. ### Standard options -Here are the standard options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). +Here are the Standard options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). #### --gcs-client-id @@ -24781,7 +26917,7 @@ Properties: ### Advanced options -Here are the advanced options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). +Here are the Advanced options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). #### --gcs-token @@ -24820,6 +26956,40 @@ Properties: - Type: string - Required: false +#### --gcs-no-check-bucket + +If set, don't attempt to check the bucket exists or create it. + +This can be useful when trying to minimise the number of transactions +rclone does if you know the bucket exists already. + + +Properties: + +- Config: no_check_bucket +- Env Var: RCLONE_GCS_NO_CHECK_BUCKET +- Type: bool +- Default: false + +#### --gcs-decompress + +If set this will decompress gzip encoded objects. + +It is possible to upload objects to GCS with "Content-Encoding: gzip" +set. Normally rclone will download these files files as compressed objects. + +If this flag is set then rclone will decompress these files with +"Content-Encoding: gzip" as they are received. This means that rclone +can't check the size and hash but the file contents will be decompressed. + + +Properties: + +- Config: decompress +- Env Var: RCLONE_GCS_DECOMPRESS +- Type: bool +- Default: false + #### --gcs-encoding The encoding for the backend. @@ -24842,8 +27012,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) # Google Drive @@ -24900,8 +27069,6 @@ Choose a number from below, or type in your own value 5 | does not allow any access to read or download file content. \ "drive.metadata.readonly" scope> 1 -ID of the root folder - leave blank normally. Fill in to access "Computers" folders. (see docs). -root_folder_id> Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. service_account_file> Remote config @@ -25003,7 +27170,7 @@ directories. ### Root folder ID -You can set the `root_folder_id` for rclone. This is the directory +This option has been moved to the advanced section. You can set the `root_folder_id` for rclone. This is the directory (identified by its `Folder ID`) that rclone considers to be the root of your drive. @@ -25351,23 +27518,28 @@ represent the currently available conversions. | Extension | Mime Type | Description | | --------- |-----------| ------------| +| bmp | image/bmp | Windows Bitmap format | | csv | text/csv | Standard CSV format for Spreadsheets | +| doc | application/msword | Classic Word file | | docx | application/vnd.openxmlformats-officedocument.wordprocessingml.document | Microsoft Office Document | | epub | application/epub+zip | E-book format | | html | text/html | An HTML Document | | jpg | image/jpeg | A JPEG Image File | -| json | application/vnd.google-apps.script+json | JSON Text Format | +| json | application/vnd.google-apps.script+json | JSON Text Format for Google Apps scripts | | odp | application/vnd.oasis.opendocument.presentation | Openoffice Presentation | | ods | application/vnd.oasis.opendocument.spreadsheet | Openoffice Spreadsheet | | ods | application/x-vnd.oasis.opendocument.spreadsheet | Openoffice Spreadsheet | | odt | application/vnd.oasis.opendocument.text | Openoffice Document | | pdf | application/pdf | Adobe PDF Format | +| pjpeg | image/pjpeg | Progressive JPEG Image | | png | image/png | PNG Image Format| | pptx | application/vnd.openxmlformats-officedocument.presentationml.presentation | Microsoft Office Powerpoint | | rtf | application/rtf | Rich Text Format | | svg | image/svg+xml | Scalable Vector Graphics Format | | tsv | text/tab-separated-values | Standard TSV format for spreadsheets | | txt | text/plain | Plain Text | +| wmf | application/x-msmetafile | Windows Meta File | +| xls | application/vnd.ms-excel | Classic Excel file | | xlsx | application/vnd.openxmlformats-officedocument.spreadsheetml.sheet | Microsoft Office Spreadsheet | | zip | application/zip | A ZIP file of HTML, Images CSS | @@ -25387,7 +27559,7 @@ Google Documents. ### Standard options -Here are the standard options specific to drive (Google Drive). +Here are the Standard options specific to drive (Google Drive). #### --drive-client-id @@ -25442,22 +27614,6 @@ Properties: - Allows read-only access to file metadata but - does not allow any access to read or download file content. -#### --drive-root-folder-id - -ID of the root folder. -Leave blank normally. - -Fill in to access "Computers" folders (see docs), or for rclone to use -a non root folder as its starting point. - - -Properties: - -- Config: root_folder_id -- Env Var: RCLONE_DRIVE_ROOT_FOLDER_ID -- Type: string -- Required: false - #### --drive-service-account-file Service Account Credentials JSON file path. @@ -25487,7 +27643,7 @@ Properties: ### Advanced options -Here are the advanced options specific to drive (Google Drive). +Here are the Advanced options specific to drive (Google Drive). #### --drive-token @@ -25526,6 +27682,22 @@ Properties: - Type: string - Required: false +#### --drive-root-folder-id + +ID of the root folder. +Leave blank normally. + +Fill in to access "Computers" folders (see docs), or for rclone to use +a non root folder as its starting point. + + +Properties: + +- Config: root_folder_id +- Env Var: RCLONE_DRIVE_ROOT_FOLDER_ID +- Type: string +- Required: false + #### --drive-service-account-credentials Service Account Credentials JSON blob. @@ -26006,6 +28178,34 @@ Properties: - Type: bool - Default: false +#### --drive-resource-key + +Resource key for accessing a link-shared file. + +If you need to access files shared with a link like this + + https://drive.google.com/drive/folders/XXX?resourcekey=YYY&usp=sharing + +Then you will need to use the first part "XXX" as the "root_folder_id" +and the second part "YYY" as the "resource_key" otherwise you will get +404 not found errors when trying to access the directory. + +See: https://developers.google.com/drive/api/guides/resource-keys + +This resource key requirement only applies to a subset of old files. + +Note also that opening the folder once in the web interface (with the +user you've authenticated rclone with) seems to be enough so that the +resource key is no needed. + + +Properties: + +- Config: resource_key +- Env Var: RCLONE_DRIVE_RESOURCE_KEY +- Type: string +- Required: false + #### --drive-encoding The encoding for the backend. @@ -26029,7 +28229,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](https://rclone.org/commands/rclone_backend/) for more +See the [backend](https://rclone.org/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command @@ -26131,7 +28331,7 @@ This will return a JSON list of objects like this With the -o config parameter it will output the list in a format suitable for adding to a config file to make aliases for all the -drives found. +drives found and a combined drive. [My Drive] type = alias @@ -26141,10 +28341,15 @@ drives found. type = alias remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=: -Adding this to the rclone config file will cause those team drives to -be accessible with the aliases shown. This may require manual editing -of the names. + [AllDrives] + type = combine + remote = "My Drive=My Drive:" "Test Drive=Test Drive:" +Adding this to the rclone config file will cause those team drives to +be accessible with the aliases shown. Any illegal charactes will be +substituted with "_" and duplicate names will have numbers suffixed. +It will also add a remote called AllDrives which shows all the shared +drives combined into one directory tree. ### untrash @@ -26201,6 +28406,18 @@ attempted if possible. Use the -i flag to see what would be copied before copying. +### exportformats + +Dump the export formats for debug purposes + + rclone backend exportformats remote: [options] [+] + +### importformats + +Dump the import formats for debug purposes + + rclone backend importformats remote: [options] [+] + ## Limitations @@ -26217,8 +28434,11 @@ and upload the files if you prefer. ### Limitations of Google Docs -Google docs will appear as size -1 in `rclone ls` and as size 0 in -anything which uses the VFS layer, e.g. `rclone mount`, `rclone serve`. +Google docs will appear as size -1 in `rclone ls`, `rclone ncdu` etc, +and as size 0 in anything which uses the VFS layer, e.g. `rclone mount` +and `rclone serve`. When calculating directory totals, e.g. in +`rclone size` and `rclone ncdu`, they will be counted in as empty +files. This is because rclone can't find out the size of the Google docs without downloading them. @@ -26297,8 +28517,9 @@ enter "Developer Contact Email" (your own email is OK); then click on "Save" (al Click again on "Credentials" on the left panel to go back to the "Credentials" screen. -(PS: if you are a GSuite user, you could also select "Internal" instead -of "External" above, but this has not been tested/documented so far). + (PS: if you are a GSuite user, you could also select "Internal" instead +of "External" above, but this will restrict API use to Google Workspace +users in your organisation). 6. Click on the "+ CREATE CREDENTIALS" button at the top of the screen, then select "OAuth client ID". @@ -26306,14 +28527,18 @@ then select "OAuth client ID". 7. Choose an application type of "Desktop app" and click "Create". (the default name is fine) 8. It will show you a client ID and client secret. Make a note of these. + + (If you selected "External" at Step 5 continue to "Publish App" in the Steps 9 and 10. + If you chose "Internal" you don't need to publish and can skip straight to + Step 11.) 9. Go to "Oauth consent screen" and press "Publish App" -10. Provide the noted client ID and client secret to rclone. - -11. Click "OAuth consent screen", then click "PUBLISH APP" button and +10. Click "OAuth consent screen", then click "PUBLISH APP" button and confirm, or add your account under "Test users". +11. Provide the noted client ID and client secret to rclone. + Be aware that, due to the "enhanced security" recently introduced by Google, you are theoretically expected to "submit your app for verification" and then wait a few weeks(!) for their response; in practice, you can go right @@ -26552,7 +28777,7 @@ This is similar to the Sharing tab in the Google Photos web interface. ### Standard options -Here are the standard options specific to google photos (Google Photos). +Here are the Standard options specific to google photos (Google Photos). #### --gphotos-client-id @@ -26596,7 +28821,7 @@ Properties: ### Advanced options -Here are the advanced options specific to google photos (Google Photos). +Here are the Advanced options specific to google photos (Google Photos). #### --gphotos-token @@ -26964,7 +29189,7 @@ or by full re-read/re-write of the files. ### Standard options -Here are the standard options specific to hasher (Better checksums for other remotes). +Here are the Standard options specific to hasher (Better checksums for other remotes). #### --hasher-remote @@ -27001,7 +29226,7 @@ Properties: ### Advanced options -Here are the advanced options specific to hasher (Better checksums for other remotes). +Here are the Advanced options specific to hasher (Better checksums for other remotes). #### --hasher-auto-size @@ -27014,6 +29239,12 @@ Properties: - Type: SizeSuffix - Default: 0 +### Metadata + +Any metadata supported by the underlying remote is read and written. + +See the [metadata](https://rclone.org/docs/#metadata) docs for more info. + ## Backend commands Here are the commands specific to the hasher backend. @@ -27024,7 +29255,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](https://rclone.org/commands/rclone_backend/) for more +See the [backend](https://rclone.org/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command @@ -27277,7 +29508,7 @@ Invalid UTF-8 bytes will also be [replaced](https://rclone.org/overview/#invalid ### Standard options -Here are the standard options specific to hdfs (Hadoop distributed file system). +Here are the Standard options specific to hdfs (Hadoop distributed file system). #### --hdfs-namenode @@ -27308,7 +29539,7 @@ Properties: ### Advanced options -Here are the advanced options specific to hdfs (Hadoop distributed file system). +Here are the Advanced options specific to hdfs (Hadoop distributed file system). #### --hdfs-service-principal-name @@ -27364,6 +29595,444 @@ Properties: - No server-side `Move` or `DirMove`. - Checksums not implemented. +# HiDrive + +Paths are specified as `remote:path` + +Paths may be as deep as required, e.g. `remote:directory/subdirectory`. + +The initial setup for hidrive involves getting a token from HiDrive +which you need to do in your browser. +`rclone config` walks you through it. + +## Configuration + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + +``` +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value +[snip] +XX / HiDrive + \ "hidrive" +[snip] +Storage> hidrive +OAuth Client Id - Leave blank normally. +client_id> +OAuth Client Secret - Leave blank normally. +client_secret> +Access permissions that rclone should use when requesting access from HiDrive. +Leave blank normally. +scope_access> +Edit advanced config? +y/n> n +Use auto config? +y/n> y +If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +type = hidrive +token = {"access_token":"xxxxxxxxxxxxxxxxxxxx","token_type":"Bearer","refresh_token":"xxxxxxxxxxxxxxxxxxxxxxx","expiry":"xxxxxxxxxxxxxxxxxxxxxxx"} +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +**You should be aware that OAuth-tokens can be used to access your account +and hence should not be shared with other persons.** +See the [below section](#keeping-your-tokens-safe) for more information. + +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from HiDrive. This only runs from the moment it opens +your browser to the moment you get back the verification code. +The webserver runs on `http://127.0.0.1:53682/`. +If local port `53682` is protected by a firewall you may need to temporarily +unblock the firewall to complete authorization. + +Once configured you can then use `rclone` like this, + +List directories in top level of your HiDrive root folder + + rclone lsd remote: + +List all the files in your HiDrive filesystem + + rclone ls remote: + +To copy a local directory to a HiDrive directory called backup + + rclone copy /home/source remote:backup + +### Keeping your tokens safe + +Any OAuth-tokens will be stored by rclone in the remote's configuration file as unencrypted text. +Anyone can use a valid refresh-token to access your HiDrive filesystem without knowing your password. +Therefore you should make sure no one else can access your configuration. + +It is possible to encrypt rclone's configuration file. +You can find information on securing your configuration file by viewing the [configuration encryption docs](https://rclone.org/docs/#configuration-encryption). + +### Invalid refresh token + +As can be verified [here](https://developer.hidrive.com/basics-flows/), +each `refresh_token` (for Native Applications) is valid for 60 days. +If used to access HiDrivei, its validity will be automatically extended. + +This means that if you + + * Don't use the HiDrive remote for 60 days + +then rclone will return an error which includes a text +that implies the refresh token is *invalid* or *expired*. + +To fix this you will need to authorize rclone to access your HiDrive account again. + +Using + + rclone config reconnect remote: + +the process is very similar to the process of initial setup exemplified before. + +### Modified time and hashes + +HiDrive allows modification times to be set on objects accurate to 1 second. + +HiDrive supports [its own hash type](https://static.hidrive.com/dev/0001) +which is used to verify the integrety of file contents after successful transfers. + +### Restricted filename characters + +HiDrive cannot store files or folders that include +`/` (0x2F) or null-bytes (0x00) in their name. +Any other characters can be used in the names of files or folders. +Additionally, files or folders cannot be named either of the following: `.` or `..` + +Therefore rclone will automatically replace these characters, +if files or folders are stored or accessed with such names. + +You can read about how this filename encoding works in general +[here](overview/#restricted-filenames). + +Keep in mind that HiDrive only supports file or folder names +with a length of 255 characters or less. + +### Transfers + +HiDrive limits file sizes per single request to a maximum of 2 GiB. +To allow storage of larger files and allow for better upload performance, +the hidrive backend will use a chunked transfer for files larger than 96 MiB. +Rclone will upload multiple parts/chunks of the file at the same time. +Chunks in the process of being uploaded are buffered in memory, +so you may want to restrict this behaviour on systems with limited resources. + +You can customize this behaviour using the following options: + +* `chunk_size`: size of file parts +* `upload_cutoff`: files larger or equal to this in size will use a chunked transfer +* `upload_concurrency`: number of file-parts to upload at the same time + +See the below section about configuration options for more details. + +### Root folder + +You can set the root folder for rclone. +This is the directory that rclone considers to be the root of your HiDrive. + +Usually, you will leave this blank, and rclone will use the root of the account. + +However, you can set this to restrict rclone to a specific folder hierarchy. + +This works by prepending the contents of the `root_prefix` option +to any paths accessed by rclone. +For example, the following two ways to access the home directory are equivalent: + + rclone lsd --hidrive-root-prefix="/users/test/" remote:path + + rclone lsd remote:/users/test/path + +See the below section about configuration options for more details. + +### Directory member count + +By default, rclone will know the number of directory members contained in a directory. +For example, `rclone lsd` uses this information. + +The acquisition of this information will result in additional time costs for HiDrive's API. +When dealing with large directory structures, it may be desirable to circumvent this time cost, +especially when this information is not explicitly needed. +For this, the `disable_fetching_member_count` option can be used. + +See the below section about configuration options for more details. + + +### Standard options + +Here are the Standard options specific to hidrive (HiDrive). + +#### --hidrive-client-id + +OAuth Client Id. + +Leave blank normally. + +Properties: + +- Config: client_id +- Env Var: RCLONE_HIDRIVE_CLIENT_ID +- Type: string +- Required: false + +#### --hidrive-client-secret + +OAuth Client Secret. + +Leave blank normally. + +Properties: + +- Config: client_secret +- Env Var: RCLONE_HIDRIVE_CLIENT_SECRET +- Type: string +- Required: false + +#### --hidrive-scope-access + +Access permissions that rclone should use when requesting access from HiDrive. + +Properties: + +- Config: scope_access +- Env Var: RCLONE_HIDRIVE_SCOPE_ACCESS +- Type: string +- Default: "rw" +- Examples: + - "rw" + - Read and write access to resources. + - "ro" + - Read-only access to resources. + +### Advanced options + +Here are the Advanced options specific to hidrive (HiDrive). + +#### --hidrive-token + +OAuth Access Token as a JSON blob. + +Properties: + +- Config: token +- Env Var: RCLONE_HIDRIVE_TOKEN +- Type: string +- Required: false + +#### --hidrive-auth-url + +Auth server URL. + +Leave blank to use the provider defaults. + +Properties: + +- Config: auth_url +- Env Var: RCLONE_HIDRIVE_AUTH_URL +- Type: string +- Required: false + +#### --hidrive-token-url + +Token server url. + +Leave blank to use the provider defaults. + +Properties: + +- Config: token_url +- Env Var: RCLONE_HIDRIVE_TOKEN_URL +- Type: string +- Required: false + +#### --hidrive-scope-role + +User-level that rclone should use when requesting access from HiDrive. + +Properties: + +- Config: scope_role +- Env Var: RCLONE_HIDRIVE_SCOPE_ROLE +- Type: string +- Default: "user" +- Examples: + - "user" + - User-level access to management permissions. + - This will be sufficient in most cases. + - "admin" + - Extensive access to management permissions. + - "owner" + - Full access to management permissions. + +#### --hidrive-root-prefix + +The root/parent folder for all paths. + +Fill in to use the specified folder as the parent for all paths given to the remote. +This way rclone can use any folder as its starting point. + +Properties: + +- Config: root_prefix +- Env Var: RCLONE_HIDRIVE_ROOT_PREFIX +- Type: string +- Default: "/" +- Examples: + - "/" + - The topmost directory accessible by rclone. + - This will be equivalent with "root" if rclone uses a regular HiDrive user account. + - "root" + - The topmost directory of the HiDrive user account + - "" + - This specifies that there is no root-prefix for your paths. + - When using this you will always need to specify paths to this remote with a valid parent e.g. "remote:/path/to/dir" or "remote:root/path/to/dir". + +#### --hidrive-endpoint + +Endpoint for the service. + +This is the URL that API-calls will be made to. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_HIDRIVE_ENDPOINT +- Type: string +- Default: "https://api.hidrive.strato.com/2.1" + +#### --hidrive-disable-fetching-member-count + +Do not fetch number of objects in directories unless it is absolutely necessary. + +Requests may be faster if the number of objects in subdirectories is not fetched. + +Properties: + +- Config: disable_fetching_member_count +- Env Var: RCLONE_HIDRIVE_DISABLE_FETCHING_MEMBER_COUNT +- Type: bool +- Default: false + +#### --hidrive-chunk-size + +Chunksize for chunked uploads. + +Any files larger than the configured cutoff (or files of unknown size) will be uploaded in chunks of this size. + +The upper limit for this is 2147483647 bytes (about 2.000Gi). +That is the maximum amount of bytes a single upload-operation will support. +Setting this above the upper limit or to a negative value will cause uploads to fail. + +Setting this to larger values may increase the upload speed at the cost of using more memory. +It can be set to smaller values smaller to save on memory. + +Properties: + +- Config: chunk_size +- Env Var: RCLONE_HIDRIVE_CHUNK_SIZE +- Type: SizeSuffix +- Default: 48Mi + +#### --hidrive-upload-cutoff + +Cutoff/Threshold for chunked uploads. + +Any files larger than this will be uploaded in chunks of the configured chunksize. + +The upper limit for this is 2147483647 bytes (about 2.000Gi). +That is the maximum amount of bytes a single upload-operation will support. +Setting this above the upper limit will cause uploads to fail. + +Properties: + +- Config: upload_cutoff +- Env Var: RCLONE_HIDRIVE_UPLOAD_CUTOFF +- Type: SizeSuffix +- Default: 96Mi + +#### --hidrive-upload-concurrency + +Concurrency for chunked uploads. + +This is the upper limit for how many transfers for the same file are running concurrently. +Setting this above to a value smaller than 1 will cause uploads to deadlock. + +If you are uploading small numbers of large files over high-speed links +and these uploads do not fully utilize your bandwidth, then increasing +this may help to speed up the transfers. + +Properties: + +- Config: upload_concurrency +- Env Var: RCLONE_HIDRIVE_UPLOAD_CONCURRENCY +- Type: int +- Default: 4 + +#### --hidrive-encoding + +The encoding for the backend. + +See the [encoding section in the overview](https://rclone.org/overview/#encoding) for more info. + +Properties: + +- Config: encoding +- Env Var: RCLONE_HIDRIVE_ENCODING +- Type: MultiEncoder +- Default: Slash,Dot + + + +## Limitations + +### Symbolic links + +HiDrive is able to store symbolic links (*symlinks*) by design, +for example, when unpacked from a zip archive. + +There exists no direct mechanism to manage native symlinks in remotes. +As such this implementation has chosen to ignore any native symlinks present in the remote. +rclone will not be able to access or show any symlinks stored in the hidrive-remote. +This means symlinks cannot be individually removed, copied, or moved, +except when removing, copying, or moving the parent folder. + +*This does not affect the `.rclonelink`-files +that rclone uses to encode and store symbolic links.* + +### Sparse files + +It is possible to store sparse files in HiDrive. + +Note that copying a sparse file will expand the holes +into null-byte (0x00) regions that will then consume disk space. +Likewise, when downloading a sparse file, +the resulting file will have null-byte regions in the place of file holes. + # HTTP The HTTP remote is a read only remote for reading files of a @@ -27413,7 +30082,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / http Connection +XX / HTTP \ "http" [snip] Storage> http @@ -27487,11 +30156,11 @@ or: ### Standard options -Here are the standard options specific to http (http Connection). +Here are the Standard options specific to http (HTTP). #### --http-url -URL of http host to connect to. +URL of HTTP host to connect to. E.g. "https://example.com", or "https://user:pass@example.com" to use a username and password. @@ -27504,7 +30173,7 @@ Properties: ### Advanced options -Here are the advanced options specific to http (http Connection). +Here are the Advanced options specific to http (HTTP). #### --http-headers @@ -27581,8 +30250,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) # Hubic @@ -27690,7 +30358,7 @@ are the same. ### Standard options -Here are the standard options specific to hubic (Hubic). +Here are the Standard options specific to hubic (Hubic). #### --hubic-client-id @@ -27720,7 +30388,7 @@ Properties: ### Advanced options -Here are the advanced options specific to hubic (Hubic). +Here are the Advanced options specific to hubic (Hubic). #### --hubic-token @@ -27818,6 +30486,279 @@ The Swift API doesn't return a correct MD5SUM for segmented files (Dynamic or Static Large Objects) so rclone won't check or use the MD5SUM for these. +# Internet Archive + +The Internet Archive backend utilizes Items on [archive.org](https://archive.org/) + +Refer to [IAS3 API documentation](https://archive.org/services/docs/api/ias3.html) for the API this backend uses. + +Paths are specified as `remote:bucket` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`. + +Once you have made a remote (see the provider specific section above) +you can use it like this: + +Unlike S3, listing up all items uploaded by you isn't supported. + +Make a new item + + rclone mkdir remote:item + +List the contents of a item + + rclone ls remote:item + +Sync `/home/local/directory` to the remote item, deleting any excess +files in the item. + + rclone sync -i /home/local/directory remote:item + +## Notes +Because of Internet Archive's architecture, it enqueues write operations (and extra post-processings) in a per-item queue. You can check item's queue at https://catalogd.archive.org/history/item-name-here . Because of that, all uploads/deletes will not show up immediately and takes some time to be available. +The per-item queue is enqueued to an another queue, Item Deriver Queue. [You can check the status of Item Deriver Queue here.](https://catalogd.archive.org/catalog.php?whereami=1) This queue has a limit, and it may block you from uploading, or even deleting. You should avoid uploading a lot of small files for better behavior. + +You can optionally wait for the server's processing to finish, by setting non-zero value to `wait_archive` key. +By making it wait, rclone can do normal file comparison. +Make sure to set a large enough value (e.g. `30m0s` for smaller files) as it can take a long time depending on server's queue. + +## About metadata +This backend supports setting, updating and reading metadata of each file. +The metadata will appear as file metadata on Internet Archive. +However, some fields are reserved by both Internet Archive and rclone. + +The following are reserved by Internet Archive: +- `name` +- `source` +- `size` +- `md5` +- `crc32` +- `sha1` +- `format` +- `old_version` +- `viruscheck` + +Trying to set values to these keys is ignored with a warning. +Only setting `mtime` is an exception. Doing so make it the identical behavior as setting ModTime. + +rclone reserves all the keys starting with `rclone-`. Setting value for these keys will give you warnings, but values are set according to request. + +If there are multiple values for a key, only the first one is returned. +This is a limitation of rclone, that supports one value per one key. +It can be triggered when you did a server-side copy. + +Reading metadata will also provide custom (non-standard nor reserved) ones. + +## Configuration + +Here is an example of making an internetarchive configuration. +Most applies to the other providers as well, any differences are described [below](#providers). + +First run + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +XX / InternetArchive Items + \ (internetarchive) +Storage> internetarchive +Option access_key_id. +IAS3 Access Key. +Leave blank for anonymous access. +You can find one here: https://archive.org/account/s3.php +Enter a value. Press Enter to leave empty. +access_key_id> XXXX +Option secret_access_key. +IAS3 Secret Key (password). +Leave blank for anonymous access. +Enter a value. Press Enter to leave empty. +secret_access_key> XXXX +Edit advanced config? +y) Yes +n) No (default) +y/n> y +Option endpoint. +IAS3 Endpoint. +Leave blank for default value. +Enter a string value. Press Enter for the default (https://s3.us.archive.org). +endpoint> +Option front_endpoint. +Host of InternetArchive Frontend. +Leave blank for default value. +Enter a string value. Press Enter for the default (https://archive.org). +front_endpoint> +Option disable_checksum. +Don't store MD5 checksum with object metadata. +Normally rclone will calculate the MD5 checksum of the input before +uploading it so it can ask the server to check the object against checksum. +This is great for data integrity checking but can cause long delays for +large files to start uploading. +Enter a boolean value (true or false). Press Enter for the default (true). +disable_checksum> true +Option encoding. +The encoding for the backend. +See the [encoding section in the overview](https://rclone.org/overview/#encoding) for more info. +Enter a encoder.MultiEncoder value. Press Enter for the default (Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot). +encoding> +Edit advanced config? +y) Yes +n) No (default) +y/n> n +-------------------- +[remote] +type = internetarchive +access_key_id = XXXX +secret_access_key = XXXX +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + + +### Standard options + +Here are the Standard options specific to internetarchive (Internet Archive). + +#### --internetarchive-access-key-id + +IAS3 Access Key. + +Leave blank for anonymous access. +You can find one here: https://archive.org/account/s3.php + +Properties: + +- Config: access_key_id +- Env Var: RCLONE_INTERNETARCHIVE_ACCESS_KEY_ID +- Type: string +- Required: false + +#### --internetarchive-secret-access-key + +IAS3 Secret Key (password). + +Leave blank for anonymous access. + +Properties: + +- Config: secret_access_key +- Env Var: RCLONE_INTERNETARCHIVE_SECRET_ACCESS_KEY +- Type: string +- Required: false + +### Advanced options + +Here are the Advanced options specific to internetarchive (Internet Archive). + +#### --internetarchive-endpoint + +IAS3 Endpoint. + +Leave blank for default value. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_INTERNETARCHIVE_ENDPOINT +- Type: string +- Default: "https://s3.us.archive.org" + +#### --internetarchive-front-endpoint + +Host of InternetArchive Frontend. + +Leave blank for default value. + +Properties: + +- Config: front_endpoint +- Env Var: RCLONE_INTERNETARCHIVE_FRONT_ENDPOINT +- Type: string +- Default: "https://archive.org" + +#### --internetarchive-disable-checksum + +Don't ask the server to test against MD5 checksum calculated by rclone. +Normally rclone will calculate the MD5 checksum of the input before +uploading it so it can ask the server to check the object against checksum. +This is great for data integrity checking but can cause long delays for +large files to start uploading. + +Properties: + +- Config: disable_checksum +- Env Var: RCLONE_INTERNETARCHIVE_DISABLE_CHECKSUM +- Type: bool +- Default: true + +#### --internetarchive-wait-archive + +Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish. +Only enable if you need to be guaranteed to be reflected after write operations. +0 to disable waiting. No errors to be thrown in case of timeout. + +Properties: + +- Config: wait_archive +- Env Var: RCLONE_INTERNETARCHIVE_WAIT_ARCHIVE +- Type: Duration +- Default: 0s + +#### --internetarchive-encoding + +The encoding for the backend. + +See the [encoding section in the overview](https://rclone.org/overview/#encoding) for more info. + +Properties: + +- Config: encoding +- Env Var: RCLONE_INTERNETARCHIVE_ENCODING +- Type: MultiEncoder +- Default: Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot + +### Metadata + +Metadata fields provided by Internet Archive. +If there are multiple values for a key, only the first one is returned. +This is a limitation of Rclone, that supports one value per one key. + +Owner is able to add custom keys. Metadata feature grabs all the keys including them. + +Here are the possible system metadata items for the internetarchive backend. + +| Name | Help | Type | Example | Read Only | +|------|------|------|---------|-----------| +| crc32 | CRC32 calculated by Internet Archive | string | 01234567 | N | +| format | Name of format identified by Internet Archive | string | Comma-Separated Values | N | +| md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | N | +| mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | +| name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | N | +| old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | N | +| rclone-ia-mtime | Time of last modification, managed by Internet Archive | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | +| rclone-mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | +| rclone-update-track | Random value used by Rclone for tracking changes inside Internet Archive | string | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | N | +| sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | N | +| size | File size in bytes | decimal number | 123456 | N | +| source | The source of the file | string | original | N | +| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | N | + +See the [metadata](https://rclone.org/docs/#metadata) docs for more info. + + + # Jottacloud Jottacloud is a cloud storage service provider from a Norwegian company, using its own datacenters @@ -27890,60 +30831,83 @@ s) Set configuration password q) Quit config n/s/q> n name> remote +Option Storage. Type of storage to configure. -Enter a string value. Press Enter for the default (""). -Choose a number from below, or type in your own value +Choose a number from below, or type in your own value. [snip] XX / Jottacloud - \ "jottacloud" + \ (jottacloud) [snip] Storage> jottacloud -** See help for jottacloud backend at: https://rclone.org/jottacloud/ ** - -Edit advanced config? (y/n) -y) Yes -n) No -y/n> n -Remote config -Use legacy authentication?. -This is only required for certain whitelabel versions of Jottacloud and not recommended for normal users. +Edit advanced config? y) Yes n) No (default) y/n> n - -Generate a personal login token here: https://www.jottacloud.com/web/secure +Option config_type. +Select authentication type. +Choose a number from below, or type in an existing string value. +Press Enter for the default (standard). + / Standard authentication. + 1 | Use this if you're a normal Jottacloud user. + \ (standard) + / Legacy authentication. + 2 | This is only required for certain whitelabel versions of Jottacloud and not recommended for normal users. + \ (legacy) + / Telia Cloud authentication. + 3 | Use this if you are using Telia Cloud. + \ (telia) + / Tele2 Cloud authentication. + 4 | Use this if you are using Tele2 Cloud. + \ (tele2) +config_type> 1 +Personal login token. +Generate here: https://www.jottacloud.com/web/secure Login Token> - -Do you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client? - +Use a non-standard device/mountpoint? +Choosing no, the default, will let you access the storage used for the archive +section of the official Jottacloud client. If you instead want to access the +sync or the backup section, for example, you must choose yes. y) Yes -n) No +n) No (default) y/n> y -Please select the device to use. Normally this will be Jotta -Choose a number from below, or type in an existing value +Option config_device. +The device to use. In standard setup the built-in Jotta device is used, +which contains predefined mountpoints for archive, sync etc. All other devices +are treated as backup devices by the official Jottacloud client. You may create +a new by entering a unique name. +Choose a number from below, or type in your own string value. +Press Enter for the default (DESKTOP-3H31129). 1 > DESKTOP-3H31129 2 > Jotta -Devices> 2 -Please select the mountpoint to user. Normally this will be Archive -Choose a number from below, or type in an existing value +config_device> 2 +Option config_mountpoint. +The mountpoint to use for the built-in device Jotta. +The standard setup is to use the Archive mountpoint. Most other mountpoints +have very limited support in rclone and should generally be avoided. +Choose a number from below, or type in an existing string value. +Press Enter for the default (Archive). 1 > Archive - 2 > Links + 2 > Shared 3 > Sync - -Mountpoints> 1 +config_mountpoint> 1 -------------------- -[jotta] +[remote] type = jottacloud +configVersion = 1 +client_id = jottacli +client_secret = +tokenURL = https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token token = {........} +username = 2940e57271a93d987d6f8a21 device = Jotta mountpoint = Archive -configVersion = 1 -------------------- -y) Yes this is OK +y) Yes this is OK (default) e) Edit this remote d) Delete this remote y/e/d> y ``` + Once configured you can then use `rclone` like this, List directories in top level of your Jottacloud @@ -27960,19 +30924,27 @@ To copy a local directory to an Jottacloud directory called backup ### Devices and Mountpoints -The official Jottacloud client registers a device for each computer you install it on, -and then creates a mountpoint for each folder you select for Backup. -The web interface uses a special device called Jotta for the Archive and Sync mountpoints. +The official Jottacloud client registers a device for each computer you install +it on, and shows them in the backup section of the user interface. For each +folder you select for backup it will create a mountpoint within this device. +A built-in device called Jotta is special, and contains mountpoints Archive, +Sync and some others, used for corresponding features in official clients. -With rclone you'll want to use the Jotta/Archive device/mountpoint in most cases, however if you -want to access files uploaded by any of the official clients rclone provides the option to select -other devices and mountpoints during config. Note that uploading files is currently not supported -to other devices than Jotta. +With rclone you'll want to use the standard Jotta/Archive device/mountpoint in +most cases. However, you may for example want to access files from the sync or +backup functionality provided by the official clients, and rclone therefore +provides the option to select other devices and mountpoints during config. -The built-in Jotta device may also contain several other mountpoints, such as: Latest, Links, Shared and Trash. -These are special mountpoints with a different internal representation than the "regular" mountpoints. -Rclone will only to a very limited degree support them. Generally you should avoid these, unless you know what you -are doing. +You are allowed to create new devices and mountpoints. All devices except the +built-in Jotta device are treated as backup devices by official Jottacloud +clients, and the mountpoints on them are individual backup sets. + +With the built-in Jotta device, only existing, built-in, mountpoints can be +selected. In addition to the mentioned Archive and Sync, it may contain +several other mountpoints such as: Latest, Links, Shared and Trash. All of +these are special mountpoints with a different internal representation than +the "regular" mountpoints. Rclone will only to a very limited degree support +them. Generally you should avoid these, unless you know what you are doing. ### --fast-list @@ -28050,7 +31022,7 @@ and the current usage. ### Advanced options -Here are the advanced options specific to jottacloud (Jottacloud). +Here are the Advanced options specific to jottacloud (Jottacloud). #### --jottacloud-md5-memory-limit @@ -28249,7 +31221,7 @@ as they can't be used in XML strings. ### Standard options -Here are the standard options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). +Here are the Standard options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). #### --koofr-provider @@ -28336,7 +31308,7 @@ Properties: ### Advanced options -Here are the advanced options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). +Here are the Advanced options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). #### --koofr-mountid @@ -28683,7 +31655,7 @@ as they can't be used in JSON strings. ### Standard options -Here are the standard options specific to mailru (Mail.ru Cloud). +Here are the Standard options specific to mailru (Mail.ru Cloud). #### --mailru-user @@ -28736,7 +31708,7 @@ Properties: ### Advanced options -Here are the advanced options specific to mailru (Mail.ru Cloud). +Here are the Advanced options specific to mailru (Mail.ru Cloud). #### --mailru-speedup-file-patterns @@ -28972,6 +31944,44 @@ Use `rclone dedupe` to fix duplicated files. ### Failure to log-in +#### Object not found + +If you are connecting to your Mega remote for the first time, +to test access and syncronisation, you may receive an error such as + +``` +Failed to create file system for "my-mega-remote:": +couldn't login: Object (typically, node or user) not found +``` + +The diagnostic steps often recommended in the [rclone forum](https://forum.rclone.org/search?q=mega) +start with the **MEGAcmd** utility. Note that this refers to +the official C++ command from https://github.com/meganz/MEGAcmd +and not the go language built command from t3rm1n4l/megacmd +that is no longer maintained. + +Follow the instructions for installing MEGAcmd and try accessing +your remote as they recommend. You can establish whether or not +you can log in using MEGAcmd, and obtain diagnostic information +to help you, and search or work with others in the forum. + +``` +MEGA CMD> login me@example.com +Password: +Fetching nodes ... +Loading transfers from local cache +Login complete as me@example.com +me@example.com:/$ +``` + +Note that some have found issues with passwords containing special +characters. If you can not log on with rclone, but MEGAcmd logs on +just fine, then consider changing your password temporarily to +pure alphanumeric characters, in case that helps. + + +#### Repeated commands blocks access + Mega remotes seem to get blocked (reject logins) under "heavy use". We haven't worked out the exact blocking rules but it seems to be related to fast paced, successive rclone commands. @@ -29019,7 +32029,7 @@ have got the remote blocked for a while. ### Standard options -Here are the standard options specific to mega (Mega). +Here are the Standard options specific to mega (Mega). #### --mega-user @@ -29047,7 +32057,7 @@ Properties: ### Advanced options -Here are the advanced options specific to mega (Mega). +Here are the Advanced options specific to mega (Mega). #### --mega-debug @@ -29164,8 +32174,7 @@ set](https://rclone.org/overview/#restricted-characters). - Akamai NetStorage -------------------------------------------------- +# Akamai NetStorage Paths are specified as `remote:` You may put subdirectories in too, e.g. `remote:/path/to/dir`. @@ -29180,6 +32189,8 @@ See all buckets rclone lsd remote: The initial setup for Netstorage involves getting an account and secret. Use `rclone config` to walk you through the setup process. +## Configuration + Here's an example of how to make a remote called `ns1`. 1. To begin the interactive configuration process, enter this command: @@ -29271,28 +32282,31 @@ y/e/d> y This remote is called `ns1` and can now be used. -### Example operations +## Example operations + Get started with rclone and NetStorage with these examples. For additional rclone commands, visit https://rclone.org/commands/. -##### See contents of a directory in your project +### See contents of a directory in your project rclone lsd ns1:/974012/testing/ -##### Sync the contents local with remote +### Sync the contents local with remote rclone sync . ns1:/974012/testing/ -##### Upload local content to remote +### Upload local content to remote rclone copy notes.txt ns1:/974012/testing/ -##### Delete content on remote +### Delete content on remote rclone delete ns1:/974012/testing/notes.txt -##### Move or copy content between CP codes. +### Move or copy content between CP codes. + Your credentials must have access to two CP codes on the same remote. You can't perform operations between different remotes. rclone move ns1:/974012/testing/notes.txt ns1:/974450/testing2/ +## Features ### Symlink Support @@ -29313,7 +32327,7 @@ With NetStorage, directories can exist in one of two forms: Rclone will intercept all file uploads and mkdir commands for the NetStorage remote and will explicitly issue the mkdir command for each directory in the uploading path. This will help with the interoperability with the other Akamai services such as SFTP and the Content Management Shell (CMShell). Rclone will not guarantee correctness of operations with implicit directories which might have been created as a result of using an upload API directly. -### ListR Feature +### `--fast-list` / ListR support NetStorage remote supports the ListR feature by using the "list" NetStorage API action to return a lexicographical list of all objects within the specified CP code, recursing into subdirectories as they're encountered. @@ -29325,7 +32339,7 @@ There are pros and cons of using the ListR method, refer to [rclone documentatio **Note**: There is a known limitation that "lsf -R" will display number of files in the directory and directory size as -1 when ListR method is used. The workaround is to pass "--disable listR" flag if these numbers are important in the output. -### Purge Feature +### Purge NetStorage remote supports the purge feature by using the "quick-delete" NetStorage API action. The quick-delete action is disabled by default for security reasons and can be enabled for the account through the Akamai portal. Rclone will first try to use quick-delete action for the purge command and if this functionality is disabled then will fall back to a standard delete method. @@ -29334,7 +32348,7 @@ NetStorage remote supports the purge feature by using the "quick-delete" NetStor ### Standard options -Here are the standard options specific to netstorage (Akamai NetStorage). +Here are the Standard options specific to netstorage (Akamai NetStorage). #### --netstorage-host @@ -29377,7 +32391,7 @@ Properties: ### Advanced options -Here are the advanced options specific to netstorage (Akamai NetStorage). +Here are the Advanced options specific to netstorage (Akamai NetStorage). #### --netstorage-protocol @@ -29408,7 +32422,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](https://rclone.org/commands/rclone_backend/) for more +See the [backend](https://rclone.org/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command @@ -29591,7 +32605,7 @@ untrusted environment such as a CI build server. ### Standard options -Here are the standard options specific to azureblob (Microsoft Azure Blob Storage). +Here are the Standard options specific to azureblob (Microsoft Azure Blob Storage). #### --azureblob-account @@ -29688,7 +32702,7 @@ Properties: ### Advanced options -Here are the advanced options specific to azureblob (Microsoft Azure Blob Storage). +Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage). #### --azureblob-msi-object-id @@ -29955,15 +32969,15 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) ## Azure Storage Emulator Support -You can test rclone with storage emulator locally, to do this make sure azure storage emulator -installed locally and set up a new remote with `rclone config` follow instructions described in -introduction, set `use_emulator` config as `true`, you do not need to provide default account name -or key if using emulator. +You can run rclone with storage emulator (usually _azurite_). + +To do this, just set up a new remote with `rclone config` following instructions described in introduction and set `use_emulator` config as `true`. You do not need to provide default account name neither an account key. + +Also, if you want to access a storage emulator instance running on a different machine, you can override _Endpoint_ parameter in advanced settings, setting it to `http(s)://:/devstoreaccount1` (e.g. `http://10.254.2.5:10000/devstoreaccount1`). # Microsoft OneDrive @@ -30082,24 +33096,45 @@ To copy a local directory to an OneDrive directory called backup ### Getting your own Client ID and Key -You can use your own Client ID if the default (`client_id` left blank) -one doesn't work for you or you see lots of throttling. The default -Client ID and Key is shared by all rclone users when performing -requests. +rclone uses a default Client ID when talking to OneDrive, unless a custom `client_id` is specified in the config. +The default Client ID and Key are shared by all rclone users when performing requests. -If you are having problems with them (E.g., seeing a lot of throttling), you can get your own -Client ID and Key by following the steps below: +You may choose to create and use your own Client ID, in case the default one does not work well for you. +For example, you might see throtting. + +#### Creating Client ID for OneDrive Personal + +To create your own Client ID, please follow these steps: 1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade and then click `New registration`. 2. Enter a name for your app, choose account type `Accounts in any organizational directory (Any Azure AD directory - Multitenant) and personal Microsoft accounts (e.g. Skype, Xbox)`, select `Web` in `Redirect URI`, then type (do not copy and paste) `http://localhost:53682/` and click Register. Copy and keep the `Application (client) ID` under the app name for later use. 3. Under `manage` select `Certificates & secrets`, click `New client secret`. Enter a description (can be anything) and set `Expires` to 24 months. Copy and keep that secret _Value_ for later use (you _won't_ be able to see this value afterwards). 4. Under `manage` select `API permissions`, click `Add a permission` and select `Microsoft Graph` then select `delegated permissions`. -5. Search and select the following permissions: `Files.Read`, `Files.ReadWrite`, `Files.Read.All`, `Files.ReadWrite.All`, `offline_access`, `User.Read`, and optionally `Sites.Read.All` (see below). Once selected click `Add permissions` at the bottom. +5. Search and select the following permissions: `Files.Read`, `Files.ReadWrite`, `Files.Read.All`, `Files.ReadWrite.All`, `offline_access`, `User.Read` and `Sites.Read.All` (if custom access scopes are configured, select the permissions accordingly). Once selected click `Add permissions` at the bottom. Now the application is complete. Run `rclone config` to create or edit a OneDrive remote. Supply the app ID and password as Client ID and Secret, respectively. rclone will walk you through the remaining steps. -The `Sites.Read.All` permission is required if you need to [search SharePoint sites when configuring the remote](https://github.com/rclone/rclone/pull/5883). However, if that permission is not assigned, you need to set `disable_site_permission` option to true in the advanced options. +The access_scopes option allows you to configure the permissions requested by rclone. +See [Microsoft Docs](https://docs.microsoft.com/en-us/graph/permissions-reference#files-permissions) for more information about the different scopes. + +The `Sites.Read.All` permission is required if you need to [search SharePoint sites when configuring the remote](https://github.com/rclone/rclone/pull/5883). However, if that permission is not assigned, you need to exclude `Sites.Read.All` from your access scopes or set `disable_site_permission` option to true in the advanced options. + +#### Creating Client ID for OneDrive Business + +The steps for OneDrive Personal may or may not work for OneDrive Business, depending on the security settings of the organization. +A common error is that the publisher of the App is not verified. + +You may try to [verify you account](https://docs.microsoft.com/en-us/azure/active-directory/develop/publisher-verification-overview), or try to limit the App to your organization only, as shown below. + +1. Make sure to create the App with your business account. +2. Follow the steps above to create an App. However, we need a different account type here: `Accounts in this organizational directory only (*** - Single tenant)`. Note that you can also change the account type aftering creating the App. +3. Find the [tenant ID](https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/active-directory-how-to-find-tenant) of your organization. +4. In the rclone config, set `auth_url` to `https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/authorize`. +5. In the rclone config, set `token_url` to `https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/token`. + +Note: If you have a special region, you may need a different host in step 4 and 5. Here are [some hints](https://github.com/rclone/rclone/blob/bc23bf11db1c78c6ebbf8ea538fbebf7058b4176/backend/onedrive/onedrive.go#L86). + ### Modification time and hashes @@ -30158,7 +33193,7 @@ the OneDrive website. ### Standard options -Here are the standard options specific to onedrive (Microsoft OneDrive). +Here are the Standard options specific to onedrive (Microsoft OneDrive). #### --onedrive-client-id @@ -30208,7 +33243,7 @@ Properties: ### Advanced options -Here are the advanced options specific to onedrive (Microsoft OneDrive). +Here are the Advanced options specific to onedrive (Microsoft OneDrive). #### --onedrive-token @@ -30300,6 +33335,28 @@ Properties: - Type: string - Required: false +#### --onedrive-access-scopes + +Set scopes to be requested by rclone. + +Choose or manually enter a custom space separated list with all scopes, that rclone should request. + + +Properties: + +- Config: access_scopes +- Env Var: RCLONE_ONEDRIVE_ACCESS_SCOPES +- Type: SpaceSepList +- Default: Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access +- Examples: + - "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access" + - Read and write access to all resources + - "Files.Read Files.Read.All Sites.Read.All offline_access" + - Read only access to all resources + - "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All offline_access" + - Read and write access to all resources, without the ability to browse SharePoint sites. + - Same as if disable_site_permission was set to true + #### --onedrive-disable-site-permission Disable the request for Sites.Read.All permission. @@ -30591,7 +33648,7 @@ are converted you will no longer need the ignore options above. It is a [known](https://github.com/OneDrive/onedrive-api-docs/issues/1068) issue that Sharepoint (not OneDrive or OneDrive for Business) may return "item not found" errors when users try to replace or delete uploaded files; this seems to -mainly affect Office files (.docx, .xlsx, etc.). As a workaround, you may use +mainly affect Office files (.docx, .xlsx, etc.) and web files (.html, .aspx, etc.). As a workaround, you may use the `--backup-dir ` command line argument so rclone moves the files to be replaced/deleted into a given backup directory (instead of directly replacing/deleting them). For example, to instruct rclone to move the files into @@ -30611,7 +33668,7 @@ Description: Using application 'rclone' is currently not supported for your orga This means that rclone can't use the OneDrive for Business API with your account. You can't do much about it, maybe write an email to your admins. -However, there are other ways to interact with your OneDrive account. Have a look at the webdav backend: https://rclone.org/webdav/#sharepoint +However, there are other ways to interact with your OneDrive account. Have a look at the WebDAV backend: https://rclone.org/webdav/#sharepoint ### invalid\_grant (AADSTS50076) #### @@ -30731,7 +33788,7 @@ as they can't be used in JSON strings. ### Standard options -Here are the standard options specific to opendrive (OpenDrive). +Here are the Standard options specific to opendrive (OpenDrive). #### --opendrive-username @@ -30759,7 +33816,7 @@ Properties: ### Advanced options -Here are the advanced options specific to opendrive (OpenDrive). +Here are the Advanced options specific to opendrive (OpenDrive). #### --opendrive-encoding @@ -30806,8 +33863,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) # QingStor @@ -30950,7 +34006,7 @@ as they can't be used in JSON strings. ### Standard options -Here are the standard options specific to qingstor (QingCloud Object Storage). +Here are the Standard options specific to qingstor (QingCloud Object Storage). #### --qingstor-env-auth @@ -31034,7 +34090,7 @@ Properties: ### Advanced options -Here are the advanced options specific to qingstor (QingCloud Object Storage). +Here are the Advanced options specific to qingstor (QingCloud Object Storage). #### --qingstor-connection-retries @@ -31124,8 +34180,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) # Sia @@ -31255,7 +34310,7 @@ rclone copy /home/source mySia:backup ### Standard options -Here are the standard options specific to sia (Sia Decentralized Cloud). +Here are the Standard options specific to sia (Sia Decentralized Cloud). #### --sia-api-url @@ -31288,7 +34343,7 @@ Properties: ### Advanced options -Here are the advanced options specific to sia (Sia Decentralized Cloud). +Here are the Advanced options specific to sia (Sia Decentralized Cloud). #### --sia-user-agent @@ -31571,7 +34626,7 @@ as they can't be used in JSON strings. ### Standard options -Here are the standard options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). +Here are the Standard options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). #### --swift-env-auth @@ -31811,7 +34866,7 @@ Properties: ### Advanced options -Here are the advanced options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). +Here are the Advanced options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). #### --swift-leave-parts-on-error @@ -32025,6 +35080,13 @@ Deleted files will be moved to the trash. Your subscription level will determine how long items stay in the trash. `rclone cleanup` can be used to empty the trash. +### Emptying the trash + +Due to an API limitation, the `rclone cleanup` command will only work if you +set your username and password in the advanced options for this backend. +Since we generally want to avoid storing user passwords in the rclone config +file, we advise you to only set this up if you need the `rclone cleanup` command to work. + ### Root folder ID You can set the `root_folder_id` for rclone. This is the directory @@ -32050,7 +35112,7 @@ the `root_folder_id` in the config. ### Standard options -Here are the standard options specific to pcloud (Pcloud). +Here are the Standard options specific to pcloud (Pcloud). #### --pcloud-client-id @@ -32080,7 +35142,7 @@ Properties: ### Advanced options -Here are the advanced options specific to pcloud (Pcloud). +Here are the Advanced options specific to pcloud (Pcloud). #### --pcloud-token @@ -32164,6 +35226,34 @@ Properties: - "eapi.pcloud.com" - EU region +#### --pcloud-username + +Your pcloud username. + +This is only required when you want to use the cleanup command. Due to a bug +in the pcloud API the required API does not support OAuth authentication so +we have to rely on user password authentication for it. + +Properties: + +- Config: username +- Env Var: RCLONE_PCLOUD_USERNAME +- Type: string +- Required: false + +#### --pcloud-password + +Your pcloud password. + +**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/). + +Properties: + +- Config: password +- Env Var: RCLONE_PCLOUD_PASSWORD +- Type: string +- Required: false + # premiumize.me @@ -32267,7 +35357,7 @@ as they can't be used in JSON strings. ### Standard options -Here are the standard options specific to premiumizeme (premiumize.me). +Here are the Standard options specific to premiumizeme (premiumize.me). #### --premiumizeme-api-key @@ -32285,7 +35375,7 @@ Properties: ### Advanced options -Here are the advanced options specific to premiumizeme (premiumize.me). +Here are the Advanced options specific to premiumizeme (premiumize.me). #### --premiumizeme-encoding @@ -32421,7 +35511,7 @@ as they can't be used in JSON strings. ### Advanced options -Here are the advanced options specific to putio (Put.io). +Here are the Advanced options specific to putio (Put.io). #### --putio-encoding @@ -32438,6 +35528,15 @@ Properties: +## Limitations + +put.io has rate limiting. When you hit a limit, rclone automatically +retries after waiting the amount of time requested by the server. + +If you want to avoid ever hitting these limits, you may use the +`--tpslimit` flag with a low number. Note that the imposed limits +may be different for different operations, and may change over time. + # Seafile This is a backend for the [Seafile](https://www.seafile.com/) storage service: @@ -32701,7 +35800,7 @@ Versions between 6.0 and 6.3 haven't been tested and might not work properly. ### Standard options -Here are the standard options specific to seafile (seafile). +Here are the Standard options specific to seafile (seafile). #### --seafile-url @@ -32793,7 +35892,7 @@ Properties: ### Advanced options -Here are the advanced options specific to seafile (seafile). +Here are the Advanced options specific to seafile (seafile). #### --seafile-create-library @@ -32829,7 +35928,7 @@ Protocol](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol). The SFTP backend can be used with a number of different providers: -- C14 +- Hetzner Storage Box - rsync.net @@ -32844,9 +35943,12 @@ would list the home directory of the user cofigured in the rclone remote config directory for remote machine (i.e. `/`) Note that some SFTP servers will need the leading / - Synology is a -good example of this. rsync.net, on the other hand, requires users to +good example of this. rsync.net and Hetzner, on the other hand, requires users to OMIT the leading /. +Note that by default rclone will try to execute shell commands on +the server, see [shell access considerations](#shell-access-considerations). + ## Configuration Here is an example of making an SFTP configuration. First run @@ -32865,7 +35967,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / SSH/SFTP Connection +XX / SSH/SFTP \ "sftp" [snip] Storage> sftp @@ -33063,6 +36165,116 @@ And then at the end of the session These commands can be used in scripts of course. +### Shell access + +Some functionality of the SFTP backend relies on remote shell access, +and the possibility to execute commands. This includes [checksum](#checksum), +and in some cases also [about](#about-command). The shell commands that +must be executed may be different on different type of shells, and also +quoting/escaping of file path arguments containing special characters may +be different. Rclone therefore needs to know what type of shell it is, +and if shell access is available at all. + +Most servers run on some version of Unix, and then a basic Unix shell can +be assumed, without further distinction. Windows 10, Server 2019, and later +can also run a SSH server, which is a port of OpenSSH (see official +[installation guide](https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse)). On a Windows server the shell handling is different: Although it can also +be set up to use a Unix type shell, e.g. Cygwin bash, the default is to +use Windows Command Prompt (cmd.exe), and PowerShell is a recommended +alternative. All of these have bahave differently, which rclone must handle. + +Rclone tries to auto-detect what type of shell is used on the server, +first time you access the SFTP remote. If a remote shell session is +successfully created, it will look for indications that it is CMD or +PowerShell, with fall-back to Unix if not something else is detected. +If unable to even create a remote shell session, then shell command +execution will be disabled entirely. The result is stored in the SFTP +remote configuration, in option `shell_type`, so that the auto-detection +only have to be performed once. If you manually set a value for this +option before first run, the auto-detection will be skipped, and if +you set a different value later this will override any existing. +Value `none` can be set to avoid any attempts at executing shell +commands, e.g. if this is not allowed on the server. + +When the server is [rclone serve sftp](https://rclone.org/commands/rclone_serve_sftp/), +the rclone SFTP remote will detect this as a Unix type shell - even +if it is running on Windows. This server does not actually have a shell, +but it accepts input commands matching the specific ones that the +SFTP backend relies on for Unix shells, e.g. `md5sum` and `df`. Also +it handles the string escape rules used for Unix shell. Treating it +as a Unix type shell from a SFTP remote will therefore always be +correct, and support all features. + +#### Shell access considerations + +The shell type auto-detection logic, described above, means that +by default rclone will try to run a shell command the first time +a new sftp remote is accessed. If you configure a sftp remote +without a config file, e.g. an [on the fly](https://rclone.org/docs/#backend-path-to-dir]) +remote, rclone will have nowhere to store the result, and it +will re-run the command on every access. To avoid this you should +explicitely set the `shell_type` option to the correct value, +or to `none` if you want to prevent rclone from executing any +remote shell commands. + +It is also important to note that, since the shell type decides +how quoting and escaping of file paths used as command-line arguments +are performed, configuring the wrong shell type may leave you exposed +to command injection exploits. Make sure to confirm the auto-detected +shell type, or explicitely set the shell type you know is correct, +or disable shell access until you know. + +### Checksum + +SFTP does not natively support checksums (file hash), but rclone +is able to use checksumming if the same login has shell access, +and can execute remote commands. If there is a command that can +calculate compatible checksums on the remote system, Rclone can +then be configured to execute this whenever a checksum is needed, +and read back the results. Currently MD5 and SHA-1 are supported. + +Normally this requires an external utility being available on +the server. By default rclone will try commands `md5sum`, `md5` +and `rclone md5sum` for MD5 checksums, and the first one found usable +will be picked. Same with `sha1sum`, `sha1` and `rclone sha1sum` +commands for SHA-1 checksums. These utilities normally need to +be in the remote's PATH to be found. + +In some cases the shell itself is capable of calculating checksums. +PowerShell is an example of such a shell. If rclone detects that the +remote shell is PowerShell, which means it most probably is a +Windows OpenSSH server, rclone will use a predefined script block +to produce the checksums when no external checksum commands are found +(see [shell access](#shell-access)). This assumes PowerShell version +4.0 or newer. + +The options `md5sum_command` and `sha1_command` can be used to customize +the command to be executed for calculation of checksums. You can for +example set a specific path to where md5sum and sha1sum executables +are located, or use them to specify some other tools that print checksums +in compatible format. The value can include command-line arguments, +or even shell script blocks as with PowerShell. Rclone has subcommands +[md5sum](https://rclone.org/commands/rclone_md5sum/) and [sha1sum](https://rclone.org/commands/rclone_sha1sum/) +that use compatible format, which means if you have an rclone executable +on the server it can be used. As mentioned above, they will be automatically +picked up if found in PATH, but if not you can set something like +`/path/to/rclone md5sum` as the value of option `md5sum_command` to +make sure a specific executable is used. + +Remote checksumming is recommended and enabled by default. First time +rclone is using a SFTP remote, if options `md5sum_command` or `sha1_command` +are not set, it will check if any of the default commands for each of them, +as described above, can be used. The result will be saved in the remote +configuration, so next time it will use the same. Value `none` +will be set if none of the default commands could be used for a specific +algorithm, and this algorithm will not be supported by the remote. + +Disabling the checksumming may be required if you are connecting to SFTP servers +which are not under your control, and to which the execution of remote shell +commands is prohibited. Set the configuration option `disable_hashcheck` +to `true` to disable checksumming entirely, or set `shell_type` to `none` +to disable all functionality based on remote shell command execution. + ### Modified time Modified times are stored on the server to 1 second precision. @@ -33074,10 +36286,26 @@ upload (for example, certain configurations of ProFTPd with mod_sftp). If you are using one of these servers, you can set the option `set_modtime = false` in your RClone backend configuration to disable this behaviour. +### About command + +The `about` command returns the total space, free space, and used +space on the remote for the disk of the specified path on the remote or, +if not set, the disk of the root on the remote. + +SFTP usually supports the [about](https://rclone.org/commands/rclone_about/) command, but +it depends on the server. If the server implements the vendor-specific +VFS statistics extension, which is normally the case with OpenSSH instances, +it will be used. If not, but the same login has access to a Unix shell, +where the `df` command is available (e.g. in the remote's PATH), then +this will be used instead. If the server shell is PowerShell, probably +with a Windows OpenSSH server, rclone will use a built-in shell command +(see [shell access](#shell-access)). If none of the above is applicable, +`about` will fail. + ### Standard options -Here are the standard options specific to sftp (SSH/SFTP Connection). +Here are the Standard options specific to sftp (SSH/SFTP). #### --sftp-host @@ -33203,7 +36431,7 @@ Properties: #### --sftp-use-insecure-cipher -Enable the use of insecure ciphers and key exchange methods. +Enable the use of insecure ciphers and key exchange methods. This enables the use of the following insecure ciphers and key exchange methods: @@ -33243,7 +36471,7 @@ Properties: ### Advanced options -Here are the advanced options specific to sftp (SSH/SFTP Connection). +Here are the Advanced options specific to sftp (SSH/SFTP). #### --sftp-known-hosts-file @@ -33281,16 +36509,16 @@ Properties: #### --sftp-path-override -Override path used by SSH connection. +Override path used by SSH shell commands. This allows checksum calculation when SFTP and SSH paths are different. This issue affects among others Synology NAS boxes. -Shared folders can be found in directories representing volumes +E.g. if shared folders can be found in directories representing volumes: rclone sync /home/local/directory remote:/directory --sftp-path-override /volume2/directory -Home directory can be found in a shared folder called "home" +E.g. if home directory can be found in a shared folder called "home": rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory @@ -33312,6 +36540,28 @@ Properties: - Type: bool - Default: true +#### --sftp-shell-type + +The type of SSH shell on remote server, if any. + +Leave blank for autodetect. + +Properties: + +- Config: shell_type +- Env Var: RCLONE_SFTP_SHELL_TYPE +- Type: string +- Required: false +- Examples: + - "none" + - No shell access + - "unix" + - Unix shell + - "powershell" + - PowerShell + - "cmd" + - Windows Command Prompt + #### --sftp-md5sum-command The command used to read md5 hashes. @@ -33452,29 +36702,82 @@ Properties: - Type: Duration - Default: 1m0s +#### --sftp-chunk-size + +Upload and download chunk size. + +This controls the maximum packet size used in the SFTP protocol. The +RFC limits this to 32768 bytes (32k), however a lot of servers +support larger sizes and setting it larger will increase transfer +speed dramatically on high latency links. + +Only use a setting higher than 32k if you always connect to the same +server or after sufficiently broad testing. + +For example using the value of 252k with OpenSSH works well with its +maximum packet size of 256k. + +If you get the error "failed to send packet header: EOF" when copying +a large file, try lowering this number. + + +Properties: + +- Config: chunk_size +- Env Var: RCLONE_SFTP_CHUNK_SIZE +- Type: SizeSuffix +- Default: 32Ki + +#### --sftp-concurrency + +The maximum number of outstanding requests for one file + +This controls the maximum number of outstanding requests for one file. +Increasing it will increase throughput on high latency links at the +cost of using more memory. + + +Properties: + +- Config: concurrency +- Env Var: RCLONE_SFTP_CONCURRENCY +- Type: int +- Default: 64 + +#### --sftp-set-env + +Environment variables to pass to sftp and commands + +Set environment variables in the form: + + VAR=value + +to be passed to the sftp client and to any commands run (eg md5sum). + +Pass multiple variables space separated, eg + + VAR1=value VAR2=value + +and pass variables with spaces in in quotes, eg + + "VAR3=value with space" "VAR4=value with space" VAR5=nospacehere + + + +Properties: + +- Config: set_env +- Env Var: RCLONE_SFTP_SET_ENV +- Type: SpaceSepList +- Default: + ## Limitations -SFTP supports checksums if the same login has shell access and `md5sum` -or `sha1sum` as well as `echo` are in the remote's PATH. -This remote checksumming (file hashing) is recommended and enabled by default. -Disabling the checksumming may be required if you are connecting to SFTP servers -which are not under your control, and to which the execution of remote commands -is prohibited. Set the configuration option `disable_hashcheck` to `true` to -disable checksumming. - -SFTP also supports `about` if the same login has shell -access and `df` are in the remote's PATH. `about` will -return the total space, free space, and used space on the remote -for the disk of the specified path on the remote or, if not set, -the disk of the root on the remote. -`about` will fail if it does not have shell -access or if `df` is not in the remote's PATH. - -Note that some SFTP servers (e.g. Synology) the paths are different for -SSH and SFTP so the hashes can't be calculated properly. For them -using `disable_hashcheck` is a good idea. +On some SFTP servers (e.g. Synology) the paths are different +for SSH and SFTP so the hashes can't be calculated properly. +For them using `disable_hashcheck` is a good idea. The only ssh agent supported under Windows is Putty's pageant. @@ -33489,23 +36792,22 @@ SFTP isn't supported under plan9 until [this issue](https://github.com/pkg/sftp/issues/156) is fixed. Note that since SFTP isn't HTTP based the following flags don't work -with it: `--dump-headers`, `--dump-bodies`, `--dump-auth` +with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`. Note that `--timeout` and `--contimeout` are both supported. - -## C14 {#c14} - -C14 is supported through the SFTP backend. - -See [C14's documentation](https://www.online.net/en/storage/c14-cold-storage) - ## rsync.net {#rsync-net} rsync.net is supported through the SFTP backend. See [rsync.net's documentation of rclone examples](https://www.rsync.net/products/rclone.html). +## Hetzner Storage Box {#hetzner-storage-box} + +Hetzner Storage Boxes are supported through the SFTP backend on port 23. + +See [Hetzner's documentation for details](https://docs.hetzner.com/robot/storage-box/access/access-ssh-rsync-borg#rclone) + # Storj [Storj](https://storj.io) is an encrypted, secure, and @@ -33718,7 +37020,7 @@ y/e/d> y ### Standard options -Here are the standard options specific to storj (Storj Decentralized Cloud Storage). +Here are the Standard options specific to storj (Storj Decentralized Cloud Storage). #### --storj-provider @@ -33918,8 +37220,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) ## Known issues @@ -34047,7 +37348,7 @@ deleted straight away. ### Standard options -Here are the standard options specific to sugarsync (Sugarsync). +Here are the Standard options specific to sugarsync (Sugarsync). #### --sugarsync-app-id @@ -34102,7 +37403,7 @@ Properties: ### Advanced options -Here are the advanced options specific to sugarsync (Sugarsync). +Here are the Advanced options specific to sugarsync (Sugarsync). #### --sugarsync-refresh-token @@ -34204,8 +37505,7 @@ this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union remote. -See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) -See [rclone about](https://rclone.org/commands/rclone_about/) +See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) # Tardigrade @@ -34310,7 +37610,7 @@ as they can't be used in XML strings. ### Standard options -Here are the standard options specific to uptobox (Uptobox). +Here are the Standard options specific to uptobox (Uptobox). #### --uptobox-access-token @@ -34327,7 +37627,7 @@ Properties: ### Advanced options -Here are the advanced options specific to uptobox (Uptobox). +Here are the Advanced options specific to uptobox (Uptobox). #### --uptobox-encoding @@ -34522,7 +37822,7 @@ The policies definition are inspired by [trapexit/mergerfs](https://github.com/t ### Standard options -Here are the standard options specific to union (Union merges the contents of several upstream fs). +Here are the Standard options specific to union (Union merges the contents of several upstream fs). #### --union-upstreams @@ -34583,6 +37883,30 @@ Properties: - Type: int - Default: 120 +### Advanced options + +Here are the Advanced options specific to union (Union merges the contents of several upstream fs). + +#### --union-min-free-space + +Minimum viable free space for lfs/eplfs policies. + +If a remote has less than this much free space then it won't be +considered for use in lfs or eplfs policies. + +Properties: + +- Config: min_free_space +- Env Var: RCLONE_UNION_MIN_FREE_SPACE +- Type: SizeSuffix +- Default: 1Gi + +### Metadata + +Any metadata supported by the underlying remote is read and written. + +See the [metadata](https://rclone.org/docs/#metadata) docs for more info. + # WebDAV @@ -34613,7 +37937,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Webdav +XX / WebDAV \ "webdav" [snip] Storage> webdav @@ -34622,7 +37946,7 @@ Choose a number from below, or type in your own value 1 / Connect to example.com \ "https://example.com" url> https://example.com/remote.php/webdav/ -Name of the Webdav site/service/software you are using +Name of the WebDAV site/service/software you are using Choose a number from below, or type in your own value 1 / Nextcloud \ "nextcloud" @@ -34692,7 +38016,7 @@ with them. ### Standard options -Here are the standard options specific to webdav (Webdav). +Here are the Standard options specific to webdav (WebDAV). #### --webdav-url @@ -34709,7 +38033,7 @@ Properties: #### --webdav-vendor -Name of the Webdav site/service/software you are using. +Name of the WebDAV site/service/software you are using. Properties: @@ -34768,7 +38092,7 @@ Properties: ### Advanced options -Here are the advanced options specific to webdav (Webdav). +Here are the Advanced options specific to webdav (WebDAV). #### --webdav-bearer-token-command @@ -35108,7 +38432,7 @@ as they can't be used in JSON strings. ### Standard options -Here are the standard options specific to yandex (Yandex Disk). +Here are the Standard options specific to yandex (Yandex Disk). #### --yandex-client-id @@ -35138,7 +38462,7 @@ Properties: ### Advanced options -Here are the advanced options specific to yandex (Yandex Disk). +Here are the Advanced options specific to yandex (Yandex Disk). #### --yandex-token @@ -35346,7 +38670,7 @@ from filenames during upload. ### Standard options -Here are the standard options specific to zoho (Zoho). +Here are the Standard options specific to zoho (Zoho). #### --zoho-client-id @@ -35395,12 +38719,16 @@ Properties: - Europe - "in" - India + - "jp" + - Japan + - "com.cn" + - China - "com.au" - Australia ### Advanced options -Here are the advanced options specific to zoho (Zoho). +Here are the Advanced options specific to zoho (Zoho). #### --zoho-token @@ -35454,6 +38782,18 @@ Properties: +## Setting up your own client_id + +For Zoho we advise you to set up your own client_id. To do so you have to complete the following steps. + +1. Log in to the [Zoho API Console](https://api-console.zoho.com) + +2. Create a new client of type "Server-based Application". The name and website don't matter, but you must add the redirect URL `http://localhost:53682/`. + +3. Once the client is created, you can go to the settings tab and enable it in other regions. + +The client id and client secret can now be used with rclone. + # Local Filesystem Local paths are specified as normal filesystem paths, e.g. `/path/to/wherever`, so @@ -35778,7 +39118,7 @@ where it isn't supported (e.g. Windows) it will be ignored. ### Advanced options -Here are the advanced options specific to local (Local Disk). +Here are the Advanced options specific to local (Local Disk). #### --local-nounc @@ -35788,8 +39128,8 @@ Properties: - Config: nounc - Env Var: RCLONE_LOCAL_NOUNC -- Type: string -- Required: false +- Type: bool +- Default: false - Examples: - "true" - Disables long file names. @@ -36014,6 +39354,31 @@ Properties: - Type: MultiEncoder - Default: Slash,Dot +### Metadata + +Depending on which OS is in use the local backend may return only some +of the system metadata. Setting system metadata is supported on all +OSes but setting user metadata is only supported on linux, freebsd, +netbsd, macOS and Solaris. It is **not** supported on Windows yet +([see pkg/attrs#47](https://github.com/pkg/xattr/issues/47)). + +User metadata is stored as extended attributes (which may not be +supported by all file systems) under the "user.*" prefix. + +Here are the possible system metadata items for the local backend. + +| Name | Help | Type | Example | Read Only | +|------|------|------|---------|-----------| +| atime | Time of last access | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N | +| btime | Time of file birth (creation) | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N | +| gid | Group ID of owner | decimal number | 500 | N | +| mode | File type and mode | octal, unix style | 0100664 | N | +| mtime | Time of last modification | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N | +| rdev | Device ID (if special file) | hexadecimal | 1abc | N | +| uid | User ID of owner | decimal number | 500 | N | + +See the [metadata](https://rclone.org/docs/#metadata) docs for more info. + ## Backend commands Here are the commands specific to the local backend. @@ -36024,7 +39389,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](https://rclone.org/commands/rclone_backend/) for more +See the [backend](https://rclone.org/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command @@ -36048,6 +39413,207 @@ Options: # Changelog +## v1.59.0 - 2022-07-09 + +[See commits](https://github.com/rclone/rclone/compare/v1.58.0...v1.59.0) + +* New backends + * [Combine](/combine) multiple remotes in one directory tree (Nick Craig-Wood) + * [Hidrive](https://rclone.org/hidrive/) (Ovidiu Victor Tatar) + * [Internet Archive](https://rclone.org/internetarchive/) (Lesmiscore (Naoya Ozaki)) + * New S3 providers + * [ArvanCloud AOS](https://rclone.org/s3/#arvan-cloud) (ehsantdy) + * [Cloudflare R2](https://rclone.org/s3/#cloudflare-r2) (Nick Craig-Wood) + * [Huawei OBS](https://rclone.org/s3/#huawei-obs) (m00594701) + * [IDrive e2](https://rclone.org/s3/#idrive-e2) (vyloy) +* New commands + * [test makefile](https://rclone.org/commands/rclone_test_makefile/): Create a single file for testing (Nick Craig-Wood) +* New Features + * [Metadata framework](https://rclone.org/docs/#metadata) to read and write system and user metadata on backends (Nick Craig-Wood) + * Implemented initially for `local`, `s3` and `internetarchive` backends + * `--metadata`/`-M` flag to control whether metadata is copied + * `--metadata-set` flag to specify metadata for uploads + * Thanks to [Manz Solutions](https://manz-solutions.at/) for sponsoring this work. + * build + * Update to go1.18 and make go1.16 the minimum required version (Nick Craig-Wood) + * Update android go build to 1.18.x and NDK to 23.1.7779620 (Nick Craig-Wood) + * All windows binaries now no longer CGO (Nick Craig-Wood) + * Add `linux/arm/v6` to docker images (Nick Craig-Wood) + * A huge number of fixes found with [staticcheck](https://staticcheck.io/) (albertony) + * Configurable version suffix independent of version number (albertony) + * check: Implement `--no-traverse` and `--no-unicode-normalization` (Nick Craig-Wood) + * config: Readability improvements (albertony) + * copyurl: Add `--header-filename` to honor the HTTP header filename directive (J-P Treen) + * filter: Allow multiple `--exclude-if-present` flags (albertony) + * fshttp: Add `--disable-http-keep-alives` to disable HTTP Keep Alives (Nick Craig-Wood) + * install.sh + * Set the modes on the files and/or directories on macOS (Michael C Tiernan - MIT-Research Computing Project) + * Pre verify sudo authorization `-v` before calling curl. (Michael C Tiernan - MIT-Research Computing Project) + * lib/encoder: Add Semicolon encoding (Nick Craig-Wood) + * lsf: Add metadata support with `M` flag (Nick Craig-Wood) + * lsjson: Add `--metadata`/`-M` flag (Nick Craig-Wood) + * ncdu + * Implement multi selection (CrossR) + * Replace termbox with tcell's termbox wrapper (eNV25) + * Display correct path in delete confirmation dialog (Roberto Ricci) + * operations + * Speed up hash checking by aborting the other hash if first returns nothing (Nick Craig-Wood) + * Use correct src/dst in some log messages (zzr93) + * rcat: Check checksums by default like copy does (Nick Craig-Wood) + * selfupdate: Replace deprecated `x/crypto/openpgp` package with `ProtonMail/go-crypto` (albertony) + * serve ftp: Check `--passive-port` arguments are correct (Nick Craig-Wood) + * size: Warn about inaccurate results when objects with unknown size (albertony) + * sync: Overlap check is now filter-sensitive so `--backup-dir` can be in the root provided it is filtered (Nick) + * test info: Check file name lengths using 1,2,3,4 byte unicode characters (Nick Craig-Wood) + * test makefile(s): `--sparse`, `--zero`, `--pattern`, `--ascii`, `--chargen` flags to control file contents (Nick Craig-Wood) + * Make sure we call the `Shutdown` method on backends (Martin Czygan) +* Bug Fixes + * accounting: Fix unknown length file transfers counting 3 transfers each (buda) + * ncdu: Fix issue where dir size is summed when file sizes are -1 (albertony) + * sync/copy/move + * Fix `--fast-list` `--create-empty-src-dirs` and `--exclude` (Nick Craig-Wood) + * Fix `--max-duration` and `--cutoff-mode soft` (Nick Craig-Wood) + * Fix fs cache unpin (Martin Czygan) + * Set proper exit code for errors that are not low-level retried (e.g. size/timestamp changing) (albertony) +* Mount + * Support `windows/arm64` (may still be problems - see [#5828](https://github.com/rclone/rclone/issues/5828)) (Nick Craig-Wood) + * Log IO errors at ERROR level (Nick Craig-Wood) + * Ignore `_netdev` mount argument (Hugal31) +* VFS + * Add `--vfs-fast-fingerprint` for less accurate but faster fingerprints (Nick Craig-Wood) + * Add `--vfs-disk-space-total-size` option to manually set the total disk space (Claudio Maradonna) + * vfscache: Fix fatal error: sync: unlock of unlocked mutex error (Nick Craig-Wood) +* Local + * Fix parsing of `--local-nounc` flag (Nick Craig-Wood) + * Add Metadata support (Nick Craig-Wood) +* Crypt + * Support metadata (Nick Craig-Wood) +* Azure Blob + * Calculate Chunksize/blocksize to stay below maxUploadParts (Leroy van Logchem) + * Use chunksize lib to determine chunksize dynamically (Derek Battams) + * Case insensitive access tier (Rob Pickerill) + * Allow remote emulator (azurite) (Lorenzo Maiorfi) +* B2 + * Add `--b2-version-at` flag to show file versions at time specified (SwazRGB) + * Use chunksize lib to determine chunksize dynamically (Derek Battams) +* Chunker + * Mark as not supporting metadata (Nick Craig-Wood) +* Compress + * Support metadata (Nick Craig-Wood) +* Drive + * Make `backend config -o config` add a combined `AllDrives:` remote (Nick Craig-Wood) + * Make `--drive-shared-with-me` work with shared drives (Nick Craig-Wood) + * Add `--drive-resource-key` for accessing link-shared files (Nick Craig-Wood) + * Add backend commands `exportformats` and `importformats` for debugging (Nick Craig-Wood) + * Fix 404 errors on copy/server side copy objects from public folder (Nick Craig-Wood) + * Update Internal OAuth consent screen docs (Phil Shackleton) + * Moved `root_folder_id` to advanced section (Abhiraj) +* Dropbox + * Migrate from deprecated api (m8rge) + * Add logs to show when poll interval limits are exceeded (Nick Craig-Wood) + * Fix nil pointer exception on dropbox impersonate user not found (Nick Craig-Wood) +* Fichier + * Parse api error codes and them accordingly (buengese) +* FTP + * Add support for `disable_utf8` option (Jason Zheng) + * Revert to upstream `github.com/jlaffaye/ftp` from our fork (Nick Craig-Wood) +* Google Cloud Storage + * Add `--gcs-no-check-bucket` to minimise transactions and perms (Nick Gooding) + * Add `--gcs-decompress` flag to decompress gzip-encoded files (Nick Craig-Wood) + * by default these will be downloaded compressed (which previously failed) +* Hasher + * Support metadata (Nick Craig-Wood) +* HTTP + * Fix missing response when using custom auth handler (albertony) +* Jottacloud + * Add support for upload to custom device and mountpoint (albertony) + * Always store username in config and use it to avoid initial API request (albertony) + * Fix issue with server-side copy when destination is in trash (albertony) + * Fix listing output of remote with special characters (albertony) +* Mailru + * Fix timeout by using int instead of time.Duration for keeping number of seconds (albertony) +* Mega + * Document using MEGAcmd to help with login failures (Art M. Gallagher) +* Onedrive + * Implement `--poll-interval` for onedrive (Hugo Laloge) + * Add access scopes option (Sven Gerber) +* Opendrive + * Resolve lag and truncate bugs (Scott Grimes) +* Pcloud + * Fix about with no free space left (buengese) + * Fix cleanup (buengese) +* S3 + * Use PUT Object instead of presigned URLs to upload single part objects (Nick Craig-Wood) + * Backend restore command to skip non-GLACIER objects (Vincent Murphy) + * Use chunksize lib to determine chunksize dynamically (Derek Battams) + * Retry RequestTimeout errors (Nick Craig-Wood) + * Implement reading and writing of metadata (Nick Craig-Wood) +* SFTP + * Add support for about and hashsum on windows server (albertony) + * Use vendor-specific VFS statistics extension for about if available (albertony) + * Add `--sftp-chunk-size` to control packets sizes for high latency links (Nick Craig-Wood) + * Add `--sftp-concurrency` to improve high latency transfers (Nick Craig-Wood) + * Add `--sftp-set-env` option to set environment variables (Nick Craig-Wood) + * Add Hetzner Storage Boxes to supported sftp backends (Anthrazz) +* Storj + * Fix put which lead to the file being unreadable when using mount (Erik van Velzen) +* Union + * Add `min_free_space` option for `lfs`/`eplfs` policies (Nick Craig-Wood) + * Fix uploading files to union of all bucket based remotes (Nick Craig-Wood) + * Fix get free space for remotes which don't support it (Nick Craig-Wood) + * Fix `eplus` policy to select correct entry for existing files (Nick Craig-Wood) + * Support metadata (Nick Craig-Wood) +* Uptobox + * Fix root path handling (buengese) +* WebDAV + * Add SharePoint in other specific regions support (Noah Hsu) +* Yandex + * Handle api error on server-side move (albertony) +* Zoho + * Add Japan and China regions (buengese) + +## v1.58.1 - 2022-04-29 + +[See commits](https://github.com/rclone/rclone/compare/v1.58.0...v1.58.1) + +* Bug Fixes + * build: Update github.com/billziss-gh to github.com/winfsp (Nick Craig-Wood) + * filter: Fix timezone of `--min-age`/`-max-age` from UTC to local as documented (Nick Craig-Wood) + * rc/js: Correct RC method names (Sơn Trần-Nguyễn) + * docs + * Fix some links to command pages (albertony) + * Add `--multi-thread-streams` note to `--transfers`. (Zsolt Ero) +* Mount + * Fix `--devname` and fusermount: unknown option 'fsname' when mounting via rc (Nick Craig-Wood) +* VFS + * Remove wording which suggests VFS is only for mounting (Nick Craig-Wood) +* Dropbox + * Fix retries of multipart uploads with incorrect_offset error (Nick Craig-Wood) +* Google Cloud Storage + * Use the s3 pacer to speed up transactions (Nick Craig-Wood) + * pacer: Default the Google pacer to a burst of 100 to fix gcs pacing (Nick Craig-Wood) +* Jottacloud + * Fix scope in token request (albertony) +* Netstorage + * Fix unescaped HTML in documentation (Nick Craig-Wood) + * Make levels of headings consistent (Nick Craig-Wood) + * Add support contacts to netstorage doc (Nil Alexandrov) +* Onedrive + * Note that sharepoint also changes web files (.html, .aspx) (GH) +* Putio + * Handle rate limit errors (Berkan Teber) + * Fix multithread download and other ranged requests (rafma0) +* S3 + * Add ChinaMobile EOS to provider list (GuoXingbin) + * Sync providers in config description with providers (Nick Craig-Wood) +* SFTP + * Fix OpenSSH 8.8+ RSA keys incompatibility (KARBOWSKI Piotr) + * Note that Scaleway C14 is deprecating SFTP in favor of S3 (Adrien Rey-Jarthon) +* Storj + * Fix bucket creation on Move (Nick Craig-Wood) +* WebDAV + * Don't override Referer if user sets it (Nick Craig-Wood) + ## v1.58.0 - 2022-03-18 [See commits](https://github.com/rclone/rclone/compare/v1.57.0...v1.58.0) @@ -39951,7 +43517,7 @@ the node running rclone would need to have lots of bandwidth. The syncs would be incremental (on a file by file basis). -Eg +e.g. rclone sync -i drive:Folder s3:bucket @@ -40038,7 +43604,7 @@ e.g. export no_proxy=localhost,127.0.0.0/8,my.host.name export NO_PROXY=$no_proxy -Note that the ftp backend does not support `ftp_proxy` yet. +Note that the FTP backend does not support `ftp_proxy` yet. ### Rclone gives x509: failed to load system roots and no roots provided error ### @@ -40747,6 +44313,57 @@ put them back in again.` >}} * Vincent Murphy * ctrl-q <34975747+ctrl-q@users.noreply.github.com> * Nil Alexandrov + * GuoXingbin <101376330+guoxingbin@users.noreply.github.com> + * Berkan Teber + * Tobias Klauser + * KARBOWSKI Piotr + * GH + * rafma0 + * Adrien Rey-Jarthon + * Nick Gooding <73336146+nickgooding@users.noreply.github.com> + * Leroy van Logchem + * Zsolt Ero + * Lesmiscore + * ehsantdy + * SwazRGB <65694696+swazrgb@users.noreply.github.com> + * Mateusz Puczyński + * Michael C Tiernan - MIT-Research Computing Project + * Kaspian <34658474+KaspianDev@users.noreply.github.com> + * Werner + * Hugal31 + * Christian Galo <36752715+cgalo5758@users.noreply.github.com> + * Erik van Velzen + * Derek Battams + * SimonLiu + * Hugo Laloge + * Mr-Kanister <68117355+Mr-Kanister@users.noreply.github.com> + * Rob Pickerill + * Andrey + * Eric Wolf <19wolf@gmail.com> + * Nick + * Jason Zheng + * Matthew Vernon + * Noah Hsu + * m00594701 + * Art M. Gallagher + * Sven Gerber <49589423+svengerber@users.noreply.github.com> + * CrossR + * Maciej Radzikowski + * Scott Grimes + * Phil Shackleton <71221528+philshacks@users.noreply.github.com> + * eNV25 + * Caleb + * J-P Treen + * Martin Czygan <53705+miku@users.noreply.github.com> + * buda + * mirekphd <36706320+mirekphd@users.noreply.github.com> + * vyloy + * Anthrazz <25553648+Anthrazz@users.noreply.github.com> + * zzr93 <34027824+zzr93@users.noreply.github.com> + * Paul Norman + * Lorenzo Maiorfi + * Claudio Maradonna + * Ovidiu Victor Tatar # Contact the rclone project # diff --git a/MANUAL.txt b/MANUAL.txt index d25a312cb36eb..d8828a0656d31 100644 --- a/MANUAL.txt +++ b/MANUAL.txt @@ -1,6 +1,6 @@ rclone(1) User Manual Nick Craig-Wood -Mar 18, 2022 +Jul 09, 2022 Rclone syncs your files to cloud storage @@ -82,7 +82,7 @@ Features - Move files to cloud storage deleting the local after verification - Check hashes and for missing/extra files - Mount your cloud storage as a network disk -- Serve local or remote files over HTTP/WebDav/FTP/SFTP/dlna +- Serve local or remote files over HTTP/WebDav/FTP/SFTP/DLNA - Experimental Web based GUI Supported providers @@ -98,8 +98,11 @@ S3, that work out of the box.) - Backblaze B2 - Box - Ceph +- China Mobile Ecloud Elastic Object Storage (EOS) +- Arvan Cloud Object Storage (AOS) - Citrix ShareFile - C14 +- Cloudflare R2 - DigitalOcean Spaces - Digi Storage - Dreamhost @@ -110,10 +113,14 @@ S3, that work out of the box.) - Google Drive - Google Photos - HDFS +- Hetzner Storage Box +- HiDrive - HTTP - Hubic +- Internet Archive - Jottacloud - IBM COS S3 +- IDrive e2 - Koofr - Mail.ru Cloud - Memset Memstore @@ -151,6 +158,19 @@ S3, that work out of the box.) - Zoho WorkDrive - The local filesystem +Virtual providers + +These backends adapt or modify other storage providers: + +- Alias: Rename existing remotes +- Cache: Cache remotes (DEPRECATED) +- Chunker: Split large files +- Combine: Combine multiple remotes into a directory tree +- Compress: Compress files +- Crypt: Encrypt files +- Hasher: Hash files +- Union: Join multiple remotes to work together + Links - Home page @@ -181,11 +201,11 @@ Script installation To install rclone on Linux/macOS/BSD systems, run: - curl https://rclone.org/install.sh | sudo bash + sudo -v ; curl https://rclone.org/install.sh | sudo bash For beta installation, run: - curl https://rclone.org/install.sh | sudo bash -s beta + sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta Note that this script checks the version of rclone installed first and won't re-download if not needed. @@ -257,7 +277,7 @@ When downloading a binary with a web browser, the browser will set the macOS gatekeeper quarantine attribute. Starting from Catalina, when attempting to run rclone, a pop-up will appear saying: - “rclone” cannot be opened because the developer cannot be verified. + "rclone" cannot be opened because the developer cannot be verified. macOS cannot verify that this app is free from malware. The simplest fix is to run @@ -346,32 +366,74 @@ Here are some commands tested on an Ubuntu 18.04.3 host: Install from source -Make sure you have at least Go go1.15 installed. Download go if -necessary. The latest release is recommended. Then +Make sure you have git and Go installed. Go version 1.16 or newer is +required, latest release is recommended. You can get it from your +package manager, or download it from golang.org/dl. Then you can run the +following: git clone https://github.com/rclone/rclone.git cd rclone go build - # If on macOS and mount is wanted, instead run: make GOTAGS=cmount - ./rclone version -This will leave you a checked out version of rclone you can modify and -send pull requests with. If you use make instead of go build then the -rclone build will have the correct version information in it. +This will check out the rclone source in subfolder rclone, which you can +later modify and send pull requests with. Then it will build the rclone +executable in the same folder. As an initial check you can now run +./rclone version (.\rclone version on Windows). -You can also build the latest stable rclone with: +Note that on macOS and Windows the mount command will not be available +unless you specify additional build tag cmount. - go get github.com/rclone/rclone + go build -tags cmount + +This assumes you have a GCC compatible C compiler (GCC or Clang) in your +PATH, as it uses cgo. But on Windows, the cgofuse library that the +cmount implementation is based on, also supports building without cgo, +i.e. by setting environment variable CGO_ENABLED to value 0 (static +linking). This is how the official Windows release of rclone is being +built, starting with version 1.59. It is still possible to build with +cgo on Windows as well, by using the MinGW port of GCC, e.g. by +installing it in a MSYS2 distribution (make sure you install it in the +classic mingw64 subsystem, the ucrt64 version is not compatible). + +Additionally, on Windows, you must install the third party utility +WinFsp, with the "Developer" feature selected. If building with cgo, you +must also set environment variable CPATH pointing to the fuse include +directory within the WinFsp installation (normally +C:\Program Files (x86)\WinFsp\inc\fuse). + +You may also add arguments -ldflags -s (with or without -tags cmount), +to omit symbol table and debug information, making the executable file +smaller, and -trimpath to remove references to local file system paths. +This is how the official rclone releases are built. + + go build -trimpath -ldflags -s -tags cmount + +Instead of executing the go build command directly, you can run it via +the Makefile, which also sets version information and copies the +resulting rclone executable into your GOPATH bin folder +($(go env GOPATH)/bin, which corresponds to ~/go/bin/rclone by default). -or the latest version (equivalent to the beta) with + make - go get github.com/rclone/rclone@master +To include mount command on macOS and Windows with Makefile build: -These will build the binary in $(go env GOPATH)/bin (~/go/bin/rclone by -default) after downloading the source to the go module cache. Note - do -not use the -u flag here. This causes go to try to update the -dependencies that rclone uses and sometimes these don't work with the -current version of rclone. + make GOTAGS=cmount + +As an alternative you can download the source, build and install rclone +in one operation, as a regular Go package. The source will be stored it +in the Go module cache, and the resulting executable will be in your +GOPATH bin folder ($(go env GOPATH)/bin, which corresponds to +~/go/bin/rclone by default). + +With Go version 1.17 or newer: + + go install github.com/rclone/rclone@latest + +With Go versions older than 1.17 (do not use the -u flag, it causes Go +to try to update the dependencies that rclone uses and sometimes these +don't work with the current version): + + go get github.com/rclone/rclone Installation with Ansible @@ -497,7 +559,7 @@ configured to run at startup. Mount command built-in service integration -For mount commands, Rclone has a built-in Windows service integration +For mount commands, rclone has a built-in Windows service integration via the third-party WinFsp library it uses. Registering as a regular Windows service easy, as you just have to execute the built-in PowerShell command New-Service (requires administrative privileges). @@ -586,6 +648,7 @@ See the following for detailed instructions for - Chunker - transparently splits large files for other remotes - Citrix ShareFile - Compress +- Combine - Crypt - to encrypt other remotes - DigitalOcean Spaces - Digi Storage @@ -597,8 +660,10 @@ See the following for detailed instructions for - Google Photos - Hasher - to handle checksums for other remotes - HDFS +- HiDrive - HTTP - Hubic +- Internet Archive - Jottacloud - Koofr - Mail.ru Cloud @@ -696,11 +761,16 @@ Synopsis Copy the source to the destination. Does not transfer files that are identical on source and destination, testing by size and modification -time or MD5SUM. Doesn't delete files from the destination. +time or MD5SUM. Doesn't delete files from the destination. If you want +to also delete files from destination, to make it match source, use the +sync command instead. Note that it is always the contents of the directory that is synced, not -the directory so when source:path is a directory, it's the contents of -source:path that are copied, not the directory name and contents. +the directory itself. So when source:path is a directory, it's the +contents of source:path that are copied, not the directory name and +contents. + +To copy single files, use the copyto command instead. If dest:path doesn't exist, it is created and the source:path contents go there. @@ -767,7 +837,8 @@ Sync the source to the destination, changing the destination only. Doesn't transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Destination is updated to match source, including deleting files if necessary (except duplicate -objects, see below). +objects, see below). If you don't want to delete files from destination, +use the copy command instead. Important: Since this can cause data loss, test first with the --dry-run or the --interactive/-i flag. @@ -779,9 +850,9 @@ errors at any point. Duplicate objects (files with the same name, on those providers that support it) are also not yet handled. It is always the contents of the directory that is synced, not the -directory so when source:path is a directory, it's the contents of -source:path that are copied, not the directory name and contents. See -extended explanation in the copy command above if unsure. +directory itself. So when source:path is a directory, it's the contents +of source:path that are copied, not the directory name and contents. See +extended explanation in the copy command if unsure. If dest:path doesn't exist, it is created and the source:path contents go there. @@ -815,6 +886,8 @@ Moves the contents of the source directory to the destination directory. Rclone will error if the source and destination overlap and the remote does not support a server-side directory move operation. +To move single files, use the moveto command instead. + If no filters are in use and if possible this will server-side move source:path into dest:path. After this source:path will no longer exist. @@ -971,6 +1044,9 @@ Checks the files in the source and destination match. It compares sizes and hashes (MD5 or SHA1) and logs a report of files that don't match. It doesn't alter the source or destination. +For the crypt remote there is a dedicated command, cryptcheck, that are +able to check the checksums of the crypted files. + If you supply the --size-only flag, it will only compare the sizes not the hashes as well. Use this for a quick check. @@ -1106,7 +1182,7 @@ Or -1 2017-01-03 14:40:54 -1 2500files -1 2017-07-08 14:39:28 -1 4000files -If you just want the directory names use "rclone lsf --dirs-only". +If you just want the directory names use rclone lsf --dirs-only. Any of the filtering options can be applied to this command. @@ -1211,6 +1287,10 @@ supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling MD5 for any remote. +For other algorithms, see the hashsum command. Running +rclone md5sum remote:path is equivalent to running +rclone hashsum MD5 remote:path. + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a @@ -1246,6 +1326,10 @@ supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling SHA-1 for any remote. +For other algorithms, see the hashsum command. Running +rclone sha1sum remote:path is equivalent to running +rclone hashsum SHA1 remote:path. + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a @@ -1274,6 +1358,23 @@ rclone size Prints the total size and number of objects in remote:path. +Synopsis + +Counts objects in the path and calculates the total size. Prints the +result to standard output. + +By default the output is in human-readable format, but shows values in +both human-readable format as well as the raw numbers (global option +--human-readable is not considered). Use option --json to format output +as JSON instead. + +Recurses by default, use --max-depth 1 to stop the recursion. + +Some backends do not always provide file sizes, see for example Google +Photos and Google Drive. Rclone will then show a notice in the log +indicating how many such files were encountered, and count them in as +empty files in the output of the size command. + rclone size remote:path [flags] Options @@ -1781,7 +1882,7 @@ SEE ALSO rclone completion -generate the autocompletion script for the specified shell +Generate the autocompletion script for the specified shell Synopsis @@ -1798,15 +1899,15 @@ See the global flags page for global options not listed here. SEE ALSO - rclone - Show help for rclone commands, flags and backends. -- rclone completion bash - generate the autocompletion script for bash -- rclone completion fish - generate the autocompletion script for fish -- rclone completion powershell - generate the autocompletion script +- rclone completion bash - Generate the autocompletion script for bash +- rclone completion fish - Generate the autocompletion script for fish +- rclone completion powershell - Generate the autocompletion script for powershell -- rclone completion zsh - generate the autocompletion script for zsh +- rclone completion zsh - Generate the autocompletion script for zsh rclone completion bash -generate the autocompletion script for bash +Generate the autocompletion script for bash Synopsis @@ -1815,12 +1916,19 @@ Generate the autocompletion script for the bash shell. This script depends on the 'bash-completion' package. If it is not installed already, you can install it via your OS's package manager. -To load completions in your current shell session: $ source <(rclone -completion bash) +To load completions in your current shell session: -To load completions for every new session, execute once: Linux: $ rclone -completion bash > /etc/bash_completion.d/rclone MacOS: $ rclone -completion bash > /usr/local/etc/bash_completion.d/rclone + source <(rclone completion bash) + +To load completions for every new session, execute once: + +Linux: + + rclone completion bash > /etc/bash_completion.d/rclone + +macOS: + + rclone completion bash > /usr/local/etc/bash_completion.d/rclone You will need to start a new shell for this setup to take effect. @@ -1835,22 +1943,24 @@ See the global flags page for global options not listed here. SEE ALSO -- rclone completion - generate the autocompletion script for the +- rclone completion - Generate the autocompletion script for the specified shell rclone completion fish -generate the autocompletion script for fish +Generate the autocompletion script for fish Synopsis Generate the autocompletion script for the fish shell. -To load completions in your current shell session: $ rclone completion -fish | source +To load completions in your current shell session: + + rclone completion fish | source -To load completions for every new session, execute once: $ rclone -completion fish > ~/.config/fish/completions/rclone.fish +To load completions for every new session, execute once: + + rclone completion fish > ~/.config/fish/completions/rclone.fish You will need to start a new shell for this setup to take effect. @@ -1865,19 +1975,20 @@ See the global flags page for global options not listed here. SEE ALSO -- rclone completion - generate the autocompletion script for the +- rclone completion - Generate the autocompletion script for the specified shell rclone completion powershell -generate the autocompletion script for powershell +Generate the autocompletion script for powershell Synopsis Generate the autocompletion script for powershell. -To load completions in your current shell session: PS C:> rclone -completion powershell | Out-String | Invoke-Expression +To load completions in your current shell session: + + rclone completion powershell | Out-String | Invoke-Expression To load completions for every new session, add the output of the above command to your powershell profile. @@ -1893,12 +2004,12 @@ See the global flags page for global options not listed here. SEE ALSO -- rclone completion - generate the autocompletion script for the +- rclone completion - Generate the autocompletion script for the specified shell rclone completion zsh -generate the autocompletion script for zsh +Generate the autocompletion script for zsh Synopsis @@ -1907,11 +2018,17 @@ Generate the autocompletion script for the zsh shell. If shell completion is not already enabled in your environment you will need to enable it. You can execute the following once: -$ echo "autoload -U compinit; compinit" >> ~/.zshrc + echo "autoload -U compinit; compinit" >> ~/.zshrc -To load completions for every new session, execute once: # Linux: $ -rclone completion zsh > "${fpath[1]}/_rclone" # macOS: $ rclone -completion zsh > /usr/local/share/zsh/site-functions/_rclone +To load completions for every new session, execute once: + +Linux: + + rclone completion zsh > "${fpath[1]}/_rclone" + +macOS: + + rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone You will need to start a new shell for this setup to take effect. @@ -1926,7 +2043,7 @@ See the global flags page for global options not listed here. SEE ALSO -- rclone completion - generate the autocompletion script for the +- rclone completion - Generate the autocompletion script for the specified shell rclone config create @@ -2468,9 +2585,12 @@ Synopsis Download a URL's content and copy it to the destination without saving it in temporary storage. -Setting --auto-filename will cause the file name to be retrieved from -the URL (after any redirections) and used in the destination path. With ---print-filename in addition, the resulting file name will be printed. +Setting --auto-filename will attempt to automatically determine the +filename from the URL (after any redirections) and used in the +destination path. With --auto-filename-header in addition, if a specific +filename is set in HTTP headers, it will be used instead of the name +from the URL. With --print-filename in addition, the resulting file name +will be printed. Setting --no-clobber will prevent overwriting file on the destination if there is one with the same name. @@ -2482,11 +2602,12 @@ to be written to standard output. Options - -a, --auto-filename Get the file name from the URL and use it for destination file path - -h, --help help for copyurl - --no-clobber Prevent overwriting file with same name - -p, --print-filename Print the resulting name from --auto-filename - --stdout Write the output to stdout rather than a file + -a, --auto-filename Get the file name from the URL and use it for destination file path + --header-filename Get the file name from the Content-Disposition header + -h, --help help for copyurl + --no-clobber Prevent overwriting file with same name + -p, --print-filename Print the resulting name from --auto-filename + --stdout Write the output to stdout rather than a file See the global flags page for global options not listed here. @@ -2586,7 +2707,7 @@ use it like this rclone cryptdecode --reverse encryptedremote: filename1 filename2 Another way to accomplish this is by using the rclone backend encode (or -decode)command. See the documentation on the crypt overlay for more +decode) command. See the documentation on the crypt overlay for more info. rclone cryptdecode encryptedremote: encryptedfilename [flags] @@ -2788,6 +2909,9 @@ supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling any hash for any remote. +For the MD5 and SHA1 algorithms there are also dedicated commands, +md5sum and sha1sum. + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a @@ -2803,6 +2927,7 @@ Run without a hash to see the list of all supported hashes, e.g. * crc32 * sha256 * dropbox + * hidrive * mailru * quickxor @@ -2879,7 +3004,7 @@ Synopsis rclone listremotes lists all the available remotes from the config file. -When uses with the -l flag it lists the types too. +When used with the --long flag it lists the types too. rclone listremotes [flags] @@ -2926,6 +3051,7 @@ just the path, but you can use these parameters to control the output: m - MimeType of object if known e - encrypted name T - tier of storage if known, e.g. "Hot" or "Cool" + M - Metadata of object in JSON blob format, eg {"key":"value"} So if you wanted the path, size and modification time, you would use --format "pst", or maybe --format "tsp" to put the path last. @@ -2940,10 +3066,10 @@ Eg 2016-06-25 18:55:40;37600;fubuwic If you specify "h" in the format you will get the MD5 hash by default, -use the "--hash" flag to change which hash you want. Note that this can -be returned as an empty string if it isn't available on the object (and -for directories), "ERROR" if there was an error reading it from the -object and "UNSUPPORTED" if that object does not support that hash type. +use the --hash flag to change which hash you want. Note that this can be +returned as an empty string if it isn't available on the object (and for +directories), "ERROR" if there was an error reading it from the object +and "UNSUPPORTED" if that object does not support that hash type. For example, to emulate the md5sum command you can use @@ -3046,15 +3172,25 @@ List directories and objects in the path in JSON format. The output is an array of Items, where each Item looks like this -{ "Hashes" : { "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f", -"MD5" : "b1946ac92492d2347c6235b4d2611184", "DropboxHash" : -"ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc" }, -"ID": "y2djkhiujf83u33", "OrigID": "UYOJVTUW00Q1RzTDA", "IsBucket" : -false, "IsDir" : false, "MimeType" : "application/octet-stream", -"ModTime" : "2017-05-31T16:15:57.034468261+01:00", "Name" : "file.txt", -"Encrypted" : "v0qpsdq8anpci8n929v3uu9338", "EncryptedPath" : -"kja9098349023498/v0qpsdq8anpci8n929v3uu9338", "Path" : -"full/path/goes/here/file.txt", "Size" : 6, "Tier" : "hot", } + { + "Hashes" : { + "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f", + "MD5" : "b1946ac92492d2347c6235b4d2611184", + "DropboxHash" : "ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc" + }, + "ID": "y2djkhiujf83u33", + "OrigID": "UYOJVTUW00Q1RzTDA", + "IsBucket" : false, + "IsDir" : false, + "MimeType" : "application/octet-stream", + "ModTime" : "2017-05-31T16:15:57.034468261+01:00", + "Name" : "file.txt", + "Encrypted" : "v0qpsdq8anpci8n929v3uu9338", + "EncryptedPath" : "kja9098349023498/v0qpsdq8anpci8n929v3uu9338", + "Path" : "full/path/goes/here/file.txt", + "Size" : 6, + "Tier" : "hot", + } If --hash is not specified the Hashes property won't be emitted. The types of hash can be specified with the --hash-type parameter (which may @@ -3076,6 +3212,9 @@ returned If --files-only is not specified directories in addition to the files will be returned. +If --metadata is set then an additional Metadata key will be returned. +This will have metdata in rclone standard format as a JSON object. + if --stat is set then a single JSON blob will be returned about the item pointed to. This will return an error if the item isn't found. However on bucket based backends (like s3, gcs, b2, azureblob etc) if the item @@ -3130,7 +3269,7 @@ bucket-based remotes). Options --dirs-only Show only directories in the listing - -M, --encrypted Show the encrypted names + --encrypted Show the encrypted names --files-only Show only files in the listing --hash Include hashes in the output (may take longer) --hash-type stringArray Show only this hash type (may be repeated) @@ -3553,7 +3692,7 @@ VFS Directory Cache Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -3706,6 +3845,37 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. Fingerprints are made from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). + +For example hash is slow with the local and sftp backends as they have +to read the entire file and hash it, and modtime is slow with the s3, +swift, ftp and qinqstor backends because they need to do an extra API +call to fetch it. + +If you use the --vfs-fast-fingerprint flag then rclone will not include +the slow operations in the fingerprint. This makes the fingerprinting +less accurate but much faster and will improve the opening time of +cached files. + +If you are running a vfs cache over local, s3 or swift backends then +using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. + VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -3746,7 +3916,7 @@ of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write @@ -3758,8 +3928,8 @@ cache file. When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of -parallel uploads of modified files from cache (the related global flag ---checkers have no effect on mount). +parallel uploads of modified files from the cache (the related global +flag --checkers has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -3777,23 +3947,22 @@ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The --vfs-case-insensitive mount flag controls how rclone handles these +The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the -mounted file system as-is. If the flag is "true" (or appears without a -value on command line), rclone may perform a "fixup" as explained below. +remote as-is. If the flag is "true" (or appears without a value on the +command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument -refers to an existing file with exactly the same name, then the case of -the existing file on the disk will be used. However, if a file name with -exactly the same name is not found but a name differing only by case -exists, rclone will transparently fixup the name. This fixup happens -only when an existing file is requested. Case sensitivity of file names -created anew by rclone is controlled by an underlying mounted file -system. +different than what is stored on the remote. If an argument refers to an +existing file with exactly the same name, then the case of the existing +file on the disk will be used. However, if a file name with exactly the +same name is not found but a name differing only by case exists, rclone +will transparently fixup the name. This fixup happens only when an +existing file is requested. Case sensitivity of file names created anew +by rclone is controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. @@ -3802,6 +3971,14 @@ depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +VFS Disk Options + +This flag allows you to manually set the statistics about the filing +system. It can be useful when those statistics cannot be read correctly +automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -3846,7 +4023,7 @@ Options --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) @@ -3854,6 +4031,8 @@ Options --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -3934,7 +4113,8 @@ builds an in memory representation. rclone ncdu can be used during this scanning phase and you will see it building up the directory structure as it goes along. -Here are the keys - press '?' to toggle the help on and off +You can interact with the user interface using key presses, press '?' to +toggle the help on and off. The supported keys are: ↑,↓ or k,j to Move →,l to enter @@ -3945,17 +4125,39 @@ Here are the keys - press '?' to toggle the help on and off u toggle human-readable format n,s,C,A sort by name,size,count,average size d delete file/directory + v select file/directory + V enter visual select mode + D delete selected files/directories y copy current path to clipboard Y display current path - ^L refresh screen + ^L refresh screen (fix screen corruption) ? to toggle help on and off - q/ESC/c-C to quit + q/ESC/^c to quit + +Listed files/directories may be prefixed by a one-character flag, some +of them combined with a description in brackes at end of line. These +flags have the following meaning: + + e means this is an empty directory, i.e. contains no files (but + may contain empty subdirectories) + ~ means this is a directory where some of the files (possibly in + subdirectories) have unknown size, and therefore the directory + size may be underestimated (and average size inaccurate, as it + is average of the files with known sizes). + . means an error occurred while reading a subdirectory, and + therefore the directory size may be underestimated (and average + size inaccurate) + ! means an error occurred while reading this directory This an homage to the ncdu tool but for rclone remotes. It is missing lots of features at the moment but is useful as it stands. -Note that it might take some time to delete big files/folders. The UI -won't respond in the meantime since the deletion is done synchronously. +Note that it might take some time to delete big files/directories. The +UI won't respond in the meantime since the deletion is done +synchronously. + +For a non-interactive listing of the remote, see the tree command. To +just get the total size of the remote you can also use the size command. rclone ncdu remote:path [flags] @@ -3989,7 +4191,7 @@ This command can also accept a password through STDIN instead of an argument by passing a hyphen as an argument. This will use the first line of STDIN as the password not including the trailing newline. -echo "secretpassword" | rclone obscure - + echo "secretpassword" | rclone obscure - If there is no data on STDIN to read, rclone obscure will default to obfuscating the hyphen itself. @@ -4034,9 +4236,9 @@ instead of key=value arguments. This is the only way of passing in more complicated values. The -o/--opt option can be used to set a key "opt" with key, value -options in the form "-o key=value" or "-o key". It can be repeated as -many times as required. This is useful for rc commands which take the -"opt" parameter which by convention is a dictionary of strings. +options in the form -o key=value or -o key. It can be repeated as many +times as required. This is useful for rc commands which take the "opt" +parameter which by convention is a dictionary of strings. -o key=value -o key2 @@ -4055,13 +4257,13 @@ Will place this in the "arg" value ["value", "value2"] -Use --loopback to connect to the rclone instance running "rclone rc". -This is very useful for testing commands without having to run an rclone -rc server, e.g.: +Use --loopback to connect to the rclone instance running rclone rc. This +is very useful for testing commands without having to run an rclone rc +server, e.g.: rclone rc --loopback operations/about fs=/ -Use "rclone rc" to see a list of all possible commands. +Use rclone rc to see a list of all possible commands. rclone rc commands parameter [flags] @@ -4106,12 +4308,12 @@ before that. The data must fit into RAM. The cutoff needs to be small enough to adhere the limits of your remote, please see there. Generally speaking, setting this cutoff too high will decrease your performance. -Use the |--size| flag to preallocate the file in advance at the remote -end and actually stream it, even if remote backend doesn't support +Use the --size flag to preallocate the file in advance at the remote end +and actually stream it, even if remote backend doesn't support streaming. -|--size| should be the exact size of the input stream in bytes. If the -size of the stream is different in length to the |--size| passed in then +--size should be the exact size of the input stream in bytes. If the +size of the stream is different in length to the --size passed in then the transfer will likely fail. Note that the upload can also not be retried because the data is not @@ -4271,8 +4473,8 @@ Serve a remote over a protocol. Synopsis -rclone serve is used to serve a remote over a given protocol. This -command requires the use of a subcommand to specify the protocol, e.g. +Serve a remote over a given protocol. Requires the use of a subcommand +to specify the protocol, e.g. rclone serve http remote: @@ -4296,7 +4498,7 @@ SEE ALSO - rclone serve http - Serve the remote over HTTP. - rclone serve restic - Serve the remote for restic's REST API. - rclone serve sftp - Serve the remote over SFTP. -- rclone serve webdav - Serve remote:path over webdav. +- rclone serve webdav - Serve remote:path over WebDAV. rclone serve dlna @@ -4304,11 +4506,11 @@ Serve remote:path over DLNA Synopsis -rclone serve dlna is a DLNA media server for media stored in an rclone -remote. Many devices, such as the Xbox and PlayStation, can -automatically discover this server in the LAN and play audio/video from -it. VLC is also supported. Service discovery uses UDP multicast packets -(SSDP) and will thus only work on LANs. +Run a DLNA media server for media stored in an rclone remote. Many +devices, such as the Xbox and PlayStation, can automatically discover +this server in the LAN and play audio/video from it. VLC is also +supported. Service discovery uses UDP multicast packets (SSDP) and will +thus only work on LANs. Rclone will list all files present in the remote, without filtering based on media formats or file extensions. Additionally, there is no @@ -4344,7 +4546,7 @@ VFS Directory Cache Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -4497,6 +4699,37 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. Fingerprints are made from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). + +For example hash is slow with the local and sftp backends as they have +to read the entire file and hash it, and modtime is slow with the s3, +swift, ftp and qinqstor backends because they need to do an extra API +call to fetch it. + +If you use the --vfs-fast-fingerprint flag then rclone will not include +the slow operations in the fingerprint. This makes the fingerprinting +less accurate but much faster and will improve the opening time of +cached files. + +If you are running a vfs cache over local, s3 or swift backends then +using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. + VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -4537,7 +4770,7 @@ of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write @@ -4549,8 +4782,8 @@ cache file. When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of -parallel uploads of modified files from cache (the related global flag ---checkers have no effect on mount). +parallel uploads of modified files from the cache (the related global +flag --checkers has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -4568,23 +4801,22 @@ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The --vfs-case-insensitive mount flag controls how rclone handles these +The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the -mounted file system as-is. If the flag is "true" (or appears without a -value on command line), rclone may perform a "fixup" as explained below. +remote as-is. If the flag is "true" (or appears without a value on the +command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument -refers to an existing file with exactly the same name, then the case of -the existing file on the disk will be used. However, if a file name with -exactly the same name is not found but a name differing only by case -exists, rclone will transparently fixup the name. This fixup happens -only when an existing file is requested. Case sensitivity of file names -created anew by rclone is controlled by an underlying mounted file -system. +different than what is stored on the remote. If an argument refers to an +existing file with exactly the same name, then the case of the existing +file on the disk will be used. However, if a file name with exactly the +same name is not found but a name differing only by case exists, rclone +will transparently fixup the name. This fixup happens only when an +existing file is requested. Case sensitivity of file names created anew +by rclone is controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. @@ -4593,6 +4825,14 @@ depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +VFS Disk Options + +This flag allows you to manually set the statistics about the filing +system. It can be useful when those statistics cannot be read correctly +automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -4623,7 +4863,7 @@ Options --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) @@ -4631,6 +4871,8 @@ Options --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -4707,7 +4949,7 @@ VFS Directory Cache Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -4860,6 +5102,37 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. Fingerprints are made from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). + +For example hash is slow with the local and sftp backends as they have +to read the entire file and hash it, and modtime is slow with the s3, +swift, ftp and qinqstor backends because they need to do an extra API +call to fetch it. + +If you use the --vfs-fast-fingerprint flag then rclone will not include +the slow operations in the fingerprint. This makes the fingerprinting +less accurate but much faster and will improve the opening time of +cached files. + +If you are running a vfs cache over local, s3 or swift backends then +using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. + VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -4900,7 +5173,7 @@ of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write @@ -4912,8 +5185,8 @@ cache file. When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of -parallel uploads of modified files from cache (the related global flag ---checkers have no effect on mount). +parallel uploads of modified files from the cache (the related global +flag --checkers has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -4931,23 +5204,22 @@ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The --vfs-case-insensitive mount flag controls how rclone handles these +The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the -mounted file system as-is. If the flag is "true" (or appears without a -value on command line), rclone may perform a "fixup" as explained below. +remote as-is. If the flag is "true" (or appears without a value on the +command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument -refers to an existing file with exactly the same name, then the case of -the existing file on the disk will be used. However, if a file name with -exactly the same name is not found but a name differing only by case -exists, rclone will transparently fixup the name. This fixup happens -only when an existing file is requested. Case sensitivity of file names -created anew by rclone is controlled by an underlying mounted file -system. +different than what is stored on the remote. If an argument refers to an +existing file with exactly the same name, then the case of the existing +file on the disk will be used. However, if a file name with exactly the +same name is not found but a name differing only by case exists, rclone +will transparently fixup the name. This fixup happens only when an +existing file is requested. Case sensitivity of file names created anew +by rclone is controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. @@ -4956,6 +5228,14 @@ depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +VFS Disk Options + +This flag allows you to manually set the statistics about the filing +system. It can be useful when those statistics cannot be read correctly +automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -5003,7 +5283,7 @@ Options --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --socket-addr string Address or absolute path (default: /run/docker/plugins/rclone.sock) --socket-gid int GID for unix socket (default: current process GID) (default 1000) --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) @@ -5013,6 +5293,8 @@ Options --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -5035,9 +5317,9 @@ Serve remote:path over FTP. Synopsis -rclone serve ftp implements a basic ftp server to serve the remote over -FTP protocol. This can be viewed with a ftp client or you can make a -remote of type ftp to read and write it. +Run a basic FTP server to serve a remote over FTP protocol. This can be +viewed with a FTP client or you can make a remote of type FTP to read +and write it. Server options @@ -5074,7 +5356,7 @@ VFS Directory Cache Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -5227,6 +5509,37 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. Fingerprints are made from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). + +For example hash is slow with the local and sftp backends as they have +to read the entire file and hash it, and modtime is slow with the s3, +swift, ftp and qinqstor backends because they need to do an extra API +call to fetch it. + +If you use the --vfs-fast-fingerprint flag then rclone will not include +the slow operations in the fingerprint. This makes the fingerprinting +less accurate but much faster and will improve the opening time of +cached files. + +If you are running a vfs cache over local, s3 or swift backends then +using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. + VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -5267,7 +5580,7 @@ of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write @@ -5279,8 +5592,8 @@ cache file. When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of -parallel uploads of modified files from cache (the related global flag ---checkers have no effect on mount). +parallel uploads of modified files from the cache (the related global +flag --checkers has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -5298,23 +5611,22 @@ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The --vfs-case-insensitive mount flag controls how rclone handles these +The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the -mounted file system as-is. If the flag is "true" (or appears without a -value on command line), rclone may perform a "fixup" as explained below. +remote as-is. If the flag is "true" (or appears without a value on the +command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument -refers to an existing file with exactly the same name, then the case of -the existing file on the disk will be used. However, if a file name with -exactly the same name is not found but a name differing only by case -exists, rclone will transparently fixup the name. This fixup happens -only when an existing file is requested. Case sensitivity of file names -created anew by rclone is controlled by an underlying mounted file -system. +different than what is stored on the remote. If an argument refers to an +existing file with exactly the same name, then the case of the existing +file on the disk will be used. However, if a file name with exactly the +same name is not found but a name differing only by case exists, rclone +will transparently fixup the name. This fixup happens only when an +existing file is requested. Case sensitivity of file names created anew +by rclone is controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. @@ -5323,6 +5635,14 @@ depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +VFS Disk Options + +This flag allows you to manually set the statistics about the filing +system. It can be useful when those statistics cannot be read correctly +automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -5428,7 +5748,7 @@ Options --passive-port string Passive port range to use (default "30000-32000") --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --public-ip string Public IP address to advertise for passive connections - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication (default "anonymous") @@ -5437,6 +5757,8 @@ Options --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -5457,9 +5779,8 @@ Serve the remote over HTTP. Synopsis -rclone serve http implements a basic web server to serve the remote over -HTTP. This can be viewed in a web browser or you can make a remote of -type http read from it. +Run a basic web server to serve a remote over HTTP. This can be viewed +in a web browser or you can make a remote of type http read from it. You can use the filter flags (e.g. --include, --exclude) to control what is served. @@ -5490,8 +5811,9 @@ accept in the HTTP header. rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading -and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl -"/rclone" and --baseurl "/rclone/" are all treated identically. +and trailing "/" on --baseurl, so --baseurl "rclone", +--baseurl "/rclone" and --baseurl "/rclone/" are all treated +identically. SSL/TLS @@ -5507,8 +5829,8 @@ authority certificate. Template ---template allows a user to specify a custom markup template for http -and webdav serve functions. The server exports the following markup to +--template allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: ----------------------------------------------------------------------- @@ -5598,7 +5920,7 @@ VFS Directory Cache Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -5751,6 +6073,37 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. Fingerprints are made from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). + +For example hash is slow with the local and sftp backends as they have +to read the entire file and hash it, and modtime is slow with the s3, +swift, ftp and qinqstor backends because they need to do an extra API +call to fetch it. + +If you use the --vfs-fast-fingerprint flag then rclone will not include +the slow operations in the fingerprint. This makes the fingerprinting +less accurate but much faster and will improve the opening time of +cached files. + +If you are running a vfs cache over local, s3 or swift backends then +using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. + VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -5791,7 +6144,7 @@ of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write @@ -5803,8 +6156,8 @@ cache file. When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of -parallel uploads of modified files from cache (the related global flag ---checkers have no effect on mount). +parallel uploads of modified files from the cache (the related global +flag --checkers has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -5822,23 +6175,22 @@ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The --vfs-case-insensitive mount flag controls how rclone handles these +The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the -mounted file system as-is. If the flag is "true" (or appears without a -value on command line), rclone may perform a "fixup" as explained below. +remote as-is. If the flag is "true" (or appears without a value on the +command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument -refers to an existing file with exactly the same name, then the case of -the existing file on the disk will be used. However, if a file name with -exactly the same name is not found but a name differing only by case -exists, rclone will transparently fixup the name. This fixup happens -only when an existing file is requested. Case sensitivity of file names -created anew by rclone is controlled by an underlying mounted file -system. +different than what is stored on the remote. If an argument refers to an +existing file with exactly the same name, then the case of the existing +file on the disk will be used. However, if a file name with exactly the +same name is not found but a name differing only by case exists, rclone +will transparently fixup the name. This fixup happens only when an +existing file is requested. Case sensitivity of file names created anew +by rclone is controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. @@ -5847,6 +6199,14 @@ depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +VFS Disk Options + +This flag allows you to manually set the statistics about the filing +system. It can be useful when those statistics cannot be read correctly +automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -5882,7 +6242,7 @@ Options --no-seek Don't allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --realm string Realm for authentication --salt string Password hashing salt (default "dlPL2MqE") --server-read-timeout duration Timeout for server reading data (default 1h0m0s) @@ -5896,6 +6256,8 @@ Options --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -5916,9 +6278,9 @@ Serve the remote for restic's REST API. Synopsis -rclone serve restic implements restic's REST backend API over HTTP. This -allows restic to use rclone as a data storage mechanism for cloud -providers that restic does not support directly. +Run a basic web server to serve a remove over restic's REST backend API +over HTTP. This allows restic to use rclone as a data storage mechanism +for cloud providers that restic does not support directly. Restic is a command-line program for doing backups. @@ -5944,7 +6306,7 @@ Where you can replace "backup" in the above by whatever path in the remote you wish to use. By default this will serve on "localhost:8080" you can change this with -use of the "--addr" flag. +use of the --addr flag. You might wish to start this server on boot. @@ -5992,7 +6354,7 @@ must end with /. Eg Private repositories -The "--private-repos" flag can be used to limit users to repositories +The--private-repos flag can be used to limit users to repositories starting with a path of //. Server options @@ -6016,11 +6378,12 @@ accept in the HTTP header. rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading -and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl -"/rclone" and --baseurl "/rclone/" are all treated identically. +and trailing "/" on --baseurl, so --baseurl "rclone", +--baseurl "/rclone" and --baseurl "/rclone/" are all treated +identically. ---template allows a user to specify a custom markup template for http -and webdav serve functions. The server exports the following markup to +--template allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: ----------------------------------------------------------------------- @@ -6092,8 +6455,8 @@ Use --realm to set the authentication realm. SSL/TLS -By default this will serve over http. If you want you can serve over -https. You will need to supply the --cert and --key flags. If you wish +By default this will serve over HTTP. If you want you can serve over +HTTPS. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also. @@ -6137,9 +6500,8 @@ Serve the remote over SFTP. Synopsis -rclone serve sftp implements an SFTP server to serve the remote over -SFTP. This can be used with an SFTP client or you can make a remote of -type sftp to use with it. +Run a SFTP server to serve a remote over SFTP. This can be used with an +SFTP client or you can make a remote of type sftp to use with it. You can use the filter flags (e.g. --include, --exclude) to control what is served. @@ -6161,13 +6523,13 @@ command when paired with the rclone sftp backend. If you don't supply a host --key then rclone will generate rsa, ecdsa and ed25519 variants, and cache them for later use in rclone's cache -directory (see "rclone help flags cache-dir") in the "serve-sftp" +directory (see rclone help flags cache-dir) in the "serve-sftp" directory. By default the server binds to localhost:2022 - if you want it to be -reachable externally then supply "--addr :2022" for example. +reachable externally then supply --addr :2022 for example. -Note that the default of "--vfs-cache-mode off" is fine for the rclone +Note that the default of --vfs-cache-mode off is fine for the rclone sftp backend, but it may not be with other SFTP clients. If --stdio is specified, rclone will serve SFTP over stdio, which can be @@ -6175,7 +6537,7 @@ used with sshd via ~/.ssh/authorized_keys, for example: restrict,command="rclone serve sftp --stdio ./photos" ssh-rsa ... -On the client you need to set "--transfers 1" when using --stdio. +On the client you need to set --transfers 1 when using --stdio. Otherwise multiple instances of the rclone server are started by OpenSSH which can lead to "corrupted on transfer" errors. This is the case because the client chooses indiscriminately which server to send @@ -6205,7 +6567,7 @@ VFS Directory Cache Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -6358,6 +6720,37 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. Fingerprints are made from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). + +For example hash is slow with the local and sftp backends as they have +to read the entire file and hash it, and modtime is slow with the s3, +swift, ftp and qinqstor backends because they need to do an extra API +call to fetch it. + +If you use the --vfs-fast-fingerprint flag then rclone will not include +the slow operations in the fingerprint. This makes the fingerprinting +less accurate but much faster and will improve the opening time of +cached files. + +If you are running a vfs cache over local, s3 or swift backends then +using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. + VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -6398,7 +6791,7 @@ of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write @@ -6410,8 +6803,8 @@ cache file. When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of -parallel uploads of modified files from cache (the related global flag ---checkers have no effect on mount). +parallel uploads of modified files from the cache (the related global +flag --checkers has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -6429,23 +6822,22 @@ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The --vfs-case-insensitive mount flag controls how rclone handles these +The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the -mounted file system as-is. If the flag is "true" (or appears without a -value on command line), rclone may perform a "fixup" as explained below. +remote as-is. If the flag is "true" (or appears without a value on the +command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument -refers to an existing file with exactly the same name, then the case of -the existing file on the disk will be used. However, if a file name with -exactly the same name is not found but a name differing only by case -exists, rclone will transparently fixup the name. This fixup happens -only when an existing file is requested. Case sensitivity of file names -created anew by rclone is controlled by an underlying mounted file -system. +different than what is stored on the remote. If an argument refers to an +existing file with exactly the same name, then the case of the existing +file on the disk will be used. However, if a file name with exactly the +same name is not found but a name differing only by case exists, rclone +will transparently fixup the name. This fixup happens only when an +existing file is requested. Case sensitivity of file names created anew +by rclone is controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. @@ -6454,6 +6846,14 @@ depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +VFS Disk Options + +This flag allows you to manually set the statistics about the filing +system. It can be useful when those statistics cannot be read correctly +automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -6558,7 +6958,7 @@ Options --no-seek Don't allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --stdio Run an sftp server on run stdin/stdout --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) @@ -6568,6 +6968,8 @@ Options --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -6584,16 +6986,15 @@ SEE ALSO rclone serve webdav -Serve remote:path over webdav. +Serve remote:path over WebDAV. Synopsis -rclone serve webdav implements a basic webdav server to serve the remote -over HTTP via the webdav protocol. This can be viewed with a webdav -client, through a web browser, or you can make a remote of type webdav -to read and write it. +Run a basic WebDAV server to serve a remote over HTTP via the WebDAV +protocol. This can be viewed with a WebDAV client, through a web +browser, or you can make a remote of type WebDAV to read and write it. -Webdav options +WebDAV options --etag-hash @@ -6602,9 +7003,7 @@ on the ModTime and Size of the object. If this flag is set to "auto" then rclone will choose the first supported hash on the backend or you can use a named hash such as "MD5" -or "SHA-1". - -Use "rclone hashsum" to see the full list. +or "SHA-1". Use the hashsum command to see the full list. Server options @@ -6627,11 +7026,12 @@ accept in the HTTP header. rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading -and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl -"/rclone" and --baseurl "/rclone/" are all treated identically. +and trailing "/" on --baseurl, so --baseurl "rclone", +--baseurl "/rclone" and --baseurl "/rclone/" are all treated +identically. ---template allows a user to specify a custom markup template for http -and webdav serve functions. The server exports the following markup to +--template allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: ----------------------------------------------------------------------- @@ -6703,8 +7103,8 @@ Use --realm to set the authentication realm. SSL/TLS -By default this will serve over http. If you want you can serve over -https. You will need to supply the --cert and --key flags. If you wish +By default this will serve over HTTP. If you want you can serve over +HTTPS. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also. @@ -6731,7 +7131,7 @@ VFS Directory Cache Using the --dir-cache-time flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -6884,6 +7284,37 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. Fingerprints are made from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). + +For example hash is slow with the local and sftp backends as they have +to read the entire file and hash it, and modtime is slow with the s3, +swift, ftp and qinqstor backends because they need to do an extra API +call to fetch it. + +If you use the --vfs-fast-fingerprint flag then rclone will not include +the slow operations in the fingerprint. This makes the fingerprinting +less accurate but much faster and will improve the opening time of +cached files. + +If you are running a vfs cache over local, s3 or swift backends then +using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. + VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -6924,7 +7355,7 @@ of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or write @@ -6936,8 +7367,8 @@ cache file. When using VFS write caching (--vfs-cache-mode with value writes or full), the global flag --transfers can be set to adjust the number of -parallel uploads of modified files from cache (the related global flag ---checkers have no effect on mount). +parallel uploads of modified files from the cache (the related global +flag --checkers has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -6955,23 +7386,22 @@ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The --vfs-case-insensitive mount flag controls how rclone handles these +The --vfs-case-insensitive VFS flag controls how rclone handles these two cases. If its value is "false", rclone passes file names to the -mounted file system as-is. If the flag is "true" (or appears without a -value on command line), rclone may perform a "fixup" as explained below. +remote as-is. If the flag is "true" (or appears without a value on the +command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument -refers to an existing file with exactly the same name, then the case of -the existing file on the disk will be used. However, if a file name with -exactly the same name is not found but a name differing only by case -exists, rclone will transparently fixup the name. This fixup happens -only when an existing file is requested. Case sensitivity of file names -created anew by rclone is controlled by an underlying mounted file -system. +different than what is stored on the remote. If an argument refers to an +existing file with exactly the same name, then the case of the existing +file on the disk will be used. However, if a file name with exactly the +same name is not found but a name differing only by case exists, rclone +will transparently fixup the name. This fixup happens only when an +existing file is requested. Case sensitivity of file names created anew +by rclone is controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. @@ -6980,6 +7410,14 @@ depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +VFS Disk Options + +This flag allows you to manually set the statistics about the filing +system. It can be useful when those statistics cannot be read correctly +automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -7089,7 +7527,7 @@ Options --no-seek Don't allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --realm string Realm for authentication (default "rclone") --server-read-timeout duration Timeout for server reading data (default 1h0m0s) --server-write-timeout duration Timeout for server writing data (default 1h0m0s) @@ -7102,6 +7540,8 @@ Options --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) @@ -7188,6 +7628,8 @@ SEE ALSO - rclone test histogram - Makes a histogram of file name characters. - rclone test info - Discovers file name or other limitations for paths. +- rclone test makefile - Make files with random contents of the size + given - rclone test makefiles - Make a random file hierarchy in a directory - rclone test memory - Load all the objects at remote:path into memory and report memory stats. @@ -7265,6 +7707,28 @@ SEE ALSO - rclone test - Run a test command +rclone test makefile + +Make files with random contents of the size given + + rclone test makefile []+ [flags] + +Options + + --ascii Fill files with random ASCII printable bytes only + --chargen Fill files with a ASCII chargen pattern + -h, --help help for makefile + --pattern Fill files with a periodic pattern + --seed int Seed for the random number generator (0 for random) (default 1) + --sparse Make the files sparse (appear to be filled with ASCII 0x00) + --zero Fill files with ASCII 0x00 + +See the global flags page for global options not listed here. + +SEE ALSO + +- rclone test - Run a test command + rclone test makefiles Make a random file hierarchy in a directory @@ -7273,6 +7737,8 @@ Make a random file hierarchy in a directory Options + --ascii Fill files with random ASCII printable bytes only + --chargen Fill files with a ASCII chargen pattern --files int Number of files to create (default 1000) --files-per-directory int Average number of files per directory (default 10) -h, --help help for makefiles @@ -7280,7 +7746,10 @@ Options --max-name-length int Maximum size of file names (default 12) --min-file-size SizeSuffix Minimum size of file to create --min-name-length int Minimum size of file names (default 4) + --pattern Fill files with a periodic pattern --seed int Seed for the random number generator (0 for random) (default 1) + --sparse Make the files sparse (appear to be filled with ASCII 0x00) + --zero Fill files with ASCII 0x00 See the global flags page for global options not listed here. @@ -7369,11 +7838,14 @@ For example 1 directories, 5 files You can use any of the filtering options with the tree command (e.g. ---include and --exclude). You can also use --fast-list. +--include and --exclude. You can also use --fast-list. The tree command has many options for controlling the listing which are -compatible with the tree command. Note that not all of them have short -options as they conflict with rclone's short options. +compatible with the tree command, for example you can include file sizes +with --size. Note that not all of them have short options as they +conflict with rclone's short options. + +For a more interactive navigation of the remote see the ncdu command. rclone tree remote:path [flags] @@ -7685,6 +8157,147 @@ This can be used when scripting to make aged backups efficiently, e.g. rclone sync -i remote:current-backup remote:previous-backup rclone sync -i /path/to/files remote:current-backup +Metadata support + +Metadata is data about a file which isn't the contents of the file. +Normally rclone only preserves the modification time and the content +(MIME) type where possible. + +Rclone supports preserving all the available metadata on files (not +directories) when using the --metadata or -M flag. + +Exactly what metadata is supported and what that support means depends +on the backend. Backends that support metadata have a metadata section +in their docs and are listed in the features table (Eg local, s3) + +Rclone only supports a one-time sync of metadata. This means that +metadata will be synced from the source object to the destination object +only when the source object has changed and needs to be re-uploaded. If +the metadata subsequently changes on the source object without changing +the object itself then it won't be synced to the destination object. +This is in line with the way rclone syncs Content-Type without the +--metadata flag. + +Using --metadata when syncing from local to local will preserve file +attributes such as file mode, owner, extended attributes (not Windows). + +Note that arbitrary metadata may be added to objects using the +--metadata-set key=value flag when the object is first uploaded. This +flag can be repeated as many times as necessary. + +Types of metadata + +Metadata is divided into two type. System metadata and User metadata. + +Metadata which the backend uses itself is called system metadata. For +example on the local backend the system metadata uid will store the user +ID of the file when used on a unix based platform. + +Arbitrary metadata is called user metadata and this can be set however +is desired. + +When objects are copied from backend to backend, they will attempt to +interpret system metadata if it is supplied. Metadata may change from +being user metadata to system metadata as objects are copied between +different backends. For example copying an object from s3 sets the +content-type metadata. In a backend which understands this (like +azureblob) this will become the Content-Type of the object. In a backend +which doesn't understand this (like the local backend) this will become +user metadata. However should the local object be copied back to s3, the +Content-Type will be set correctly. + +Metadata framework + +Rclone implements a metadata framework which can read metadata from an +object and write it to the object when (and only when) it is being +uploaded. + +This metadata is stored as a dictionary with string keys and string +values. + +There are some limits on the names of the keys (these may be clarified +further in the future). + +- must be lower case +- may be a-z 0-9 containing . - or _ +- length is backend dependent + +Each backend can provide system metadata that it understands. Some +backends can also store arbitrary user metadata. + +Where possible the key names are standardized, so, for example, it is +possible to copy object metadata from s3 to azureblob for example and +metadata will be translated apropriately. + +Some backends have limits on the size of the metadata and rclone will +give errors on upload if they are exceeded. + +Metadata preservation + +The goal of the implementation is to + +1. Preserve metadata if at all possible +2. Interpret metadata if at all possible + +The consequences of 1 is that you can copy an S3 object to a local disk +then back to S3 losslessly. Likewise you can copy a local file with file +attributes and xattrs from local disk to s3 and back again losslessly. + +The consequence of 2 is that you can copy an S3 object with metadata to +Azureblob (say) and have the metadata appear on the Azureblob object +also. + +Standard system metadata + +Here is a table of standard system metadata which, if appropriate, a +backend may implement. + + ---------------------------------------------------------------------------------------------- + key description example + ---------------------------------- --------------------- ------------------------------------- + mode File type and mode: 0100664 + octal, unix style + + uid User ID of owner: 500 + decimal number + + gid Group ID of owner: 500 + decimal number + + rdev Device ID (if special 0 + file) => hexadecimal + + atime Time of last access: 2006-01-02T15:04:05.999999999Z07:00 + RFC 3339 + + mtime Time of last 2006-01-02T15:04:05.999999999Z07:00 + modification: RFC + 3339 + + btime Time of file creation 2006-01-02T15:04:05.999999999Z07:00 + (birth): RFC 3339 + + cache-control Cache-Control header no-cache + + content-disposition Content-Disposition inline + header + + content-encoding Content-Encoding gzip + header + + content-language Content-Language en-US + header + + content-type Content-Type header text/plain + ---------------------------------------------------------------------------------------------- + +The metadata keys mtime and content-type will take precedence if +supplied in the metadata over reading the Content-Type or modification +time of the source object. + +Hashes are not included in system metadata as there is a well defined +way of reading those already. + Options Rclone has a number of options to control its behaviour. @@ -7896,12 +8509,22 @@ held in memory before the transfers start. --checkers=N -The number of checkers to run in parallel. Checkers do the equality -checking of files during a sync. For some storage systems (e.g. S3, -Swift, Dropbox) this can take a significant amount of time so they are -run in parallel. +Originally controlling just the number of file checkers to run in +parallel, e.g. by rclone copy. Now a fairly universal parallelism +control used by rclone in several places. + +Note: checkers do the equality checking of files during a sync. For some +storage systems (e.g. S3, Swift, Dropbox) this can take a significant +amount of time so they are run in parallel. -The default is to run 8 checkers in parallel. +The default is to run 8 checkers in parallel. However, in case of +slow-reacting backends you may need to lower (rather than increase) this +default by setting --checkers to 4 or less threads. This is especially +advised if you are experiencing backend server crashes during file +checking phase (e.g. on subsequent or top-up backups where little or no +file copying is done and checking takes up most of the time). Increase +this setting only with utmost care, while monitoring your server health +and file checking throughput. -c, --checksum @@ -8047,8 +8670,9 @@ See --compare-dest and --backup-dir. --dedupe-mode MODE Mode to run dedupe command in. One of interactive, skip, first, newest, -oldest, rename. The default is interactive. See the dedupe command for -more information as to what these options mean. +oldest, rename. The default is interactive. +See the dedupe command for more information as to what these options +mean. --disable FEATURE,FEATURE,... @@ -8444,6 +9068,17 @@ When the limit is reached all transfers will stop immediately. Rclone will exit with exit code 8 if the transfer limit is reached. +--metadata / -M + +Setting this flag enables rclone to copy the metadata from the source to +the destination. For local backends this is ownership, permissions, +xattr etc. See the #metadata for more info. + +--metadata-set key=value + +Add metadata key = value when uploading. This can be repeated as many +times as required. See the #metadata for more info. + --cutoff-mode=hard|soft|cautious This modifies the behavior of --max-transfer Defaults to @@ -9047,6 +9682,9 @@ timeouts or bigger if you have lots of bandwidth and a fast remote. The default is to run 4 file transfers in parallel. +Look at --multi-thread-streams if you would like to control single file +transfers. + -u, --update This forces rclone to skip any files which exist on the destination and @@ -9119,6 +9757,9 @@ With -vv rclone will become very verbose telling you about every file it considers and transfers. Please send bug reports with a log with this setting. +When setting verbosity as an environment variable, use RCLONE_VERBOSE=1 +or RCLONE_VERBOSE=2 for -v and -vv respectively. + -V, --version Prints the version number @@ -9360,6 +10001,7 @@ For the filtering options - --filter-from - --exclude - --exclude-from +- --exclude-if-present - --include - --include-from - --files-from @@ -9466,6 +10108,9 @@ the environment variable setting. Or to always use the trash in drive --drive-use-trash, set RCLONE_DRIVE_USE_TRASH=true. +Verbosity is slightly different, the environment variable equivalent of +--verbose or -v is RCLONE_VERBOSE=1, or for -vv, RCLONE_VERBOSE=2. + The same parser is used for the options and the environment variables so they take exactly the same form. @@ -9647,6 +10292,29 @@ Now transfer it to the remote box (scp, cut paste, ftp, sftp, etc.) and place it in the correct place (use rclone config file on the remote box to find out where). +Configuring using SSH Tunnel + +Linux and MacOS users can utilize SSH Tunnel to redirect the headless +box port 53682 to local machine by using the following command: + + ssh -L localhost:53682:localhost:53682 username@remote_server + +Then on the headless box run rclone config and answer Y to the +Use auto config? question. + + ... + Remote config + Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine + y) Yes (default) + n) No + y/n> y + +Then copy and paste the auth url +http://127.0.0.1:53682/auth?state=xxxxxxxxxxxx to the browser on your +local machine, complete the auth and it is done. + Filtering, includes and excludes Filter flags determine which files rclone sync, move, ls, lsl, md5sum, @@ -9774,7 +10442,11 @@ regular expression syntax. The regular expressions used are as defined in the Go regular expression reference. Regular expressions should be enclosed in {{ }}. They will match only the last path segment if the glob doesn't start with / or the -whole path name if it does. +whole path name if it does. Note that rclone does not attempt to parse +the supplied regular expression, meaning that using any regular +expression filter will prevent rclone from using directory filter rules, +as it will instead check every path against the supplied regular +expression(s). Here is how the {{regexp}} is transformed into an full regular expression to match the entire path: @@ -9906,10 +10578,14 @@ remote by avoiding listing unnecessary directories. Whether optimisation is desirable depends on the specific filter rules and source remote content. +If any regular expression filters are in use, then no directory +recursion optimisation is possible, as rclone must check every path +against the supplied regular expression(s). + Directory recursion optimisation occurs if either: - A source remote does not support the rclone ListR primitive. local, - sftp, Microsoft OneDrive and WebDav do not support ListR. Google + sftp, Microsoft OneDrive and WebDAV do not support ListR. Google Drive and most bucket type storage do. Full list - On other remotes (those that support ListR), if the rclone command @@ -10380,7 +11056,8 @@ Exclude directory based on a file The --exclude-if-present flag controls whether a directory is within the scope of an rclone command based on the presence of a named file within -it. +it. The flag can be repeated to check for multiple file names, presence +of any of them will exclude the directory. This flag has a priority over other filter flags. @@ -10394,8 +11071,6 @@ E.g. for the following directory structure: The command rclone ls --exclude-if-present .ignore dir1 does not list dir3, file3 or .ignore. ---exclude-if-present can only be used once in an rclone command. - Common pitfalls The most frequent filter support issues on the rclone forum are: @@ -10518,7 +11193,7 @@ Remote controlling rclone with its API If rclone is run with the --rc flag then it starts an HTTP server which can be used to remote control rclone using its API. -You can either use the rclone rc command to access the API or use HTTP +You can either use the rc command to access the API or use HTTP directly. If you just want to run a remote control then see the rcd command. @@ -10666,6 +11341,16 @@ use these credentials in the request. Default Off. +--rc-baseurl + +Prefix for URLs. + +Default is root + +--rc-template + +User-specified template. + Accessing the remote control via the rclone rc command Rclone itself implements the remote control protocol in its rclone rc @@ -11023,7 +11708,7 @@ This takes the following parameters: - state - state to restart with - used with continue - result - result to restart with - used with continue -See the config create command command for more information on the above. +See the config create command for more information on the above. Authentication is required for this call. @@ -11033,7 +11718,7 @@ Parameters: - name - name of remote to delete -See the config delete command command for more information on the above. +See the config delete command for more information on the above. Authentication is required for this call. @@ -11043,7 +11728,7 @@ Returns a JSON object: - key: value Where keys are remote names and values are the config parameters. -See the config dump command command for more information on the above. +See the config dump command for more information on the above. Authentication is required for this call. @@ -11053,7 +11738,7 @@ Parameters: - name - name of remote to get -See the config dump command command for more information on the above. +See the config dump command for more information on the above. Authentication is required for this call. @@ -11061,7 +11746,7 @@ config/listremotes: Lists the remotes in the config file. Returns - remotes - array of remote names -See the listremotes command command for more information on the above. +See the listremotes command for more information on the above. Authentication is required for this call. @@ -11072,8 +11757,7 @@ This takes the following parameters: - name - name of remote - parameters - a map of { "key": "value" } pairs -See the config password command command for more information on the -above. +See the config password command for more information on the above. Authentication is required for this call. @@ -11081,8 +11765,7 @@ config/providers: Shows how providers are configured in the config file. Returns a JSON object: - providers - array of objects -See the config providers command command for more information on the -above. +See the config providers command for more information on the above. Authentication is required for this call. @@ -11102,7 +11785,7 @@ This takes the following parameters: - state - state to restart with - used with continue - result - result to restart with - used with continue -See the config update command command for more information on the above. +See the config update command for more information on the above. Authentication is required for this call. @@ -11552,7 +12235,7 @@ This takes the following parameters: The result is as returned from rclone about --json -See the about command command for more information on the above. +See the about command for more information on the above. Authentication is required for this call. @@ -11562,7 +12245,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" -See the cleanup command command for more information on the above. +See the cleanup command for more information on the above. Authentication is required for this call. @@ -11586,8 +12269,9 @@ This takes the following parameters: - remote - a path within that remote e.g. "dir" - url - string, URL to read from - autoFilename - boolean, set to true to retrieve destination file - name from url See the copyurl command command for more information - on the above. + name from url + +See the copyurl command for more information on the above. Authentication is required for this call. @@ -11597,7 +12281,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" -See the delete command command for more information on the above. +See the delete command for more information on the above. Authentication is required for this call. @@ -11608,7 +12292,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the deletefile command command for more information on the above. +See the deletefile command for more information on the above. Authentication is required for this call. @@ -11621,46 +12305,103 @@ This takes the following parameters: This returns info about the remote passed in; { - // optional features and whether they are available or not - "Features": { - "About": true, - "BucketBased": false, - "CanHaveEmptyDirectories": true, - "CaseInsensitive": false, - "ChangeNotify": false, - "CleanUp": false, - "Copy": false, - "DirCacheFlush": false, - "DirMove": true, - "DuplicateFiles": false, - "GetTier": false, - "ListR": false, - "MergeDirs": false, - "Move": true, - "OpenWriterAt": true, - "PublicLink": false, - "Purge": true, - "PutStream": true, - "PutUnchecked": false, - "ReadMimeType": false, - "ServerSideAcrossConfigs": false, - "SetTier": false, - "SetWrapper": false, - "UnWrap": false, - "WrapFs": false, - "WriteMimeType": false - }, - // Names of hashes available - "Hashes": [ - "MD5", - "SHA-1", - "DropboxHash", - "QuickXorHash" - ], - "Name": "local", // Name as created - "Precision": 1, // Precision of timestamps in ns - "Root": "/", // Path as created - "String": "Local file system at /" // how the remote will appear in logs + // optional features and whether they are available or not + "Features": { + "About": true, + "BucketBased": false, + "BucketBasedRootOK": false, + "CanHaveEmptyDirectories": true, + "CaseInsensitive": false, + "ChangeNotify": false, + "CleanUp": false, + "Command": true, + "Copy": false, + "DirCacheFlush": false, + "DirMove": true, + "Disconnect": false, + "DuplicateFiles": false, + "GetTier": false, + "IsLocal": true, + "ListR": false, + "MergeDirs": false, + "MetadataInfo": true, + "Move": true, + "OpenWriterAt": true, + "PublicLink": false, + "Purge": true, + "PutStream": true, + "PutUnchecked": false, + "ReadMetadata": true, + "ReadMimeType": false, + "ServerSideAcrossConfigs": false, + "SetTier": false, + "SetWrapper": false, + "Shutdown": false, + "SlowHash": true, + "SlowModTime": false, + "UnWrap": false, + "UserInfo": false, + "UserMetadata": true, + "WrapFs": false, + "WriteMetadata": true, + "WriteMimeType": false + }, + // Names of hashes available + "Hashes": [ + "md5", + "sha1", + "whirlpool", + "crc32", + "sha256", + "dropbox", + "mailru", + "quickxor" + ], + "Name": "local", // Name as created + "Precision": 1, // Precision of timestamps in ns + "Root": "/", // Path as created + "String": "Local file system at /", // how the remote will appear in logs + // Information about the system metadata for this backend + "MetadataInfo": { + "System": { + "atime": { + "Help": "Time of last access", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "btime": { + "Help": "Time of file birth (creation)", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "gid": { + "Help": "Group ID of owner", + "Type": "decimal number", + "Example": "500" + }, + "mode": { + "Help": "File type and mode", + "Type": "octal, unix style", + "Example": "0100664" + }, + "mtime": { + "Help": "Time of last modification", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "rdev": { + "Help": "Device ID (if special file)", + "Type": "hexadecimal", + "Example": "1abc" + }, + "uid": { + "Help": "User ID of owner", + "Type": "decimal number", + "Example": "500" + } + }, + "Help": "Textual help string\n" + } } This command does not have a command line equivalent so use this @@ -11683,6 +12424,7 @@ This takes the following parameters: - noMimeType - If set don't show mime types - dirsOnly - If set only show directories - filesOnly - If set only show files + - metadata - If set return metadata of objects also - hashTypes - array of strings of hash types to show if showHash set @@ -11702,7 +12444,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the mkdir command command for more information on the above. +See the mkdir command for more information on the above. Authentication is required for this call. @@ -11732,7 +12474,7 @@ Returns: - url - URL of the resource -See the link command command for more information on the above. +See the link command for more information on the above. Authentication is required for this call. @@ -11743,7 +12485,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the purge command command for more information on the above. +See the purge command for more information on the above. Authentication is required for this call. @@ -11754,7 +12496,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -See the rmdir command command for more information on the above. +See the rmdir command for more information on the above. Authentication is required for this call. @@ -11764,8 +12506,9 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -- leaveRoot - boolean, set to true not to delete the root See the - rmdirs command command for more information on the above. +- leaveRoot - boolean, set to true not to delete the root + +See the rmdirs command for more information on the above. Authentication is required for this call. @@ -11780,7 +12523,7 @@ Returns: - count - number of files - bytes - number of bytes in those files -See the size command command for more information on the above. +See the size command for more information on the above. Authentication is required for this call. @@ -11811,8 +12554,9 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" -- each part in body represents a file to be uploaded See the - uploadfile command command for more information on the above. +- each part in body represents a file to be uploaded + +See the uploadfile command for more information on the above. Authentication is required for this call. @@ -12035,7 +12779,7 @@ This takes the following parameters: - createEmptySrcDirs - create empty src directories on destination if set -See the copy command command for more information on the above. +See the copy command for more information on the above. Authentication is required for this call. @@ -12049,7 +12793,7 @@ This takes the following parameters: set - deleteEmptySrcDirs - delete empty src directories if set -See the move command command for more information on the above. +See the move command for more information on the above. Authentication is required for this call. @@ -12062,7 +12806,7 @@ This takes the following parameters: - createEmptySrcDirs - create empty src directories on destination if set -See the sync command command for more information on the above. +See the sync command for more information on the above. Authentication is required for this call. @@ -12380,47 +13124,49 @@ Features Here is an overview of the major features of each cloud storage system. - Name Hash ModTime Case Insensitive Duplicate Files MIME Type - ------------------------------ ------------- --------- ------------------ ----------------- ----------- - 1Fichier Whirlpool No No Yes R - Akamai Netstorage MD5, SHA256 Yes No No R - Amazon Drive MD5 No Yes No R - Amazon S3 (or S3 compatible) MD5 Yes No No R/W - Backblaze B2 SHA1 Yes No No R/W - Box SHA1 Yes Yes No - - Citrix ShareFile MD5 Yes Yes No - - Dropbox DBHASH ¹ Yes Yes No - - Enterprise File Fabric - Yes Yes No R/W - FTP - No No No - - Google Cloud Storage MD5 Yes No No R/W - Google Drive MD5 Yes No Yes R/W - Google Photos - No No Yes R - HDFS - Yes No No - - HTTP - No No No R - Hubic MD5 Yes No No R/W - Jottacloud MD5 Yes Yes No R - Koofr MD5 No Yes No - - Mail.ru Cloud Mailru ⁶ Yes Yes No - - Mega - No No Yes - - Memory MD5 Yes No No - - Microsoft Azure Blob Storage MD5 Yes No No R/W - Microsoft OneDrive SHA1 ⁵ Yes Yes No R - OpenDrive MD5 Yes Yes Partial ⁸ - - OpenStack Swift MD5 Yes No No R/W - pCloud MD5, SHA1 ⁷ Yes No No W - premiumize.me - No Yes No R - put.io CRC-32 Yes No Yes R - QingStor MD5 No No No R/W - Seafile - No No No - - SFTP MD5, SHA1 ² Yes Depends No - - Sia - No No No - - SugarSync - No No No - - Storj - Yes No No - - Uptobox - No No Yes - - WebDAV MD5, SHA1 ³ Yes ⁴ Depends No - - Yandex Disk MD5 Yes No No R - Zoho WorkDrive - No No No - - The local filesystem All Yes Depends No - + Name Hash ModTime Case Insensitive Duplicate Files MIME Type Metadata + ------------------------------ ------------------ --------- ------------------ ----------------- ----------- ---------- + 1Fichier Whirlpool - No Yes R - + Akamai Netstorage MD5, SHA256 R/W No No R - + Amazon Drive MD5 - Yes No R - + Amazon S3 (or S3 compatible) MD5 R/W No No R/W RWU + Backblaze B2 SHA1 R/W No No R/W - + Box SHA1 R/W Yes No - - + Citrix ShareFile MD5 R/W Yes No - - + Dropbox DBHASH ¹ R Yes No - - + Enterprise File Fabric - R/W Yes No R/W - + FTP - R/W ¹⁰ No No - - + Google Cloud Storage MD5 R/W No No R/W - + Google Drive MD5 R/W No Yes R/W - + Google Photos - - No Yes R - + HDFS - R/W No No - - + HiDrive HiDrive ¹² R/W No No - - + HTTP - R No No R - + Hubic MD5 R/W No No R/W - + Internet Archive MD5, SHA1, CRC32 R/W ¹¹ No No - RWU + Jottacloud MD5 R/W Yes No R - + Koofr MD5 - Yes No - - + Mail.ru Cloud Mailru ⁶ R/W Yes No - - + Mega - - No Yes - - + Memory MD5 R/W No No - - + Microsoft Azure Blob Storage MD5 R/W No No R/W - + Microsoft OneDrive SHA1 ⁵ R/W Yes No R - + OpenDrive MD5 R/W Yes Partial ⁸ - - + OpenStack Swift MD5 R/W No No R/W - + pCloud MD5, SHA1 ⁷ R No No W - + premiumize.me - - Yes No R - + put.io CRC-32 R/W No Yes R - + QingStor MD5 - ⁹ No No R/W - + Seafile - - No No - - + SFTP MD5, SHA1 ² R/W Depends No - - + Sia - - No No - - + SugarSync - - No No - - + Storj - R No No - - + Uptobox - - No Yes - - + WebDAV MD5, SHA1 ³ R ⁴ Depends No - - + Yandex Disk MD5 R/W No No R - + Zoho WorkDrive - - No No - - + The local filesystem All R/W Depends No - RWU Notes @@ -12447,6 +13193,17 @@ platform has been determined to allow duplicate files, and it is possible to create them with rclone. It may be that this is a mistake or an unsupported feature. +⁹ QingStor does not support SetModTime for objects bigger than 5 GiB. + +¹⁰ FTP supports modtimes for the major FTP servers, and also others if +they advertised required protocol extensions. See this for more details. + +¹¹ Internet Archive requires option wait_archive to be set to a non-zero +value for full modtime support. + +¹² HiDrive supports its own custom hash. It combines SHA1 sums for each +4 KiB block hierarchically to a single top-level sum. + Hash The cloud storage system supports various hash types of the objects. The @@ -12459,13 +13216,34 @@ systems they must support a common hash type. ModTime -The cloud storage system supports setting modification times on objects. -If it does then this enables a using the modification times as part of -the sync. If not then only the size will be checked by default, though -the MD5SUM can be checked with the --checksum flag. - -All cloud storage systems support some kind of date on the object and -these will be set when transferring from the cloud storage system. +Allmost all cloud storage systems store some sort of timestamp on +objects, but several of them not something that is appropriate to use +for syncing. E.g. some backends will only write a timestamp that +represent the time of the upload. To be relevant for syncing it should +be able to store the modification time of the source object. If this is +not the case, rclone will only check the file size by default, though +can be configured to check the file hash (with the --checksum flag). +Ideally it should also be possible to change the timestamp of an +existing file without having to re-upload it. + +Storage systems with a - in the ModTime column, means the modification +read on objects is not the modification time of the file when uploaded. +It is most likely the time the file was uploaded, or possibly something +else (like the time the picture was taken in Google Photos). + +Storage systems with a R (for read-only) in the ModTime column, means +the it keeps modification times on objects, and updates them when +uploading objects, but it does not support changing only the +modification time (SetModTime operation) without re-uploading, possibly +not even without deleting existing first. Some operations in rclone, +such as copy and sync commands, will automatically check for SetModTime +support and re-upload if necessary to keep the modification times in +sync. Other commands will not work without SetModTime support, e.g. +touch command on an existing file will fail, and changes to modification +time only on a files in a mount will be silently ignored. + +Storage systems with R/W (for read/write) in the ModTime column, means +they do also support modtime-only operations. Case Insensitive @@ -12665,35 +13443,77 @@ of all possible values by passing an invalid value to this flag, e.g. --local-encoding "help". The command rclone help flags encoding will show you the defaults for the backends. - Encoding Characters - --------------- ------------------------------------------------------------- - Asterisk * - BackQuote ` - BackSlash \ - Colon : - CrLf CR 0x0D, LF 0x0A - Ctl All control characters 0x00-0x1F - Del DEL 0x7F - Dollar $ - Dot . or .. as entire string - DoubleQuote " - Hash # - InvalidUtf8 An invalid UTF-8 character (e.g. latin1) - LeftCrLfHtVt CR 0x0D, LF 0x0A,HT 0x09, VT 0x0B on the left of a string - LeftPeriod . on the left of a string - LeftSpace SPACE on the left of a string - LeftTilde ~ on the left of a string - LtGt <, > - None No characters are encoded - Percent % - Pipe | - Question ? - RightCrLfHtVt CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string - RightPeriod . on the right of a string - RightSpace SPACE on the right of a string - SingleQuote ' - Slash / - SquareBracket [, ] + ---------------------------------------------------------------------------------- + Encoding Characters Encoded as + ---------------------- ------------------------ ---------------------------------- + Asterisk * ＊ + + BackQuote ` ｀ + + BackSlash \ ＼ + + Colon : ： + + CrLf CR 0x0D, LF 0x0A ␍, ␊ + + Ctl All control characters ␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟ + 0x00-0x1F + + Del DEL 0x7F ␡ + + Dollar $ ＄ + + Dot . or .. as entire string ．, ．． + + DoubleQuote " ＂ + + Hash # ＃ + + InvalidUtf8 An invalid UTF-8 � + character (e.g. latin1) + + LeftCrLfHtVt CR 0x0D, LF 0x0A, HT ␍, ␊, ␉, ␋ + 0x09, VT 0x0B on the + left of a string + + LeftPeriod . on the left of a . + string + + LeftSpace SPACE on the left of a ␠ + string + + LeftTilde ~ on the left of a ～ + string + + LtGt <, > ＜, ＞ + + None No characters are + encoded + + Percent % ％ + + Pipe | ｜ + + Question ? ？ + + RightCrLfHtVt CR 0x0D, LF 0x0A, HT ␍, ␊, ␉, ␋ + 0x09, VT 0x0B on the + right of a string + + RightPeriod . on the right of a . + string + + RightSpace SPACE on the right of a ␠ + string + + Semicolon ; ； + + SingleQuote ' ＇ + + Slash / ／ + + SquareBracket [, ] ［, ］ + ---------------------------------------------------------------------------------- Encoding example: FTP @@ -12764,6 +13584,22 @@ which supports writing (W) then rclone will preserve the MIME types. Otherwise they will be guessed from the extension, or the remote itself may assign the MIME type. +Metadata + +Backends may or may support reading or writing metadata. They may +support reading and writing system metadata (metadata intrinsic to that +backend) and/or user metadata (general purpose metadata). + +The levels of metadata support are + + Key Explanation + ----- ----------------------------------------------------------------- + R Read only System Metadata + RW Read and write System Metadata + RWU Read and write System Metadata and read and write User Metadata + +See the metadata docs for more info. + Optional Features All rclone remotes support a base command set. Other features depend @@ -12772,8 +13608,9 @@ upon backend-specific capabilities. Name Purge Copy Move DirMove CleanUp ListR StreamUpload LinkSharing About EmptyDir ------------------------------ ------- ------ ------ --------- --------- ------- -------------- ------------- ------- ---------- 1Fichier No Yes Yes No No No No Yes No Yes + Akamai Netstorage Yes No No No No Yes Yes No No Yes Amazon Drive Yes No Yes Yes No No No No No Yes - Amazon S3 No Yes No No Yes Yes Yes Yes No No + Amazon S3 (or S3 compatible) No Yes No No Yes Yes Yes Yes No No Backblaze B2 No Yes No No Yes Yes Yes Yes No No Box Yes Yes Yes Yes Yes ‡‡ No Yes Yes Yes Yes Citrix ShareFile Yes Yes Yes Yes No No Yes No No Yes @@ -12784,9 +13621,12 @@ upon backend-specific capabilities. Google Drive Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes Google Photos No No No No No No No No No No HDFS Yes No Yes Yes No No Yes No Yes Yes + HiDrive Yes Yes Yes Yes No No Yes No No Yes HTTP No No No No No No No No No Yes Hubic Yes † Yes No No No Yes Yes No Yes No + Internet Archive No Yes No No Yes Yes No Yes Yes No Jottacloud Yes Yes Yes Yes Yes Yes No Yes Yes Yes + Koofr Yes Yes Yes Yes No No Yes Yes Yes Yes Mail.ru Cloud Yes Yes Yes Yes Yes No No Yes Yes Yes Mega Yes No Yes Yes Yes No No Yes Yes Yes Memory No Yes No No No Yes Yes No No No @@ -12800,6 +13640,7 @@ upon backend-specific capabilities. QingStor No Yes No No Yes Yes No No No No Seafile Yes Yes Yes Yes Yes Yes Yes Yes Yes Yes SFTP No No Yes Yes No No Yes No Yes Yes + Sia No No No No No No Yes No No Yes SugarSync Yes Yes Yes Yes No No Yes Yes No Yes Storj Yes † No Yes No No Yes Yes No No No Uptobox No Yes Yes Yes No No No No No No @@ -12924,6 +13765,7 @@ These flags are available for every command. --delete-during When synchronizing, delete files during transfer --delete-excluded Delete files on dest excluded from sync --disable string Disable a comma separated list of features (use --disable help to see a list) + --disable-http-keep-alives Disable HTTP keep-alives and use each connection once. --disable-http2 Disable HTTP/2 in the global transport -n, --dry-run Do a trial run with no permanent changes --dscp string Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21 @@ -12933,7 +13775,7 @@ These flags are available for every command. --error-on-no-transfer Sets exit code 9 if no files are transferred, useful in scripts --exclude stringArray Exclude files matching pattern --exclude-from stringArray Read exclude patterns from file (use - to read from stdin) - --exclude-if-present string Exclude directories if filename is present + --exclude-if-present stringArray Exclude directories if filename is present --expect-continue-timeout duration Timeout when using expect / 100-continue in HTTP (default 1s) --fast-list Use recursive list if available; uses more memory but fewer transactions --files-from stringArray Read list of source-file names from file (use - to read from stdin) @@ -12972,6 +13814,8 @@ These flags are available for every command. --max-stats-groups int Maximum number of stats groups to keep in memory, on max oldest is discarded (default 1000) --max-transfer SizeSuffix Maximum size of data to transfer (default off) --memprofile string Write memory profile to file + -M, --metadata If set, preserve metadata when copying objects + --metadata-set stringArray Add metadata key=value when uploading --min-age Duration Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off) --min-size SizeSuffix Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off) --modify-window duration Max time diff to be considered the same (default 1ns) @@ -13043,7 +13887,7 @@ These flags are available for every command. --use-json-log Use json log format --use-mmap Use mmap allocator (see docs) --use-server-modtime Use server modified time instead of object metadata - --user-agent string Set the user-agent to a specified string (default "rclone/v1.58.0") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.59.0") -v, --verbose count Print lots more stuff (repeat for more) Backend Flags @@ -13096,6 +13940,7 @@ and may be set in the config file. --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --b2-version-at Time Show file versions as they were at the specified time (default off) --b2-versions Include old versions in directory listings --box-access-token string Box App Primary Access Token --box-auth-url string Auth server URL @@ -13135,6 +13980,7 @@ and may be set in the config file. --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks --chunker-hash-type string Choose how chunker handles hash sums (default "md5") --chunker-remote string Remote to chunk/unchunk + --combine-upstreams SpaceSepList Upstreams for combining --compress-level int GZIP compression level (-2 to 9) (default -1) --compress-mode string Compression mode (default "gzip") --compress-ram-cache-limit SizeSuffix Some remotes don't allow the upload of files with unknown size (default 20Mi) @@ -13167,6 +14013,7 @@ and may be set in the config file. --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) + --drive-resource-key string Resource key for accessing a link-shared file --drive-root-folder-id string ID of the root folder --drive-scope string Scope that rclone should use when requesting access from drive --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs @@ -13222,6 +14069,7 @@ and may be set in the config file. --ftp-disable-epsv Disable using EPSV even if server advertises support --ftp-disable-mlsd Disable using MLSD even if server advertises support --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) + --ftp-disable-utf8 Disable using UTF-8 even if server advertises support --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) --ftp-host string FTP host to connect to @@ -13240,8 +14088,10 @@ and may be set in the config file. --gcs-bucket-policy-only Access checks should use bucket-level IAM policies --gcs-client-id string OAuth Client Id --gcs-client-secret string OAuth Client Secret + --gcs-decompress If set this will decompress gzip encoded objects --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) --gcs-location string Location for the newly created buckets + --gcs-no-check-bucket If set, don't attempt to check the bucket exists or create it --gcs-object-acl string Access Control List for new objects --gcs-project-number string Project number --gcs-service-account-file string Service Account Credentials JSON file path @@ -13267,10 +14117,24 @@ and may be set in the config file. --hdfs-namenode string Hadoop name node and port --hdfs-service-principal-name string Kerberos service principal name for the namenode --hdfs-username string Hadoop user name + --hidrive-auth-url string Auth server URL + --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) + --hidrive-client-id string OAuth Client Id + --hidrive-client-secret string OAuth Client Secret + --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary + --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --hidrive-endpoint string Endpoint for the service (default "https://api.hidrive.strato.com/2.1") + --hidrive-root-prefix string The root/parent folder for all paths (default "/") + --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default "rw") + --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default "user") + --hidrive-token string OAuth Access Token as a JSON blob + --hidrive-token-url string Token server url + --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) + --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) --http-headers CommaSepList Set HTTP headers for all transactions --http-no-head Don't use HEAD requests --http-no-slash Set this if the site doesn't end directories with / - --http-url string URL of http host to connect to + --http-url string URL of HTTP host to connect to --hubic-auth-url string Auth server URL --hubic-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) --hubic-client-id string OAuth Client Id @@ -13279,6 +14143,13 @@ and may be set in the config file. --hubic-no-chunk Don't chunk files during streaming upload --hubic-token string OAuth Access Token as a JSON blob --hubic-token-url string Token server url + --internetarchive-access-key-id string IAS3 Access Key + --internetarchive-disable-checksum Don't ask the server to test against MD5 checksum calculated by rclone (default true) + --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) + --internetarchive-endpoint string IAS3 Endpoint (default "https://s3.us.archive.org") + --internetarchive-front-endpoint string Host of InternetArchive Frontend (default "https://archive.org") + --internetarchive-secret-access-key string IAS3 Secret Key (password) + --internetarchive-wait-archive Duration Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s) --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) --jottacloud-hard-delete Delete files permanently rather than putting them into the trash --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) @@ -13300,7 +14171,7 @@ and may be set in the config file. --local-no-preallocate Disable preallocation of disk space for transferred files --local-no-set-modtime Disable setting modtime --local-no-sparse Disable sparse files for multi-thread downloads - --local-nounc string Disable UNC (long path names) conversion on Windows + --local-nounc Disable UNC (long path names) conversion on Windows --local-unicode-normalization Apply unicode NFC normalization to paths and filenames --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) @@ -13321,11 +14192,11 @@ and may be set in the config file. --netstorage-protocol string Select between HTTP or HTTPS protocol (default "https") --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) -x, --one-file-system Don't cross filesystem boundaries (unix/macOS only) + --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) --onedrive-auth-url string Auth server URL --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) --onedrive-client-id string OAuth Client Id --onedrive-client-secret string OAuth Client Secret - --onedrive-disable-site-permission Disable the request for Sites.Read.All permission --onedrive-drive-id string The ID of the drive to use --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) @@ -13349,9 +14220,11 @@ and may be set in the config file. --pcloud-client-secret string OAuth Client Secret --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) --pcloud-hostname string Hostname to connect to (default "api.pcloud.com") + --pcloud-password string Your pcloud password (obscured) --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default "d0") --pcloud-token string OAuth Access Token as a JSON blob --pcloud-token-url string Token server url + --pcloud-username string Your pcloud username --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) --qingstor-access-key-id string QingStor Access Key ID @@ -13404,6 +14277,7 @@ and may be set in the config file. --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) + --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads --s3-v2-auth If true use v2 authentication --seafile-2fa Two-factor authentication ('true' if the account has 2FA enabled) --seafile-create-library Should rclone create a library if it doesn't exist @@ -13414,6 +14288,8 @@ and may be set in the config file. --seafile-url string URL of seafile host to connect to --seafile-user string User name (usually email address) --sftp-ask-password Allow asking for SFTP password when needed + --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) + --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) --sftp-disable-concurrent-reads If set don't use concurrent reads --sftp-disable-concurrent-writes If set don't use concurrent writes --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available @@ -13426,12 +14302,14 @@ and may be set in the config file. --sftp-known-hosts-file string Optional path to known_hosts file --sftp-md5sum-command string The command used to read md5 hashes --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) - --sftp-path-override string Override path used by SSH connection + --sftp-path-override string Override path used by SSH shell commands --sftp-port int SSH port number (default 22) --sftp-pubkey-file string Optional path to public key file --sftp-server-command string Specifies the path or command to run a sftp server on the remote host + --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands --sftp-set-modtime Set the modified time on the remote if set (default true) --sftp-sha1sum-command string The command used to read sha1 hashes + --sftp-shell-type string The type of SSH shell on remote server, if any --sftp-skip-links Set to skip any symlinks and any other non regular files --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default "sftp") --sftp-use-fstat If set use fstat instead of stat @@ -13488,6 +14366,7 @@ and may be set in the config file. --union-action-policy string Policy to choose upstream on ACTION category (default "epall") --union-cache-time int Cache time of usage and free space (in seconds) (default 120) --union-create-policy string Policy to choose upstream on CREATE category (default "epmfs") + --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) --union-search-policy string Policy to choose upstream on SEARCH category (default "ff") --union-upstreams string List of space separated upstreams --uptobox-access-token string Your access token @@ -13499,7 +14378,7 @@ and may be set in the config file. --webdav-pass string Password (obscured) --webdav-url string URL of http host to connect to --webdav-user string User name - --webdav-vendor string Name of the Webdav site/service/software you are using + --webdav-vendor string Name of the WebDAV site/service/software you are using --yandex-auth-url string Auth server URL --yandex-client-id string OAuth Client Id --yandex-client-secret string OAuth Client Secret @@ -14116,7 +14995,7 @@ Command line syntax Arbitrary rclone flags may be specified on the bisync command line, for example -rclone bsync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s +rclone bisync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s Note that interactions of various rclone flags with bisync process flow has not been fully tested yet. @@ -14406,7 +15285,7 @@ Supported backends Bisync is considered BETA and has been tested with the following backends: - Local filesystem - Google Drive - Dropbox - OneDrive - S3 - -SFTP +SFTP - Yandex Disk It has not been fully tested with other services yet. If it works, or sorta works, please let us know and we'll update the list. Run the test @@ -14741,7 +15620,7 @@ Google Doc files Google docs exist as virtual files on Google Drive and cannot be transferred to other filesystems natively. While it is possible to export a Google doc to a normal file (with .xlsx extension, for -example), it's not possible to import a normal file back into a Google +example), it is not possible to import a normal file back into a Google document. Bisync's handling of Google Doc files is to flag them in the run log @@ -15232,7 +16111,7 @@ strings. Standard options -Here are the standard options specific to fichier (1Fichier). +Here are the Standard options specific to fichier (1Fichier). --fichier-api-key @@ -15247,7 +16126,7 @@ Properties: Advanced options -Here are the advanced options specific to fichier (1Fichier). +Here are the Advanced options specific to fichier (1Fichier). --fichier-shared-folder @@ -15308,7 +16187,7 @@ rclone about is not supported by the 1Fichier backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Alias @@ -15395,7 +16274,7 @@ Copy another local directory to the alias directory called source Standard options -Here are the standard options specific to alias (Alias for an existing +Here are the Standard options specific to alias (Alias for an existing remote). --alias-remote @@ -15562,7 +16441,7 @@ amazon.co.uk email and password should work here just fine. Standard options -Here are the standard options specific to amazon cloud drive (Amazon +Here are the Standard options specific to amazon cloud drive (Amazon Drive). --acd-client-id @@ -15593,7 +16472,7 @@ Properties: Advanced options -Here are the advanced options specific to amazon cloud drive (Amazon +Here are the Advanced options specific to amazon cloud drive (Amazon Drive). --acd-token @@ -15737,7 +16616,7 @@ without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Amazon S3 Storage Providers @@ -15746,9 +16625,14 @@ The S3 backend can be used with a number of different providers: - AWS S3 - Alibaba Cloud (Aliyun) Object Storage System (OSS) - Ceph +- China Mobile Ecloud Elastic Object Storage (EOS) +- Cloudflare R2 +- Arvan Cloud Object Storage (AOS) - DigitalOcean Spaces - Dreamhost +- Huawei OBS - IBM COS S3 +- IDrive e2 - Minio - RackCorp Object Storage - Scaleway @@ -15803,7 +16687,7 @@ This will guide you through an interactive setup process. Type of storage to configure. Choose a number from below, or type in your own value [snip] - XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, Dreamhost, IBM COS, Minio, and Tencent COS + XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -16303,10 +17187,11 @@ be uploaded as multipart. Standard options -Here are the standard options specific to s3 (Amazon S3 Compliant -Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, -Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent -COS). +Here are the Standard options specific to s3 (Amazon S3 Compliant +Storage Providers including AWS, Alibaba, Ceph, China Mobile, +Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, +IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, +StackPath, Storj, Tencent COS and Wasabi). --s3-provider @@ -16325,12 +17210,22 @@ Properties: - Alibaba Cloud Object Storage System (OSS) formerly Aliyun - "Ceph" - Ceph Object Storage + - "ChinaMobile" + - China Mobile Ecloud Elastic Object Storage (EOS) + - "Cloudflare" + - Cloudflare R2 Storage + - "ArvanCloud" + - Arvan Cloud Object Storage (AOS) - "DigitalOcean" - Digital Ocean Spaces - "Dreamhost" - Dreamhost DreamObjects + - "HuaweiOBS" + - Huawei Object Storage Service - "IBMCOS" - IBM COS S3 + - "IDrive" + - IDrive e2 - "LyveCloud" - Seagate Lyve Cloud - "Minio" @@ -16556,6 +17451,68 @@ Properties: - Amsterdam, The Netherlands - "fr-par" - Paris, France + - "pl-waw" + - Warsaw, Poland + +--s3-region + +Region to connect to. - the location where your bucket will be created +and your data stored. Need bo be same with your endpoint. + +Properties: + +- Config: region +- Env Var: RCLONE_S3_REGION +- Provider: HuaweiOBS +- Type: string +- Required: false +- Examples: + - "af-south-1" + - AF-Johannesburg + - "ap-southeast-2" + - AP-Bangkok + - "ap-southeast-3" + - AP-Singapore + - "cn-east-3" + - CN East-Shanghai1 + - "cn-east-2" + - CN East-Shanghai2 + - "cn-north-1" + - CN North-Beijing1 + - "cn-north-4" + - CN North-Beijing4 + - "cn-south-1" + - CN South-Guangzhou + - "ap-southeast-1" + - CN-Hong Kong + - "sa-argentina-1" + - LA-Buenos Aires1 + - "sa-peru-1" + - LA-Lima1 + - "na-mexico-1" + - LA-Mexico City1 + - "sa-chile-1" + - LA-Santiago2 + - "sa-brazil-1" + - LA-Sao Paulo1 + - "ru-northwest-2" + - RU-Moscow2 + +--s3-region + +Region to connect to. + +Properties: + +- Config: region +- Env Var: RCLONE_S3_REGION +- Provider: Cloudflare +- Type: string +- Required: false +- Examples: + - "auto" + - R2 buckets are automatically distributed across Cloudflare's + data centers for low latency. --s3-region @@ -16567,7 +17524,8 @@ Properties: - Config: region - Env Var: RCLONE_S3_REGION -- Provider: !AWS,Alibaba,RackCorp,Scaleway,Storj,TencentCOS +- Provider: + !AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive - Type: string - Required: false - Examples: @@ -16594,6 +17552,98 @@ Properties: --s3-endpoint +Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Provider: ChinaMobile +- Type: string +- Required: false +- Examples: + - "eos-wuxi-1.cmecloud.cn" + - The default endpoint - a good choice if you are unsure. + - East China (Suzhou) + - "eos-jinan-1.cmecloud.cn" + - East China (Jinan) + - "eos-ningbo-1.cmecloud.cn" + - East China (Hangzhou) + - "eos-shanghai-1.cmecloud.cn" + - East China (Shanghai-1) + - "eos-zhengzhou-1.cmecloud.cn" + - Central China (Zhengzhou) + - "eos-hunan-1.cmecloud.cn" + - Central China (Changsha-1) + - "eos-zhuzhou-1.cmecloud.cn" + - Central China (Changsha-2) + - "eos-guangzhou-1.cmecloud.cn" + - South China (Guangzhou-2) + - "eos-dongguan-1.cmecloud.cn" + - South China (Guangzhou-3) + - "eos-beijing-1.cmecloud.cn" + - North China (Beijing-1) + - "eos-beijing-2.cmecloud.cn" + - North China (Beijing-2) + - "eos-beijing-4.cmecloud.cn" + - North China (Beijing-3) + - "eos-huhehaote-1.cmecloud.cn" + - North China (Huhehaote) + - "eos-chengdu-1.cmecloud.cn" + - Southwest China (Chengdu) + - "eos-chongqing-1.cmecloud.cn" + - Southwest China (Chongqing) + - "eos-guiyang-1.cmecloud.cn" + - Southwest China (Guiyang) + - "eos-xian-1.cmecloud.cn" + - Nouthwest China (Xian) + - "eos-yunnan.cmecloud.cn" + - Yunnan China (Kunming) + - "eos-yunnan-2.cmecloud.cn" + - Yunnan China (Kunming-2) + - "eos-tianjin-1.cmecloud.cn" + - Tianjin China (Tianjin) + - "eos-jilin-1.cmecloud.cn" + - Jilin China (Changchun) + - "eos-hubei-1.cmecloud.cn" + - Hubei China (Xiangyan) + - "eos-jiangxi-1.cmecloud.cn" + - Jiangxi China (Nanchang) + - "eos-gansu-1.cmecloud.cn" + - Gansu China (Lanzhou) + - "eos-shanxi-1.cmecloud.cn" + - Shanxi China (Taiyuan) + - "eos-liaoning-1.cmecloud.cn" + - Liaoning China (Shenyang) + - "eos-hebei-1.cmecloud.cn" + - Hebei China (Shijiazhuang) + - "eos-fujian-1.cmecloud.cn" + - Fujian China (Xiamen) + - "eos-guangxi-1.cmecloud.cn" + - Guangxi China (Nanning) + - "eos-anhui-1.cmecloud.cn" + - Anhui China (Huainan) + +--s3-endpoint + +Endpoint for Arvan Cloud Object Storage (AOS) API. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Provider: ArvanCloud +- Type: string +- Required: false +- Examples: + - "s3.ir-thr-at1.arvanstorage.com" + - The default endpoint - a good choice if you are unsure. + - Tehran Iran (Asiatech) + - "s3.ir-tbz-sh1.arvanstorage.com" + - Tabriz Iran (Shahriar) + +--s3-endpoint + Endpoint for IBM COS S3 API. Specify if using an IBM COS On Premise. @@ -16796,6 +17846,49 @@ Properties: --s3-endpoint +Endpoint for OBS API. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Provider: HuaweiOBS +- Type: string +- Required: false +- Examples: + - "obs.af-south-1.myhuaweicloud.com" + - AF-Johannesburg + - "obs.ap-southeast-2.myhuaweicloud.com" + - AP-Bangkok + - "obs.ap-southeast-3.myhuaweicloud.com" + - AP-Singapore + - "obs.cn-east-3.myhuaweicloud.com" + - CN East-Shanghai1 + - "obs.cn-east-2.myhuaweicloud.com" + - CN East-Shanghai2 + - "obs.cn-north-1.myhuaweicloud.com" + - CN North-Beijing1 + - "obs.cn-north-4.myhuaweicloud.com" + - CN North-Beijing4 + - "obs.cn-south-1.myhuaweicloud.com" + - CN South-Guangzhou + - "obs.ap-southeast-1.myhuaweicloud.com" + - CN-Hong Kong + - "obs.sa-argentina-1.myhuaweicloud.com" + - LA-Buenos Aires1 + - "obs.sa-peru-1.myhuaweicloud.com" + - LA-Lima1 + - "obs.na-mexico-1.myhuaweicloud.com" + - LA-Mexico City1 + - "obs.sa-chile-1.myhuaweicloud.com" + - LA-Santiago2 + - "obs.sa-brazil-1.myhuaweicloud.com" + - LA-Sao Paulo1 + - "obs.ru-northwest-2.myhuaweicloud.com" + - RU-Moscow2 + +--s3-endpoint + Endpoint for Scaleway Object Storage. Properties: @@ -16810,6 +17903,8 @@ Properties: - Amsterdam Endpoint - "s3.fr-par.scw.cloud" - Paris Endpoint + - "s3.pl-waw.scw.cloud" + - Warsaw Endpoint --s3-endpoint @@ -16962,7 +18057,7 @@ Properties: - Config: endpoint - Env Var: RCLONE_S3_ENDPOINT - Provider: - !AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,Storj,RackCorp + !AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp - Type: string - Required: false - Examples: @@ -16992,6 +18087,8 @@ Properties: - Wasabi AP Northeast 1 (Tokyo) endpoint - "s3.ap-northeast-2.wasabisys.com" - Wasabi AP Northeast 2 (Osaka) endpoint + - "s3.ir-thr-at1.arvanstorage.com" + - ArvanCloud Tehran Iran (Asiatech) endpoint --s3-location-constraint @@ -17060,6 +18157,100 @@ Properties: --s3-location-constraint +Location constraint - must match endpoint. + +Used when creating buckets only. + +Properties: + +- Config: location_constraint +- Env Var: RCLONE_S3_LOCATION_CONSTRAINT +- Provider: ChinaMobile +- Type: string +- Required: false +- Examples: + - "wuxi1" + - East China (Suzhou) + - "jinan1" + - East China (Jinan) + - "ningbo1" + - East China (Hangzhou) + - "shanghai1" + - East China (Shanghai-1) + - "zhengzhou1" + - Central China (Zhengzhou) + - "hunan1" + - Central China (Changsha-1) + - "zhuzhou1" + - Central China (Changsha-2) + - "guangzhou1" + - South China (Guangzhou-2) + - "dongguan1" + - South China (Guangzhou-3) + - "beijing1" + - North China (Beijing-1) + - "beijing2" + - North China (Beijing-2) + - "beijing4" + - North China (Beijing-3) + - "huhehaote1" + - North China (Huhehaote) + - "chengdu1" + - Southwest China (Chengdu) + - "chongqing1" + - Southwest China (Chongqing) + - "guiyang1" + - Southwest China (Guiyang) + - "xian1" + - Nouthwest China (Xian) + - "yunnan" + - Yunnan China (Kunming) + - "yunnan2" + - Yunnan China (Kunming-2) + - "tianjin1" + - Tianjin China (Tianjin) + - "jilin1" + - Jilin China (Changchun) + - "hubei1" + - Hubei China (Xiangyan) + - "jiangxi1" + - Jiangxi China (Nanchang) + - "gansu1" + - Gansu China (Lanzhou) + - "shanxi1" + - Shanxi China (Taiyuan) + - "liaoning1" + - Liaoning China (Shenyang) + - "hebei1" + - Hebei China (Shijiazhuang) + - "fujian1" + - Fujian China (Xiamen) + - "guangxi1" + - Guangxi China (Nanning) + - "anhui1" + - Anhui China (Huainan) + +--s3-location-constraint + +Location constraint - must match endpoint. + +Used when creating buckets only. + +Properties: + +- Config: location_constraint +- Env Var: RCLONE_S3_LOCATION_CONSTRAINT +- Provider: ArvanCloud +- Type: string +- Required: false +- Examples: + - "ir-thr-at1" + - Tehran Iran (Asiatech) + - "ir-tbz-sh1" + - Tabriz Iran (Shahriar) + +--s3-location-constraint + Location constraint - must match endpoint when using IBM Cloud Public. For on-prem COS, do not make a selection from this list, hit enter. @@ -17200,7 +18391,7 @@ Properties: - Config: location_constraint - Env Var: RCLONE_S3_LOCATION_CONSTRAINT - Provider: - !AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,Storj,TencentCOS + !AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS - Type: string - Required: false @@ -17221,7 +18412,7 @@ Properties: - Config: acl - Env Var: RCLONE_S3_ACL -- Provider: !Storj +- Provider: !Storj,Cloudflare - Type: string - Required: false - Examples: @@ -17282,7 +18473,7 @@ Properties: - Config: server_side_encryption - Env Var: RCLONE_S3_SERVER_SIDE_ENCRYPTION -- Provider: AWS,Ceph,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -17364,6 +18555,42 @@ Properties: --s3-storage-class +The storage class to use when storing new objects in ChinaMobile. + +Properties: + +- Config: storage_class +- Env Var: RCLONE_S3_STORAGE_CLASS +- Provider: ChinaMobile +- Type: string +- Required: false +- Examples: + - "" + - Default + - "STANDARD" + - Standard storage class + - "GLACIER" + - Archive storage mode + - "STANDARD_IA" + - Infrequent access storage mode + +--s3-storage-class + +The storage class to use when storing new objects in ArvanCloud. + +Properties: + +- Config: storage_class +- Env Var: RCLONE_S3_STORAGE_CLASS +- Provider: ArvanCloud +- Type: string +- Required: false +- Examples: + - "STANDARD" + - Standard storage class + +--s3-storage-class + The storage class to use when storing new objects in Tencent COS. Properties: @@ -17407,10 +18634,11 @@ Properties: Advanced options -Here are the advanced options specific to s3 (Amazon S3 Compliant -Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, -Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent -COS). +Here are the Advanced options specific to s3 (Amazon S3 Compliant +Storage Providers including AWS, Alibaba, Ceph, China Mobile, +Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, +IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, +StackPath, Storj, Tencent COS and Wasabi). --s3-bucket-acl @@ -17464,7 +18692,7 @@ Properties: - Config: sse_customer_algorithm - Env Var: RCLONE_S3_SSE_CUSTOMER_ALGORITHM -- Provider: AWS,Ceph,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -17482,7 +18710,7 @@ Properties: - Config: sse_customer_key - Env Var: RCLONE_S3_SSE_CUSTOMER_KEY -- Provider: AWS,Ceph,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -17501,7 +18729,7 @@ Properties: - Config: sse_customer_key_md5 - Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_MD5 -- Provider: AWS,Ceph,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -17546,6 +18774,12 @@ this means that by default the maximum size of a file you can stream upload is 48 GiB. If you wish to stream upload larger files then you will need to increase chunk_size. +Increasing the chunk size decreases the accuracy of the progress +statistics displayed with "-P" flag. Rclone treats chunk as sent when +it's buffered by the AWS SDK, when in fact it may still be uploading. A +bigger chunk size means a bigger AWS SDK buffer and progress reporting +more deviating from the truth. + Properties: - Config: chunk_size @@ -17946,6 +19180,61 @@ Properties: - Type: Tristate - Default: unset +--s3-use-presigned-request + +Whether to use a presigned request or PutObject for single part uploads + +If this is false rclone will use PutObject from the AWS SDK to upload an +object. + +Versions of rclone < 1.59 use presigned requests to upload a single part +object and setting this flag to true will re-enable that functionality. +This shouldn't be necessary except in exceptional circumstances or for +testing. + +Properties: + +- Config: use_presigned_request +- Env Var: RCLONE_S3_USE_PRESIGNED_REQUEST +- Type: bool +- Default: false + +Metadata + +User metadata is stored as x-amz-meta- keys. S3 metadata keys are case +insensitive and are always returned in lower case. + +Here are the possible system metadata items for the s3 backend. + + ------------------------------------------------------------------------------------------------------------------ + Name Help Type Example Read Only + --------------------- --------------------- ----------- ------------------------------------- -------------------- + btime Time of file birth RFC 3339 2006-01-02T15:04:05.999999999Z07:00 Y + (creation) read from + Last-Modified header + + cache-control Cache-Control header string no-cache N + + content-disposition Content-Disposition string inline N + header + + content-encoding Content-Encoding string gzip N + header + + content-language Content-Language string en-US N + header + + content-type Content-Type header string text/plain N + + mtime Time of last RFC 3339 2006-01-02T15:04:05.999999999Z07:00 N + modification, read + from rclone metadata + + tier Tier of the object string GLACIER Y + ------------------------------------------------------------------------------------------------------------------ + +See the metadata docs for more info. + Backend commands Here are the commands specific to the s3 backend. @@ -17956,8 +19245,8 @@ Run them with The help below will explain what arguments each command takes. -See the "rclone backend" command for more info on how to pass options -and arguments. +See the backend command for more info on how to pass options and +arguments. These can be run on a running backend using the rc command backend/command. @@ -18102,9 +19391,14 @@ AWS Snowball is a hardware appliance used for transferring bulk data back to AWS. Its main software interface is S3 object storage. To use rclone with AWS Snowball Edge devices, configure as standard for -an 'S3 Compatible Service' be sure to set upload_cutoff = 0 otherwise -you will run into authentication header issues as the snowball device -does not support query parameter based authentication. +an 'S3 Compatible Service'. + +If using rclone pre v1.59 be sure to set upload_cutoff = 0 otherwise you +will run into authentication header issues as the snowball device does +not support query parameter based authentication. + +With rclone v1.59 or later setting upload_cutoff should not be +necessary. eg. @@ -18139,10 +19433,10 @@ config: server_side_encryption = storage_class = -If you are using an older version of CEPH, e.g. 10.2.x Jewel, then you -may need to supply the parameter --s3-upload-cutoff 0 or put this in the -config file as upload_cutoff 0 to work around a bug which causes -uploading of small files to fail. +If you are using an older version of CEPH (e.g. 10.2.x Jewel) and a +version of rclone before v1.59 then you may need to supply the parameter +--s3-upload-cutoff 0 or put this in the config file as upload_cutoff 0 +to work around a bug which causes uploading of small files to fail. Note also that Ceph sometimes puts / in the passwords it gives users. If you read the secret access key using the command line tools you will get @@ -18167,6 +19461,101 @@ removed). Because this is a json dump, it is encoding the / as \/, so if you use the secret key as xxxxxx/xxxx it will work fine. +Cloudflare R2 + +Cloudflare R2 Storage allows developers to store large amounts of +unstructured data without the costly egress bandwidth fees associated +with typical cloud storage services. + +Here is an example of making a Cloudflare R2 configuration. First run: + + rclone config + +This will guide you through an interactive setup process. + +Note that all buckets are private, and all are stored in the same "auto" +region. It is necessary to use Cloudflare workers to share the content +of a bucket publicly. + + No remotes found, make a new one? + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> r2 + Option Storage. + Type of storage to configure. + Choose a number from below, or type in your own value. + ... + XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \ (s3) + ... + Storage> s3 + Option provider. + Choose your S3 provider. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + ... + XX / Cloudflare R2 Storage + \ (Cloudflare) + ... + provider> Cloudflare + Option env_auth. + Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). + Only applies if access_key_id and secret_access_key is blank. + Choose a number from below, or type in your own boolean value (true or false). + Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) + env_auth> 1 + Option access_key_id. + AWS Access Key ID. + Leave blank for anonymous access or runtime credentials. + Enter a value. Press Enter to leave empty. + access_key_id> ACCESS_KEY + Option secret_access_key. + AWS Secret Access Key (password). + Leave blank for anonymous access or runtime credentials. + Enter a value. Press Enter to leave empty. + secret_access_key> SECRET_ACCESS_KEY + Option region. + Region to connect to. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + 1 / R2 buckets are automatically distributed across Cloudflare's data centers for low latency. + \ (auto) + region> 1 + Option endpoint. + Endpoint for S3 API. + Required when using an S3 clone. + Enter a value. Press Enter to leave empty. + endpoint> https://ACCOUNT_ID.r2.cloudflarestorage.com + Edit advanced config? + y) Yes + n) No (default) + y/n> n + -------------------- + y) Yes this is OK (default) + e) Edit this remote + d) Delete this remote + y/e/d> y + +This will leave your config looking something like: + + [r2] + type = s3 + provider = Cloudflare + access_key_id = ACCESS_KEY + secret_access_key = SECRET_ACCESS_KEY + region = auto + endpoint = https://ACCOUNT_ID.r2.cloudflarestorage.com + acl = private + +Now run rclone lsf r2: to see your buckets and rclone lsf r2:bucket to +look within a bucket. + Dreamhost Dreamhost DreamObjects is an object storage system based on CEPH. @@ -18237,6 +19626,135 @@ example: rclone mkdir spaces:my-new-space rclone copy /path/to/files spaces:my-new-space +Huawei OBS + +Object Storage Service (OBS) provides stable, secure, efficient, and +easy-to-use cloud storage that lets you store virtually any volume of +unstructured data in any format and access it from anywhere. + +OBS provides an S3 interface, you can copy and modify the following +configuration and add it to your rclone configuration file. + + [obs] + type = s3 + provider = HuaweiOBS + access_key_id = your-access-key-id + secret_access_key = your-secret-access-key + region = af-south-1 + endpoint = obs.af-south-1.myhuaweicloud.com + acl = private + +Or you can also configure via the interactive command line: + + No remotes found, make a new one? + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> obs + Option Storage. + Type of storage to configure. + Choose a number from below, or type in your own value. + [snip] + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \ (s3) + [snip] + Storage> 5 + Option provider. + Choose your S3 provider. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + [snip] + 9 / Huawei Object Storage Service + \ (HuaweiOBS) + [snip] + provider> 9 + Option env_auth. + Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). + Only applies if access_key_id and secret_access_key is blank. + Choose a number from below, or type in your own boolean value (true or false). + Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) + env_auth> 1 + Option access_key_id. + AWS Access Key ID. + Leave blank for anonymous access or runtime credentials. + Enter a value. Press Enter to leave empty. + access_key_id> your-access-key-id + Option secret_access_key. + AWS Secret Access Key (password). + Leave blank for anonymous access or runtime credentials. + Enter a value. Press Enter to leave empty. + secret_access_key> your-secret-access-key + Option region. + Region to connect to. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + 1 / AF-Johannesburg + \ (af-south-1) + 2 / AP-Bangkok + \ (ap-southeast-2) + [snip] + region> 1 + Option endpoint. + Endpoint for OBS API. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + 1 / AF-Johannesburg + \ (obs.af-south-1.myhuaweicloud.com) + 2 / AP-Bangkok + \ (obs.ap-southeast-2.myhuaweicloud.com) + [snip] + endpoint> 1 + Option acl. + Canned ACL used when creating buckets and storing or copying objects. + This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too. + For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + Note that this ACL is applied when server-side copying objects as S3 + doesn't copy the ACL from the source but rather writes a fresh one. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \ (private) + [snip] + acl> 1 + Edit advanced config? + y) Yes + n) No (default) + y/n> + -------------------- + [obs] + type = s3 + provider = HuaweiOBS + access_key_id = your-access-key-id + secret_access_key = your-secret-access-key + region = af-south-1 + endpoint = obs.af-south-1.myhuaweicloud.com + acl = private + -------------------- + y) Yes this is OK (default) + e) Edit this remote + d) Delete this remote + y/e/d> y + Current remotes: + + Name Type + ==== ==== + obs s3 + + e) Edit existing remote + n) New remote + d) Delete remote + r) Rename remote + c) Copy remote + s) Set configuration password + q) Quit config + e/n/d/r/c/s/q> q + IBM COS (S3) Information stored with IBM Cloud Object Storage is encrypted and @@ -18268,12 +19786,12 @@ To configure access to IBM COS S3, follow the steps below: \ "alias" 2 / Amazon Drive \ "amazon cloud drive" - 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, Minio, IBM COS) + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS) \ "s3" 4 / Backblaze B2 \ "b2" [snip] - 23 / http Connection + 23 / HTTP \ "http" Storage> 3 @@ -18408,6 +19926,113 @@ To configure access to IBM COS S3, follow the steps below: 6) Delete a file on remote. rclone delete IBM-COS-XREGION:newbucket/file.txt +IDrive e2 + +Here is an example of making an IDrive e2 configuration. First run: + + rclone config + +This will guide you through an interactive setup process. + + No remotes found, make a new one? + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + + Enter name for new remote. + name> e2 + + Option Storage. + Type of storage to configure. + Choose a number from below, or type in your own value. + [snip] + XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \ (s3) + [snip] + Storage> s3 + + Option provider. + Choose your S3 provider. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + [snip] + XX / IDrive e2 + \ (IDrive) + [snip] + provider> IDrive + + Option env_auth. + Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). + Only applies if access_key_id and secret_access_key is blank. + Choose a number from below, or type in your own boolean value (true or false). + Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) + env_auth> + + Option access_key_id. + AWS Access Key ID. + Leave blank for anonymous access or runtime credentials. + Enter a value. Press Enter to leave empty. + access_key_id> YOUR_ACCESS_KEY + + Option secret_access_key. + AWS Secret Access Key (password). + Leave blank for anonymous access or runtime credentials. + Enter a value. Press Enter to leave empty. + secret_access_key> YOUR_SECRET_KEY + + Option acl. + Canned ACL used when creating buckets and storing or copying objects. + This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too. + For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + Note that this ACL is applied when server-side copying objects as S3 + doesn't copy the ACL from the source but rather writes a fresh one. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \ (private) + / Owner gets FULL_CONTROL. + 2 | The AllUsers group gets READ access. + \ (public-read) + / Owner gets FULL_CONTROL. + 3 | The AllUsers group gets READ and WRITE access. + | Granting this on a bucket is generally not recommended. + \ (public-read-write) + / Owner gets FULL_CONTROL. + 4 | The AuthenticatedUsers group gets READ access. + \ (authenticated-read) + / Object owner gets FULL_CONTROL. + 5 | Bucket owner gets READ access. + | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ (bucket-owner-read) + / Both the object owner and the bucket owner get FULL_CONTROL over the object. + 6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \ (bucket-owner-full-control) + acl> + + Edit advanced config? + y) Yes + n) No (default) + y/n> + + Configuration complete. + Options: + - type: s3 + - provider: IDrive + - access_key_id: YOUR_ACCESS_KEY + - secret_access_key: YOUR_SECRET_KEY + - endpoint: q9d9.la12.idrivee2-5.com + Keep this "e2" remote? + y) Yes this is OK (default) + e) Edit this remote + d) Delete this remote + y/e/d> y + Minio Minio is an object storage server built for cloud application developers @@ -18517,6 +20142,14 @@ rclone like this: server_side_encryption = storage_class = +C14 Cold Storage is the low-cost S3 Glacier alternative from Scaleway +and it works the same way as on S3 by accepting the "GLACIER" +storage_class. So you can configure your remote with the +storage_class = GLACIER option to upload directly to C14. Don't forget +that in this state you can't read files back after, you will need to +restore them to "STANDARD" storage_class first before being able to read +them (see "restore" section above) + Seagate Lyve Cloud Seagate Lyve Cloud is an S3 compatible object storage platform from @@ -18539,7 +20172,7 @@ Choose s3 backend Type of storage to configure. Choose a number from below, or type in your own value. [snip] - XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS + XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \ (s3) [snip] Storage> s3 @@ -18703,7 +20336,7 @@ rclone like this. Type of storage to configure. Choose a number from below, or type in your own value [snip] - XX / Amazon S3 (also Dreamhost, Ceph, Minio) + XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) \ "s3" [snip] Storage> s3 @@ -18813,7 +20446,7 @@ This will guide you through an interactive setup process. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [snip] - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -18902,6 +20535,360 @@ This will guide you through an interactive setup process. d) Delete this remote y/e/d> y +China Mobile Ecloud Elastic Object Storage (EOS) + +Here is an example of making an China Mobile Ecloud Elastic Object +Storage (EOS) configuration. First run: + + rclone config + +This will guide you through an interactive setup process. + + No remotes found, make a new one? + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> ChinaMobile + Option Storage. + Type of storage to configure. + Choose a number from below, or type in your own value. + ... + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS + \ (s3) + ... + Storage> s3 + Option provider. + Choose your S3 provider. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + ... + 4 / China Mobile Ecloud Elastic Object Storage (EOS) + \ (ChinaMobile) + ... + provider> ChinaMobile + Option env_auth. + Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). + Only applies if access_key_id and secret_access_key is blank. + Choose a number from below, or type in your own boolean value (true or false). + Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) + env_auth> + Option access_key_id. + AWS Access Key ID. + Leave blank for anonymous access or runtime credentials. + Enter a value. Press Enter to leave empty. + access_key_id> accesskeyid + Option secret_access_key. + AWS Secret Access Key (password). + Leave blank for anonymous access or runtime credentials. + Enter a value. Press Enter to leave empty. + secret_access_key> secretaccesskey + Option endpoint. + Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + / The default endpoint - a good choice if you are unsure. + 1 | East China (Suzhou) + \ (eos-wuxi-1.cmecloud.cn) + 2 / East China (Jinan) + \ (eos-jinan-1.cmecloud.cn) + 3 / East China (Hangzhou) + \ (eos-ningbo-1.cmecloud.cn) + 4 / East China (Shanghai-1) + \ (eos-shanghai-1.cmecloud.cn) + 5 / Central China (Zhengzhou) + \ (eos-zhengzhou-1.cmecloud.cn) + 6 / Central China (Changsha-1) + \ (eos-hunan-1.cmecloud.cn) + 7 / Central China (Changsha-2) + \ (eos-zhuzhou-1.cmecloud.cn) + 8 / South China (Guangzhou-2) + \ (eos-guangzhou-1.cmecloud.cn) + 9 / South China (Guangzhou-3) + \ (eos-dongguan-1.cmecloud.cn) + 10 / North China (Beijing-1) + \ (eos-beijing-1.cmecloud.cn) + 11 / North China (Beijing-2) + \ (eos-beijing-2.cmecloud.cn) + 12 / North China (Beijing-3) + \ (eos-beijing-4.cmecloud.cn) + 13 / North China (Huhehaote) + \ (eos-huhehaote-1.cmecloud.cn) + 14 / Southwest China (Chengdu) + \ (eos-chengdu-1.cmecloud.cn) + 15 / Southwest China (Chongqing) + \ (eos-chongqing-1.cmecloud.cn) + 16 / Southwest China (Guiyang) + \ (eos-guiyang-1.cmecloud.cn) + 17 / Nouthwest China (Xian) + \ (eos-xian-1.cmecloud.cn) + 18 / Yunnan China (Kunming) + \ (eos-yunnan.cmecloud.cn) + 19 / Yunnan China (Kunming-2) + \ (eos-yunnan-2.cmecloud.cn) + 20 / Tianjin China (Tianjin) + \ (eos-tianjin-1.cmecloud.cn) + 21 / Jilin China (Changchun) + \ (eos-jilin-1.cmecloud.cn) + 22 / Hubei China (Xiangyan) + \ (eos-hubei-1.cmecloud.cn) + 23 / Jiangxi China (Nanchang) + \ (eos-jiangxi-1.cmecloud.cn) + 24 / Gansu China (Lanzhou) + \ (eos-gansu-1.cmecloud.cn) + 25 / Shanxi China (Taiyuan) + \ (eos-shanxi-1.cmecloud.cn) + 26 / Liaoning China (Shenyang) + \ (eos-liaoning-1.cmecloud.cn) + 27 / Hebei China (Shijiazhuang) + \ (eos-hebei-1.cmecloud.cn) + 28 / Fujian China (Xiamen) + \ (eos-fujian-1.cmecloud.cn) + 29 / Guangxi China (Nanning) + \ (eos-guangxi-1.cmecloud.cn) + 30 / Anhui China (Huainan) + \ (eos-anhui-1.cmecloud.cn) + endpoint> 1 + Option location_constraint. + Location constraint - must match endpoint. + Used when creating buckets only. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + 1 / East China (Suzhou) + \ (wuxi1) + 2 / East China (Jinan) + \ (jinan1) + 3 / East China (Hangzhou) + \ (ningbo1) + 4 / East China (Shanghai-1) + \ (shanghai1) + 5 / Central China (Zhengzhou) + \ (zhengzhou1) + 6 / Central China (Changsha-1) + \ (hunan1) + 7 / Central China (Changsha-2) + \ (zhuzhou1) + 8 / South China (Guangzhou-2) + \ (guangzhou1) + 9 / South China (Guangzhou-3) + \ (dongguan1) + 10 / North China (Beijing-1) + \ (beijing1) + 11 / North China (Beijing-2) + \ (beijing2) + 12 / North China (Beijing-3) + \ (beijing4) + 13 / North China (Huhehaote) + \ (huhehaote1) + 14 / Southwest China (Chengdu) + \ (chengdu1) + 15 / Southwest China (Chongqing) + \ (chongqing1) + 16 / Southwest China (Guiyang) + \ (guiyang1) + 17 / Nouthwest China (Xian) + \ (xian1) + 18 / Yunnan China (Kunming) + \ (yunnan) + 19 / Yunnan China (Kunming-2) + \ (yunnan2) + 20 / Tianjin China (Tianjin) + \ (tianjin1) + 21 / Jilin China (Changchun) + \ (jilin1) + 22 / Hubei China (Xiangyan) + \ (hubei1) + 23 / Jiangxi China (Nanchang) + \ (jiangxi1) + 24 / Gansu China (Lanzhou) + \ (gansu1) + 25 / Shanxi China (Taiyuan) + \ (shanxi1) + 26 / Liaoning China (Shenyang) + \ (liaoning1) + 27 / Hebei China (Shijiazhuang) + \ (hebei1) + 28 / Fujian China (Xiamen) + \ (fujian1) + 29 / Guangxi China (Nanning) + \ (guangxi1) + 30 / Anhui China (Huainan) + \ (anhui1) + location_constraint> 1 + Option acl. + Canned ACL used when creating buckets and storing or copying objects. + This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too. + For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + Note that this ACL is applied when server-side copying objects as S3 + doesn't copy the ACL from the source but rather writes a fresh one. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \ (private) + / Owner gets FULL_CONTROL. + 2 | The AllUsers group gets READ access. + \ (public-read) + / Owner gets FULL_CONTROL. + 3 | The AllUsers group gets READ and WRITE access. + | Granting this on a bucket is generally not recommended. + \ (public-read-write) + / Owner gets FULL_CONTROL. + 4 | The AuthenticatedUsers group gets READ access. + \ (authenticated-read) + / Object owner gets FULL_CONTROL. + acl> private + Option server_side_encryption. + The server-side encryption algorithm used when storing this object in S3. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + 1 / None + \ () + 2 / AES256 + \ (AES256) + server_side_encryption> + Option storage_class. + The storage class to use when storing new objects in ChinaMobile. + Choose a number from below, or type in your own value. + Press Enter to leave empty. + 1 / Default + \ () + 2 / Standard storage class + \ (STANDARD) + 3 / Archive storage mode + \ (GLACIER) + 4 / Infrequent access storage mode + \ (STANDARD_IA) + storage_class> + Edit advanced config? + y) Yes + n) No (default) + y/n> n + -------------------- + [ChinaMobile] + type = s3 + provider = ChinaMobile + access_key_id = accesskeyid + secret_access_key = secretaccesskey + endpoint = eos-wuxi-1.cmecloud.cn + location_constraint = wuxi1 + acl = private + -------------------- + y) Yes this is OK (default) + e) Edit this remote + d) Delete this remote + y/e/d> y + +ArvanCloud + +ArvanCloud ArvanCloud Object Storage goes beyond the limited traditional +file storage. It gives you access to backup and archived files and +allows sharing. Files like profile image in the app, images sent by +users or scanned documents can be stored securely and easily in our +Object Storage service. + +ArvanCloud provides an S3 interface which can be configured for use with +rclone like this. + + No remotes found, make a new one? + n) New remote + s) Set configuration password + n/s> n + name> ArvanCloud + Type of storage to configure. + Choose a number from below, or type in your own value + [snip] + XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) + \ "s3" + [snip] + Storage> s3 + Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. + Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" + env_auth> 1 + AWS Access Key ID - leave blank for anonymous access or runtime credentials. + access_key_id> YOURACCESSKEY + AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. + secret_access_key> YOURSECRETACCESSKEY + Region to connect to. + Choose a number from below, or type in your own value + / The default endpoint - a good choice if you are unsure. + 1 | US Region, Northern Virginia, or Pacific Northwest. + | Leave location constraint empty. + \ "us-east-1" + [snip] + region> + Endpoint for S3 API. + Leave blank if using ArvanCloud to use the default endpoint for the region. + Specify if using an S3 clone such as Ceph. + endpoint> s3.arvanstorage.com + Location constraint - must be set to match the Region. Used when creating buckets only. + Choose a number from below, or type in your own value + 1 / Empty for Iran-Tehran Region. + \ "" + [snip] + location_constraint> + Canned ACL used when creating buckets and/or storing objects in S3. + For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" + [snip] + acl> + The server-side encryption algorithm used when storing this object in S3. + Choose a number from below, or type in your own value + 1 / None + \ "" + 2 / AES256 + \ "AES256" + server_side_encryption> + The storage class to use when storing objects in S3. + Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Standard storage class + \ "STANDARD" + storage_class> + Remote config + -------------------- + [ArvanCloud] + env_auth = false + access_key_id = YOURACCESSKEY + secret_access_key = YOURSECRETACCESSKEY + region = ir-thr-at1 + endpoint = s3.arvanstorage.com + location_constraint = + acl = + server_side_encryption = + storage_class = + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +This will leave the config file looking like this. + + [ArvanCloud] + type = s3 + provider = ArvanCloud + env_auth = false + access_key_id = YOURACCESSKEY + secret_access_key = YOURSECRETACCESSKEY + region = + endpoint = s3.arvanstorage.com + location_constraint = + acl = + server_side_encryption = + storage_class = + Tencent COS Tencent Cloud Object Storage (COS) is a distributed storage service @@ -18932,7 +20919,7 @@ To configure access to Tencent COS, follow the steps below: \ "alias" 3 / Amazon Drive \ "amazon cloud drive" - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -19136,7 +21123,7 @@ rclone about is not supported by the S3 backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Backblaze B2 @@ -19303,6 +21290,11 @@ remove the file instead of hiding it. Old versions of files, where available, are visible using the --b2-versions flag. +It is also possible to view a bucket as it was at a certain point in +time, using the --b2-version-at flag. This will show the file versions +as they were at that time, showing files that have been deleted +afterwards, and hiding files that were created since. + If you wish to remove all the old versions then you can use the rclone cleanup remote:bucket command which will delete all the old versions of files, leaving the current ones intact. You can also supply @@ -19428,7 +21420,7 @@ you can then use the authorization token (the part of the url from the Standard options -Here are the standard options specific to b2 (Backblaze B2). +Here are the Standard options specific to b2 (Backblaze B2). --b2-account @@ -19465,7 +21457,7 @@ Properties: Advanced options -Here are the advanced options specific to b2 (Backblaze B2). +Here are the Advanced options specific to b2 (Backblaze B2). --b2-endpoint @@ -19515,6 +21507,20 @@ Properties: - Type: bool - Default: false +--b2-version-at + +Show file versions as they were at the specified time. + +Note that when using this no file write operations are permitted, so you +can't upload files or delete them. + +Properties: + +- Config: version_at +- Env Var: RCLONE_B2_VERSION_AT +- Type: Time +- Default: off + --b2-upload-cutoff Cutoff for switching to chunked upload. @@ -19664,7 +21670,7 @@ rclone about is not supported by the B2 backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Box @@ -19920,7 +21926,7 @@ https://app.box.com/folder/11xxxxxxxxx8 in the browser, then you use Standard options -Here are the standard options specific to box (Box). +Here are the Standard options specific to box (Box). --box-client-id @@ -19993,7 +21999,7 @@ Properties: Advanced options -Here are the advanced options specific to box (Box). +Here are the Advanced options specific to box (Box). --box-token @@ -20115,7 +22121,7 @@ rclone about is not supported by the Box backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Cache (DEPRECATED) @@ -20434,7 +22440,7 @@ delete cached data (chunks) as well (optional, false by default) Standard options -Here are the standard options specific to cache (Cache a remote). +Here are the Standard options specific to cache (Cache a remote). --cache-remote @@ -20551,7 +22557,7 @@ Properties: Advanced options -Here are the advanced options specific to cache (Cache a remote). +Here are the Advanced options specific to cache (Cache a remote). --cache-plex-token @@ -20800,8 +22806,8 @@ Run them with The help below will explain what arguments each command takes. -See the "rclone backend" command for more info on how to pass options -and arguments. +See the backend command for more info on how to pass options and +arguments. These can be run on a running backend using the rc command backend/command. @@ -21115,7 +23121,7 @@ Changing transactions is dangerous and requires explicit migration. Standard options -Here are the standard options specific to chunker (Transparently +Here are the Standard options specific to chunker (Transparently chunk/split large files). --chunker-remote @@ -21176,7 +23182,7 @@ Properties: Advanced options -Here are the advanced options specific to chunker (Transparently +Here are the Advanced options specific to chunker (Transparently chunk/split large files). --chunker-name-format @@ -21425,7 +23431,7 @@ strings. Standard options -Here are the standard options specific to sharefile (Citrix Sharefile). +Here are the Standard options specific to sharefile (Citrix Sharefile). --sharefile-root-folder-id @@ -21455,7 +23461,7 @@ Properties: Advanced options -Here are the advanced options specific to sharefile (Citrix Sharefile). +Here are the Advanced options specific to sharefile (Citrix Sharefile). --sharefile-upload-cutoff @@ -21526,7 +23532,7 @@ without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Crypt @@ -21923,7 +23929,7 @@ remote instead of rclone check which can't check the checksums properly. Standard options -Here are the standard options specific to crypt (Encrypt/Decrypt a +Here are the Standard options specific to crypt (Encrypt/Decrypt a remote). --crypt-remote @@ -22008,7 +24014,7 @@ Properties: Advanced options -Here are the advanced options specific to crypt (Encrypt/Decrypt a +Here are the Advanced options specific to crypt (Encrypt/Decrypt a remote). --crypt-server-side-across-configs @@ -22091,6 +24097,12 @@ Properties: - Unicode codepoint instead of UTF-8 byte length. (Eg. Onedrive) +Metadata + +Any metadata supported by the underlying remote is read and written. + +See the metadata docs for more info. + Backend commands Here are the commands specific to the crypt backend. @@ -22101,8 +24113,8 @@ Run them with The help below will explain what arguments each command takes. -See the "rclone backend" command for more info on how to pass options -and arguments. +See the backend command for more info on how to pass options and +arguments. These can be run on a running backend using the rc command backend/command. @@ -22358,7 +24370,7 @@ compression backend. Standard options -Here are the standard options specific to compress (Compress a remote). +Here are the Standard options specific to compress (Compress a remote). --compress-remote @@ -22387,7 +24399,7 @@ Properties: Advanced options -Here are the advanced options specific to compress (Compress a remote). +Here are the Advanced options specific to compress (Compress a remote). --compress-level @@ -22422,6 +24434,157 @@ Properties: - Type: SizeSuffix - Default: 20Mi +Metadata + +Any metadata supported by the underlying remote is read and written. + +See the metadata docs for more info. + +Combine + +The combine backend joins remotes together into a single directory tree. + +For example you might have a remote for images on one provider: + + $ rclone tree s3:imagesbucket + / + ├── image1.jpg + └── image2.jpg + +And a remote for files on another: + + $ rclone tree drive:important/files + / + ├── file1.txt + └── file2.txt + +The combine backend can join these together into a synthetic directory +structure like this: + + $ rclone tree combined: + / + ├── files + │ ├── file1.txt + │ └── file2.txt + └── images + ├── image1.jpg + └── image2.jpg + +You'd do this by specifying an upstreams parameter in the config like +this + + upstreams = images=s3:imagesbucket files=drive:important/files + +During the initial setup with rclone config you will specify the +upstreams remotes as a space separated list. The upstream remotes can +either be a local paths or other remotes. + +Configuration + +Here is an example of how to make a combine called remote for the +example above. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found, make a new one? + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Option Storage. + Type of storage to configure. + Choose a number from below, or type in your own value. + ... + XX / Combine several remotes into one + \ (combine) + ... + Storage> combine + Option upstreams. + Upstreams for combining + These should be in the form + dir=remote:path dir2=remote2:path + Where before the = is specified the root directory and after is the remote to + put there. + Embedded spaces can be added using quotes + "dir=remote:path with space" "dir2=remote2:path with space" + Enter a fs.SpaceSepList value. + upstreams> images=s3:imagesbucket files=drive:important/files + -------------------- + [remote] + type = combine + upstreams = images=s3:imagesbucket files=drive:important/files + -------------------- + y) Yes this is OK (default) + e) Edit this remote + d) Delete this remote + y/e/d> y + +Configuring for Google Drive Shared Drives + +Rclone has a convenience feature for making a combine backend for all +the shared drives you have access to. + +Assuming your main (non shared drive) Google drive remote is called +drive: you would run + + rclone backend -o config drives drive: + +This would produce something like this: + + [My Drive] + type = alias + remote = drive,team_drive=0ABCDEF-01234567890,root_folder_id=: + + [Test Drive] + type = alias + remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=: + + [AllDrives] + type = combine + remote = "My Drive=My Drive:" "Test Drive=Test Drive:" + +If you then add that config to your config file (find it with +rclone config file) then you can access all the shared drives in one +place with the AllDrives: remote. + +See the Google Drive docs for full info. + +Standard options + +Here are the Standard options specific to combine (Combine several +remotes into one). + +--combine-upstreams + +Upstreams for combining + +These should be in the form + + dir=remote:path dir2=remote2:path + +Where before the = is specified the root directory and after is the +remote to put there. + +Embedded spaces can be added using quotes + + "dir=remote:path with space" "dir2=remote2:path with space" + +Properties: + +- Config: upstreams +- Env Var: RCLONE_COMBINE_UPSTREAMS +- Type: SpaceSepList +- Default: + +Metadata + +Any metadata supported by the underlying remote is read and written. + +See the metadata docs for more info. + Dropbox Paths are specified as remote:path @@ -22592,7 +24755,7 @@ finishes up the last batch using this mode. Standard options -Here are the standard options specific to dropbox (Dropbox). +Here are the Standard options specific to dropbox (Dropbox). --dropbox-client-id @@ -22622,7 +24785,7 @@ Properties: Advanced options -Here are the advanced options specific to dropbox (Dropbox). +Here are the Advanced options specific to dropbox (Dropbox). --dropbox-token @@ -23045,7 +25208,7 @@ The ID for "S3 Storage" would be 120673761. Standard options -Here are the standard options specific to filefabric (Enterprise File +Here are the Standard options specific to filefabric (Enterprise File Fabric). --filefabric-url @@ -23104,7 +25267,7 @@ Properties: Advanced options -Here are the advanced options specific to filefabric (Enterprise File +Here are the Advanced options specific to filefabric (Enterprise File Fabric). --filefabric-token @@ -23196,7 +25359,7 @@ address as password. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [snip] - XX / FTP Connection + XX / FTP \ "ftp" [snip] Storage> ftp @@ -23292,7 +25455,7 @@ VsFTPd. Just hit a selection number when prompted. Standard options -Here are the standard options specific to ftp (FTP Connection). +Here are the Standard options specific to ftp (FTP). --ftp-host @@ -23375,7 +25538,7 @@ Properties: Advanced options -Here are the advanced options specific to ftp (FTP Connection). +Here are the Advanced options specific to ftp (FTP). --ftp-concurrency @@ -23421,6 +25584,17 @@ Properties: - Type: bool - Default: false +--ftp-disable-utf8 + +Disable using UTF-8 even if server advertises support. + +Properties: + +- Config: disable_utf8 +- Env Var: RCLONE_FTP_DISABLE_UTF8 +- Type: bool +- Default: false + --ftp-writing-mdtm Use MDTM to set modification time (VsFtpd quirk) @@ -23545,7 +25719,7 @@ rclone about is not supported by the FTP backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about The implementation of : --dump headers, --dump bodies, --dump auth for debugging isn't the same as for rclone HTTP based backends - it has less @@ -23845,7 +26019,7 @@ strings. Standard options -Here are the standard options specific to google cloud storage (Google +Here are the Standard options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). --gcs-client-id @@ -24123,7 +26297,7 @@ Properties: Advanced options -Here are the advanced options specific to google cloud storage (Google +Here are the Advanced options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). --gcs-token @@ -24163,6 +26337,40 @@ Properties: - Type: string - Required: false +--gcs-no-check-bucket + +If set, don't attempt to check the bucket exists or create it. + +This can be useful when trying to minimise the number of transactions +rclone does if you know the bucket exists already. + +Properties: + +- Config: no_check_bucket +- Env Var: RCLONE_GCS_NO_CHECK_BUCKET +- Type: bool +- Default: false + +--gcs-decompress + +If set this will decompress gzip encoded objects. + +It is possible to upload objects to GCS with "Content-Encoding: gzip" +set. Normally rclone will download these files files as compressed +objects. + +If this flag is set then rclone will decompress these files with +"Content-Encoding: gzip" as they are received. This means that rclone +can't check the size and hash but the file contents will be +decompressed. + +Properties: + +- Config: decompress +- Env Var: RCLONE_GCS_DECOMPRESS +- Type: bool +- Default: false + --gcs-encoding The encoding for the backend. @@ -24183,7 +26391,7 @@ Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Google Drive @@ -24240,8 +26448,6 @@ This will guide you through an interactive setup process: 5 | does not allow any access to read or download file content. \ "drive.metadata.readonly" scope> 1 - ID of the root folder - leave blank normally. Fill in to access "Computers" folders. (see docs). - root_folder_id> Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. service_account_file> Remote config @@ -24341,9 +26547,9 @@ directories. Root folder ID -You can set the root_folder_id for rclone. This is the directory -(identified by its Folder ID) that rclone considers to be the root of -your drive. +This option has been moved to the advanced section. You can set the +root_folder_id for rclone. This is the directory (identified by its +Folder ID) that rclone considers to be the root of your drive. Normally you will leave this blank and rclone will determine the correct root to use itself. @@ -24697,9 +26903,13 @@ represent the currently available conversions. -------------------------------------------------------------------------------------------------------------------------- Extension Mime Type Description ------------------- --------------------------------------------------------------------------- -------------------------- + bmp image/bmp Windows Bitmap format + csv text/csv Standard CSV format for Spreadsheets + doc application/msword Classic Word file + docx application/vnd.openxmlformats-officedocument.wordprocessingml.document Microsoft Office Document epub application/epub+zip E-book format @@ -24708,7 +26918,8 @@ represent the currently available conversions. jpg image/jpeg A JPEG Image File - json application/vnd.google-apps.script+json JSON Text Format + json application/vnd.google-apps.script+json JSON Text Format for + Google Apps scripts odp application/vnd.oasis.opendocument.presentation Openoffice Presentation @@ -24720,6 +26931,8 @@ represent the currently available conversions. pdf application/pdf Adobe PDF Format + pjpeg image/pjpeg Progressive JPEG Image + png image/png PNG Image Format pptx application/vnd.openxmlformats-officedocument.presentationml.presentation Microsoft Office @@ -24735,6 +26948,10 @@ represent the currently available conversions. txt text/plain Plain Text + wmf application/x-msmetafile Windows Meta File + + xls application/vnd.ms-excel Classic Excel file + xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet Microsoft Office Spreadsheet @@ -24757,7 +26974,7 @@ Documents. Standard options -Here are the standard options specific to drive (Google Drive). +Here are the Standard options specific to drive (Google Drive). --drive-client-id @@ -24813,20 +27030,6 @@ Properties: - Allows read-only access to file metadata but - does not allow any access to read or download file content. ---drive-root-folder-id - -ID of the root folder. Leave blank normally. - -Fill in to access "Computers" folders (see docs), or for rclone to use a -non root folder as its starting point. - -Properties: - -- Config: root_folder_id -- Env Var: RCLONE_DRIVE_ROOT_FOLDER_ID -- Type: string -- Required: false - --drive-service-account-file Service Account Credentials JSON file path. @@ -24857,7 +27060,7 @@ Properties: Advanced options -Here are the advanced options specific to drive (Google Drive). +Here are the Advanced options specific to drive (Google Drive). --drive-token @@ -24896,6 +27099,20 @@ Properties: - Type: string - Required: false +--drive-root-folder-id + +ID of the root folder. Leave blank normally. + +Fill in to access "Computers" folders (see docs), or for rclone to use a +non root folder as its starting point. + +Properties: + +- Config: root_folder_id +- Env Var: RCLONE_DRIVE_ROOT_FOLDER_ID +- Type: string +- Required: false + --drive-service-account-credentials Service Account Credentials JSON blob. @@ -25372,6 +27589,33 @@ Properties: - Type: bool - Default: false +--drive-resource-key + +Resource key for accessing a link-shared file. + +If you need to access files shared with a link like this + + https://drive.google.com/drive/folders/XXX?resourcekey=YYY&usp=sharing + +Then you will need to use the first part "XXX" as the "root_folder_id" +and the second part "YYY" as the "resource_key" otherwise you will get +404 not found errors when trying to access the directory. + +See: https://developers.google.com/drive/api/guides/resource-keys + +This resource key requirement only applies to a subset of old files. + +Note also that opening the folder once in the web interface (with the +user you've authenticated rclone with) seems to be enough so that the +resource key is no needed. + +Properties: + +- Config: resource_key +- Env Var: RCLONE_DRIVE_RESOURCE_KEY +- Type: string +- Required: false + --drive-encoding The encoding for the backend. @@ -25395,8 +27639,8 @@ Run them with The help below will explain what arguments each command takes. -See the "rclone backend" command for more info on how to pass options -and arguments. +See the backend command for more info on how to pass options and +arguments. These can be run on a running backend using the rc command backend/command. @@ -25496,7 +27740,7 @@ This will return a JSON list of objects like this With the -o config parameter it will output the list in a format suitable for adding to a config file to make aliases for all the drives -found. +found and a combined drive. [My Drive] type = alias @@ -25506,9 +27750,15 @@ found. type = alias remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=: + [AllDrives] + type = combine + remote = "My Drive=My Drive:" "Test Drive=Test Drive:" + Adding this to the rclone config file will cause those team drives to be -accessible with the aliases shown. This may require manual editing of -the names. +accessible with the aliases shown. Any illegal charactes will be +substituted with "_" and duplicate names will have numbers suffixed. It +will also add a remote called AllDrives which shows all the shared +drives combined into one directory tree. untrash @@ -25562,6 +27812,18 @@ attempted if possible. Use the -i flag to see what would be copied before copying. +exportformats + +Dump the export formats for debug purposes + + rclone backend exportformats remote: [options] [+] + +importformats + +Dump the import formats for debug purposes + + rclone backend importformats remote: [options] [+] + Limitations Drive has quite a lot of rate limiting. This causes rclone to be limited @@ -25576,8 +27838,10 @@ upload the files if you prefer. Limitations of Google Docs -Google docs will appear as size -1 in rclone ls and as size 0 in -anything which uses the VFS layer, e.g. rclone mount, rclone serve. +Google docs will appear as size -1 in rclone ls, rclone ncdu etc, and as +size 0 in anything which uses the VFS layer, e.g. rclone mount and +rclone serve. When calculating directory totals, e.g. in rclone size and +rclone ncdu, they will be counted in as empty files. This is because rclone can't find out the size of the Google docs without downloading them. @@ -25661,8 +27925,9 @@ Here is how to create your own Google Drive client ID for rclone: Click again on "Credentials" on the left panel to go back to the "Credentials" screen. -(PS: if you are a GSuite user, you could also select "Internal" instead -of "External" above, but this has not been tested/documented so far). + (PS: if you are a GSuite user, you could also select "Internal" + instead of "External" above, but this will restrict API use to + Google Workspace users in your organisation). 6. Click on the "+ CREATE CREDENTIALS" button at the top of the screen, then select "OAuth client ID". @@ -25673,13 +27938,17 @@ of "External" above, but this has not been tested/documented so far). 8. It will show you a client ID and client secret. Make a note of these. -9. Go to "Oauth consent screen" and press "Publish App" + (If you selected "External" at Step 5 continue to "Publish App" in + the Steps 9 and 10. If you chose "Internal" you don't need to + publish and can skip straight to Step 11.) -10. Provide the noted client ID and client secret to rclone. +9. Go to "Oauth consent screen" and press "Publish App" -11. Click "OAuth consent screen", then click "PUBLISH APP" button and +10. Click "OAuth consent screen", then click "PUBLISH APP" button and confirm, or add your account under "Test users". +11. Provide the noted client ID and client secret to rclone. + Be aware that, due to the "enhanced security" recently introduced by Google, you are theoretically expected to "submit your app for verification" and then wait a few weeks(!) for their response; in @@ -25911,7 +28180,7 @@ is similar to the Sharing tab in the Google Photos web interface. Standard options -Here are the standard options specific to google photos (Google Photos). +Here are the Standard options specific to google photos (Google Photos). --gphotos-client-id @@ -25955,7 +28224,7 @@ Properties: Advanced options -Here are the advanced options specific to google photos (Google Photos). +Here are the Advanced options specific to google photos (Google Photos). --gphotos-token @@ -26311,7 +28580,7 @@ Configuration reference Standard options -Here are the standard options specific to hasher (Better checksums for +Here are the Standard options specific to hasher (Better checksums for other remotes). --hasher-remote @@ -26350,7 +28619,7 @@ Properties: Advanced options -Here are the advanced options specific to hasher (Better checksums for +Here are the Advanced options specific to hasher (Better checksums for other remotes). --hasher-auto-size @@ -26365,6 +28634,12 @@ Properties: - Type: SizeSuffix - Default: 0 +Metadata + +Any metadata supported by the underlying remote is read and written. + +See the metadata docs for more info. + Backend commands Here are the commands specific to the hasher backend. @@ -26375,8 +28650,8 @@ Run them with The help below will explain what arguments each command takes. -See the "rclone backend" command for more info on how to pass options -and arguments. +See the backend command for more info on how to pass options and +arguments. These can be run on a running backend using the rc command backend/command. @@ -26617,7 +28892,7 @@ Invalid UTF-8 bytes will also be replaced. Standard options -Here are the standard options specific to hdfs (Hadoop distributed file +Here are the Standard options specific to hdfs (Hadoop distributed file system). --hdfs-namenode @@ -26649,7 +28924,7 @@ Properties: Advanced options -Here are the advanced options specific to hdfs (Hadoop distributed file +Here are the Advanced options specific to hdfs (Hadoop distributed file system). --hdfs-service-principal-name @@ -26704,6 +28979,460 @@ Limitations - No server-side Move or DirMove. - Checksums not implemented. +HiDrive + +Paths are specified as remote:path + +Paths may be as deep as required, e.g. remote:directory/subdirectory. + +The initial setup for hidrive involves getting a token from HiDrive +which you need to do in your browser. rclone config walks you through +it. + +Configuration + +Here is an example of how to make a remote called remote. First run: + + rclone config + +This will guide you through an interactive setup process: + + No remotes found - make a new one + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Type of storage to configure. + Choose a number from below, or type in your own value + [snip] + XX / HiDrive + \ "hidrive" + [snip] + Storage> hidrive + OAuth Client Id - Leave blank normally. + client_id> + OAuth Client Secret - Leave blank normally. + client_secret> + Access permissions that rclone should use when requesting access from HiDrive. + Leave blank normally. + scope_access> + Edit advanced config? + y/n> n + Use auto config? + y/n> y + If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx + Log in and authorize rclone for access + Waiting for code... + Got code + -------------------- + [remote] + type = hidrive + token = {"access_token":"xxxxxxxxxxxxxxxxxxxx","token_type":"Bearer","refresh_token":"xxxxxxxxxxxxxxxxxxxxxxx","expiry":"xxxxxxxxxxxxxxxxxxxxxxx"} + -------------------- + y) Yes this is OK (default) + e) Edit this remote + d) Delete this remote + y/e/d> y + +You should be aware that OAuth-tokens can be used to access your account +and hence should not be shared with other persons. See the below section +for more information. + +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from HiDrive. This only runs from the moment it opens +your browser to the moment you get back the verification code. The +webserver runs on http://127.0.0.1:53682/. If local port 53682 is +protected by a firewall you may need to temporarily unblock the firewall +to complete authorization. + +Once configured you can then use rclone like this, + +List directories in top level of your HiDrive root folder + + rclone lsd remote: + +List all the files in your HiDrive filesystem + + rclone ls remote: + +To copy a local directory to a HiDrive directory called backup + + rclone copy /home/source remote:backup + +Keeping your tokens safe + +Any OAuth-tokens will be stored by rclone in the remote's configuration +file as unencrypted text. Anyone can use a valid refresh-token to access +your HiDrive filesystem without knowing your password. Therefore you +should make sure no one else can access your configuration. + +It is possible to encrypt rclone's configuration file. You can find +information on securing your configuration file by viewing the +configuration encryption docs. + +Invalid refresh token + +As can be verified here, each refresh_token (for Native Applications) is +valid for 60 days. If used to access HiDrivei, its validity will be +automatically extended. + +This means that if you + +- Don't use the HiDrive remote for 60 days + +then rclone will return an error which includes a text that implies the +refresh token is invalid or expired. + +To fix this you will need to authorize rclone to access your HiDrive +account again. + +Using + + rclone config reconnect remote: + +the process is very similar to the process of initial setup exemplified +before. + +Modified time and hashes + +HiDrive allows modification times to be set on objects accurate to 1 +second. + +HiDrive supports its own hash type which is used to verify the integrety +of file contents after successful transfers. + +Restricted filename characters + +HiDrive cannot store files or folders that include / (0x2F) or +null-bytes (0x00) in their name. Any other characters can be used in the +names of files or folders. Additionally, files or folders cannot be +named either of the following: . or .. + +Therefore rclone will automatically replace these characters, if files +or folders are stored or accessed with such names. + +You can read about how this filename encoding works in general here. + +Keep in mind that HiDrive only supports file or folder names with a +length of 255 characters or less. + +Transfers + +HiDrive limits file sizes per single request to a maximum of 2 GiB. To +allow storage of larger files and allow for better upload performance, +the hidrive backend will use a chunked transfer for files larger than 96 +MiB. Rclone will upload multiple parts/chunks of the file at the same +time. Chunks in the process of being uploaded are buffered in memory, so +you may want to restrict this behaviour on systems with limited +resources. + +You can customize this behaviour using the following options: + +- chunk_size: size of file parts +- upload_cutoff: files larger or equal to this in size will use a + chunked transfer +- upload_concurrency: number of file-parts to upload at the same time + +See the below section about configuration options for more details. + +Root folder + +You can set the root folder for rclone. This is the directory that +rclone considers to be the root of your HiDrive. + +Usually, you will leave this blank, and rclone will use the root of the +account. + +However, you can set this to restrict rclone to a specific folder +hierarchy. + +This works by prepending the contents of the root_prefix option to any +paths accessed by rclone. For example, the following two ways to access +the home directory are equivalent: + + rclone lsd --hidrive-root-prefix="/users/test/" remote:path + + rclone lsd remote:/users/test/path + +See the below section about configuration options for more details. + +Directory member count + +By default, rclone will know the number of directory members contained +in a directory. For example, rclone lsd uses this information. + +The acquisition of this information will result in additional time costs +for HiDrive's API. When dealing with large directory structures, it may +be desirable to circumvent this time cost, especially when this +information is not explicitly needed. For this, the +disable_fetching_member_count option can be used. + +See the below section about configuration options for more details. + +Standard options + +Here are the Standard options specific to hidrive (HiDrive). + +--hidrive-client-id + +OAuth Client Id. + +Leave blank normally. + +Properties: + +- Config: client_id +- Env Var: RCLONE_HIDRIVE_CLIENT_ID +- Type: string +- Required: false + +--hidrive-client-secret + +OAuth Client Secret. + +Leave blank normally. + +Properties: + +- Config: client_secret +- Env Var: RCLONE_HIDRIVE_CLIENT_SECRET +- Type: string +- Required: false + +--hidrive-scope-access + +Access permissions that rclone should use when requesting access from +HiDrive. + +Properties: + +- Config: scope_access +- Env Var: RCLONE_HIDRIVE_SCOPE_ACCESS +- Type: string +- Default: "rw" +- Examples: + - "rw" + - Read and write access to resources. + - "ro" + - Read-only access to resources. + +Advanced options + +Here are the Advanced options specific to hidrive (HiDrive). + +--hidrive-token + +OAuth Access Token as a JSON blob. + +Properties: + +- Config: token +- Env Var: RCLONE_HIDRIVE_TOKEN +- Type: string +- Required: false + +--hidrive-auth-url + +Auth server URL. + +Leave blank to use the provider defaults. + +Properties: + +- Config: auth_url +- Env Var: RCLONE_HIDRIVE_AUTH_URL +- Type: string +- Required: false + +--hidrive-token-url + +Token server url. + +Leave blank to use the provider defaults. + +Properties: + +- Config: token_url +- Env Var: RCLONE_HIDRIVE_TOKEN_URL +- Type: string +- Required: false + +--hidrive-scope-role + +User-level that rclone should use when requesting access from HiDrive. + +Properties: + +- Config: scope_role +- Env Var: RCLONE_HIDRIVE_SCOPE_ROLE +- Type: string +- Default: "user" +- Examples: + - "user" + - User-level access to management permissions. + - This will be sufficient in most cases. + - "admin" + - Extensive access to management permissions. + - "owner" + - Full access to management permissions. + +--hidrive-root-prefix + +The root/parent folder for all paths. + +Fill in to use the specified folder as the parent for all paths given to +the remote. This way rclone can use any folder as its starting point. + +Properties: + +- Config: root_prefix +- Env Var: RCLONE_HIDRIVE_ROOT_PREFIX +- Type: string +- Default: "/" +- Examples: + - "/" + - The topmost directory accessible by rclone. + - This will be equivalent with "root" if rclone uses a regular + HiDrive user account. + - "root" + - The topmost directory of the HiDrive user account + - "" + - This specifies that there is no root-prefix for your paths. + - When using this you will always need to specify paths to + this remote with a valid parent e.g. "remote:/path/to/dir" + or "remote:root/path/to/dir". + +--hidrive-endpoint + +Endpoint for the service. + +This is the URL that API-calls will be made to. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_HIDRIVE_ENDPOINT +- Type: string +- Default: "https://api.hidrive.strato.com/2.1" + +--hidrive-disable-fetching-member-count + +Do not fetch number of objects in directories unless it is absolutely +necessary. + +Requests may be faster if the number of objects in subdirectories is not +fetched. + +Properties: + +- Config: disable_fetching_member_count +- Env Var: RCLONE_HIDRIVE_DISABLE_FETCHING_MEMBER_COUNT +- Type: bool +- Default: false + +--hidrive-chunk-size + +Chunksize for chunked uploads. + +Any files larger than the configured cutoff (or files of unknown size) +will be uploaded in chunks of this size. + +The upper limit for this is 2147483647 bytes (about 2.000Gi). That is +the maximum amount of bytes a single upload-operation will support. +Setting this above the upper limit or to a negative value will cause +uploads to fail. + +Setting this to larger values may increase the upload speed at the cost +of using more memory. It can be set to smaller values smaller to save on +memory. + +Properties: + +- Config: chunk_size +- Env Var: RCLONE_HIDRIVE_CHUNK_SIZE +- Type: SizeSuffix +- Default: 48Mi + +--hidrive-upload-cutoff + +Cutoff/Threshold for chunked uploads. + +Any files larger than this will be uploaded in chunks of the configured +chunksize. + +The upper limit for this is 2147483647 bytes (about 2.000Gi). That is +the maximum amount of bytes a single upload-operation will support. +Setting this above the upper limit will cause uploads to fail. + +Properties: + +- Config: upload_cutoff +- Env Var: RCLONE_HIDRIVE_UPLOAD_CUTOFF +- Type: SizeSuffix +- Default: 96Mi + +--hidrive-upload-concurrency + +Concurrency for chunked uploads. + +This is the upper limit for how many transfers for the same file are +running concurrently. Setting this above to a value smaller than 1 will +cause uploads to deadlock. + +If you are uploading small numbers of large files over high-speed links +and these uploads do not fully utilize your bandwidth, then increasing +this may help to speed up the transfers. + +Properties: + +- Config: upload_concurrency +- Env Var: RCLONE_HIDRIVE_UPLOAD_CONCURRENCY +- Type: int +- Default: 4 + +--hidrive-encoding + +The encoding for the backend. + +See the encoding section in the overview for more info. + +Properties: + +- Config: encoding +- Env Var: RCLONE_HIDRIVE_ENCODING +- Type: MultiEncoder +- Default: Slash,Dot + +Limitations + +Symbolic links + +HiDrive is able to store symbolic links (symlinks) by design, for +example, when unpacked from a zip archive. + +There exists no direct mechanism to manage native symlinks in remotes. +As such this implementation has chosen to ignore any native symlinks +present in the remote. rclone will not be able to access or show any +symlinks stored in the hidrive-remote. This means symlinks cannot be +individually removed, copied, or moved, except when removing, copying, +or moving the parent folder. + +This does not affect the .rclonelink-files that rclone uses to encode +and store symbolic links. + +Sparse files + +It is possible to store sparse files in HiDrive. + +Note that copying a sparse file will expand the holes into null-byte +(0x00) regions that will then consume disk space. Likewise, when +downloading a sparse file, the resulting file will have null-byte +regions in the place of file holes. + HTTP The HTTP remote is a read only remote for reading files of a webserver. @@ -26750,7 +29479,7 @@ This will guide you through an interactive setup process: Type of storage to configure. Choose a number from below, or type in your own value [snip] - XX / http Connection + XX / HTTP \ "http" [snip] Storage> http @@ -26823,11 +29552,11 @@ or: Standard options -Here are the standard options specific to http (http Connection). +Here are the Standard options specific to http (HTTP). --http-url -URL of http host to connect to. +URL of HTTP host to connect to. E.g. "https://example.com", or "https://user:pass@example.com" to use a username and password. @@ -26841,7 +29570,7 @@ Properties: Advanced options -Here are the advanced options specific to http (http Connection). +Here are the Advanced options specific to http (HTTP). --http-headers @@ -26918,7 +29647,7 @@ rclone about is not supported by the HTTP backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Hubic @@ -27023,7 +29752,7 @@ are the same. Standard options -Here are the standard options specific to hubic (Hubic). +Here are the Standard options specific to hubic (Hubic). --hubic-client-id @@ -27053,7 +29782,7 @@ Properties: Advanced options -Here are the advanced options specific to hubic (Hubic). +Here are the Advanced options specific to hubic (Hubic). --hubic-token @@ -27148,6 +29877,331 @@ The Swift API doesn't return a correct MD5SUM for segmented files (Dynamic or Static Large Objects) so rclone won't check or use the MD5SUM for these. +Internet Archive + +The Internet Archive backend utilizes Items on archive.org + +Refer to IAS3 API documentation for the API this backend uses. + +Paths are specified as remote:bucket (or remote: for the lsd command.) +You may put subdirectories in too, e.g. remote:item/path/to/dir. + +Once you have made a remote (see the provider specific section above) +you can use it like this: + +Unlike S3, listing up all items uploaded by you isn't supported. + +Make a new item + + rclone mkdir remote:item + +List the contents of a item + + rclone ls remote:item + +Sync /home/local/directory to the remote item, deleting any excess files +in the item. + + rclone sync -i /home/local/directory remote:item + +Notes + +Because of Internet Archive's architecture, it enqueues write operations +(and extra post-processings) in a per-item queue. You can check item's +queue at https://catalogd.archive.org/history/item-name-here . Because +of that, all uploads/deletes will not show up immediately and takes some +time to be available. The per-item queue is enqueued to an another +queue, Item Deriver Queue. You can check the status of Item Deriver +Queue here. This queue has a limit, and it may block you from uploading, +or even deleting. You should avoid uploading a lot of small files for +better behavior. + +You can optionally wait for the server's processing to finish, by +setting non-zero value to wait_archive key. By making it wait, rclone +can do normal file comparison. Make sure to set a large enough value +(e.g. 30m0s for smaller files) as it can take a long time depending on +server's queue. + +About metadata + +This backend supports setting, updating and reading metadata of each +file. The metadata will appear as file metadata on Internet Archive. +However, some fields are reserved by both Internet Archive and rclone. + +The following are reserved by Internet Archive: - name - source - size - +md5 - crc32 - sha1 - format - old_version - viruscheck + +Trying to set values to these keys is ignored with a warning. Only +setting mtime is an exception. Doing so make it the identical behavior +as setting ModTime. + +rclone reserves all the keys starting with rclone-. Setting value for +these keys will give you warnings, but values are set according to +request. + +If there are multiple values for a key, only the first one is returned. +This is a limitation of rclone, that supports one value per one key. It +can be triggered when you did a server-side copy. + +Reading metadata will also provide custom (non-standard nor reserved) +ones. + +Configuration + +Here is an example of making an internetarchive configuration. Most +applies to the other providers as well, any differences are described +below. + +First run + + rclone config + +This will guide you through an interactive setup process. + + No remotes found, make a new one? + n) New remote + s) Set configuration password + q) Quit config + n/s/q> n + name> remote + Option Storage. + Type of storage to configure. + Choose a number from below, or type in your own value. + XX / InternetArchive Items + \ (internetarchive) + Storage> internetarchive + Option access_key_id. + IAS3 Access Key. + Leave blank for anonymous access. + You can find one here: https://archive.org/account/s3.php + Enter a value. Press Enter to leave empty. + access_key_id> XXXX + Option secret_access_key. + IAS3 Secret Key (password). + Leave blank for anonymous access. + Enter a value. Press Enter to leave empty. + secret_access_key> XXXX + Edit advanced config? + y) Yes + n) No (default) + y/n> y + Option endpoint. + IAS3 Endpoint. + Leave blank for default value. + Enter a string value. Press Enter for the default (https://s3.us.archive.org). + endpoint> + Option front_endpoint. + Host of InternetArchive Frontend. + Leave blank for default value. + Enter a string value. Press Enter for the default (https://archive.org). + front_endpoint> + Option disable_checksum. + Don't store MD5 checksum with object metadata. + Normally rclone will calculate the MD5 checksum of the input before + uploading it so it can ask the server to check the object against checksum. + This is great for data integrity checking but can cause long delays for + large files to start uploading. + Enter a boolean value (true or false). Press Enter for the default (true). + disable_checksum> true + Option encoding. + The encoding for the backend. + See the [encoding section in the overview](https://rclone.org/overview/#encoding) for more info. + Enter a encoder.MultiEncoder value. Press Enter for the default (Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot). + encoding> + Edit advanced config? + y) Yes + n) No (default) + y/n> n + -------------------- + [remote] + type = internetarchive + access_key_id = XXXX + secret_access_key = XXXX + -------------------- + y) Yes this is OK (default) + e) Edit this remote + d) Delete this remote + y/e/d> y + +Standard options + +Here are the Standard options specific to internetarchive (Internet +Archive). + +--internetarchive-access-key-id + +IAS3 Access Key. + +Leave blank for anonymous access. You can find one here: +https://archive.org/account/s3.php + +Properties: + +- Config: access_key_id +- Env Var: RCLONE_INTERNETARCHIVE_ACCESS_KEY_ID +- Type: string +- Required: false + +--internetarchive-secret-access-key + +IAS3 Secret Key (password). + +Leave blank for anonymous access. + +Properties: + +- Config: secret_access_key +- Env Var: RCLONE_INTERNETARCHIVE_SECRET_ACCESS_KEY +- Type: string +- Required: false + +Advanced options + +Here are the Advanced options specific to internetarchive (Internet +Archive). + +--internetarchive-endpoint + +IAS3 Endpoint. + +Leave blank for default value. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_INTERNETARCHIVE_ENDPOINT +- Type: string +- Default: "https://s3.us.archive.org" + +--internetarchive-front-endpoint + +Host of InternetArchive Frontend. + +Leave blank for default value. + +Properties: + +- Config: front_endpoint +- Env Var: RCLONE_INTERNETARCHIVE_FRONT_ENDPOINT +- Type: string +- Default: "https://archive.org" + +--internetarchive-disable-checksum + +Don't ask the server to test against MD5 checksum calculated by rclone. +Normally rclone will calculate the MD5 checksum of the input before +uploading it so it can ask the server to check the object against +checksum. This is great for data integrity checking but can cause long +delays for large files to start uploading. + +Properties: + +- Config: disable_checksum +- Env Var: RCLONE_INTERNETARCHIVE_DISABLE_CHECKSUM +- Type: bool +- Default: true + +--internetarchive-wait-archive + +Timeout for waiting the server's processing tasks (specifically archive +and book_op) to finish. Only enable if you need to be guaranteed to be +reflected after write operations. 0 to disable waiting. No errors to be +thrown in case of timeout. + +Properties: + +- Config: wait_archive +- Env Var: RCLONE_INTERNETARCHIVE_WAIT_ARCHIVE +- Type: Duration +- Default: 0s + +--internetarchive-encoding + +The encoding for the backend. + +See the encoding section in the overview for more info. + +Properties: + +- Config: encoding +- Env Var: RCLONE_INTERNETARCHIVE_ENCODING +- Type: MultiEncoder +- Default: Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot + +Metadata + +Metadata fields provided by Internet Archive. If there are multiple +values for a key, only the first one is returned. This is a limitation +of Rclone, that supports one value per one key. + +Owner is able to add custom keys. Metadata feature grabs all the keys +including them. + +Here are the possible system metadata items for the internetarchive +backend. + + ---------------------------------------------------------------------------------------------------------------------- + Name Help Type Example Read Only + --------------------- ------------------ ----------- -------------------------------------------- -------------------- + crc32 CRC32 calculated string 01234567 N + by Internet + Archive + + format Name of format string Comma-Separated Values N + identified by + Internet Archive + + md5 MD5 hash string 01234567012345670123456701234567 N + calculated by + Internet Archive + + mtime Time of last RFC 3339 2006-01-02T15:04:05.999999999Z N + modification, + managed by Rclone + + name Full file path, filename backend/internetarchive/internetarchive.go N + without the bucket + part + + old_version Whether the file boolean true N + was replaced and + moved by + keep-old-version + flag + + rclone-ia-mtime Time of last RFC 3339 2006-01-02T15:04:05.999999999Z N + modification, + managed by + Internet Archive + + rclone-mtime Time of last RFC 3339 2006-01-02T15:04:05.999999999Z N + modification, + managed by Rclone + + rclone-update-track Random value used string aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa N + by Rclone for + tracking changes + inside Internet + Archive + + sha1 SHA1 hash string 0123456701234567012345670123456701234567 N + calculated by + Internet Archive + + size File size in bytes decimal 123456 N + number + + source The source of the string original N + file + + viruscheck The last time unixtime 1654191352 N + viruscheck process + was run for the + file (?) + ---------------------------------------------------------------------------------------------------------------------- + +See the metadata docs for more info. + Jottacloud Jottacloud is a cloud storage service provider from a Norwegian company, @@ -27220,56 +30274,78 @@ This will guide you through an interactive setup process: q) Quit config n/s/q> n name> remote + Option Storage. Type of storage to configure. - Enter a string value. Press Enter for the default (""). - Choose a number from below, or type in your own value + Choose a number from below, or type in your own value. [snip] XX / Jottacloud - \ "jottacloud" + \ (jottacloud) [snip] Storage> jottacloud - ** See help for jottacloud backend at: https://rclone.org/jottacloud/ ** - - Edit advanced config? (y/n) - y) Yes - n) No - y/n> n - Remote config - Use legacy authentication?. - This is only required for certain whitelabel versions of Jottacloud and not recommended for normal users. + Edit advanced config? y) Yes n) No (default) y/n> n - - Generate a personal login token here: https://www.jottacloud.com/web/secure + Option config_type. + Select authentication type. + Choose a number from below, or type in an existing string value. + Press Enter for the default (standard). + / Standard authentication. + 1 | Use this if you're a normal Jottacloud user. + \ (standard) + / Legacy authentication. + 2 | This is only required for certain whitelabel versions of Jottacloud and not recommended for normal users. + \ (legacy) + / Telia Cloud authentication. + 3 | Use this if you are using Telia Cloud. + \ (telia) + / Tele2 Cloud authentication. + 4 | Use this if you are using Tele2 Cloud. + \ (tele2) + config_type> 1 + Personal login token. + Generate here: https://www.jottacloud.com/web/secure Login Token> - - Do you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client? - + Use a non-standard device/mountpoint? + Choosing no, the default, will let you access the storage used for the archive + section of the official Jottacloud client. If you instead want to access the + sync or the backup section, for example, you must choose yes. y) Yes - n) No + n) No (default) y/n> y - Please select the device to use. Normally this will be Jotta - Choose a number from below, or type in an existing value + Option config_device. + The device to use. In standard setup the built-in Jotta device is used, + which contains predefined mountpoints for archive, sync etc. All other devices + are treated as backup devices by the official Jottacloud client. You may create + a new by entering a unique name. + Choose a number from below, or type in your own string value. + Press Enter for the default (DESKTOP-3H31129). 1 > DESKTOP-3H31129 2 > Jotta - Devices> 2 - Please select the mountpoint to user. Normally this will be Archive - Choose a number from below, or type in an existing value + config_device> 2 + Option config_mountpoint. + The mountpoint to use for the built-in device Jotta. + The standard setup is to use the Archive mountpoint. Most other mountpoints + have very limited support in rclone and should generally be avoided. + Choose a number from below, or type in an existing string value. + Press Enter for the default (Archive). 1 > Archive - 2 > Links + 2 > Shared 3 > Sync - - Mountpoints> 1 + config_mountpoint> 1 -------------------- - [jotta] + [remote] type = jottacloud + configVersion = 1 + client_id = jottacli + client_secret = + tokenURL = https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token token = {........} + username = 2940e57271a93d987d6f8a21 device = Jotta mountpoint = Archive - configVersion = 1 -------------------- - y) Yes this is OK + y) Yes this is OK (default) e) Edit this remote d) Delete this remote y/e/d> y @@ -27291,21 +30367,30 @@ To copy a local directory to an Jottacloud directory called backup Devices and Mountpoints The official Jottacloud client registers a device for each computer you -install it on, and then creates a mountpoint for each folder you select -for Backup. The web interface uses a special device called Jotta for the -Archive and Sync mountpoints. - -With rclone you'll want to use the Jotta/Archive device/mountpoint in -most cases, however if you want to access files uploaded by any of the -official clients rclone provides the option to select other devices and -mountpoints during config. Note that uploading files is currently not -supported to other devices than Jotta. - -The built-in Jotta device may also contain several other mountpoints, -such as: Latest, Links, Shared and Trash. These are special mountpoints -with a different internal representation than the "regular" mountpoints. -Rclone will only to a very limited degree support them. Generally you -should avoid these, unless you know what you are doing. +install it on, and shows them in the backup section of the user +interface. For each folder you select for backup it will create a +mountpoint within this device. A built-in device called Jotta is +special, and contains mountpoints Archive, Sync and some others, used +for corresponding features in official clients. + +With rclone you'll want to use the standard Jotta/Archive +device/mountpoint in most cases. However, you may for example want to +access files from the sync or backup functionality provided by the +official clients, and rclone therefore provides the option to select +other devices and mountpoints during config. + +You are allowed to create new devices and mountpoints. All devices +except the built-in Jotta device are treated as backup devices by +official Jottacloud clients, and the mountpoints on them are individual +backup sets. + +With the built-in Jotta device, only existing, built-in, mountpoints can +be selected. In addition to the mentioned Archive and Sync, it may +contain several other mountpoints such as: Latest, Links, Shared and +Trash. All of these are special mountpoints with a different internal +representation than the "regular" mountpoints. Rclone will only to a +very limited degree support them. Generally you should avoid these, +unless you know what you are doing. --fast-list @@ -27384,7 +30469,7 @@ current usage. Advanced options -Here are the advanced options specific to jottacloud (Jottacloud). +Here are the Advanced options specific to jottacloud (Jottacloud). --jottacloud-md5-memory-limit @@ -27584,7 +30669,7 @@ strings. Standard options -Here are the standard options specific to koofr (Koofr, Digi Storage and +Here are the Standard options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). --koofr-provider @@ -27674,7 +30759,7 @@ Properties: Advanced options -Here are the advanced options specific to koofr (Koofr, Digi Storage and +Here are the Advanced options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). --koofr-mountid @@ -28024,7 +31109,7 @@ strings. Standard options -Here are the standard options specific to mailru (Mail.ru Cloud). +Here are the Standard options specific to mailru (Mail.ru Cloud). --mailru-user @@ -28078,7 +31163,7 @@ Properties: Advanced options -Here are the advanced options specific to mailru (Mail.ru Cloud). +Here are the Advanced options specific to mailru (Mail.ru Cloud). --mailru-speedup-file-patterns @@ -28315,6 +31400,38 @@ Use rclone dedupe to fix duplicated files. Failure to log-in +Object not found + +If you are connecting to your Mega remote for the first time, to test +access and syncronisation, you may receive an error such as + + Failed to create file system for "my-mega-remote:": + couldn't login: Object (typically, node or user) not found + +The diagnostic steps often recommended in the rclone forum start with +the MEGAcmd utility. Note that this refers to the official C++ command +from https://github.com/meganz/MEGAcmd and not the go language built +command from t3rm1n4l/megacmd that is no longer maintained. + +Follow the instructions for installing MEGAcmd and try accessing your +remote as they recommend. You can establish whether or not you can log +in using MEGAcmd, and obtain diagnostic information to help you, and +search or work with others in the forum. + + MEGA CMD> login me@example.com + Password: + Fetching nodes ... + Loading transfers from local cache + Login complete as me@example.com + me@example.com:/$ + +Note that some have found issues with passwords containing special +characters. If you can not log on with rclone, but MEGAcmd logs on just +fine, then consider changing your password temporarily to pure +alphanumeric characters, in case that helps. + +Repeated commands blocks access + Mega remotes seem to get blocked (reject logins) under "heavy use". We haven't worked out the exact blocking rules but it seems to be related to fast paced, successive rclone commands. @@ -28361,7 +31478,7 @@ got the remote blocked for a while. Standard options -Here are the standard options specific to mega (Mega). +Here are the Standard options specific to mega (Mega). --mega-user @@ -28389,7 +31506,7 @@ Properties: Advanced options -Here are the advanced options specific to mega (Mega). +Here are the Advanced options specific to mega (Mega). --mega-debug @@ -28515,6 +31632,8 @@ See all buckets rclone lsd remote: The initial setup for Netstorage involves getting an account and secret. Use rclone config to walk you through the setup process. +Configuration + Here's an example of how to make a remote called ns1. 1. To begin the interactive configuration process, enter this command: @@ -28621,6 +31740,8 @@ You can't perform operations between different remotes. rclone move ns1:/974012/testing/notes.txt ns1:/974450/testing2/ +Features + Symlink Support The Netstorage backend changes the rclone --links, -l behavior. When @@ -28669,7 +31790,7 @@ Content Management Shell (CMShell). Rclone will not guarantee correctness of operations with implicit directories which might have been created as a result of using an upload API directly. -ListR Feature +--fast-list / ListR support NetStorage remote supports the ListR feature by using the "list" NetStorage API action to return a lexicographical list of all objects @@ -28696,7 +31817,7 @@ files in the directory and directory size as -1 when ListR method is used. The workaround is to pass "--disable listR" flag if these numbers are important in the output. -Purge Feature +Purge NetStorage remote supports the purge feature by using the "quick-delete" NetStorage API action. The quick-delete action is disabled by default @@ -28712,7 +31833,7 @@ accessible. Standard options -Here are the standard options specific to netstorage (Akamai +Here are the Standard options specific to netstorage (Akamai NetStorage). --netstorage-host @@ -28757,7 +31878,7 @@ Properties: Advanced options -Here are the advanced options specific to netstorage (Akamai +Here are the Advanced options specific to netstorage (Akamai NetStorage). --netstorage-protocol @@ -28789,8 +31910,8 @@ Run them with The help below will explain what arguments each command takes. -See the "rclone backend" command for more info on how to pass options -and arguments. +See the backend command for more info on how to pass options and +arguments. These can be run on a running backend using the rc command backend/command. @@ -28966,7 +32087,7 @@ untrusted environment such as a CI build server. Standard options -Here are the standard options specific to azureblob (Microsoft Azure +Here are the Standard options specific to azureblob (Microsoft Azure Blob Storage). --azureblob-account @@ -29067,7 +32188,7 @@ Properties: Advanced options -Here are the advanced options specific to azureblob (Microsoft Azure +Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage). --azureblob-msi-object-id @@ -29332,15 +32453,21 @@ backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Azure Storage Emulator Support -You can test rclone with storage emulator locally, to do this make sure -azure storage emulator installed locally and set up a new remote with -rclone config follow instructions described in introduction, set -use_emulator config as true, you do not need to provide default account -name or key if using emulator. +You can run rclone with storage emulator (usually azurite). + +To do this, just set up a new remote with rclone config following +instructions described in introduction and set use_emulator config as +true. You do not need to provide default account name neither an account +key. + +Also, if you want to access a storage emulator instance running on a +different machine, you can override Endpoint parameter in advanced +settings, setting it to http(s)://:/devstoreaccount1 (e.g. +http://10.254.2.5:10000/devstoreaccount1). Microsoft OneDrive @@ -29457,12 +32584,16 @@ To copy a local directory to an OneDrive directory called backup Getting your own Client ID and Key -You can use your own Client ID if the default (client_id left blank) one -doesn't work for you or you see lots of throttling. The default Client -ID and Key is shared by all rclone users when performing requests. +rclone uses a default Client ID when talking to OneDrive, unless a +custom client_id is specified in the config. The default Client ID and +Key are shared by all rclone users when performing requests. + +You may choose to create and use your own Client ID, in case the default +one does not work well for you. For example, you might see throtting. -If you are having problems with them (E.g., seeing a lot of throttling), -you can get your own Client ID and Key by following the steps below: +Creating Client ID for OneDrive Personal + +To create your own Client ID, please follow these steps: 1. Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade @@ -29480,17 +32611,47 @@ you can get your own Client ID and Key by following the steps below: select Microsoft Graph then select delegated permissions. 5. Search and select the following permissions: Files.Read, Files.ReadWrite, Files.Read.All, Files.ReadWrite.All, - offline_access, User.Read, and optionally Sites.Read.All (see - below). Once selected click Add permissions at the bottom. + offline_access, User.Read and Sites.Read.All (if custom access + scopes are configured, select the permissions accordingly). Once + selected click Add permissions at the bottom. Now the application is complete. Run rclone config to create or edit a OneDrive remote. Supply the app ID and password as Client ID and Secret, respectively. rclone will walk you through the remaining steps. +The access_scopes option allows you to configure the permissions +requested by rclone. See Microsoft Docs for more information about the +different scopes. + The Sites.Read.All permission is required if you need to search SharePoint sites when configuring the remote. However, if that -permission is not assigned, you need to set disable_site_permission -option to true in the advanced options. +permission is not assigned, you need to exclude Sites.Read.All from your +access scopes or set disable_site_permission option to true in the +advanced options. + +Creating Client ID for OneDrive Business + +The steps for OneDrive Personal may or may not work for OneDrive +Business, depending on the security settings of the organization. A +common error is that the publisher of the App is not verified. + +You may try to verify you account, or try to limit the App to your +organization only, as shown below. + +1. Make sure to create the App with your business account. +2. Follow the steps above to create an App. However, we need a + different account type here: + Accounts in this organizational directory only (*** - Single tenant). + Note that you can also change the account type aftering creating the + App. +3. Find the tenant ID of your organization. +4. In the rclone config, set auth_url to + https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/authorize. +5. In the rclone config, set token_url to + https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/token. + +Note: If you have a special region, you may need a different host in +step 4 and 5. Here are some hints. Modification time and hashes @@ -29547,7 +32708,7 @@ the OneDrive website. Standard options -Here are the standard options specific to onedrive (Microsoft OneDrive). +Here are the Standard options specific to onedrive (Microsoft OneDrive). --onedrive-client-id @@ -29597,7 +32758,7 @@ Properties: Advanced options -Here are the advanced options specific to onedrive (Microsoft OneDrive). +Here are the Advanced options specific to onedrive (Microsoft OneDrive). --onedrive-token @@ -29691,6 +32852,32 @@ Properties: - Type: string - Required: false +--onedrive-access-scopes + +Set scopes to be requested by rclone. + +Choose or manually enter a custom space separated list with all scopes, +that rclone should request. + +Properties: + +- Config: access_scopes +- Env Var: RCLONE_ONEDRIVE_ACCESS_SCOPES +- Type: SpaceSepList +- Default: Files.Read Files.ReadWrite Files.Read.All + Files.ReadWrite.All Sites.Read.All offline_access +- Examples: + - "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All + Sites.Read.All offline_access" + - Read and write access to all resources + - "Files.Read Files.Read.All Sites.Read.All offline_access" + - Read only access to all resources + - "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All + offline_access" + - Read and write access to all resources, without the ability + to browse SharePoint sites. + - Same as if disable_site_permission was set to true + --onedrive-disable-site-permission Disable the request for Sites.Read.All permission. @@ -29994,12 +33181,12 @@ Replacing/deleting existing files on Sharepoint gets "item not found" It is a known issue that Sharepoint (not OneDrive or OneDrive for Business) may return "item not found" errors when users try to replace or delete uploaded files; this seems to mainly affect Office files -(.docx, .xlsx, etc.). As a workaround, you may use the ---backup-dir command line argument so rclone moves the -files to be replaced/deleted into a given backup directory (instead of -directly replacing/deleting them). For example, to instruct rclone to -move the files into the directory rclone-backup-dir on backend -mysharepoint, you may use: +(.docx, .xlsx, etc.) and web files (.html, .aspx, etc.). As a +workaround, you may use the --backup-dir command line +argument so rclone moves the files to be replaced/deleted into a given +backup directory (instead of directly replacing/deleting them). For +example, to instruct rclone to move the files into the directory +rclone-backup-dir on backend mysharepoint, you may use: --backup-dir mysharepoint:rclone-backup-dir @@ -30014,7 +33201,7 @@ account. You can't do much about it, maybe write an email to your admins. However, there are other ways to interact with your OneDrive account. -Have a look at the webdav backend: https://rclone.org/webdav/#sharepoint +Have a look at the WebDAV backend: https://rclone.org/webdav/#sharepoint invalid_grant (AADSTS50076) @@ -30135,7 +33322,7 @@ strings. Standard options -Here are the standard options specific to opendrive (OpenDrive). +Here are the Standard options specific to opendrive (OpenDrive). --opendrive-username @@ -30163,7 +33350,7 @@ Properties: Advanced options -Here are the advanced options specific to opendrive (OpenDrive). +Here are the Advanced options specific to opendrive (OpenDrive). --opendrive-encoding @@ -30208,7 +33395,7 @@ rclone about is not supported by the OpenDrive backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about QingStor @@ -30348,7 +33535,7 @@ strings. Standard options -Here are the standard options specific to qingstor (QingCloud Object +Here are the Standard options specific to qingstor (QingCloud Object Storage). --qingstor-env-auth @@ -30434,7 +33621,7 @@ Properties: Advanced options -Here are the advanced options specific to qingstor (QingCloud Object +Here are the Advanced options specific to qingstor (QingCloud Object Storage). --qingstor-connection-retries @@ -30522,7 +33709,7 @@ rclone about is not supported by the qingstor backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Sia @@ -30641,7 +33828,7 @@ Once configured, you can then use rclone like this: Standard options -Here are the standard options specific to sia (Sia Decentralized Cloud). +Here are the Standard options specific to sia (Sia Decentralized Cloud). --sia-api-url @@ -30676,7 +33863,7 @@ Properties: Advanced options -Here are the advanced options specific to sia (Sia Decentralized Cloud). +Here are the Advanced options specific to sia (Sia Decentralized Cloud). --sia-user-agent @@ -30948,7 +34135,7 @@ strings. Standard options -Here are the standard options specific to swift (OpenStack Swift +Here are the Standard options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). --swift-env-auth @@ -31193,7 +34380,7 @@ Properties: Advanced options -Here are the advanced options specific to swift (OpenStack Swift +Here are the Advanced options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). --swift-leave-parts-on-error @@ -31410,6 +34597,14 @@ Deleted files will be moved to the trash. Your subscription level will determine how long items stay in the trash. rclone cleanup can be used to empty the trash. +Emptying the trash + +Due to an API limitation, the rclone cleanup command will only work if +you set your username and password in the advanced options for this +backend. Since we generally want to avoid storing user passwords in the +rclone config file, we advise you to only set this up if you need the +rclone cleanup command to work. + Root folder ID You can set the root_folder_id for rclone. This is the directory @@ -31433,7 +34628,7 @@ config. Standard options -Here are the standard options specific to pcloud (Pcloud). +Here are the Standard options specific to pcloud (Pcloud). --pcloud-client-id @@ -31463,7 +34658,7 @@ Properties: Advanced options -Here are the advanced options specific to pcloud (Pcloud). +Here are the Advanced options specific to pcloud (Pcloud). --pcloud-token @@ -31546,6 +34741,35 @@ Properties: - "eapi.pcloud.com" - EU region +--pcloud-username + +Your pcloud username. + +This is only required when you want to use the cleanup command. Due to a +bug in the pcloud API the required API does not support OAuth +authentication so we have to rely on user password authentication for +it. + +Properties: + +- Config: username +- Env Var: RCLONE_PCLOUD_USERNAME +- Type: string +- Required: false + +--pcloud-password + +Your pcloud password. + +NB Input to this must be obscured - see rclone obscure. + +Properties: + +- Config: password +- Env Var: RCLONE_PCLOUD_PASSWORD +- Type: string +- Required: false + premiumize.me Paths are specified as remote:path @@ -31645,7 +34869,7 @@ strings. Standard options -Here are the standard options specific to premiumizeme (premiumize.me). +Here are the Standard options specific to premiumizeme (premiumize.me). --premiumizeme-api-key @@ -31662,7 +34886,7 @@ Properties: Advanced options -Here are the advanced options specific to premiumizeme (premiumize.me). +Here are the Advanced options specific to premiumizeme (premiumize.me). --premiumizeme-encoding @@ -31792,7 +35016,7 @@ strings. Advanced options -Here are the advanced options specific to putio (Put.io). +Here are the Advanced options specific to putio (Put.io). --putio-encoding @@ -31807,6 +35031,15 @@ Properties: - Type: MultiEncoder - Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot +Limitations + +put.io has rate limiting. When you hit a limit, rclone automatically +retries after waiting the amount of time requested by the server. + +If you want to avoid ever hitting these limits, you may use the +--tpslimit flag with a low number. Note that the imposed limits may be +different for different operations, and may change over time. + Seafile This is a backend for the Seafile storage service: - It works with both @@ -32065,7 +35298,7 @@ haven't been tested and might not work properly. Standard options -Here are the standard options specific to seafile (seafile). +Here are the Standard options specific to seafile (seafile). --seafile-url @@ -32157,7 +35390,7 @@ Properties: Advanced options -Here are the advanced options specific to seafile (seafile). +Here are the Advanced options specific to seafile (seafile). --seafile-create-library @@ -32189,7 +35422,7 @@ SFTP is the Secure (or SSH) File Transfer Protocol. The SFTP backend can be used with a number of different providers: -- C14 +- Hetzner Storage Box - rsync.net SFTP runs over SSH v2 and is installed as standard with most modern SSH @@ -32203,8 +35436,11 @@ config (i.e /home/sftpuser). However, rclone lsd remote:/ would list the root directory for remote machine (i.e. /) Note that some SFTP servers will need the leading / - Synology is a good -example of this. rsync.net, on the other hand, requires users to OMIT -the leading /. +example of this. rsync.net and Hetzner, on the other hand, requires +users to OMIT the leading /. + +Note that by default rclone will try to execute shell commands on the +server, see shell access considerations. Configuration @@ -32223,7 +35459,7 @@ This will guide you through an interactive setup process. Type of storage to configure. Choose a number from below, or type in your own value [snip] - XX / SSH/SFTP Connection + XX / SSH/SFTP \ "sftp" [snip] Storage> sftp @@ -32417,6 +35653,114 @@ And then at the end of the session These commands can be used in scripts of course. +Shell access + +Some functionality of the SFTP backend relies on remote shell access, +and the possibility to execute commands. This includes checksum, and in +some cases also about. The shell commands that must be executed may be +different on different type of shells, and also quoting/escaping of file +path arguments containing special characters may be different. Rclone +therefore needs to know what type of shell it is, and if shell access is +available at all. + +Most servers run on some version of Unix, and then a basic Unix shell +can be assumed, without further distinction. Windows 10, Server 2019, +and later can also run a SSH server, which is a port of OpenSSH (see +official installation guide). On a Windows server the shell handling is +different: Although it can also be set up to use a Unix type shell, e.g. +Cygwin bash, the default is to use Windows Command Prompt (cmd.exe), and +PowerShell is a recommended alternative. All of these have bahave +differently, which rclone must handle. + +Rclone tries to auto-detect what type of shell is used on the server, +first time you access the SFTP remote. If a remote shell session is +successfully created, it will look for indications that it is CMD or +PowerShell, with fall-back to Unix if not something else is detected. If +unable to even create a remote shell session, then shell command +execution will be disabled entirely. The result is stored in the SFTP +remote configuration, in option shell_type, so that the auto-detection +only have to be performed once. If you manually set a value for this +option before first run, the auto-detection will be skipped, and if you +set a different value later this will override any existing. Value none +can be set to avoid any attempts at executing shell commands, e.g. if +this is not allowed on the server. + +When the server is rclone serve sftp, the rclone SFTP remote will detect +this as a Unix type shell - even if it is running on Windows. This +server does not actually have a shell, but it accepts input commands +matching the specific ones that the SFTP backend relies on for Unix +shells, e.g. md5sum and df. Also it handles the string escape rules used +for Unix shell. Treating it as a Unix type shell from a SFTP remote will +therefore always be correct, and support all features. + +Shell access considerations + +The shell type auto-detection logic, described above, means that by +default rclone will try to run a shell command the first time a new sftp +remote is accessed. If you configure a sftp remote without a config +file, e.g. an on the fly remote, rclone will have nowhere to store the +result, and it will re-run the command on every access. To avoid this +you should explicitely set the shell_type option to the correct value, +or to none if you want to prevent rclone from executing any remote shell +commands. + +It is also important to note that, since the shell type decides how +quoting and escaping of file paths used as command-line arguments are +performed, configuring the wrong shell type may leave you exposed to +command injection exploits. Make sure to confirm the auto-detected shell +type, or explicitely set the shell type you know is correct, or disable +shell access until you know. + +Checksum + +SFTP does not natively support checksums (file hash), but rclone is able +to use checksumming if the same login has shell access, and can execute +remote commands. If there is a command that can calculate compatible +checksums on the remote system, Rclone can then be configured to execute +this whenever a checksum is needed, and read back the results. Currently +MD5 and SHA-1 are supported. + +Normally this requires an external utility being available on the +server. By default rclone will try commands md5sum, md5 and +rclone md5sum for MD5 checksums, and the first one found usable will be +picked. Same with sha1sum, sha1 and rclone sha1sum commands for SHA-1 +checksums. These utilities normally need to be in the remote's PATH to +be found. + +In some cases the shell itself is capable of calculating checksums. +PowerShell is an example of such a shell. If rclone detects that the +remote shell is PowerShell, which means it most probably is a Windows +OpenSSH server, rclone will use a predefined script block to produce the +checksums when no external checksum commands are found (see shell +access). This assumes PowerShell version 4.0 or newer. + +The options md5sum_command and sha1_command can be used to customize the +command to be executed for calculation of checksums. You can for example +set a specific path to where md5sum and sha1sum executables are located, +or use them to specify some other tools that print checksums in +compatible format. The value can include command-line arguments, or even +shell script blocks as with PowerShell. Rclone has subcommands md5sum +and sha1sum that use compatible format, which means if you have an +rclone executable on the server it can be used. As mentioned above, they +will be automatically picked up if found in PATH, but if not you can set +something like /path/to/rclone md5sum as the value of option +md5sum_command to make sure a specific executable is used. + +Remote checksumming is recommended and enabled by default. First time +rclone is using a SFTP remote, if options md5sum_command or sha1_command +are not set, it will check if any of the default commands for each of +them, as described above, can be used. The result will be saved in the +remote configuration, so next time it will use the same. Value none will +be set if none of the default commands could be used for a specific +algorithm, and this algorithm will not be supported by the remote. + +Disabling the checksumming may be required if you are connecting to SFTP +servers which are not under your control, and to which the execution of +remote shell commands is prohibited. Set the configuration option +disable_hashcheck to true to disable checksumming entirely, or set +shell_type to none to disable all functionality based on remote shell +command execution. + Modified time Modified times are stored on the server to 1 second precision. @@ -32429,9 +35773,24 @@ mod_sftp). If you are using one of these servers, you can set the option set_modtime = false in your RClone backend configuration to disable this behaviour. +About command + +The about command returns the total space, free space, and used space on +the remote for the disk of the specified path on the remote or, if not +set, the disk of the root on the remote. + +SFTP usually supports the about command, but it depends on the server. +If the server implements the vendor-specific VFS statistics extension, +which is normally the case with OpenSSH instances, it will be used. If +not, but the same login has access to a Unix shell, where the df command +is available (e.g. in the remote's PATH), then this will be used +instead. If the server shell is PowerShell, probably with a Windows +OpenSSH server, rclone will use a built-in shell command (see shell +access). If none of the above is applicable, about will fail. + Standard options -Here are the standard options specific to sftp (SSH/SFTP Connection). +Here are the Standard options specific to sftp (SSH/SFTP). --sftp-host @@ -32607,7 +35966,7 @@ Properties: Advanced options -Here are the advanced options specific to sftp (SSH/SFTP Connection). +Here are the Advanced options specific to sftp (SSH/SFTP). --sftp-known-hosts-file @@ -32644,16 +36003,16 @@ Properties: --sftp-path-override -Override path used by SSH connection. +Override path used by SSH shell commands. This allows checksum calculation when SFTP and SSH paths are different. This issue affects among others Synology NAS boxes. -Shared folders can be found in directories representing volumes +E.g. if shared folders can be found in directories representing volumes: rclone sync /home/local/directory remote:/directory --sftp-path-override /volume2/directory -Home directory can be found in a shared folder called "home" +E.g. if home directory can be found in a shared folder called "home": rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory @@ -32675,6 +36034,28 @@ Properties: - Type: bool - Default: true +--sftp-shell-type + +The type of SSH shell on remote server, if any. + +Leave blank for autodetect. + +Properties: + +- Config: shell_type +- Env Var: RCLONE_SFTP_SHELL_TYPE +- Type: string +- Required: false +- Examples: + - "none" + - No shell access + - "unix" + - Unix shell + - "powershell" + - PowerShell + - "cmd" + - Windows Command Prompt + --sftp-md5sum-command The command used to read md5 hashes. @@ -32812,25 +36193,75 @@ Properties: - Type: Duration - Default: 1m0s +--sftp-chunk-size + +Upload and download chunk size. + +This controls the maximum packet size used in the SFTP protocol. The RFC +limits this to 32768 bytes (32k), however a lot of servers support +larger sizes and setting it larger will increase transfer speed +dramatically on high latency links. + +Only use a setting higher than 32k if you always connect to the same +server or after sufficiently broad testing. + +For example using the value of 252k with OpenSSH works well with its +maximum packet size of 256k. + +If you get the error "failed to send packet header: EOF" when copying a +large file, try lowering this number. + +Properties: + +- Config: chunk_size +- Env Var: RCLONE_SFTP_CHUNK_SIZE +- Type: SizeSuffix +- Default: 32Ki + +--sftp-concurrency + +The maximum number of outstanding requests for one file + +This controls the maximum number of outstanding requests for one file. +Increasing it will increase throughput on high latency links at the cost +of using more memory. + +Properties: + +- Config: concurrency +- Env Var: RCLONE_SFTP_CONCURRENCY +- Type: int +- Default: 64 + +--sftp-set-env + +Environment variables to pass to sftp and commands + +Set environment variables in the form: + + VAR=value + +to be passed to the sftp client and to any commands run (eg md5sum). + +Pass multiple variables space separated, eg + + VAR1=value VAR2=value + +and pass variables with spaces in in quotes, eg + + "VAR3=value with space" "VAR4=value with space" VAR5=nospacehere + +Properties: + +- Config: set_env +- Env Var: RCLONE_SFTP_SET_ENV +- Type: SpaceSepList +- Default: + Limitations -SFTP supports checksums if the same login has shell access and md5sum or -sha1sum as well as echo are in the remote's PATH. This remote -checksumming (file hashing) is recommended and enabled by default. -Disabling the checksumming may be required if you are connecting to SFTP -servers which are not under your control, and to which the execution of -remote commands is prohibited. Set the configuration option -disable_hashcheck to true to disable checksumming. - -SFTP also supports about if the same login has shell access and df are -in the remote's PATH. about will return the total space, free space, and -used space on the remote for the disk of the specified path on the -remote or, if not set, the disk of the root on the remote. about will -fail if it does not have shell access or if df is not in the remote's -PATH. - -Note that some SFTP servers (e.g. Synology) the paths are different for -SSH and SFTP so the hashes can't be calculated properly. For them using +On some SFTP servers (e.g. Synology) the paths are different for SSH and +SFTP so the hashes can't be calculated properly. For them using disable_hashcheck is a good idea. The only ssh agent supported under Windows is Putty's pageant. @@ -32844,22 +36275,22 @@ found in this paper. SFTP isn't supported under plan9 until this issue is fixed. Note that since SFTP isn't HTTP based the following flags don't work -with it: --dump-headers, --dump-bodies, --dump-auth +with it: --dump-headers, --dump-bodies, --dump-auth. Note that --timeout and --contimeout are both supported. -C14 - -C14 is supported through the SFTP backend. - -See C14's documentation - rsync.net rsync.net is supported through the SFTP backend. See rsync.net's documentation of rclone examples. +Hetzner Storage Box + +Hetzner Storage Boxes are supported through the SFTP backend on port 23. + +See Hetzner's documentation for details + Storj Storj is an encrypted, secure, and cost-effective object storage service @@ -33065,7 +36496,7 @@ Setup with API key and passphrase Standard options -Here are the standard options specific to storj (Storj Decentralized +Here are the Standard options specific to storj (Storj Decentralized Cloud Storage). --storj-provider @@ -33269,7 +36700,7 @@ without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Known issues @@ -33404,7 +36835,7 @@ deleted straight away. Standard options -Here are the standard options specific to sugarsync (Sugarsync). +Here are the Standard options specific to sugarsync (Sugarsync). --sugarsync-app-id @@ -33459,7 +36890,7 @@ Properties: Advanced options -Here are the advanced options specific to sugarsync (Sugarsync). +Here are the Advanced options specific to sugarsync (Sugarsync). --sugarsync-refresh-token @@ -33558,7 +36989,7 @@ rclone about is not supported by the SugarSync backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. -See List of backends that do not support rclone about See rclone about +See List of backends that do not support rclone about and rclone about Tardigrade @@ -33663,7 +37094,7 @@ strings. Standard options -Here are the standard options specific to uptobox (Uptobox). +Here are the Standard options specific to uptobox (Uptobox). --uptobox-access-token @@ -33680,7 +37111,7 @@ Properties: Advanced options -Here are the advanced options specific to uptobox (Uptobox). +Here are the Advanced options specific to uptobox (Uptobox). --uptobox-encoding @@ -33950,7 +37381,7 @@ much larger latency of remote file systems. Standard options -Here are the standard options specific to union (Union merges the +Here are the Standard options specific to union (Union merges the contents of several upstream fs). --union-upstreams @@ -34013,6 +37444,31 @@ Properties: - Type: int - Default: 120 +Advanced options + +Here are the Advanced options specific to union (Union merges the +contents of several upstream fs). + +--union-min-free-space + +Minimum viable free space for lfs/eplfs policies. + +If a remote has less than this much free space then it won't be +considered for use in lfs or eplfs policies. + +Properties: + +- Config: min_free_space +- Env Var: RCLONE_UNION_MIN_FREE_SPACE +- Type: SizeSuffix +- Default: 1Gi + +Metadata + +Any metadata supported by the underlying remote is read and written. + +See the metadata docs for more info. + WebDAV Paths are specified as remote:path @@ -34040,7 +37496,7 @@ This will guide you through an interactive setup process: Type of storage to configure. Choose a number from below, or type in your own value [snip] - XX / Webdav + XX / WebDAV \ "webdav" [snip] Storage> webdav @@ -34049,7 +37505,7 @@ This will guide you through an interactive setup process: 1 / Connect to example.com \ "https://example.com" url> https://example.com/remote.php/webdav/ - Name of the Webdav site/service/software you are using + Name of the WebDAV site/service/software you are using Choose a number from below, or type in your own value 1 / Nextcloud \ "nextcloud" @@ -34116,7 +37572,7 @@ objects, or only on objects which had a hash uploaded with them. Standard options -Here are the standard options specific to webdav (Webdav). +Here are the Standard options specific to webdav (WebDAV). --webdav-url @@ -34133,7 +37589,7 @@ Properties: --webdav-vendor -Name of the Webdav site/service/software you are using. +Name of the WebDAV site/service/software you are using. Properties: @@ -34194,7 +37650,7 @@ Properties: Advanced options -Here are the advanced options specific to webdav (Webdav). +Here are the Advanced options specific to webdav (WebDAV). --webdav-bearer-token-command @@ -34521,7 +37977,7 @@ strings. Standard options -Here are the standard options specific to yandex (Yandex Disk). +Here are the Standard options specific to yandex (Yandex Disk). --yandex-client-id @@ -34551,7 +38007,7 @@ Properties: Advanced options -Here are the advanced options specific to yandex (Yandex Disk). +Here are the Advanced options specific to yandex (Yandex Disk). --yandex-token @@ -34753,7 +38209,7 @@ removed from filenames during upload. Standard options -Here are the standard options specific to zoho (Zoho). +Here are the Standard options specific to zoho (Zoho). --zoho-client-id @@ -34801,12 +38257,16 @@ Properties: - Europe - "in" - India + - "jp" + - Japan + - "com.cn" + - China - "com.au" - Australia Advanced options -Here are the advanced options specific to zoho (Zoho). +Here are the Advanced options specific to zoho (Zoho). --zoho-token @@ -34858,6 +38318,22 @@ Properties: - Type: MultiEncoder - Default: Del,Ctl,InvalidUtf8 +Setting up your own client_id + +For Zoho we advise you to set up your own client_id. To do so you have +to complete the following steps. + +1. Log in to the Zoho API Console + +2. Create a new client of type "Server-based Application". The name and + website don't matter, but you must add the redirect URL + http://localhost:53682/. + +3. Once the client is created, you can go to the settings tab and + enable it in other regions. + +The client id and client secret can now be used with rclone. + Local Filesystem Local paths are specified as normal filesystem paths, e.g. @@ -35156,7 +38632,7 @@ it isn't supported (e.g. Windows) it will be ignored. Advanced options -Here are the advanced options specific to local (Local Disk). +Here are the Advanced options specific to local (Local Disk). --local-nounc @@ -35166,8 +38642,8 @@ Properties: - Config: nounc - Env Var: RCLONE_LOCAL_NOUNC -- Type: string -- Required: false +- Type: bool +- Default: false - Examples: - "true" - Disables long file names. @@ -35390,6 +38866,47 @@ Properties: - Type: MultiEncoder - Default: Slash,Dot +Metadata + +Depending on which OS is in use the local backend may return only some +of the system metadata. Setting system metadata is supported on all OSes +but setting user metadata is only supported on linux, freebsd, netbsd, +macOS and Solaris. It is not supported on Windows yet (see +pkg/attrs#47). + +User metadata is stored as extended attributes (which may not be +supported by all file systems) under the "user.*" prefix. + +Here are the possible system metadata items for the local backend. + + --------------------------------------------------------------------------------------------------- + Name Help Type Example Read Only + ----------- -------------- ------------- ------------------------------------- -------------------- + atime Time of last RFC 3339 2006-01-02T15:04:05.999999999Z07:00 N + access + + btime Time of file RFC 3339 2006-01-02T15:04:05.999999999Z07:00 N + birth + (creation) + + gid Group ID of decimal 500 N + owner number + + mode File type and octal, unix 0100664 N + mode style + + mtime Time of last RFC 3339 2006-01-02T15:04:05.999999999Z07:00 N + modification + + rdev Device ID (if hexadecimal 1abc N + special file) + + uid User ID of decimal 500 N + owner number + --------------------------------------------------------------------------------------------------- + +See the metadata docs for more info. + Backend commands Here are the commands specific to the local backend. @@ -35400,8 +38917,8 @@ Run them with The help below will explain what arguments each command takes. -See the "rclone backend" command for more info on how to pass options -and arguments. +See the backend command for more info on how to pass options and +arguments. These can be run on a running backend using the rc command backend/command. @@ -35422,6 +38939,276 @@ Options: Changelog +v1.59.0 - 2022-07-09 + +See commits + +- New backends + - Combine multiple remotes in one directory tree (Nick Craig-Wood) + - Hidrive (Ovidiu Victor Tatar) + - Internet Archive (Lesmiscore (Naoya Ozaki)) + - New S3 providers + - ArvanCloud AOS (ehsantdy) + - Cloudflare R2 (Nick Craig-Wood) + - Huawei OBS (m00594701) + - IDrive e2 (vyloy) +- New commands + - test makefile: Create a single file for testing (Nick + Craig-Wood) +- New Features + - Metadata framework to read and write system and user metadata on + backends (Nick Craig-Wood) + - Implemented initially for local, s3 and internetarchive + backends + - --metadata/-M flag to control whether metadata is copied + - --metadata-set flag to specify metadata for uploads + - Thanks to Manz Solutions for sponsoring this work. + - build + - Update to go1.18 and make go1.16 the minimum required + version (Nick Craig-Wood) + - Update android go build to 1.18.x and NDK to 23.1.7779620 + (Nick Craig-Wood) + - All windows binaries now no longer CGO (Nick Craig-Wood) + - Add linux/arm/v6 to docker images (Nick Craig-Wood) + - A huge number of fixes found with staticcheck (albertony) + - Configurable version suffix independent of version number + (albertony) + - check: Implement --no-traverse and --no-unicode-normalization + (Nick Craig-Wood) + - config: Readability improvements (albertony) + - copyurl: Add --header-filename to honor the HTTP header filename + directive (J-P Treen) + - filter: Allow multiple --exclude-if-present flags (albertony) + - fshttp: Add --disable-http-keep-alives to disable HTTP Keep + Alives (Nick Craig-Wood) + - install.sh + - Set the modes on the files and/or directories on macOS + (Michael C Tiernan - MIT-Research Computing Project) + - Pre verify sudo authorization -v before calling curl. + (Michael C Tiernan - MIT-Research Computing Project) + - lib/encoder: Add Semicolon encoding (Nick Craig-Wood) + - lsf: Add metadata support with M flag (Nick Craig-Wood) + - lsjson: Add --metadata/-M flag (Nick Craig-Wood) + - ncdu + - Implement multi selection (CrossR) + - Replace termbox with tcell's termbox wrapper (eNV25) + - Display correct path in delete confirmation dialog (Roberto + Ricci) + - operations + - Speed up hash checking by aborting the other hash if first + returns nothing (Nick Craig-Wood) + - Use correct src/dst in some log messages (zzr93) + - rcat: Check checksums by default like copy does (Nick + Craig-Wood) + - selfupdate: Replace deprecated x/crypto/openpgp package with + ProtonMail/go-crypto (albertony) + - serve ftp: Check --passive-port arguments are correct (Nick + Craig-Wood) + - size: Warn about inaccurate results when objects with unknown + size (albertony) + - sync: Overlap check is now filter-sensitive so --backup-dir can + be in the root provided it is filtered (Nick) + - test info: Check file name lengths using 1,2,3,4 byte unicode + characters (Nick Craig-Wood) + - test makefile(s): --sparse, --zero, --pattern, --ascii, + --chargen flags to control file contents (Nick Craig-Wood) + - Make sure we call the Shutdown method on backends (Martin + Czygan) +- Bug Fixes + - accounting: Fix unknown length file transfers counting 3 + transfers each (buda) + - ncdu: Fix issue where dir size is summed when file sizes are -1 + (albertony) + - sync/copy/move + - Fix --fast-list --create-empty-src-dirs and --exclude (Nick + Craig-Wood) + - Fix --max-duration and --cutoff-mode soft (Nick Craig-Wood) + - Fix fs cache unpin (Martin Czygan) + - Set proper exit code for errors that are not low-level retried + (e.g. size/timestamp changing) (albertony) +- Mount + - Support windows/arm64 (may still be problems - see #5828) (Nick + Craig-Wood) + - Log IO errors at ERROR level (Nick Craig-Wood) + - Ignore _netdev mount argument (Hugal31) +- VFS + - Add --vfs-fast-fingerprint for less accurate but faster + fingerprints (Nick Craig-Wood) + - Add --vfs-disk-space-total-size option to manually set the total + disk space (Claudio Maradonna) + - vfscache: Fix fatal error: sync: unlock of unlocked mutex error + (Nick Craig-Wood) +- Local + - Fix parsing of --local-nounc flag (Nick Craig-Wood) + - Add Metadata support (Nick Craig-Wood) +- Crypt + - Support metadata (Nick Craig-Wood) +- Azure Blob + - Calculate Chunksize/blocksize to stay below maxUploadParts + (Leroy van Logchem) + - Use chunksize lib to determine chunksize dynamically (Derek + Battams) + - Case insensitive access tier (Rob Pickerill) + - Allow remote emulator (azurite) (Lorenzo Maiorfi) +- B2 + - Add --b2-version-at flag to show file versions at time specified + (SwazRGB) + - Use chunksize lib to determine chunksize dynamically (Derek + Battams) +- Chunker + - Mark as not supporting metadata (Nick Craig-Wood) +- Compress + - Support metadata (Nick Craig-Wood) +- Drive + - Make backend config -o config add a combined AllDrives: remote + (Nick Craig-Wood) + - Make --drive-shared-with-me work with shared drives (Nick + Craig-Wood) + - Add --drive-resource-key for accessing link-shared files (Nick + Craig-Wood) + - Add backend commands exportformats and importformats for + debugging (Nick Craig-Wood) + - Fix 404 errors on copy/server side copy objects from public + folder (Nick Craig-Wood) + - Update Internal OAuth consent screen docs (Phil Shackleton) + - Moved root_folder_id to advanced section (Abhiraj) +- Dropbox + - Migrate from deprecated api (m8rge) + - Add logs to show when poll interval limits are exceeded (Nick + Craig-Wood) + - Fix nil pointer exception on dropbox impersonate user not found + (Nick Craig-Wood) +- Fichier + - Parse api error codes and them accordingly (buengese) +- FTP + - Add support for disable_utf8 option (Jason Zheng) + - Revert to upstream github.com/jlaffaye/ftp from our fork (Nick + Craig-Wood) +- Google Cloud Storage + - Add --gcs-no-check-bucket to minimise transactions and perms + (Nick Gooding) + - Add --gcs-decompress flag to decompress gzip-encoded files (Nick + Craig-Wood) + - by default these will be downloaded compressed (which + previously failed) +- Hasher + - Support metadata (Nick Craig-Wood) +- HTTP + - Fix missing response when using custom auth handler (albertony) +- Jottacloud + - Add support for upload to custom device and mountpoint + (albertony) + - Always store username in config and use it to avoid initial API + request (albertony) + - Fix issue with server-side copy when destination is in trash + (albertony) + - Fix listing output of remote with special characters (albertony) +- Mailru + - Fix timeout by using int instead of time.Duration for keeping + number of seconds (albertony) +- Mega + - Document using MEGAcmd to help with login failures (Art M. + Gallagher) +- Onedrive + - Implement --poll-interval for onedrive (Hugo Laloge) + - Add access scopes option (Sven Gerber) +- Opendrive + - Resolve lag and truncate bugs (Scott Grimes) +- Pcloud + - Fix about with no free space left (buengese) + - Fix cleanup (buengese) +- S3 + - Use PUT Object instead of presigned URLs to upload single part + objects (Nick Craig-Wood) + - Backend restore command to skip non-GLACIER objects (Vincent + Murphy) + - Use chunksize lib to determine chunksize dynamically (Derek + Battams) + - Retry RequestTimeout errors (Nick Craig-Wood) + - Implement reading and writing of metadata (Nick Craig-Wood) +- SFTP + - Add support for about and hashsum on windows server (albertony) + - Use vendor-specific VFS statistics extension for about if + available (albertony) + - Add --sftp-chunk-size to control packets sizes for high latency + links (Nick Craig-Wood) + - Add --sftp-concurrency to improve high latency transfers (Nick + Craig-Wood) + - Add --sftp-set-env option to set environment variables (Nick + Craig-Wood) + - Add Hetzner Storage Boxes to supported sftp backends (Anthrazz) +- Storj + - Fix put which lead to the file being unreadable when using mount + (Erik van Velzen) +- Union + - Add min_free_space option for lfs/eplfs policies (Nick + Craig-Wood) + - Fix uploading files to union of all bucket based remotes (Nick + Craig-Wood) + - Fix get free space for remotes which don't support it (Nick + Craig-Wood) + - Fix eplus policy to select correct entry for existing files + (Nick Craig-Wood) + - Support metadata (Nick Craig-Wood) +- Uptobox + - Fix root path handling (buengese) +- WebDAV + - Add SharePoint in other specific regions support (Noah Hsu) +- Yandex + - Handle api error on server-side move (albertony) +- Zoho + - Add Japan and China regions (buengese) + +v1.58.1 - 2022-04-29 + +See commits + +- Bug Fixes + - build: Update github.com/billziss-gh to github.com/winfsp (Nick + Craig-Wood) + - filter: Fix timezone of --min-age/-max-age from UTC to local as + documented (Nick Craig-Wood) + - rc/js: Correct RC method names (Sơn Trần-Nguyễn) + - docs + - Fix some links to command pages (albertony) + - Add --multi-thread-streams note to --transfers. (Zsolt Ero) +- Mount + - Fix --devname and fusermount: unknown option 'fsname' when + mounting via rc (Nick Craig-Wood) +- VFS + - Remove wording which suggests VFS is only for mounting (Nick + Craig-Wood) +- Dropbox + - Fix retries of multipart uploads with incorrect_offset error + (Nick Craig-Wood) +- Google Cloud Storage + - Use the s3 pacer to speed up transactions (Nick Craig-Wood) + - pacer: Default the Google pacer to a burst of 100 to fix gcs + pacing (Nick Craig-Wood) +- Jottacloud + - Fix scope in token request (albertony) +- Netstorage + - Fix unescaped HTML in documentation (Nick Craig-Wood) + - Make levels of headings consistent (Nick Craig-Wood) + - Add support contacts to netstorage doc (Nil Alexandrov) +- Onedrive + - Note that sharepoint also changes web files (.html, .aspx) (GH) +- Putio + - Handle rate limit errors (Berkan Teber) + - Fix multithread download and other ranged requests (rafma0) +- S3 + - Add ChinaMobile EOS to provider list (GuoXingbin) + - Sync providers in config description with providers (Nick + Craig-Wood) +- SFTP + - Fix OpenSSH 8.8+ RSA keys incompatibility (KARBOWSKI Piotr) + - Note that Scaleway C14 is deprecating SFTP in favor of S3 + (Adrien Rey-Jarthon) +- Storj + - Fix bucket creation on Move (Nick Craig-Wood) +- WebDAV + - Don't override Referer if user sets it (Nick Craig-Wood) + v1.58.0 - 2022-03-18 See commits @@ -40460,7 +44247,7 @@ node running rclone would need to have lots of bandwidth. The syncs would be incremental (on a file by file basis). -Eg +e.g. rclone sync -i drive:Folder s3:bucket @@ -40543,7 +44330,7 @@ e.g. export no_proxy=localhost,127.0.0.0/8,my.host.name export NO_PROXY=$no_proxy -Note that the ftp backend does not support ftp_proxy yet. +Note that the FTP backend does not support ftp_proxy yet. Rclone gives x509: failed to load system roots and no roots provided error @@ -41251,6 +45038,57 @@ email addresses removed from here need to be addeed to bin/.ignore-emails to mak - Vincent Murphy vdm@vdm.ie - ctrl-q 34975747+ctrl-q@users.noreply.github.com - Nil Alexandrov nalexand@akamai.com +- GuoXingbin 101376330+guoxingbin@users.noreply.github.com +- Berkan Teber berkan@berkanteber.com +- Tobias Klauser tklauser@distanz.ch +- KARBOWSKI Piotr piotr.karbowski@gmail.com +- GH geeklihui@foxmail.com +- rafma0 int.main@gmail.com +- Adrien Rey-Jarthon jobs@adrienjarthon.com +- Nick Gooding 73336146+nickgooding@users.noreply.github.com +- Leroy van Logchem lr.vanlogchem@gmail.com +- Zsolt Ero zsolt.ero@gmail.com +- Lesmiscore nao20010128@gmail.com +- ehsantdy ehsan.tadayon@arvancloud.com +- SwazRGB 65694696+swazrgb@users.noreply.github.com +- Mateusz Puczyński mati6095@gmail.com +- Michael C Tiernan - MIT-Research Computing Project mtiernan@mit.edu +- Kaspian 34658474+KaspianDev@users.noreply.github.com +- Werner EvilOlaf@users.noreply.github.com +- Hugal31 hugo.laloge@gmail.com +- Christian Galo 36752715+cgalo5758@users.noreply.github.com +- Erik van Velzen erik@evanv.nl +- Derek Battams derek@battams.ca +- SimonLiu simonliu009@users.noreply.github.com +- Hugo Laloge hla@lescompanions.com +- Mr-Kanister 68117355+Mr-Kanister@users.noreply.github.com +- Rob Pickerill r.pickerill@gmail.com +- Andrey to.merge@gmail.com +- Eric Wolf 19wolf@gmail.com +- Nick nick.naumann@mailbox.tu-dresden.de +- Jason Zheng jszheng17@gmail.com +- Matthew Vernon mvernon@wikimedia.org +- Noah Hsu i@nn.ci +- m00594701 mengpengbo@huawei.com +- Art M. Gallagher artmg50@gmail.com +- Sven Gerber 49589423+svengerber@users.noreply.github.com +- CrossR r.cross@lancaster.ac.uk +- Maciej Radzikowski maciej@radzikowski.com.pl +- Scott Grimes scott.grimes@spaciq.com +- Phil Shackleton 71221528+philshacks@users.noreply.github.com +- eNV25 env252525@gmail.com +- Caleb inventor96@users.noreply.github.com +- J-P Treen jp@wraptious.com +- Martin Czygan 53705+miku@users.noreply.github.com +- buda sandrojijavadze@protonmail.com +- mirekphd 36706320+mirekphd@users.noreply.github.com +- vyloy vyloy@qq.com +- Anthrazz 25553648+Anthrazz@users.noreply.github.com +- zzr93 34027824+zzr93@users.noreply.github.com +- Paul Norman penorman@mac.com +- Lorenzo Maiorfi maiorfi@gmail.com +- Claudio Maradonna penguyman@stronzi.org +- Ovidiu Victor Tatar ovi.tatar@googlemail.com Contact the rclone project diff --git a/docs/content/alias.md b/docs/content/alias.md index 69278aa3bf536..aa041f6f4eed1 100644 --- a/docs/content/alias.md +++ b/docs/content/alias.md @@ -91,7 +91,7 @@ Copy another local directory to the alias directory called source {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/alias/alias.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to alias (Alias for an existing remote). +Here are the Standard options specific to alias (Alias for an existing remote). #### --alias-remote diff --git a/docs/content/amazonclouddrive.md b/docs/content/amazonclouddrive.md index 478caa5caebc0..74ff53df6c61a 100644 --- a/docs/content/amazonclouddrive.md +++ b/docs/content/amazonclouddrive.md @@ -160,7 +160,7 @@ rclone it will take you to an `amazon.com` page to log in. Your {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/amazonclouddrive/amazonclouddrive.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to amazon cloud drive (Amazon Drive). +Here are the Standard options specific to amazon cloud drive (Amazon Drive). #### --acd-client-id @@ -190,7 +190,7 @@ Properties: ### Advanced options -Here are the advanced options specific to amazon cloud drive (Amazon Drive). +Here are the Advanced options specific to amazon cloud drive (Amazon Drive). #### --acd-token diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md index fa2205ae58993..b9841e0e3256e 100644 --- a/docs/content/azureblob.md +++ b/docs/content/azureblob.md @@ -158,7 +158,7 @@ untrusted environment such as a CI build server. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/azureblob/azureblob.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to azureblob (Microsoft Azure Blob Storage). +Here are the Standard options specific to azureblob (Microsoft Azure Blob Storage). #### --azureblob-account @@ -255,7 +255,7 @@ Properties: ### Advanced options -Here are the advanced options specific to azureblob (Microsoft Azure Blob Storage). +Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage). #### --azureblob-msi-object-id diff --git a/docs/content/b2.md b/docs/content/b2.md index 1df9298d41754..bbc8a17c648c0 100644 --- a/docs/content/b2.md +++ b/docs/content/b2.md @@ -328,7 +328,7 @@ https://f002.backblazeb2.com/file/bucket/path/folder/file3?Authorization=xxxxxxx {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/b2/b2.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to b2 (Backblaze B2). +Here are the Standard options specific to b2 (Backblaze B2). #### --b2-account @@ -365,7 +365,7 @@ Properties: ### Advanced options -Here are the advanced options specific to b2 (Backblaze B2). +Here are the Advanced options specific to b2 (Backblaze B2). #### --b2-endpoint @@ -415,6 +415,20 @@ Properties: - Type: bool - Default: false +#### --b2-version-at + +Show file versions as they were at the specified time. + +Note that when using this no file write operations are permitted, +so you can't upload files or delete them. + +Properties: + +- Config: version_at +- Env Var: RCLONE_B2_VERSION_AT +- Type: Time +- Default: off + #### --b2-upload-cutoff Cutoff for switching to chunked upload. diff --git a/docs/content/box.md b/docs/content/box.md index d0cdb603d5469..9da20764a03de 100644 --- a/docs/content/box.md +++ b/docs/content/box.md @@ -267,7 +267,7 @@ the `root_folder_id` in the config. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/box/box.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to box (Box). +Here are the Standard options specific to box (Box). #### --box-client-id @@ -341,7 +341,7 @@ Properties: ### Advanced options -Here are the advanced options specific to box (Box). +Here are the Advanced options specific to box (Box). #### --box-token diff --git a/docs/content/cache.md b/docs/content/cache.md index 76e8fa4f0c8b7..592f9eef4f2f0 100644 --- a/docs/content/cache.md +++ b/docs/content/cache.md @@ -307,7 +307,7 @@ Params: {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/cache/cache.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to cache (Cache a remote). +Here are the Standard options specific to cache (Cache a remote). #### --cache-remote @@ -423,7 +423,7 @@ Properties: ### Advanced options -Here are the advanced options specific to cache (Cache a remote). +Here are the Advanced options specific to cache (Cache a remote). #### --cache-plex-token @@ -672,7 +672,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](/commands/rclone_backend/) for more +See the [backend](/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command diff --git a/docs/content/changelog.md b/docs/content/changelog.md index fab1db463b877..1cb4255b63fba 100644 --- a/docs/content/changelog.md +++ b/docs/content/changelog.md @@ -5,6 +5,165 @@ description: "Rclone Changelog" # Changelog +## v1.59.0 - 2022-07-09 + +[See commits](https://github.com/rclone/rclone/compare/v1.58.0...v1.59.0) + +* New backends + * [Combine](/combine) multiple remotes in one directory tree (Nick Craig-Wood) + * [Hidrive](/hidrive/) (Ovidiu Victor Tatar) + * [Internet Archive](/internetarchive/) (Lesmiscore (Naoya Ozaki)) + * New S3 providers + * [ArvanCloud AOS](/s3/#arvan-cloud) (ehsantdy) + * [Cloudflare R2](/s3/#cloudflare-r2) (Nick Craig-Wood) + * [Huawei OBS](/s3/#huawei-obs) (m00594701) + * [IDrive e2](/s3/#idrive-e2) (vyloy) +* New commands + * [test makefile](/commands/rclone_test_makefile/): Create a single file for testing (Nick Craig-Wood) +* New Features + * [Metadata framework](/docs/#metadata) to read and write system and user metadata on backends (Nick Craig-Wood) + * Implemented initially for `local`, `s3` and `internetarchive` backends + * `--metadata`/`-M` flag to control whether metadata is copied + * `--metadata-set` flag to specify metadata for uploads + * Thanks to [Manz Solutions](https://manz-solutions.at/) for sponsoring this work. + * build + * Update to go1.18 and make go1.16 the minimum required version (Nick Craig-Wood) + * Update android go build to 1.18.x and NDK to 23.1.7779620 (Nick Craig-Wood) + * All windows binaries now no longer CGO (Nick Craig-Wood) + * Add `linux/arm/v6` to docker images (Nick Craig-Wood) + * A huge number of fixes found with [staticcheck](https://staticcheck.io/) (albertony) + * Configurable version suffix independent of version number (albertony) + * check: Implement `--no-traverse` and `--no-unicode-normalization` (Nick Craig-Wood) + * config: Readability improvements (albertony) + * copyurl: Add `--header-filename` to honor the HTTP header filename directive (J-P Treen) + * filter: Allow multiple `--exclude-if-present` flags (albertony) + * fshttp: Add `--disable-http-keep-alives` to disable HTTP Keep Alives (Nick Craig-Wood) + * install.sh + * Set the modes on the files and/or directories on macOS (Michael C Tiernan - MIT-Research Computing Project) + * Pre verify sudo authorization `-v` before calling curl. (Michael C Tiernan - MIT-Research Computing Project) + * lib/encoder: Add Semicolon encoding (Nick Craig-Wood) + * lsf: Add metadata support with `M` flag (Nick Craig-Wood) + * lsjson: Add `--metadata`/`-M` flag (Nick Craig-Wood) + * ncdu + * Implement multi selection (CrossR) + * Replace termbox with tcell's termbox wrapper (eNV25) + * Display correct path in delete confirmation dialog (Roberto Ricci) + * operations + * Speed up hash checking by aborting the other hash if first returns nothing (Nick Craig-Wood) + * Use correct src/dst in some log messages (zzr93) + * rcat: Check checksums by default like copy does (Nick Craig-Wood) + * selfupdate: Replace deprecated `x/crypto/openpgp` package with `ProtonMail/go-crypto` (albertony) + * serve ftp: Check `--passive-port` arguments are correct (Nick Craig-Wood) + * size: Warn about inaccurate results when objects with unknown size (albertony) + * sync: Overlap check is now filter-sensitive so `--backup-dir` can be in the root provided it is filtered (Nick) + * test info: Check file name lengths using 1,2,3,4 byte unicode characters (Nick Craig-Wood) + * test makefile(s): `--sparse`, `--zero`, `--pattern`, `--ascii`, `--chargen` flags to control file contents (Nick Craig-Wood) + * Make sure we call the `Shutdown` method on backends (Martin Czygan) +* Bug Fixes + * accounting: Fix unknown length file transfers counting 3 transfers each (buda) + * ncdu: Fix issue where dir size is summed when file sizes are -1 (albertony) + * sync/copy/move + * Fix `--fast-list` `--create-empty-src-dirs` and `--exclude` (Nick Craig-Wood) + * Fix `--max-duration` and `--cutoff-mode soft` (Nick Craig-Wood) + * Fix fs cache unpin (Martin Czygan) + * Set proper exit code for errors that are not low-level retried (e.g. size/timestamp changing) (albertony) +* Mount + * Support `windows/arm64` (may still be problems - see [#5828](https://github.com/rclone/rclone/issues/5828)) (Nick Craig-Wood) + * Log IO errors at ERROR level (Nick Craig-Wood) + * Ignore `_netdev` mount argument (Hugal31) +* VFS + * Add `--vfs-fast-fingerprint` for less accurate but faster fingerprints (Nick Craig-Wood) + * Add `--vfs-disk-space-total-size` option to manually set the total disk space (Claudio Maradonna) + * vfscache: Fix fatal error: sync: unlock of unlocked mutex error (Nick Craig-Wood) +* Local + * Fix parsing of `--local-nounc` flag (Nick Craig-Wood) + * Add Metadata support (Nick Craig-Wood) +* Crypt + * Support metadata (Nick Craig-Wood) +* Azure Blob + * Calculate Chunksize/blocksize to stay below maxUploadParts (Leroy van Logchem) + * Use chunksize lib to determine chunksize dynamically (Derek Battams) + * Case insensitive access tier (Rob Pickerill) + * Allow remote emulator (azurite) (Lorenzo Maiorfi) +* B2 + * Add `--b2-version-at` flag to show file versions at time specified (SwazRGB) + * Use chunksize lib to determine chunksize dynamically (Derek Battams) +* Chunker + * Mark as not supporting metadata (Nick Craig-Wood) +* Compress + * Support metadata (Nick Craig-Wood) +* Drive + * Make `backend config -o config` add a combined `AllDrives:` remote (Nick Craig-Wood) + * Make `--drive-shared-with-me` work with shared drives (Nick Craig-Wood) + * Add `--drive-resource-key` for accessing link-shared files (Nick Craig-Wood) + * Add backend commands `exportformats` and `importformats` for debugging (Nick Craig-Wood) + * Fix 404 errors on copy/server side copy objects from public folder (Nick Craig-Wood) + * Update Internal OAuth consent screen docs (Phil Shackleton) + * Moved `root_folder_id` to advanced section (Abhiraj) +* Dropbox + * Migrate from deprecated api (m8rge) + * Add logs to show when poll interval limits are exceeded (Nick Craig-Wood) + * Fix nil pointer exception on dropbox impersonate user not found (Nick Craig-Wood) +* Fichier + * Parse api error codes and them accordingly (buengese) +* FTP + * Add support for `disable_utf8` option (Jason Zheng) + * Revert to upstream `github.com/jlaffaye/ftp` from our fork (Nick Craig-Wood) +* Google Cloud Storage + * Add `--gcs-no-check-bucket` to minimise transactions and perms (Nick Gooding) + * Add `--gcs-decompress` flag to decompress gzip-encoded files (Nick Craig-Wood) + * by default these will be downloaded compressed (which previously failed) +* Hasher + * Support metadata (Nick Craig-Wood) +* HTTP + * Fix missing response when using custom auth handler (albertony) +* Jottacloud + * Add support for upload to custom device and mountpoint (albertony) + * Always store username in config and use it to avoid initial API request (albertony) + * Fix issue with server-side copy when destination is in trash (albertony) + * Fix listing output of remote with special characters (albertony) +* Mailru + * Fix timeout by using int instead of time.Duration for keeping number of seconds (albertony) +* Mega + * Document using MEGAcmd to help with login failures (Art M. Gallagher) +* Onedrive + * Implement `--poll-interval` for onedrive (Hugo Laloge) + * Add access scopes option (Sven Gerber) +* Opendrive + * Resolve lag and truncate bugs (Scott Grimes) +* Pcloud + * Fix about with no free space left (buengese) + * Fix cleanup (buengese) +* S3 + * Use PUT Object instead of presigned URLs to upload single part objects (Nick Craig-Wood) + * Backend restore command to skip non-GLACIER objects (Vincent Murphy) + * Use chunksize lib to determine chunksize dynamically (Derek Battams) + * Retry RequestTimeout errors (Nick Craig-Wood) + * Implement reading and writing of metadata (Nick Craig-Wood) +* SFTP + * Add support for about and hashsum on windows server (albertony) + * Use vendor-specific VFS statistics extension for about if available (albertony) + * Add `--sftp-chunk-size` to control packets sizes for high latency links (Nick Craig-Wood) + * Add `--sftp-concurrency` to improve high latency transfers (Nick Craig-Wood) + * Add `--sftp-set-env` option to set environment variables (Nick Craig-Wood) + * Add Hetzner Storage Boxes to supported sftp backends (Anthrazz) +* Storj + * Fix put which lead to the file being unreadable when using mount (Erik van Velzen) +* Union + * Add `min_free_space` option for `lfs`/`eplfs` policies (Nick Craig-Wood) + * Fix uploading files to union of all bucket based remotes (Nick Craig-Wood) + * Fix get free space for remotes which don't support it (Nick Craig-Wood) + * Fix `eplus` policy to select correct entry for existing files (Nick Craig-Wood) + * Support metadata (Nick Craig-Wood) +* Uptobox + * Fix root path handling (buengese) +* WebDAV + * Add SharePoint in other specific regions support (Noah Hsu) +* Yandex + * Handle api error on server-side move (albertony) +* Zoho + * Add Japan and China regions (buengese) + ## v1.58.1 - 2022-04-29 [See commits](https://github.com/rclone/rclone/compare/v1.58.0...v1.58.1) diff --git a/docs/content/chunker.md b/docs/content/chunker.md index a4ee805d0db2f..4ba8491db71ed 100644 --- a/docs/content/chunker.md +++ b/docs/content/chunker.md @@ -313,7 +313,7 @@ Changing `transactions` is dangerous and requires explicit migration. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/chunker/chunker.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to chunker (Transparently chunk/split large files). +Here are the Standard options specific to chunker (Transparently chunk/split large files). #### --chunker-remote @@ -372,7 +372,7 @@ Properties: ### Advanced options -Here are the advanced options specific to chunker (Transparently chunk/split large files). +Here are the Advanced options specific to chunker (Transparently chunk/split large files). #### --chunker-name-format diff --git a/docs/content/combine.md b/docs/content/combine.md index 98e174a04de3e..c3b5ed322a5d7 100644 --- a/docs/content/combine.md +++ b/docs/content/combine.md @@ -127,7 +127,7 @@ See [the Google Drive docs](/drive/#drives) for full info. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/combine/combine.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to combine (Combine several remotes into one). +Here are the Standard options specific to combine (Combine several remotes into one). #### --combine-upstreams @@ -153,4 +153,10 @@ Properties: - Type: SpaceSepList - Default: +### Metadata + +Any metadata supported by the underlying remote is read and written. + +See the [metadata](/docs/#metadata) docs for more info. + {{< rem autogenerated options stop >}} diff --git a/docs/content/commands/rclone.md b/docs/content/commands/rclone.md index e1d88dd8e999e..e98dc744369ed 100644 --- a/docs/content/commands/rclone.md +++ b/docs/content/commands/rclone.md @@ -42,7 +42,7 @@ See the [global flags page](/flags/) for global options not listed here. * [rclone check](/commands/rclone_check/) - Checks the files in the source and destination match. * [rclone checksum](/commands/rclone_checksum/) - Checks the files in the source against a SUM file. * [rclone cleanup](/commands/rclone_cleanup/) - Clean up the remote if possible. -* [rclone completion](/commands/rclone_completion/) - generate the autocompletion script for the specified shell +* [rclone completion](/commands/rclone_completion/) - Generate the autocompletion script for the specified shell * [rclone config](/commands/rclone_config/) - Enter an interactive configuration session. * [rclone copy](/commands/rclone_copy/) - Copy files from source to dest, skipping identical files. * [rclone copyto](/commands/rclone_copyto/) - Copy files from source to dest, skipping identical files. diff --git a/docs/content/commands/rclone_check.md b/docs/content/commands/rclone_check.md index 25a27ab83ff25..112633fabbcc2 100644 --- a/docs/content/commands/rclone_check.md +++ b/docs/content/commands/rclone_check.md @@ -16,6 +16,10 @@ Checks the files in the source and destination match. It compares sizes and hashes (MD5 or SHA1) and logs a report of files that don't match. It doesn't alter the source or destination. +For the [crypt](/crypt/) remote there is a dedicated command, +[cryptcheck](/commands/rclone_cryptcheck/), that are able to check +the checksums of the crypted files. + If you supply the `--size-only` flag, it will only compare the sizes not the hashes as well. Use this for a quick check. diff --git a/docs/content/commands/rclone_completion.md b/docs/content/commands/rclone_completion.md index b30d71dfe482f..9193c9868f2ee 100644 --- a/docs/content/commands/rclone_completion.md +++ b/docs/content/commands/rclone_completion.md @@ -1,17 +1,16 @@ --- title: "rclone completion" -description: "generate the autocompletion script for the specified shell" +description: "Generate the autocompletion script for the specified shell" slug: rclone_completion url: /commands/rclone_completion/ # autogenerated - DO NOT EDIT, instead edit the source code in cmd/completion/ and as part of making a release run "make commanddocs" --- # rclone completion -generate the autocompletion script for the specified shell +Generate the autocompletion script for the specified shell ## Synopsis - Generate the autocompletion script for rclone for the specified shell. See each sub-command's help for details on how to use the generated script. @@ -27,8 +26,8 @@ See the [global flags page](/flags/) for global options not listed here. ## SEE ALSO * [rclone](/commands/rclone/) - Show help for rclone commands, flags and backends. -* [rclone completion bash](/commands/rclone_completion_bash/) - generate the autocompletion script for bash -* [rclone completion fish](/commands/rclone_completion_fish/) - generate the autocompletion script for fish -* [rclone completion powershell](/commands/rclone_completion_powershell/) - generate the autocompletion script for powershell -* [rclone completion zsh](/commands/rclone_completion_zsh/) - generate the autocompletion script for zsh +* [rclone completion bash](/commands/rclone_completion_bash/) - Generate the autocompletion script for bash +* [rclone completion fish](/commands/rclone_completion_fish/) - Generate the autocompletion script for fish +* [rclone completion powershell](/commands/rclone_completion_powershell/) - Generate the autocompletion script for powershell +* [rclone completion zsh](/commands/rclone_completion_zsh/) - Generate the autocompletion script for zsh diff --git a/docs/content/commands/rclone_completion_bash.md b/docs/content/commands/rclone_completion_bash.md index ce59b639ff7ff..b3c24a6e44682 100644 --- a/docs/content/commands/rclone_completion_bash.md +++ b/docs/content/commands/rclone_completion_bash.md @@ -1,33 +1,37 @@ --- title: "rclone completion bash" -description: "generate the autocompletion script for bash" +description: "Generate the autocompletion script for bash" slug: rclone_completion_bash url: /commands/rclone_completion_bash/ # autogenerated - DO NOT EDIT, instead edit the source code in cmd/completion/bash/ and as part of making a release run "make commanddocs" --- # rclone completion bash -generate the autocompletion script for bash +Generate the autocompletion script for bash ## Synopsis - Generate the autocompletion script for the bash shell. This script depends on the 'bash-completion' package. If it is not installed already, you can install it via your OS's package manager. To load completions in your current shell session: -$ source <(rclone completion bash) + + source <(rclone completion bash) To load completions for every new session, execute once: -Linux: - $ rclone completion bash > /etc/bash_completion.d/rclone -MacOS: - $ rclone completion bash > /usr/local/etc/bash_completion.d/rclone + +### Linux: + + rclone completion bash > /etc/bash_completion.d/rclone + +### macOS: + + rclone completion bash > /usr/local/etc/bash_completion.d/rclone You will need to start a new shell for this setup to take effect. - + ``` rclone completion bash @@ -44,5 +48,5 @@ See the [global flags page](/flags/) for global options not listed here. ## SEE ALSO -* [rclone completion](/commands/rclone_completion/) - generate the autocompletion script for the specified shell +* [rclone completion](/commands/rclone_completion/) - Generate the autocompletion script for the specified shell diff --git a/docs/content/commands/rclone_completion_fish.md b/docs/content/commands/rclone_completion_fish.md index 62645463b18a5..5e09dadfbb915 100644 --- a/docs/content/commands/rclone_completion_fish.md +++ b/docs/content/commands/rclone_completion_fish.md @@ -1,24 +1,25 @@ --- title: "rclone completion fish" -description: "generate the autocompletion script for fish" +description: "Generate the autocompletion script for fish" slug: rclone_completion_fish url: /commands/rclone_completion_fish/ # autogenerated - DO NOT EDIT, instead edit the source code in cmd/completion/fish/ and as part of making a release run "make commanddocs" --- # rclone completion fish -generate the autocompletion script for fish +Generate the autocompletion script for fish ## Synopsis - Generate the autocompletion script for the fish shell. To load completions in your current shell session: -$ rclone completion fish | source + + rclone completion fish | source To load completions for every new session, execute once: -$ rclone completion fish > ~/.config/fish/completions/rclone.fish + + rclone completion fish > ~/.config/fish/completions/rclone.fish You will need to start a new shell for this setup to take effect. @@ -38,5 +39,5 @@ See the [global flags page](/flags/) for global options not listed here. ## SEE ALSO -* [rclone completion](/commands/rclone_completion/) - generate the autocompletion script for the specified shell +* [rclone completion](/commands/rclone_completion/) - Generate the autocompletion script for the specified shell diff --git a/docs/content/commands/rclone_completion_powershell.md b/docs/content/commands/rclone_completion_powershell.md index 9bd523e76e83d..8dfbafc458f9d 100644 --- a/docs/content/commands/rclone_completion_powershell.md +++ b/docs/content/commands/rclone_completion_powershell.md @@ -1,21 +1,21 @@ --- title: "rclone completion powershell" -description: "generate the autocompletion script for powershell" +description: "Generate the autocompletion script for powershell" slug: rclone_completion_powershell url: /commands/rclone_completion_powershell/ # autogenerated - DO NOT EDIT, instead edit the source code in cmd/completion/powershell/ and as part of making a release run "make commanddocs" --- # rclone completion powershell -generate the autocompletion script for powershell +Generate the autocompletion script for powershell ## Synopsis - Generate the autocompletion script for powershell. To load completions in your current shell session: -PS C:\> rclone completion powershell | Out-String | Invoke-Expression + + rclone completion powershell | Out-String | Invoke-Expression To load completions for every new session, add the output of the above command to your powershell profile. @@ -36,5 +36,5 @@ See the [global flags page](/flags/) for global options not listed here. ## SEE ALSO -* [rclone completion](/commands/rclone_completion/) - generate the autocompletion script for the specified shell +* [rclone completion](/commands/rclone_completion/) - Generate the autocompletion script for the specified shell diff --git a/docs/content/commands/rclone_completion_zsh.md b/docs/content/commands/rclone_completion_zsh.md index 2e487e67438dd..b48faa25a43fd 100644 --- a/docs/content/commands/rclone_completion_zsh.md +++ b/docs/content/commands/rclone_completion_zsh.md @@ -1,29 +1,32 @@ --- title: "rclone completion zsh" -description: "generate the autocompletion script for zsh" +description: "Generate the autocompletion script for zsh" slug: rclone_completion_zsh url: /commands/rclone_completion_zsh/ # autogenerated - DO NOT EDIT, instead edit the source code in cmd/completion/zsh/ and as part of making a release run "make commanddocs" --- # rclone completion zsh -generate the autocompletion script for zsh +Generate the autocompletion script for zsh ## Synopsis - Generate the autocompletion script for the zsh shell. If shell completion is not already enabled in your environment you will need to enable it. You can execute the following once: -$ echo "autoload -U compinit; compinit" >> ~/.zshrc + echo "autoload -U compinit; compinit" >> ~/.zshrc To load completions for every new session, execute once: -# Linux: -$ rclone completion zsh > "${fpath[1]}/_rclone" -# macOS: -$ rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone + +### Linux: + + rclone completion zsh > "${fpath[1]}/_rclone" + +### macOS: + + rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone You will need to start a new shell for this setup to take effect. @@ -43,5 +46,5 @@ See the [global flags page](/flags/) for global options not listed here. ## SEE ALSO -* [rclone completion](/commands/rclone_completion/) - generate the autocompletion script for the specified shell +* [rclone completion](/commands/rclone_completion/) - Generate the autocompletion script for the specified shell diff --git a/docs/content/commands/rclone_copy.md b/docs/content/commands/rclone_copy.md index 163b0289b102c..022f3e7b08e5f 100644 --- a/docs/content/commands/rclone_copy.md +++ b/docs/content/commands/rclone_copy.md @@ -14,13 +14,18 @@ Copy files from source to dest, skipping identical files. Copy the source to the destination. Does not transfer files that are identical on source and destination, testing by size and modification -time or MD5SUM. Doesn't delete files from the destination. +time or MD5SUM. Doesn't delete files from the destination. If you +want to also delete files from destination, to make it match source, +use the [sync](/commands/rclone_sync/) command instead. Note that it is always the contents of the directory that is synced, -not the directory so when source:path is a directory, it's the +not the directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. +To copy single files, use the [copyto](/commands/rclone_copyto/) +command instead. + If dest:path doesn't exist, it is created and the source:path contents go there. diff --git a/docs/content/commands/rclone_copyto.md b/docs/content/commands/rclone_copyto.md index 93da1fea879d8..a5cfa974e4d93 100644 --- a/docs/content/commands/rclone_copyto.md +++ b/docs/content/commands/rclone_copyto.md @@ -16,8 +16,8 @@ If source:path is a file or directory then it copies it to a file or directory named dest:path. This can be used to upload single files to other than their current -name. If the source is a directory then it acts exactly like the copy -command. +name. If the source is a directory then it acts exactly like the +[copy](/commands/rclone_copy/) command. So diff --git a/docs/content/commands/rclone_copyurl.md b/docs/content/commands/rclone_copyurl.md index 928bec5995af7..1188f0647b9ce 100644 --- a/docs/content/commands/rclone_copyurl.md +++ b/docs/content/commands/rclone_copyurl.md @@ -15,10 +15,11 @@ Copy url content to dest. Download a URL's content and copy it to the destination without saving it in temporary storage. -Setting `--auto-filename` will cause the file name to be retrieved from -the URL (after any redirections) and used in the destination -path. With `--print-filename` in addition, the resulting file name will -be printed. +Setting `--auto-filename` will attempt to automatically determine the filename from the URL +(after any redirections) and used in the destination path. +With `--auto-filename-header` in +addition, if a specific filename is set in HTTP headers, it will be used instead of the name from the URL. +With `--print-filename` in addition, the resulting file name will be printed. Setting `--no-clobber` will prevent overwriting file on the destination if there is one with the same name. @@ -34,11 +35,12 @@ rclone copyurl https://example.com dest:path [flags] ## Options ``` - -a, --auto-filename Get the file name from the URL and use it for destination file path - -h, --help help for copyurl - --no-clobber Prevent overwriting file with same name - -p, --print-filename Print the resulting name from --auto-filename - --stdout Write the output to stdout rather than a file + -a, --auto-filename Get the file name from the URL and use it for destination file path + --header-filename Get the file name from the Content-Disposition header + -h, --help help for copyurl + --no-clobber Prevent overwriting file with same name + -p, --print-filename Print the resulting name from --auto-filename + --stdout Write the output to stdout rather than a file ``` See the [global flags page](/flags/) for global options not listed here. diff --git a/docs/content/commands/rclone_cryptcheck.md b/docs/content/commands/rclone_cryptcheck.md index cf4d701ad961b..3e81e53e9b8be 100644 --- a/docs/content/commands/rclone_cryptcheck.md +++ b/docs/content/commands/rclone_cryptcheck.md @@ -12,9 +12,9 @@ Cryptcheck checks the integrity of a crypted remote. ## Synopsis -rclone cryptcheck checks a remote against a crypted remote. This is -the equivalent of running rclone check, but able to check the -checksums of the crypted remote. +rclone cryptcheck checks a remote against a [crypted](/crypt/) remote. +This is the equivalent of running rclone [check](/commands/rclone_check/), +but able to check the checksums of the crypted remote. For it to work the underlying remote of the cryptedremote must support some kind of checksum. diff --git a/docs/content/commands/rclone_cryptdecode.md b/docs/content/commands/rclone_cryptdecode.md index b2e5623f0915e..c4836ec8d583a 100644 --- a/docs/content/commands/rclone_cryptdecode.md +++ b/docs/content/commands/rclone_cryptdecode.md @@ -15,7 +15,7 @@ Cryptdecode returns unencrypted file names. rclone cryptdecode returns unencrypted file names when provided with a list of encrypted file names. List limit is 10 items. -If you supply the --reverse flag, it will return encrypted file names. +If you supply the `--reverse` flag, it will return encrypted file names. use it like this @@ -23,8 +23,8 @@ use it like this rclone cryptdecode --reverse encryptedremote: filename1 filename2 -Another way to accomplish this is by using the `rclone backend encode` (or `decode`)command. -See the documentation on the `crypt` overlay for more info. +Another way to accomplish this is by using the `rclone backend encode` (or `decode`) command. +See the documentation on the [crypt](/crypt/) overlay for more info. ``` diff --git a/docs/content/commands/rclone_dedupe.md b/docs/content/commands/rclone_dedupe.md index 345bec0d05499..6b77f17cbf317 100644 --- a/docs/content/commands/rclone_dedupe.md +++ b/docs/content/commands/rclone_dedupe.md @@ -22,7 +22,7 @@ Opendrive) that can have duplicate file names. It can be run on wrapping backend (e.g. crypt) if they wrap a backend which supports duplicate file names. -However if --by-hash is passed in then dedupe will find files with +However if `--by-hash` is passed in then dedupe will find files with duplicate hashes instead which will work on any backend which supports at least one hash. This can be used to find files with duplicate content. This is known as deduping by hash. diff --git a/docs/content/commands/rclone_delete.md b/docs/content/commands/rclone_delete.md index 0951011f5bf7e..09076a46ae028 100644 --- a/docs/content/commands/rclone_delete.md +++ b/docs/content/commands/rclone_delete.md @@ -12,16 +12,16 @@ Remove the files in path. ## Synopsis -Remove the files in path. Unlike `purge` it obeys include/exclude -filters so can be used to selectively delete files. +Remove the files in path. Unlike [purge](/commands/rclone_purge/) it +obeys include/exclude filters so can be used to selectively delete files. `rclone delete` only deletes files but leaves the directory structure alone. If you want to delete a directory and all of its contents use -the `purge` command. +the [purge](/commands/rclone_purge/) command. If you supply the `--rmdirs` flag, it will remove all empty directories along with it. -You can also use the separate command `rmdir` or `rmdirs` to -delete empty directories only. +You can also use the separate command [rmdir](/commands/rclone_rmdir/) or +[rmdirs](/commands/rclone_rmdirs/) to delete empty directories only. For example, to delete all files bigger than 100 MiB, you may first want to check what would be deleted (use either): diff --git a/docs/content/commands/rclone_genautocomplete.md b/docs/content/commands/rclone_genautocomplete.md index c55828cc1fcf0..3838dda4a8bd7 100644 --- a/docs/content/commands/rclone_genautocomplete.md +++ b/docs/content/commands/rclone_genautocomplete.md @@ -13,7 +13,7 @@ Output completion script for a given shell. Generates a shell completion script for rclone. -Run with --help to list the supported shells. +Run with `--help` to list the supported shells. ## Options diff --git a/docs/content/commands/rclone_hashsum.md b/docs/content/commands/rclone_hashsum.md index 4c85d8b663006..0c34c0cd65bd0 100644 --- a/docs/content/commands/rclone_hashsum.md +++ b/docs/content/commands/rclone_hashsum.md @@ -21,6 +21,9 @@ not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling any hash for any remote. +For the MD5 and SHA1 algorithms there are also dedicated commands, +[md5sum](/commands/rclone_md5sum/) and [sha1sum](/commands/rclone_sha1sum/). + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, @@ -36,6 +39,7 @@ Run without a hash to see the list of all supported hashes, e.g. * crc32 * sha256 * dropbox + * hidrive * mailru * quickxor diff --git a/docs/content/commands/rclone_listremotes.md b/docs/content/commands/rclone_listremotes.md index 98fe86311029b..54e4317c8e954 100644 --- a/docs/content/commands/rclone_listremotes.md +++ b/docs/content/commands/rclone_listremotes.md @@ -14,7 +14,7 @@ List all the remotes in the config file. rclone listremotes lists all the available remotes from the config file. -When uses with the -l flag it lists the types too. +When used with the `--long` flag it lists the types too. ``` diff --git a/docs/content/commands/rclone_lsd.md b/docs/content/commands/rclone_lsd.md index cccd67b0fd7af..fbd9b2c92f1fc 100644 --- a/docs/content/commands/rclone_lsd.md +++ b/docs/content/commands/rclone_lsd.md @@ -13,7 +13,7 @@ List all directories/containers/buckets in the path. Lists the directories in the source path to standard output. Does not -recurse by default. Use the -R flag to recurse. +recurse by default. Use the `-R` flag to recurse. This command lists the total size of the directory (if known, -1 if not), the modification time (if known, the current time if not), the @@ -31,7 +31,7 @@ Or -1 2017-01-03 14:40:54 -1 2500files -1 2017-07-08 14:39:28 -1 4000files -If you just want the directory names use "rclone lsf --dirs-only". +If you just want the directory names use `rclone lsf --dirs-only`. Any of the filtering options can be applied to this command. diff --git a/docs/content/commands/rclone_lsf.md b/docs/content/commands/rclone_lsf.md index a48a737fe84d8..2cdd3ce5cf2c8 100644 --- a/docs/content/commands/rclone_lsf.md +++ b/docs/content/commands/rclone_lsf.md @@ -26,7 +26,7 @@ Eg ferejej3gux/ fubuwic -Use the --format option to control what gets listed. By default this +Use the `--format` option to control what gets listed. By default this is just the path, but you can use these parameters to control the output: @@ -39,9 +39,10 @@ output: m - MimeType of object if known e - encrypted name T - tier of storage if known, e.g. "Hot" or "Cool" + M - Metadata of object in JSON blob format, eg {"key":"value"} So if you wanted the path, size and modification time, you would use ---format "pst", or maybe --format "tsp" to put the path last. +`--format "pst"`, or maybe `--format "tsp"` to put the path last. Eg @@ -53,7 +54,7 @@ Eg 2016-06-25 18:55:40;37600;fubuwic If you specify "h" in the format you will get the MD5 hash by default, -use the "--hash" flag to change which hash you want. Note that this +use the `--hash` flag to change which hash you want. Note that this can be returned as an empty string if it isn't available on the object (and for directories), "ERROR" if there was an error reading it from the object and "UNSUPPORTED" if that object does not support that hash @@ -75,7 +76,7 @@ Eg (Though "rclone md5sum ." is an easier way of typing this.) By default the separator is ";" this can be changed with the ---separator flag. Note that separators aren't escaped in the path so +`--separator` flag. Note that separators aren't escaped in the path so putting it last is a good strategy. Eg @@ -97,8 +98,8 @@ Eg test.sh,449 "this file contains a comma, in the file name.txt",6 -Note that the --absolute parameter is useful for making lists of files -to pass to an rclone copy with the --files-from-raw flag. +Note that the `--absolute` parameter is useful for making lists of files +to pass to an rclone copy with the `--files-from-raw` flag. For example, to find all the files modified within one day and copy those only (without traversing the whole directory structure): diff --git a/docs/content/commands/rclone_lsjson.md b/docs/content/commands/rclone_lsjson.md index 79bdcbafd8ea7..abf8a39cacd45 100644 --- a/docs/content/commands/rclone_lsjson.md +++ b/docs/content/commands/rclone_lsjson.md @@ -15,7 +15,7 @@ List directories and objects in the path in JSON format. The output is an array of Items, where each Item looks like this - { + { "Hashes" : { "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f", "MD5" : "b1946ac92492d2347c6235b4d2611184", @@ -33,29 +33,32 @@ The output is an array of Items, where each Item looks like this "Path" : "full/path/goes/here/file.txt", "Size" : 6, "Tier" : "hot", - } + } -If --hash is not specified the Hashes property won't be emitted. The -types of hash can be specified with the --hash-type parameter (which -may be repeated). If --hash-type is set then it implies --hash. +If `--hash` is not specified the Hashes property won't be emitted. The +types of hash can be specified with the `--hash-type` parameter (which +may be repeated). If `--hash-type` is set then it implies `--hash`. -If --no-modtime is specified then ModTime will be blank. This can +If `--no-modtime` is specified then ModTime will be blank. This can speed things up on remotes where reading the ModTime takes an extra request (e.g. s3, swift). -If --no-mimetype is specified then MimeType will be blank. This can +If `--no-mimetype` is specified then MimeType will be blank. This can speed things up on remotes where reading the MimeType takes an extra request (e.g. s3, swift). -If --encrypted is not specified the Encrypted won't be emitted. +If `--encrypted` is not specified the Encrypted won't be emitted. -If --dirs-only is not specified files in addition to directories are +If `--dirs-only` is not specified files in addition to directories are returned -If --files-only is not specified directories in addition to the files +If `--files-only` is not specified directories in addition to the files will be returned. -if --stat is set then a single JSON blob will be returned about the +If `--metadata` is set then an additional Metadata key will be returned. +This will have metdata in rclone standard format as a JSON object. + +if `--stat` is set then a single JSON blob will be returned about the item pointed to. This will return an error if the item isn't found. However on bucket based backends (like s3, gcs, b2, azureblob etc) if the item isn't found it will return an empty directory as it isn't @@ -64,7 +67,7 @@ possible to tell empty directories from missing directories there. The Path field will only show folders below the remote path being listed. If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt" will be "subfolder/file.txt", not "remote:path/subfolder/file.txt". -When used without --recursive the Path will always be the same as Name. +When used without `--recursive` the Path will always be the same as Name. If the directory is a bucket in a bucket-based backend, then "IsBucket" will be set to true. This key won't be present unless it is @@ -112,7 +115,7 @@ rclone lsjson remote:path [flags] ``` --dirs-only Show only directories in the listing - -M, --encrypted Show the encrypted names + --encrypted Show the encrypted names --files-only Show only files in the listing --hash Include hashes in the output (may take longer) --hash-type stringArray Show only this hash type (may be repeated) diff --git a/docs/content/commands/rclone_md5sum.md b/docs/content/commands/rclone_md5sum.md index ea58ff627b8eb..de2b68fb0baed 100644 --- a/docs/content/commands/rclone_md5sum.md +++ b/docs/content/commands/rclone_md5sum.md @@ -20,6 +20,10 @@ not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling MD5 for any remote. +For other algorithms, see the [hashsum](/commands/rclone_hashsum/) +command. Running `rclone md5sum remote:path` is equivalent +to running `rclone hashsum MD5 remote:path`. + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, diff --git a/docs/content/commands/rclone_mount.md b/docs/content/commands/rclone_mount.md index 8a7d3cefd6ccd..2b88e0720570f 100644 --- a/docs/content/commands/rclone_mount.md +++ b/docs/content/commands/rclone_mount.md @@ -75,10 +75,10 @@ at all, then 1 PiB is set as both the total and the free size. To run rclone mount on Windows, you will need to download and install [WinFsp](http://www.secfs.net/winfsp/). -[WinFsp](https://github.com/billziss-gh/winfsp) is an open-source +[WinFsp](https://github.com/winfsp/winfsp) is an open-source Windows File System Proxy which makes it easy to write user space file systems for Windows. It provides a FUSE emulation layer which rclone -uses combination with [cgofuse](https://github.com/billziss-gh/cgofuse). +uses combination with [cgofuse](https://github.com/winfsp/cgofuse). Both of these packages are by Bill Zissimopoulos who was very helpful during the implementation of rclone mount for Windows. @@ -228,7 +228,7 @@ from Microsoft's Sysinternals suite, which has option `-s` to start processes as the SYSTEM account. Another alternative is to run the mount command from a Windows Scheduled Task, or a Windows Service, configured to run as the SYSTEM account. A third alternative is to use the -[WinFsp.Launcher infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)). +[WinFsp.Launcher infrastructure](https://github.com/winfsp/winfsp/wiki/WinFsp-Service-Architecture)). Note that when running rclone as another user, it will not use the configuration file from your profile unless you tell it to with the [`--config`](https://rclone.org/docs/#config-config-file) option. @@ -410,7 +410,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -567,6 +567,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -607,7 +639,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -619,7 +651,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -636,28 +668,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -705,7 +744,7 @@ rclone mount remote:path /path/to/mountpoint [flags] --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) @@ -713,6 +752,8 @@ rclone mount remote:path /path/to/mountpoint [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) diff --git a/docs/content/commands/rclone_move.md b/docs/content/commands/rclone_move.md index 0a53c1eeabc71..ce97800a2aeea 100644 --- a/docs/content/commands/rclone_move.md +++ b/docs/content/commands/rclone_move.md @@ -16,6 +16,9 @@ Moves the contents of the source directory to the destination directory. Rclone will error if the source and destination overlap and the remote does not support a server-side directory move operation. +To move single files, use the [moveto](/commands/rclone_moveto/) +command instead. + If no filters are in use and if possible this will server-side move `source:path` into `dest:path`. After this `source:path` will no longer exist. @@ -26,7 +29,8 @@ move will be used, otherwise it will copy it (server-side if possible) into `dest:path` then delete the original (if no errors on copy) in `source:path`. -If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag. +If you want to delete empty source directories after move, use the +`--delete-empty-src-dirs` flag. See the [--no-traverse](/docs/#no-traverse) option for controlling whether rclone lists the destination directory or not. Supplying this diff --git a/docs/content/commands/rclone_moveto.md b/docs/content/commands/rclone_moveto.md index 7ba6f6545ecb4..6f8488f83a4fb 100644 --- a/docs/content/commands/rclone_moveto.md +++ b/docs/content/commands/rclone_moveto.md @@ -17,7 +17,7 @@ directory named dest:path. This can be used to rename files or upload single files to other than their existing name. If the source is a directory then it acts exactly -like the move command. +like the [move](/commands/rclone_move/) command. So diff --git a/docs/content/commands/rclone_ncdu.md b/docs/content/commands/rclone_ncdu.md index 6e48ee2b86038..e1f51e6049176 100644 --- a/docs/content/commands/rclone_ncdu.md +++ b/docs/content/commands/rclone_ncdu.md @@ -23,7 +23,8 @@ builds an in memory representation. rclone ncdu can be used during this scanning phase and you will see it building up the directory structure as it goes along. -Here are the keys - press '?' to toggle the help on and off +You can interact with the user interface using key presses, +press '?' to toggle the help on and off. The supported keys are: ↑,↓ or k,j to Move →,l to enter @@ -34,19 +35,41 @@ Here are the keys - press '?' to toggle the help on and off u toggle human-readable format n,s,C,A sort by name,size,count,average size d delete file/directory + v select file/directory + V enter visual select mode + D delete selected files/directories y copy current path to clipboard Y display current path - ^L refresh screen + ^L refresh screen (fix screen corruption) ? to toggle help on and off - q/ESC/c-C to quit + q/ESC/^c to quit + +Listed files/directories may be prefixed by a one-character flag, +some of them combined with a description in brackes at end of line. +These flags have the following meaning: + + e means this is an empty directory, i.e. contains no files (but + may contain empty subdirectories) + ~ means this is a directory where some of the files (possibly in + subdirectories) have unknown size, and therefore the directory + size may be underestimated (and average size inaccurate, as it + is average of the files with known sizes). + . means an error occurred while reading a subdirectory, and + therefore the directory size may be underestimated (and average + size inaccurate) + ! means an error occurred while reading this directory This an homage to the [ncdu tool](https://dev.yorhel.nl/ncdu) but for rclone remotes. It is missing lots of features at the moment but is useful as it stands. -Note that it might take some time to delete big files/folders. The +Note that it might take some time to delete big files/directories. The UI won't respond in the meantime since the deletion is done synchronously. +For a non-interactive listing of the remote, see the +[tree](/commands/rclone_tree/) command. To just get the total size of +the remote you can also use the [size](/commands/rclone_size/) command. + ``` rclone ncdu remote:path [flags] diff --git a/docs/content/commands/rclone_obscure.md b/docs/content/commands/rclone_obscure.md index f9eda751b513e..0a77772a889ce 100644 --- a/docs/content/commands/rclone_obscure.md +++ b/docs/content/commands/rclone_obscure.md @@ -26,7 +26,7 @@ This command can also accept a password through STDIN instead of an argument by passing a hyphen as an argument. This will use the first line of STDIN as the password not including the trailing newline. -echo "secretpassword" | rclone obscure - + echo "secretpassword" | rclone obscure - If there is no data on STDIN to read, rclone obscure will default to obfuscating the hyphen itself. diff --git a/docs/content/commands/rclone_purge.md b/docs/content/commands/rclone_purge.md index c15c103208e66..6b87acc56ed6a 100644 --- a/docs/content/commands/rclone_purge.md +++ b/docs/content/commands/rclone_purge.md @@ -13,9 +13,10 @@ Remove the path and all of its contents. Remove the path and all of its contents. Note that this does not obey -include/exclude filters - everything will be removed. Use the `delete` -command if you want to selectively delete files. To delete empty directories only, -use command `rmdir` or `rmdirs`. +include/exclude filters - everything will be removed. Use the +[delete](/commands/rclone_delete/) command if you want to selectively +delete files. To delete empty directories only, use command +[rmdir](/commands/rclone_rmdir/) or [rmdirs](/commands/rclone_rmdirs/). **Important**: Since this can cause data loss, test first with the `--dry-run` or the `--interactive`/`-i` flag. diff --git a/docs/content/commands/rclone_rc.md b/docs/content/commands/rclone_rc.md index ae3df6f6c09fa..4d2dbb07f7bc2 100644 --- a/docs/content/commands/rclone_rc.md +++ b/docs/content/commands/rclone_rc.md @@ -13,26 +13,26 @@ Run a command against a running rclone. -This runs a command against a running rclone. Use the --url flag to +This runs a command against a running rclone. Use the `--url` flag to specify an non default URL to connect on. This can be either a ":port" which is taken to mean "http://localhost:port" or a "host:port" which is taken to mean "http://host:port" -A username and password can be passed in with --user and --pass. +A username and password can be passed in with `--user` and `--pass`. -Note that --rc-addr, --rc-user, --rc-pass will be read also for --url, ---user, --pass. +Note that `--rc-addr`, `--rc-user`, `--rc-pass` will be read also for +`--url`, `--user`, `--pass`. Arguments should be passed in as parameter=value. The result will be returned as a JSON object by default. -The --json parameter can be used to pass in a JSON blob as an input +The `--json` parameter can be used to pass in a JSON blob as an input instead of key=value arguments. This is the only way of passing in more complicated values. -The -o/--opt option can be used to set a key "opt" with key, value -options in the form "-o key=value" or "-o key". It can be repeated as +The `-o`/`--opt` option can be used to set a key "opt" with key, value +options in the form `-o key=value` or `-o key`. It can be repeated as many times as required. This is useful for rc commands which take the "opt" parameter which by convention is a dictionary of strings. @@ -43,7 +43,7 @@ Will place this in the "opt" value {"key":"value", "key2","") -The -a/--arg option can be used to set strings in the "arg" value. It +The `-a`/`--arg` option can be used to set strings in the "arg" value. It can be repeated as many times as required. This is useful for rc commands which take the "arg" parameter which by convention is a list of strings. @@ -54,13 +54,13 @@ Will place this in the "arg" value ["value", "value2"] -Use --loopback to connect to the rclone instance running "rclone rc". +Use `--loopback` to connect to the rclone instance running `rclone rc`. This is very useful for testing commands without having to run an rclone rc server, e.g.: rclone rc --loopback operations/about fs=/ -Use "rclone rc" to see a list of all possible commands. +Use `rclone rc` to see a list of all possible commands. ``` rclone rc commands parameter [flags] diff --git a/docs/content/commands/rclone_rcat.md b/docs/content/commands/rclone_rcat.md index 2085e2c2399f2..86d4c793e9c4a 100644 --- a/docs/content/commands/rclone_rcat.md +++ b/docs/content/commands/rclone_rcat.md @@ -30,11 +30,11 @@ must fit into RAM. The cutoff needs to be small enough to adhere the limits of your remote, please see there. Generally speaking, setting this cutoff too high will decrease your performance. -Use the |--size| flag to preallocate the file in advance at the remote end +Use the `--size` flag to preallocate the file in advance at the remote end and actually stream it, even if remote backend doesn't support streaming. -|--size| should be the exact size of the input stream in bytes. If the -size of the stream is different in length to the |--size| passed in +`--size` should be the exact size of the input stream in bytes. If the +size of the stream is different in length to the `--size` passed in then the transfer will likely fail. Note that the upload can also not be retried because the data is diff --git a/docs/content/commands/rclone_rmdir.md b/docs/content/commands/rclone_rmdir.md index f9d84cdfd8ef0..c8424e24a0ee5 100644 --- a/docs/content/commands/rclone_rmdir.md +++ b/docs/content/commands/rclone_rmdir.md @@ -14,10 +14,10 @@ Remove the empty directory at path. This removes empty directory given by path. Will not remove the path if it has any objects in it, not even empty subdirectories. Use -command `rmdirs` (or `delete` with option `--rmdirs`) -to do that. +command [rmdirs](/commands/rclone_rmdirs/) (or [delete](/commands/rclone_delete/) +with option `--rmdirs`) to do that. -To delete a path and any objects in it, use `purge` command. +To delete a path and any objects in it, use [purge](/commands/rclone_purge/) command. ``` diff --git a/docs/content/commands/rclone_rmdirs.md b/docs/content/commands/rclone_rmdirs.md index 1e90a055bb89f..ba5dc56fc1fec 100644 --- a/docs/content/commands/rclone_rmdirs.md +++ b/docs/content/commands/rclone_rmdirs.md @@ -17,15 +17,16 @@ that only contain empty directories), that it finds under the path. The root path itself will also be removed if it is empty, unless you supply the `--leave-root` flag. -Use command `rmdir` to delete just the empty directory -given by path, not recurse. +Use command [rmdir](/commands/rclone_rmdir/) to delete just the empty +directory given by path, not recurse. This is useful for tidying up remotes that rclone has left a lot of -empty directories in. For example the `delete` command will -delete files but leave the directory structure (unless used with -option `--rmdirs`). +empty directories in. For example the [delete](/commands/rclone_delete/) +command will delete files but leave the directory structure (unless +used with option `--rmdirs`). -To delete a path and any objects in it, use `purge` command. +To delete a path and any objects in it, use [purge](/commands/rclone_purge/) +command. ``` diff --git a/docs/content/commands/rclone_serve.md b/docs/content/commands/rclone_serve.md index 7663f10fbee8e..12d8141a82a04 100644 --- a/docs/content/commands/rclone_serve.md +++ b/docs/content/commands/rclone_serve.md @@ -11,8 +11,8 @@ Serve a remote over a protocol. ## Synopsis -rclone serve is used to serve a remote over a given protocol. This -command requires the use of a subcommand to specify the protocol, e.g. +Serve a remote over a given protocol. Requires the use of a +subcommand to specify the protocol, e.g. rclone serve http remote: @@ -40,5 +40,5 @@ See the [global flags page](/flags/) for global options not listed here. * [rclone serve http](/commands/rclone_serve_http/) - Serve the remote over HTTP. * [rclone serve restic](/commands/rclone_serve_restic/) - Serve the remote for restic's REST API. * [rclone serve sftp](/commands/rclone_serve_sftp/) - Serve the remote over SFTP. -* [rclone serve webdav](/commands/rclone_serve_webdav/) - Serve remote:path over webdav. +* [rclone serve webdav](/commands/rclone_serve_webdav/) - Serve remote:path over WebDAV. diff --git a/docs/content/commands/rclone_serve_dlna.md b/docs/content/commands/rclone_serve_dlna.md index 0ab69644e1dae..0343debb40b14 100644 --- a/docs/content/commands/rclone_serve_dlna.md +++ b/docs/content/commands/rclone_serve_dlna.md @@ -11,14 +11,16 @@ Serve remote:path over DLNA ## Synopsis -rclone serve dlna is a DLNA media server for media stored in an rclone remote. Many -devices, such as the Xbox and PlayStation, can automatically discover this server in the LAN -and play audio/video from it. VLC is also supported. Service discovery uses UDP multicast -packets (SSDP) and will thus only work on LANs. +Run a DLNA media server for media stored in an rclone remote. Many +devices, such as the Xbox and PlayStation, can automatically discover +this server in the LAN and play audio/video from it. VLC is also +supported. Service discovery uses UDP multicast packets (SSDP) and +will thus only work on LANs. -Rclone will list all files present in the remote, without filtering based on media formats or -file extensions. Additionally, there is no media transcoding support. This means that some -players might show files that they are not able to play back correctly. +Rclone will list all files present in the remote, without filtering +based on media formats or file extensions. Additionally, there is no +media transcoding support. This means that some players might show +files that they are not able to play back correctly. ## Server options @@ -51,7 +53,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -208,6 +210,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -248,7 +282,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -260,7 +294,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -277,28 +311,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -332,7 +373,7 @@ rclone serve dlna remote:path [flags] --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) @@ -340,6 +381,8 @@ rclone serve dlna remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) diff --git a/docs/content/commands/rclone_serve_docker.md b/docs/content/commands/rclone_serve_docker.md index 976c7a710407e..b294968a080fe 100644 --- a/docs/content/commands/rclone_serve_docker.md +++ b/docs/content/commands/rclone_serve_docker.md @@ -69,7 +69,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -226,6 +226,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -266,7 +298,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -278,7 +310,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -295,28 +327,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -367,7 +406,7 @@ rclone serve docker [flags] --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --socket-addr string Address or absolute path (default: /run/docker/plugins/rclone.sock) --socket-gid int GID for unix socket (default: current process GID) (default 1000) --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) @@ -377,6 +416,8 @@ rclone serve docker [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) diff --git a/docs/content/commands/rclone_serve_ftp.md b/docs/content/commands/rclone_serve_ftp.md index 5dfd4633fa529..3274a92c491d1 100644 --- a/docs/content/commands/rclone_serve_ftp.md +++ b/docs/content/commands/rclone_serve_ftp.md @@ -12,9 +12,9 @@ Serve remote:path over FTP. ## Synopsis -rclone serve ftp implements a basic ftp server to serve the -remote over FTP protocol. This can be viewed with a ftp client -or you can make a remote of type ftp to read and write it. +Run a basic FTP server to serve a remote over FTP protocol. +This can be viewed with a FTP client or you can make a remote of +type FTP to read and write it. ## Server options @@ -50,7 +50,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -207,6 +207,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -247,7 +279,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -259,7 +291,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -276,28 +308,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -416,7 +455,7 @@ rclone serve ftp remote:path [flags] --passive-port string Passive port range to use (default "30000-32000") --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --public-ip string Public IP address to advertise for passive connections - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication (default "anonymous") @@ -425,6 +464,8 @@ rclone serve ftp remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) diff --git a/docs/content/commands/rclone_serve_http.md b/docs/content/commands/rclone_serve_http.md index 418c18377ca96..329bc14207f58 100644 --- a/docs/content/commands/rclone_serve_http.md +++ b/docs/content/commands/rclone_serve_http.md @@ -11,59 +11,59 @@ Serve the remote over HTTP. ## Synopsis -rclone serve http implements a basic web server to serve the remote -over HTTP. This can be viewed in a web browser or you can make a -remote of type http read from it. +Run a basic web server to serve a remote over HTTP. +This can be viewed in a web browser or you can make a remote of type +http read from it. -You can use the filter flags (e.g. --include, --exclude) to control what +You can use the filter flags (e.g. `--include`, `--exclude`) to control what is served. -The server will log errors. Use -v to see access logs. +The server will log errors. Use `-v` to see access logs. ---bwlimit will be respected for file transfers. Use --stats to +`--bwlimit` will be respected for file transfers. Use `--stats` to control the stats printing. ## Server options -Use --addr to specify which IP address and port the server should -listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all +Use `--addr` to specify which IP address and port the server should +listen on, eg `--addr 1.2.3.4:8000` or `--addr :8080` to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. -If you set --addr to listen on a public or LAN accessible IP address +If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. ---server-read-timeout and --server-write-timeout can be used to +`--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. ---max-header-bytes controls the maximum number of bytes the server will +`--max-header-bytes` controls the maximum number of bytes the server will accept in the HTTP header. ---baseurl controls the URL prefix that rclone serves from. By default -rclone will serve from the root. If you used --baseurl "/rclone" then +`--baseurl` controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used `--baseurl "/rclone"` then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically -inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", ---baseurl "/rclone" and --baseurl "/rclone/" are all treated +inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, +`--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. ### SSL/TLS By default this will serve over http. If you want you can serve over -https. You will need to supply the --cert and --key flags. If you -wish to do client side certificate validation then you will need to -supply --client-ca also. +https. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. ---cert should be a either a PEM encoded certificate or a concatenation -of that with the CA certificate. --key should be the PEM encoded -private key and --client-ca should be the PEM encoded client +`--cert` should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client certificate authority certificate. ### Template ---template allows a user to specify a custom markup template for http -and webdav serve functions. The server exports the following markup +`--template` allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: | Parameter | Description | @@ -90,9 +90,9 @@ to be used within the template to server pages: By default this will serve files without needing a login. You can either use an htpasswd file which can take lots of users, or -set a single username and password with the --user and --pass flags. +set a single username and password with the `--user` and `--pass` flags. -Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is +Use `--htpasswd /path/to/htpasswd` to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. @@ -104,9 +104,9 @@ To create an htpasswd file: The password file can be updated while rclone is running. -Use --realm to set the authentication realm. +Use `--realm` to set the authentication realm. -Use --salt to change the password hashing salt from the default. +Use `--salt` to change the password hashing salt from the default. ## VFS - Virtual File System @@ -126,7 +126,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -283,6 +283,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -323,7 +355,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -335,7 +367,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -352,28 +384,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -412,7 +451,7 @@ rclone serve http remote:path [flags] --no-seek Don't allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --realm string Realm for authentication --salt string Password hashing salt (default "dlPL2MqE") --server-read-timeout duration Timeout for server reading data (default 1h0m0s) @@ -426,6 +465,8 @@ rclone serve http remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) diff --git a/docs/content/commands/rclone_serve_restic.md b/docs/content/commands/rclone_serve_restic.md index 2dec5ed5a5dbd..881e697f00615 100644 --- a/docs/content/commands/rclone_serve_restic.md +++ b/docs/content/commands/rclone_serve_restic.md @@ -11,8 +11,8 @@ Serve the remote for restic's REST API. ## Synopsis -rclone serve restic implements restic's REST backend API -over HTTP. This allows restic to use rclone as a data storage +Run a basic web server to serve a remove over restic's REST backend +API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly. [Restic](https://restic.net/) is a command-line program for doing @@ -20,8 +20,8 @@ backups. The server will log errors. Use -v to see access logs. ---bwlimit will be respected for file transfers. Use --stats to -control the stats printing. +`--bwlimit` will be respected for file transfers. +Use `--stats` to control the stats printing. ## Setting up rclone for use by restic ### @@ -40,11 +40,11 @@ Where you can replace "backup" in the above by whatever path in the remote you wish to use. By default this will serve on "localhost:8080" you can change this -with use of the "--addr" flag. +with use of the `--addr` flag. You might wish to start this server on boot. -Adding --cache-objects=false will cause rclone to stop caching objects +Adding `--cache-objects=false` will cause rclone to stop caching objects returned from the List call. Caching is normally desirable as it speeds up downloading objects, saves transactions and uses very little memory. @@ -90,36 +90,36 @@ these **must** end with /. Eg ### Private repositories #### -The "--private-repos" flag can be used to limit users to repositories starting +The`--private-repos` flag can be used to limit users to repositories starting with a path of `//`. ## Server options -Use --addr to specify which IP address and port the server should -listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all -IPs. By default it only listens on localhost. You can use port +Use `--addr` to specify which IP address and port the server should +listen on, e.g. `--addr 1.2.3.4:8000` or `--addr :8080` to +listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. -If you set --addr to listen on a public or LAN accessible IP address +If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. ---server-read-timeout and --server-write-timeout can be used to +`--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. ---max-header-bytes controls the maximum number of bytes the server will +`--max-header-bytes` controls the maximum number of bytes the server will accept in the HTTP header. ---baseurl controls the URL prefix that rclone serves from. By default -rclone will serve from the root. If you used --baseurl "/rclone" then +`--baseurl` controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used `--baseurl "/rclone"` then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically -inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", ---baseurl "/rclone" and --baseurl "/rclone/" are all treated +inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, +`--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. ---template allows a user to specify a custom markup template for http -and webdav serve functions. The server exports the following markup +`--template` allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: | Parameter | Description | @@ -146,9 +146,9 @@ to be used within the template to server pages: By default this will serve files without needing a login. You can either use an htpasswd file which can take lots of users, or -set a single username and password with the --user and --pass flags. +set a single username and password with the `--user` and `--pass` flags. -Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is +Use `--htpasswd /path/to/htpasswd` to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. @@ -160,18 +160,18 @@ To create an htpasswd file: The password file can be updated while rclone is running. -Use --realm to set the authentication realm. +Use `--realm` to set the authentication realm. ### SSL/TLS -By default this will serve over http. If you want you can serve over -https. You will need to supply the --cert and --key flags. If you -wish to do client side certificate validation then you will need to -supply --client-ca also. +By default this will serve over HTTP. If you want you can serve over +HTTPS. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. ---cert should be either a PEM encoded certificate or a concatenation -of that with the CA certificate. --key should be the PEM encoded -private key and --client-ca should be the PEM encoded client +`--cert` should be either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client certificate authority certificate. diff --git a/docs/content/commands/rclone_serve_sftp.md b/docs/content/commands/rclone_serve_sftp.md index 5995b97855b0f..226e2976946c2 100644 --- a/docs/content/commands/rclone_serve_sftp.md +++ b/docs/content/commands/rclone_serve_sftp.md @@ -11,21 +11,21 @@ Serve the remote over SFTP. ## Synopsis -rclone serve sftp implements an SFTP server to serve the remote -over SFTP. This can be used with an SFTP client or you can make a -remote of type sftp to use with it. +Run a SFTP server to serve a remote over SFTP. This can be used +with an SFTP client or you can make a remote of type sftp to use with it. -You can use the filter flags (e.g. --include, --exclude) to control what +You can use the filter flags (e.g. `--include`, `--exclude`) to control what is served. -The server will log errors. Use -v to see access logs. +The server will log errors. Use `-v` to see access logs. ---bwlimit will be respected for file transfers. Use --stats to -control the stats printing. +`--bwlimit` will be respected for file transfers. +Use `--stats` to control the stats printing. -You must provide some means of authentication, either with --user/--pass, -an authorized keys file (specify location with --authorized-keys - the -default is the same as ssh), an --auth-proxy, or set the --no-auth flag for no +You must provide some means of authentication, either with +`--user`/`--pass`, an authorized keys file (specify location with +`--authorized-keys` - the default is the same as ssh), an +`--auth-proxy`, or set the `--no-auth` flag for no authentication when logging in. Note that this also implements a small number of shell commands so @@ -33,30 +33,30 @@ that it can provide md5sum/sha1sum/df information for the rclone sftp backend. This means that is can support SHA1SUMs, MD5SUMs and the about command when paired with the rclone sftp backend. -If you don't supply a host --key then rclone will generate rsa, ecdsa +If you don't supply a host `--key` then rclone will generate rsa, ecdsa and ed25519 variants, and cache them for later use in rclone's cache -directory (see "rclone help flags cache-dir") in the "serve-sftp" +directory (see `rclone help flags cache-dir`) in the "serve-sftp" directory. By default the server binds to localhost:2022 - if you want it to be -reachable externally then supply "--addr :2022" for example. +reachable externally then supply `--addr :2022` for example. -Note that the default of "--vfs-cache-mode off" is fine for the rclone +Note that the default of `--vfs-cache-mode off` is fine for the rclone sftp backend, but it may not be with other SFTP clients. -If --stdio is specified, rclone will serve SFTP over stdio, which can +If `--stdio` is specified, rclone will serve SFTP over stdio, which can be used with sshd via ~/.ssh/authorized_keys, for example: restrict,command="rclone serve sftp --stdio ./photos" ssh-rsa ... -On the client you need to set "--transfers 1" when using --stdio. +On the client you need to set `--transfers 1` when using `--stdio`. Otherwise multiple instances of the rclone server are started by OpenSSH which can lead to "corrupted on transfer" errors. This is the case because the client chooses indiscriminately which server to send commands to while the servers all have different views of the state of the filing system. The "restrict" in authorized_keys prevents SHA1SUMs and MD5SUMs from beeing -used. Omitting "restrict" and using --sftp-path-override to enable +used. Omitting "restrict" and using `--sftp-path-override` to enable checksumming is possible but less secure and you could use the SFTP server provided by OpenSSH in this case. @@ -79,7 +79,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -236,6 +236,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -276,7 +308,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -288,7 +320,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -305,28 +337,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -444,7 +483,7 @@ rclone serve sftp remote:path [flags] --no-seek Don't allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --stdio Run an sftp server on run stdin/stdout --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) @@ -454,6 +493,8 @@ rclone serve sftp remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) diff --git a/docs/content/commands/rclone_serve_webdav.md b/docs/content/commands/rclone_serve_webdav.md index a240e67ab1f0d..5209719f43f4c 100644 --- a/docs/content/commands/rclone_serve_webdav.md +++ b/docs/content/commands/rclone_serve_webdav.md @@ -1,23 +1,21 @@ --- title: "rclone serve webdav" -description: "Serve remote:path over webdav." +description: "Serve remote:path over WebDAV." slug: rclone_serve_webdav url: /commands/rclone_serve_webdav/ # autogenerated - DO NOT EDIT, instead edit the source code in cmd/serve/webdav/ and as part of making a release run "make commanddocs" --- # rclone serve webdav -Serve remote:path over webdav. +Serve remote:path over WebDAV. ## Synopsis +Run a basic WebDAV server to serve a remote over HTTP via the +WebDAV protocol. This can be viewed with a WebDAV client, through a web +browser, or you can make a remote of type WebDAV to read and write it. -rclone serve webdav implements a basic webdav server to serve the -remote over HTTP via the webdav protocol. This can be viewed with a -webdav client, through a web browser, or you can make a remote of -type webdav to read and write it. - -## Webdav options +## WebDAV options ### --etag-hash @@ -26,38 +24,37 @@ based on the ModTime and Size of the object. If this flag is set to "auto" then rclone will choose the first supported hash on the backend or you can use a named hash such as -"MD5" or "SHA-1". - -Use "rclone hashsum" to see the full list. +"MD5" or "SHA-1". Use the [hashsum](/commands/rclone_hashsum/) command +to see the full list. ## Server options -Use --addr to specify which IP address and port the server should -listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all -IPs. By default it only listens on localhost. You can use port +Use `--addr` to specify which IP address and port the server should +listen on, e.g. `--addr 1.2.3.4:8000` or `--addr :8080` to +listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. -If you set --addr to listen on a public or LAN accessible IP address +If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. ---server-read-timeout and --server-write-timeout can be used to +`--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. ---max-header-bytes controls the maximum number of bytes the server will +`--max-header-bytes` controls the maximum number of bytes the server will accept in the HTTP header. ---baseurl controls the URL prefix that rclone serves from. By default -rclone will serve from the root. If you used --baseurl "/rclone" then +`--baseurl` controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used `--baseurl "/rclone"` then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically -inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", ---baseurl "/rclone" and --baseurl "/rclone/" are all treated +inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, +`--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. ---template allows a user to specify a custom markup template for http -and webdav serve functions. The server exports the following markup +`--template` allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: | Parameter | Description | @@ -84,9 +81,9 @@ to be used within the template to server pages: By default this will serve files without needing a login. You can either use an htpasswd file which can take lots of users, or -set a single username and password with the --user and --pass flags. +set a single username and password with the `--user` and `--pass` flags. -Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is +Use `--htpasswd /path/to/htpasswd` to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. @@ -98,18 +95,18 @@ To create an htpasswd file: The password file can be updated while rclone is running. -Use --realm to set the authentication realm. +Use `--realm` to set the authentication realm. ### SSL/TLS -By default this will serve over http. If you want you can serve over -https. You will need to supply the --cert and --key flags. If you -wish to do client side certificate validation then you will need to -supply --client-ca also. +By default this will serve over HTTP. If you want you can serve over +HTTPS. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. ---cert should be either a PEM encoded certificate or a concatenation -of that with the CA certificate. --key should be the PEM encoded -private key and --client-ca should be the PEM encoded client +`--cert` should be either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client certificate authority certificate. ## VFS - Virtual File System @@ -130,7 +127,7 @@ about files and directories (but not the data) in memory. Using the `--dir-cache-time` flag, you can control how long a directory should be considered up to date and not refreshed from the -backend. Changes made through the mount will appear immediately or +backend. Changes made through the VFS will appear immediately or invalidate the cache. --dir-cache-time duration Time to cache directory entries for (default 5m0s) @@ -287,6 +284,38 @@ FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn't support sparse files and it will log an ERROR message if one is detected. +### Fingerprinting + +Various parts of the VFS use fingerprinting to see if a local file +copy has changed relative to a remote file. Fingerprints are made +from: + +- size +- modification time +- hash + +where available on an object. + +On some backends some of these attributes are slow to read (they take +an extra API call per object, or extra work per object). + +For example `hash` is slow with the `local` and `sftp` backends as +they have to read the entire file and hash it, and `modtime` is slow +with the `s3`, `swift`, `ftp` and `qinqstor` backends because they +need to do an extra API call to fetch it. + +If you use the `--vfs-fast-fingerprint` flag then rclone will not +include the slow operations in the fingerprint. This makes the +fingerprinting less accurate but much faster and will improve the +opening time of cached files. + +If you are running a vfs cache over `local`, `s3` or `swift` backends +then using this flag is recommended. + +Note that if you change the value of this flag, the fingerprints of +the files in the cache may be invalidated and the files will need to +be downloaded again. + ## VFS Chunked Reading When rclone reads files from a remote it reads them in chunks. This @@ -327,7 +356,7 @@ read of the modification time takes a transaction. --no-checksum Don't compare checksums on up/download. --no-modtime Don't read/write the modification time (can speed things up). --no-seek Don't allow seeking in files. - --read-only Mount read-only. + --read-only Only allow read-only access. Sometimes rclone is delivered reads or writes out of order. Rather than seeking rclone will wait a short time for the in sequence read or @@ -339,7 +368,7 @@ on disk cache file. When using VFS write caching (`--vfs-cache-mode` with value writes or full), the global flag `--transfers` can be set to adjust the number of parallel uploads of -modified files from cache (the related global flag `--checkers` have no effect on mount). +modified files from the cache (the related global flag `--checkers` has no effect on the VFS). --transfers int Number of file transfers to run in parallel (default 4) @@ -356,28 +385,35 @@ It is not allowed for two files in the same directory to differ only by case. Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. -The `--vfs-case-insensitive` mount flag controls how rclone handles these -two cases. If its value is "false", rclone passes file names to the mounted -file system as-is. If the flag is "true" (or appears without a value on +The `--vfs-case-insensitive` VFS flag controls how rclone handles these +two cases. If its value is "false", rclone passes file names to the remote +as-is. If the flag is "true" (or appears without a value on the command line), rclone may perform a "fixup" as explained below. The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. If an argument refers +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is -controlled by an underlying mounted file system. +controlled by the underlying remote. Note that case sensitivity of the operating system running rclone (the target) -may differ from case sensitivity of a file system mounted by rclone (the source). +may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether "fixup" is performed to satisfy the target. If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: "true" on Windows and macOS, "false" otherwise. If the flag is provided without a value, then it is "true". +## VFS Disk Options + +This flag allows you to manually set the statistics about the filing system. +It can be useful when those statistics cannot be read correctly automatically. + + --vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) + ## Alternate report of used bytes Some backends, most notably S3, do not report the amount of bytes used. @@ -500,7 +536,7 @@ rclone serve webdav remote:path [flags] --no-seek Don't allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --realm string Realm for authentication (default "rclone") --server-read-timeout duration Timeout for server reading data (default 1h0m0s) --server-write-timeout duration Timeout for server writing data (default 1h0m0s) @@ -513,6 +549,8 @@ rclone serve webdav remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) diff --git a/docs/content/commands/rclone_sha1sum.md b/docs/content/commands/rclone_sha1sum.md index 522b371e56473..a61b15e4593c0 100644 --- a/docs/content/commands/rclone_sha1sum.md +++ b/docs/content/commands/rclone_sha1sum.md @@ -20,6 +20,10 @@ not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling SHA-1 for any remote. +For other algorithms, see the [hashsum](/commands/rclone_hashsum/) +command. Running `rclone sha1sum remote:path` is equivalent +to running `rclone hashsum SHA1 remote:path`. + This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, diff --git a/docs/content/commands/rclone_size.md b/docs/content/commands/rclone_size.md index ce8e4552d2107..7f75fe981d148 100644 --- a/docs/content/commands/rclone_size.md +++ b/docs/content/commands/rclone_size.md @@ -9,6 +9,28 @@ url: /commands/rclone_size/ Prints the total size and number of objects in remote:path. +## Synopsis + + +Counts objects in the path and calculates the total size. Prints the +result to standard output. + +By default the output is in human-readable format, but shows values in +both human-readable format as well as the raw numbers (global option +`--human-readable` is not considered). Use option `--json` +to format output as JSON instead. + +Recurses by default, use `--max-depth 1` to stop the +recursion. + +Some backends do not always provide file sizes, see for example +[Google Photos](/googlephotos/#size) and +[Google Drive](/drive/#limitations-of-google-docs). +Rclone will then show a notice in the log indicating how many such +files were encountered, and count them in as empty files in the output +of the size command. + + ``` rclone size remote:path [flags] ``` diff --git a/docs/content/commands/rclone_sync.md b/docs/content/commands/rclone_sync.md index 5596240e1e9a9..0f7da84a244c1 100644 --- a/docs/content/commands/rclone_sync.md +++ b/docs/content/commands/rclone_sync.md @@ -16,7 +16,9 @@ Sync the source to the destination, changing the destination only. Doesn't transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Destination is updated to match source, including deleting files -if necessary (except duplicate objects, see below). +if necessary (except duplicate objects, see below). If you don't +want to delete files from destination, use the +[copy](/commands/rclone_copy/) command instead. **Important**: Since this can cause data loss, test first with the `--dry-run` or the `--interactive`/`-i` flag. @@ -30,7 +32,7 @@ those providers that support it) are also not yet handled. It is always the contents of the directory that is synced, not the directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. See -extended explanation in the `copy` command above if unsure. +extended explanation in the [copy](/commands/rclone_copy/) command if unsure. If dest:path doesn't exist, it is created and the source:path contents go there. diff --git a/docs/content/commands/rclone_test.md b/docs/content/commands/rclone_test.md index 2217f68795cc3..d04ccb803b159 100644 --- a/docs/content/commands/rclone_test.md +++ b/docs/content/commands/rclone_test.md @@ -37,6 +37,7 @@ See the [global flags page](/flags/) for global options not listed here. * [rclone test changenotify](/commands/rclone_test_changenotify/) - Log any change notify requests for the remote passed in. * [rclone test histogram](/commands/rclone_test_histogram/) - Makes a histogram of file name characters. * [rclone test info](/commands/rclone_test_info/) - Discovers file name or other limitations for paths. +* [rclone test makefile](/commands/rclone_test_makefile/) - Make files with random contents of the size given * [rclone test makefiles](/commands/rclone_test_makefiles/) - Make a random file hierarchy in a directory * [rclone test memory](/commands/rclone_test_memory/) - Load all the objects at remote:path into memory and report memory stats. diff --git a/docs/content/commands/rclone_test_makefile.md b/docs/content/commands/rclone_test_makefile.md new file mode 100644 index 0000000000000..5acddb5c19256 --- /dev/null +++ b/docs/content/commands/rclone_test_makefile.md @@ -0,0 +1,33 @@ +--- +title: "rclone test makefile" +description: "Make files with random contents of the size given" +slug: rclone_test_makefile +url: /commands/rclone_test_makefile/ +# autogenerated - DO NOT EDIT, instead edit the source code in cmd/test/makefile/ and as part of making a release run "make commanddocs" +--- +# rclone test makefile + +Make files with random contents of the size given + +``` +rclone test makefile []+ [flags] +``` + +## Options + +``` + --ascii Fill files with random ASCII printable bytes only + --chargen Fill files with a ASCII chargen pattern + -h, --help help for makefile + --pattern Fill files with a periodic pattern + --seed int Seed for the random number generator (0 for random) (default 1) + --sparse Make the files sparse (appear to be filled with ASCII 0x00) + --zero Fill files with ASCII 0x00 +``` + +See the [global flags page](/flags/) for global options not listed here. + +## SEE ALSO + +* [rclone test](/commands/rclone_test/) - Run a test command + diff --git a/docs/content/commands/rclone_test_makefiles.md b/docs/content/commands/rclone_test_makefiles.md index f0816d14ef25e..ad8e3f14b819b 100644 --- a/docs/content/commands/rclone_test_makefiles.md +++ b/docs/content/commands/rclone_test_makefiles.md @@ -16,6 +16,8 @@ rclone test makefiles [flags] ## Options ``` + --ascii Fill files with random ASCII printable bytes only + --chargen Fill files with a ASCII chargen pattern --files int Number of files to create (default 1000) --files-per-directory int Average number of files per directory (default 10) -h, --help help for makefiles @@ -23,7 +25,10 @@ rclone test makefiles [flags] --max-name-length int Maximum size of file names (default 12) --min-file-size SizeSuffix Minimum size of file to create --min-name-length int Minimum size of file names (default 4) + --pattern Fill files with a periodic pattern --seed int Seed for the random number generator (0 for random) (default 1) + --sparse Make the files sparse (appear to be filled with ASCII 0x00) + --zero Fill files with ASCII 0x00 ``` See the [global flags page](/flags/) for global options not listed here. diff --git a/docs/content/commands/rclone_tree.md b/docs/content/commands/rclone_tree.md index 5de357227c0f4..1995a108f5d9c 100644 --- a/docs/content/commands/rclone_tree.md +++ b/docs/content/commands/rclone_tree.md @@ -29,12 +29,16 @@ For example 1 directories, 5 files You can use any of the filtering options with the tree command (e.g. ---include and --exclude). You can also use --fast-list. +`--include` and `--exclude`. You can also use `--fast-list`. The tree command has many options for controlling the listing which -are compatible with the tree command. Note that not all of them have +are compatible with the tree command, for example you can include file +sizes with `--size`. Note that not all of them have short options as they conflict with rclone's short options. +For a more interactive navigation of the remote see the +[ncdu](/commands/rclone_ncdu/) command. + ``` rclone tree remote:path [flags] diff --git a/docs/content/compress.md b/docs/content/compress.md index f0ff203abde81..9a2d6b974ecbf 100644 --- a/docs/content/compress.md +++ b/docs/content/compress.md @@ -90,7 +90,7 @@ size of the uncompressed file. The file names should not be changed by anything {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/compress/compress.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to compress (Compress a remote). +Here are the Standard options specific to compress (Compress a remote). #### --compress-remote @@ -119,7 +119,7 @@ Properties: ### Advanced options -Here are the advanced options specific to compress (Compress a remote). +Here are the Advanced options specific to compress (Compress a remote). #### --compress-level @@ -156,4 +156,10 @@ Properties: - Type: SizeSuffix - Default: 20Mi +### Metadata + +Any metadata supported by the underlying remote is read and written. + +See the [metadata](/docs/#metadata) docs for more info. + {{< rem autogenerated options stop >}} diff --git a/docs/content/crypt.md b/docs/content/crypt.md index 99b9f629c6fc4..92390bda39f1c 100644 --- a/docs/content/crypt.md +++ b/docs/content/crypt.md @@ -419,7 +419,7 @@ check the checksums properly. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/crypt/crypt.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to crypt (Encrypt/Decrypt a remote). +Here are the Standard options specific to crypt (Encrypt/Decrypt a remote). #### --crypt-remote @@ -504,7 +504,7 @@ Properties: ### Advanced options -Here are the advanced options specific to crypt (Encrypt/Decrypt a remote). +Here are the Advanced options specific to crypt (Encrypt/Decrypt a remote). #### --crypt-server-side-across-configs @@ -584,6 +584,12 @@ Properties: - Encode using base32768. Suitable if your remote counts UTF-16 or - Unicode codepoint instead of UTF-8 byte length. (Eg. Onedrive) +### Metadata + +Any metadata supported by the underlying remote is read and written. + +See the [metadata](/docs/#metadata) docs for more info. + ## Backend commands Here are the commands specific to the crypt backend. @@ -594,7 +600,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](/commands/rclone_backend/) for more +See the [backend](/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command diff --git a/docs/content/drive.md b/docs/content/drive.md index 08d1bb8330cf5..a4fdf8ccb7f0f 100644 --- a/docs/content/drive.md +++ b/docs/content/drive.md @@ -548,7 +548,7 @@ Google Documents. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/drive/drive.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to drive (Google Drive). +Here are the Standard options specific to drive (Google Drive). #### --drive-client-id @@ -603,22 +603,6 @@ Properties: - Allows read-only access to file metadata but - does not allow any access to read or download file content. -#### --drive-root-folder-id - -ID of the root folder. -Leave blank normally. - -Fill in to access "Computers" folders (see docs), or for rclone to use -a non root folder as its starting point. - - -Properties: - -- Config: root_folder_id -- Env Var: RCLONE_DRIVE_ROOT_FOLDER_ID -- Type: string -- Required: false - #### --drive-service-account-file Service Account Credentials JSON file path. @@ -648,7 +632,7 @@ Properties: ### Advanced options -Here are the advanced options specific to drive (Google Drive). +Here are the Advanced options specific to drive (Google Drive). #### --drive-token @@ -687,6 +671,22 @@ Properties: - Type: string - Required: false +#### --drive-root-folder-id + +ID of the root folder. +Leave blank normally. + +Fill in to access "Computers" folders (see docs), or for rclone to use +a non root folder as its starting point. + + +Properties: + +- Config: root_folder_id +- Env Var: RCLONE_DRIVE_ROOT_FOLDER_ID +- Type: string +- Required: false + #### --drive-service-account-credentials Service Account Credentials JSON blob. @@ -1167,6 +1167,34 @@ Properties: - Type: bool - Default: false +#### --drive-resource-key + +Resource key for accessing a link-shared file. + +If you need to access files shared with a link like this + + https://drive.google.com/drive/folders/XXX?resourcekey=YYY&usp=sharing + +Then you will need to use the first part "XXX" as the "root_folder_id" +and the second part "YYY" as the "resource_key" otherwise you will get +404 not found errors when trying to access the directory. + +See: https://developers.google.com/drive/api/guides/resource-keys + +This resource key requirement only applies to a subset of old files. + +Note also that opening the folder once in the web interface (with the +user you've authenticated rclone with) seems to be enough so that the +resource key is no needed. + + +Properties: + +- Config: resource_key +- Env Var: RCLONE_DRIVE_RESOURCE_KEY +- Type: string +- Required: false + #### --drive-encoding The encoding for the backend. @@ -1190,7 +1218,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](/commands/rclone_backend/) for more +See the [backend](/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command @@ -1292,7 +1320,7 @@ This will return a JSON list of objects like this With the -o config parameter it will output the list in a format suitable for adding to a config file to make aliases for all the -drives found. +drives found and a combined drive. [My Drive] type = alias @@ -1302,10 +1330,15 @@ drives found. type = alias remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=: -Adding this to the rclone config file will cause those team drives to -be accessible with the aliases shown. This may require manual editing -of the names. + [AllDrives] + type = combine + remote = "My Drive=My Drive:" "Test Drive=Test Drive:" +Adding this to the rclone config file will cause those team drives to +be accessible with the aliases shown. Any illegal charactes will be +substituted with "_" and duplicate names will have numbers suffixed. +It will also add a remote called AllDrives which shows all the shared +drives combined into one directory tree. ### untrash @@ -1362,6 +1395,18 @@ attempted if possible. Use the -i flag to see what would be copied before copying. +### exportformats + +Dump the export formats for debug purposes + + rclone backend exportformats remote: [options] [+] + +### importformats + +Dump the import formats for debug purposes + + rclone backend importformats remote: [options] [+] + {{< rem autogenerated options stop >}} ## Limitations diff --git a/docs/content/dropbox.md b/docs/content/dropbox.md index 690a74c0f8bab..62b351c51789b 100644 --- a/docs/content/dropbox.md +++ b/docs/content/dropbox.md @@ -182,7 +182,7 @@ finishes up the last batch using this mode. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/dropbox/dropbox.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to dropbox (Dropbox). +Here are the Standard options specific to dropbox (Dropbox). #### --dropbox-client-id @@ -212,7 +212,7 @@ Properties: ### Advanced options -Here are the advanced options specific to dropbox (Dropbox). +Here are the Advanced options specific to dropbox (Dropbox). #### --dropbox-token diff --git a/docs/content/fichier.md b/docs/content/fichier.md index 38ea450d93c6e..0e2259b693e4b 100644 --- a/docs/content/fichier.md +++ b/docs/content/fichier.md @@ -116,7 +116,7 @@ as they can't be used in JSON strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/fichier/fichier.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to fichier (1Fichier). +Here are the Standard options specific to fichier (1Fichier). #### --fichier-api-key @@ -131,7 +131,7 @@ Properties: ### Advanced options -Here are the advanced options specific to fichier (1Fichier). +Here are the Advanced options specific to fichier (1Fichier). #### --fichier-shared-folder diff --git a/docs/content/filefabric.md b/docs/content/filefabric.md index 66c1e47320f3b..225573ba576ea 100644 --- a/docs/content/filefabric.md +++ b/docs/content/filefabric.md @@ -154,7 +154,7 @@ The ID for "S3 Storage" would be `120673761`. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/filefabric/filefabric.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to filefabric (Enterprise File Fabric). +Here are the Standard options specific to filefabric (Enterprise File Fabric). #### --filefabric-url @@ -213,7 +213,7 @@ Properties: ### Advanced options -Here are the advanced options specific to filefabric (Enterprise File Fabric). +Here are the Advanced options specific to filefabric (Enterprise File Fabric). #### --filefabric-token diff --git a/docs/content/flags.md b/docs/content/flags.md index bfc367e63afdc..cd9801851b9a5 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -38,6 +38,7 @@ These flags are available for every command. --delete-during When synchronizing, delete files during transfer --delete-excluded Delete files on dest excluded from sync --disable string Disable a comma separated list of features (use --disable help to see a list) + --disable-http-keep-alives Disable HTTP keep-alives and use each connection once. --disable-http2 Disable HTTP/2 in the global transport -n, --dry-run Do a trial run with no permanent changes --dscp string Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21 @@ -86,6 +87,8 @@ These flags are available for every command. --max-stats-groups int Maximum number of stats groups to keep in memory, on max oldest is discarded (default 1000) --max-transfer SizeSuffix Maximum size of data to transfer (default off) --memprofile string Write memory profile to file + -M, --metadata If set, preserve metadata when copying objects + --metadata-set stringArray Add metadata key=value when uploading --min-age Duration Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off) --min-size SizeSuffix Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off) --modify-window duration Max time diff to be considered the same (default 1ns) @@ -157,7 +160,7 @@ These flags are available for every command. --use-json-log Use json log format --use-mmap Use mmap allocator (see docs) --use-server-modtime Use server modified time instead of object metadata - --user-agent string Set the user-agent to a specified string (default "rclone/v1.58.0") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.59.0") -v, --verbose count Print lots more stuff (repeat for more) ``` @@ -212,6 +215,7 @@ and may be set in the config file. --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --b2-version-at Time Show file versions as they were at the specified time (default off) --b2-versions Include old versions in directory listings --box-access-token string Box App Primary Access Token --box-auth-url string Auth server URL @@ -251,6 +255,7 @@ and may be set in the config file. --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks --chunker-hash-type string Choose how chunker handles hash sums (default "md5") --chunker-remote string Remote to chunk/unchunk + --combine-upstreams SpaceSepList Upstreams for combining --compress-level int GZIP compression level (-2 to 9) (default -1) --compress-mode string Compression mode (default "gzip") --compress-ram-cache-limit SizeSuffix Some remotes don't allow the upload of files with unknown size (default 20Mi) @@ -283,6 +288,7 @@ and may be set in the config file. --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) + --drive-resource-key string Resource key for accessing a link-shared file --drive-root-folder-id string ID of the root folder --drive-scope string Scope that rclone should use when requesting access from drive --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs @@ -337,8 +343,8 @@ and may be set in the config file. --ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited --ftp-disable-epsv Disable using EPSV even if server advertises support --ftp-disable-mlsd Disable using MLSD even if server advertises support - --ftp-disable-utf8 Disable using UTF-8 even if server advertises support --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) + --ftp-disable-utf8 Disable using UTF-8 even if server advertises support --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) --ftp-host string FTP host to connect to @@ -357,8 +363,10 @@ and may be set in the config file. --gcs-bucket-policy-only Access checks should use bucket-level IAM policies --gcs-client-id string OAuth Client Id --gcs-client-secret string OAuth Client Secret + --gcs-decompress If set this will decompress gzip encoded objects --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) --gcs-location string Location for the newly created buckets + --gcs-no-check-bucket If set, don't attempt to check the bucket exists or create it --gcs-object-acl string Access Control List for new objects --gcs-project-number string Project number --gcs-service-account-file string Service Account Credentials JSON file path @@ -384,10 +392,24 @@ and may be set in the config file. --hdfs-namenode string Hadoop name node and port --hdfs-service-principal-name string Kerberos service principal name for the namenode --hdfs-username string Hadoop user name + --hidrive-auth-url string Auth server URL + --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) + --hidrive-client-id string OAuth Client Id + --hidrive-client-secret string OAuth Client Secret + --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary + --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --hidrive-endpoint string Endpoint for the service (default "https://api.hidrive.strato.com/2.1") + --hidrive-root-prefix string The root/parent folder for all paths (default "/") + --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default "rw") + --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default "user") + --hidrive-token string OAuth Access Token as a JSON blob + --hidrive-token-url string Token server url + --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) + --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) --http-headers CommaSepList Set HTTP headers for all transactions --http-no-head Don't use HEAD requests --http-no-slash Set this if the site doesn't end directories with / - --http-url string URL of http host to connect to + --http-url string URL of HTTP host to connect to --hubic-auth-url string Auth server URL --hubic-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) --hubic-client-id string OAuth Client Id @@ -396,6 +418,13 @@ and may be set in the config file. --hubic-no-chunk Don't chunk files during streaming upload --hubic-token string OAuth Access Token as a JSON blob --hubic-token-url string Token server url + --internetarchive-access-key-id string IAS3 Access Key + --internetarchive-disable-checksum Don't ask the server to test against MD5 checksum calculated by rclone (default true) + --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) + --internetarchive-endpoint string IAS3 Endpoint (default "https://s3.us.archive.org") + --internetarchive-front-endpoint string Host of InternetArchive Frontend (default "https://archive.org") + --internetarchive-secret-access-key string IAS3 Secret Key (password) + --internetarchive-wait-archive Duration Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s) --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) --jottacloud-hard-delete Delete files permanently rather than putting them into the trash --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) @@ -417,7 +446,7 @@ and may be set in the config file. --local-no-preallocate Disable preallocation of disk space for transferred files --local-no-set-modtime Disable setting modtime --local-no-sparse Disable sparse files for multi-thread downloads - --local-nounc string Disable UNC (long path names) conversion on Windows + --local-nounc Disable UNC (long path names) conversion on Windows --local-unicode-normalization Apply unicode NFC normalization to paths and filenames --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) @@ -438,11 +467,11 @@ and may be set in the config file. --netstorage-protocol string Select between HTTP or HTTPS protocol (default "https") --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) -x, --one-file-system Don't cross filesystem boundaries (unix/macOS only) + --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) --onedrive-auth-url string Auth server URL --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) --onedrive-client-id string OAuth Client Id --onedrive-client-secret string OAuth Client Secret - --onedrive-disable-site-permission Disable the request for Sites.Read.All permission --onedrive-drive-id string The ID of the drive to use --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) @@ -466,9 +495,11 @@ and may be set in the config file. --pcloud-client-secret string OAuth Client Secret --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) --pcloud-hostname string Hostname to connect to (default "api.pcloud.com") + --pcloud-password string Your pcloud password (obscured) --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default "d0") --pcloud-token string OAuth Access Token as a JSON blob --pcloud-token-url string Token server url + --pcloud-username string Your pcloud username --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) --qingstor-access-key-id string QingStor Access Key ID @@ -521,6 +552,7 @@ and may be set in the config file. --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) + --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads --s3-v2-auth If true use v2 authentication --seafile-2fa Two-factor authentication ('true' if the account has 2FA enabled) --seafile-create-library Should rclone create a library if it doesn't exist @@ -531,6 +563,8 @@ and may be set in the config file. --seafile-url string URL of seafile host to connect to --seafile-user string User name (usually email address) --sftp-ask-password Allow asking for SFTP password when needed + --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) + --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) --sftp-disable-concurrent-reads If set don't use concurrent reads --sftp-disable-concurrent-writes If set don't use concurrent writes --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available @@ -543,12 +577,14 @@ and may be set in the config file. --sftp-known-hosts-file string Optional path to known_hosts file --sftp-md5sum-command string The command used to read md5 hashes --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) - --sftp-path-override string Override path used by SSH connection + --sftp-path-override string Override path used by SSH shell commands --sftp-port int SSH port number (default 22) --sftp-pubkey-file string Optional path to public key file --sftp-server-command string Specifies the path or command to run a sftp server on the remote host + --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands --sftp-set-modtime Set the modified time on the remote if set (default true) --sftp-sha1sum-command string The command used to read sha1 hashes + --sftp-shell-type string The type of SSH shell on remote server, if any --sftp-skip-links Set to skip any symlinks and any other non regular files --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default "sftp") --sftp-use-fstat If set use fstat instead of stat @@ -605,6 +641,7 @@ and may be set in the config file. --union-action-policy string Policy to choose upstream on ACTION category (default "epall") --union-cache-time int Cache time of usage and free space (in seconds) (default 120) --union-create-policy string Policy to choose upstream on CREATE category (default "epmfs") + --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) --union-search-policy string Policy to choose upstream on SEARCH category (default "ff") --union-upstreams string List of space separated upstreams --uptobox-access-token string Your access token @@ -616,7 +653,7 @@ and may be set in the config file. --webdav-pass string Password (obscured) --webdav-url string URL of http host to connect to --webdav-user string User name - --webdav-vendor string Name of the Webdav site/service/software you are using + --webdav-vendor string Name of the WebDAV site/service/software you are using --yandex-auth-url string Auth server URL --yandex-client-id string OAuth Client Id --yandex-client-secret string OAuth Client Secret diff --git a/docs/content/ftp.md b/docs/content/ftp.md index f2d4a4229ede5..617ca38d76272 100644 --- a/docs/content/ftp.md +++ b/docs/content/ftp.md @@ -138,7 +138,7 @@ Just hit a selection number when prompted. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/ftp/ftp.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to ftp (FTP Connection). +Here are the Standard options specific to ftp (FTP). #### --ftp-host @@ -221,7 +221,7 @@ Properties: ### Advanced options -Here are the advanced options specific to ftp (FTP Connection). +Here are the Advanced options specific to ftp (FTP). #### --ftp-concurrency diff --git a/docs/content/googlecloudstorage.md b/docs/content/googlecloudstorage.md index 3c1b6e2bc58cc..61b751d15d29e 100644 --- a/docs/content/googlecloudstorage.md +++ b/docs/content/googlecloudstorage.md @@ -273,7 +273,7 @@ as they can't be used in JSON strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/googlecloudstorage/googlecloudstorage.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). +Here are the Standard options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). #### --gcs-client-id @@ -548,7 +548,7 @@ Properties: ### Advanced options -Here are the advanced options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). +Here are the Advanced options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). #### --gcs-token @@ -587,6 +587,40 @@ Properties: - Type: string - Required: false +#### --gcs-no-check-bucket + +If set, don't attempt to check the bucket exists or create it. + +This can be useful when trying to minimise the number of transactions +rclone does if you know the bucket exists already. + + +Properties: + +- Config: no_check_bucket +- Env Var: RCLONE_GCS_NO_CHECK_BUCKET +- Type: bool +- Default: false + +#### --gcs-decompress + +If set this will decompress gzip encoded objects. + +It is possible to upload objects to GCS with "Content-Encoding: gzip" +set. Normally rclone will download these files files as compressed objects. + +If this flag is set then rclone will decompress these files with +"Content-Encoding: gzip" as they are received. This means that rclone +can't check the size and hash but the file contents will be decompressed. + + +Properties: + +- Config: decompress +- Env Var: RCLONE_GCS_DECOMPRESS +- Type: bool +- Default: false + #### --gcs-encoding The encoding for the backend. diff --git a/docs/content/googlephotos.md b/docs/content/googlephotos.md index 66ecabad4b437..da46f015687f0 100644 --- a/docs/content/googlephotos.md +++ b/docs/content/googlephotos.md @@ -224,7 +224,7 @@ This is similar to the Sharing tab in the Google Photos web interface. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/googlephotos/googlephotos.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to google photos (Google Photos). +Here are the Standard options specific to google photos (Google Photos). #### --gphotos-client-id @@ -268,7 +268,7 @@ Properties: ### Advanced options -Here are the advanced options specific to google photos (Google Photos). +Here are the Advanced options specific to google photos (Google Photos). #### --gphotos-token diff --git a/docs/content/hasher.md b/docs/content/hasher.md index 459dbe3202271..c92392af7ff90 100644 --- a/docs/content/hasher.md +++ b/docs/content/hasher.md @@ -172,7 +172,7 @@ or by full re-read/re-write of the files. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/hasher/hasher.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to hasher (Better checksums for other remotes). +Here are the Standard options specific to hasher (Better checksums for other remotes). #### --hasher-remote @@ -209,7 +209,7 @@ Properties: ### Advanced options -Here are the advanced options specific to hasher (Better checksums for other remotes). +Here are the Advanced options specific to hasher (Better checksums for other remotes). #### --hasher-auto-size @@ -222,6 +222,12 @@ Properties: - Type: SizeSuffix - Default: 0 +### Metadata + +Any metadata supported by the underlying remote is read and written. + +See the [metadata](/docs/#metadata) docs for more info. + ## Backend commands Here are the commands specific to the hasher backend. @@ -232,7 +238,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](/commands/rclone_backend/) for more +See the [backend](/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command diff --git a/docs/content/hdfs.md b/docs/content/hdfs.md index 5cf77fab51d91..d5c1d69e5c176 100644 --- a/docs/content/hdfs.md +++ b/docs/content/hdfs.md @@ -151,7 +151,7 @@ Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8). {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/hdfs/hdfs.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to hdfs (Hadoop distributed file system). +Here are the Standard options specific to hdfs (Hadoop distributed file system). #### --hdfs-namenode @@ -182,7 +182,7 @@ Properties: ### Advanced options -Here are the advanced options specific to hdfs (Hadoop distributed file system). +Here are the Advanced options specific to hdfs (Hadoop distributed file system). #### --hdfs-service-principal-name diff --git a/docs/content/hidrive.md b/docs/content/hidrive.md index 2d667a9e6d255..68375d1a22001 100644 --- a/docs/content/hidrive.md +++ b/docs/content/hidrive.md @@ -193,7 +193,7 @@ See the below section about configuration options for more details. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/hidrive/hidrive.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to hidrive (HiDrive). +Here are the Standard options specific to hidrive (HiDrive). #### --hidrive-client-id @@ -239,7 +239,7 @@ Properties: ### Advanced options -Here are the advanced options specific to hidrive (HiDrive). +Here are the Advanced options specific to hidrive (HiDrive). #### --hidrive-token @@ -346,25 +346,6 @@ Properties: - Type: bool - Default: false -#### --hidrive-disable-unicode-normalization - -Do not apply Unicode "Normalization Form C" to remote paths. - -In Unicode there are multiple valid representations for the same abstract character. -They (should) result in the same visual appearance, but are represented by different byte-sequences. -This is known as canonical equivalence. - -In HiDrive paths are always represented as byte-sequences. -This means that two paths that are canonically equivalent (and therefore look the same) are treated as two distinct paths. -As this behaviour may be undesired, by default rclone will apply unicode normalization to paths it will access. - -Properties: - -- Config: disable_unicode_normalization -- Env Var: RCLONE_HIDRIVE_DISABLE_UNICODE_NORMALIZATION -- Type: bool -- Default: false - #### --hidrive-chunk-size Chunksize for chunked uploads. diff --git a/docs/content/http.md b/docs/content/http.md index 10ef35098324c..d98052c598d81 100644 --- a/docs/content/http.md +++ b/docs/content/http.md @@ -126,11 +126,11 @@ or: {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/http/http.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to http (http Connection). +Here are the Standard options specific to http (HTTP). #### --http-url -URL of http host to connect to. +URL of HTTP host to connect to. E.g. "https://example.com", or "https://user:pass@example.com" to use a username and password. @@ -143,7 +143,7 @@ Properties: ### Advanced options -Here are the advanced options specific to http (http Connection). +Here are the Advanced options specific to http (HTTP). #### --http-headers diff --git a/docs/content/hubic.md b/docs/content/hubic.md index 017e901d4abf8..3000bc6af4290 100644 --- a/docs/content/hubic.md +++ b/docs/content/hubic.md @@ -109,7 +109,7 @@ are the same. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/hubic/hubic.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to hubic (Hubic). +Here are the Standard options specific to hubic (Hubic). #### --hubic-client-id @@ -139,7 +139,7 @@ Properties: ### Advanced options -Here are the advanced options specific to hubic (Hubic). +Here are the Advanced options specific to hubic (Hubic). #### --hubic-token diff --git a/docs/content/internetarchive.md b/docs/content/internetarchive.md index 622db4d608bf3..1bdb05962713d 100644 --- a/docs/content/internetarchive.md +++ b/docs/content/internetarchive.md @@ -146,7 +146,7 @@ y/e/d> y {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/internetarchive/internetarchive.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to internetarchive (Internet Archive). +Here are the Standard options specific to internetarchive (Internet Archive). #### --internetarchive-access-key-id @@ -177,7 +177,7 @@ Properties: ### Advanced options -Here are the advanced options specific to internetarchive (Internet Archive). +Here are the Advanced options specific to internetarchive (Internet Archive). #### --internetarchive-endpoint @@ -246,4 +246,32 @@ Properties: - Type: MultiEncoder - Default: Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot +### Metadata + +Metadata fields provided by Internet Archive. +If there are multiple values for a key, only the first one is returned. +This is a limitation of Rclone, that supports one value per one key. + +Owner is able to add custom keys. Metadata feature grabs all the keys including them. + +Here are the possible system metadata items for the internetarchive backend. + +| Name | Help | Type | Example | Read Only | +|------|------|------|---------|-----------| +| crc32 | CRC32 calculated by Internet Archive | string | 01234567 | N | +| format | Name of format identified by Internet Archive | string | Comma-Separated Values | N | +| md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | N | +| mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | +| name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | N | +| old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | N | +| rclone-ia-mtime | Time of last modification, managed by Internet Archive | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | +| rclone-mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | +| rclone-update-track | Random value used by Rclone for tracking changes inside Internet Archive | string | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | N | +| sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | N | +| size | File size in bytes | decimal number | 123456 | N | +| source | The source of the file | string | original | N | +| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | N | + +See the [metadata](/docs/#metadata) docs for more info. + {{< rem autogenerated options stop >}} diff --git a/docs/content/jottacloud.md b/docs/content/jottacloud.md index ef7ea558627e7..7d815e8c4b0af 100644 --- a/docs/content/jottacloud.md +++ b/docs/content/jottacloud.md @@ -266,7 +266,7 @@ and the current usage. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/jottacloud/jottacloud.go then run make backenddocs" >}} ### Advanced options -Here are the advanced options specific to jottacloud (Jottacloud). +Here are the Advanced options specific to jottacloud (Jottacloud). #### --jottacloud-md5-memory-limit diff --git a/docs/content/koofr.md b/docs/content/koofr.md index d1bd22976a39b..2025d64a318e4 100644 --- a/docs/content/koofr.md +++ b/docs/content/koofr.md @@ -113,7 +113,7 @@ as they can't be used in XML strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/koofr/koofr.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). +Here are the Standard options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). #### --koofr-provider @@ -200,7 +200,7 @@ Properties: ### Advanced options -Here are the advanced options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). +Here are the Advanced options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). #### --koofr-mountid diff --git a/docs/content/local.md b/docs/content/local.md index c17a22c1a8a70..f123aa74b1e26 100644 --- a/docs/content/local.md +++ b/docs/content/local.md @@ -327,7 +327,7 @@ where it isn't supported (e.g. Windows) it will be ignored. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/local/local.go then run make backenddocs" >}} ### Advanced options -Here are the advanced options specific to local (Local Disk). +Here are the Advanced options specific to local (Local Disk). #### --local-nounc @@ -337,8 +337,8 @@ Properties: - Config: nounc - Env Var: RCLONE_LOCAL_NOUNC -- Type: string -- Required: false +- Type: bool +- Default: false - Examples: - "true" - Disables long file names. @@ -586,7 +586,6 @@ Here are the possible system metadata items for the local backend. | rdev | Device ID (if special file) | hexadecimal | 1abc | N | | uid | User ID of owner | decimal number | 500 | N | - See the [metadata](/docs/#metadata) docs for more info. ## Backend commands @@ -599,7 +598,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](/commands/rclone_backend/) for more +See the [backend](/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command diff --git a/docs/content/mailru.md b/docs/content/mailru.md index 04a165a5c0897..96bd5aabb1acd 100644 --- a/docs/content/mailru.md +++ b/docs/content/mailru.md @@ -156,7 +156,7 @@ as they can't be used in JSON strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/mailru/mailru.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to mailru (Mail.ru Cloud). +Here are the Standard options specific to mailru (Mail.ru Cloud). #### --mailru-user @@ -209,7 +209,7 @@ Properties: ### Advanced options -Here are the advanced options specific to mailru (Mail.ru Cloud). +Here are the Advanced options specific to mailru (Mail.ru Cloud). #### --mailru-speedup-file-patterns diff --git a/docs/content/mega.md b/docs/content/mega.md index 27882e34232fd..cd26c70108ece 100644 --- a/docs/content/mega.md +++ b/docs/content/mega.md @@ -192,7 +192,7 @@ have got the remote blocked for a while. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/mega/mega.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to mega (Mega). +Here are the Standard options specific to mega (Mega). #### --mega-user @@ -220,7 +220,7 @@ Properties: ### Advanced options -Here are the advanced options specific to mega (Mega). +Here are the Advanced options specific to mega (Mega). #### --mega-debug diff --git a/docs/content/netstorage.md b/docs/content/netstorage.md index 428fcc3617a83..0c1bef5bdda4c 100644 --- a/docs/content/netstorage.md +++ b/docs/content/netstorage.md @@ -177,7 +177,7 @@ NetStorage remote supports the purge feature by using the "quick-delete" NetStor {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/netstorage/netstorage.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to netstorage (Akamai NetStorage). +Here are the Standard options specific to netstorage (Akamai NetStorage). #### --netstorage-host @@ -220,7 +220,7 @@ Properties: ### Advanced options -Here are the advanced options specific to netstorage (Akamai NetStorage). +Here are the Advanced options specific to netstorage (Akamai NetStorage). #### --netstorage-protocol @@ -251,7 +251,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](/commands/rclone_backend/) for more +See the [backend](/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command @@ -277,10 +277,4 @@ the object that will be the target of the symlink (for example, /links/mylink). Include the file extension for the object, if applicable. `rclone backend symlink ` -## Support - -If you have any questions or issues, please contact [Akamai Technical Support -via Control Center or by -phone](https://control.akamai.com/apps/support-ui/#/contact-support). - {{< rem autogenerated options stop >}} diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index 36f3c89fbe07c..b3141578753c5 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -217,7 +217,7 @@ the OneDrive website. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/onedrive/onedrive.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to onedrive (Microsoft OneDrive). +Here are the Standard options specific to onedrive (Microsoft OneDrive). #### --onedrive-client-id @@ -267,7 +267,7 @@ Properties: ### Advanced options -Here are the advanced options specific to onedrive (Microsoft OneDrive). +Here are the Advanced options specific to onedrive (Microsoft OneDrive). #### --onedrive-token @@ -359,6 +359,28 @@ Properties: - Type: string - Required: false +#### --onedrive-access-scopes + +Set scopes to be requested by rclone. + +Choose or manually enter a custom space separated list with all scopes, that rclone should request. + + +Properties: + +- Config: access_scopes +- Env Var: RCLONE_ONEDRIVE_ACCESS_SCOPES +- Type: SpaceSepList +- Default: Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access +- Examples: + - "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access" + - Read and write access to all resources + - "Files.Read Files.Read.All Sites.Read.All offline_access" + - Read only access to all resources + - "Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All offline_access" + - Read and write access to all resources, without the ability to browse SharePoint sites. + - Same as if disable_site_permission was set to true + #### --onedrive-disable-site-permission Disable the request for Sites.Read.All permission. diff --git a/docs/content/opendrive.md b/docs/content/opendrive.md index a39072d3f863f..772bc051fa434 100644 --- a/docs/content/opendrive.md +++ b/docs/content/opendrive.md @@ -102,7 +102,7 @@ as they can't be used in JSON strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/opendrive/opendrive.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to opendrive (OpenDrive). +Here are the Standard options specific to opendrive (OpenDrive). #### --opendrive-username @@ -130,7 +130,7 @@ Properties: ### Advanced options -Here are the advanced options specific to opendrive (OpenDrive). +Here are the Advanced options specific to opendrive (OpenDrive). #### --opendrive-encoding diff --git a/docs/content/pcloud.md b/docs/content/pcloud.md index eb11905dd5987..6e0c4458ab088 100644 --- a/docs/content/pcloud.md +++ b/docs/content/pcloud.md @@ -144,7 +144,7 @@ the `root_folder_id` in the config. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/pcloud/pcloud.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to pcloud (Pcloud). +Here are the Standard options specific to pcloud (Pcloud). #### --pcloud-client-id @@ -174,7 +174,7 @@ Properties: ### Advanced options -Here are the advanced options specific to pcloud (Pcloud). +Here are the Advanced options specific to pcloud (Pcloud). #### --pcloud-token diff --git a/docs/content/premiumizeme.md b/docs/content/premiumizeme.md index ed71c09ea803b..e8039764d1eef 100644 --- a/docs/content/premiumizeme.md +++ b/docs/content/premiumizeme.md @@ -104,7 +104,7 @@ as they can't be used in JSON strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/premiumizeme/premiumizeme.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to premiumizeme (premiumize.me). +Here are the Standard options specific to premiumizeme (premiumize.me). #### --premiumizeme-api-key @@ -122,7 +122,7 @@ Properties: ### Advanced options -Here are the advanced options specific to premiumizeme (premiumize.me). +Here are the Advanced options specific to premiumizeme (premiumize.me). #### --premiumizeme-encoding diff --git a/docs/content/putio.md b/docs/content/putio.md index f80a1f3bdcc5b..d98da528e57ee 100644 --- a/docs/content/putio.md +++ b/docs/content/putio.md @@ -111,7 +111,7 @@ as they can't be used in JSON strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/putio/putio.go then run make backenddocs" >}} ### Advanced options -Here are the advanced options specific to putio (Put.io). +Here are the Advanced options specific to putio (Put.io). #### --putio-encoding diff --git a/docs/content/qingstor.md b/docs/content/qingstor.md index 2cffe23b170de..971e59bc6f94d 100644 --- a/docs/content/qingstor.md +++ b/docs/content/qingstor.md @@ -144,7 +144,7 @@ as they can't be used in JSON strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/qingstor/qingstor.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to qingstor (QingCloud Object Storage). +Here are the Standard options specific to qingstor (QingCloud Object Storage). #### --qingstor-env-auth @@ -228,7 +228,7 @@ Properties: ### Advanced options -Here are the advanced options specific to qingstor (QingCloud Object Storage). +Here are the Advanced options specific to qingstor (QingCloud Object Storage). #### --qingstor-connection-retries diff --git a/docs/content/rc.md b/docs/content/rc.md index 8ea6b77d6a655..fc6316e51974e 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -544,6 +544,7 @@ This takes the following parameters: - state - state to restart with - used with continue - result - result to restart with - used with continue + See the [config create](/commands/rclone_config_create/) command for more information on the above. **Authentication is required for this call.** @@ -595,6 +596,7 @@ This takes the following parameters: - name - name of remote - parameters - a map of \{ "key": "value" \} pairs + See the [config password](/commands/rclone_config_password/) command for more information on the above. **Authentication is required for this call.** @@ -623,6 +625,7 @@ This takes the following parameters: - state - state to restart with - used with continue - result - result to restart with - used with continue + See the [config update](/commands/rclone_config_update/) command for more information on the above. **Authentication is required for this call.** @@ -1069,7 +1072,7 @@ This takes the following parameters: The result is as returned from rclone about --json -See the [about](/commands/rclone_size/) command for more information on the above. +See the [about](/commands/rclone_about/) command for more information on the above. **Authentication is required for this call.** @@ -1101,7 +1104,7 @@ This takes the following parameters: - fs - a remote name string e.g. "drive:" - remote - a path within that remote e.g. "dir" - url - string, URL to read from -- autoFilename - boolean, set to true to retrieve destination file name from url + - autoFilename - boolean, set to true to retrieve destination file name from url See the [copyurl](/commands/rclone_copyurl/) command for more information on the above. @@ -1138,46 +1141,103 @@ This returns info about the remote passed in; ``` { - // optional features and whether they are available or not - "Features": { - "About": true, - "BucketBased": false, - "CanHaveEmptyDirectories": true, - "CaseInsensitive": false, - "ChangeNotify": false, - "CleanUp": false, - "Copy": false, - "DirCacheFlush": false, - "DirMove": true, - "DuplicateFiles": false, - "GetTier": false, - "ListR": false, - "MergeDirs": false, - "Move": true, - "OpenWriterAt": true, - "PublicLink": false, - "Purge": true, - "PutStream": true, - "PutUnchecked": false, - "ReadMimeType": false, - "ServerSideAcrossConfigs": false, - "SetTier": false, - "SetWrapper": false, - "UnWrap": false, - "WrapFs": false, - "WriteMimeType": false - }, - // Names of hashes available - "Hashes": [ - "MD5", - "SHA-1", - "DropboxHash", - "QuickXorHash" - ], - "Name": "local", // Name as created - "Precision": 1, // Precision of timestamps in ns - "Root": "/", // Path as created - "String": "Local file system at /" // how the remote will appear in logs + // optional features and whether they are available or not + "Features": { + "About": true, + "BucketBased": false, + "BucketBasedRootOK": false, + "CanHaveEmptyDirectories": true, + "CaseInsensitive": false, + "ChangeNotify": false, + "CleanUp": false, + "Command": true, + "Copy": false, + "DirCacheFlush": false, + "DirMove": true, + "Disconnect": false, + "DuplicateFiles": false, + "GetTier": false, + "IsLocal": true, + "ListR": false, + "MergeDirs": false, + "MetadataInfo": true, + "Move": true, + "OpenWriterAt": true, + "PublicLink": false, + "Purge": true, + "PutStream": true, + "PutUnchecked": false, + "ReadMetadata": true, + "ReadMimeType": false, + "ServerSideAcrossConfigs": false, + "SetTier": false, + "SetWrapper": false, + "Shutdown": false, + "SlowHash": true, + "SlowModTime": false, + "UnWrap": false, + "UserInfo": false, + "UserMetadata": true, + "WrapFs": false, + "WriteMetadata": true, + "WriteMimeType": false + }, + // Names of hashes available + "Hashes": [ + "md5", + "sha1", + "whirlpool", + "crc32", + "sha256", + "dropbox", + "mailru", + "quickxor" + ], + "Name": "local", // Name as created + "Precision": 1, // Precision of timestamps in ns + "Root": "/", // Path as created + "String": "Local file system at /", // how the remote will appear in logs + // Information about the system metadata for this backend + "MetadataInfo": { + "System": { + "atime": { + "Help": "Time of last access", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "btime": { + "Help": "Time of file birth (creation)", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "gid": { + "Help": "Group ID of owner", + "Type": "decimal number", + "Example": "500" + }, + "mode": { + "Help": "File type and mode", + "Type": "octal, unix style", + "Example": "0100664" + }, + "mtime": { + "Help": "Time of last modification", + "Type": "RFC 3339", + "Example": "2006-01-02T15:04:05.999999999Z07:00" + }, + "rdev": { + "Help": "Device ID (if special file)", + "Type": "hexadecimal", + "Example": "1abc" + }, + "uid": { + "Help": "User ID of owner", + "Type": "decimal number", + "Example": "500" + } + }, + "Help": "Textual help string\n" + } } ``` @@ -1200,6 +1260,7 @@ This takes the following parameters: - noMimeType - If set don't show mime types - dirsOnly - If set only show directories - filesOnly - If set only show files + - metadata - If set return metadata of objects also - hashTypes - array of strings of hash types to show if showHash set Returns: @@ -1207,7 +1268,7 @@ Returns: - list - This is an array of objects as described in the lsjson command -See the [lsjson](/commands/rclone_lsjson/) for more information on the above and examples. +See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples. **Authentication is required for this call.** @@ -1294,7 +1355,6 @@ Returns: - count - number of files - bytes - number of bytes in those files -- sizeless - number of files with unknown size, included in count but not accounted for in bytes See the [size](/commands/rclone_size/) command for more information on the above. @@ -1316,7 +1376,7 @@ The result is Note that if you are only interested in files then it is much more efficient to set the filesOnly flag in the options. -See the [lsjson](/commands/rclone_lsjson/) for more information on the above and examples. +See the [lsjson](/commands/rclone_lsjson/) command for more information on the above and examples. **Authentication is required for this call.** @@ -1542,6 +1602,7 @@ This takes the following parameters: - dstFs - a remote name string e.g. "drive:dst" for the destination - createEmptySrcDirs - create empty src directories on destination if set + See the [copy](/commands/rclone_copy/) command for more information on the above. **Authentication is required for this call.** @@ -1555,6 +1616,7 @@ This takes the following parameters: - createEmptySrcDirs - create empty src directories on destination if set - deleteEmptySrcDirs - delete empty src directories if set + See the [move](/commands/rclone_move/) command for more information on the above. **Authentication is required for this call.** diff --git a/docs/content/s3.md b/docs/content/s3.md index ddf9fd23dec58..e68b2f52ac32e 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -13,7 +13,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}} {{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}} {{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}} -{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}} +{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud" >}} {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}} {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} {{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}} @@ -571,7 +571,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS). +Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi). #### --s3-provider @@ -592,6 +592,8 @@ Properties: - Ceph Object Storage - "ChinaMobile" - China Mobile Ecloud Elastic Object Storage (EOS) + - "Cloudflare" + - Cloudflare R2 Storage - "ArvanCloud" - Arvan Cloud Object Storage (AOS) - "DigitalOcean" @@ -828,6 +830,67 @@ Properties: - Amsterdam, The Netherlands - "fr-par" - Paris, France + - "pl-waw" + - Warsaw, Poland + +#### --s3-region + +Region to connect to. - the location where your bucket will be created and your data stored. Need bo be same with your endpoint. + + +Properties: + +- Config: region +- Env Var: RCLONE_S3_REGION +- Provider: HuaweiOBS +- Type: string +- Required: false +- Examples: + - "af-south-1" + - AF-Johannesburg + - "ap-southeast-2" + - AP-Bangkok + - "ap-southeast-3" + - AP-Singapore + - "cn-east-3" + - CN East-Shanghai1 + - "cn-east-2" + - CN East-Shanghai2 + - "cn-north-1" + - CN North-Beijing1 + - "cn-north-4" + - CN North-Beijing4 + - "cn-south-1" + - CN South-Guangzhou + - "ap-southeast-1" + - CN-Hong Kong + - "sa-argentina-1" + - LA-Buenos Aires1 + - "sa-peru-1" + - LA-Lima1 + - "na-mexico-1" + - LA-Mexico City1 + - "sa-chile-1" + - LA-Santiago2 + - "sa-brazil-1" + - LA-Sao Paulo1 + - "ru-northwest-2" + - RU-Moscow2 + +#### --s3-region + +Region to connect to. + +Properties: + +- Config: region +- Env Var: RCLONE_S3_REGION +- Provider: Cloudflare +- Type: string +- Required: false +- Examples: + - "auto" + - R2 buckets are automatically distributed across Cloudflare's data centers for low latency. #### --s3-region @@ -839,7 +902,7 @@ Properties: - Config: region - Env Var: RCLONE_S3_REGION -- Provider: !AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive +- Provider: !AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive - Type: string - Required: false - Examples: @@ -868,6 +931,8 @@ Properties: Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API. +Properties: + - Config: endpoint - Env Var: RCLONE_S3_ENDPOINT - Provider: ChinaMobile @@ -925,7 +990,7 @@ Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API. - Gansu China (Lanzhou) - "eos-shanxi-1.cmecloud.cn" - Shanxi China (Taiyuan) - - eos-liaoning-1.cmecloud.cn" + - "eos-liaoning-1.cmecloud.cn" - Liaoning China (Shenyang) - "eos-hebei-1.cmecloud.cn" - Hebei China (Shijiazhuang) @@ -940,6 +1005,8 @@ Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API. Endpoint for Arvan Cloud Object Storage (AOS) API. +Properties: + - Config: endpoint - Env Var: RCLONE_S3_ENDPOINT - Provider: ArvanCloud @@ -952,50 +1019,6 @@ Endpoint for Arvan Cloud Object Storage (AOS) API. - "s3.ir-tbz-sh1.arvanstorage.com" - Tabriz Iran (Shahriar) -#### --s3-endpoint - -Endpoint for Huawei Cloud Object Storage Service (OBS) API. - -- Config: endpoint -- Env Var: RCLONE_S3_ENDPOINT -- Provider: HuaweiOBS -- Type: string -- Required: false -- Examples: - - "obs.af-south-1.myhuaweicloud.com" - - AF-Johannesburg Endpoint - - "obs.ap-southeast-2.myhuaweicloud.com" - - AP-Bangkok Endpoint - - "obs.ap-southeast-3.myhuaweicloud.com" - - AP-Singapore Endpoint - - "obs.cn-east-3.myhuaweicloud.com" - - CN East-Shanghai1 Endpoint - - "obs.cn-east-2.myhuaweicloud.com" - - CN East-Shanghai2 Endpoint - - "obs.cn-north-1.myhuaweicloud.com" - - CN North-Beijing1 Endpoint - - "obs.cn-north-4.myhuaweicloud.com" - - CN North-Beijing4 Endpoint - - "obs.cn-south-1.myhuaweicloud.com" - - CN South-Guangzhou Endpoint - - "obs.ap-southeast-1.myhuaweicloud.com" - - CN-Hong Kong Endpoint - - "obs.sa-argentina-1.myhuaweicloud.com" - - LA-Buenos Aires1 Endpoint - - "obs.sa-peru-1.myhuaweicloud.com" - - LA-Lima1 Endpoint - - "obs.na-mexico-1.myhuaweicloud.com" - - LA-Mexico City1 Endpoint - - "obs.sa-chile-1.myhuaweicloud.com" - - LA-Santiago2 Endpoint - - "obs.sa-brazil-1.myhuaweicloud.com" - - LA-Sao Paulo1 Endpoint - - "obs.ru-northwest-2.myhuaweicloud.com" - - RU-Moscow2 Endpoint - - - - #### --s3-endpoint Endpoint for IBM COS S3 API. @@ -1200,6 +1223,49 @@ Properties: #### --s3-endpoint +Endpoint for OBS API. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Provider: HuaweiOBS +- Type: string +- Required: false +- Examples: + - "obs.af-south-1.myhuaweicloud.com" + - AF-Johannesburg + - "obs.ap-southeast-2.myhuaweicloud.com" + - AP-Bangkok + - "obs.ap-southeast-3.myhuaweicloud.com" + - AP-Singapore + - "obs.cn-east-3.myhuaweicloud.com" + - CN East-Shanghai1 + - "obs.cn-east-2.myhuaweicloud.com" + - CN East-Shanghai2 + - "obs.cn-north-1.myhuaweicloud.com" + - CN North-Beijing1 + - "obs.cn-north-4.myhuaweicloud.com" + - CN North-Beijing4 + - "obs.cn-south-1.myhuaweicloud.com" + - CN South-Guangzhou + - "obs.ap-southeast-1.myhuaweicloud.com" + - CN-Hong Kong + - "obs.sa-argentina-1.myhuaweicloud.com" + - LA-Buenos Aires1 + - "obs.sa-peru-1.myhuaweicloud.com" + - LA-Lima1 + - "obs.na-mexico-1.myhuaweicloud.com" + - LA-Mexico City1 + - "obs.sa-chile-1.myhuaweicloud.com" + - LA-Santiago2 + - "obs.sa-brazil-1.myhuaweicloud.com" + - LA-Sao Paulo1 + - "obs.ru-northwest-2.myhuaweicloud.com" + - RU-Moscow2 + +#### --s3-endpoint + Endpoint for Scaleway Object Storage. Properties: @@ -1214,6 +1280,8 @@ Properties: - Amsterdam Endpoint - "s3.fr-par.scw.cloud" - Paris Endpoint + - "s3.pl-waw.scw.cloud" + - Warsaw Endpoint #### --s3-endpoint @@ -1365,7 +1433,7 @@ Properties: - Config: endpoint - Env Var: RCLONE_S3_ENDPOINT -- Provider: !AWS,IBMCOS,IDrive,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,HuaweiOBS +- Provider: !AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp - Type: string - Required: false - Examples: @@ -1465,6 +1533,100 @@ Properties: #### --s3-location-constraint +Location constraint - must match endpoint. + +Used when creating buckets only. + +Properties: + +- Config: location_constraint +- Env Var: RCLONE_S3_LOCATION_CONSTRAINT +- Provider: ChinaMobile +- Type: string +- Required: false +- Examples: + - "wuxi1" + - East China (Suzhou) + - "jinan1" + - East China (Jinan) + - "ningbo1" + - East China (Hangzhou) + - "shanghai1" + - East China (Shanghai-1) + - "zhengzhou1" + - Central China (Zhengzhou) + - "hunan1" + - Central China (Changsha-1) + - "zhuzhou1" + - Central China (Changsha-2) + - "guangzhou1" + - South China (Guangzhou-2) + - "dongguan1" + - South China (Guangzhou-3) + - "beijing1" + - North China (Beijing-1) + - "beijing2" + - North China (Beijing-2) + - "beijing4" + - North China (Beijing-3) + - "huhehaote1" + - North China (Huhehaote) + - "chengdu1" + - Southwest China (Chengdu) + - "chongqing1" + - Southwest China (Chongqing) + - "guiyang1" + - Southwest China (Guiyang) + - "xian1" + - Nouthwest China (Xian) + - "yunnan" + - Yunnan China (Kunming) + - "yunnan2" + - Yunnan China (Kunming-2) + - "tianjin1" + - Tianjin China (Tianjin) + - "jilin1" + - Jilin China (Changchun) + - "hubei1" + - Hubei China (Xiangyan) + - "jiangxi1" + - Jiangxi China (Nanchang) + - "gansu1" + - Gansu China (Lanzhou) + - "shanxi1" + - Shanxi China (Taiyuan) + - "liaoning1" + - Liaoning China (Shenyang) + - "hebei1" + - Hebei China (Shijiazhuang) + - "fujian1" + - Fujian China (Xiamen) + - "guangxi1" + - Guangxi China (Nanning) + - "anhui1" + - Anhui China (Huainan) + +#### --s3-location-constraint + +Location constraint - must match endpoint. + +Used when creating buckets only. + +Properties: + +- Config: location_constraint +- Env Var: RCLONE_S3_LOCATION_CONSTRAINT +- Provider: ArvanCloud +- Type: string +- Required: false +- Examples: + - "ir-thr-at1" + - Tehran Iran (Asiatech) + - "ir-tbz-sh1" + - Tabriz Iran (Shahriar) + +#### --s3-location-constraint + Location constraint - must match endpoint when using IBM Cloud Public. For on-prem COS, do not make a selection from this list, hit enter. @@ -1604,7 +1766,7 @@ Properties: - Config: location_constraint - Env Var: RCLONE_S3_LOCATION_CONSTRAINT -- Provider: !AWS,IBMCOS,IDrive,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS,HuaweiOBS +- Provider: !AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS - Type: string - Required: false @@ -1623,7 +1785,7 @@ Properties: - Config: acl - Env Var: RCLONE_S3_ACL -- Provider: !Storj +- Provider: !Storj,Cloudflare - Type: string - Required: false - Examples: @@ -1676,7 +1838,7 @@ Properties: - Config: server_side_encryption - Env Var: RCLONE_S3_SERVER_SIDE_ENCRYPTION -- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -1760,6 +1922,8 @@ Properties: The storage class to use when storing new objects in ChinaMobile. +Properties: + - Config: storage_class - Env Var: RCLONE_S3_STORAGE_CLASS - Provider: ChinaMobile @@ -1779,6 +1943,8 @@ The storage class to use when storing new objects in ChinaMobile. The storage class to use when storing new objects in ArvanCloud. +Properties: + - Config: storage_class - Env Var: RCLONE_S3_STORAGE_CLASS - Provider: ArvanCloud @@ -1832,7 +1998,7 @@ Properties: ### Advanced options -Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS). +Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi). #### --s3-bucket-acl @@ -1884,7 +2050,7 @@ Properties: - Config: sse_customer_algorithm - Env Var: RCLONE_S3_SSE_CUSTOMER_ALGORITHM -- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -1901,7 +2067,7 @@ Properties: - Config: sse_customer_key - Env Var: RCLONE_S3_SSE_CUSTOMER_KEY -- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -1919,7 +2085,7 @@ Properties: - Config: sse_customer_key_md5 - Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_MD5 -- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio +- Provider: AWS,Ceph,ChinaMobile,Minio - Type: string - Required: false - Examples: @@ -1964,6 +2130,13 @@ most 10,000 chunks, this means that by default the maximum size of a file you can stream upload is 48 GiB. If you wish to stream upload larger files then you will need to increase chunk_size. +Increasing the chunk size decreases the accuracy of the progress +statistics displayed with "-P" flag. Rclone treats chunk as sent when +it's buffered by the AWS SDK, when in fact it may still be uploading. +A bigger chunk size means a bigger AWS SDK buffer and progress +reporting more deviating from the truth. + + Properties: - Config: chunk_size @@ -2369,6 +2542,26 @@ Properties: - Type: Tristate - Default: unset +#### --s3-use-presigned-request + +Whether to use a presigned request or PutObject for single part uploads + +If this is false rclone will use PutObject from the AWS SDK to upload +an object. + +Versions of rclone < 1.59 use presigned requests to upload a single +part object and setting this flag to true will re-enable that +functionality. This shouldn't be necessary except in exceptional +circumstances or for testing. + + +Properties: + +- Config: use_presigned_request +- Env Var: RCLONE_S3_USE_PRESIGNED_REQUEST +- Type: bool +- Default: false + ### Metadata User metadata is stored as x-amz-meta- keys. S3 metadata keys are case insensitive and are always returned in lower case. @@ -2386,7 +2579,6 @@ Here are the possible system metadata items for the s3 backend. | mtime | Time of last modification, read from rclone metadata | RFC 3339 | 2006-01-02T15:04:05.999999999Z07:00 | N | | tier | Tier of the object | string | GLACIER | **Y** | - See the [metadata](/docs/#metadata) docs for more info. ## Backend commands @@ -2399,7 +2591,7 @@ Run them with The help below will explain what arguments each command takes. -See [the "rclone backend" command](/commands/rclone_backend/) for more +See the [backend](/commands/rclone_backend/) command for more info on how to pass options and arguments. These can be run on a running backend using the rc command @@ -3991,7 +4183,7 @@ d) Delete this remote y/e/d> y ``` -### ArvanCloud +### ArvanCloud {#arvan-cloud} [ArvanCloud](https://www.arvancloud.com/en/products/cloud-storage) ArvanCloud Object Storage goes beyond the limited traditional file storage. It gives you access to backup and archived files and allows sharing. diff --git a/docs/content/seafile.md b/docs/content/seafile.md index 407de6f4161be..132a405a13e2b 100644 --- a/docs/content/seafile.md +++ b/docs/content/seafile.md @@ -266,7 +266,7 @@ Versions between 6.0 and 6.3 haven't been tested and might not work properly. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/seafile/seafile.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to seafile (seafile). +Here are the Standard options specific to seafile (seafile). #### --seafile-url @@ -358,7 +358,7 @@ Properties: ### Advanced options -Here are the advanced options specific to seafile (seafile). +Here are the Advanced options specific to seafile (seafile). #### --seafile-create-library diff --git a/docs/content/sftp.md b/docs/content/sftp.md index ed5eec9de3d68..8f3066a930504 100644 --- a/docs/content/sftp.md +++ b/docs/content/sftp.md @@ -388,7 +388,7 @@ with a Windows OpenSSH server, rclone will use a built-in shell command {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/sftp/sftp.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to sftp (SSH/SFTP Connection). +Here are the Standard options specific to sftp (SSH/SFTP). #### --sftp-host @@ -514,7 +514,7 @@ Properties: #### --sftp-use-insecure-cipher -Enable the use of insecure ciphers and key exchange methods. +Enable the use of insecure ciphers and key exchange methods. This enables the use of the following insecure ciphers and key exchange methods: @@ -554,7 +554,7 @@ Properties: ### Advanced options -Here are the advanced options specific to sftp (SSH/SFTP Connection). +Here are the Advanced options specific to sftp (SSH/SFTP). #### --sftp-known-hosts-file @@ -592,16 +592,16 @@ Properties: #### --sftp-path-override -Override path used by SSH connection. +Override path used by SSH shell commands. This allows checksum calculation when SFTP and SSH paths are different. This issue affects among others Synology NAS boxes. -Shared folders can be found in directories representing volumes +E.g. if shared folders can be found in directories representing volumes: rclone sync /home/local/directory remote:/directory --sftp-path-override /volume2/directory -Home directory can be found in a shared folder called "home" +E.g. if home directory can be found in a shared folder called "home": rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory @@ -623,6 +623,28 @@ Properties: - Type: bool - Default: true +#### --sftp-shell-type + +The type of SSH shell on remote server, if any. + +Leave blank for autodetect. + +Properties: + +- Config: shell_type +- Env Var: RCLONE_SFTP_SHELL_TYPE +- Type: string +- Required: false +- Examples: + - "none" + - No shell access + - "unix" + - Unix shell + - "powershell" + - PowerShell + - "cmd" + - Windows Command Prompt + #### --sftp-md5sum-command The command used to read md5 hashes. @@ -763,6 +785,75 @@ Properties: - Type: Duration - Default: 1m0s +#### --sftp-chunk-size + +Upload and download chunk size. + +This controls the maximum packet size used in the SFTP protocol. The +RFC limits this to 32768 bytes (32k), however a lot of servers +support larger sizes and setting it larger will increase transfer +speed dramatically on high latency links. + +Only use a setting higher than 32k if you always connect to the same +server or after sufficiently broad testing. + +For example using the value of 252k with OpenSSH works well with its +maximum packet size of 256k. + +If you get the error "failed to send packet header: EOF" when copying +a large file, try lowering this number. + + +Properties: + +- Config: chunk_size +- Env Var: RCLONE_SFTP_CHUNK_SIZE +- Type: SizeSuffix +- Default: 32Ki + +#### --sftp-concurrency + +The maximum number of outstanding requests for one file + +This controls the maximum number of outstanding requests for one file. +Increasing it will increase throughput on high latency links at the +cost of using more memory. + + +Properties: + +- Config: concurrency +- Env Var: RCLONE_SFTP_CONCURRENCY +- Type: int +- Default: 64 + +#### --sftp-set-env + +Environment variables to pass to sftp and commands + +Set environment variables in the form: + + VAR=value + +to be passed to the sftp client and to any commands run (eg md5sum). + +Pass multiple variables space separated, eg + + VAR1=value VAR2=value + +and pass variables with spaces in in quotes, eg + + "VAR3=value with space" "VAR4=value with space" VAR5=nospacehere + + + +Properties: + +- Config: set_env +- Env Var: RCLONE_SFTP_SET_ENV +- Type: SpaceSepList +- Default: + {{< rem autogenerated options stop >}} ## Limitations diff --git a/docs/content/sharefile.md b/docs/content/sharefile.md index c33651b90557c..19cdc939700a3 100644 --- a/docs/content/sharefile.md +++ b/docs/content/sharefile.md @@ -150,7 +150,7 @@ as they can't be used in JSON strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/sharefile/sharefile.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to sharefile (Citrix Sharefile). +Here are the Standard options specific to sharefile (Citrix Sharefile). #### --sharefile-root-folder-id @@ -179,7 +179,7 @@ Properties: ### Advanced options -Here are the advanced options specific to sharefile (Citrix Sharefile). +Here are the Advanced options specific to sharefile (Citrix Sharefile). #### --sharefile-upload-cutoff diff --git a/docs/content/sia.md b/docs/content/sia.md index 0d3d920312a93..9cdb24b25bec2 100644 --- a/docs/content/sia.md +++ b/docs/content/sia.md @@ -132,7 +132,7 @@ rclone copy /home/source mySia:backup {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/sia/sia.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to sia (Sia Decentralized Cloud). +Here are the Standard options specific to sia (Sia Decentralized Cloud). #### --sia-api-url @@ -165,7 +165,7 @@ Properties: ### Advanced options -Here are the advanced options specific to sia (Sia Decentralized Cloud). +Here are the Advanced options specific to sia (Sia Decentralized Cloud). #### --sia-user-agent diff --git a/docs/content/storj.md b/docs/content/storj.md index 836718e38d221..6a8e416239e22 100644 --- a/docs/content/storj.md +++ b/docs/content/storj.md @@ -215,7 +215,7 @@ y/e/d> y {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/storj/storj.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to storj (Storj Decentralized Cloud Storage). +Here are the Standard options specific to storj (Storj Decentralized Cloud Storage). #### --storj-provider diff --git a/docs/content/sugarsync.md b/docs/content/sugarsync.md index 8e66557230022..58c163b325931 100644 --- a/docs/content/sugarsync.md +++ b/docs/content/sugarsync.md @@ -123,7 +123,7 @@ deleted straight away. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/sugarsync/sugarsync.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to sugarsync (Sugarsync). +Here are the Standard options specific to sugarsync (Sugarsync). #### --sugarsync-app-id @@ -178,7 +178,7 @@ Properties: ### Advanced options -Here are the advanced options specific to sugarsync (Sugarsync). +Here are the Advanced options specific to sugarsync (Sugarsync). #### --sugarsync-refresh-token diff --git a/docs/content/swift.md b/docs/content/swift.md index 870179ca0f12a..70cde652f2d61 100644 --- a/docs/content/swift.md +++ b/docs/content/swift.md @@ -245,7 +245,7 @@ as they can't be used in JSON strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/swift/swift.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). +Here are the Standard options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). #### --swift-env-auth @@ -485,7 +485,7 @@ Properties: ### Advanced options -Here are the advanced options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). +Here are the Advanced options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). #### --swift-leave-parts-on-error diff --git a/docs/content/union.md b/docs/content/union.md index be0f81007300f..d7f37fa7194c8 100644 --- a/docs/content/union.md +++ b/docs/content/union.md @@ -174,7 +174,7 @@ The policies definition are inspired by [trapexit/mergerfs](https://github.com/t {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/union/union.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to union (Union merges the contents of several upstream fs). +Here are the Standard options specific to union (Union merges the contents of several upstream fs). #### --union-upstreams @@ -235,4 +235,28 @@ Properties: - Type: int - Default: 120 +### Advanced options + +Here are the Advanced options specific to union (Union merges the contents of several upstream fs). + +#### --union-min-free-space + +Minimum viable free space for lfs/eplfs policies. + +If a remote has less than this much free space then it won't be +considered for use in lfs or eplfs policies. + +Properties: + +- Config: min_free_space +- Env Var: RCLONE_UNION_MIN_FREE_SPACE +- Type: SizeSuffix +- Default: 1Gi + +### Metadata + +Any metadata supported by the underlying remote is read and written. + +See the [metadata](/docs/#metadata) docs for more info. + {{< rem autogenerated options stop >}} diff --git a/docs/content/uptobox.md b/docs/content/uptobox.md index 626c96ffd1ec0..83f63916bc7c3 100644 --- a/docs/content/uptobox.md +++ b/docs/content/uptobox.md @@ -101,7 +101,7 @@ as they can't be used in XML strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/uptobox/uptobox.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to uptobox (Uptobox). +Here are the Standard options specific to uptobox (Uptobox). #### --uptobox-access-token @@ -118,7 +118,7 @@ Properties: ### Advanced options -Here are the advanced options specific to uptobox (Uptobox). +Here are the Advanced options specific to uptobox (Uptobox). #### --uptobox-encoding diff --git a/docs/content/webdav.md b/docs/content/webdav.md index 766a5cc8cd28b..9780941ebe4b0 100644 --- a/docs/content/webdav.md +++ b/docs/content/webdav.md @@ -110,7 +110,7 @@ with them. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/webdav/webdav.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to webdav (Webdav). +Here are the Standard options specific to webdav (WebDAV). #### --webdav-url @@ -127,7 +127,7 @@ Properties: #### --webdav-vendor -Name of the Webdav site/service/software you are using. +Name of the WebDAV site/service/software you are using. Properties: @@ -186,7 +186,7 @@ Properties: ### Advanced options -Here are the advanced options specific to webdav (Webdav). +Here are the Advanced options specific to webdav (WebDAV). #### --webdav-bearer-token-command diff --git a/docs/content/yandex.md b/docs/content/yandex.md index d61093ff261fb..37176a40af752 100644 --- a/docs/content/yandex.md +++ b/docs/content/yandex.md @@ -116,7 +116,7 @@ as they can't be used in JSON strings. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/yandex/yandex.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to yandex (Yandex Disk). +Here are the Standard options specific to yandex (Yandex Disk). #### --yandex-client-id @@ -146,7 +146,7 @@ Properties: ### Advanced options -Here are the advanced options specific to yandex (Yandex Disk). +Here are the Advanced options specific to yandex (Yandex Disk). #### --yandex-token diff --git a/docs/content/zoho.md b/docs/content/zoho.md index ba155dfed10f3..187eb5ff1c64d 100644 --- a/docs/content/zoho.md +++ b/docs/content/zoho.md @@ -127,7 +127,7 @@ from filenames during upload. {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/zoho/zoho.go then run make backenddocs" >}} ### Standard options -Here are the standard options specific to zoho (Zoho). +Here are the Standard options specific to zoho (Zoho). #### --zoho-client-id @@ -176,12 +176,16 @@ Properties: - Europe - "in" - India + - "jp" + - Japan + - "com.cn" + - China - "com.au" - Australia ### Advanced options -Here are the advanced options specific to zoho (Zoho). +Here are the Advanced options specific to zoho (Zoho). #### --zoho-token diff --git a/rclone.1 b/rclone.1 index 693b003af1d88..6b3c7fd376a86 100644 --- a/rclone.1 +++ b/rclone.1 @@ -1,7 +1,7 @@ .\"t .\" Automatically generated by Pandoc 2.9.2.1 .\" -.TH "rclone" "1" "Mar 18, 2022" "User Manual" "" +.TH "rclone" "1" "Jul 09, 2022" "User Manual" "" .hy .SH Rclone syncs your files to cloud storage .PP @@ -133,7 +133,7 @@ a network disk .IP \[bu] 2 Serve (https://rclone.org/commands/rclone_serve/) local or remote files over -HTTP (https://rclone.org/commands/rclone_serve_http/)/WebDav (https://rclone.org/commands/rclone_serve_webdav/)/FTP (https://rclone.org/commands/rclone_serve_ftp/)/SFTP (https://rclone.org/commands/rclone_serve_sftp/)/dlna (https://rclone.org/commands/rclone_serve_dlna/) +HTTP (https://rclone.org/commands/rclone_serve_http/)/WebDav (https://rclone.org/commands/rclone_serve_webdav/)/FTP (https://rclone.org/commands/rclone_serve_ftp/)/SFTP (https://rclone.org/commands/rclone_serve_sftp/)/DLNA (https://rclone.org/commands/rclone_serve_dlna/) .IP \[bu] 2 Experimental Web based GUI (https://rclone.org/gui/) .SS Supported providers @@ -157,10 +157,16 @@ Box .IP \[bu] 2 Ceph .IP \[bu] 2 +China Mobile Ecloud Elastic Object Storage (EOS) +.IP \[bu] 2 +Arvan Cloud Object Storage (AOS) +.IP \[bu] 2 Citrix ShareFile .IP \[bu] 2 C14 .IP \[bu] 2 +Cloudflare R2 +.IP \[bu] 2 DigitalOcean Spaces .IP \[bu] 2 Digi Storage @@ -181,14 +187,22 @@ Google Photos .IP \[bu] 2 HDFS .IP \[bu] 2 +Hetzner Storage Box +.IP \[bu] 2 +HiDrive +.IP \[bu] 2 HTTP .IP \[bu] 2 Hubic .IP \[bu] 2 +Internet Archive +.IP \[bu] 2 Jottacloud .IP \[bu] 2 IBM COS S3 .IP \[bu] 2 +IDrive e2 +.IP \[bu] 2 Koofr .IP \[bu] 2 Mail.ru Cloud @@ -260,8 +274,26 @@ Yandex Disk Zoho WorkDrive .IP \[bu] 2 The local filesystem +.SS Virtual providers .PP -Links +These backends adapt or modify other storage providers: +.IP \[bu] 2 +Alias: Rename existing remotes +.IP \[bu] 2 +Cache: Cache remotes (DEPRECATED) +.IP \[bu] 2 +Chunker: Split large files +.IP \[bu] 2 +Combine: Combine multiple remotes into a directory tree +.IP \[bu] 2 +Compress: Compress files +.IP \[bu] 2 +Crypt: Encrypt files +.IP \[bu] 2 +Hasher: Hash files +.IP \[bu] 2 +Union: Join multiple remotes to work together +.SS Links .IP \[bu] 2 Home page (https://rclone.org/) .IP \[bu] 2 @@ -300,7 +332,7 @@ To install rclone on Linux/macOS/BSD systems, run: .IP .nf \f[C] -curl https://rclone.org/install.sh | sudo bash +sudo -v ; curl https://rclone.org/install.sh | sudo bash \f[R] .fi .PP @@ -308,7 +340,7 @@ For beta installation, run: .IP .nf \f[C] -curl https://rclone.org/install.sh | sudo bash -s beta +sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta \f[R] .fi .PP @@ -425,7 +457,7 @@ pop-up will appear saying: .IP .nf \f[C] -\[lq]rclone\[rq] cannot be opened because the developer cannot be verified. +\[dq]rclone\[dq] cannot be opened because the developer cannot be verified. macOS cannot verify that this app is free from malware. \f[R] .fi @@ -538,48 +570,114 @@ kill %1 .fi .SS Install from source .PP -Make sure you have at least Go (https://golang.org/) go1.15 installed. -Download go (https://golang.org/dl/) if necessary. -The latest release is recommended. -Then +Make sure you have git and Go (https://golang.org/) installed. +Go version 1.16 or newer is required, latest release is recommended. +You can get it from your package manager, or download it from +golang.org/dl (https://golang.org/dl/). +Then you can run the following: .IP .nf \f[C] git clone https://github.com/rclone/rclone.git cd rclone go build -# If on macOS and mount is wanted, instead run: make GOTAGS=cmount -\&./rclone version \f[R] .fi .PP -This will leave you a checked out version of rclone you can modify and -send pull requests with. -If you use \f[C]make\f[R] instead of \f[C]go build\f[R] then the rclone -build will have the correct version information in it. +This will check out the rclone source in subfolder rclone, which you can +later modify and send pull requests with. +Then it will build the rclone executable in the same folder. +As an initial check you can now run \f[C]./rclone version\f[R] +(\f[C].\[rs]rclone version\f[R] on Windows). .PP -You can also build the latest stable rclone with: +Note that on macOS and Windows the +mount (https://rclone.org/commands/rclone_mount/) command will not be +available unless you specify additional build tag \f[C]cmount\f[R]. .IP .nf \f[C] -go get github.com/rclone/rclone +go build -tags cmount +\f[R] +.fi +.PP +This assumes you have a GCC compatible C compiler (GCC or Clang) in your +PATH, as it uses cgo (https://pkg.go.dev/cmd/cgo). +But on Windows, the cgofuse (https://github.com/winfsp/cgofuse) library +that the cmount implementation is based on, also supports building +without cgo (https://github.com/golang/go/wiki/WindowsDLLs), i.e. +by setting environment variable CGO_ENABLED to value 0 (static linking). +This is how the official Windows release of rclone is being built, +starting with version 1.59. +It is still possible to build with cgo on Windows as well, by using the +MinGW port of GCC, e.g. +by installing it in a MSYS2 (https://www.msys2.org) distribution (make +sure you install it in the classic mingw64 subsystem, the ucrt64 version +is not compatible). +.PP +Additionally, on Windows, you must install the third party utility +WinFsp (http://www.secfs.net/winfsp/), with the \[dq]Developer\[dq] +feature selected. +If building with cgo, you must also set environment variable CPATH +pointing to the fuse include directory within the WinFsp installation +(normally +\f[C]C:\[rs]Program Files (x86)\[rs]WinFsp\[rs]inc\[rs]fuse\f[R]). +.PP +You may also add arguments \f[C]-ldflags -s\f[R] (with or without +\f[C]-tags cmount\f[R]), to omit symbol table and debug information, +making the executable file smaller, and \f[C]-trimpath\f[R] to remove +references to local file system paths. +This is how the official rclone releases are built. +.IP +.nf +\f[C] +go build -trimpath -ldflags -s -tags cmount +\f[R] +.fi +.PP +Instead of executing the \f[C]go build\f[R] command directly, you can +run it via the Makefile, which also sets version information and copies +the resulting rclone executable into your GOPATH bin folder +(\f[C]$(go env GOPATH)/bin\f[R], which corresponds to +\f[C]\[ti]/go/bin/rclone\f[R] by default). +.IP +.nf +\f[C] +make +\f[R] +.fi +.PP +To include mount command on macOS and Windows with Makefile build: +.IP +.nf +\f[C] +make GOTAGS=cmount \f[R] .fi .PP -or the latest version (equivalent to the beta) with +As an alternative you can download the source, build and install rclone +in one operation, as a regular Go package. +The source will be stored it in the Go module cache, and the resulting +executable will be in your GOPATH bin folder +(\f[C]$(go env GOPATH)/bin\f[R], which corresponds to +\f[C]\[ti]/go/bin/rclone\f[R] by default). +.PP +With Go version 1.17 or newer: .IP .nf \f[C] -go get github.com/rclone/rclone\[at]master +go install github.com/rclone/rclone\[at]latest \f[R] .fi .PP -These will build the binary in \f[C]$(go env GOPATH)/bin\f[R] -(\f[C]\[ti]/go/bin/rclone\f[R] by default) after downloading the source -to the go module cache. -Note - do \f[B]not\f[R] use the \f[C]-u\f[R] flag here. -This causes go to try to update the dependencies that rclone uses and -sometimes these don\[aq]t work with the current version of rclone. +With Go versions older than 1.17 (do \f[B]not\f[R] use the \f[C]-u\f[R] +flag, it causes Go to try to update the dependencies that rclone uses +and sometimes these don\[aq]t work with the current version): +.IP +.nf +\f[C] +go get github.com/rclone/rclone +\f[R] +.fi .SS Installation with Ansible .PP This can be done with Stefan Weichinger\[aq]s ansible @@ -730,7 +828,7 @@ that executes your rclone command, as an alternative to scheduled task configured to run at startup. .SS Mount command built-in service integration .PP -For mount commands, Rclone has a built-in Windows service integration +For mount commands, rclone has a built-in Windows service integration via the third-party WinFsp library it uses. Registering as a regular Windows service easy, as you just have to execute the built-in PowerShell command \f[C]New-Service\f[R] (requires @@ -844,6 +942,8 @@ Citrix ShareFile (https://rclone.org/sharefile/) .IP \[bu] 2 Compress (https://rclone.org/compress/) .IP \[bu] 2 +Combine (https://rclone.org/combine/) +.IP \[bu] 2 Crypt (https://rclone.org/crypt/) - to encrypt other remotes .IP \[bu] 2 DigitalOcean Spaces (https://rclone.org/s3/#digitalocean-spaces) @@ -867,10 +967,14 @@ remotes .IP \[bu] 2 HDFS (https://rclone.org/hdfs/) .IP \[bu] 2 +HiDrive (https://rclone.org/hidrive/) +.IP \[bu] 2 HTTP (https://rclone.org/http/) .IP \[bu] 2 Hubic (https://rclone.org/hubic/) .IP \[bu] 2 +Internet Archive (https://rclone.org/internetarchive/) +.IP \[bu] 2 Jottacloud (https://rclone.org/jottacloud/) .IP \[bu] 2 Koofr (https://rclone.org/koofr/) @@ -1032,10 +1136,17 @@ Copy the source to the destination. Does not transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Doesn\[aq]t delete files from the destination. +If you want to also delete files from destination, to make it match +source, use the sync (https://rclone.org/commands/rclone_sync/) command +instead. .PP Note that it is always the contents of the directory that is synced, not -the directory so when source:path is a directory, it\[aq]s the contents -of source:path that are copied, not the directory name and contents. +the directory itself. +So when source:path is a directory, it\[aq]s the contents of source:path +that are copied, not the directory name and contents. +.PP +To copy single files, use the +copyto (https://rclone.org/commands/rclone_copyto/) command instead. .PP If dest:path doesn\[aq]t exist, it is created and the source:path contents go there. @@ -1133,6 +1244,8 @@ Doesn\[aq]t transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Destination is updated to match source, including deleting files if necessary (except duplicate objects, see below). +If you don\[aq]t want to delete files from destination, use the +copy (https://rclone.org/commands/rclone_copy/) command instead. .PP \f[B]Important\f[R]: Since this can cause data loss, test first with the \f[C]--dry-run\f[R] or the \f[C]--interactive\f[R]/\f[C]-i\f[R] flag. @@ -1149,9 +1262,11 @@ Duplicate objects (files with the same name, on those providers that support it) are also not yet handled. .PP It is always the contents of the directory that is synced, not the -directory so when source:path is a directory, it\[aq]s the contents of -source:path that are copied, not the directory name and contents. -See extended explanation in the \f[C]copy\f[R] command above if unsure. +directory itself. +So when source:path is a directory, it\[aq]s the contents of source:path +that are copied, not the directory name and contents. +See extended explanation in the +copy (https://rclone.org/commands/rclone_copy/) command if unsure. .PP If dest:path doesn\[aq]t exist, it is created and the source:path contents go there. @@ -1195,6 +1310,9 @@ Moves the contents of the source directory to the destination directory. Rclone will error if the source and destination overlap and the remote does not support a server-side directory move operation. .PP +To move single files, use the +moveto (https://rclone.org/commands/rclone_moveto/) command instead. +.PP If no filters are in use and if possible this will server-side move \f[C]source:path\f[R] into \f[C]dest:path\f[R]. After this \f[C]source:path\f[R] will no longer exist. @@ -1206,7 +1324,7 @@ If possible a server-side move will be used, otherwise it will copy it original (if no errors on copy) in \f[C]source:path\f[R]. .PP If you want to delete empty source directories after move, use the ---delete-empty-src-dirs flag. +\f[C]--delete-empty-src-dirs\f[R] flag. .PP See the --no-traverse (https://rclone.org/docs/#no-traverse) option for controlling whether rclone lists the destination directory or not. @@ -1246,18 +1364,20 @@ Remove the files in path. .SS Synopsis .PP Remove the files in path. -Unlike \f[C]purge\f[R] it obeys include/exclude filters so can be used -to selectively delete files. +Unlike purge (https://rclone.org/commands/rclone_purge/) it obeys +include/exclude filters so can be used to selectively delete files. .PP \f[C]rclone delete\f[R] only deletes files but leaves the directory structure alone. If you want to delete a directory and all of its contents use the -\f[C]purge\f[R] command. +purge (https://rclone.org/commands/rclone_purge/) command. .PP If you supply the \f[C]--rmdirs\f[R] flag, it will remove all empty directories along with it. -You can also use the separate command \f[C]rmdir\f[R] or -\f[C]rmdirs\f[R] to delete empty directories only. +You can also use the separate command +rmdir (https://rclone.org/commands/rclone_rmdir/) or +rmdirs (https://rclone.org/commands/rclone_rmdirs/) to delete empty +directories only. .PP For example, to delete all files bigger than 100 MiB, you may first want to check what would be deleted (use either): @@ -1311,10 +1431,11 @@ Remove the path and all of its contents. Remove the path and all of its contents. Note that this does not obey include/exclude filters - everything will be removed. -Use the \f[C]delete\f[R] command if you want to selectively delete -files. -To delete empty directories only, use command \f[C]rmdir\f[R] or -\f[C]rmdirs\f[R]. +Use the delete (https://rclone.org/commands/rclone_delete/) command if +you want to selectively delete files. +To delete empty directories only, use command +rmdir (https://rclone.org/commands/rclone_rmdir/) or +rmdirs (https://rclone.org/commands/rclone_rmdirs/). .PP \f[B]Important\f[R]: Since this can cause data loss, test first with the \f[C]--dry-run\f[R] or the \f[C]--interactive\f[R]/\f[C]-i\f[R] flag. @@ -1369,10 +1490,12 @@ Remove the empty directory at path. This removes empty directory given by path. Will not remove the path if it has any objects in it, not even empty subdirectories. -Use command \f[C]rmdirs\f[R] (or \f[C]delete\f[R] with option +Use command rmdirs (https://rclone.org/commands/rclone_rmdirs/) (or +delete (https://rclone.org/commands/rclone_delete/) with option \f[C]--rmdirs\f[R]) to do that. .PP -To delete a path and any objects in it, use \f[C]purge\f[R] command. +To delete a path and any objects in it, use +purge (https://rclone.org/commands/rclone_purge/) command. .IP .nf \f[C] @@ -1403,6 +1526,10 @@ It compares sizes and hashes (MD5 or SHA1) and logs a report of files that don\[aq]t match. It doesn\[aq]t alter the source or destination. .PP +For the crypt (https://rclone.org/crypt/) remote there is a dedicated +command, cryptcheck (https://rclone.org/commands/rclone_cryptcheck/), +that are able to check the checksums of the crypted files. +.PP If you supply the \f[C]--size-only\f[R] flag, it will only compare the sizes not the hashes as well. Use this for a quick check. @@ -1554,7 +1681,7 @@ List all directories/containers/buckets in the path. .PP Lists the directories in the source path to standard output. Does not recurse by default. -Use the -R flag to recurse. +Use the \f[C]-R\f[R] flag to recurse. .PP This command lists the total size of the directory (if known, -1 if not), the modification time (if known, the current time if not), the @@ -1580,8 +1707,8 @@ $ rclone lsd drive:test \f[R] .fi .PP -If you just want the directory names use \[dq]rclone lsf ---dirs-only\[dq]. +If you just want the directory names use +\f[C]rclone lsf --dirs-only\f[R]. .PP Any of the filtering options can be applied to this command. .PP @@ -1714,6 +1841,11 @@ If MD5 is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling MD5 for any remote. .PP +For other algorithms, see the +hashsum (https://rclone.org/commands/rclone_hashsum/) command. +Running \f[C]rclone md5sum remote:path\f[R] is equivalent to running +\f[C]rclone hashsum MD5 remote:path\f[R]. +.PP This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a @@ -1755,6 +1887,11 @@ If SHA-1 is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling SHA-1 for any remote. .PP +For other algorithms, see the +hashsum (https://rclone.org/commands/rclone_hashsum/) command. +Running \f[C]rclone sha1sum remote:path\f[R] is equivalent to running +\f[C]rclone hashsum SHA1 remote:path\f[R]. +.PP This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a @@ -1789,6 +1926,24 @@ commands, flags and backends. .SH rclone size .PP Prints the total size and number of objects in remote:path. +.SS Synopsis +.PP +Counts objects in the path and calculates the total size. +Prints the result to standard output. +.PP +By default the output is in human-readable format, but shows values in +both human-readable format as well as the raw numbers (global option +\f[C]--human-readable\f[R] is not considered). +Use option \f[C]--json\f[R] to format output as JSON instead. +.PP +Recurses by default, use \f[C]--max-depth 1\f[R] to stop the recursion. +.PP +Some backends do not always provide file sizes, see for example Google +Photos (https://rclone.org/googlephotos/#size) and Google +Drive (https://rclone.org/drive/#limitations-of-google-docs). +Rclone will then show a notice in the log indicating how many such files +were encountered, and count them in as empty files in the output of the +size command. .IP .nf \f[C] @@ -1926,9 +2081,9 @@ Google Drive, Opendrive) that can have duplicate file names. It can be run on wrapping backends (e.g. crypt) if they wrap a backend which supports duplicate file names. .PP -However if --by-hash is passed in then dedupe will find files with -duplicate hashes instead which will work on any backend which supports -at least one hash. +However if \f[C]--by-hash\f[R] is passed in then dedupe will find files +with duplicate hashes instead which will work on any backend which +supports at least one hash. This can be used to find files with duplicate content. This is known as deduping by hash. .PP @@ -2473,7 +2628,7 @@ rclone (https://rclone.org/commands/rclone/) - Show help for rclone commands, flags and backends. .SH rclone completion .PP -generate the autocompletion script for the specified shell +Generate the autocompletion script for the specified shell .SS Synopsis .PP Generate the autocompletion script for rclone for the specified shell. @@ -2495,23 +2650,23 @@ rclone (https://rclone.org/commands/rclone/) - Show help for rclone commands, flags and backends. .IP \[bu] 2 rclone completion -bash (https://rclone.org/commands/rclone_completion_bash/) - generate +bash (https://rclone.org/commands/rclone_completion_bash/) - Generate the autocompletion script for bash .IP \[bu] 2 rclone completion -fish (https://rclone.org/commands/rclone_completion_fish/) - generate +fish (https://rclone.org/commands/rclone_completion_fish/) - Generate the autocompletion script for fish .IP \[bu] 2 rclone completion powershell (https://rclone.org/commands/rclone_completion_powershell/) - -generate the autocompletion script for powershell +Generate the autocompletion script for powershell .IP \[bu] 2 rclone completion -zsh (https://rclone.org/commands/rclone_completion_zsh/) - generate the +zsh (https://rclone.org/commands/rclone_completion_zsh/) - Generate the autocompletion script for zsh .SH rclone completion bash .PP -generate the autocompletion script for bash +Generate the autocompletion script for bash .SS Synopsis .PP Generate the autocompletion script for the bash shell. @@ -2520,12 +2675,29 @@ This script depends on the \[aq]bash-completion\[aq] package. If it is not installed already, you can install it via your OS\[aq]s package manager. .PP -To load completions in your current shell session: $ source <(rclone -completion bash) +To load completions in your current shell session: +.IP +.nf +\f[C] +source <(rclone completion bash) +\f[R] +.fi .PP -To load completions for every new session, execute once: Linux: $ rclone -completion bash > /etc/bash_completion.d/rclone MacOS: $ rclone -completion bash > /usr/local/etc/bash_completion.d/rclone +To load completions for every new session, execute once: +.SS Linux: +.IP +.nf +\f[C] +rclone completion bash > /etc/bash_completion.d/rclone +\f[R] +.fi +.SS macOS: +.IP +.nf +\f[C] +rclone completion bash > /usr/local/etc/bash_completion.d/rclone +\f[R] +.fi .PP You will need to start a new shell for this setup to take effect. .IP @@ -2548,19 +2720,29 @@ not listed here. .SS SEE ALSO .IP \[bu] 2 rclone completion (https://rclone.org/commands/rclone_completion/) - -generate the autocompletion script for the specified shell +Generate the autocompletion script for the specified shell .SH rclone completion fish .PP -generate the autocompletion script for fish +Generate the autocompletion script for fish .SS Synopsis .PP Generate the autocompletion script for the fish shell. .PP -To load completions in your current shell session: $ rclone completion -fish | source +To load completions in your current shell session: +.IP +.nf +\f[C] +rclone completion fish | source +\f[R] +.fi .PP -To load completions for every new session, execute once: $ rclone -completion fish > \[ti]/.config/fish/completions/rclone.fish +To load completions for every new session, execute once: +.IP +.nf +\f[C] +rclone completion fish > \[ti]/.config/fish/completions/rclone.fish +\f[R] +.fi .PP You will need to start a new shell for this setup to take effect. .IP @@ -2583,16 +2765,21 @@ not listed here. .SS SEE ALSO .IP \[bu] 2 rclone completion (https://rclone.org/commands/rclone_completion/) - -generate the autocompletion script for the specified shell +Generate the autocompletion script for the specified shell .SH rclone completion powershell .PP -generate the autocompletion script for powershell +Generate the autocompletion script for powershell .SS Synopsis .PP Generate the autocompletion script for powershell. .PP -To load completions in your current shell session: PS C:> rclone -completion powershell | Out-String | Invoke-Expression +To load completions in your current shell session: +.IP +.nf +\f[C] +rclone completion powershell | Out-String | Invoke-Expression +\f[R] +.fi .PP To load completions for every new session, add the output of the above command to your powershell profile. @@ -2616,10 +2803,10 @@ not listed here. .SS SEE ALSO .IP \[bu] 2 rclone completion (https://rclone.org/commands/rclone_completion/) - -generate the autocompletion script for the specified shell +Generate the autocompletion script for the specified shell .SH rclone completion zsh .PP -generate the autocompletion script for zsh +Generate the autocompletion script for zsh .SS Synopsis .PP Generate the autocompletion script for the zsh shell. @@ -2627,12 +2814,28 @@ Generate the autocompletion script for the zsh shell. If shell completion is not already enabled in your environment you will need to enable it. You can execute the following once: +.IP +.nf +\f[C] +echo \[dq]autoload -U compinit; compinit\[dq] >> \[ti]/.zshrc +\f[R] +.fi .PP -$ echo \[dq]autoload -U compinit; compinit\[dq] >> \[ti]/.zshrc -.PP -To load completions for every new session, execute once: # Linux: $ -rclone completion zsh > \[dq]${fpath[1]}/_rclone\[dq] # macOS: $ rclone -completion zsh > /usr/local/share/zsh/site-functions/_rclone +To load completions for every new session, execute once: +.SS Linux: +.IP +.nf +\f[C] +rclone completion zsh > \[dq]${fpath[1]}/_rclone\[dq] +\f[R] +.fi +.SS macOS: +.IP +.nf +\f[C] +rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone +\f[R] +.fi .PP You will need to start a new shell for this setup to take effect. .IP @@ -2655,7 +2858,7 @@ not listed here. .SS SEE ALSO .IP \[bu] 2 rclone completion (https://rclone.org/commands/rclone_completion/) - -generate the autocompletion script for the specified shell +Generate the autocompletion script for the specified shell .SH rclone config create .PP Create a new remote with name, type and options. @@ -3314,7 +3517,8 @@ directory named dest:path. .PP This can be used to upload single files to other than their current name. -If the source is a directory then it acts exactly like the copy command. +If the source is a directory then it acts exactly like the +copy (https://rclone.org/commands/rclone_copy/) command. .PP So .IP @@ -3373,9 +3577,12 @@ Copy url content to dest. Download a URL\[aq]s content and copy it to the destination without saving it in temporary storage. .PP -Setting \f[C]--auto-filename\f[R] will cause the file name to be -retrieved from the URL (after any redirections) and used in the -destination path. +Setting \f[C]--auto-filename\f[R] will attempt to automatically +determine the filename from the URL (after any redirections) and used in +the destination path. +With \f[C]--auto-filename-header\f[R] in addition, if a specific +filename is set in HTTP headers, it will be used instead of the name +from the URL. With \f[C]--print-filename\f[R] in addition, the resulting file name will be printed. .PP @@ -3394,11 +3601,12 @@ rclone copyurl https://example.com dest:path [flags] .IP .nf \f[C] - -a, --auto-filename Get the file name from the URL and use it for destination file path - -h, --help help for copyurl - --no-clobber Prevent overwriting file with same name - -p, --print-filename Print the resulting name from --auto-filename - --stdout Write the output to stdout rather than a file + -a, --auto-filename Get the file name from the URL and use it for destination file path + --header-filename Get the file name from the Content-Disposition header + -h, --help help for copyurl + --no-clobber Prevent overwriting file with same name + -p, --print-filename Print the resulting name from --auto-filename + --stdout Write the output to stdout rather than a file \f[R] .fi .PP @@ -3413,8 +3621,10 @@ commands, flags and backends. Cryptcheck checks the integrity of a crypted remote. .SS Synopsis .PP -rclone cryptcheck checks a remote against a crypted remote. -This is the equivalent of running rclone check, but able to check the +rclone cryptcheck checks a remote against a +crypted (https://rclone.org/crypt/) remote. +This is the equivalent of running rclone +check (https://rclone.org/commands/rclone_check/), but able to check the checksums of the crypted remote. .PP For it to work the underlying remote of the cryptedremote must support @@ -3513,7 +3723,8 @@ rclone cryptdecode returns unencrypted file names when provided with a list of encrypted file names. List limit is 10 items. .PP -If you supply the --reverse flag, it will return encrypted file names. +If you supply the \f[C]--reverse\f[R] flag, it will return encrypted +file names. .PP use it like this .IP @@ -3526,8 +3737,9 @@ rclone cryptdecode --reverse encryptedremote: filename1 filename2 .fi .PP Another way to accomplish this is by using the -\f[C]rclone backend encode\f[R] (or \f[C]decode\f[R])command. -See the documentation on the \f[C]crypt\f[R] overlay for more info. +\f[C]rclone backend encode\f[R] (or \f[C]decode\f[R]) command. +See the documentation on the crypt (https://rclone.org/crypt/) overlay +for more info. .IP .nf \f[C] @@ -3584,7 +3796,7 @@ Output completion script for a given shell. .SS Synopsis .PP Generates a shell completion script for rclone. -Run with --help to list the supported shells. +Run with \f[C]--help\f[R] to list the supported shells. .SS Options .IP .nf @@ -3804,6 +4016,10 @@ If the hash is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling any hash for any remote. .PP +For the MD5 and SHA1 algorithms there are also dedicated commands, +md5sum (https://rclone.org/commands/rclone_md5sum/) and +sha1sum (https://rclone.org/commands/rclone_sha1sum/). +.PP This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a @@ -3821,6 +4037,7 @@ Supported hashes are: * crc32 * sha256 * dropbox + * hidrive * mailru * quickxor \f[R] @@ -3920,7 +4137,7 @@ List all the remotes in the config file. .PP rclone listremotes lists all the available remotes from the config file. .PP -When uses with the -l flag it lists the types too. +When used with the \f[C]--long\f[R] flag it lists the types too. .IP .nf \f[C] @@ -3966,7 +4183,7 @@ fubuwic \f[R] .fi .PP -Use the --format option to control what gets listed. +Use the \f[C]--format\f[R] option to control what gets listed. By default this is just the path, but you can use these parameters to control the output: .IP @@ -3981,12 +4198,13 @@ o - Original ID of underlying object m - MimeType of object if known e - encrypted name T - tier of storage if known, e.g. \[dq]Hot\[dq] or \[dq]Cool\[dq] +M - Metadata of object in JSON blob format, eg {\[dq]key\[dq]:\[dq]value\[dq]} \f[R] .fi .PP So if you wanted the path, size and modification time, you would use ---format \[dq]pst\[dq], or maybe --format \[dq]tsp\[dq] to put the path -last. +\f[C]--format \[dq]pst\[dq]\f[R], or maybe +\f[C]--format \[dq]tsp\[dq]\f[R] to put the path last. .PP Eg .IP @@ -4002,7 +4220,7 @@ $ rclone lsf --format \[dq]tsp\[dq] swift:bucket .fi .PP If you specify \[dq]h\[dq] in the format you will get the MD5 hash by -default, use the \[dq]--hash\[dq] flag to change which hash you want. +default, use the \f[C]--hash\f[R] flag to change which hash you want. Note that this can be returned as an empty string if it isn\[aq]t available on the object (and for directories), \[dq]ERROR\[dq] if there was an error reading it from the object and \[dq]UNSUPPORTED\[dq] if @@ -4032,7 +4250,7 @@ cd65ac234e6fea5925974a51cdd865cc canole (Though \[dq]rclone md5sum .\[dq] is an easier way of typing this.) .PP By default the separator is \[dq];\[dq] this can be changed with the ---separator flag. +\f[C]--separator\f[R] flag. Note that separators aren\[aq]t escaped in the path so putting it last is a good strategy. .PP @@ -4063,8 +4281,9 @@ test.sh,449 \f[R] .fi .PP -Note that the --absolute parameter is useful for making lists of files -to pass to an rclone copy with the --files-from-raw flag. +Note that the \f[C]--absolute\f[R] parameter is useful for making lists +of files to pass to an rclone copy with the \f[C]--files-from-raw\f[R] +flag. .PP For example, to find all the files modified within one day and copy those only (without traversing the whole directory structure): @@ -4141,46 +4360,62 @@ List directories and objects in the path in JSON format. List directories and objects in the path in JSON format. .PP The output is an array of Items, where each Item looks like this +.IP +.nf +\f[C] +{ + \[dq]Hashes\[dq] : { + \[dq]SHA-1\[dq] : \[dq]f572d396fae9206628714fb2ce00f72e94f2258f\[dq], + \[dq]MD5\[dq] : \[dq]b1946ac92492d2347c6235b4d2611184\[dq], + \[dq]DropboxHash\[dq] : \[dq]ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc\[dq] + }, + \[dq]ID\[dq]: \[dq]y2djkhiujf83u33\[dq], + \[dq]OrigID\[dq]: \[dq]UYOJVTUW00Q1RzTDA\[dq], + \[dq]IsBucket\[dq] : false, + \[dq]IsDir\[dq] : false, + \[dq]MimeType\[dq] : \[dq]application/octet-stream\[dq], + \[dq]ModTime\[dq] : \[dq]2017-05-31T16:15:57.034468261+01:00\[dq], + \[dq]Name\[dq] : \[dq]file.txt\[dq], + \[dq]Encrypted\[dq] : \[dq]v0qpsdq8anpci8n929v3uu9338\[dq], + \[dq]EncryptedPath\[dq] : \[dq]kja9098349023498/v0qpsdq8anpci8n929v3uu9338\[dq], + \[dq]Path\[dq] : \[dq]full/path/goes/here/file.txt\[dq], + \[dq]Size\[dq] : 6, + \[dq]Tier\[dq] : \[dq]hot\[dq], +} +\f[R] +.fi +.PP +If \f[C]--hash\f[R] is not specified the Hashes property won\[aq]t be +emitted. +The types of hash can be specified with the \f[C]--hash-type\f[R] +parameter (which may be repeated). +If \f[C]--hash-type\f[R] is set then it implies \f[C]--hash\f[R]. .PP -{ \[dq]Hashes\[dq] : { \[dq]SHA-1\[dq] : -\[dq]f572d396fae9206628714fb2ce00f72e94f2258f\[dq], \[dq]MD5\[dq] : -\[dq]b1946ac92492d2347c6235b4d2611184\[dq], \[dq]DropboxHash\[dq] : -\[dq]ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc\[dq] -}, \[dq]ID\[dq]: \[dq]y2djkhiujf83u33\[dq], \[dq]OrigID\[dq]: -\[dq]UYOJVTUW00Q1RzTDA\[dq], \[dq]IsBucket\[dq] : false, \[dq]IsDir\[dq] -: false, \[dq]MimeType\[dq] : \[dq]application/octet-stream\[dq], -\[dq]ModTime\[dq] : \[dq]2017-05-31T16:15:57.034468261+01:00\[dq], -\[dq]Name\[dq] : \[dq]file.txt\[dq], \[dq]Encrypted\[dq] : -\[dq]v0qpsdq8anpci8n929v3uu9338\[dq], \[dq]EncryptedPath\[dq] : -\[dq]kja9098349023498/v0qpsdq8anpci8n929v3uu9338\[dq], \[dq]Path\[dq] : -\[dq]full/path/goes/here/file.txt\[dq], \[dq]Size\[dq] : 6, -\[dq]Tier\[dq] : \[dq]hot\[dq], } -.PP -If --hash is not specified the Hashes property won\[aq]t be emitted. -The types of hash can be specified with the --hash-type parameter (which -may be repeated). -If --hash-type is set then it implies --hash. -.PP -If --no-modtime is specified then ModTime will be blank. +If \f[C]--no-modtime\f[R] is specified then ModTime will be blank. This can speed things up on remotes where reading the ModTime takes an extra request (e.g. s3, swift). .PP -If --no-mimetype is specified then MimeType will be blank. +If \f[C]--no-mimetype\f[R] is specified then MimeType will be blank. This can speed things up on remotes where reading the MimeType takes an extra request (e.g. s3, swift). .PP -If --encrypted is not specified the Encrypted won\[aq]t be emitted. +If \f[C]--encrypted\f[R] is not specified the Encrypted won\[aq]t be +emitted. .PP -If --dirs-only is not specified files in addition to directories are -returned +If \f[C]--dirs-only\f[R] is not specified files in addition to +directories are returned .PP -If --files-only is not specified directories in addition to the files -will be returned. +If \f[C]--files-only\f[R] is not specified directories in addition to +the files will be returned. .PP -if --stat is set then a single JSON blob will be returned about the item -pointed to. +If \f[C]--metadata\f[R] is set then an additional Metadata key will be +returned. +This will have metdata in rclone standard format as a JSON object. +.PP +if \f[C]--stat\f[R] is set then a single JSON blob will be returned +about the item pointed to. This will return an error if the item isn\[aq]t found. However on bucket based backends (like s3, gcs, b2, azureblob etc) if the item isn\[aq]t found it will return an empty directory as it @@ -4192,7 +4427,8 @@ listed. If \[dq]remote:path\[dq] contains the file \[dq]subfolder/file.txt\[dq], the Path for \[dq]file.txt\[dq] will be \[dq]subfolder/file.txt\[dq], not \[dq]remote:path/subfolder/file.txt\[dq]. -When used without --recursive the Path will always be the same as Name. +When used without \f[C]--recursive\f[R] the Path will always be the same +as Name. .PP If the directory is a bucket in a bucket-based backend, then \[dq]IsBucket\[dq] will be set to true. @@ -4249,7 +4485,7 @@ rclone lsjson remote:path [flags] .nf \f[C] --dirs-only Show only directories in the listing - -M, --encrypted Show the encrypted names + --encrypted Show the encrypted names --files-only Show only files in the listing --hash Include hashes in the output (may take longer) --hash-type stringArray Show only this hash type (may be repeated) @@ -4357,11 +4593,11 @@ feature at all, then 1 PiB is set as both the total and the free size. To run rclone mount on Windows, you will need to download and install WinFsp (http://www.secfs.net/winfsp/). .PP -WinFsp (https://github.com/billziss-gh/winfsp) is an open-source Windows -File System Proxy which makes it easy to write user space file systems -for Windows. +WinFsp (https://github.com/winfsp/winfsp) is an open-source Windows File +System Proxy which makes it easy to write user space file systems for +Windows. It provides a FUSE emulation layer which rclone uses combination with -cgofuse (https://github.com/billziss-gh/cgofuse). +cgofuse (https://github.com/winfsp/cgofuse). Both of these packages are by Bill Zissimopoulos who was very helpful during the implementation of rclone mount for Windows. .SS Mounting modes on windows @@ -4564,7 +4800,7 @@ to start processes as the SYSTEM account. Another alternative is to run the mount command from a Windows Scheduled Task, or a Windows Service, configured to run as the SYSTEM account. A third alternative is to use the WinFsp.Launcher -infrastructure (https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture)). +infrastructure (https://github.com/winfsp/winfsp/wiki/WinFsp-Service-Architecture)). Note that when running rclone as another user, it will not use the configuration file from your profile unless you tell it to with the \f[C]--config\f[R] (https://rclone.org/docs/#config-config-file) option. @@ -4781,7 +5017,7 @@ files and directories (but not the data) in memory. Using the \f[C]--dir-cache-time\f[R] flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. .IP .nf @@ -4972,6 +5208,40 @@ In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn\[aq]t support sparse files and it will log an ERROR message if one is detected. +.SS Fingerprinting +.PP +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. +Fingerprints are made from: +.IP \[bu] 2 +size +.IP \[bu] 2 +modification time +.IP \[bu] 2 +hash +.PP +where available on an object. +.PP +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). +.PP +For example \f[C]hash\f[R] is slow with the \f[C]local\f[R] and +\f[C]sftp\f[R] backends as they have to read the entire file and hash +it, and \f[C]modtime\f[R] is slow with the \f[C]s3\f[R], +\f[C]swift\f[R], \f[C]ftp\f[R] and \f[C]qinqstor\f[R] backends because +they need to do an extra API call to fetch it. +.PP +If you use the \f[C]--vfs-fast-fingerprint\f[R] flag then rclone will +not include the slow operations in the fingerprint. +This makes the fingerprinting less accurate but much faster and will +improve the opening time of cached files. +.PP +If you are running a vfs cache over \f[C]local\f[R], \f[C]s3\f[R] or +\f[C]swift\f[R] backends then using this flag is recommended. +.PP +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. .SS VFS Chunked Reading .PP When rclone reads files from a remote it reads them in chunks. @@ -5023,7 +5293,7 @@ transaction. --no-checksum Don\[aq]t compare checksums on up/download. --no-modtime Don\[aq]t read/write the modification time (can speed things up). --no-seek Don\[aq]t allow seeking in files. ---read-only Mount read-only. +--read-only Only allow read-only access. \f[R] .fi .PP @@ -5041,8 +5311,8 @@ These flags only come into effect when not using an on disk cache file. .PP When using VFS write caching (\f[C]--vfs-cache-mode\f[R] with value writes or full), the global flag \f[C]--transfers\f[R] can be set to -adjust the number of parallel uploads of modified files from cache (the -related global flag \f[C]--checkers\f[R] have no effect on mount). +adjust the number of parallel uploads of modified files from the cache +(the related global flag \f[C]--checkers\f[R] has no effect on the VFS). .IP .nf \f[C] @@ -5065,15 +5335,15 @@ Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. .PP -The \f[C]--vfs-case-insensitive\f[R] mount flag controls how rclone +The \f[C]--vfs-case-insensitive\f[R] VFS flag controls how rclone handles these two cases. -If its value is \[dq]false\[dq], rclone passes file names to the mounted -file system as-is. -If the flag is \[dq]true\[dq] (or appears without a value on command +If its value is \[dq]false\[dq], rclone passes file names to the remote +as-is. +If the flag is \[dq]true\[dq] (or appears without a value on the command line), rclone may perform a \[dq]fixup\[dq] as explained below. .PP The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a @@ -5081,10 +5351,10 @@ name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by -an underlying mounted file system. +the underlying remote. .PP Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether \[dq]fixup\[dq] is performed to satisfy the target. @@ -5093,6 +5363,18 @@ If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: \[dq]true\[dq] on Windows and macOS, \[dq]false\[dq] otherwise. If the flag is provided without a value, then it is \[dq]true\[dq]. +.SS VFS Disk Options +.PP +This flag allows you to manually set the statistics about the filing +system. +It can be useful when those statistics cannot be read correctly +automatically. +.IP +.nf +\f[C] +--vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) +\f[R] +.fi .SS Alternate report of used bytes .PP Some backends, most notably S3, do not report the amount of bytes used. @@ -5144,7 +5426,7 @@ rclone mount remote:path /path/to/mountpoint [flags] --noapplexattr Ignore all \[dq]com.apple.*\[dq] extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) @@ -5152,6 +5434,8 @@ rclone mount remote:path /path/to/mountpoint [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) @@ -5180,7 +5464,8 @@ directory named dest:path. .PP This can be used to rename files or upload single files to other than their existing name. -If the source is a directory then it acts exactly like the move command. +If the source is a directory then it acts exactly like the +move (https://rclone.org/commands/rclone_move/) command. .PP So .IP @@ -5249,7 +5534,9 @@ builds an in memory representation. rclone ncdu can be used during this scanning phase and you will see it building up the directory structure as it goes along. .PP -Here are the keys - press \[aq]?\[aq] to toggle the help on and off +You can interact with the user interface using key presses, press +\[aq]?\[aq] to toggle the help on and off. +The supported keys are: .IP .nf \f[C] @@ -5262,11 +5549,33 @@ Here are the keys - press \[aq]?\[aq] to toggle the help on and off u toggle human-readable format n,s,C,A sort by name,size,count,average size d delete file/directory + v select file/directory + V enter visual select mode + D delete selected files/directories y copy current path to clipboard Y display current path - \[ha]L refresh screen + \[ha]L refresh screen (fix screen corruption) ? to toggle help on and off - q/ESC/c-C to quit + q/ESC/\[ha]c to quit +\f[R] +.fi +.PP +Listed files/directories may be prefixed by a one-character flag, some +of them combined with a description in brackes at end of line. +These flags have the following meaning: +.IP +.nf +\f[C] +e means this is an empty directory, i.e. contains no files (but + may contain empty subdirectories) +\[ti] means this is a directory where some of the files (possibly in + subdirectories) have unknown size, and therefore the directory + size may be underestimated (and average size inaccurate, as it + is average of the files with known sizes). +\&. means an error occurred while reading a subdirectory, and + therefore the directory size may be underestimated (and average + size inaccurate) +! means an error occurred while reading this directory \f[R] .fi .PP @@ -5274,9 +5583,14 @@ This an homage to the ncdu tool (https://dev.yorhel.nl/ncdu) but for rclone remotes. It is missing lots of features at the moment but is useful as it stands. .PP -Note that it might take some time to delete big files/folders. +Note that it might take some time to delete big files/directories. The UI won\[aq]t respond in the meantime since the deletion is done synchronously. +.PP +For a non-interactive listing of the remote, see the +tree (https://rclone.org/commands/rclone_tree/) command. +To just get the total size of the remote you can also use the +size (https://rclone.org/commands/rclone_size/) command. .IP .nf \f[C] @@ -5317,8 +5631,12 @@ This command can also accept a password through STDIN instead of an argument by passing a hyphen as an argument. This will use the first line of STDIN as the password not including the trailing newline. -.PP +.IP +.nf +\f[C] echo \[dq]secretpassword\[dq] | rclone obscure - +\f[R] +.fi .PP If there is no data on STDIN to read, rclone obscure will default to obfuscating the hyphen itself. @@ -5352,26 +5670,30 @@ Run a command against a running rclone. .SS Synopsis .PP This runs a command against a running rclone. -Use the --url flag to specify an non default URL to connect on. +Use the \f[C]--url\f[R] flag to specify an non default URL to connect +on. This can be either a \[dq]:port\[dq] which is taken to mean \[dq]http://localhost:port\[dq] or a \[dq]host:port\[dq] which is taken to mean \[dq]http://host:port\[dq] .PP -A username and password can be passed in with --user and --pass. +A username and password can be passed in with \f[C]--user\f[R] and +\f[C]--pass\f[R]. .PP -Note that --rc-addr, --rc-user, --rc-pass will be read also for --url, ---user, --pass. +Note that \f[C]--rc-addr\f[R], \f[C]--rc-user\f[R], \f[C]--rc-pass\f[R] +will be read also for \f[C]--url\f[R], \f[C]--user\f[R], +\f[C]--pass\f[R]. .PP Arguments should be passed in as parameter=value. .PP The result will be returned as a JSON object by default. .PP -The --json parameter can be used to pass in a JSON blob as an input -instead of key=value arguments. +The \f[C]--json\f[R] parameter can be used to pass in a JSON blob as an +input instead of key=value arguments. This is the only way of passing in more complicated values. .PP -The -o/--opt option can be used to set a key \[dq]opt\[dq] with key, -value options in the form \[dq]-o key=value\[dq] or \[dq]-o key\[dq]. +The \f[C]-o\f[R]/\f[C]--opt\f[R] option can be used to set a key +\[dq]opt\[dq] with key, value options in the form \f[C]-o key=value\f[R] +or \f[C]-o key\f[R]. It can be repeated as many times as required. This is useful for rc commands which take the \[dq]opt\[dq] parameter which by convention is a dictionary of strings. @@ -5390,8 +5712,8 @@ Will place this in the \[dq]opt\[dq] value \f[R] .fi .PP -The -a/--arg option can be used to set strings in the \[dq]arg\[dq] -value. +The \f[C]-a\f[R]/\f[C]--arg\f[R] option can be used to set strings in +the \[dq]arg\[dq] value. It can be repeated as many times as required. This is useful for rc commands which take the \[dq]arg\[dq] parameter which by convention is a list of strings. @@ -5410,8 +5732,8 @@ Will place this in the \[dq]arg\[dq] value \f[R] .fi .PP -Use --loopback to connect to the rclone instance running \[dq]rclone -rc\[dq]. +Use \f[C]--loopback\f[R] to connect to the rclone instance running +\f[C]rclone rc\f[R]. This is very useful for testing commands without having to run an rclone rc server, e.g.: .IP @@ -5421,7 +5743,7 @@ rclone rc --loopback operations/about fs=/ \f[R] .fi .PP -Use \[dq]rclone rc\[dq] to see a list of all possible commands. +Use \f[C]rclone rc\f[R] to see a list of all possible commands. .IP .nf \f[C] @@ -5481,13 +5803,13 @@ please see there. Generally speaking, setting this cutoff too high will decrease your performance. .PP -Use the |--size| flag to preallocate the file in advance at the remote -end and actually stream it, even if remote backend doesn\[aq]t support -streaming. +Use the \f[C]--size\f[R] flag to preallocate the file in advance at the +remote end and actually stream it, even if remote backend doesn\[aq]t +support streaming. .PP -|--size| should be the exact size of the input stream in bytes. -If the size of the stream is different in length to the |--size| passed -in then the transfer will likely fail. +\f[C]--size\f[R] should be the exact size of the input stream in bytes. +If the size of the stream is different in length to the \f[C]--size\f[R] +passed in then the transfer will likely fail. .PP Note that the upload can also not be retried because the data is not kept around until the upload succeeds. @@ -5559,15 +5881,17 @@ that only contain empty directories), that it finds under the path. The root path itself will also be removed if it is empty, unless you supply the \f[C]--leave-root\f[R] flag. .PP -Use command \f[C]rmdir\f[R] to delete just the empty directory given by -path, not recurse. +Use command rmdir (https://rclone.org/commands/rclone_rmdir/) to delete +just the empty directory given by path, not recurse. .PP This is useful for tidying up remotes that rclone has left a lot of empty directories in. -For example the \f[C]delete\f[R] command will delete files but leave the -directory structure (unless used with option \f[C]--rmdirs\f[R]). +For example the delete (https://rclone.org/commands/rclone_delete/) +command will delete files but leave the directory structure (unless used +with option \f[C]--rmdirs\f[R]). .PP -To delete a path and any objects in it, use \f[C]purge\f[R] command. +To delete a path and any objects in it, use +purge (https://rclone.org/commands/rclone_purge/) command. .IP .nf \f[C] @@ -5689,9 +6013,8 @@ commands, flags and backends. Serve a remote over a protocol. .SS Synopsis .PP -rclone serve is used to serve a remote over a given protocol. -This command requires the use of a subcommand to specify the protocol, -e.g. +Serve a remote over a given protocol. +Requires the use of a subcommand to specify the protocol, e.g. .IP .nf \f[C] @@ -5740,14 +6063,13 @@ rclone serve sftp (https://rclone.org/commands/rclone_serve_sftp/) - Serve the remote over SFTP. .IP \[bu] 2 rclone serve webdav (https://rclone.org/commands/rclone_serve_webdav/) - -Serve remote:path over webdav. +Serve remote:path over WebDAV. .SH rclone serve dlna .PP Serve remote:path over DLNA .SS Synopsis .PP -rclone serve dlna is a DLNA media server for media stored in an rclone -remote. +Run a DLNA media server for media stored in an rclone remote. Many devices, such as the Xbox and PlayStation, can automatically discover this server in the LAN and play audio/video from it. VLC is also supported. @@ -5790,7 +6112,7 @@ files and directories (but not the data) in memory. Using the \f[C]--dir-cache-time\f[R] flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. .IP .nf @@ -5981,6 +6303,40 @@ In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn\[aq]t support sparse files and it will log an ERROR message if one is detected. +.SS Fingerprinting +.PP +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. +Fingerprints are made from: +.IP \[bu] 2 +size +.IP \[bu] 2 +modification time +.IP \[bu] 2 +hash +.PP +where available on an object. +.PP +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). +.PP +For example \f[C]hash\f[R] is slow with the \f[C]local\f[R] and +\f[C]sftp\f[R] backends as they have to read the entire file and hash +it, and \f[C]modtime\f[R] is slow with the \f[C]s3\f[R], +\f[C]swift\f[R], \f[C]ftp\f[R] and \f[C]qinqstor\f[R] backends because +they need to do an extra API call to fetch it. +.PP +If you use the \f[C]--vfs-fast-fingerprint\f[R] flag then rclone will +not include the slow operations in the fingerprint. +This makes the fingerprinting less accurate but much faster and will +improve the opening time of cached files. +.PP +If you are running a vfs cache over \f[C]local\f[R], \f[C]s3\f[R] or +\f[C]swift\f[R] backends then using this flag is recommended. +.PP +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. .SS VFS Chunked Reading .PP When rclone reads files from a remote it reads them in chunks. @@ -6032,7 +6388,7 @@ transaction. --no-checksum Don\[aq]t compare checksums on up/download. --no-modtime Don\[aq]t read/write the modification time (can speed things up). --no-seek Don\[aq]t allow seeking in files. ---read-only Mount read-only. +--read-only Only allow read-only access. \f[R] .fi .PP @@ -6050,8 +6406,8 @@ These flags only come into effect when not using an on disk cache file. .PP When using VFS write caching (\f[C]--vfs-cache-mode\f[R] with value writes or full), the global flag \f[C]--transfers\f[R] can be set to -adjust the number of parallel uploads of modified files from cache (the -related global flag \f[C]--checkers\f[R] have no effect on mount). +adjust the number of parallel uploads of modified files from the cache +(the related global flag \f[C]--checkers\f[R] has no effect on the VFS). .IP .nf \f[C] @@ -6074,15 +6430,15 @@ Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. .PP -The \f[C]--vfs-case-insensitive\f[R] mount flag controls how rclone +The \f[C]--vfs-case-insensitive\f[R] VFS flag controls how rclone handles these two cases. -If its value is \[dq]false\[dq], rclone passes file names to the mounted -file system as-is. -If the flag is \[dq]true\[dq] (or appears without a value on command +If its value is \[dq]false\[dq], rclone passes file names to the remote +as-is. +If the flag is \[dq]true\[dq] (or appears without a value on the command line), rclone may perform a \[dq]fixup\[dq] as explained below. .PP The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a @@ -6090,10 +6446,10 @@ name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by -an underlying mounted file system. +the underlying remote. .PP Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether \[dq]fixup\[dq] is performed to satisfy the target. @@ -6102,6 +6458,18 @@ If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: \[dq]true\[dq] on Windows and macOS, \[dq]false\[dq] otherwise. If the flag is provided without a value, then it is \[dq]true\[dq]. +.SS VFS Disk Options +.PP +This flag allows you to manually set the statistics about the filing +system. +It can be useful when those statistics cannot be read correctly +automatically. +.IP +.nf +\f[C] +--vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) +\f[R] +.fi .SS Alternate report of used bytes .PP Some backends, most notably S3, do not report the amount of bytes used. @@ -6139,7 +6507,7 @@ rclone serve dlna remote:path [flags] --no-modtime Don\[aq]t read/write the modification time (can speed things up) --no-seek Don\[aq]t allow seeking in files --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) @@ -6147,6 +6515,8 @@ rclone serve dlna remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) @@ -6236,7 +6606,7 @@ files and directories (but not the data) in memory. Using the \f[C]--dir-cache-time\f[R] flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. .IP .nf @@ -6427,6 +6797,40 @@ In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn\[aq]t support sparse files and it will log an ERROR message if one is detected. +.SS Fingerprinting +.PP +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. +Fingerprints are made from: +.IP \[bu] 2 +size +.IP \[bu] 2 +modification time +.IP \[bu] 2 +hash +.PP +where available on an object. +.PP +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). +.PP +For example \f[C]hash\f[R] is slow with the \f[C]local\f[R] and +\f[C]sftp\f[R] backends as they have to read the entire file and hash +it, and \f[C]modtime\f[R] is slow with the \f[C]s3\f[R], +\f[C]swift\f[R], \f[C]ftp\f[R] and \f[C]qinqstor\f[R] backends because +they need to do an extra API call to fetch it. +.PP +If you use the \f[C]--vfs-fast-fingerprint\f[R] flag then rclone will +not include the slow operations in the fingerprint. +This makes the fingerprinting less accurate but much faster and will +improve the opening time of cached files. +.PP +If you are running a vfs cache over \f[C]local\f[R], \f[C]s3\f[R] or +\f[C]swift\f[R] backends then using this flag is recommended. +.PP +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. .SS VFS Chunked Reading .PP When rclone reads files from a remote it reads them in chunks. @@ -6478,7 +6882,7 @@ transaction. --no-checksum Don\[aq]t compare checksums on up/download. --no-modtime Don\[aq]t read/write the modification time (can speed things up). --no-seek Don\[aq]t allow seeking in files. ---read-only Mount read-only. +--read-only Only allow read-only access. \f[R] .fi .PP @@ -6496,8 +6900,8 @@ These flags only come into effect when not using an on disk cache file. .PP When using VFS write caching (\f[C]--vfs-cache-mode\f[R] with value writes or full), the global flag \f[C]--transfers\f[R] can be set to -adjust the number of parallel uploads of modified files from cache (the -related global flag \f[C]--checkers\f[R] have no effect on mount). +adjust the number of parallel uploads of modified files from the cache +(the related global flag \f[C]--checkers\f[R] has no effect on the VFS). .IP .nf \f[C] @@ -6520,15 +6924,15 @@ Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. .PP -The \f[C]--vfs-case-insensitive\f[R] mount flag controls how rclone +The \f[C]--vfs-case-insensitive\f[R] VFS flag controls how rclone handles these two cases. -If its value is \[dq]false\[dq], rclone passes file names to the mounted -file system as-is. -If the flag is \[dq]true\[dq] (or appears without a value on command +If its value is \[dq]false\[dq], rclone passes file names to the remote +as-is. +If the flag is \[dq]true\[dq] (or appears without a value on the command line), rclone may perform a \[dq]fixup\[dq] as explained below. .PP The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a @@ -6536,10 +6940,10 @@ name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by -an underlying mounted file system. +the underlying remote. .PP Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether \[dq]fixup\[dq] is performed to satisfy the target. @@ -6548,6 +6952,18 @@ If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: \[dq]true\[dq] on Windows and macOS, \[dq]false\[dq] otherwise. If the flag is provided without a value, then it is \[dq]true\[dq]. +.SS VFS Disk Options +.PP +This flag allows you to manually set the statistics about the filing +system. +It can be useful when those statistics cannot be read correctly +automatically. +.IP +.nf +\f[C] +--vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) +\f[R] +.fi .SS Alternate report of used bytes .PP Some backends, most notably S3, do not report the amount of bytes used. @@ -6602,7 +7018,7 @@ rclone serve docker [flags] --noapplexattr Ignore all \[dq]com.apple.*\[dq] extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --socket-addr string Address or absolute path (default: /run/docker/plugins/rclone.sock) --socket-gid int GID for unix socket (default: current process GID) (default 1000) --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) @@ -6612,6 +7028,8 @@ rclone serve docker [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) @@ -6635,10 +7053,9 @@ remote over a protocol. Serve remote:path over FTP. .SS Synopsis .PP -rclone serve ftp implements a basic ftp server to serve the remote over -FTP protocol. -This can be viewed with a ftp client or you can make a remote of type -ftp to read and write it. +Run a basic FTP server to serve a remote over FTP protocol. +This can be viewed with a FTP client or you can make a remote of type +FTP to read and write it. .SS Server options .PP Use --addr to specify which IP address and port the server should listen @@ -6674,7 +7091,7 @@ files and directories (but not the data) in memory. Using the \f[C]--dir-cache-time\f[R] flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. .IP .nf @@ -6865,6 +7282,40 @@ In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn\[aq]t support sparse files and it will log an ERROR message if one is detected. +.SS Fingerprinting +.PP +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. +Fingerprints are made from: +.IP \[bu] 2 +size +.IP \[bu] 2 +modification time +.IP \[bu] 2 +hash +.PP +where available on an object. +.PP +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). +.PP +For example \f[C]hash\f[R] is slow with the \f[C]local\f[R] and +\f[C]sftp\f[R] backends as they have to read the entire file and hash +it, and \f[C]modtime\f[R] is slow with the \f[C]s3\f[R], +\f[C]swift\f[R], \f[C]ftp\f[R] and \f[C]qinqstor\f[R] backends because +they need to do an extra API call to fetch it. +.PP +If you use the \f[C]--vfs-fast-fingerprint\f[R] flag then rclone will +not include the slow operations in the fingerprint. +This makes the fingerprinting less accurate but much faster and will +improve the opening time of cached files. +.PP +If you are running a vfs cache over \f[C]local\f[R], \f[C]s3\f[R] or +\f[C]swift\f[R] backends then using this flag is recommended. +.PP +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. .SS VFS Chunked Reading .PP When rclone reads files from a remote it reads them in chunks. @@ -6916,7 +7367,7 @@ transaction. --no-checksum Don\[aq]t compare checksums on up/download. --no-modtime Don\[aq]t read/write the modification time (can speed things up). --no-seek Don\[aq]t allow seeking in files. ---read-only Mount read-only. +--read-only Only allow read-only access. \f[R] .fi .PP @@ -6934,8 +7385,8 @@ These flags only come into effect when not using an on disk cache file. .PP When using VFS write caching (\f[C]--vfs-cache-mode\f[R] with value writes or full), the global flag \f[C]--transfers\f[R] can be set to -adjust the number of parallel uploads of modified files from cache (the -related global flag \f[C]--checkers\f[R] have no effect on mount). +adjust the number of parallel uploads of modified files from the cache +(the related global flag \f[C]--checkers\f[R] has no effect on the VFS). .IP .nf \f[C] @@ -6958,15 +7409,15 @@ Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. .PP -The \f[C]--vfs-case-insensitive\f[R] mount flag controls how rclone +The \f[C]--vfs-case-insensitive\f[R] VFS flag controls how rclone handles these two cases. -If its value is \[dq]false\[dq], rclone passes file names to the mounted -file system as-is. -If the flag is \[dq]true\[dq] (or appears without a value on command +If its value is \[dq]false\[dq], rclone passes file names to the remote +as-is. +If the flag is \[dq]true\[dq] (or appears without a value on the command line), rclone may perform a \[dq]fixup\[dq] as explained below. .PP The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a @@ -6974,10 +7425,10 @@ name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by -an underlying mounted file system. +the underlying remote. .PP Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether \[dq]fixup\[dq] is performed to satisfy the target. @@ -6986,6 +7437,18 @@ If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: \[dq]true\[dq] on Windows and macOS, \[dq]false\[dq] otherwise. If the flag is provided without a value, then it is \[dq]true\[dq]. +.SS VFS Disk Options +.PP +This flag allows you to manually set the statistics about the filing +system. +It can be useful when those statistics cannot be read correctly +automatically. +.IP +.nf +\f[C] +--vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) +\f[R] +.fi .SS Alternate report of used bytes .PP Some backends, most notably S3, do not report the amount of bytes used. @@ -7120,7 +7583,7 @@ rclone serve ftp remote:path [flags] --passive-port string Passive port range to use (default \[dq]30000-32000\[dq]) --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --public-ip string Public IP address to advertise for passive connections - --read-only Mount read-only + --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication (default \[dq]anonymous\[dq]) @@ -7129,6 +7592,8 @@ rclone serve ftp remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) @@ -7150,60 +7615,63 @@ remote over a protocol. Serve the remote over HTTP. .SS Synopsis .PP -rclone serve http implements a basic web server to serve the remote over -HTTP. +Run a basic web server to serve a remote over HTTP. This can be viewed in a web browser or you can make a remote of type http read from it. .PP You can use the filter flags (e.g. ---include, --exclude) to control what is served. +\f[C]--include\f[R], \f[C]--exclude\f[R]) to control what is served. .PP The server will log errors. -Use -v to see access logs. +Use \f[C]-v\f[R] to see access logs. .PP ---bwlimit will be respected for file transfers. -Use --stats to control the stats printing. +\f[C]--bwlimit\f[R] will be respected for file transfers. +Use \f[C]--stats\f[R] to control the stats printing. .SS Server options .PP -Use --addr to specify which IP address and port the server should listen -on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. +Use \f[C]--addr\f[R] to specify which IP address and port the server +should listen on, eg \f[C]--addr 1.2.3.4:8000\f[R] or +\f[C]--addr :8080\f[R] to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. .PP -If you set --addr to listen on a public or LAN accessible IP address -then using Authentication is advised - see the next section for info. +If you set \f[C]--addr\f[R] to listen on a public or LAN accessible IP +address then using Authentication is advised - see the next section for +info. .PP ---server-read-timeout and --server-write-timeout can be used to control -the timeouts on the server. +\f[C]--server-read-timeout\f[R] and \f[C]--server-write-timeout\f[R] can +be used to control the timeouts on the server. Note that this is the total time for a transfer. .PP ---max-header-bytes controls the maximum number of bytes the server will -accept in the HTTP header. +\f[C]--max-header-bytes\f[R] controls the maximum number of bytes the +server will accept in the HTTP header. .PP ---baseurl controls the URL prefix that rclone serves from. +\f[C]--baseurl\f[R] controls the URL prefix that rclone serves from. By default rclone will serve from the root. -If you used --baseurl \[dq]/rclone\[dq] then rclone would serve from a -URL starting with \[dq]/rclone/\[dq]. +If you used \f[C]--baseurl \[dq]/rclone\[dq]\f[R] then rclone would +serve from a URL starting with \[dq]/rclone/\[dq]. This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing \[dq]/\[dq] on ---baseurl, so --baseurl \[dq]rclone\[dq], --baseurl \[dq]/rclone\[dq] -and --baseurl \[dq]/rclone/\[dq] are all treated identically. +\f[C]--baseurl\f[R], so \f[C]--baseurl \[dq]rclone\[dq]\f[R], +\f[C]--baseurl \[dq]/rclone\[dq]\f[R] and +\f[C]--baseurl \[dq]/rclone/\[dq]\f[R] are all treated identically. .SS SSL/TLS .PP By default this will serve over http. If you want you can serve over https. -You will need to supply the --cert and --key flags. +You will need to supply the \f[C]--cert\f[R] and \f[C]--key\f[R] flags. If you wish to do client side certificate validation then you will need -to supply --client-ca also. +to supply \f[C]--client-ca\f[R] also. .PP ---cert should be a either a PEM encoded certificate or a concatenation -of that with the CA certificate. ---key should be the PEM encoded private key and --client-ca should be -the PEM encoded client certificate authority certificate. +\f[C]--cert\f[R] should be a either a PEM encoded certificate or a +concatenation of that with the CA certificate. +\f[C]--key\f[R] should be the PEM encoded private key and +\f[C]--client-ca\f[R] should be the PEM encoded client certificate +authority certificate. .SS Template .PP ---template allows a user to specify a custom markup template for http -and webdav serve functions. +\f[C]--template\f[R] allows a user to specify a custom markup template +for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: .PP @@ -7303,9 +7771,10 @@ T} By default this will serve files without needing a login. .PP You can either use an htpasswd file which can take lots of users, or set -a single username and password with the --user and --pass flags. +a single username and password with the \f[C]--user\f[R] and +\f[C]--pass\f[R] flags. .PP -Use --htpasswd /path/to/htpasswd to provide an htpasswd file. +Use \f[C]--htpasswd /path/to/htpasswd\f[R] to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. @@ -7322,9 +7791,10 @@ htpasswd -B htpasswd anotherUser .PP The password file can be updated while rclone is running. .PP -Use --realm to set the authentication realm. +Use \f[C]--realm\f[R] to set the authentication realm. .PP -Use --salt to change the password hashing salt from the default. +Use \f[C]--salt\f[R] to change the password hashing salt from the +default. .SS VFS - Virtual File System .PP This command uses the VFS layer. @@ -7344,7 +7814,7 @@ files and directories (but not the data) in memory. Using the \f[C]--dir-cache-time\f[R] flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. .IP .nf @@ -7535,6 +8005,40 @@ In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn\[aq]t support sparse files and it will log an ERROR message if one is detected. +.SS Fingerprinting +.PP +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. +Fingerprints are made from: +.IP \[bu] 2 +size +.IP \[bu] 2 +modification time +.IP \[bu] 2 +hash +.PP +where available on an object. +.PP +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). +.PP +For example \f[C]hash\f[R] is slow with the \f[C]local\f[R] and +\f[C]sftp\f[R] backends as they have to read the entire file and hash +it, and \f[C]modtime\f[R] is slow with the \f[C]s3\f[R], +\f[C]swift\f[R], \f[C]ftp\f[R] and \f[C]qinqstor\f[R] backends because +they need to do an extra API call to fetch it. +.PP +If you use the \f[C]--vfs-fast-fingerprint\f[R] flag then rclone will +not include the slow operations in the fingerprint. +This makes the fingerprinting less accurate but much faster and will +improve the opening time of cached files. +.PP +If you are running a vfs cache over \f[C]local\f[R], \f[C]s3\f[R] or +\f[C]swift\f[R] backends then using this flag is recommended. +.PP +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. .SS VFS Chunked Reading .PP When rclone reads files from a remote it reads them in chunks. @@ -7586,7 +8090,7 @@ transaction. --no-checksum Don\[aq]t compare checksums on up/download. --no-modtime Don\[aq]t read/write the modification time (can speed things up). --no-seek Don\[aq]t allow seeking in files. ---read-only Mount read-only. +--read-only Only allow read-only access. \f[R] .fi .PP @@ -7604,8 +8108,8 @@ These flags only come into effect when not using an on disk cache file. .PP When using VFS write caching (\f[C]--vfs-cache-mode\f[R] with value writes or full), the global flag \f[C]--transfers\f[R] can be set to -adjust the number of parallel uploads of modified files from cache (the -related global flag \f[C]--checkers\f[R] have no effect on mount). +adjust the number of parallel uploads of modified files from the cache +(the related global flag \f[C]--checkers\f[R] has no effect on the VFS). .IP .nf \f[C] @@ -7628,15 +8132,15 @@ Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. .PP -The \f[C]--vfs-case-insensitive\f[R] mount flag controls how rclone +The \f[C]--vfs-case-insensitive\f[R] VFS flag controls how rclone handles these two cases. -If its value is \[dq]false\[dq], rclone passes file names to the mounted -file system as-is. -If the flag is \[dq]true\[dq] (or appears without a value on command +If its value is \[dq]false\[dq], rclone passes file names to the remote +as-is. +If the flag is \[dq]true\[dq] (or appears without a value on the command line), rclone may perform a \[dq]fixup\[dq] as explained below. .PP The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a @@ -7644,10 +8148,10 @@ name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by -an underlying mounted file system. +the underlying remote. .PP Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether \[dq]fixup\[dq] is performed to satisfy the target. @@ -7656,6 +8160,18 @@ If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: \[dq]true\[dq] on Windows and macOS, \[dq]false\[dq] otherwise. If the flag is provided without a value, then it is \[dq]true\[dq]. +.SS VFS Disk Options +.PP +This flag allows you to manually set the statistics about the filing +system. +It can be useful when those statistics cannot be read correctly +automatically. +.IP +.nf +\f[C] +--vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) +\f[R] +.fi .SS Alternate report of used bytes .PP Some backends, most notably S3, do not report the amount of bytes used. @@ -7698,7 +8214,7 @@ rclone serve http remote:path [flags] --no-seek Don\[aq]t allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --realm string Realm for authentication --salt string Password hashing salt (default \[dq]dlPL2MqE\[dq]) --server-read-timeout duration Timeout for server reading data (default 1h0m0s) @@ -7712,6 +8228,8 @@ rclone serve http remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) @@ -7733,7 +8251,8 @@ remote over a protocol. Serve the remote for restic\[aq]s REST API. .SS Synopsis .PP -rclone serve restic implements restic\[aq]s REST backend API over HTTP. +Run a basic web server to serve a remove over restic\[aq]s REST backend +API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly. .PP @@ -7743,8 +8262,8 @@ backups. The server will log errors. Use -v to see access logs. .PP ---bwlimit will be respected for file transfers. -Use --stats to control the stats printing. +\f[C]--bwlimit\f[R] will be respected for file transfers. +Use \f[C]--stats\f[R] to control the stats printing. .SS Setting up rclone for use by restic .PP First set up a remote for your chosen cloud @@ -7767,12 +8286,12 @@ Where you can replace \[dq]backup\[dq] in the above by whatever path in the remote you wish to use. .PP By default this will serve on \[dq]localhost:8080\[dq] you can change -this with use of the \[dq]--addr\[dq] flag. +this with use of the \f[C]--addr\f[R] flag. .PP You might wish to start this server on boot. .PP -Adding --cache-objects=false will cause rclone to stop caching objects -returned from the List call. +Adding \f[C]--cache-objects=false\f[R] will cause rclone to stop caching +objects returned from the List call. Caching is normally desirable as it speeds up downloading objects, saves transactions and uses very little memory. .SS Setting up restic to use rclone @@ -7824,37 +8343,40 @@ $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user2repo/ .fi .SS Private repositories .PP -The \[dq]--private-repos\[dq] flag can be used to limit users to +The\f[C]--private-repos\f[R] flag can be used to limit users to repositories starting with a path of \f[C]//\f[R]. .SS Server options .PP -Use --addr to specify which IP address and port the server should listen -on, e.g. ---addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. +Use \f[C]--addr\f[R] to specify which IP address and port the server +should listen on, e.g. +\f[C]--addr 1.2.3.4:8000\f[R] or \f[C]--addr :8080\f[R] to listen to all +IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. .PP -If you set --addr to listen on a public or LAN accessible IP address -then using Authentication is advised - see the next section for info. +If you set \f[C]--addr\f[R] to listen on a public or LAN accessible IP +address then using Authentication is advised - see the next section for +info. .PP ---server-read-timeout and --server-write-timeout can be used to control -the timeouts on the server. +\f[C]--server-read-timeout\f[R] and \f[C]--server-write-timeout\f[R] can +be used to control the timeouts on the server. Note that this is the total time for a transfer. .PP ---max-header-bytes controls the maximum number of bytes the server will -accept in the HTTP header. +\f[C]--max-header-bytes\f[R] controls the maximum number of bytes the +server will accept in the HTTP header. .PP ---baseurl controls the URL prefix that rclone serves from. +\f[C]--baseurl\f[R] controls the URL prefix that rclone serves from. By default rclone will serve from the root. -If you used --baseurl \[dq]/rclone\[dq] then rclone would serve from a -URL starting with \[dq]/rclone/\[dq]. +If you used \f[C]--baseurl \[dq]/rclone\[dq]\f[R] then rclone would +serve from a URL starting with \[dq]/rclone/\[dq]. This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing \[dq]/\[dq] on ---baseurl, so --baseurl \[dq]rclone\[dq], --baseurl \[dq]/rclone\[dq] -and --baseurl \[dq]/rclone/\[dq] are all treated identically. +\f[C]--baseurl\f[R], so \f[C]--baseurl \[dq]rclone\[dq]\f[R], +\f[C]--baseurl \[dq]/rclone\[dq]\f[R] and +\f[C]--baseurl \[dq]/rclone/\[dq]\f[R] are all treated identically. .PP ---template allows a user to specify a custom markup template for http -and webdav serve functions. +\f[C]--template\f[R] allows a user to specify a custom markup template +for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: .PP @@ -7954,9 +8476,10 @@ T} By default this will serve files without needing a login. .PP You can either use an htpasswd file which can take lots of users, or set -a single username and password with the --user and --pass flags. +a single username and password with the \f[C]--user\f[R] and +\f[C]--pass\f[R] flags. .PP -Use --htpasswd /path/to/htpasswd to provide an htpasswd file. +Use \f[C]--htpasswd /path/to/htpasswd\f[R] to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. @@ -7973,19 +8496,20 @@ htpasswd -B htpasswd anotherUser .PP The password file can be updated while rclone is running. .PP -Use --realm to set the authentication realm. +Use \f[C]--realm\f[R] to set the authentication realm. .SS SSL/TLS .PP -By default this will serve over http. -If you want you can serve over https. -You will need to supply the --cert and --key flags. +By default this will serve over HTTP. +If you want you can serve over HTTPS. +You will need to supply the \f[C]--cert\f[R] and \f[C]--key\f[R] flags. If you wish to do client side certificate validation then you will need -to supply --client-ca also. +to supply \f[C]--client-ca\f[R] also. .PP ---cert should be either a PEM encoded certificate or a concatenation of -that with the CA certificate. ---key should be the PEM encoded private key and --client-ca should be -the PEM encoded client certificate authority certificate. +\f[C]--cert\f[R] should be either a PEM encoded certificate or a +concatenation of that with the CA certificate. +\f[C]--key\f[R] should be the PEM encoded private key and +\f[C]--client-ca\f[R] should be the PEM encoded client certificate +authority certificate. .IP .nf \f[C] @@ -8028,24 +8552,24 @@ remote over a protocol. Serve the remote over SFTP. .SS Synopsis .PP -rclone serve sftp implements an SFTP server to serve the remote over -SFTP. +Run a SFTP server to serve a remote over SFTP. This can be used with an SFTP client or you can make a remote of type sftp to use with it. .PP You can use the filter flags (e.g. ---include, --exclude) to control what is served. +\f[C]--include\f[R], \f[C]--exclude\f[R]) to control what is served. .PP The server will log errors. -Use -v to see access logs. +Use \f[C]-v\f[R] to see access logs. .PP ---bwlimit will be respected for file transfers. -Use --stats to control the stats printing. +\f[C]--bwlimit\f[R] will be respected for file transfers. +Use \f[C]--stats\f[R] to control the stats printing. .PP You must provide some means of authentication, either with ---user/--pass, an authorized keys file (specify location with ---authorized-keys - the default is the same as ssh), an --auth-proxy, or -set the --no-auth flag for no authentication when logging in. +\f[C]--user\f[R]/\f[C]--pass\f[R], an authorized keys file (specify +location with \f[C]--authorized-keys\f[R] - the default is the same as +ssh), an \f[C]--auth-proxy\f[R], or set the \f[C]--no-auth\f[R] flag for +no authentication when logging in. .PP Note that this also implements a small number of shell commands so that it can provide md5sum/sha1sum/df information for the rclone sftp @@ -8053,19 +8577,19 @@ backend. This means that is can support SHA1SUMs, MD5SUMs and the about command when paired with the rclone sftp backend. .PP -If you don\[aq]t supply a host --key then rclone will generate rsa, -ecdsa and ed25519 variants, and cache them for later use in rclone\[aq]s -cache directory (see \[dq]rclone help flags cache-dir\[dq]) in the -\[dq]serve-sftp\[dq] directory. +If you don\[aq]t supply a host \f[C]--key\f[R] then rclone will generate +rsa, ecdsa and ed25519 variants, and cache them for later use in +rclone\[aq]s cache directory (see \f[C]rclone help flags cache-dir\f[R]) +in the \[dq]serve-sftp\[dq] directory. .PP By default the server binds to localhost:2022 - if you want it to be -reachable externally then supply \[dq]--addr :2022\[dq] for example. +reachable externally then supply \f[C]--addr :2022\f[R] for example. .PP -Note that the default of \[dq]--vfs-cache-mode off\[dq] is fine for the +Note that the default of \f[C]--vfs-cache-mode off\f[R] is fine for the rclone sftp backend, but it may not be with other SFTP clients. .PP -If --stdio is specified, rclone will serve SFTP over stdio, which can be -used with sshd via \[ti]/.ssh/authorized_keys, for example: +If \f[C]--stdio\f[R] is specified, rclone will serve SFTP over stdio, +which can be used with sshd via \[ti]/.ssh/authorized_keys, for example: .IP .nf \f[C] @@ -8073,8 +8597,8 @@ restrict,command=\[dq]rclone serve sftp --stdio ./photos\[dq] ssh-rsa ... \f[R] .fi .PP -On the client you need to set \[dq]--transfers 1\[dq] when using ---stdio. +On the client you need to set \f[C]--transfers 1\f[R] when using +\f[C]--stdio\f[R]. Otherwise multiple instances of the rclone server are started by OpenSSH which can lead to \[dq]corrupted on transfer\[dq] errors. This is the case because the client chooses indiscriminately which @@ -8083,9 +8607,9 @@ the state of the filing system. .PP The \[dq]restrict\[dq] in authorized_keys prevents SHA1SUMs and MD5SUMs from beeing used. -Omitting \[dq]restrict\[dq] and using --sftp-path-override to enable -checksumming is possible but less secure and you could use the SFTP -server provided by OpenSSH in this case. +Omitting \[dq]restrict\[dq] and using \f[C]--sftp-path-override\f[R] to +enable checksumming is possible but less secure and you could use the +SFTP server provided by OpenSSH in this case. .SS VFS - Virtual File System .PP This command uses the VFS layer. @@ -8105,7 +8629,7 @@ files and directories (but not the data) in memory. Using the \f[C]--dir-cache-time\f[R] flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. .IP .nf @@ -8296,6 +8820,40 @@ In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn\[aq]t support sparse files and it will log an ERROR message if one is detected. +.SS Fingerprinting +.PP +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. +Fingerprints are made from: +.IP \[bu] 2 +size +.IP \[bu] 2 +modification time +.IP \[bu] 2 +hash +.PP +where available on an object. +.PP +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). +.PP +For example \f[C]hash\f[R] is slow with the \f[C]local\f[R] and +\f[C]sftp\f[R] backends as they have to read the entire file and hash +it, and \f[C]modtime\f[R] is slow with the \f[C]s3\f[R], +\f[C]swift\f[R], \f[C]ftp\f[R] and \f[C]qinqstor\f[R] backends because +they need to do an extra API call to fetch it. +.PP +If you use the \f[C]--vfs-fast-fingerprint\f[R] flag then rclone will +not include the slow operations in the fingerprint. +This makes the fingerprinting less accurate but much faster and will +improve the opening time of cached files. +.PP +If you are running a vfs cache over \f[C]local\f[R], \f[C]s3\f[R] or +\f[C]swift\f[R] backends then using this flag is recommended. +.PP +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. .SS VFS Chunked Reading .PP When rclone reads files from a remote it reads them in chunks. @@ -8347,7 +8905,7 @@ transaction. --no-checksum Don\[aq]t compare checksums on up/download. --no-modtime Don\[aq]t read/write the modification time (can speed things up). --no-seek Don\[aq]t allow seeking in files. ---read-only Mount read-only. +--read-only Only allow read-only access. \f[R] .fi .PP @@ -8365,8 +8923,8 @@ These flags only come into effect when not using an on disk cache file. .PP When using VFS write caching (\f[C]--vfs-cache-mode\f[R] with value writes or full), the global flag \f[C]--transfers\f[R] can be set to -adjust the number of parallel uploads of modified files from cache (the -related global flag \f[C]--checkers\f[R] have no effect on mount). +adjust the number of parallel uploads of modified files from the cache +(the related global flag \f[C]--checkers\f[R] has no effect on the VFS). .IP .nf \f[C] @@ -8389,15 +8947,15 @@ Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. .PP -The \f[C]--vfs-case-insensitive\f[R] mount flag controls how rclone +The \f[C]--vfs-case-insensitive\f[R] VFS flag controls how rclone handles these two cases. -If its value is \[dq]false\[dq], rclone passes file names to the mounted -file system as-is. -If the flag is \[dq]true\[dq] (or appears without a value on command +If its value is \[dq]false\[dq], rclone passes file names to the remote +as-is. +If the flag is \[dq]true\[dq] (or appears without a value on the command line), rclone may perform a \[dq]fixup\[dq] as explained below. .PP The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a @@ -8405,10 +8963,10 @@ name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by -an underlying mounted file system. +the underlying remote. .PP Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether \[dq]fixup\[dq] is performed to satisfy the target. @@ -8417,6 +8975,18 @@ If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: \[dq]true\[dq] on Windows and macOS, \[dq]false\[dq] otherwise. If the flag is provided without a value, then it is \[dq]true\[dq]. +.SS VFS Disk Options +.PP +This flag allows you to manually set the statistics about the filing +system. +It can be useful when those statistics cannot be read correctly +automatically. +.IP +.nf +\f[C] +--vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) +\f[R] +.fi .SS Alternate report of used bytes .PP Some backends, most notably S3, do not report the amount of bytes used. @@ -8550,7 +9120,7 @@ rclone serve sftp remote:path [flags] --no-seek Don\[aq]t allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --stdio Run an sftp server on run stdin/stdout --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) @@ -8560,6 +9130,8 @@ rclone serve sftp remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) @@ -8578,14 +9150,14 @@ rclone serve (https://rclone.org/commands/rclone_serve/) - Serve a remote over a protocol. .SH rclone serve webdav .PP -Serve remote:path over webdav. +Serve remote:path over WebDAV. .SS Synopsis .PP -rclone serve webdav implements a basic webdav server to serve the remote -over HTTP via the webdav protocol. -This can be viewed with a webdav client, through a web browser, or you -can make a remote of type webdav to read and write it. -.SS Webdav options +Run a basic WebDAV server to serve a remote over HTTP via the WebDAV +protocol. +This can be viewed with a WebDAV client, through a web browser, or you +can make a remote of type WebDAV to read and write it. +.SS WebDAV options .SS --etag-hash .PP This controls the ETag header. @@ -8595,37 +9167,40 @@ object. If this flag is set to \[dq]auto\[dq] then rclone will choose the first supported hash on the backend or you can use a named hash such as \[dq]MD5\[dq] or \[dq]SHA-1\[dq]. -.PP -Use \[dq]rclone hashsum\[dq] to see the full list. +Use the hashsum (https://rclone.org/commands/rclone_hashsum/) command to +see the full list. .SS Server options .PP -Use --addr to specify which IP address and port the server should listen -on, e.g. ---addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. +Use \f[C]--addr\f[R] to specify which IP address and port the server +should listen on, e.g. +\f[C]--addr 1.2.3.4:8000\f[R] or \f[C]--addr :8080\f[R] to listen to all +IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. .PP -If you set --addr to listen on a public or LAN accessible IP address -then using Authentication is advised - see the next section for info. +If you set \f[C]--addr\f[R] to listen on a public or LAN accessible IP +address then using Authentication is advised - see the next section for +info. .PP ---server-read-timeout and --server-write-timeout can be used to control -the timeouts on the server. +\f[C]--server-read-timeout\f[R] and \f[C]--server-write-timeout\f[R] can +be used to control the timeouts on the server. Note that this is the total time for a transfer. .PP ---max-header-bytes controls the maximum number of bytes the server will -accept in the HTTP header. +\f[C]--max-header-bytes\f[R] controls the maximum number of bytes the +server will accept in the HTTP header. .PP ---baseurl controls the URL prefix that rclone serves from. +\f[C]--baseurl\f[R] controls the URL prefix that rclone serves from. By default rclone will serve from the root. -If you used --baseurl \[dq]/rclone\[dq] then rclone would serve from a -URL starting with \[dq]/rclone/\[dq]. +If you used \f[C]--baseurl \[dq]/rclone\[dq]\f[R] then rclone would +serve from a URL starting with \[dq]/rclone/\[dq]. This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing \[dq]/\[dq] on ---baseurl, so --baseurl \[dq]rclone\[dq], --baseurl \[dq]/rclone\[dq] -and --baseurl \[dq]/rclone/\[dq] are all treated identically. +\f[C]--baseurl\f[R], so \f[C]--baseurl \[dq]rclone\[dq]\f[R], +\f[C]--baseurl \[dq]/rclone\[dq]\f[R] and +\f[C]--baseurl \[dq]/rclone/\[dq]\f[R] are all treated identically. .PP ---template allows a user to specify a custom markup template for http -and webdav serve functions. +\f[C]--template\f[R] allows a user to specify a custom markup template +for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: .PP @@ -8725,9 +9300,10 @@ T} By default this will serve files without needing a login. .PP You can either use an htpasswd file which can take lots of users, or set -a single username and password with the --user and --pass flags. +a single username and password with the \f[C]--user\f[R] and +\f[C]--pass\f[R] flags. .PP -Use --htpasswd /path/to/htpasswd to provide an htpasswd file. +Use \f[C]--htpasswd /path/to/htpasswd\f[R] to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended. @@ -8744,19 +9320,20 @@ htpasswd -B htpasswd anotherUser .PP The password file can be updated while rclone is running. .PP -Use --realm to set the authentication realm. +Use \f[C]--realm\f[R] to set the authentication realm. .SS SSL/TLS .PP -By default this will serve over http. -If you want you can serve over https. -You will need to supply the --cert and --key flags. +By default this will serve over HTTP. +If you want you can serve over HTTPS. +You will need to supply the \f[C]--cert\f[R] and \f[C]--key\f[R] flags. If you wish to do client side certificate validation then you will need -to supply --client-ca also. +to supply \f[C]--client-ca\f[R] also. .PP ---cert should be either a PEM encoded certificate or a concatenation of -that with the CA certificate. ---key should be the PEM encoded private key and --client-ca should be -the PEM encoded client certificate authority certificate. +\f[C]--cert\f[R] should be either a PEM encoded certificate or a +concatenation of that with the CA certificate. +\f[C]--key\f[R] should be the PEM encoded private key and +\f[C]--client-ca\f[R] should be the PEM encoded client certificate +authority certificate. .SS VFS - Virtual File System .PP This command uses the VFS layer. @@ -8776,7 +9353,7 @@ files and directories (but not the data) in memory. Using the \f[C]--dir-cache-time\f[R] flag, you can control how long a directory should be considered up to date and not refreshed from the backend. -Changes made through the mount will appear immediately or invalidate the +Changes made through the VFS will appear immediately or invalidate the cache. .IP .nf @@ -8967,6 +9544,40 @@ In particular FAT/exFAT do not. Rclone will perform very badly if the cache directory is on a filesystem which doesn\[aq]t support sparse files and it will log an ERROR message if one is detected. +.SS Fingerprinting +.PP +Various parts of the VFS use fingerprinting to see if a local file copy +has changed relative to a remote file. +Fingerprints are made from: +.IP \[bu] 2 +size +.IP \[bu] 2 +modification time +.IP \[bu] 2 +hash +.PP +where available on an object. +.PP +On some backends some of these attributes are slow to read (they take an +extra API call per object, or extra work per object). +.PP +For example \f[C]hash\f[R] is slow with the \f[C]local\f[R] and +\f[C]sftp\f[R] backends as they have to read the entire file and hash +it, and \f[C]modtime\f[R] is slow with the \f[C]s3\f[R], +\f[C]swift\f[R], \f[C]ftp\f[R] and \f[C]qinqstor\f[R] backends because +they need to do an extra API call to fetch it. +.PP +If you use the \f[C]--vfs-fast-fingerprint\f[R] flag then rclone will +not include the slow operations in the fingerprint. +This makes the fingerprinting less accurate but much faster and will +improve the opening time of cached files. +.PP +If you are running a vfs cache over \f[C]local\f[R], \f[C]s3\f[R] or +\f[C]swift\f[R] backends then using this flag is recommended. +.PP +Note that if you change the value of this flag, the fingerprints of the +files in the cache may be invalidated and the files will need to be +downloaded again. .SS VFS Chunked Reading .PP When rclone reads files from a remote it reads them in chunks. @@ -9018,7 +9629,7 @@ transaction. --no-checksum Don\[aq]t compare checksums on up/download. --no-modtime Don\[aq]t read/write the modification time (can speed things up). --no-seek Don\[aq]t allow seeking in files. ---read-only Mount read-only. +--read-only Only allow read-only access. \f[R] .fi .PP @@ -9036,8 +9647,8 @@ These flags only come into effect when not using an on disk cache file. .PP When using VFS write caching (\f[C]--vfs-cache-mode\f[R] with value writes or full), the global flag \f[C]--transfers\f[R] can be set to -adjust the number of parallel uploads of modified files from cache (the -related global flag \f[C]--checkers\f[R] have no effect on mount). +adjust the number of parallel uploads of modified files from the cache +(the related global flag \f[C]--checkers\f[R] has no effect on the VFS). .IP .nf \f[C] @@ -9060,15 +9671,15 @@ Usually file systems on macOS are case-insensitive. It is possible to make macOS file systems case-sensitive but that is not the default. .PP -The \f[C]--vfs-case-insensitive\f[R] mount flag controls how rclone +The \f[C]--vfs-case-insensitive\f[R] VFS flag controls how rclone handles these two cases. -If its value is \[dq]false\[dq], rclone passes file names to the mounted -file system as-is. -If the flag is \[dq]true\[dq] (or appears without a value on command +If its value is \[dq]false\[dq], rclone passes file names to the remote +as-is. +If the flag is \[dq]true\[dq] (or appears without a value on the command line), rclone may perform a \[dq]fixup\[dq] as explained below. .PP The user may specify a file name to open/delete/rename/etc with a case -different than what is stored on mounted file system. +different than what is stored on the remote. If an argument refers to an existing file with exactly the same name, then the case of the existing file on the disk will be used. However, if a file name with exactly the same name is not found but a @@ -9076,10 +9687,10 @@ name differing only by case exists, rclone will transparently fixup the name. This fixup happens only when an existing file is requested. Case sensitivity of file names created anew by rclone is controlled by -an underlying mounted file system. +the underlying remote. .PP Note that case sensitivity of the operating system running rclone (the -target) may differ from case sensitivity of a file system mounted by +target) may differ from case sensitivity of a file system presented by rclone (the source). The flag controls whether \[dq]fixup\[dq] is performed to satisfy the target. @@ -9088,6 +9699,18 @@ If the flag is not provided on the command line, then its default value depends on the operating system where rclone runs: \[dq]true\[dq] on Windows and macOS, \[dq]false\[dq] otherwise. If the flag is provided without a value, then it is \[dq]true\[dq]. +.SS VFS Disk Options +.PP +This flag allows you to manually set the statistics about the filing +system. +It can be useful when those statistics cannot be read correctly +automatically. +.IP +.nf +\f[C] +--vfs-disk-space-total-size Manually set the total disk space size (example: 256G, default: -1) +\f[R] +.fi .SS Alternate report of used bytes .PP Some backends, most notably S3, do not report the amount of bytes used. @@ -9226,7 +9849,7 @@ rclone serve webdav remote:path [flags] --no-seek Don\[aq]t allow seeking in files --pass string Password for authentication --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) - --read-only Mount read-only + --read-only Only allow read-only access --realm string Realm for authentication (default \[dq]rclone\[dq]) --server-read-timeout duration Timeout for server reading data (default 1h0m0s) --server-write-timeout duration Timeout for server writing data (default 1h0m0s) @@ -9239,6 +9862,8 @@ rclone serve webdav remote:path [flags] --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match + --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) + --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) @@ -9361,6 +9986,9 @@ histogram of file name characters. rclone test info (https://rclone.org/commands/rclone_test_info/) - Discovers file name or other limitations for paths. .IP \[bu] 2 +rclone test makefile (https://rclone.org/commands/rclone_test_makefile/) +- Make files with random contents of the size given +.IP \[bu] 2 rclone test makefiles (https://rclone.org/commands/rclone_test_makefiles/) - Make a random file hierarchy in a directory @@ -9461,6 +10089,35 @@ not listed here. .IP \[bu] 2 rclone test (https://rclone.org/commands/rclone_test/) - Run a test command +.SH rclone test makefile +.PP +Make files with random contents of the size given +.IP +.nf +\f[C] +rclone test makefile []+ [flags] +\f[R] +.fi +.SS Options +.IP +.nf +\f[C] + --ascii Fill files with random ASCII printable bytes only + --chargen Fill files with a ASCII chargen pattern + -h, --help help for makefile + --pattern Fill files with a periodic pattern + --seed int Seed for the random number generator (0 for random) (default 1) + --sparse Make the files sparse (appear to be filled with ASCII 0x00) + --zero Fill files with ASCII 0x00 +\f[R] +.fi +.PP +See the global flags page (https://rclone.org/flags/) for global options +not listed here. +.SS SEE ALSO +.IP \[bu] 2 +rclone test (https://rclone.org/commands/rclone_test/) - Run a test +command .SH rclone test makefiles .PP Make a random file hierarchy in a directory @@ -9474,6 +10131,8 @@ rclone test makefiles [flags] .IP .nf \f[C] + --ascii Fill files with random ASCII printable bytes only + --chargen Fill files with a ASCII chargen pattern --files int Number of files to create (default 1000) --files-per-directory int Average number of files per directory (default 10) -h, --help help for makefiles @@ -9481,7 +10140,10 @@ rclone test makefiles [flags] --max-name-length int Maximum size of file names (default 12) --min-file-size SizeSuffix Minimum size of file to create --min-name-length int Minimum size of file names (default 4) + --pattern Fill files with a periodic pattern --seed int Seed for the random number generator (0 for random) (default 1) + --sparse Make the files sparse (appear to be filled with ASCII 0x00) + --zero Fill files with ASCII 0x00 \f[R] .fi .PP @@ -9595,13 +10257,17 @@ $ rclone tree remote:path .fi .PP You can use any of the filtering options with the tree command (e.g. ---include and --exclude). -You can also use --fast-list. +\f[C]--include\f[R] and \f[C]--exclude\f[R]. +You can also use \f[C]--fast-list\f[R]. .PP The tree command has many options for controlling the listing which are -compatible with the tree command. +compatible with the tree command, for example you can include file sizes +with \f[C]--size\f[R]. Note that not all of them have short options as they conflict with rclone\[aq]s short options. +.PP +For a more interactive navigation of the remote see the +ncdu (https://rclone.org/commands/rclone_ncdu/) command. .IP .nf \f[C] @@ -10057,6 +10723,215 @@ rclone sync -i remote:current-backup remote:previous-backup rclone sync -i /path/to/files remote:current-backup \f[R] .fi +.SS Metadata support +.PP +Metadata is data about a file which isn\[aq]t the contents of the file. +Normally rclone only preserves the modification time and the content +(MIME) type where possible. +.PP +Rclone supports preserving all the available metadata on files (not +directories) when using the \f[C]--metadata\f[R] or \f[C]-M\f[R] flag. +.PP +Exactly what metadata is supported and what that support means depends +on the backend. +Backends that support metadata have a metadata section in their docs and +are listed in the features table (https://rclone.org/overview/#features) +(Eg local (https://rclone.org/local/#metadata), s3) +.PP +Rclone only supports a one-time sync of metadata. +This means that metadata will be synced from the source object to the +destination object only when the source object has changed and needs to +be re-uploaded. +If the metadata subsequently changes on the source object without +changing the object itself then it won\[aq]t be synced to the +destination object. +This is in line with the way rclone syncs \f[C]Content-Type\f[R] without +the \f[C]--metadata\f[R] flag. +.PP +Using \f[C]--metadata\f[R] when syncing from local to local will +preserve file attributes such as file mode, owner, extended attributes +(not Windows). +.PP +Note that arbitrary metadata may be added to objects using the +\f[C]--metadata-set key=value\f[R] flag when the object is first +uploaded. +This flag can be repeated as many times as necessary. +.SS Types of metadata +.PP +Metadata is divided into two type. +System metadata and User metadata. +.PP +Metadata which the backend uses itself is called system metadata. +For example on the local backend the system metadata \f[C]uid\f[R] will +store the user ID of the file when used on a unix based platform. +.PP +Arbitrary metadata is called user metadata and this can be set however +is desired. +.PP +When objects are copied from backend to backend, they will attempt to +interpret system metadata if it is supplied. +Metadata may change from being user metadata to system metadata as +objects are copied between different backends. +For example copying an object from s3 sets the \f[C]content-type\f[R] +metadata. +In a backend which understands this (like \f[C]azureblob\f[R]) this will +become the Content-Type of the object. +In a backend which doesn\[aq]t understand this (like the \f[C]local\f[R] +backend) this will become user metadata. +However should the local object be copied back to s3, the Content-Type +will be set correctly. +.SS Metadata framework +.PP +Rclone implements a metadata framework which can read metadata from an +object and write it to the object when (and only when) it is being +uploaded. +.PP +This metadata is stored as a dictionary with string keys and string +values. +.PP +There are some limits on the names of the keys (these may be clarified +further in the future). +.IP \[bu] 2 +must be lower case +.IP \[bu] 2 +may be \f[C]a-z\f[R] \f[C]0-9\f[R] containing \f[C].\f[R] \f[C]-\f[R] or +\f[C]_\f[R] +.IP \[bu] 2 +length is backend dependent +.PP +Each backend can provide system metadata that it understands. +Some backends can also store arbitrary user metadata. +.PP +Where possible the key names are standardized, so, for example, it is +possible to copy object metadata from s3 to azureblob for example and +metadata will be translated apropriately. +.PP +Some backends have limits on the size of the metadata and rclone will +give errors on upload if they are exceeded. +.SS Metadata preservation +.PP +The goal of the implementation is to +.IP "1." 3 +Preserve metadata if at all possible +.IP "2." 3 +Interpret metadata if at all possible +.PP +The consequences of 1 is that you can copy an S3 object to a local disk +then back to S3 losslessly. +Likewise you can copy a local file with file attributes and xattrs from +local disk to s3 and back again losslessly. +.PP +The consequence of 2 is that you can copy an S3 object with metadata to +Azureblob (say) and have the metadata appear on the Azureblob object +also. +.SS Standard system metadata +.PP +Here is a table of standard system metadata which, if appropriate, a +backend may implement. +.PP +.TS +tab(@); +lw(34.2n) lw(21.2n) lw(14.7n). +T{ +key +T}@T{ +description +T}@T{ +example +T} +_ +T{ +mode +T}@T{ +File type and mode: octal, unix style +T}@T{ +0100664 +T} +T{ +uid +T}@T{ +User ID of owner: decimal number +T}@T{ +500 +T} +T{ +gid +T}@T{ +Group ID of owner: decimal number +T}@T{ +500 +T} +T{ +rdev +T}@T{ +Device ID (if special file) => hexadecimal +T}@T{ +0 +T} +T{ +atime +T}@T{ +Time of last access: RFC 3339 +T}@T{ +2006-01-02T15:04:05.999999999Z07:00 +T} +T{ +mtime +T}@T{ +Time of last modification: RFC 3339 +T}@T{ +2006-01-02T15:04:05.999999999Z07:00 +T} +T{ +btime +T}@T{ +Time of file creation (birth): RFC 3339 +T}@T{ +2006-01-02T15:04:05.999999999Z07:00 +T} +T{ +cache-control +T}@T{ +Cache-Control header +T}@T{ +no-cache +T} +T{ +content-disposition +T}@T{ +Content-Disposition header +T}@T{ +inline +T} +T{ +content-encoding +T}@T{ +Content-Encoding header +T}@T{ +gzip +T} +T{ +content-language +T}@T{ +Content-Language header +T}@T{ +en-US +T} +T{ +content-type +T}@T{ +Content-Type header +T}@T{ +text/plain +T} +.TE +.PP +The metadata keys \f[C]mtime\f[R] and \f[C]content-type\f[R] will take +precedence if supplied in the metadata over reading the +\f[C]Content-Type\f[R] or modification time of the source object. +.PP +Hashes are not included in system metadata as there is a well defined +way of reading those already. .SS Options .PP Rclone has a number of options to control its behaviour. @@ -10325,13 +11200,27 @@ This means that all the info on the objects to transfer is held in memory before the transfers start. .SS --checkers=N .PP -The number of checkers to run in parallel. -Checkers do the equality checking of files during a sync. +Originally controlling just the number of file checkers to run in +parallel, e.g. +by \f[C]rclone copy\f[R]. +Now a fairly universal parallelism control used by \f[C]rclone\f[R] in +several places. +.PP +Note: checkers do the equality checking of files during a sync. For some storage systems (e.g. S3, Swift, Dropbox) this can take a significant amount of time so they are run in parallel. .PP The default is to run 8 checkers in parallel. +However, in case of slow-reacting backends you may need to lower (rather +than increase) this default by setting \f[C]--checkers\f[R] to 4 or less +threads. +This is especially advised if you are experiencing backend server +crashes during file checking phase (e.g. +on subsequent or top-up backups where little or no file copying is done +and checking takes up most of the time). +Increase this setting only with utmost care, while monitoring your +server health and file checking throughput. .SS -c, --checksum .PP Normally rclone will look at modification time and size of files to see @@ -10509,6 +11398,9 @@ Mode to run dedupe command in. One of \f[C]interactive\f[R], \f[C]skip\f[R], \f[C]first\f[R], \f[C]newest\f[R], \f[C]oldest\f[R], \f[C]rename\f[R]. The default is \f[C]interactive\f[R]. +.PD 0 +.P +.PD See the dedupe command for more information as to what these options mean. .SS --disable FEATURE,FEATURE,... @@ -10691,25 +11583,28 @@ For counts the SI standard notation is used, e.g. prefix \f[C]k\f[R] for kilo. Used with file counts, \f[C]1k\f[R] means 1000 files. .PP -The various list commands output raw numbers by default. +The various list (https://rclone.org/commands/rclone_ls/) commands +output raw numbers by default. Option \f[C]--human-readable\f[R] will make them output values in human-readable format instead (with the short unit prefix). .PP -The about command outputs human-readable by default, with a -command-specific option \f[C]--full\f[R] to output the raw numbers -instead. +The about (https://rclone.org/commands/rclone_about/) command outputs +human-readable by default, with a command-specific option +\f[C]--full\f[R] to output the raw numbers instead. .PP -Command size outputs both human-readable and raw numbers in the same -output. +Command size (https://rclone.org/commands/rclone_size/) outputs both +human-readable and raw numbers in the same output. .PP -The tree command also considers \f[C]--human-readable\f[R], but it will -not use the exact same notation as the other commands: It rounds to one -decimal, and uses single letter suffix, e.g. +The tree (https://rclone.org/commands/rclone_tree/) command also +considers \f[C]--human-readable\f[R], but it will not use the exact same +notation as the other commands: It rounds to one decimal, and uses +single letter suffix, e.g. \f[C]K\f[R] instead of \f[C]Ki\f[R]. The reason for this is that it relies on an external library. .PP -The interactive command ncdu shows human-readable by default, and -responds to key \f[C]u\f[R] for toggling human-readable format. +The interactive command ncdu (https://rclone.org/commands/rclone_ncdu/) +shows human-readable by default, and responds to key \f[C]u\f[R] for +toggling human-readable format. .SS --ignore-case-sync .PP Using this option will cause rclone to ignore the case of the files when @@ -10962,6 +11857,17 @@ Defaults to off. When the limit is reached all transfers will stop immediately. .PP Rclone will exit with exit code 8 if the transfer limit is reached. +.SS --metadata / -M +.PP +Setting this flag enables rclone to copy the metadata from the source to +the destination. +For local backends this is ownership, permissions, xattr etc. +See the #metadata for more info. +.SS --metadata-set key=value +.PP +Add metadata \f[C]key\f[R] = \f[C]value\f[R] when uploading. +This can be repeated as many times as required. +See the #metadata for more info. .SS --cutoff-mode=hard|soft|cautious .PP This modifies the behavior of \f[C]--max-transfer\f[R] Defaults to @@ -11634,6 +12540,9 @@ is giving a lot of timeouts or bigger if you have lots of bandwidth and a fast remote. .PP The default is to run 4 file transfers in parallel. +.PP +Look at --multi-thread-streams if you would like to control single file +transfers. .SS -u, --update .PP This forces rclone to skip any files which exist on the destination and @@ -11709,6 +12618,10 @@ transferred and a small number of significant events. With \f[C]-vv\f[R] rclone will become very verbose telling you about every file it considers and transfers. Please send bug reports with a log with this setting. +.PP +When setting verbosity as an environment variable, use +\f[C]RCLONE_VERBOSE=1\f[R] or \f[C]RCLONE_VERBOSE=2\f[R] for +\f[C]-v\f[R] and \f[C]-vv\f[R] respectively. .SS -V, --version .PP Prints the version number @@ -11985,6 +12898,8 @@ For the filtering options .IP \[bu] 2 \f[C]--exclude-from\f[R] .IP \[bu] 2 +\f[C]--exclude-if-present\f[R] +.IP \[bu] 2 \f[C]--include\f[R] .IP \[bu] 2 \f[C]--include-from\f[R] @@ -12113,6 +13028,10 @@ variable setting. Or to always use the trash in drive \f[C]--drive-use-trash\f[R], set \f[C]RCLONE_DRIVE_USE_TRASH=true\f[R]. .PP +Verbosity is slightly different, the environment variable equivalent of +\f[C]--verbose\f[R] or \f[C]-v\f[R] is \f[C]RCLONE_VERBOSE=1\f[R], or +for \f[C]-vv\f[R], \f[C]RCLONE_VERBOSE=2\f[R]. +.PP The same parser is used for the options and the environment variables so they take exactly the same form. .PP @@ -12354,6 +13273,36 @@ Configuration file is stored at: Now transfer it to the remote box (scp, cut paste, ftp, sftp, etc.) and place it in the correct place (use \f[C]rclone config file\f[R] on the remote box to find out where). +.SS Configuring using SSH Tunnel +.PP +Linux and MacOS users can utilize SSH Tunnel to redirect the headless +box port 53682 to local machine by using the following command: +.IP +.nf +\f[C] +ssh -L localhost:53682:localhost:53682 username\[at]remote_server +\f[R] +.fi +.PP +Then on the headless box run \f[C]rclone\f[R] config and answer +\f[C]Y\f[R] to the \f[C]Use auto config?\f[R] question. +.IP +.nf +\f[C] +\&... +Remote config +Use auto config? + * Say Y if not sure + * Say N if you are working on a remote or headless machine +y) Yes (default) +n) No +y/n> y +\f[R] +.fi +.PP +Then copy and paste the auth url +\f[C]http://127.0.0.1:53682/auth?state=xxxxxxxxxxxx\f[R] to the browser +on your local machine, complete the auth and it is done. .SH Filtering, includes and excludes .PP Filter flags determine which files rclone \f[C]sync\f[R], @@ -12528,6 +13477,10 @@ reference (https://golang.org/pkg/regexp/syntax/). Regular expressions should be enclosed in \f[C]{{\f[R] \f[C]}}\f[R]. They will match only the last path segment if the glob doesn\[aq]t start with \f[C]/\f[R] or the whole path name if it does. +Note that rclone does not attempt to parse the supplied regular +expression, meaning that using any regular expression filter will +prevent rclone from using directory filter rules, as it will instead +check every path against the supplied regular expression(s). .PP Here is how the \f[C]{{regexp}}\f[R] is transformed into an full regular expression to match the entire path: @@ -12842,10 +13795,14 @@ unnecessary directories. Whether optimisation is desirable depends on the specific filter rules and source remote content. .PP +If any regular expression filters are in use, then no directory +recursion optimisation is possible, as rclone must check every path +against the supplied regular expression(s). +.PP Directory recursion optimisation occurs if either: .IP \[bu] 2 A source remote does not support the rclone \f[C]ListR\f[R] primitive. -local, sftp, Microsoft OneDrive and WebDav do not support +local, sftp, Microsoft OneDrive and WebDAV do not support \f[C]ListR\f[R]. Google Drive and most bucket type storage do. Full list (https://rclone.org/overview/#optional-features) @@ -13475,6 +14432,8 @@ Useful for debugging. The \f[C]--exclude-if-present\f[R] flag controls whether a directory is within the scope of an rclone command based on the presence of a named file within it. +The flag can be repeated to check for multiple file names, presence of +any of them will exclude the directory. .PP This flag has a priority over other filter flags. .PP @@ -13492,9 +14451,6 @@ dir1/dir2/dir3/.ignore .PP The command \f[C]rclone ls --exclude-if-present .ignore dir1\f[R] does not list \f[C]dir3\f[R], \f[C]file3\f[R] or \f[C].ignore\f[R]. -.PP -\f[C]--exclude-if-present\f[R] can only be used once in an rclone -command. .SS Common pitfalls .PP The most frequent filter support issues on the rclone @@ -13644,11 +14600,11 @@ forum (https://forum.rclone.org/). If rclone is run with the \f[C]--rc\f[R] flag then it starts an HTTP server which can be used to remote control rclone using its API. .PP -You can either use the rclone rc command to access the API or use HTTP +You can either use the rc command to access the API or use HTTP directly. .PP -If you just want to run a remote control then see the rcd -command (https://rclone.org/commands/rclone_rcd/). +If you just want to run a remote control then see the +rcd (https://rclone.org/commands/rclone_rcd/) command. .SS Supported parameters .SS --rc .PP @@ -13772,6 +14728,14 @@ The alternative is to use \f[C]--rc-user\f[R] and \f[C]--rc-pass\f[R] and use these credentials in the request. .PP Default Off. +.SS --rc-baseurl +.PP +Prefix for URLs. +.PP +Default is root +.SS --rc-template +.PP +User-specified template. .SS Accessing the remote control via the rclone rc command .PP Rclone itself implements the remote control protocol in its @@ -14258,8 +15222,8 @@ state - state to restart with - used with continue result - result to restart with - used with continue .RE .PP -See the config create -command (https://rclone.org/commands/rclone_config_create/) command for +See the config +create (https://rclone.org/commands/rclone_config_create/) command for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] @@ -14269,8 +15233,8 @@ Parameters: .IP \[bu] 2 name - name of remote to delete .PP -See the config delete -command (https://rclone.org/commands/rclone_config_delete/) command for +See the config +delete (https://rclone.org/commands/rclone_config_delete/) command for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] @@ -14280,9 +15244,8 @@ Returns a JSON object: - key: value .PP Where keys are remote names and values are the config parameters. .PP -See the config dump -command (https://rclone.org/commands/rclone_config_dump/) command for -more information on the above. +See the config dump (https://rclone.org/commands/rclone_config_dump/) +command for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS config/get: Get a remote in the config file. @@ -14291,18 +15254,16 @@ Parameters: .IP \[bu] 2 name - name of remote to get .PP -See the config dump -command (https://rclone.org/commands/rclone_config_dump/) command for -more information on the above. +See the config dump (https://rclone.org/commands/rclone_config_dump/) +command for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS config/listremotes: Lists the remotes in the config file. .PP Returns - remotes - array of remote names .PP -See the listremotes -command (https://rclone.org/commands/rclone_listremotes/) command for -more information on the above. +See the listremotes (https://rclone.org/commands/rclone_listremotes/) +command for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS config/password: password the config for a remote. @@ -14313,8 +15274,8 @@ name - name of remote .IP \[bu] 2 parameters - a map of { \[dq]key\[dq]: \[dq]value\[dq] } pairs .PP -See the config password -command (https://rclone.org/commands/rclone_config_password/) command +See the config +password (https://rclone.org/commands/rclone_config_password/) command for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] @@ -14322,8 +15283,8 @@ for more information on the above. .PP Returns a JSON object: - providers - array of objects .PP -See the config providers -command (https://rclone.org/commands/rclone_config_providers/) command +See the config +providers (https://rclone.org/commands/rclone_config_providers/) command for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] @@ -14354,8 +15315,8 @@ state - state to restart with - used with continue result - result to restart with - used with continue .RE .PP -See the config update -command (https://rclone.org/commands/rclone_config_update/) command for +See the config +update (https://rclone.org/commands/rclone_config_update/) command for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] @@ -14895,8 +15856,8 @@ fs - a remote name string e.g. .PP The result is as returned from rclone about --json .PP -See the about command (https://rclone.org/commands/rclone_size/) command -for more information on the above. +See the about (https://rclone.org/commands/rclone_about/) command for +more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS operations/cleanup: Remove trashed files in the remote or path @@ -14906,8 +15867,8 @@ This takes the following parameters: fs - a remote name string e.g. \[dq]drive:\[dq] .PP -See the cleanup command (https://rclone.org/commands/rclone_cleanup/) -command for more information on the above. +See the cleanup (https://rclone.org/commands/rclone_cleanup/) command +for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS operations/copyfile: Copy a file from source remote to destination remote @@ -14940,9 +15901,10 @@ remote - a path within that remote e.g. url - string, URL to read from .IP \[bu] 2 autoFilename - boolean, set to true to retrieve destination file name -from url See the copyurl -command (https://rclone.org/commands/rclone_copyurl/) command for more -information on the above. +from url +.PP +See the copyurl (https://rclone.org/commands/rclone_copyurl/) command +for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS operations/delete: Remove files in the path @@ -14952,8 +15914,8 @@ This takes the following parameters: fs - a remote name string e.g. \[dq]drive:\[dq] .PP -See the delete command (https://rclone.org/commands/rclone_delete/) -command for more information on the above. +See the delete (https://rclone.org/commands/rclone_delete/) command for +more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS operations/deletefile: Remove the single file pointed to @@ -14966,9 +15928,8 @@ fs - a remote name string e.g. remote - a path within that remote e.g. \[dq]dir\[dq] .PP -See the deletefile -command (https://rclone.org/commands/rclone_deletefile/) command for -more information on the above. +See the deletefile (https://rclone.org/commands/rclone_deletefile/) +command for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS operations/fsinfo: Return information about the remote @@ -14983,46 +15944,103 @@ This returns info about the remote passed in; .nf \f[C] { - // optional features and whether they are available or not - \[dq]Features\[dq]: { - \[dq]About\[dq]: true, - \[dq]BucketBased\[dq]: false, - \[dq]CanHaveEmptyDirectories\[dq]: true, - \[dq]CaseInsensitive\[dq]: false, - \[dq]ChangeNotify\[dq]: false, - \[dq]CleanUp\[dq]: false, - \[dq]Copy\[dq]: false, - \[dq]DirCacheFlush\[dq]: false, - \[dq]DirMove\[dq]: true, - \[dq]DuplicateFiles\[dq]: false, - \[dq]GetTier\[dq]: false, - \[dq]ListR\[dq]: false, - \[dq]MergeDirs\[dq]: false, - \[dq]Move\[dq]: true, - \[dq]OpenWriterAt\[dq]: true, - \[dq]PublicLink\[dq]: false, - \[dq]Purge\[dq]: true, - \[dq]PutStream\[dq]: true, - \[dq]PutUnchecked\[dq]: false, - \[dq]ReadMimeType\[dq]: false, - \[dq]ServerSideAcrossConfigs\[dq]: false, - \[dq]SetTier\[dq]: false, - \[dq]SetWrapper\[dq]: false, - \[dq]UnWrap\[dq]: false, - \[dq]WrapFs\[dq]: false, - \[dq]WriteMimeType\[dq]: false - }, - // Names of hashes available - \[dq]Hashes\[dq]: [ - \[dq]MD5\[dq], - \[dq]SHA-1\[dq], - \[dq]DropboxHash\[dq], - \[dq]QuickXorHash\[dq] - ], - \[dq]Name\[dq]: \[dq]local\[dq], // Name as created - \[dq]Precision\[dq]: 1, // Precision of timestamps in ns - \[dq]Root\[dq]: \[dq]/\[dq], // Path as created - \[dq]String\[dq]: \[dq]Local file system at /\[dq] // how the remote will appear in logs + // optional features and whether they are available or not + \[dq]Features\[dq]: { + \[dq]About\[dq]: true, + \[dq]BucketBased\[dq]: false, + \[dq]BucketBasedRootOK\[dq]: false, + \[dq]CanHaveEmptyDirectories\[dq]: true, + \[dq]CaseInsensitive\[dq]: false, + \[dq]ChangeNotify\[dq]: false, + \[dq]CleanUp\[dq]: false, + \[dq]Command\[dq]: true, + \[dq]Copy\[dq]: false, + \[dq]DirCacheFlush\[dq]: false, + \[dq]DirMove\[dq]: true, + \[dq]Disconnect\[dq]: false, + \[dq]DuplicateFiles\[dq]: false, + \[dq]GetTier\[dq]: false, + \[dq]IsLocal\[dq]: true, + \[dq]ListR\[dq]: false, + \[dq]MergeDirs\[dq]: false, + \[dq]MetadataInfo\[dq]: true, + \[dq]Move\[dq]: true, + \[dq]OpenWriterAt\[dq]: true, + \[dq]PublicLink\[dq]: false, + \[dq]Purge\[dq]: true, + \[dq]PutStream\[dq]: true, + \[dq]PutUnchecked\[dq]: false, + \[dq]ReadMetadata\[dq]: true, + \[dq]ReadMimeType\[dq]: false, + \[dq]ServerSideAcrossConfigs\[dq]: false, + \[dq]SetTier\[dq]: false, + \[dq]SetWrapper\[dq]: false, + \[dq]Shutdown\[dq]: false, + \[dq]SlowHash\[dq]: true, + \[dq]SlowModTime\[dq]: false, + \[dq]UnWrap\[dq]: false, + \[dq]UserInfo\[dq]: false, + \[dq]UserMetadata\[dq]: true, + \[dq]WrapFs\[dq]: false, + \[dq]WriteMetadata\[dq]: true, + \[dq]WriteMimeType\[dq]: false + }, + // Names of hashes available + \[dq]Hashes\[dq]: [ + \[dq]md5\[dq], + \[dq]sha1\[dq], + \[dq]whirlpool\[dq], + \[dq]crc32\[dq], + \[dq]sha256\[dq], + \[dq]dropbox\[dq], + \[dq]mailru\[dq], + \[dq]quickxor\[dq] + ], + \[dq]Name\[dq]: \[dq]local\[dq], // Name as created + \[dq]Precision\[dq]: 1, // Precision of timestamps in ns + \[dq]Root\[dq]: \[dq]/\[dq], // Path as created + \[dq]String\[dq]: \[dq]Local file system at /\[dq], // how the remote will appear in logs + // Information about the system metadata for this backend + \[dq]MetadataInfo\[dq]: { + \[dq]System\[dq]: { + \[dq]atime\[dq]: { + \[dq]Help\[dq]: \[dq]Time of last access\[dq], + \[dq]Type\[dq]: \[dq]RFC 3339\[dq], + \[dq]Example\[dq]: \[dq]2006-01-02T15:04:05.999999999Z07:00\[dq] + }, + \[dq]btime\[dq]: { + \[dq]Help\[dq]: \[dq]Time of file birth (creation)\[dq], + \[dq]Type\[dq]: \[dq]RFC 3339\[dq], + \[dq]Example\[dq]: \[dq]2006-01-02T15:04:05.999999999Z07:00\[dq] + }, + \[dq]gid\[dq]: { + \[dq]Help\[dq]: \[dq]Group ID of owner\[dq], + \[dq]Type\[dq]: \[dq]decimal number\[dq], + \[dq]Example\[dq]: \[dq]500\[dq] + }, + \[dq]mode\[dq]: { + \[dq]Help\[dq]: \[dq]File type and mode\[dq], + \[dq]Type\[dq]: \[dq]octal, unix style\[dq], + \[dq]Example\[dq]: \[dq]0100664\[dq] + }, + \[dq]mtime\[dq]: { + \[dq]Help\[dq]: \[dq]Time of last modification\[dq], + \[dq]Type\[dq]: \[dq]RFC 3339\[dq], + \[dq]Example\[dq]: \[dq]2006-01-02T15:04:05.999999999Z07:00\[dq] + }, + \[dq]rdev\[dq]: { + \[dq]Help\[dq]: \[dq]Device ID (if special file)\[dq], + \[dq]Type\[dq]: \[dq]hexadecimal\[dq], + \[dq]Example\[dq]: \[dq]1abc\[dq] + }, + \[dq]uid\[dq]: { + \[dq]Help\[dq]: \[dq]User ID of owner\[dq], + \[dq]Type\[dq]: \[dq]decimal number\[dq], + \[dq]Example\[dq]: \[dq]500\[dq] + } + }, + \[dq]Help\[dq]: \[dq]Textual help string\[rs]n\[dq] + } } \f[R] .fi @@ -15064,6 +16082,8 @@ dirsOnly - If set only show directories .IP \[bu] 2 filesOnly - If set only show files .IP \[bu] 2 +metadata - If set return metadata of objects also +.IP \[bu] 2 hashTypes - array of strings of hash types to show if showHash set .RE .PP @@ -15075,7 +16095,7 @@ list This is an array of objects as described in the lsjson command .RE .PP -See the lsjson command (https://rclone.org/commands/rclone_lsjson/) for +See the lsjson (https://rclone.org/commands/rclone_lsjson/) command for more information on the above and examples. .PP \f[B]Authentication is required for this call.\f[R] @@ -15089,8 +16109,8 @@ fs - a remote name string e.g. remote - a path within that remote e.g. \[dq]dir\[dq] .PP -See the mkdir command (https://rclone.org/commands/rclone_mkdir/) -command for more information on the above. +See the mkdir (https://rclone.org/commands/rclone_mkdir/) command for +more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS operations/movefile: Move a file from source remote to destination remote @@ -15130,8 +16150,8 @@ Returns: .IP \[bu] 2 url - URL of the resource .PP -See the link command (https://rclone.org/commands/rclone_link/) command -for more information on the above. +See the link (https://rclone.org/commands/rclone_link/) command for more +information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS operations/purge: Remove a directory or container and all of its contents @@ -15144,8 +16164,8 @@ fs - a remote name string e.g. remote - a path within that remote e.g. \[dq]dir\[dq] .PP -See the purge command (https://rclone.org/commands/rclone_purge/) -command for more information on the above. +See the purge (https://rclone.org/commands/rclone_purge/) command for +more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS operations/rmdir: Remove an empty directory or container @@ -15158,8 +16178,8 @@ fs - a remote name string e.g. remote - a path within that remote e.g. \[dq]dir\[dq] .PP -See the rmdir command (https://rclone.org/commands/rclone_rmdir/) -command for more information on the above. +See the rmdir (https://rclone.org/commands/rclone_rmdir/) command for +more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS operations/rmdirs: Remove all the empty directories in the path @@ -15172,9 +16192,10 @@ fs - a remote name string e.g. remote - a path within that remote e.g. \[dq]dir\[dq] .IP \[bu] 2 -leaveRoot - boolean, set to true not to delete the root See the rmdirs -command (https://rclone.org/commands/rclone_rmdirs/) command for more -information on the above. +leaveRoot - boolean, set to true not to delete the root +.PP +See the rmdirs (https://rclone.org/commands/rclone_rmdirs/) command for +more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS operations/size: Count the number of bytes and files in remote @@ -15190,8 +16211,8 @@ count - number of files .IP \[bu] 2 bytes - number of bytes in those files .PP -See the size command (https://rclone.org/commands/rclone_size/) command -for more information on the above. +See the size (https://rclone.org/commands/rclone_size/) command for more +information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS operations/stat: Give information about the supplied file or directory @@ -15216,7 +16237,7 @@ Will be null if not found. Note that if you are only interested in files then it is much more efficient to set the filesOnly flag in the options. .PP -See the lsjson command (https://rclone.org/commands/rclone_lsjson/) for +See the lsjson (https://rclone.org/commands/rclone_lsjson/) command for more information on the above and examples. .PP \f[B]Authentication is required for this call.\f[R] @@ -15230,9 +16251,10 @@ fs - a remote name string e.g. remote - a path within that remote e.g. \[dq]dir\[dq] .IP \[bu] 2 -each part in body represents a file to be uploaded See the uploadfile -command (https://rclone.org/commands/rclone_uploadfile/) command for -more information on the above. +each part in body represents a file to be uploaded +.PP +See the uploadfile (https://rclone.org/commands/rclone_uploadfile/) +command for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS options/blocks: List all the option blocks @@ -15487,8 +16509,8 @@ dstFs - a remote name string e.g. .IP \[bu] 2 createEmptySrcDirs - create empty src directories on destination if set .PP -See the copy command (https://rclone.org/commands/rclone_copy/) command -for more information on the above. +See the copy (https://rclone.org/commands/rclone_copy/) command for more +information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS sync/move: move a directory from source remote to destination remote @@ -15505,8 +16527,8 @@ createEmptySrcDirs - create empty src directories on destination if set .IP \[bu] 2 deleteEmptySrcDirs - delete empty src directories if set .PP -See the move command (https://rclone.org/commands/rclone_move/) command -for more information on the above. +See the move (https://rclone.org/commands/rclone_move/) command for more +information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS sync/sync: sync a directory from source remote to destination remote @@ -15521,8 +16543,8 @@ dstFs - a remote name string e.g. .IP \[bu] 2 createEmptySrcDirs - create empty src directories on destination if set .PP -See the sync command (https://rclone.org/commands/rclone_sync/) command -for more information on the above. +See the sync (https://rclone.org/commands/rclone_sync/) command for more +information on the above. .PP \f[B]Authentication is required for this call.\f[R] .SS vfs/forget: Forget files or directories in the directory cache. @@ -15943,7 +16965,7 @@ Here is an overview of the major features of each cloud storage system. .PP .TS tab(@); -l c c c c c. +l c c c c c c. T{ Name T}@T{ @@ -15956,6 +16978,8 @@ T}@T{ Duplicate Files T}@T{ MIME Type +T}@T{ +Metadata T} _ T{ @@ -15963,507 +16987,615 @@ T{ T}@T{ Whirlpool T}@T{ -No +- T}@T{ No T}@T{ Yes T}@T{ R +T}@T{ +- T} T{ Akamai Netstorage T}@T{ MD5, SHA256 T}@T{ -Yes +R/W T}@T{ No T}@T{ No T}@T{ R +T}@T{ +- T} T{ Amazon Drive T}@T{ MD5 T}@T{ -No +- T}@T{ Yes T}@T{ No T}@T{ R +T}@T{ +- T} T{ Amazon S3 (or S3 compatible) T}@T{ MD5 T}@T{ -Yes +R/W T}@T{ No T}@T{ No T}@T{ R/W +T}@T{ +RWU T} T{ Backblaze B2 T}@T{ SHA1 T}@T{ -Yes +R/W T}@T{ No T}@T{ No T}@T{ R/W +T}@T{ +- T} T{ Box T}@T{ SHA1 T}@T{ -Yes +R/W T}@T{ Yes T}@T{ No T}@T{ - +T}@T{ +- T} T{ Citrix ShareFile T}@T{ MD5 T}@T{ -Yes +R/W T}@T{ Yes T}@T{ No T}@T{ - +T}@T{ +- T} T{ Dropbox T}@T{ DBHASH \[S1] T}@T{ -Yes +R T}@T{ Yes T}@T{ No T}@T{ - +T}@T{ +- T} T{ Enterprise File Fabric T}@T{ - T}@T{ -Yes +R/W T}@T{ Yes T}@T{ No T}@T{ R/W +T}@T{ +- T} T{ FTP T}@T{ - T}@T{ -No +R/W \[S1]\[u2070] T}@T{ No T}@T{ No T}@T{ - +T}@T{ +- T} T{ Google Cloud Storage T}@T{ MD5 T}@T{ -Yes +R/W T}@T{ No T}@T{ No T}@T{ R/W +T}@T{ +- T} T{ Google Drive T}@T{ MD5 T}@T{ -Yes +R/W T}@T{ No T}@T{ Yes T}@T{ R/W +T}@T{ +- T} T{ Google Photos T}@T{ - T}@T{ -No +- T}@T{ No T}@T{ Yes T}@T{ R +T}@T{ +- T} T{ HDFS T}@T{ - T}@T{ -Yes +R/W +T}@T{ +No +T}@T{ +No +T}@T{ +- +T}@T{ +- +T} +T{ +HiDrive +T}@T{ +HiDrive \[S1]\[S2] +T}@T{ +R/W T}@T{ No T}@T{ No T}@T{ - +T}@T{ +- T} T{ HTTP T}@T{ - T}@T{ -No +R T}@T{ No T}@T{ No T}@T{ R +T}@T{ +- T} T{ Hubic T}@T{ MD5 T}@T{ -Yes +R/W T}@T{ No T}@T{ No T}@T{ R/W +T}@T{ +- +T} +T{ +Internet Archive +T}@T{ +MD5, SHA1, CRC32 +T}@T{ +R/W \[S1]\[S1] +T}@T{ +No +T}@T{ +No +T}@T{ +- +T}@T{ +RWU T} T{ Jottacloud T}@T{ MD5 T}@T{ -Yes +R/W T}@T{ Yes T}@T{ No T}@T{ R +T}@T{ +- T} T{ Koofr T}@T{ MD5 T}@T{ -No +- T}@T{ Yes T}@T{ No T}@T{ - +T}@T{ +- T} T{ Mail.ru Cloud T}@T{ Mailru \[u2076] T}@T{ -Yes +R/W T}@T{ Yes T}@T{ No T}@T{ - +T}@T{ +- T} T{ Mega T}@T{ - T}@T{ -No +- T}@T{ No T}@T{ Yes T}@T{ - +T}@T{ +- T} T{ Memory T}@T{ MD5 T}@T{ -Yes +R/W T}@T{ No T}@T{ No T}@T{ - +T}@T{ +- T} T{ Microsoft Azure Blob Storage T}@T{ MD5 T}@T{ -Yes +R/W T}@T{ No T}@T{ No T}@T{ R/W +T}@T{ +- T} T{ Microsoft OneDrive T}@T{ SHA1 \[u2075] T}@T{ -Yes +R/W T}@T{ Yes T}@T{ No T}@T{ R +T}@T{ +- T} T{ OpenDrive T}@T{ MD5 T}@T{ -Yes +R/W T}@T{ Yes T}@T{ Partial \[u2078] T}@T{ - +T}@T{ +- T} T{ OpenStack Swift T}@T{ MD5 T}@T{ -Yes +R/W T}@T{ No T}@T{ No T}@T{ R/W +T}@T{ +- T} T{ pCloud T}@T{ MD5, SHA1 \[u2077] T}@T{ -Yes +R T}@T{ No T}@T{ No T}@T{ W +T}@T{ +- T} T{ premiumize.me T}@T{ - T}@T{ -No +- T}@T{ Yes T}@T{ No T}@T{ R +T}@T{ +- T} T{ put.io T}@T{ CRC-32 T}@T{ -Yes +R/W T}@T{ No T}@T{ Yes T}@T{ R +T}@T{ +- T} T{ QingStor T}@T{ MD5 T}@T{ -No +- \[u2079] T}@T{ No T}@T{ No T}@T{ R/W +T}@T{ +- T} T{ Seafile T}@T{ - T}@T{ -No +- T}@T{ No T}@T{ No T}@T{ - +T}@T{ +- T} T{ SFTP T}@T{ MD5, SHA1 \[S2] T}@T{ -Yes +R/W T}@T{ Depends T}@T{ No T}@T{ - +T}@T{ +- T} T{ Sia T}@T{ - T}@T{ -No +- T}@T{ No T}@T{ No T}@T{ - +T}@T{ +- T} T{ SugarSync T}@T{ - T}@T{ -No +- T}@T{ No T}@T{ No T}@T{ - +T}@T{ +- T} T{ Storj T}@T{ - T}@T{ -Yes +R T}@T{ No T}@T{ No T}@T{ - +T}@T{ +- T} T{ Uptobox T}@T{ - T}@T{ -No +- T}@T{ No T}@T{ Yes T}@T{ - +T}@T{ +- T} T{ WebDAV T}@T{ MD5, SHA1 \[S3] T}@T{ -Yes \[u2074] +R \[u2074] T}@T{ Depends T}@T{ No T}@T{ - +T}@T{ +- T} T{ Yandex Disk T}@T{ MD5 T}@T{ -Yes +R/W T}@T{ No T}@T{ No T}@T{ R +T}@T{ +- T} T{ Zoho WorkDrive T}@T{ - T}@T{ -No +- T}@T{ No T}@T{ No T}@T{ - +T}@T{ +- T} T{ The local filesystem T}@T{ All T}@T{ -Yes +R/W T}@T{ Depends T}@T{ No T}@T{ - +T}@T{ +RWU T} .TE .SS Notes @@ -16494,6 +17626,21 @@ their web client interface or other stock clients, but the underlying storage platform has been determined to allow duplicate files, and it is possible to create them with \f[C]rclone\f[R]. It may be that this is a mistake or an unsupported feature. +.PP +\[u2079] QingStor does not support SetModTime for objects bigger than 5 +GiB. +.PP +\[S1]\[u2070] FTP supports modtimes for the major FTP servers, and also +others if they advertised required protocol extensions. +See this (https://rclone.org/ftp/#modified-time) for more details. +.PP +\[S1]\[S1] Internet Archive requires option \f[C]wait_archive\f[R] to be +set to a non-zero value for full modtime support. +.PP +\[S1]\[S2] HiDrive supports its own custom +hash (https://static.hidrive.com/dev/0001). +It combines SHA1 sums for each 4 KiB block hierarchically to a single +top-level sum. .SS Hash .PP The cloud storage system supports various hash types of the objects. @@ -16505,14 +17652,41 @@ To use the verify checksums when transferring between cloud storage systems they must support a common hash type. .SS ModTime .PP -The cloud storage system supports setting modification times on objects. -If it does then this enables a using the modification times as part of -the sync. -If not then only the size will be checked by default, though the MD5SUM -can be checked with the \f[C]--checksum\f[R] flag. +Allmost all cloud storage systems store some sort of timestamp on +objects, but several of them not something that is appropriate to use +for syncing. +E.g. +some backends will only write a timestamp that represent the time of the +upload. +To be relevant for syncing it should be able to store the modification +time of the source object. +If this is not the case, rclone will only check the file size by +default, though can be configured to check the file hash (with the +\f[C]--checksum\f[R] flag). +Ideally it should also be possible to change the timestamp of an +existing file without having to re-upload it. +.PP +Storage systems with a \f[C]-\f[R] in the ModTime column, means the +modification read on objects is not the modification time of the file +when uploaded. +It is most likely the time the file was uploaded, or possibly something +else (like the time the picture was taken in Google Photos). +.PP +Storage systems with a \f[C]R\f[R] (for read-only) in the ModTime +column, means the it keeps modification times on objects, and updates +them when uploading objects, but it does not support changing only the +modification time (\f[C]SetModTime\f[R] operation) without re-uploading, +possibly not even without deleting existing first. +Some operations in rclone, such as \f[C]copy\f[R] and \f[C]sync\f[R] +commands, will automatically check for \f[C]SetModTime\f[R] support and +re-upload if necessary to keep the modification times in sync. +Other commands will not work without \f[C]SetModTime\f[R] support, e.g. +\f[C]touch\f[R] command on an existing file will fail, and changes to +modification time only on a files in a \f[C]mount\f[R] will be silently +ignored. .PP -All cloud storage systems support some kind of date on the object and -these will be set when transferring from the cloud storage system. +Storage systems with \f[C]R/W\f[R] (for read/write) in the ModTime +column, means they do also support modtime-only operations. .SS Case Insensitive .PP If a cloud storage systems is case sensitive then it is possible to have @@ -16972,148 +18146,212 @@ defaults for the backends. .PP .TS tab(@); -l l. +lw(21.7n) lw(24.1n) lw(24.1n). T{ Encoding T}@T{ Characters +T}@T{ +Encoded as T} _ T{ Asterisk T}@T{ \f[C]*\f[R] +T}@T{ +\f[C]\[uFF0A]\f[R] T} T{ BackQuote T}@T{ \f[C]\[ga]\f[R] +T}@T{ +\f[C]\[uFF40]\f[R] T} T{ BackSlash T}@T{ \f[C]\[rs]\f[R] +T}@T{ +\f[C]\[uFF3C]\f[R] T} T{ Colon T}@T{ \f[C]:\f[R] +T}@T{ +\f[C]\[uFF1A]\f[R] T} T{ CrLf T}@T{ CR 0x0D, LF 0x0A +T}@T{ +\f[C]\[u240D]\f[R], \f[C]\[u240A]\f[R] T} T{ Ctl T}@T{ All control characters 0x00-0x1F +T}@T{ +\f[C]\[u2400]\[u2401]\[u2402]\[u2403]\[u2404]\[u2405]\[u2406]\[u2407]\[u2408]\[u2409]\[u240A]\[u240B]\[u240C]\[u240D]\[u240E]\[u240F]\[u2410]\[u2411]\[u2412]\[u2413]\[u2414]\[u2415]\[u2416]\[u2417]\[u2418]\[u2419]\[u241A]\[u241B]\[u241C]\[u241D]\[u241E]\[u241F]\f[R] T} T{ Del T}@T{ DEL 0x7F +T}@T{ +\f[C]\[u2421]\f[R] T} T{ Dollar T}@T{ \f[C]$\f[R] +T}@T{ +\f[C]\[uFF04]\f[R] T} T{ Dot T}@T{ \f[C].\f[R] or \f[C]..\f[R] as entire string +T}@T{ +\f[C]\[uFF0E]\f[R], \f[C]\[uFF0E]\[uFF0E]\f[R] T} T{ DoubleQuote T}@T{ \f[C]\[dq]\f[R] +T}@T{ +\f[C]\[uFF02]\f[R] T} T{ Hash T}@T{ \f[C]#\f[R] +T}@T{ +\f[C]\[uFF03]\f[R] T} T{ InvalidUtf8 T}@T{ An invalid UTF-8 character (e.g. latin1) +T}@T{ +\f[C]\[uFFFD]\f[R] T} T{ LeftCrLfHtVt T}@T{ -CR 0x0D, LF 0x0A,HT 0x09, VT 0x0B on the left of a string +CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the left of a string +T}@T{ +\f[C]\[u240D]\f[R], \f[C]\[u240A]\f[R], \f[C]\[u2409]\f[R], +\f[C]\[u240B]\f[R] T} T{ LeftPeriod T}@T{ \f[C].\f[R] on the left of a string +T}@T{ +\f[C].\f[R] T} T{ LeftSpace T}@T{ SPACE on the left of a string +T}@T{ +\f[C]\[u2420]\f[R] T} T{ LeftTilde T}@T{ \f[C]\[ti]\f[R] on the left of a string +T}@T{ +\f[C]\[uFF5E]\f[R] T} T{ LtGt T}@T{ \f[C]<\f[R], \f[C]>\f[R] +T}@T{ +\f[C]\[uFF1C]\f[R], \f[C]\[uFF1E]\f[R] T} T{ None T}@T{ No characters are encoded +T}@T{ T} T{ Percent T}@T{ \f[C]%\f[R] +T}@T{ +\f[C]\[uFF05]\f[R] T} T{ Pipe T}@T{ | +T}@T{ +\f[C]\[uFF5C]\f[R] T} T{ Question T}@T{ \f[C]?\f[R] +T}@T{ +\f[C]\[uFF1F]\f[R] T} T{ RightCrLfHtVt T}@T{ CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string +T}@T{ +\f[C]\[u240D]\f[R], \f[C]\[u240A]\f[R], \f[C]\[u2409]\f[R], +\f[C]\[u240B]\f[R] T} T{ RightPeriod T}@T{ \f[C].\f[R] on the right of a string +T}@T{ +\f[C].\f[R] T} T{ RightSpace T}@T{ SPACE on the right of a string +T}@T{ +\f[C]\[u2420]\f[R] +T} +T{ +Semicolon +T}@T{ +\f[C];\f[R] +T}@T{ +\f[C]\[uFF1B]\f[R] T} T{ SingleQuote T}@T{ \f[C]\[aq]\f[R] +T}@T{ +\f[C]\[uFF07]\f[R] T} T{ Slash T}@T{ \f[C]/\f[R] +T}@T{ +\f[C]\[uFF0F]\f[R] T} T{ SquareBracket T}@T{ \f[C][\f[R], \f[C]]\f[R] +T}@T{ +\f[C]\[uFF3B]\f[R], \f[C]\[uFF3D]\f[R] T} .TE .SS Encoding example: FTP @@ -17215,6 +18453,41 @@ a remote which supports writing (\f[C]W\f[R]) then rclone will preserve the MIME types. Otherwise they will be guessed from the extension, or the remote itself may assign the MIME type. +.SS Metadata +.PP +Backends may or may support reading or writing metadata. +They may support reading and writing system metadata (metadata intrinsic +to that backend) and/or user metadata (general purpose metadata). +.PP +The levels of metadata support are +.PP +.TS +tab(@); +l l. +T{ +Key +T}@T{ +Explanation +T} +_ +T{ +\f[C]R\f[R] +T}@T{ +Read only System Metadata +T} +T{ +\f[C]RW\f[R] +T}@T{ +Read and write System Metadata +T} +T{ +\f[C]RWU\f[R] +T}@T{ +Read and write System Metadata and read and write User Metadata +T} +.TE +.PP +See the metadata docs (https://rclone.org/docs/#metadata) for more info. .SS Optional Features .PP All rclone remotes support a base command set. @@ -17271,13 +18544,7 @@ T}@T{ Yes T} T{ -Amazon Drive -T}@T{ -Yes -T}@T{ -No -T}@T{ -Yes +Akamai Netstorage T}@T{ Yes T}@T{ @@ -17289,14 +18556,7 @@ No T}@T{ No T}@T{ -No -T}@T{ Yes -T} -T{ -Amazon S3 -T}@T{ -No T}@T{ Yes T}@T{ @@ -17305,80 +18565,24 @@ T}@T{ No T}@T{ Yes -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -No -T}@T{ -No T} T{ -Backblaze B2 -T}@T{ -No -T}@T{ -Yes -T}@T{ -No -T}@T{ -No -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes +Amazon Drive T}@T{ Yes T}@T{ No T}@T{ -No -T} -T{ -Box -T}@T{ Yes T}@T{ Yes T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes \[dd]\[dd] -T}@T{ No T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes -T} -T{ -Citrix ShareFile -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes -T}@T{ No T}@T{ No T}@T{ -Yes -T}@T{ No T}@T{ No @@ -17386,13 +18590,9 @@ T}@T{ Yes T} T{ -Dropbox -T}@T{ -Yes -T}@T{ -Yes +Amazon S3 (or S3 compatible) T}@T{ -Yes +No T}@T{ Yes T}@T{ @@ -17407,42 +18607,13 @@ T}@T{ Yes T}@T{ Yes -T} -T{ -Enterprise File Fabric -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -No T}@T{ No T}@T{ No -T}@T{ -No -T}@T{ -Yes T} T{ -FTP -T}@T{ -No -T}@T{ -No -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -No +Backblaze B2 T}@T{ No T}@T{ @@ -17453,32 +18624,19 @@ T}@T{ No T}@T{ Yes -T} -T{ -Google Cloud Storage T}@T{ Yes T}@T{ Yes T}@T{ -No -T}@T{ -No -T}@T{ -No -T}@T{ -Yes -T}@T{ Yes T}@T{ No T}@T{ No -T}@T{ -No T} T{ -Google Drive +Box T}@T{ Yes T}@T{ @@ -17488,9 +18646,9 @@ Yes T}@T{ Yes T}@T{ -Yes +Yes \[dd]\[dd] T}@T{ -Yes +No T}@T{ Yes T}@T{ @@ -17501,67 +18659,21 @@ T}@T{ Yes T} T{ -Google Photos -T}@T{ -No -T}@T{ -No -T}@T{ -No -T}@T{ -No -T}@T{ -No -T}@T{ -No -T}@T{ -No -T}@T{ -No -T}@T{ -No -T}@T{ -No -T} -T{ -HDFS -T}@T{ -Yes -T}@T{ -No +Citrix ShareFile T}@T{ Yes T}@T{ Yes T}@T{ -No -T}@T{ -No -T}@T{ -Yes -T}@T{ -No -T}@T{ Yes T}@T{ Yes -T} -T{ -HTTP -T}@T{ -No -T}@T{ -No -T}@T{ -No T}@T{ No T}@T{ No T}@T{ -No -T}@T{ -No +Yes T}@T{ No T}@T{ @@ -17570,45 +18682,22 @@ T}@T{ Yes T} T{ -Hubic -T}@T{ -Yes \[dg] +Dropbox T}@T{ Yes T}@T{ -No -T}@T{ -No -T}@T{ -No -T}@T{ Yes T}@T{ Yes T}@T{ -No -T}@T{ Yes T}@T{ No -T} -T{ -Jottacloud T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes +No T}@T{ Yes T}@T{ -No -T}@T{ Yes T}@T{ Yes @@ -17616,7 +18705,7 @@ T}@T{ Yes T} T{ -Mail.ru Cloud +Enterprise File Fabric T}@T{ Yes T}@T{ @@ -17632,24 +18721,14 @@ No T}@T{ No T}@T{ -Yes +No T}@T{ -Yes +No T}@T{ Yes T} T{ -Mega -T}@T{ -Yes -T}@T{ -No -T}@T{ -Yes -T}@T{ -Yes -T}@T{ -Yes +FTP T}@T{ No T}@T{ @@ -17659,33 +18738,319 @@ Yes T}@T{ Yes T}@T{ -Yes -T} -T{ -Memory -T}@T{ No T}@T{ -Yes -T}@T{ No T}@T{ -No -T}@T{ -No -T}@T{ -Yes -T}@T{ Yes T}@T{ No T}@T{ No T}@T{ -No +Yes T} T{ -Microsoft Azure Blob Storage +Google Cloud Storage +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +No +T} +T{ +Google Drive +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T} +T{ +Google Photos +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T} +T{ +HDFS +T}@T{ +Yes +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T} +T{ +HiDrive +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T} +T{ +HTTP +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T} +T{ +Hubic +T}@T{ +Yes \[dg] +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T}@T{ +Yes +T}@T{ +No +T} +T{ +Internet Archive +T}@T{ +No +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T} +T{ +Jottacloud +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T} +T{ +Koofr +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T} +T{ +Mail.ru Cloud +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T} +T{ +Mega +T}@T{ +Yes +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +Yes +T} +T{ +Memory +T}@T{ +No +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +No +T} +T{ +Microsoft Azure Blob Storage T}@T{ Yes T}@T{ @@ -17915,6 +19280,29 @@ T}@T{ Yes T} T{ +Sia +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T}@T{ +No +T}@T{ +No +T}@T{ +Yes +T} +T{ SugarSync T}@T{ Yes @@ -18197,6 +19585,7 @@ These flags are available for every command. --delete-during When synchronizing, delete files during transfer --delete-excluded Delete files on dest excluded from sync --disable string Disable a comma separated list of features (use --disable help to see a list) + --disable-http-keep-alives Disable HTTP keep-alives and use each connection once. --disable-http2 Disable HTTP/2 in the global transport -n, --dry-run Do a trial run with no permanent changes --dscp string Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21 @@ -18206,7 +19595,7 @@ These flags are available for every command. --error-on-no-transfer Sets exit code 9 if no files are transferred, useful in scripts --exclude stringArray Exclude files matching pattern --exclude-from stringArray Read exclude patterns from file (use - to read from stdin) - --exclude-if-present string Exclude directories if filename is present + --exclude-if-present stringArray Exclude directories if filename is present --expect-continue-timeout duration Timeout when using expect / 100-continue in HTTP (default 1s) --fast-list Use recursive list if available; uses more memory but fewer transactions --files-from stringArray Read list of source-file names from file (use - to read from stdin) @@ -18245,6 +19634,8 @@ These flags are available for every command. --max-stats-groups int Maximum number of stats groups to keep in memory, on max oldest is discarded (default 1000) --max-transfer SizeSuffix Maximum size of data to transfer (default off) --memprofile string Write memory profile to file + -M, --metadata If set, preserve metadata when copying objects + --metadata-set stringArray Add metadata key=value when uploading --min-age Duration Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off) --min-size SizeSuffix Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off) --modify-window duration Max time diff to be considered the same (default 1ns) @@ -18316,7 +19707,7 @@ These flags are available for every command. --use-json-log Use json log format --use-mmap Use mmap allocator (see docs) --use-server-modtime Use server modified time instead of object metadata - --user-agent string Set the user-agent to a specified string (default \[dq]rclone/v1.58.0\[dq]) + --user-agent string Set the user-agent to a specified string (default \[dq]rclone/v1.59.0\[dq]) -v, --verbose count Print lots more stuff (repeat for more) \f[R] .fi @@ -18372,6 +19763,7 @@ They control the backends and may be set in the config file. --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --b2-version-at Time Show file versions as they were at the specified time (default off) --b2-versions Include old versions in directory listings --box-access-token string Box App Primary Access Token --box-auth-url string Auth server URL @@ -18411,6 +19803,7 @@ They control the backends and may be set in the config file. --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks --chunker-hash-type string Choose how chunker handles hash sums (default \[dq]md5\[dq]) --chunker-remote string Remote to chunk/unchunk + --combine-upstreams SpaceSepList Upstreams for combining --compress-level int GZIP compression level (-2 to 9) (default -1) --compress-mode string Compression mode (default \[dq]gzip\[dq]) --compress-ram-cache-limit SizeSuffix Some remotes don\[aq]t allow the upload of files with unknown size (default 20Mi) @@ -18443,6 +19836,7 @@ They control the backends and may be set in the config file. --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) + --drive-resource-key string Resource key for accessing a link-shared file --drive-root-folder-id string ID of the root folder --drive-scope string Scope that rclone should use when requesting access from drive --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs @@ -18498,6 +19892,7 @@ They control the backends and may be set in the config file. --ftp-disable-epsv Disable using EPSV even if server advertises support --ftp-disable-mlsd Disable using MLSD even if server advertises support --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) + --ftp-disable-utf8 Disable using UTF-8 even if server advertises support --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) --ftp-host string FTP host to connect to @@ -18516,8 +19911,10 @@ They control the backends and may be set in the config file. --gcs-bucket-policy-only Access checks should use bucket-level IAM policies --gcs-client-id string OAuth Client Id --gcs-client-secret string OAuth Client Secret + --gcs-decompress If set this will decompress gzip encoded objects --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) --gcs-location string Location for the newly created buckets + --gcs-no-check-bucket If set, don\[aq]t attempt to check the bucket exists or create it --gcs-object-acl string Access Control List for new objects --gcs-project-number string Project number --gcs-service-account-file string Service Account Credentials JSON file path @@ -18543,10 +19940,24 @@ They control the backends and may be set in the config file. --hdfs-namenode string Hadoop name node and port --hdfs-service-principal-name string Kerberos service principal name for the namenode --hdfs-username string Hadoop user name + --hidrive-auth-url string Auth server URL + --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) + --hidrive-client-id string OAuth Client Id + --hidrive-client-secret string OAuth Client Secret + --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary + --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --hidrive-endpoint string Endpoint for the service (default \[dq]https://api.hidrive.strato.com/2.1\[dq]) + --hidrive-root-prefix string The root/parent folder for all paths (default \[dq]/\[dq]) + --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default \[dq]rw\[dq]) + --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default \[dq]user\[dq]) + --hidrive-token string OAuth Access Token as a JSON blob + --hidrive-token-url string Token server url + --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) + --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) --http-headers CommaSepList Set HTTP headers for all transactions --http-no-head Don\[aq]t use HEAD requests --http-no-slash Set this if the site doesn\[aq]t end directories with / - --http-url string URL of http host to connect to + --http-url string URL of HTTP host to connect to --hubic-auth-url string Auth server URL --hubic-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) --hubic-client-id string OAuth Client Id @@ -18555,6 +19966,13 @@ They control the backends and may be set in the config file. --hubic-no-chunk Don\[aq]t chunk files during streaming upload --hubic-token string OAuth Access Token as a JSON blob --hubic-token-url string Token server url + --internetarchive-access-key-id string IAS3 Access Key + --internetarchive-disable-checksum Don\[aq]t ask the server to test against MD5 checksum calculated by rclone (default true) + --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) + --internetarchive-endpoint string IAS3 Endpoint (default \[dq]https://s3.us.archive.org\[dq]) + --internetarchive-front-endpoint string Host of InternetArchive Frontend (default \[dq]https://archive.org\[dq]) + --internetarchive-secret-access-key string IAS3 Secret Key (password) + --internetarchive-wait-archive Duration Timeout for waiting the server\[aq]s processing tasks (specifically archive and book_op) to finish (default 0s) --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) --jottacloud-hard-delete Delete files permanently rather than putting them into the trash --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) @@ -18576,7 +19994,7 @@ They control the backends and may be set in the config file. --local-no-preallocate Disable preallocation of disk space for transferred files --local-no-set-modtime Disable setting modtime --local-no-sparse Disable sparse files for multi-thread downloads - --local-nounc string Disable UNC (long path names) conversion on Windows + --local-nounc Disable UNC (long path names) conversion on Windows --local-unicode-normalization Apply unicode NFC normalization to paths and filenames --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) @@ -18597,11 +20015,11 @@ They control the backends and may be set in the config file. --netstorage-protocol string Select between HTTP or HTTPS protocol (default \[dq]https\[dq]) --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) -x, --one-file-system Don\[aq]t cross filesystem boundaries (unix/macOS only) + --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) --onedrive-auth-url string Auth server URL --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) --onedrive-client-id string OAuth Client Id --onedrive-client-secret string OAuth Client Secret - --onedrive-disable-site-permission Disable the request for Sites.Read.All permission --onedrive-drive-id string The ID of the drive to use --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) @@ -18625,9 +20043,11 @@ They control the backends and may be set in the config file. --pcloud-client-secret string OAuth Client Secret --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) --pcloud-hostname string Hostname to connect to (default \[dq]api.pcloud.com\[dq]) + --pcloud-password string Your pcloud password (obscured) --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default \[dq]d0\[dq]) --pcloud-token string OAuth Access Token as a JSON blob --pcloud-token-url string Token server url + --pcloud-username string Your pcloud username --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) --qingstor-access-key-id string QingStor Access Key ID @@ -18680,6 +20100,7 @@ They control the backends and may be set in the config file. --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) + --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads --s3-v2-auth If true use v2 authentication --seafile-2fa Two-factor authentication (\[aq]true\[aq] if the account has 2FA enabled) --seafile-create-library Should rclone create a library if it doesn\[aq]t exist @@ -18690,6 +20111,8 @@ They control the backends and may be set in the config file. --seafile-url string URL of seafile host to connect to --seafile-user string User name (usually email address) --sftp-ask-password Allow asking for SFTP password when needed + --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) + --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) --sftp-disable-concurrent-reads If set don\[aq]t use concurrent reads --sftp-disable-concurrent-writes If set don\[aq]t use concurrent writes --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available @@ -18702,12 +20125,14 @@ They control the backends and may be set in the config file. --sftp-known-hosts-file string Optional path to known_hosts file --sftp-md5sum-command string The command used to read md5 hashes --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) - --sftp-path-override string Override path used by SSH connection + --sftp-path-override string Override path used by SSH shell commands --sftp-port int SSH port number (default 22) --sftp-pubkey-file string Optional path to public key file --sftp-server-command string Specifies the path or command to run a sftp server on the remote host + --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands --sftp-set-modtime Set the modified time on the remote if set (default true) --sftp-sha1sum-command string The command used to read sha1 hashes + --sftp-shell-type string The type of SSH shell on remote server, if any --sftp-skip-links Set to skip any symlinks and any other non regular files --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default \[dq]sftp\[dq]) --sftp-use-fstat If set use fstat instead of stat @@ -18764,6 +20189,7 @@ They control the backends and may be set in the config file. --union-action-policy string Policy to choose upstream on ACTION category (default \[dq]epall\[dq]) --union-cache-time int Cache time of usage and free space (in seconds) (default 120) --union-create-policy string Policy to choose upstream on CREATE category (default \[dq]epmfs\[dq]) + --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) --union-search-policy string Policy to choose upstream on SEARCH category (default \[dq]ff\[dq]) --union-upstreams string List of space separated upstreams --uptobox-access-token string Your access token @@ -18775,7 +20201,7 @@ They control the backends and may be set in the config file. --webdav-pass string Password (obscured) --webdav-url string URL of http host to connect to --webdav-user string User name - --webdav-vendor string Name of the Webdav site/service/software you are using + --webdav-vendor string Name of the WebDAV site/service/software you are using --yandex-auth-url string Auth server URL --yandex-client-id string OAuth Client Id --yandex-client-secret string OAuth Client Secret @@ -19629,7 +21055,7 @@ Optional Flags: .PP Arbitrary rclone flags may be specified on the bisync command line (https://rclone.org/commands/rclone_bisync/), for example -\f[C]rclone bsync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s\f[R] +\f[C]rclone bisync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s\f[R] Note that interactions of various rclone flags with bisync process flow has not been fully tested yet. .SS Paths @@ -20030,7 +21456,7 @@ aborted run (requires a \f[C]--resync\f[R] to recover). .PP Bisync is considered \f[I]BETA\f[R] and has been tested with the following backends: - Local filesystem - Google Drive - Dropbox - -OneDrive - S3 - SFTP +OneDrive - S3 - SFTP - Yandex Disk .PP It has not been fully tested with other services yet. If it works, or sorta works, please let us know and we\[aq]ll update the @@ -20452,8 +21878,8 @@ consider using the flag Google docs exist as virtual files on Google Drive and cannot be transferred to other filesystems natively. While it is possible to export a Google doc to a normal file (with -\f[C].xlsx\f[R] extension, for example), it\[aq]s not possible to import -a normal file back into a Google document. +\f[C].xlsx\f[R] extension, for example), it is not possible to import a +normal file back into a Google document. .PP Bisync\[aq]s handling of Google Doc files is to flag them in the run log output for user\[aq]s attention and ignore them for any file transfers, @@ -21160,7 +22586,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in JSON strings. .SS Standard options .PP -Here are the standard options specific to fichier (1Fichier). +Here are the Standard options specific to fichier (1Fichier). .SS --fichier-api-key .PP Your API Key, get it from https://1fichier.com/console/params.pl. @@ -21176,7 +22602,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to fichier (1Fichier). +Here are the Advanced options specific to fichier (1Fichier). .SS --fichier-shared-folder .PP If you want to download a shared folder, add this parameter. @@ -21249,7 +22675,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SH Alias .PP @@ -21360,7 +22786,7 @@ rclone copy /home/source remote:source .fi .SS Standard options .PP -Here are the standard options specific to alias (Alias for an existing +Here are the Standard options specific to alias (Alias for an existing remote). .SS --alias-remote .PP @@ -21576,7 +23002,7 @@ Your \f[C]amazon.co.uk\f[R] email and password should work here just fine. .SS Standard options .PP -Here are the standard options specific to amazon cloud drive (Amazon +Here are the Standard options specific to amazon cloud drive (Amazon Drive). .SS --acd-client-id .PP @@ -21610,7 +23036,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to amazon cloud drive (Amazon +Here are the Advanced options specific to amazon cloud drive (Amazon Drive). .SS --acd-token .PP @@ -21773,7 +23199,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SH Amazon S3 Storage Providers .PP @@ -21785,12 +23211,22 @@ Alibaba Cloud (Aliyun) Object Storage System (OSS) .IP \[bu] 2 Ceph .IP \[bu] 2 +China Mobile Ecloud Elastic Object Storage (EOS) +.IP \[bu] 2 +Cloudflare R2 +.IP \[bu] 2 +Arvan Cloud Object Storage (AOS) +.IP \[bu] 2 DigitalOcean Spaces .IP \[bu] 2 Dreamhost .IP \[bu] 2 +Huawei OBS +.IP \[bu] 2 IBM COS S3 .IP \[bu] 2 +IDrive e2 +.IP \[bu] 2 Minio .IP \[bu] 2 RackCorp Object Storage @@ -21876,7 +23312,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, Dreamhost, IBM COS, Minio, and Tencent COS +XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS \[rs] \[dq]s3\[dq] [snip] Storage> s3 @@ -22500,10 +23936,11 @@ A simple solution is to set the \f[C]--s3-upload-cutoff 0\f[R] and force all the files to be uploaded as multipart. .SS Standard options .PP -Here are the standard options specific to s3 (Amazon S3 Compliant -Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, -Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent -COS). +Here are the Standard options specific to s3 (Amazon S3 Compliant +Storage Providers including AWS, Alibaba, Ceph, China Mobile, +Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, +IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, +StackPath, Storj, Tencent COS and Wasabi). .SS --s3-provider .PP Choose your S3 provider. @@ -22539,6 +23976,24 @@ Alibaba Cloud Object Storage System (OSS) formerly Aliyun Ceph Object Storage .RE .IP \[bu] 2 +\[dq]ChinaMobile\[dq] +.RS 2 +.IP \[bu] 2 +China Mobile Ecloud Elastic Object Storage (EOS) +.RE +.IP \[bu] 2 +\[dq]Cloudflare\[dq] +.RS 2 +.IP \[bu] 2 +Cloudflare R2 Storage +.RE +.IP \[bu] 2 +\[dq]ArvanCloud\[dq] +.RS 2 +.IP \[bu] 2 +Arvan Cloud Object Storage (AOS) +.RE +.IP \[bu] 2 \[dq]DigitalOcean\[dq] .RS 2 .IP \[bu] 2 @@ -22551,12 +24006,24 @@ Digital Ocean Spaces Dreamhost DreamObjects .RE .IP \[bu] 2 +\[dq]HuaweiOBS\[dq] +.RS 2 +.IP \[bu] 2 +Huawei Object Storage Service +.RE +.IP \[bu] 2 \[dq]IBMCOS\[dq] .RS 2 .IP \[bu] 2 IBM COS S3 .RE .IP \[bu] 2 +\[dq]IDrive\[dq] +.RS 2 +.IP \[bu] 2 +IDrive e2 +.RE +.IP \[bu] 2 \[dq]LyveCloud\[dq] .RS 2 .IP \[bu] 2 @@ -23070,6 +24537,149 @@ Amsterdam, The Netherlands .IP \[bu] 2 Paris, France .RE +.IP \[bu] 2 +\[dq]pl-waw\[dq] +.RS 2 +.IP \[bu] 2 +Warsaw, Poland +.RE +.RE +.SS --s3-region +.PP +Region to connect to. +- the location where your bucket will be created and your data stored. +Need bo be same with your endpoint. +.PP +Properties: +.IP \[bu] 2 +Config: region +.IP \[bu] 2 +Env Var: RCLONE_S3_REGION +.IP \[bu] 2 +Provider: HuaweiOBS +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]af-south-1\[dq] +.RS 2 +.IP \[bu] 2 +AF-Johannesburg +.RE +.IP \[bu] 2 +\[dq]ap-southeast-2\[dq] +.RS 2 +.IP \[bu] 2 +AP-Bangkok +.RE +.IP \[bu] 2 +\[dq]ap-southeast-3\[dq] +.RS 2 +.IP \[bu] 2 +AP-Singapore +.RE +.IP \[bu] 2 +\[dq]cn-east-3\[dq] +.RS 2 +.IP \[bu] 2 +CN East-Shanghai1 +.RE +.IP \[bu] 2 +\[dq]cn-east-2\[dq] +.RS 2 +.IP \[bu] 2 +CN East-Shanghai2 +.RE +.IP \[bu] 2 +\[dq]cn-north-1\[dq] +.RS 2 +.IP \[bu] 2 +CN North-Beijing1 +.RE +.IP \[bu] 2 +\[dq]cn-north-4\[dq] +.RS 2 +.IP \[bu] 2 +CN North-Beijing4 +.RE +.IP \[bu] 2 +\[dq]cn-south-1\[dq] +.RS 2 +.IP \[bu] 2 +CN South-Guangzhou +.RE +.IP \[bu] 2 +\[dq]ap-southeast-1\[dq] +.RS 2 +.IP \[bu] 2 +CN-Hong Kong +.RE +.IP \[bu] 2 +\[dq]sa-argentina-1\[dq] +.RS 2 +.IP \[bu] 2 +LA-Buenos Aires1 +.RE +.IP \[bu] 2 +\[dq]sa-peru-1\[dq] +.RS 2 +.IP \[bu] 2 +LA-Lima1 +.RE +.IP \[bu] 2 +\[dq]na-mexico-1\[dq] +.RS 2 +.IP \[bu] 2 +LA-Mexico City1 +.RE +.IP \[bu] 2 +\[dq]sa-chile-1\[dq] +.RS 2 +.IP \[bu] 2 +LA-Santiago2 +.RE +.IP \[bu] 2 +\[dq]sa-brazil-1\[dq] +.RS 2 +.IP \[bu] 2 +LA-Sao Paulo1 +.RE +.IP \[bu] 2 +\[dq]ru-northwest-2\[dq] +.RS 2 +.IP \[bu] 2 +RU-Moscow2 +.RE +.RE +.SS --s3-region +.PP +Region to connect to. +.PP +Properties: +.IP \[bu] 2 +Config: region +.IP \[bu] 2 +Env Var: RCLONE_S3_REGION +.IP \[bu] 2 +Provider: Cloudflare +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]auto\[dq] +.RS 2 +.IP \[bu] 2 +R2 buckets are automatically distributed across Cloudflare\[aq]s data +centers for low latency. +.RE .RE .SS --s3-region .PP @@ -23084,7 +24694,8 @@ Config: region .IP \[bu] 2 Env Var: RCLONE_S3_REGION .IP \[bu] 2 -Provider: !AWS,Alibaba,RackCorp,Scaleway,Storj,TencentCOS +Provider: +!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive .IP \[bu] 2 Type: string .IP \[bu] 2 @@ -23129,6 +24740,240 @@ Type: string Required: false .SS --s3-endpoint .PP +Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API. +.PP +Properties: +.IP \[bu] 2 +Config: endpoint +.IP \[bu] 2 +Env Var: RCLONE_S3_ENDPOINT +.IP \[bu] 2 +Provider: ChinaMobile +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]eos-wuxi-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +The default endpoint - a good choice if you are unsure. +.IP \[bu] 2 +East China (Suzhou) +.RE +.IP \[bu] 2 +\[dq]eos-jinan-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +East China (Jinan) +.RE +.IP \[bu] 2 +\[dq]eos-ningbo-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +East China (Hangzhou) +.RE +.IP \[bu] 2 +\[dq]eos-shanghai-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +East China (Shanghai-1) +.RE +.IP \[bu] 2 +\[dq]eos-zhengzhou-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Central China (Zhengzhou) +.RE +.IP \[bu] 2 +\[dq]eos-hunan-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Central China (Changsha-1) +.RE +.IP \[bu] 2 +\[dq]eos-zhuzhou-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Central China (Changsha-2) +.RE +.IP \[bu] 2 +\[dq]eos-guangzhou-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +South China (Guangzhou-2) +.RE +.IP \[bu] 2 +\[dq]eos-dongguan-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +South China (Guangzhou-3) +.RE +.IP \[bu] 2 +\[dq]eos-beijing-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +North China (Beijing-1) +.RE +.IP \[bu] 2 +\[dq]eos-beijing-2.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +North China (Beijing-2) +.RE +.IP \[bu] 2 +\[dq]eos-beijing-4.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +North China (Beijing-3) +.RE +.IP \[bu] 2 +\[dq]eos-huhehaote-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +North China (Huhehaote) +.RE +.IP \[bu] 2 +\[dq]eos-chengdu-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Southwest China (Chengdu) +.RE +.IP \[bu] 2 +\[dq]eos-chongqing-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Southwest China (Chongqing) +.RE +.IP \[bu] 2 +\[dq]eos-guiyang-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Southwest China (Guiyang) +.RE +.IP \[bu] 2 +\[dq]eos-xian-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Nouthwest China (Xian) +.RE +.IP \[bu] 2 +\[dq]eos-yunnan.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Yunnan China (Kunming) +.RE +.IP \[bu] 2 +\[dq]eos-yunnan-2.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Yunnan China (Kunming-2) +.RE +.IP \[bu] 2 +\[dq]eos-tianjin-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Tianjin China (Tianjin) +.RE +.IP \[bu] 2 +\[dq]eos-jilin-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Jilin China (Changchun) +.RE +.IP \[bu] 2 +\[dq]eos-hubei-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Hubei China (Xiangyan) +.RE +.IP \[bu] 2 +\[dq]eos-jiangxi-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Jiangxi China (Nanchang) +.RE +.IP \[bu] 2 +\[dq]eos-gansu-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Gansu China (Lanzhou) +.RE +.IP \[bu] 2 +\[dq]eos-shanxi-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Shanxi China (Taiyuan) +.RE +.IP \[bu] 2 +\[dq]eos-liaoning-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Liaoning China (Shenyang) +.RE +.IP \[bu] 2 +\[dq]eos-hebei-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Hebei China (Shijiazhuang) +.RE +.IP \[bu] 2 +\[dq]eos-fujian-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Fujian China (Xiamen) +.RE +.IP \[bu] 2 +\[dq]eos-guangxi-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Guangxi China (Nanning) +.RE +.IP \[bu] 2 +\[dq]eos-anhui-1.cmecloud.cn\[dq] +.RS 2 +.IP \[bu] 2 +Anhui China (Huainan) +.RE +.RE +.SS --s3-endpoint +.PP +Endpoint for Arvan Cloud Object Storage (AOS) API. +.PP +Properties: +.IP \[bu] 2 +Config: endpoint +.IP \[bu] 2 +Env Var: RCLONE_S3_ENDPOINT +.IP \[bu] 2 +Provider: ArvanCloud +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]s3.ir-thr-at1.arvanstorage.com\[dq] +.RS 2 +.IP \[bu] 2 +The default endpoint - a good choice if you are unsure. +.IP \[bu] 2 +Tehran Iran (Asiatech) +.RE +.IP \[bu] 2 +\[dq]s3.ir-tbz-sh1.arvanstorage.com\[dq] +.RS 2 +.IP \[bu] 2 +Tabriz Iran (Shahriar) +.RE +.RE +.SS --s3-endpoint +.PP Endpoint for IBM COS S3 API. .PP Specify if using an IBM COS On Premise. @@ -23691,6 +25536,115 @@ Middle East 1 (Dubai) .RE .SS --s3-endpoint .PP +Endpoint for OBS API. +.PP +Properties: +.IP \[bu] 2 +Config: endpoint +.IP \[bu] 2 +Env Var: RCLONE_S3_ENDPOINT +.IP \[bu] 2 +Provider: HuaweiOBS +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]obs.af-south-1.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +AF-Johannesburg +.RE +.IP \[bu] 2 +\[dq]obs.ap-southeast-2.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +AP-Bangkok +.RE +.IP \[bu] 2 +\[dq]obs.ap-southeast-3.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +AP-Singapore +.RE +.IP \[bu] 2 +\[dq]obs.cn-east-3.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +CN East-Shanghai1 +.RE +.IP \[bu] 2 +\[dq]obs.cn-east-2.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +CN East-Shanghai2 +.RE +.IP \[bu] 2 +\[dq]obs.cn-north-1.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +CN North-Beijing1 +.RE +.IP \[bu] 2 +\[dq]obs.cn-north-4.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +CN North-Beijing4 +.RE +.IP \[bu] 2 +\[dq]obs.cn-south-1.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +CN South-Guangzhou +.RE +.IP \[bu] 2 +\[dq]obs.ap-southeast-1.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +CN-Hong Kong +.RE +.IP \[bu] 2 +\[dq]obs.sa-argentina-1.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +LA-Buenos Aires1 +.RE +.IP \[bu] 2 +\[dq]obs.sa-peru-1.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +LA-Lima1 +.RE +.IP \[bu] 2 +\[dq]obs.na-mexico-1.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +LA-Mexico City1 +.RE +.IP \[bu] 2 +\[dq]obs.sa-chile-1.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +LA-Santiago2 +.RE +.IP \[bu] 2 +\[dq]obs.sa-brazil-1.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +LA-Sao Paulo1 +.RE +.IP \[bu] 2 +\[dq]obs.ru-northwest-2.myhuaweicloud.com\[dq] +.RS 2 +.IP \[bu] 2 +RU-Moscow2 +.RE +.RE +.SS --s3-endpoint +.PP Endpoint for Scaleway Object Storage. .PP Properties: @@ -23719,6 +25673,12 @@ Amsterdam Endpoint .IP \[bu] 2 Paris Endpoint .RE +.IP \[bu] 2 +\[dq]s3.pl-waw.scw.cloud\[dq] +.RS 2 +.IP \[bu] 2 +Warsaw Endpoint +.RE .RE .SS --s3-endpoint .PP @@ -24073,7 +26033,7 @@ Config: endpoint Env Var: RCLONE_S3_ENDPOINT .IP \[bu] 2 Provider: -!AWS,IBMCOS,TencentCOS,Alibaba,Scaleway,StackPath,Storj,RackCorp +!AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp .IP \[bu] 2 Type: string .IP \[bu] 2 @@ -24159,6 +26119,12 @@ Wasabi AP Northeast 1 (Tokyo) endpoint .IP \[bu] 2 Wasabi AP Northeast 2 (Osaka) endpoint .RE +.IP \[bu] 2 +\[dq]s3.ir-thr-at1.arvanstorage.com\[dq] +.RS 2 +.IP \[bu] 2 +ArvanCloud Tehran Iran (Asiatech) endpoint +.RE .RE .SS --s3-location-constraint .PP @@ -24333,6 +26299,240 @@ AWS GovCloud (US) Region .RE .SS --s3-location-constraint .PP +Location constraint - must match endpoint. +.PP +Used when creating buckets only. +.PP +Properties: +.IP \[bu] 2 +Config: location_constraint +.IP \[bu] 2 +Env Var: RCLONE_S3_LOCATION_CONSTRAINT +.IP \[bu] 2 +Provider: ChinaMobile +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]wuxi1\[dq] +.RS 2 +.IP \[bu] 2 +East China (Suzhou) +.RE +.IP \[bu] 2 +\[dq]jinan1\[dq] +.RS 2 +.IP \[bu] 2 +East China (Jinan) +.RE +.IP \[bu] 2 +\[dq]ningbo1\[dq] +.RS 2 +.IP \[bu] 2 +East China (Hangzhou) +.RE +.IP \[bu] 2 +\[dq]shanghai1\[dq] +.RS 2 +.IP \[bu] 2 +East China (Shanghai-1) +.RE +.IP \[bu] 2 +\[dq]zhengzhou1\[dq] +.RS 2 +.IP \[bu] 2 +Central China (Zhengzhou) +.RE +.IP \[bu] 2 +\[dq]hunan1\[dq] +.RS 2 +.IP \[bu] 2 +Central China (Changsha-1) +.RE +.IP \[bu] 2 +\[dq]zhuzhou1\[dq] +.RS 2 +.IP \[bu] 2 +Central China (Changsha-2) +.RE +.IP \[bu] 2 +\[dq]guangzhou1\[dq] +.RS 2 +.IP \[bu] 2 +South China (Guangzhou-2) +.RE +.IP \[bu] 2 +\[dq]dongguan1\[dq] +.RS 2 +.IP \[bu] 2 +South China (Guangzhou-3) +.RE +.IP \[bu] 2 +\[dq]beijing1\[dq] +.RS 2 +.IP \[bu] 2 +North China (Beijing-1) +.RE +.IP \[bu] 2 +\[dq]beijing2\[dq] +.RS 2 +.IP \[bu] 2 +North China (Beijing-2) +.RE +.IP \[bu] 2 +\[dq]beijing4\[dq] +.RS 2 +.IP \[bu] 2 +North China (Beijing-3) +.RE +.IP \[bu] 2 +\[dq]huhehaote1\[dq] +.RS 2 +.IP \[bu] 2 +North China (Huhehaote) +.RE +.IP \[bu] 2 +\[dq]chengdu1\[dq] +.RS 2 +.IP \[bu] 2 +Southwest China (Chengdu) +.RE +.IP \[bu] 2 +\[dq]chongqing1\[dq] +.RS 2 +.IP \[bu] 2 +Southwest China (Chongqing) +.RE +.IP \[bu] 2 +\[dq]guiyang1\[dq] +.RS 2 +.IP \[bu] 2 +Southwest China (Guiyang) +.RE +.IP \[bu] 2 +\[dq]xian1\[dq] +.RS 2 +.IP \[bu] 2 +Nouthwest China (Xian) +.RE +.IP \[bu] 2 +\[dq]yunnan\[dq] +.RS 2 +.IP \[bu] 2 +Yunnan China (Kunming) +.RE +.IP \[bu] 2 +\[dq]yunnan2\[dq] +.RS 2 +.IP \[bu] 2 +Yunnan China (Kunming-2) +.RE +.IP \[bu] 2 +\[dq]tianjin1\[dq] +.RS 2 +.IP \[bu] 2 +Tianjin China (Tianjin) +.RE +.IP \[bu] 2 +\[dq]jilin1\[dq] +.RS 2 +.IP \[bu] 2 +Jilin China (Changchun) +.RE +.IP \[bu] 2 +\[dq]hubei1\[dq] +.RS 2 +.IP \[bu] 2 +Hubei China (Xiangyan) +.RE +.IP \[bu] 2 +\[dq]jiangxi1\[dq] +.RS 2 +.IP \[bu] 2 +Jiangxi China (Nanchang) +.RE +.IP \[bu] 2 +\[dq]gansu1\[dq] +.RS 2 +.IP \[bu] 2 +Gansu China (Lanzhou) +.RE +.IP \[bu] 2 +\[dq]shanxi1\[dq] +.RS 2 +.IP \[bu] 2 +Shanxi China (Taiyuan) +.RE +.IP \[bu] 2 +\[dq]liaoning1\[dq] +.RS 2 +.IP \[bu] 2 +Liaoning China (Shenyang) +.RE +.IP \[bu] 2 +\[dq]hebei1\[dq] +.RS 2 +.IP \[bu] 2 +Hebei China (Shijiazhuang) +.RE +.IP \[bu] 2 +\[dq]fujian1\[dq] +.RS 2 +.IP \[bu] 2 +Fujian China (Xiamen) +.RE +.IP \[bu] 2 +\[dq]guangxi1\[dq] +.RS 2 +.IP \[bu] 2 +Guangxi China (Nanning) +.RE +.IP \[bu] 2 +\[dq]anhui1\[dq] +.RS 2 +.IP \[bu] 2 +Anhui China (Huainan) +.RE +.RE +.SS --s3-location-constraint +.PP +Location constraint - must match endpoint. +.PP +Used when creating buckets only. +.PP +Properties: +.IP \[bu] 2 +Config: location_constraint +.IP \[bu] 2 +Env Var: RCLONE_S3_LOCATION_CONSTRAINT +.IP \[bu] 2 +Provider: ArvanCloud +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]ir-thr-at1\[dq] +.RS 2 +.IP \[bu] 2 +Tehran Iran (Asiatech) +.RE +.IP \[bu] 2 +\[dq]ir-tbz-sh1\[dq] +.RS 2 +.IP \[bu] 2 +Tabriz Iran (Shahriar) +.RE +.RE +.SS --s3-location-constraint +.PP Location constraint - must match endpoint when using IBM Cloud Public. .PP For on-prem COS, do not make a selection from this list, hit enter. @@ -24692,7 +26892,7 @@ Config: location_constraint Env Var: RCLONE_S3_LOCATION_CONSTRAINT .IP \[bu] 2 Provider: -!AWS,IBMCOS,Alibaba,RackCorp,Scaleway,StackPath,Storj,TencentCOS +!AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS .IP \[bu] 2 Type: string .IP \[bu] 2 @@ -24716,7 +26916,7 @@ Config: acl .IP \[bu] 2 Env Var: RCLONE_S3_ACL .IP \[bu] 2 -Provider: !Storj +Provider: !Storj,Cloudflare .IP \[bu] 2 Type: string .IP \[bu] 2 @@ -24843,7 +27043,7 @@ Config: server_side_encryption .IP \[bu] 2 Env Var: RCLONE_S3_SERVER_SIDE_ENCRYPTION .IP \[bu] 2 -Provider: AWS,Ceph,Minio +Provider: AWS,Ceph,ChinaMobile,Minio .IP \[bu] 2 Type: string .IP \[bu] 2 @@ -25019,6 +27219,74 @@ Infrequent access storage mode .RE .SS --s3-storage-class .PP +The storage class to use when storing new objects in ChinaMobile. +.PP +Properties: +.IP \[bu] 2 +Config: storage_class +.IP \[bu] 2 +Env Var: RCLONE_S3_STORAGE_CLASS +.IP \[bu] 2 +Provider: ChinaMobile +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]\[dq] +.RS 2 +.IP \[bu] 2 +Default +.RE +.IP \[bu] 2 +\[dq]STANDARD\[dq] +.RS 2 +.IP \[bu] 2 +Standard storage class +.RE +.IP \[bu] 2 +\[dq]GLACIER\[dq] +.RS 2 +.IP \[bu] 2 +Archive storage mode +.RE +.IP \[bu] 2 +\[dq]STANDARD_IA\[dq] +.RS 2 +.IP \[bu] 2 +Infrequent access storage mode +.RE +.RE +.SS --s3-storage-class +.PP +The storage class to use when storing new objects in ArvanCloud. +.PP +Properties: +.IP \[bu] 2 +Config: storage_class +.IP \[bu] 2 +Env Var: RCLONE_S3_STORAGE_CLASS +.IP \[bu] 2 +Provider: ArvanCloud +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]STANDARD\[dq] +.RS 2 +.IP \[bu] 2 +Standard storage class +.RE +.RE +.SS --s3-storage-class +.PP The storage class to use when storing new objects in Tencent COS. .PP Properties: @@ -25103,10 +27371,11 @@ Prices are lower, but it needs to be restored first to be accessed. .RE .SS Advanced options .PP -Here are the advanced options specific to s3 (Amazon S3 Compliant -Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, -Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent -COS). +Here are the Advanced options specific to s3 (Amazon S3 Compliant +Storage Providers including AWS, Alibaba, Ceph, China Mobile, +Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, +IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, +StackPath, Storj, Tencent COS and Wasabi). .SS --s3-bucket-acl .PP Canned ACL used when creating buckets. @@ -25190,7 +27459,7 @@ Config: sse_customer_algorithm .IP \[bu] 2 Env Var: RCLONE_S3_SSE_CUSTOMER_ALGORITHM .IP \[bu] 2 -Provider: AWS,Ceph,Minio +Provider: AWS,Ceph,ChinaMobile,Minio .IP \[bu] 2 Type: string .IP \[bu] 2 @@ -25222,7 +27491,7 @@ Config: sse_customer_key .IP \[bu] 2 Env Var: RCLONE_S3_SSE_CUSTOMER_KEY .IP \[bu] 2 -Provider: AWS,Ceph,Minio +Provider: AWS,Ceph,ChinaMobile,Minio .IP \[bu] 2 Type: string .IP \[bu] 2 @@ -25251,7 +27520,7 @@ Config: sse_customer_key_md5 .IP \[bu] 2 Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_MD5 .IP \[bu] 2 -Provider: AWS,Ceph,Minio +Provider: AWS,Ceph,ChinaMobile,Minio .IP \[bu] 2 Type: string .IP \[bu] 2 @@ -25308,6 +27577,13 @@ stream upload is 48 GiB. If you wish to stream upload larger files then you will need to increase chunk_size. .PP +Increasing the chunk size decreases the accuracy of the progress +statistics displayed with \[dq]-P\[dq] flag. +Rclone treats chunk as sent when it\[aq]s buffered by the AWS SDK, when +in fact it may still be uploading. +A bigger chunk size means a bigger AWS SDK buffer and progress reporting +more deviating from the truth. +.PP Properties: .IP \[bu] 2 Config: chunk_size @@ -25777,6 +28053,141 @@ Env Var: RCLONE_S3_USE_MULTIPART_ETAG Type: Tristate .IP \[bu] 2 Default: unset +.SS --s3-use-presigned-request +.PP +Whether to use a presigned request or PutObject for single part uploads +.PP +If this is false rclone will use PutObject from the AWS SDK to upload an +object. +.PP +Versions of rclone < 1.59 use presigned requests to upload a single part +object and setting this flag to true will re-enable that functionality. +This shouldn\[aq]t be necessary except in exceptional circumstances or +for testing. +.PP +Properties: +.IP \[bu] 2 +Config: use_presigned_request +.IP \[bu] 2 +Env Var: RCLONE_S3_USE_PRESIGNED_REQUEST +.IP \[bu] 2 +Type: bool +.IP \[bu] 2 +Default: false +.SS Metadata +.PP +User metadata is stored as x-amz-meta- keys. +S3 metadata keys are case insensitive and are always returned in lower +case. +.PP +Here are the possible system metadata items for the s3 backend. +.PP +.TS +tab(@); +lw(11.1n) lw(11.1n) lw(11.1n) lw(16.6n) lw(20.3n). +T{ +Name +T}@T{ +Help +T}@T{ +Type +T}@T{ +Example +T}@T{ +Read Only +T} +_ +T{ +btime +T}@T{ +Time of file birth (creation) read from Last-Modified header +T}@T{ +RFC 3339 +T}@T{ +2006-01-02T15:04:05.999999999Z07:00 +T}@T{ +\f[B]Y\f[R] +T} +T{ +cache-control +T}@T{ +Cache-Control header +T}@T{ +string +T}@T{ +no-cache +T}@T{ +N +T} +T{ +content-disposition +T}@T{ +Content-Disposition header +T}@T{ +string +T}@T{ +inline +T}@T{ +N +T} +T{ +content-encoding +T}@T{ +Content-Encoding header +T}@T{ +string +T}@T{ +gzip +T}@T{ +N +T} +T{ +content-language +T}@T{ +Content-Language header +T}@T{ +string +T}@T{ +en-US +T}@T{ +N +T} +T{ +content-type +T}@T{ +Content-Type header +T}@T{ +string +T}@T{ +text/plain +T}@T{ +N +T} +T{ +mtime +T}@T{ +Time of last modification, read from rclone metadata +T}@T{ +RFC 3339 +T}@T{ +2006-01-02T15:04:05.999999999Z07:00 +T}@T{ +N +T} +T{ +tier +T}@T{ +Tier of the object +T}@T{ +string +T}@T{ +GLACIER +T}@T{ +\f[B]Y\f[R] +T} +.TE +.PP +See the metadata (https://rclone.org/docs/#metadata) docs for more info. .SS Backend commands .PP Here are the commands specific to the s3 backend. @@ -25791,9 +28202,8 @@ rclone backend COMMAND remote: .PP The help below will explain what arguments each command takes. .PP -See the \[dq]rclone backend\[dq] -command (https://rclone.org/commands/rclone_backend/) for more info on -how to pass options and arguments. +See the backend (https://rclone.org/commands/rclone_backend/) command +for more info on how to pass options and arguments. .PP These can be run on a running backend using the rc command backend/command (https://rclone.org/rc/#backend-command). @@ -25982,10 +28392,14 @@ used for transferring bulk data back to AWS. Its main software interface is S3 object storage. .PP To use rclone with AWS Snowball Edge devices, configure as standard for -an \[aq]S3 Compatible Service\[aq] be sure to set -\f[C]upload_cutoff = 0\f[R] otherwise you will run into authentication -header issues as the snowball device does not support query parameter -based authentication. +an \[aq]S3 Compatible Service\[aq]. +.PP +If using rclone pre v1.59 be sure to set \f[C]upload_cutoff = 0\f[R] +otherwise you will run into authentication header issues as the snowball +device does not support query parameter based authentication. +.PP +With rclone v1.59 or later setting \f[C]upload_cutoff\f[R] should not be +necessary. .PP eg. .IP @@ -26027,11 +28441,11 @@ storage_class = \f[R] .fi .PP -If you are using an older version of CEPH, e.g. -10.2.x Jewel, then you may need to supply the parameter -\f[C]--s3-upload-cutoff 0\f[R] or put this in the config file as -\f[C]upload_cutoff 0\f[R] to work around a bug which causes uploading of -small files to fail. +If you are using an older version of CEPH (e.g. +10.2.x Jewel) and a version of rclone before v1.59 then you may need to +supply the parameter \f[C]--s3-upload-cutoff 0\f[R] or put this in the +config file as \f[C]upload_cutoff 0\f[R] to work around a bug which +causes uploading of small files to fail. .PP Note also that Ceph sometimes puts \f[C]/\f[R] in the passwords it gives users. @@ -26061,6 +28475,115 @@ removed). Because this is a json dump, it is encoding the \f[C]/\f[R] as \f[C]\[rs]/\f[R], so if you use the secret key as \f[C]xxxxxx/xxxx\f[R] it will work fine. +.SS Cloudflare R2 +.PP +Cloudflare R2 (https://blog.cloudflare.com/r2-open-beta/) Storage allows +developers to store large amounts of unstructured data without the +costly egress bandwidth fees associated with typical cloud storage +services. +.PP +Here is an example of making a Cloudflare R2 configuration. +First run: +.IP +.nf +\f[C] +rclone config +\f[R] +.fi +.PP +This will guide you through an interactive setup process. +.PP +Note that all buckets are private, and all are stored in the same +\[dq]auto\[dq] region. +It is necessary to use Cloudflare workers to share the content of a +bucket publicly. +.IP +.nf +\f[C] +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> r2 +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +\&... +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \[rs] (s3) +\&... +Storage> s3 +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. +\&... +XX / Cloudflare R2 Storage + \[rs] (Cloudflare) +\&... +provider> Cloudflare +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \[rs] (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \[rs] (true) +env_auth> 1 +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> ACCESS_KEY +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> SECRET_ACCESS_KEY +Option region. +Region to connect to. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / R2 buckets are automatically distributed across Cloudflare\[aq]s data centers for low latency. + \[rs] (auto) +region> 1 +Option endpoint. +Endpoint for S3 API. +Required when using an S3 clone. +Enter a value. Press Enter to leave empty. +endpoint> https://ACCOUNT_ID.r2.cloudflarestorage.com +Edit advanced config? +y) Yes +n) No (default) +y/n> n +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +\f[R] +.fi +.PP +This will leave your config looking something like: +.IP +.nf +\f[C] +[r2] +type = s3 +provider = Cloudflare +access_key_id = ACCESS_KEY +secret_access_key = SECRET_ACCESS_KEY +region = auto +endpoint = https://ACCOUNT_ID.r2.cloudflarestorage.com +acl = private +\f[R] +.fi +.PP +Now run \f[C]rclone lsf r2:\f[R] to see your buckets and +\f[C]rclone lsf r2:bucket\f[R] to look within a bucket. .SS Dreamhost .PP Dreamhost DreamObjects (https://www.dreamhost.com/cloud/storage/) is an @@ -26151,6 +28674,142 @@ rclone mkdir spaces:my-new-space rclone copy /path/to/files spaces:my-new-space \f[R] .fi +.SS Huawei OBS +.PP +Object Storage Service (OBS) provides stable, secure, efficient, and +easy-to-use cloud storage that lets you store virtually any volume of +unstructured data in any format and access it from anywhere. +.PP +OBS provides an S3 interface, you can copy and modify the following +configuration and add it to your rclone configuration file. +.IP +.nf +\f[C] +[obs] +type = s3 +provider = HuaweiOBS +access_key_id = your-access-key-id +secret_access_key = your-secret-access-key +region = af-south-1 +endpoint = obs.af-south-1.myhuaweicloud.com +acl = private +\f[R] +.fi +.PP +Or you can also configure via the interactive command line: +.IP +.nf +\f[C] +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> obs +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +[snip] + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \[rs] (s3) +[snip] +Storage> 5 +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. +[snip] + 9 / Huawei Object Storage Service + \[rs] (HuaweiOBS) +[snip] +provider> 9 +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \[rs] (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \[rs] (true) +env_auth> 1 +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> your-access-key-id +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> your-secret-access-key +Option region. +Region to connect to. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / AF-Johannesburg + \[rs] (af-south-1) + 2 / AP-Bangkok + \[rs] (ap-southeast-2) +[snip] +region> 1 +Option endpoint. +Endpoint for OBS API. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / AF-Johannesburg + \[rs] (obs.af-south-1.myhuaweicloud.com) + 2 / AP-Bangkok + \[rs] (obs.ap-southeast-2.myhuaweicloud.com) +[snip] +endpoint> 1 +Option acl. +Canned ACL used when creating buckets and storing or copying objects. +This ACL is used for creating objects and if bucket_acl isn\[aq]t set, for creating buckets too. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Note that this ACL is applied when server-side copying objects as S3 +doesn\[aq]t copy the ACL from the source but rather writes a fresh one. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \[rs] (private) +[snip] +acl> 1 +Edit advanced config? +y) Yes +n) No (default) +y/n> +-------------------- +[obs] +type = s3 +provider = HuaweiOBS +access_key_id = your-access-key-id +secret_access_key = your-secret-access-key +region = af-south-1 +endpoint = obs.af-south-1.myhuaweicloud.com +acl = private +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +Current remotes: + +Name Type +==== ==== +obs s3 + +e) Edit existing remote +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> q +\f[R] +.fi .SS IBM COS (S3) .PP Information stored with IBM Cloud Object Storage is encrypted and @@ -26192,12 +28851,12 @@ Choose a number from below, or type in your own value \[rs] \[dq]alias\[dq] 2 / Amazon Drive \[rs] \[dq]amazon cloud drive\[dq] - 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, Minio, IBM COS) + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS) \[rs] \[dq]s3\[dq] 4 / Backblaze B2 \[rs] \[dq]b2\[dq] [snip] - 23 / http Connection + 23 / HTTP \[rs] \[dq]http\[dq] Storage> 3 \f[R] @@ -26365,6 +29024,122 @@ Execute rclone commands rclone delete IBM-COS-XREGION:newbucket/file.txt \f[R] .fi +.SS IDrive e2 +.PP +Here is an example of making an IDrive e2 (https://www.idrive.com/e2/) +configuration. +First run: +.IP +.nf +\f[C] +rclone config +\f[R] +.fi +.PP +This will guide you through an interactive setup process. +.IP +.nf +\f[C] +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n + +Enter name for new remote. +name> e2 + +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +[snip] +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \[rs] (s3) +[snip] +Storage> s3 + +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. +[snip] +XX / IDrive e2 + \[rs] (IDrive) +[snip] +provider> IDrive + +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \[rs] (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \[rs] (true) +env_auth> + +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> YOUR_ACCESS_KEY + +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> YOUR_SECRET_KEY + +Option acl. +Canned ACL used when creating buckets and storing or copying objects. +This ACL is used for creating objects and if bucket_acl isn\[aq]t set, for creating buckets too. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Note that this ACL is applied when server-side copying objects as S3 +doesn\[aq]t copy the ACL from the source but rather writes a fresh one. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \[rs] (private) + / Owner gets FULL_CONTROL. + 2 | The AllUsers group gets READ access. + \[rs] (public-read) + / Owner gets FULL_CONTROL. + 3 | The AllUsers group gets READ and WRITE access. + | Granting this on a bucket is generally not recommended. + \[rs] (public-read-write) + / Owner gets FULL_CONTROL. + 4 | The AuthenticatedUsers group gets READ access. + \[rs] (authenticated-read) + / Object owner gets FULL_CONTROL. + 5 | Bucket owner gets READ access. + | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \[rs] (bucket-owner-read) + / Both the object owner and the bucket owner get FULL_CONTROL over the object. + 6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. + \[rs] (bucket-owner-full-control) +acl> + +Edit advanced config? +y) Yes +n) No (default) +y/n> + +Configuration complete. +Options: +- type: s3 +- provider: IDrive +- access_key_id: YOUR_ACCESS_KEY +- secret_access_key: YOUR_SECRET_KEY +- endpoint: q9d9.la12.idrivee2-5.com +Keep this \[dq]e2\[dq] remote? +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +\f[R] +.fi .SS Minio .PP Minio (https://minio.io/) is an object storage server built for cloud @@ -26502,6 +29277,15 @@ server_side_encryption = storage_class = \f[R] .fi +.PP +C14 Cold Storage (https://www.online.net/en/storage/c14-cold-storage) is +the low-cost S3 Glacier alternative from Scaleway and it works the same +way as on S3 by accepting the \[dq]GLACIER\[dq] \f[C]storage_class\f[R]. +So you can configure your remote with the +\f[C]storage_class = GLACIER\f[R] option to upload directly to C14. +Don\[aq]t forget that in this state you can\[aq]t read files back after, +you will need to restore them to \[dq]STANDARD\[dq] storage_class first +before being able to read them (see \[dq]restore\[dq] section above) .SS Seagate Lyve Cloud .PP Seagate Lyve @@ -26533,7 +29317,7 @@ Choose \f[C]s3\f[R] backend Type of storage to configure. Choose a number from below, or type in your own value. [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \[rs] (s3) [snip] Storage> s3 @@ -26750,7 +29534,7 @@ name> wasabi Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 (also Dreamhost, Ceph, Minio) +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) \[rs] \[dq]s3\[dq] [snip] Storage> s3 @@ -26872,7 +29656,7 @@ Type of storage to configure. Enter a string value. Press Enter for the default (\[dq]\[dq]). Choose a number from below, or type in your own value [snip] - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS \[rs] \[dq]s3\[dq] [snip] Storage> s3 @@ -26962,6 +29746,378 @@ d) Delete this remote y/e/d> y \f[R] .fi +.SS China Mobile Ecloud Elastic Object Storage (EOS) +.PP +Here is an example of making an China Mobile Ecloud Elastic Object +Storage (EOS) (https:///ecloud.10086.cn/home/product-introduction/eos/) +configuration. +First run: +.IP +.nf +\f[C] +rclone config +\f[R] +.fi +.PP +This will guide you through an interactive setup process. +.IP +.nf +\f[C] +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> ChinaMobile +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. + ... + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS + \[rs] (s3) + ... +Storage> s3 +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + ... + 4 / China Mobile Ecloud Elastic Object Storage (EOS) + \[rs] (ChinaMobile) + ... +provider> ChinaMobile +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \[rs] (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \[rs] (true) +env_auth> +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> accesskeyid +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> secretaccesskey +Option endpoint. +Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + / The default endpoint - a good choice if you are unsure. + 1 | East China (Suzhou) + \[rs] (eos-wuxi-1.cmecloud.cn) + 2 / East China (Jinan) + \[rs] (eos-jinan-1.cmecloud.cn) + 3 / East China (Hangzhou) + \[rs] (eos-ningbo-1.cmecloud.cn) + 4 / East China (Shanghai-1) + \[rs] (eos-shanghai-1.cmecloud.cn) + 5 / Central China (Zhengzhou) + \[rs] (eos-zhengzhou-1.cmecloud.cn) + 6 / Central China (Changsha-1) + \[rs] (eos-hunan-1.cmecloud.cn) + 7 / Central China (Changsha-2) + \[rs] (eos-zhuzhou-1.cmecloud.cn) + 8 / South China (Guangzhou-2) + \[rs] (eos-guangzhou-1.cmecloud.cn) + 9 / South China (Guangzhou-3) + \[rs] (eos-dongguan-1.cmecloud.cn) +10 / North China (Beijing-1) + \[rs] (eos-beijing-1.cmecloud.cn) +11 / North China (Beijing-2) + \[rs] (eos-beijing-2.cmecloud.cn) +12 / North China (Beijing-3) + \[rs] (eos-beijing-4.cmecloud.cn) +13 / North China (Huhehaote) + \[rs] (eos-huhehaote-1.cmecloud.cn) +14 / Southwest China (Chengdu) + \[rs] (eos-chengdu-1.cmecloud.cn) +15 / Southwest China (Chongqing) + \[rs] (eos-chongqing-1.cmecloud.cn) +16 / Southwest China (Guiyang) + \[rs] (eos-guiyang-1.cmecloud.cn) +17 / Nouthwest China (Xian) + \[rs] (eos-xian-1.cmecloud.cn) +18 / Yunnan China (Kunming) + \[rs] (eos-yunnan.cmecloud.cn) +19 / Yunnan China (Kunming-2) + \[rs] (eos-yunnan-2.cmecloud.cn) +20 / Tianjin China (Tianjin) + \[rs] (eos-tianjin-1.cmecloud.cn) +21 / Jilin China (Changchun) + \[rs] (eos-jilin-1.cmecloud.cn) +22 / Hubei China (Xiangyan) + \[rs] (eos-hubei-1.cmecloud.cn) +23 / Jiangxi China (Nanchang) + \[rs] (eos-jiangxi-1.cmecloud.cn) +24 / Gansu China (Lanzhou) + \[rs] (eos-gansu-1.cmecloud.cn) +25 / Shanxi China (Taiyuan) + \[rs] (eos-shanxi-1.cmecloud.cn) +26 / Liaoning China (Shenyang) + \[rs] (eos-liaoning-1.cmecloud.cn) +27 / Hebei China (Shijiazhuang) + \[rs] (eos-hebei-1.cmecloud.cn) +28 / Fujian China (Xiamen) + \[rs] (eos-fujian-1.cmecloud.cn) +29 / Guangxi China (Nanning) + \[rs] (eos-guangxi-1.cmecloud.cn) +30 / Anhui China (Huainan) + \[rs] (eos-anhui-1.cmecloud.cn) +endpoint> 1 +Option location_constraint. +Location constraint - must match endpoint. +Used when creating buckets only. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / East China (Suzhou) + \[rs] (wuxi1) + 2 / East China (Jinan) + \[rs] (jinan1) + 3 / East China (Hangzhou) + \[rs] (ningbo1) + 4 / East China (Shanghai-1) + \[rs] (shanghai1) + 5 / Central China (Zhengzhou) + \[rs] (zhengzhou1) + 6 / Central China (Changsha-1) + \[rs] (hunan1) + 7 / Central China (Changsha-2) + \[rs] (zhuzhou1) + 8 / South China (Guangzhou-2) + \[rs] (guangzhou1) + 9 / South China (Guangzhou-3) + \[rs] (dongguan1) +10 / North China (Beijing-1) + \[rs] (beijing1) +11 / North China (Beijing-2) + \[rs] (beijing2) +12 / North China (Beijing-3) + \[rs] (beijing4) +13 / North China (Huhehaote) + \[rs] (huhehaote1) +14 / Southwest China (Chengdu) + \[rs] (chengdu1) +15 / Southwest China (Chongqing) + \[rs] (chongqing1) +16 / Southwest China (Guiyang) + \[rs] (guiyang1) +17 / Nouthwest China (Xian) + \[rs] (xian1) +18 / Yunnan China (Kunming) + \[rs] (yunnan) +19 / Yunnan China (Kunming-2) + \[rs] (yunnan2) +20 / Tianjin China (Tianjin) + \[rs] (tianjin1) +21 / Jilin China (Changchun) + \[rs] (jilin1) +22 / Hubei China (Xiangyan) + \[rs] (hubei1) +23 / Jiangxi China (Nanchang) + \[rs] (jiangxi1) +24 / Gansu China (Lanzhou) + \[rs] (gansu1) +25 / Shanxi China (Taiyuan) + \[rs] (shanxi1) +26 / Liaoning China (Shenyang) + \[rs] (liaoning1) +27 / Hebei China (Shijiazhuang) + \[rs] (hebei1) +28 / Fujian China (Xiamen) + \[rs] (fujian1) +29 / Guangxi China (Nanning) + \[rs] (guangxi1) +30 / Anhui China (Huainan) + \[rs] (anhui1) +location_constraint> 1 +Option acl. +Canned ACL used when creating buckets and storing or copying objects. +This ACL is used for creating objects and if bucket_acl isn\[aq]t set, for creating buckets too. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Note that this ACL is applied when server-side copying objects as S3 +doesn\[aq]t copy the ACL from the source but rather writes a fresh one. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \[rs] (private) + / Owner gets FULL_CONTROL. + 2 | The AllUsers group gets READ access. + \[rs] (public-read) + / Owner gets FULL_CONTROL. + 3 | The AllUsers group gets READ and WRITE access. + | Granting this on a bucket is generally not recommended. + \[rs] (public-read-write) + / Owner gets FULL_CONTROL. + 4 | The AuthenticatedUsers group gets READ access. + \[rs] (authenticated-read) + / Object owner gets FULL_CONTROL. +acl> private +Option server_side_encryption. +The server-side encryption algorithm used when storing this object in S3. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / None + \[rs] () + 2 / AES256 + \[rs] (AES256) +server_side_encryption> +Option storage_class. +The storage class to use when storing new objects in ChinaMobile. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / Default + \[rs] () + 2 / Standard storage class + \[rs] (STANDARD) + 3 / Archive storage mode + \[rs] (GLACIER) + 4 / Infrequent access storage mode + \[rs] (STANDARD_IA) +storage_class> +Edit advanced config? +y) Yes +n) No (default) +y/n> n +-------------------- +[ChinaMobile] +type = s3 +provider = ChinaMobile +access_key_id = accesskeyid +secret_access_key = secretaccesskey +endpoint = eos-wuxi-1.cmecloud.cn +location_constraint = wuxi1 +acl = private +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +\f[R] +.fi +.SS ArvanCloud +.PP +ArvanCloud (https://www.arvancloud.com/en/products/cloud-storage) +ArvanCloud Object Storage goes beyond the limited traditional file +storage. +It gives you access to backup and archived files and allows sharing. +Files like profile image in the app, images sent by users or scanned +documents can be stored securely and easily in our Object Storage +service. +.PP +ArvanCloud provides an S3 interface which can be configured for use with +rclone like this. +.IP +.nf +\f[C] +No remotes found, make a new one? +n) New remote +s) Set configuration password +n/s> n +name> ArvanCloud +Type of storage to configure. +Choose a number from below, or type in your own value +[snip] +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) + \[rs] \[dq]s3\[dq] +[snip] +Storage> s3 +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \[rs] \[dq]false\[dq] + 2 / Get AWS credentials from the environment (env vars or IAM) + \[rs] \[dq]true\[dq] +env_auth> 1 +AWS Access Key ID - leave blank for anonymous access or runtime credentials. +access_key_id> YOURACCESSKEY +AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. +secret_access_key> YOURSECRETACCESSKEY +Region to connect to. +Choose a number from below, or type in your own value + / The default endpoint - a good choice if you are unsure. + 1 | US Region, Northern Virginia, or Pacific Northwest. + | Leave location constraint empty. + \[rs] \[dq]us-east-1\[dq] +[snip] +region> +Endpoint for S3 API. +Leave blank if using ArvanCloud to use the default endpoint for the region. +Specify if using an S3 clone such as Ceph. +endpoint> s3.arvanstorage.com +Location constraint - must be set to match the Region. Used when creating buckets only. +Choose a number from below, or type in your own value + 1 / Empty for Iran-Tehran Region. + \[rs] \[dq]\[dq] +[snip] +location_constraint> +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \[rs] \[dq]private\[dq] +[snip] +acl> +The server-side encryption algorithm used when storing this object in S3. +Choose a number from below, or type in your own value + 1 / None + \[rs] \[dq]\[dq] + 2 / AES256 + \[rs] \[dq]AES256\[dq] +server_side_encryption> +The storage class to use when storing objects in S3. +Choose a number from below, or type in your own value + 1 / Default + \[rs] \[dq]\[dq] + 2 / Standard storage class + \[rs] \[dq]STANDARD\[dq] +storage_class> +Remote config +-------------------- +[ArvanCloud] +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +region = ir-thr-at1 +endpoint = s3.arvanstorage.com +location_constraint = +acl = +server_side_encryption = +storage_class = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +\f[R] +.fi +.PP +This will leave the config file looking like this. +.IP +.nf +\f[C] +[ArvanCloud] +type = s3 +provider = ArvanCloud +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +region = +endpoint = s3.arvanstorage.com +location_constraint = +acl = +server_side_encryption = +storage_class = +\f[R] +.fi .SS Tencent COS .PP Tencent Cloud Object Storage @@ -27004,7 +30160,7 @@ Choose a number from below, or type in your own value \[rs] \[dq]alias\[dq] 3 / Amazon Drive \[rs] \[dq]amazon cloud drive\[dq] - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS \[rs] \[dq]s3\[dq] [snip] Storage> s3 @@ -27246,7 +30402,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SH Backblaze B2 .PP @@ -27463,6 +30619,12 @@ instead of hiding it. Old versions of files, where available, are visible using the \f[C]--b2-versions\f[R] flag. .PP +It is also possible to view a bucket as it was at a certain point in +time, using the \f[C]--b2-version-at\f[R] flag. +This will show the file versions as they were at that time, showing +files that have been deleted afterwards, and hiding files that were +created since. +.PP If you wish to remove all the old versions then you can use the \f[C]rclone cleanup remote:bucket\f[R] command which will delete all the old versions of files, leaving the current ones intact. @@ -27636,7 +30798,7 @@ https://f002.backblazeb2.com/file/bucket/path/folder/file3?Authorization=xxxxxxx .fi .SS Standard options .PP -Here are the standard options specific to b2 (Backblaze B2). +Here are the Standard options specific to b2 (Backblaze B2). .SS --b2-account .PP Account ID or Application Key ID. @@ -27678,7 +30840,7 @@ Type: bool Default: false .SS Advanced options .PP -Here are the advanced options specific to b2 (Backblaze B2). +Here are the Advanced options specific to b2 (Backblaze B2). .SS --b2-endpoint .PP Endpoint for the service. @@ -27737,6 +30899,22 @@ Env Var: RCLONE_B2_VERSIONS Type: bool .IP \[bu] 2 Default: false +.SS --b2-version-at +.PP +Show file versions as they were at the specified time. +.PP +Note that when using this no file write operations are permitted, so you +can\[aq]t upload files or delete them. +.PP +Properties: +.IP \[bu] 2 +Config: version_at +.IP \[bu] 2 +Env Var: RCLONE_B2_VERSION_AT +.IP \[bu] 2 +Type: Time +.IP \[bu] 2 +Default: off .SS --b2-upload-cutoff .PP Cutoff for switching to chunked upload. @@ -27912,7 +31090,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SH Box .PP @@ -28234,7 +31412,7 @@ you use \f[C]11xxxxxxxxx8\f[R] as the \f[C]root_folder_id\f[R] in the config. .SS Standard options .PP -Here are the standard options specific to box (Box). +Here are the Standard options specific to box (Box). .SS --box-client-id .PP OAuth Client Id. @@ -28327,7 +31505,7 @@ Rclone should act on behalf of a service account. .RE .SS Advanced options .PP -Here are the advanced options specific to box (Box). +Here are the Advanced options specific to box (Box). .SS --box-token .PP OAuth Access Token as a JSON blob. @@ -28469,7 +31647,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SH Cache (DEPRECATED) .PP @@ -28821,7 +31999,7 @@ Params: - \f[B]remote\f[R] = path to remote \f[B](required)\f[R] - \f[I](optional, false by default)\f[R] .SS Standard options .PP -Here are the standard options specific to cache (Cache a remote). +Here are the Standard options specific to cache (Cache a remote). .SS --cache-remote .PP Remote to cache. @@ -29000,7 +32178,7 @@ Examples: .RE .SS Advanced options .PP -Here are the advanced options specific to cache (Cache a remote). +Here are the Advanced options specific to cache (Cache a remote). .SS --cache-plex-token .PP The plex token for authentication - auto set normally. @@ -29286,9 +32464,8 @@ rclone backend COMMAND remote: .PP The help below will explain what arguments each command takes. .PP -See the \[dq]rclone backend\[dq] -command (https://rclone.org/commands/rclone_backend/) for more info on -how to pass options and arguments. +See the backend (https://rclone.org/commands/rclone_backend/) command +for more info on how to pass options and arguments. .PP These can be run on a running backend using the rc command backend/command (https://rclone.org/rc/#backend-command). @@ -29647,7 +32824,7 @@ Changing \f[C]transactions\f[R] is dangerous and requires explicit migration. .SS Standard options .PP -Here are the standard options specific to chunker (Transparently +Here are the Standard options specific to chunker (Transparently chunk/split large files). .SS --chunker-remote .PP @@ -29746,7 +32923,7 @@ Similar to \[dq]md5quick\[dq] but prefers SHA1 over MD5. .RE .SS Advanced options .PP -Here are the advanced options specific to chunker (Transparently +Here are the Advanced options specific to chunker (Transparently chunk/split large files). .SS --chunker-name-format .PP @@ -30151,7 +33328,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in JSON strings. .SS Standard options .PP -Here are the standard options specific to sharefile (Citrix Sharefile). +Here are the Standard options specific to sharefile (Citrix Sharefile). .SS --sharefile-root-folder-id .PP ID of the root folder. @@ -30206,7 +33383,7 @@ connectors. .RE .SS Advanced options .PP -Here are the advanced options specific to sharefile (Citrix Sharefile). +Here are the Advanced options specific to sharefile (Citrix Sharefile). .SS --sharefile-upload-cutoff .PP Cutoff for switching to multipart upload. @@ -30286,7 +33463,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SH Crypt .PP @@ -30753,7 +33930,7 @@ crypted remote instead of \f[C]rclone check\f[R] which can\[aq]t check the checksums properly. .SS Standard options .PP -Here are the standard options specific to crypt (Encrypt/Decrypt a +Here are the Standard options specific to crypt (Encrypt/Decrypt a remote). .SS --crypt-remote .PP @@ -30880,7 +34057,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to crypt (Encrypt/Decrypt a +Here are the Advanced options specific to crypt (Encrypt/Decrypt a remote). .SS --crypt-server-side-across-configs .PP @@ -31001,6 +34178,11 @@ Unicode codepoint instead of UTF-8 byte length. Onedrive) .RE .RE +.SS Metadata +.PP +Any metadata supported by the underlying remote is read and written. +.PP +See the metadata (https://rclone.org/docs/#metadata) docs for more info. .SS Backend commands .PP Here are the commands specific to the crypt backend. @@ -31015,9 +34197,8 @@ rclone backend COMMAND remote: .PP The help below will explain what arguments each command takes. .PP -See the \[dq]rclone backend\[dq] -command (https://rclone.org/commands/rclone_backend/) for more info on -how to pass options and arguments. +See the backend (https://rclone.org/commands/rclone_backend/) command +for more info on how to pass options and arguments. .PP These can be run on a running backend using the rc command backend/command (https://rclone.org/rc/#backend-command). @@ -31302,7 +34483,7 @@ The file names should not be changed by anything other than the rclone compression backend. .SS Standard options .PP -Here are the standard options specific to compress (Compress a remote). +Here are the Standard options specific to compress (Compress a remote). .SS --compress-remote .PP Remote to compress. @@ -31341,7 +34522,7 @@ Standard gzip compression with fastest parameters. .RE .SS Advanced options .PP -Here are the advanced options specific to compress (Compress a remote). +Here are the Advanced options specific to compress (Compress a remote). .SS --compress-level .PP GZIP compression level (-2 to 9). @@ -31381,6 +34562,196 @@ Env Var: RCLONE_COMPRESS_RAM_CACHE_LIMIT Type: SizeSuffix .IP \[bu] 2 Default: 20Mi +.SS Metadata +.PP +Any metadata supported by the underlying remote is read and written. +.PP +See the metadata (https://rclone.org/docs/#metadata) docs for more info. +.SH Combine +.PP +The \f[C]combine\f[R] backend joins remotes together into a single +directory tree. +.PP +For example you might have a remote for images on one provider: +.IP +.nf +\f[C] +$ rclone tree s3:imagesbucket +/ +\[u251C]\[u2500]\[u2500] image1.jpg +\[u2514]\[u2500]\[u2500] image2.jpg +\f[R] +.fi +.PP +And a remote for files on another: +.IP +.nf +\f[C] +$ rclone tree drive:important/files +/ +\[u251C]\[u2500]\[u2500] file1.txt +\[u2514]\[u2500]\[u2500] file2.txt +\f[R] +.fi +.PP +The \f[C]combine\f[R] backend can join these together into a synthetic +directory structure like this: +.IP +.nf +\f[C] +$ rclone tree combined: +/ +\[u251C]\[u2500]\[u2500] files +\[br] \[u251C]\[u2500]\[u2500] file1.txt +\[br] \[u2514]\[u2500]\[u2500] file2.txt +\[u2514]\[u2500]\[u2500] images + \[u251C]\[u2500]\[u2500] image1.jpg + \[u2514]\[u2500]\[u2500] image2.jpg +\f[R] +.fi +.PP +You\[aq]d do this by specifying an \f[C]upstreams\f[R] parameter in the +config like this +.IP +.nf +\f[C] +upstreams = images=s3:imagesbucket files=drive:important/files +\f[R] +.fi +.PP +During the initial setup with \f[C]rclone config\f[R] you will specify +the upstreams remotes as a space separated list. +The upstream remotes can either be a local paths or other remotes. +.SS Configuration +.PP +Here is an example of how to make a combine called \f[C]remote\f[R] for +the example above. +First run: +.IP +.nf +\f[C] + rclone config +\f[R] +.fi +.PP +This will guide you through an interactive setup process: +.IP +.nf +\f[C] +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +\&... +XX / Combine several remotes into one + \[rs] (combine) +\&... +Storage> combine +Option upstreams. +Upstreams for combining +These should be in the form + dir=remote:path dir2=remote2:path +Where before the = is specified the root directory and after is the remote to +put there. +Embedded spaces can be added using quotes + \[dq]dir=remote:path with space\[dq] \[dq]dir2=remote2:path with space\[dq] +Enter a fs.SpaceSepList value. +upstreams> images=s3:imagesbucket files=drive:important/files +-------------------- +[remote] +type = combine +upstreams = images=s3:imagesbucket files=drive:important/files +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +\f[R] +.fi +.SS Configuring for Google Drive Shared Drives +.PP +Rclone has a convenience feature for making a combine backend for all +the shared drives you have access to. +.PP +Assuming your main (non shared drive) Google drive remote is called +\f[C]drive:\f[R] you would run +.IP +.nf +\f[C] +rclone backend -o config drives drive: +\f[R] +.fi +.PP +This would produce something like this: +.IP +.nf +\f[C] +[My Drive] +type = alias +remote = drive,team_drive=0ABCDEF-01234567890,root_folder_id=: + +[Test Drive] +type = alias +remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=: + +[AllDrives] +type = combine +remote = \[dq]My Drive=My Drive:\[dq] \[dq]Test Drive=Test Drive:\[dq] +\f[R] +.fi +.PP +If you then add that config to your config file (find it with +\f[C]rclone config file\f[R]) then you can access all the shared drives +in one place with the \f[C]AllDrives:\f[R] remote. +.PP +See the Google Drive docs (https://rclone.org/drive/#drives) for full +info. +.SS Standard options +.PP +Here are the Standard options specific to combine (Combine several +remotes into one). +.SS --combine-upstreams +.PP +Upstreams for combining +.PP +These should be in the form +.IP +.nf +\f[C] +dir=remote:path dir2=remote2:path +\f[R] +.fi +.PP +Where before the = is specified the root directory and after is the +remote to put there. +.PP +Embedded spaces can be added using quotes +.IP +.nf +\f[C] +\[dq]dir=remote:path with space\[dq] \[dq]dir2=remote2:path with space\[dq] +\f[R] +.fi +.PP +Properties: +.IP \[bu] 2 +Config: upstreams +.IP \[bu] 2 +Env Var: RCLONE_COMBINE_UPSTREAMS +.IP \[bu] 2 +Type: SpaceSepList +.IP \[bu] 2 +Default: +.SS Metadata +.PP +Any metadata supported by the underlying remote is read and written. +.PP +See the metadata (https://rclone.org/docs/#metadata) docs for more info. .SH Dropbox .PP Paths are specified as \f[C]remote:path\f[R] @@ -31629,7 +35000,7 @@ Note that there may be a pause when quitting rclone while rclone finishes up the last batch using this mode. .SS Standard options .PP -Here are the standard options specific to dropbox (Dropbox). +Here are the Standard options specific to dropbox (Dropbox). .SS --dropbox-client-id .PP OAuth Client Id. @@ -31662,7 +35033,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to dropbox (Dropbox). +Here are the Advanced options specific to dropbox (Dropbox). .SS --dropbox-token .PP OAuth Access Token as a JSON blob. @@ -32160,7 +35531,7 @@ $ rclone lsf --dirs-only -Fip --csv filefabric: The ID for \[dq]S3 Storage\[dq] would be \f[C]120673761\f[R]. .SS Standard options .PP -Here are the standard options specific to filefabric (Enterprise File +Here are the Standard options specific to filefabric (Enterprise File Fabric). .SS --filefabric-url .PP @@ -32239,7 +35610,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to filefabric (Enterprise File +Here are the Advanced options specific to filefabric (Enterprise File Fabric). .SS --filefabric-token .PP @@ -32348,7 +35719,7 @@ Type of storage to configure. Enter a string value. Press Enter for the default (\[dq]\[dq]). Choose a number from below, or type in your own value [snip] -XX / FTP Connection +XX / FTP \[rs] \[dq]ftp\[dq] [snip] Storage> ftp @@ -32498,7 +35869,7 @@ VsFTPd. Just hit a selection number when prompted. .SS Standard options .PP -Here are the standard options specific to ftp (FTP Connection). +Here are the Standard options specific to ftp (FTP). .SS --ftp-host .PP FTP host to connect to. @@ -32595,7 +35966,7 @@ Type: bool Default: false .SS Advanced options .PP -Here are the advanced options specific to ftp (FTP Connection). +Here are the Advanced options specific to ftp (FTP). .SS --ftp-concurrency .PP Maximum number of FTP simultaneous connections, 0 for unlimited. @@ -32648,6 +36019,19 @@ Env Var: RCLONE_FTP_DISABLE_MLSD Type: bool .IP \[bu] 2 Default: false +.SS --ftp-disable-utf8 +.PP +Disable using UTF-8 even if server advertises support. +.PP +Properties: +.IP \[bu] 2 +Config: disable_utf8 +.IP \[bu] 2 +Env Var: RCLONE_FTP_DISABLE_UTF8 +.IP \[bu] 2 +Type: bool +.IP \[bu] 2 +Default: false .SS --ftp-writing-mdtm .PP Use MDTM to set modification time (VsFtpd quirk) @@ -32812,7 +36196,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .PP The implementation of : \f[C]--dump headers\f[R], @@ -33195,7 +36579,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in JSON strings. .SS Standard options .PP -Here are the standard options specific to google cloud storage (Google +Here are the Standard options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). .SS --gcs-client-id .PP @@ -33734,7 +37118,7 @@ Durable reduced availability storage class .RE .SS Advanced options .PP -Here are the advanced options specific to google cloud storage (Google +Here are the Advanced options specific to google cloud storage (Google Cloud Storage (this is not Google Drive)). .SS --gcs-token .PP @@ -33779,6 +37163,44 @@ Env Var: RCLONE_GCS_TOKEN_URL Type: string .IP \[bu] 2 Required: false +.SS --gcs-no-check-bucket +.PP +If set, don\[aq]t attempt to check the bucket exists or create it. +.PP +This can be useful when trying to minimise the number of transactions +rclone does if you know the bucket exists already. +.PP +Properties: +.IP \[bu] 2 +Config: no_check_bucket +.IP \[bu] 2 +Env Var: RCLONE_GCS_NO_CHECK_BUCKET +.IP \[bu] 2 +Type: bool +.IP \[bu] 2 +Default: false +.SS --gcs-decompress +.PP +If set this will decompress gzip encoded objects. +.PP +It is possible to upload objects to GCS with \[dq]Content-Encoding: +gzip\[dq] set. +Normally rclone will download these files files as compressed objects. +.PP +If this flag is set then rclone will decompress these files with +\[dq]Content-Encoding: gzip\[dq] as they are received. +This means that rclone can\[aq]t check the size and hash but the file +contents will be decompressed. +.PP +Properties: +.IP \[bu] 2 +Config: decompress +.IP \[bu] 2 +Env Var: RCLONE_GCS_DECOMPRESS +.IP \[bu] 2 +Type: bool +.IP \[bu] 2 +Default: false .SS --gcs-encoding .PP The encoding for the backend. @@ -33804,7 +37226,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SH Google Drive .PP @@ -33867,8 +37289,6 @@ Choose a number from below, or type in your own value 5 | does not allow any access to read or download file content. \[rs] \[dq]drive.metadata.readonly\[dq] scope> 1 -ID of the root folder - leave blank normally. Fill in to access \[dq]Computers\[dq] folders. (see docs). -root_folder_id> Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. service_account_file> Remote config @@ -33979,6 +37399,7 @@ It does not allow rclone to download or upload data, or rename or delete files or directories. .SS Root folder ID .PP +This option has been moved to the advanced section. You can set the \f[C]root_folder_id\f[R] for rclone. This is the directory (identified by its \f[C]Folder ID\f[R]) that rclone considers to be the root of your drive. @@ -34495,6 +37916,13 @@ Description T} _ T{ +bmp +T}@T{ +image/bmp +T}@T{ +Windows Bitmap format +T} +T{ csv T}@T{ text/csv @@ -34502,6 +37930,13 @@ T}@T{ Standard CSV format for Spreadsheets T} T{ +doc +T}@T{ +application/msword +T}@T{ +Classic Word file +T} +T{ docx T}@T{ application/vnd.openxmlformats-officedocument.wordprocessingml.document @@ -34534,7 +37969,7 @@ json T}@T{ application/vnd.google-apps.script+json T}@T{ -JSON Text Format +JSON Text Format for Google Apps scripts T} T{ odp @@ -34572,6 +38007,13 @@ T}@T{ Adobe PDF Format T} T{ +pjpeg +T}@T{ +image/pjpeg +T}@T{ +Progressive JPEG Image +T} +T{ png T}@T{ image/png @@ -34614,6 +38056,20 @@ T}@T{ Plain Text T} T{ +wmf +T}@T{ +application/x-msmetafile +T}@T{ +Windows Meta File +T} +T{ +xls +T}@T{ +application/vnd.ms-excel +T}@T{ +Classic Excel file +T} +T{ xlsx T}@T{ application/vnd.openxmlformats-officedocument.spreadsheetml.sheet @@ -34678,7 +38134,7 @@ T} .TE .SS Standard options .PP -Here are the standard options specific to drive (Google Drive). +Here are the Standard options specific to drive (Google Drive). .SS --drive-client-id .PP Google Application Client Id Setting your own is recommended. @@ -34766,23 +38222,6 @@ Allows read-only access to file metadata but does not allow any access to read or download file content. .RE .RE -.SS --drive-root-folder-id -.PP -ID of the root folder. -Leave blank normally. -.PP -Fill in to access \[dq]Computers\[dq] folders (see docs), or for rclone -to use a non root folder as its starting point. -.PP -Properties: -.IP \[bu] 2 -Config: root_folder_id -.IP \[bu] 2 -Env Var: RCLONE_DRIVE_ROOT_FOLDER_ID -.IP \[bu] 2 -Type: string -.IP \[bu] 2 -Required: false .SS --drive-service-account-file .PP Service Account Credentials JSON file path. @@ -34817,7 +38256,7 @@ Type: bool Default: false .SS Advanced options .PP -Here are the advanced options specific to drive (Google Drive). +Here are the Advanced options specific to drive (Google Drive). .SS --drive-token .PP OAuth Access Token as a JSON blob. @@ -34861,6 +38300,23 @@ Env Var: RCLONE_DRIVE_TOKEN_URL Type: string .IP \[bu] 2 Required: false +.SS --drive-root-folder-id +.PP +ID of the root folder. +Leave blank normally. +.PP +Fill in to access \[dq]Computers\[dq] folders (see docs), or for rclone +to use a non root folder as its starting point. +.PP +Properties: +.IP \[bu] 2 +Config: root_folder_id +.IP \[bu] 2 +Env Var: RCLONE_DRIVE_ROOT_FOLDER_ID +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false .SS --drive-service-account-credentials .PP Service Account Credentials JSON blob. @@ -35410,6 +38866,40 @@ Env Var: RCLONE_DRIVE_SKIP_DANGLING_SHORTCUTS Type: bool .IP \[bu] 2 Default: false +.SS --drive-resource-key +.PP +Resource key for accessing a link-shared file. +.PP +If you need to access files shared with a link like this +.IP +.nf +\f[C] +https://drive.google.com/drive/folders/XXX?resourcekey=YYY&usp=sharing +\f[R] +.fi +.PP +Then you will need to use the first part \[dq]XXX\[dq] as the +\[dq]root_folder_id\[dq] and the second part \[dq]YYY\[dq] as the +\[dq]resource_key\[dq] otherwise you will get 404 not found errors when +trying to access the directory. +.PP +See: https://developers.google.com/drive/api/guides/resource-keys +.PP +This resource key requirement only applies to a subset of old files. +.PP +Note also that opening the folder once in the web interface (with the +user you\[aq]ve authenticated rclone with) seems to be enough so that +the resource key is no needed. +.PP +Properties: +.IP \[bu] 2 +Config: resource_key +.IP \[bu] 2 +Env Var: RCLONE_DRIVE_RESOURCE_KEY +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false .SS --drive-encoding .PP The encoding for the backend. @@ -35440,9 +38930,8 @@ rclone backend COMMAND remote: .PP The help below will explain what arguments each command takes. .PP -See the \[dq]rclone backend\[dq] -command (https://rclone.org/commands/rclone_backend/) for more info on -how to pass options and arguments. +See the backend (https://rclone.org/commands/rclone_backend/) command +for more info on how to pass options and arguments. .PP These can be run on a running backend using the rc command backend/command (https://rclone.org/rc/#backend-command). @@ -35578,7 +39067,7 @@ This will return a JSON list of objects like this .PP With the -o config parameter it will output the list in a format suitable for adding to a config file to make aliases for all the drives -found. +found and a combined drive. .IP .nf \f[C] @@ -35589,12 +39078,19 @@ remote = drive,team_drive=0ABCDEF-01234567890,root_folder_id=: [Test Drive] type = alias remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=: + +[AllDrives] +type = combine +remote = \[dq]My Drive=My Drive:\[dq] \[dq]Test Drive=Test Drive:\[dq] \f[R] .fi .PP Adding this to the rclone config file will cause those team drives to be accessible with the aliases shown. -This may require manual editing of the names. +Any illegal charactes will be substituted with \[dq]_\[dq] and duplicate +names will have numbers suffixed. +It will also add a remote called AllDrives which shows all the shared +drives combined into one directory tree. .SS untrash .PP Untrash files and directories @@ -35666,6 +39162,24 @@ If the destination is a drive backend then server-side copying will be attempted if possible. .PP Use the -i flag to see what would be copied before copying. +.SS exportformats +.PP +Dump the export formats for debug purposes +.IP +.nf +\f[C] +rclone backend exportformats remote: [options] [+] +\f[R] +.fi +.SS importformats +.PP +Dump the import formats for debug purposes +.IP +.nf +\f[C] +rclone backend importformats remote: [options] [+] +\f[R] +.fi .SS Limitations .PP Drive has quite a lot of rate limiting. @@ -35681,9 +39195,13 @@ You can disable server-side copies with \f[C]--disable copy\f[R] to download and upload the files if you prefer. .SS Limitations of Google Docs .PP -Google docs will appear as size -1 in \f[C]rclone ls\f[R] and as size 0 -in anything which uses the VFS layer, e.g. -\f[C]rclone mount\f[R], \f[C]rclone serve\f[R]. +Google docs will appear as size -1 in \f[C]rclone ls\f[R], +\f[C]rclone ncdu\f[R] etc, and as size 0 in anything which uses the VFS +layer, e.g. +\f[C]rclone mount\f[R] and \f[C]rclone serve\f[R]. +When calculating directory totals, e.g. +in \f[C]rclone size\f[R] and \f[C]rclone ncdu\f[R], they will be counted +in as empty files. .PP This is because rclone can\[aq]t find out the size of the Google docs without downloading them. @@ -35746,21 +39264,21 @@ recommended to stay under that number as if you use more than that, it will cause rclone to rate limit and make things slower. .PP Here is how to create your own Google Drive client ID for rclone: -.IP "1." 3 +.IP " 1." 4 Log into the Google API Console (https://console.developers.google.com/) with your Google account. It doesn\[aq]t matter what Google account you use. (It need not be the same account as the Google Drive you want to access) -.IP "2." 3 +.IP " 2." 4 Select a project or create a new project. -.IP "3." 3 +.IP " 3." 4 Under \[dq]ENABLE APIS AND SERVICES\[dq] search for \[dq]Drive\[dq], and enable the \[dq]Google Drive API\[dq]. -.IP "4." 3 +.IP " 4." 4 Click \[dq]Credentials\[dq] in the left-side panel (not \[dq]Create credentials\[dq], which opens the wizard), then \[dq]Create credentials\[dq] -.IP "5." 3 +.IP " 5." 4 If you already configured an \[dq]Oauth Consent Screen\[dq], then skip to the next step; if not, click on \[dq]CONFIGURE CONSENT SCREEN\[dq] button (near the top right corner of the right panel), then select @@ -35771,10 +39289,12 @@ enter an \[dq]Application name\[dq] (\[dq]rclone\[dq] is OK); enter \[dq]Save\[dq] (all other data is optional). Click again on \[dq]Credentials\[dq] on the left panel to go back to the \[dq]Credentials\[dq] screen. +.RS 4 .PP (PS: if you are a GSuite user, you could also select \[dq]Internal\[dq] -instead of \[dq]External\[dq] above, but this has not been -tested/documented so far). +instead of \[dq]External\[dq] above, but this will restrict API use to +Google Workspace users in your organisation). +.RE .IP " 6." 4 Click on the \[dq]+ CREATE CREDENTIALS\[dq] button at the top of the screen, then select \[dq]OAuth client ID\[dq]. @@ -35785,13 +39305,20 @@ Choose an application type of \[dq]Desktop app\[dq] and click .IP " 8." 4 It will show you a client ID and client secret. Make a note of these. +.RS 4 +.PP +(If you selected \[dq]External\[dq] at Step 5 continue to \[dq]Publish +App\[dq] in the Steps 9 and 10. +If you chose \[dq]Internal\[dq] you don\[aq]t need to publish and can +skip straight to Step 11.) +.RE .IP " 9." 4 Go to \[dq]Oauth consent screen\[dq] and press \[dq]Publish App\[dq] .IP "10." 4 -Provide the noted client ID and client secret to rclone. -.IP "11." 4 Click \[dq]OAuth consent screen\[dq], then click \[dq]PUBLISH APP\[dq] button and confirm, or add your account under \[dq]Test users\[dq]. +.IP "11." 4 +Provide the noted client ID and client secret to rclone. .PP Be aware that, due to the \[dq]enhanced security\[dq] recently introduced by Google, you are theoretically expected to \[dq]submit your @@ -36080,7 +39607,7 @@ you. This is similar to the Sharing tab in the Google Photos web interface. .SS Standard options .PP -Here are the standard options specific to google photos (Google Photos). +Here are the Standard options specific to google photos (Google Photos). .SS --gphotos-client-id .PP OAuth Client Id. @@ -36129,7 +39656,7 @@ Type: bool Default: false .SS Advanced options .PP -Here are the advanced options specific to google photos (Google Photos). +Here are the Advanced options specific to google photos (Google Photos). .SS --gphotos-token .PP OAuth Access Token as a JSON blob. @@ -36544,7 +40071,7 @@ the files. .SS Configuration reference .SS Standard options .PP -Here are the standard options specific to hasher (Better checksums for +Here are the Standard options specific to hasher (Better checksums for other remotes). .SS --hasher-remote .PP @@ -36589,7 +40116,7 @@ Type: Duration Default: off .SS Advanced options .PP -Here are the advanced options specific to hasher (Better checksums for +Here are the Advanced options specific to hasher (Better checksums for other remotes). .SS --hasher-auto-size .PP @@ -36605,6 +40132,11 @@ Env Var: RCLONE_HASHER_AUTO_SIZE Type: SizeSuffix .IP \[bu] 2 Default: 0 +.SS Metadata +.PP +Any metadata supported by the underlying remote is read and written. +.PP +See the metadata (https://rclone.org/docs/#metadata) docs for more info. .SS Backend commands .PP Here are the commands specific to the hasher backend. @@ -36619,9 +40151,8 @@ rclone backend COMMAND remote: .PP The help below will explain what arguments each command takes. .PP -See the \[dq]rclone backend\[dq] -command (https://rclone.org/commands/rclone_backend/) for more info on -how to pass options and arguments. +See the backend (https://rclone.org/commands/rclone_backend/) command +for more info on how to pass options and arguments. .PP These can be run on a running backend using the rc command backend/command (https://rclone.org/rc/#backend-command). @@ -36932,7 +40463,7 @@ Invalid UTF-8 bytes will also be replaced (https://rclone.org/overview/#invalid-utf8). .SS Standard options .PP -Here are the standard options specific to hdfs (Hadoop distributed file +Here are the Standard options specific to hdfs (Hadoop distributed file system). .SS --hdfs-namenode .PP @@ -36975,7 +40506,7 @@ Connect to hdfs as root. .RE .SS Advanced options .PP -Here are the advanced options specific to hdfs (Hadoop distributed file +Here are the Advanced options specific to hdfs (Hadoop distributed file system). .SS --hdfs-service-principal-name .PP @@ -37047,6 +40578,569 @@ Default: Slash,Colon,Del,Ctl,InvalidUtf8,Dot No server-side \f[C]Move\f[R] or \f[C]DirMove\f[R]. .IP \[bu] 2 Checksums not implemented. +.SH HiDrive +.PP +Paths are specified as \f[C]remote:path\f[R] +.PP +Paths may be as deep as required, e.g. +\f[C]remote:directory/subdirectory\f[R]. +.PP +The initial setup for hidrive involves getting a token from HiDrive +which you need to do in your browser. +\f[C]rclone config\f[R] walks you through it. +.SS Configuration +.PP +Here is an example of how to make a remote called \f[C]remote\f[R]. +First run: +.IP +.nf +\f[C] + rclone config +\f[R] +.fi +.PP +This will guide you through an interactive setup process: +.IP +.nf +\f[C] +No remotes found - make a new one +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Type of storage to configure. +Choose a number from below, or type in your own value +[snip] +XX / HiDrive + \[rs] \[dq]hidrive\[dq] +[snip] +Storage> hidrive +OAuth Client Id - Leave blank normally. +client_id> +OAuth Client Secret - Leave blank normally. +client_secret> +Access permissions that rclone should use when requesting access from HiDrive. +Leave blank normally. +scope_access> +Edit advanced config? +y/n> n +Use auto config? +y/n> y +If your browser doesn\[aq]t open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx +Log in and authorize rclone for access +Waiting for code... +Got code +-------------------- +[remote] +type = hidrive +token = {\[dq]access_token\[dq]:\[dq]xxxxxxxxxxxxxxxxxxxx\[dq],\[dq]token_type\[dq]:\[dq]Bearer\[dq],\[dq]refresh_token\[dq]:\[dq]xxxxxxxxxxxxxxxxxxxxxxx\[dq],\[dq]expiry\[dq]:\[dq]xxxxxxxxxxxxxxxxxxxxxxx\[dq]} +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +\f[R] +.fi +.PP +\f[B]You should be aware that OAuth-tokens can be used to access your +account and hence should not be shared with other persons.\f[R] See the +below section for more information. +.PP +See the remote setup docs (https://rclone.org/remote_setup/) for how to +set it up on a machine with no Internet browser available. +.PP +Note that rclone runs a webserver on your local machine to collect the +token as returned from HiDrive. +This only runs from the moment it opens your browser to the moment you +get back the verification code. +The webserver runs on \f[C]http://127.0.0.1:53682/\f[R]. +If local port \f[C]53682\f[R] is protected by a firewall you may need to +temporarily unblock the firewall to complete authorization. +.PP +Once configured you can then use \f[C]rclone\f[R] like this, +.PP +List directories in top level of your HiDrive root folder +.IP +.nf +\f[C] +rclone lsd remote: +\f[R] +.fi +.PP +List all the files in your HiDrive filesystem +.IP +.nf +\f[C] +rclone ls remote: +\f[R] +.fi +.PP +To copy a local directory to a HiDrive directory called backup +.IP +.nf +\f[C] +rclone copy /home/source remote:backup +\f[R] +.fi +.SS Keeping your tokens safe +.PP +Any OAuth-tokens will be stored by rclone in the remote\[aq]s +configuration file as unencrypted text. +Anyone can use a valid refresh-token to access your HiDrive filesystem +without knowing your password. +Therefore you should make sure no one else can access your +configuration. +.PP +It is possible to encrypt rclone\[aq]s configuration file. +You can find information on securing your configuration file by viewing +the configuration encryption +docs (https://rclone.org/docs/#configuration-encryption). +.SS Invalid refresh token +.PP +As can be verified here (https://developer.hidrive.com/basics-flows/), +each \f[C]refresh_token\f[R] (for Native Applications) is valid for 60 +days. +If used to access HiDrivei, its validity will be automatically extended. +.PP +This means that if you +.IP \[bu] 2 +Don\[aq]t use the HiDrive remote for 60 days +.PP +then rclone will return an error which includes a text that implies the +refresh token is \f[I]invalid\f[R] or \f[I]expired\f[R]. +.PP +To fix this you will need to authorize rclone to access your HiDrive +account again. +.PP +Using +.IP +.nf +\f[C] +rclone config reconnect remote: +\f[R] +.fi +.PP +the process is very similar to the process of initial setup exemplified +before. +.SS Modified time and hashes +.PP +HiDrive allows modification times to be set on objects accurate to 1 +second. +.PP +HiDrive supports its own hash type (https://static.hidrive.com/dev/0001) +which is used to verify the integrety of file contents after successful +transfers. +.SS Restricted filename characters +.PP +HiDrive cannot store files or folders that include \f[C]/\f[R] (0x2F) or +null-bytes (0x00) in their name. +Any other characters can be used in the names of files or folders. +Additionally, files or folders cannot be named either of the following: +\f[C].\f[R] or \f[C]..\f[R] +.PP +Therefore rclone will automatically replace these characters, if files +or folders are stored or accessed with such names. +.PP +You can read about how this filename encoding works in general here. +.PP +Keep in mind that HiDrive only supports file or folder names with a +length of 255 characters or less. +.SS Transfers +.PP +HiDrive limits file sizes per single request to a maximum of 2 GiB. +To allow storage of larger files and allow for better upload +performance, the hidrive backend will use a chunked transfer for files +larger than 96 MiB. +Rclone will upload multiple parts/chunks of the file at the same time. +Chunks in the process of being uploaded are buffered in memory, so you +may want to restrict this behaviour on systems with limited resources. +.PP +You can customize this behaviour using the following options: +.IP \[bu] 2 +\f[C]chunk_size\f[R]: size of file parts +.IP \[bu] 2 +\f[C]upload_cutoff\f[R]: files larger or equal to this in size will use +a chunked transfer +.IP \[bu] 2 +\f[C]upload_concurrency\f[R]: number of file-parts to upload at the same +time +.PP +See the below section about configuration options for more details. +.SS Root folder +.PP +You can set the root folder for rclone. +This is the directory that rclone considers to be the root of your +HiDrive. +.PP +Usually, you will leave this blank, and rclone will use the root of the +account. +.PP +However, you can set this to restrict rclone to a specific folder +hierarchy. +.PP +This works by prepending the contents of the \f[C]root_prefix\f[R] +option to any paths accessed by rclone. +For example, the following two ways to access the home directory are +equivalent: +.IP +.nf +\f[C] +rclone lsd --hidrive-root-prefix=\[dq]/users/test/\[dq] remote:path + +rclone lsd remote:/users/test/path +\f[R] +.fi +.PP +See the below section about configuration options for more details. +.SS Directory member count +.PP +By default, rclone will know the number of directory members contained +in a directory. +For example, \f[C]rclone lsd\f[R] uses this information. +.PP +The acquisition of this information will result in additional time costs +for HiDrive\[aq]s API. +When dealing with large directory structures, it may be desirable to +circumvent this time cost, especially when this information is not +explicitly needed. +For this, the \f[C]disable_fetching_member_count\f[R] option can be +used. +.PP +See the below section about configuration options for more details. +.SS Standard options +.PP +Here are the Standard options specific to hidrive (HiDrive). +.SS --hidrive-client-id +.PP +OAuth Client Id. +.PP +Leave blank normally. +.PP +Properties: +.IP \[bu] 2 +Config: client_id +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_CLIENT_ID +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --hidrive-client-secret +.PP +OAuth Client Secret. +.PP +Leave blank normally. +.PP +Properties: +.IP \[bu] 2 +Config: client_secret +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_CLIENT_SECRET +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --hidrive-scope-access +.PP +Access permissions that rclone should use when requesting access from +HiDrive. +.PP +Properties: +.IP \[bu] 2 +Config: scope_access +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_SCOPE_ACCESS +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Default: \[dq]rw\[dq] +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]rw\[dq] +.RS 2 +.IP \[bu] 2 +Read and write access to resources. +.RE +.IP \[bu] 2 +\[dq]ro\[dq] +.RS 2 +.IP \[bu] 2 +Read-only access to resources. +.RE +.RE +.SS Advanced options +.PP +Here are the Advanced options specific to hidrive (HiDrive). +.SS --hidrive-token +.PP +OAuth Access Token as a JSON blob. +.PP +Properties: +.IP \[bu] 2 +Config: token +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_TOKEN +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --hidrive-auth-url +.PP +Auth server URL. +.PP +Leave blank to use the provider defaults. +.PP +Properties: +.IP \[bu] 2 +Config: auth_url +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_AUTH_URL +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --hidrive-token-url +.PP +Token server url. +.PP +Leave blank to use the provider defaults. +.PP +Properties: +.IP \[bu] 2 +Config: token_url +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_TOKEN_URL +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --hidrive-scope-role +.PP +User-level that rclone should use when requesting access from HiDrive. +.PP +Properties: +.IP \[bu] 2 +Config: scope_role +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_SCOPE_ROLE +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Default: \[dq]user\[dq] +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]user\[dq] +.RS 2 +.IP \[bu] 2 +User-level access to management permissions. +.IP \[bu] 2 +This will be sufficient in most cases. +.RE +.IP \[bu] 2 +\[dq]admin\[dq] +.RS 2 +.IP \[bu] 2 +Extensive access to management permissions. +.RE +.IP \[bu] 2 +\[dq]owner\[dq] +.RS 2 +.IP \[bu] 2 +Full access to management permissions. +.RE +.RE +.SS --hidrive-root-prefix +.PP +The root/parent folder for all paths. +.PP +Fill in to use the specified folder as the parent for all paths given to +the remote. +This way rclone can use any folder as its starting point. +.PP +Properties: +.IP \[bu] 2 +Config: root_prefix +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_ROOT_PREFIX +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Default: \[dq]/\[dq] +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]/\[dq] +.RS 2 +.IP \[bu] 2 +The topmost directory accessible by rclone. +.IP \[bu] 2 +This will be equivalent with \[dq]root\[dq] if rclone uses a regular +HiDrive user account. +.RE +.IP \[bu] 2 +\[dq]root\[dq] +.RS 2 +.IP \[bu] 2 +The topmost directory of the HiDrive user account +.RE +.IP \[bu] 2 +\[dq]\[dq] +.RS 2 +.IP \[bu] 2 +This specifies that there is no root-prefix for your paths. +.IP \[bu] 2 +When using this you will always need to specify paths to this remote +with a valid parent e.g. +\[dq]remote:/path/to/dir\[dq] or \[dq]remote:root/path/to/dir\[dq]. +.RE +.RE +.SS --hidrive-endpoint +.PP +Endpoint for the service. +.PP +This is the URL that API-calls will be made to. +.PP +Properties: +.IP \[bu] 2 +Config: endpoint +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_ENDPOINT +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Default: \[dq]https://api.hidrive.strato.com/2.1\[dq] +.SS --hidrive-disable-fetching-member-count +.PP +Do not fetch number of objects in directories unless it is absolutely +necessary. +.PP +Requests may be faster if the number of objects in subdirectories is not +fetched. +.PP +Properties: +.IP \[bu] 2 +Config: disable_fetching_member_count +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_DISABLE_FETCHING_MEMBER_COUNT +.IP \[bu] 2 +Type: bool +.IP \[bu] 2 +Default: false +.SS --hidrive-chunk-size +.PP +Chunksize for chunked uploads. +.PP +Any files larger than the configured cutoff (or files of unknown size) +will be uploaded in chunks of this size. +.PP +The upper limit for this is 2147483647 bytes (about 2.000Gi). +That is the maximum amount of bytes a single upload-operation will +support. +Setting this above the upper limit or to a negative value will cause +uploads to fail. +.PP +Setting this to larger values may increase the upload speed at the cost +of using more memory. +It can be set to smaller values smaller to save on memory. +.PP +Properties: +.IP \[bu] 2 +Config: chunk_size +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_CHUNK_SIZE +.IP \[bu] 2 +Type: SizeSuffix +.IP \[bu] 2 +Default: 48Mi +.SS --hidrive-upload-cutoff +.PP +Cutoff/Threshold for chunked uploads. +.PP +Any files larger than this will be uploaded in chunks of the configured +chunksize. +.PP +The upper limit for this is 2147483647 bytes (about 2.000Gi). +That is the maximum amount of bytes a single upload-operation will +support. +Setting this above the upper limit will cause uploads to fail. +.PP +Properties: +.IP \[bu] 2 +Config: upload_cutoff +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_UPLOAD_CUTOFF +.IP \[bu] 2 +Type: SizeSuffix +.IP \[bu] 2 +Default: 96Mi +.SS --hidrive-upload-concurrency +.PP +Concurrency for chunked uploads. +.PP +This is the upper limit for how many transfers for the same file are +running concurrently. +Setting this above to a value smaller than 1 will cause uploads to +deadlock. +.PP +If you are uploading small numbers of large files over high-speed links +and these uploads do not fully utilize your bandwidth, then increasing +this may help to speed up the transfers. +.PP +Properties: +.IP \[bu] 2 +Config: upload_concurrency +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_UPLOAD_CONCURRENCY +.IP \[bu] 2 +Type: int +.IP \[bu] 2 +Default: 4 +.SS --hidrive-encoding +.PP +The encoding for the backend. +.PP +See the encoding section in the +overview (https://rclone.org/overview/#encoding) for more info. +.PP +Properties: +.IP \[bu] 2 +Config: encoding +.IP \[bu] 2 +Env Var: RCLONE_HIDRIVE_ENCODING +.IP \[bu] 2 +Type: MultiEncoder +.IP \[bu] 2 +Default: Slash,Dot +.SS Limitations +.SS Symbolic links +.PP +HiDrive is able to store symbolic links (\f[I]symlinks\f[R]) by design, +for example, when unpacked from a zip archive. +.PP +There exists no direct mechanism to manage native symlinks in remotes. +As such this implementation has chosen to ignore any native symlinks +present in the remote. +rclone will not be able to access or show any symlinks stored in the +hidrive-remote. +This means symlinks cannot be individually removed, copied, or moved, +except when removing, copying, or moving the parent folder. +.PP +\f[I]This does not affect the \f[CI].rclonelink\f[I]-files that rclone +uses to encode and store symbolic links.\f[R] +.SS Sparse files +.PP +It is possible to store sparse files in HiDrive. +.PP +Note that copying a sparse file will expand the holes into null-byte +(0x00) regions that will then consume disk space. +Likewise, when downloading a sparse file, the resulting file will have +null-byte regions in the place of file holes. .SH HTTP .PP The HTTP remote is a read only remote for reading files of a webserver. @@ -37106,7 +41200,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / http Connection +XX / HTTP \[rs] \[dq]http\[dq] [snip] Storage> http @@ -37196,10 +41290,10 @@ rclone lsd :http,url=\[aq]https://beta.rclone.org\[aq]: .fi .SS Standard options .PP -Here are the standard options specific to http (http Connection). +Here are the Standard options specific to http (HTTP). .SS --http-url .PP -URL of http host to connect to. +URL of HTTP host to connect to. .PP E.g. \[dq]https://example.com\[dq], or @@ -37217,7 +41311,7 @@ Type: string Required: true .SS Advanced options .PP -Here are the advanced options specific to http (http Connection). +Here are the Advanced options specific to http (HTTP). .SS --http-headers .PP Set HTTP headers for all transactions. @@ -37304,7 +41398,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SH Hubic .PP @@ -37435,7 +41529,7 @@ Note that Hubic wraps the Swift backend, so most of the properties of are the same. .SS Standard options .PP -Here are the standard options specific to hubic (Hubic). +Here are the Standard options specific to hubic (Hubic). .SS --hubic-client-id .PP OAuth Client Id. @@ -37468,7 +41562,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to hubic (Hubic). +Here are the Advanced options specific to hubic (Hubic). .SS --hubic-token .PP OAuth Access Token as a JSON blob. @@ -37575,6 +41669,473 @@ credentials and ignores the expires field returned by the Hubic API. The Swift API doesn\[aq]t return a correct MD5SUM for segmented files (Dynamic or Static Large Objects) so rclone won\[aq]t check or use the MD5SUM for these. +.SH Internet Archive +.PP +The Internet Archive backend utilizes Items on +archive.org (https://archive.org/) +.PP +Refer to IAS3 API +documentation (https://archive.org/services/docs/api/ias3.html) for the +API this backend uses. +.PP +Paths are specified as \f[C]remote:bucket\f[R] (or \f[C]remote:\f[R] for +the \f[C]lsd\f[R] command.) You may put subdirectories in too, e.g. +\f[C]remote:item/path/to/dir\f[R]. +.PP +Once you have made a remote (see the provider specific section above) +you can use it like this: +.PP +Unlike S3, listing up all items uploaded by you isn\[aq]t supported. +.PP +Make a new item +.IP +.nf +\f[C] +rclone mkdir remote:item +\f[R] +.fi +.PP +List the contents of a item +.IP +.nf +\f[C] +rclone ls remote:item +\f[R] +.fi +.PP +Sync \f[C]/home/local/directory\f[R] to the remote item, deleting any +excess files in the item. +.IP +.nf +\f[C] +rclone sync -i /home/local/directory remote:item +\f[R] +.fi +.SS Notes +.PP +Because of Internet Archive\[aq]s architecture, it enqueues write +operations (and extra post-processings) in a per-item queue. +You can check item\[aq]s queue at +https://catalogd.archive.org/history/item-name-here . +Because of that, all uploads/deletes will not show up immediately and +takes some time to be available. +The per-item queue is enqueued to an another queue, Item Deriver Queue. +You can check the status of Item Deriver Queue +here. (https://catalogd.archive.org/catalog.php?whereami=1) This queue +has a limit, and it may block you from uploading, or even deleting. +You should avoid uploading a lot of small files for better behavior. +.PP +You can optionally wait for the server\[aq]s processing to finish, by +setting non-zero value to \f[C]wait_archive\f[R] key. +By making it wait, rclone can do normal file comparison. +Make sure to set a large enough value (e.g. +\f[C]30m0s\f[R] for smaller files) as it can take a long time depending +on server\[aq]s queue. +.SS About metadata +.PP +This backend supports setting, updating and reading metadata of each +file. +The metadata will appear as file metadata on Internet Archive. +However, some fields are reserved by both Internet Archive and rclone. +.PP +The following are reserved by Internet Archive: - \f[C]name\f[R] - +\f[C]source\f[R] - \f[C]size\f[R] - \f[C]md5\f[R] - \f[C]crc32\f[R] - +\f[C]sha1\f[R] - \f[C]format\f[R] - \f[C]old_version\f[R] - +\f[C]viruscheck\f[R] +.PP +Trying to set values to these keys is ignored with a warning. +Only setting \f[C]mtime\f[R] is an exception. +Doing so make it the identical behavior as setting ModTime. +.PP +rclone reserves all the keys starting with \f[C]rclone-\f[R]. +Setting value for these keys will give you warnings, but values are set +according to request. +.PP +If there are multiple values for a key, only the first one is returned. +This is a limitation of rclone, that supports one value per one key. +It can be triggered when you did a server-side copy. +.PP +Reading metadata will also provide custom (non-standard nor reserved) +ones. +.SS Configuration +.PP +Here is an example of making an internetarchive configuration. +Most applies to the other providers as well, any differences are +described below. +.PP +First run +.IP +.nf +\f[C] +rclone config +\f[R] +.fi +.PP +This will guide you through an interactive setup process. +.IP +.nf +\f[C] +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +XX / InternetArchive Items + \[rs] (internetarchive) +Storage> internetarchive +Option access_key_id. +IAS3 Access Key. +Leave blank for anonymous access. +You can find one here: https://archive.org/account/s3.php +Enter a value. Press Enter to leave empty. +access_key_id> XXXX +Option secret_access_key. +IAS3 Secret Key (password). +Leave blank for anonymous access. +Enter a value. Press Enter to leave empty. +secret_access_key> XXXX +Edit advanced config? +y) Yes +n) No (default) +y/n> y +Option endpoint. +IAS3 Endpoint. +Leave blank for default value. +Enter a string value. Press Enter for the default (https://s3.us.archive.org). +endpoint> +Option front_endpoint. +Host of InternetArchive Frontend. +Leave blank for default value. +Enter a string value. Press Enter for the default (https://archive.org). +front_endpoint> +Option disable_checksum. +Don\[aq]t store MD5 checksum with object metadata. +Normally rclone will calculate the MD5 checksum of the input before +uploading it so it can ask the server to check the object against checksum. +This is great for data integrity checking but can cause long delays for +large files to start uploading. +Enter a boolean value (true or false). Press Enter for the default (true). +disable_checksum> true +Option encoding. +The encoding for the backend. +See the [encoding section in the overview](https://rclone.org/overview/#encoding) for more info. +Enter a encoder.MultiEncoder value. Press Enter for the default (Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot). +encoding> +Edit advanced config? +y) Yes +n) No (default) +y/n> n +-------------------- +[remote] +type = internetarchive +access_key_id = XXXX +secret_access_key = XXXX +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +\f[R] +.fi +.SS Standard options +.PP +Here are the Standard options specific to internetarchive (Internet +Archive). +.SS --internetarchive-access-key-id +.PP +IAS3 Access Key. +.PP +Leave blank for anonymous access. +You can find one here: https://archive.org/account/s3.php +.PP +Properties: +.IP \[bu] 2 +Config: access_key_id +.IP \[bu] 2 +Env Var: RCLONE_INTERNETARCHIVE_ACCESS_KEY_ID +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --internetarchive-secret-access-key +.PP +IAS3 Secret Key (password). +.PP +Leave blank for anonymous access. +.PP +Properties: +.IP \[bu] 2 +Config: secret_access_key +.IP \[bu] 2 +Env Var: RCLONE_INTERNETARCHIVE_SECRET_ACCESS_KEY +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS Advanced options +.PP +Here are the Advanced options specific to internetarchive (Internet +Archive). +.SS --internetarchive-endpoint +.PP +IAS3 Endpoint. +.PP +Leave blank for default value. +.PP +Properties: +.IP \[bu] 2 +Config: endpoint +.IP \[bu] 2 +Env Var: RCLONE_INTERNETARCHIVE_ENDPOINT +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Default: \[dq]https://s3.us.archive.org\[dq] +.SS --internetarchive-front-endpoint +.PP +Host of InternetArchive Frontend. +.PP +Leave blank for default value. +.PP +Properties: +.IP \[bu] 2 +Config: front_endpoint +.IP \[bu] 2 +Env Var: RCLONE_INTERNETARCHIVE_FRONT_ENDPOINT +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Default: \[dq]https://archive.org\[dq] +.SS --internetarchive-disable-checksum +.PP +Don\[aq]t ask the server to test against MD5 checksum calculated by +rclone. +Normally rclone will calculate the MD5 checksum of the input before +uploading it so it can ask the server to check the object against +checksum. +This is great for data integrity checking but can cause long delays for +large files to start uploading. +.PP +Properties: +.IP \[bu] 2 +Config: disable_checksum +.IP \[bu] 2 +Env Var: RCLONE_INTERNETARCHIVE_DISABLE_CHECKSUM +.IP \[bu] 2 +Type: bool +.IP \[bu] 2 +Default: true +.SS --internetarchive-wait-archive +.PP +Timeout for waiting the server\[aq]s processing tasks (specifically +archive and book_op) to finish. +Only enable if you need to be guaranteed to be reflected after write +operations. +0 to disable waiting. +No errors to be thrown in case of timeout. +.PP +Properties: +.IP \[bu] 2 +Config: wait_archive +.IP \[bu] 2 +Env Var: RCLONE_INTERNETARCHIVE_WAIT_ARCHIVE +.IP \[bu] 2 +Type: Duration +.IP \[bu] 2 +Default: 0s +.SS --internetarchive-encoding +.PP +The encoding for the backend. +.PP +See the encoding section in the +overview (https://rclone.org/overview/#encoding) for more info. +.PP +Properties: +.IP \[bu] 2 +Config: encoding +.IP \[bu] 2 +Env Var: RCLONE_INTERNETARCHIVE_ENCODING +.IP \[bu] 2 +Type: MultiEncoder +.IP \[bu] 2 +Default: Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot +.SS Metadata +.PP +Metadata fields provided by Internet Archive. +If there are multiple values for a key, only the first one is returned. +This is a limitation of Rclone, that supports one value per one key. +.PP +Owner is able to add custom keys. +Metadata feature grabs all the keys including them. +.PP +Here are the possible system metadata items for the internetarchive +backend. +.PP +.TS +tab(@); +lw(11.1n) lw(11.1n) lw(11.1n) lw(16.6n) lw(20.3n). +T{ +Name +T}@T{ +Help +T}@T{ +Type +T}@T{ +Example +T}@T{ +Read Only +T} +_ +T{ +crc32 +T}@T{ +CRC32 calculated by Internet Archive +T}@T{ +string +T}@T{ +01234567 +T}@T{ +N +T} +T{ +format +T}@T{ +Name of format identified by Internet Archive +T}@T{ +string +T}@T{ +Comma-Separated Values +T}@T{ +N +T} +T{ +md5 +T}@T{ +MD5 hash calculated by Internet Archive +T}@T{ +string +T}@T{ +01234567012345670123456701234567 +T}@T{ +N +T} +T{ +mtime +T}@T{ +Time of last modification, managed by Rclone +T}@T{ +RFC 3339 +T}@T{ +2006-01-02T15:04:05.999999999Z +T}@T{ +N +T} +T{ +name +T}@T{ +Full file path, without the bucket part +T}@T{ +filename +T}@T{ +backend/internetarchive/internetarchive.go +T}@T{ +N +T} +T{ +old_version +T}@T{ +Whether the file was replaced and moved by keep-old-version flag +T}@T{ +boolean +T}@T{ +true +T}@T{ +N +T} +T{ +rclone-ia-mtime +T}@T{ +Time of last modification, managed by Internet Archive +T}@T{ +RFC 3339 +T}@T{ +2006-01-02T15:04:05.999999999Z +T}@T{ +N +T} +T{ +rclone-mtime +T}@T{ +Time of last modification, managed by Rclone +T}@T{ +RFC 3339 +T}@T{ +2006-01-02T15:04:05.999999999Z +T}@T{ +N +T} +T{ +rclone-update-track +T}@T{ +Random value used by Rclone for tracking changes inside Internet Archive +T}@T{ +string +T}@T{ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +T}@T{ +N +T} +T{ +sha1 +T}@T{ +SHA1 hash calculated by Internet Archive +T}@T{ +string +T}@T{ +0123456701234567012345670123456701234567 +T}@T{ +N +T} +T{ +size +T}@T{ +File size in bytes +T}@T{ +decimal number +T}@T{ +123456 +T}@T{ +N +T} +T{ +source +T}@T{ +The source of the file +T}@T{ +string +T}@T{ +original +T}@T{ +N +T} +T{ +viruscheck +T}@T{ +The last time viruscheck process was run for the file (?) +T}@T{ +unixtime +T}@T{ +1654191352 +T}@T{ +N +T} +.TE +.PP +See the metadata (https://rclone.org/docs/#metadata) docs for more info. .SH Jottacloud .PP Jottacloud is a cloud storage service provider from a Norwegian company, @@ -37655,56 +42216,78 @@ s) Set configuration password q) Quit config n/s/q> n name> remote +Option Storage. Type of storage to configure. -Enter a string value. Press Enter for the default (\[dq]\[dq]). -Choose a number from below, or type in your own value +Choose a number from below, or type in your own value. [snip] XX / Jottacloud - \[rs] \[dq]jottacloud\[dq] + \[rs] (jottacloud) [snip] Storage> jottacloud -** See help for jottacloud backend at: https://rclone.org/jottacloud/ ** - -Edit advanced config? (y/n) -y) Yes -n) No -y/n> n -Remote config -Use legacy authentication?. -This is only required for certain whitelabel versions of Jottacloud and not recommended for normal users. +Edit advanced config? y) Yes n) No (default) y/n> n - -Generate a personal login token here: https://www.jottacloud.com/web/secure +Option config_type. +Select authentication type. +Choose a number from below, or type in an existing string value. +Press Enter for the default (standard). + / Standard authentication. + 1 | Use this if you\[aq]re a normal Jottacloud user. + \[rs] (standard) + / Legacy authentication. + 2 | This is only required for certain whitelabel versions of Jottacloud and not recommended for normal users. + \[rs] (legacy) + / Telia Cloud authentication. + 3 | Use this if you are using Telia Cloud. + \[rs] (telia) + / Tele2 Cloud authentication. + 4 | Use this if you are using Tele2 Cloud. + \[rs] (tele2) +config_type> 1 +Personal login token. +Generate here: https://www.jottacloud.com/web/secure Login Token> - -Do you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client? - +Use a non-standard device/mountpoint? +Choosing no, the default, will let you access the storage used for the archive +section of the official Jottacloud client. If you instead want to access the +sync or the backup section, for example, you must choose yes. y) Yes -n) No +n) No (default) y/n> y -Please select the device to use. Normally this will be Jotta -Choose a number from below, or type in an existing value +Option config_device. +The device to use. In standard setup the built-in Jotta device is used, +which contains predefined mountpoints for archive, sync etc. All other devices +are treated as backup devices by the official Jottacloud client. You may create +a new by entering a unique name. +Choose a number from below, or type in your own string value. +Press Enter for the default (DESKTOP-3H31129). 1 > DESKTOP-3H31129 2 > Jotta -Devices> 2 -Please select the mountpoint to user. Normally this will be Archive -Choose a number from below, or type in an existing value +config_device> 2 +Option config_mountpoint. +The mountpoint to use for the built-in device Jotta. +The standard setup is to use the Archive mountpoint. Most other mountpoints +have very limited support in rclone and should generally be avoided. +Choose a number from below, or type in an existing string value. +Press Enter for the default (Archive). 1 > Archive - 2 > Links + 2 > Shared 3 > Sync - -Mountpoints> 1 +config_mountpoint> 1 -------------------- -[jotta] +[remote] type = jottacloud +configVersion = 1 +client_id = jottacli +client_secret = +tokenURL = https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token token = {........} +username = 2940e57271a93d987d6f8a21 device = Jotta mountpoint = Archive -configVersion = 1 -------------------- -y) Yes this is OK +y) Yes this is OK (default) e) Edit this remote d) Delete this remote y/e/d> y @@ -37739,22 +42322,32 @@ rclone copy /home/source remote:backup .SS Devices and Mountpoints .PP The official Jottacloud client registers a device for each computer you -install it on, and then creates a mountpoint for each folder you select -for Backup. -The web interface uses a special device called Jotta for the Archive and -Sync mountpoints. -.PP -With rclone you\[aq]ll want to use the Jotta/Archive device/mountpoint -in most cases, however if you want to access files uploaded by any of -the official clients rclone provides the option to select other devices -and mountpoints during config. -Note that uploading files is currently not supported to other devices -than Jotta. -.PP -The built-in Jotta device may also contain several other mountpoints, -such as: Latest, Links, Shared and Trash. -These are special mountpoints with a different internal representation -than the \[dq]regular\[dq] mountpoints. +install it on, and shows them in the backup section of the user +interface. +For each folder you select for backup it will create a mountpoint within +this device. +A built-in device called Jotta is special, and contains mountpoints +Archive, Sync and some others, used for corresponding features in +official clients. +.PP +With rclone you\[aq]ll want to use the standard Jotta/Archive +device/mountpoint in most cases. +However, you may for example want to access files from the sync or +backup functionality provided by the official clients, and rclone +therefore provides the option to select other devices and mountpoints +during config. +.PP +You are allowed to create new devices and mountpoints. +All devices except the built-in Jotta device are treated as backup +devices by official Jottacloud clients, and the mountpoints on them are +individual backup sets. +.PP +With the built-in Jotta device, only existing, built-in, mountpoints can +be selected. +In addition to the mentioned Archive and Sync, it may contain several +other mountpoints such as: Latest, Links, Shared and Trash. +All of these are special mountpoints with a different internal +representation than the \[dq]regular\[dq] mountpoints. Rclone will only to a very limited degree support them. Generally you should avoid these, unless you know what you are doing. .SS --fast-list @@ -37893,7 +42486,7 @@ To view your current quota you can use the limit (unless it is unlimited) and the current usage. .SS Advanced options .PP -Here are the advanced options specific to jottacloud (Jottacloud). +Here are the Advanced options specific to jottacloud (Jottacloud). .SS --jottacloud-md5-memory-limit .PP Files bigger than this will be cached on disk to calculate the MD5 if @@ -38145,7 +42738,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in XML strings. .SS Standard options .PP -Here are the standard options specific to koofr (Koofr, Digi Storage and +Here are the Standard options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). .SS --koofr-provider .PP @@ -38269,7 +42862,7 @@ Type: string Required: true .SS Advanced options .PP -Here are the advanced options specific to koofr (Koofr, Digi Storage and +Here are the Advanced options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers). .SS --koofr-mountid .PP @@ -38728,7 +43321,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in JSON strings. .SS Standard options .PP -Here are the standard options specific to mailru (Mail.ru Cloud). +Here are the Standard options specific to mailru (Mail.ru Cloud). .SS --mailru-user .PP User name (usually email). @@ -38801,7 +43394,7 @@ Disable .RE .SS Advanced options .PP -Here are the advanced options specific to mailru (Mail.ru Cloud). +Here are the Advanced options specific to mailru (Mail.ru Cloud). .SS --mailru-speedup-file-patterns .PP Comma separated list of file name patterns eligible for speedup (put by @@ -39154,6 +43747,48 @@ messages in the log about duplicates. .PP Use \f[C]rclone dedupe\f[R] to fix duplicated files. .SS Failure to log-in +.SS Object not found +.PP +If you are connecting to your Mega remote for the first time, to test +access and syncronisation, you may receive an error such as +.IP +.nf +\f[C] +Failed to create file system for \[dq]my-mega-remote:\[dq]: +couldn\[aq]t login: Object (typically, node or user) not found +\f[R] +.fi +.PP +The diagnostic steps often recommended in the rclone +forum (https://forum.rclone.org/search?q=mega) start with the +\f[B]MEGAcmd\f[R] utility. +Note that this refers to the official C++ command from +https://github.com/meganz/MEGAcmd and not the go language built command +from t3rm1n4l/megacmd that is no longer maintained. +.PP +Follow the instructions for installing MEGAcmd and try accessing your +remote as they recommend. +You can establish whether or not you can log in using MEGAcmd, and +obtain diagnostic information to help you, and search or work with +others in the forum. +.IP +.nf +\f[C] +MEGA CMD> login me\[at]example.com +Password: +Fetching nodes ... +Loading transfers from local cache +Login complete as me\[at]example.com +me\[at]example.com:/$ +\f[R] +.fi +.PP +Note that some have found issues with passwords containing special +characters. +If you can not log on with rclone, but MEGAcmd logs on just fine, then +consider changing your password temporarily to pure alphanumeric +characters, in case that helps. +.SS Repeated commands blocks access .PP Mega remotes seem to get blocked (reject logins) under \[dq]heavy use\[dq]. @@ -39205,7 +43840,7 @@ and you are sure the user and the password are correct, likely you have got the remote blocked for a while. .SS Standard options .PP -Here are the standard options specific to mega (Mega). +Here are the Standard options specific to mega (Mega). .SS --mega-user .PP User name. @@ -39237,7 +43872,7 @@ Type: string Required: true .SS Advanced options .PP -Here are the advanced options specific to mega (Mega). +Here are the Advanced options specific to mega (Mega). .SS --mega-debug .PP Output more debug from Mega. @@ -39360,7 +43995,7 @@ to 1 nS. .PP The memory backend replaces the default restricted characters set (https://rclone.org/overview/#restricted-characters). -.SS Akamai NetStorage +.SH Akamai NetStorage .PP Paths are specified as \f[C]remote:\f[R] You may put subdirectories in too, e.g. @@ -39377,6 +44012,7 @@ For example, this is commonly configured with or without a CP code: * See all buckets rclone lsd remote: The initial setup for Netstorage involves getting an account and secret. Use \f[C]rclone config\f[R] to walk you through the setup process. +.SS Configuration .PP Here\[aq]s an example of how to make a remote called \f[C]ns1\f[R]. .IP "1." 3 @@ -39535,6 +44171,7 @@ You can\[aq]t perform operations between different remotes. rclone move ns1:/974012/testing/notes.txt ns1:/974450/testing2/ \f[R] .fi +.SS Features .SS Symlink Support .PP The Netstorage backend changes the rclone \f[C]--links, -l\f[R] @@ -39591,7 +44228,7 @@ such as SFTP and the Content Management Shell (CMShell). Rclone will not guarantee correctness of operations with implicit directories which might have been created as a result of using an upload API directly. -.SS ListR Feature +.SS \f[C]--fast-list\f[R] / ListR support .PP NetStorage remote supports the ListR feature by using the \[dq]list\[dq] NetStorage API action to return a lexicographical list of all objects @@ -39621,7 +44258,7 @@ display number of files in the directory and directory size as -1 when ListR method is used. The workaround is to pass \[dq]--disable listR\[dq] flag if these numbers are important in the output. -.SS Purge Feature +.SS Purge .PP NetStorage remote supports the purge feature by using the \[dq]quick-delete\[dq] NetStorage API action. @@ -39639,7 +44276,7 @@ immediately and objects targeted for quick-delete may still be accessible. .SS Standard options .PP -Here are the standard options specific to netstorage (Akamai +Here are the Standard options specific to netstorage (Akamai NetStorage). .SS --netstorage-host .PP @@ -39690,7 +44327,7 @@ Type: string Required: true .SS Advanced options .PP -Here are the advanced options specific to netstorage (Akamai +Here are the Advanced options specific to netstorage (Akamai NetStorage). .SS --netstorage-protocol .PP @@ -39738,9 +44375,8 @@ rclone backend COMMAND remote: .PP The help below will explain what arguments each command takes. .PP -See the \[dq]rclone backend\[dq] -command (https://rclone.org/commands/rclone_backend/) for more info on -how to pass options and arguments. +See the backend (https://rclone.org/commands/rclone_backend/) command +for more info on how to pass options and arguments. .PP These can be run on a running backend using the rc command backend/command (https://rclone.org/rc/#backend-command). @@ -39997,7 +44633,7 @@ parties access to a single container or putting credentials into an untrusted environment such as a CI build server. .SS Standard options .PP -Here are the standard options specific to azureblob (Microsoft Azure +Here are the Standard options specific to azureblob (Microsoft Azure Blob Storage). .SS --azureblob-account .PP @@ -40118,7 +44754,7 @@ Type: bool Default: false .SS Advanced options .PP -Here are the advanced options specific to azureblob (Microsoft Azure +Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage). .SS --azureblob-msi-object-id .PP @@ -40450,15 +45086,22 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SS Azure Storage Emulator Support .PP -You can test rclone with storage emulator locally, to do this make sure -azure storage emulator installed locally and set up a new remote with -\f[C]rclone config\f[R] follow instructions described in introduction, -set \f[C]use_emulator\f[R] config as \f[C]true\f[R], you do not need to -provide default account name or key if using emulator. +You can run rclone with storage emulator (usually \f[I]azurite\f[R]). +.PP +To do this, just set up a new remote with \f[C]rclone config\f[R] +following instructions described in introduction and set +\f[C]use_emulator\f[R] config as \f[C]true\f[R]. +You do not need to provide default account name neither an account key. +.PP +Also, if you want to access a storage emulator instance running on a +different machine, you can override \f[I]Endpoint\f[R] parameter in +advanced settings, setting it to +\f[C]http(s)://:/devstoreaccount1\f[R] (e.g. +\f[C]http://10.254.2.5:10000/devstoreaccount1\f[R]). .SH Microsoft OneDrive .PP Paths are specified as \f[C]remote:path\f[R] @@ -40595,13 +45238,17 @@ rclone copy /home/source remote:backup .fi .SS Getting your own Client ID and Key .PP -You can use your own Client ID if the default (\f[C]client_id\f[R] left -blank) one doesn\[aq]t work for you or you see lots of throttling. -The default Client ID and Key is shared by all rclone users when +rclone uses a default Client ID when talking to OneDrive, unless a +custom \f[C]client_id\f[R] is specified in the config. +The default Client ID and Key are shared by all rclone users when performing requests. .PP -If you are having problems with them (E.g., seeing a lot of throttling), -you can get your own Client ID and Key by following the steps below: +You may choose to create and use your own Client ID, in case the default +one does not work well for you. +For example, you might see throtting. +.SS Creating Client ID for OneDrive Personal +.PP +To create your own Client ID, please follow these steps: .IP "1." 3 Open https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationsListBlade @@ -40628,8 +45275,8 @@ select \f[C]delegated permissions\f[R]. Search and select the following permissions: \f[C]Files.Read\f[R], \f[C]Files.ReadWrite\f[R], \f[C]Files.Read.All\f[R], \f[C]Files.ReadWrite.All\f[R], \f[C]offline_access\f[R], -\f[C]User.Read\f[R], and optionally \f[C]Sites.Read.All\f[R] (see -below). +\f[C]User.Read\f[R] and \f[C]Sites.Read.All\f[R] (if custom access +scopes are configured, select the permissions accordingly). Once selected click \f[C]Add permissions\f[R] at the bottom. .PP Now the application is complete. @@ -40637,12 +45284,51 @@ Run \f[C]rclone config\f[R] to create or edit a OneDrive remote. Supply the app ID and password as Client ID and Secret, respectively. rclone will walk you through the remaining steps. .PP +The access_scopes option allows you to configure the permissions +requested by rclone. +See Microsoft +Docs (https://docs.microsoft.com/en-us/graph/permissions-reference#files-permissions) +for more information about the different scopes. +.PP The \f[C]Sites.Read.All\f[R] permission is required if you need to search SharePoint sites when configuring the remote (https://github.com/rclone/rclone/pull/5883). -However, if that permission is not assigned, you need to set +However, if that permission is not assigned, you need to exclude +\f[C]Sites.Read.All\f[R] from your access scopes or set \f[C]disable_site_permission\f[R] option to true in the advanced options. +.SS Creating Client ID for OneDrive Business +.PP +The steps for OneDrive Personal may or may not work for OneDrive +Business, depending on the security settings of the organization. +A common error is that the publisher of the App is not verified. +.PP +You may try to verify you +account (https://docs.microsoft.com/en-us/azure/active-directory/develop/publisher-verification-overview), +or try to limit the App to your organization only, as shown below. +.IP "1." 3 +Make sure to create the App with your business account. +.IP "2." 3 +Follow the steps above to create an App. +However, we need a different account type here: +\f[C]Accounts in this organizational directory only (*** - Single tenant)\f[R]. +Note that you can also change the account type aftering creating the +App. +.IP "3." 3 +Find the tenant +ID (https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/active-directory-how-to-find-tenant) +of your organization. +.IP "4." 3 +In the rclone config, set \f[C]auth_url\f[R] to +\f[C]https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/authorize\f[R]. +.IP "5." 3 +In the rclone config, set \f[C]token_url\f[R] to +\f[C]https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/token\f[R]. +.PP +Note: If you have a special region, you may need a different host in +step 4 and 5. +Here are some +hints (https://github.com/rclone/rclone/blob/bc23bf11db1c78c6ebbf8ea538fbebf7058b4176/backend/onedrive/onedrive.go#L86). .SS Modification time and hashes .PP OneDrive allows modification times to be set on objects accurate to 1 @@ -40800,7 +45486,7 @@ empty the trash, so you will have to do that with one of Microsoft\[aq]s apps or via the OneDrive website. .SS Standard options .PP -Here are the standard options specific to onedrive (Microsoft OneDrive). +Here are the Standard options specific to onedrive (Microsoft OneDrive). .SS --onedrive-client-id .PP OAuth Client Id. @@ -40874,7 +45560,7 @@ Azure and Office 365 operated by 21Vianet in China .RE .SS Advanced options .PP -Here are the advanced options specific to onedrive (Microsoft OneDrive). +Here are the Advanced options specific to onedrive (Microsoft OneDrive). .SS --onedrive-token .PP OAuth Access Token as a JSON blob. @@ -40982,6 +45668,50 @@ Env Var: RCLONE_ONEDRIVE_ROOT_FOLDER_ID Type: string .IP \[bu] 2 Required: false +.SS --onedrive-access-scopes +.PP +Set scopes to be requested by rclone. +.PP +Choose or manually enter a custom space separated list with all scopes, +that rclone should request. +.PP +Properties: +.IP \[bu] 2 +Config: access_scopes +.IP \[bu] 2 +Env Var: RCLONE_ONEDRIVE_ACCESS_SCOPES +.IP \[bu] 2 +Type: SpaceSepList +.IP \[bu] 2 +Default: Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All +Sites.Read.All offline_access +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All +Sites.Read.All offline_access\[dq] +.RS 2 +.IP \[bu] 2 +Read and write access to all resources +.RE +.IP \[bu] 2 +\[dq]Files.Read Files.Read.All Sites.Read.All offline_access\[dq] +.RS 2 +.IP \[bu] 2 +Read only access to all resources +.RE +.IP \[bu] 2 +\[dq]Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All +offline_access\[dq] +.RS 2 +.IP \[bu] 2 +Read and write access to all resources, without the ability to browse +SharePoint sites. +.IP \[bu] 2 +Same as if disable_site_permission was set to true +.RE +.RE .SS --onedrive-disable-site-permission .PP Disable the request for Sites.Read.All permission. @@ -41370,7 +46100,7 @@ known (https://github.com/OneDrive/onedrive-api-docs/issues/1068) issue that Sharepoint (not OneDrive or OneDrive for Business) may return \[dq]item not found\[dq] errors when users try to replace or delete uploaded files; this seems to mainly affect Office files (.docx, .xlsx, -etc.). +etc.) and web files (.html, .aspx, etc.). As a workaround, you may use the \f[C]--backup-dir \f[R] command line argument so rclone moves the files to be replaced/deleted into a given backup directory (instead of directly replacing/deleting @@ -41399,7 +46129,7 @@ your account. You can\[aq]t do much about it, maybe write an email to your admins. .PP However, there are other ways to interact with your OneDrive account. -Have a look at the webdav backend: https://rclone.org/webdav/#sharepoint +Have a look at the WebDAV backend: https://rclone.org/webdav/#sharepoint .SS invalid_grant (AADSTS50076) .IP .nf @@ -41653,7 +46383,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in JSON strings. .SS Standard options .PP -Here are the standard options specific to opendrive (OpenDrive). +Here are the Standard options specific to opendrive (OpenDrive). .SS --opendrive-username .PP Username. @@ -41685,7 +46415,7 @@ Type: string Required: true .SS Advanced options .PP -Here are the advanced options specific to opendrive (OpenDrive). +Here are the Advanced options specific to opendrive (OpenDrive). .SS --opendrive-encoding .PP The encoding for the backend. @@ -41739,7 +46469,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SH QingStor .PP @@ -41918,7 +46648,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in JSON strings. .SS Standard options .PP -Here are the standard options specific to qingstor (QingCloud Object +Here are the Standard options specific to qingstor (QingCloud Object Storage). .SS --qingstor-env-auth .PP @@ -42042,7 +46772,7 @@ Needs location constraint gd2a. .RE .SS Advanced options .PP -Here are the advanced options specific to qingstor (QingCloud Object +Here are the Advanced options specific to qingstor (QingCloud Object Storage). .SS --qingstor-connection-retries .PP @@ -42142,7 +46872,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SH Sia .PP @@ -42298,7 +47028,7 @@ rclone copy /home/source mySia:backup .fi .SS Standard options .PP -Here are the standard options specific to sia (Sia Decentralized Cloud). +Here are the Standard options specific to sia (Sia Decentralized Cloud). .SS --sia-api-url .PP Sia daemon API URL, like http://sia.daemon.host:9980. @@ -42337,7 +47067,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to sia (Sia Decentralized Cloud). +Here are the Advanced options specific to sia (Sia Decentralized Cloud). .SS --sia-user-agent .PP Siad User Agent @@ -42682,7 +47412,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in JSON strings. .SS Standard options .PP -Here are the standard options specific to swift (OpenStack Swift +Here are the Standard options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). .SS --swift-env-auth .PP @@ -43032,7 +47762,7 @@ OVH Public Cloud Archive .RE .SS Advanced options .PP -Here are the advanced options specific to swift (OpenStack Swift +Here are the Advanced options specific to swift (OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)). .SS --swift-leave-parts-on-error .PP @@ -43294,6 +48024,14 @@ be used in JSON strings. Deleted files will be moved to the trash. Your subscription level will determine how long items stay in the trash. \f[C]rclone cleanup\f[R] can be used to empty the trash. +.SS Emptying the trash +.PP +Due to an API limitation, the \f[C]rclone cleanup\f[R] command will only +work if you set your username and password in the advanced options for +this backend. +Since we generally want to avoid storing user passwords in the rclone +config file, we advise you to only set this up if you need the +\f[C]rclone cleanup\f[R] command to work. .SS Root folder ID .PP You can set the \f[C]root_folder_id\f[R] for rclone. @@ -43317,7 +48055,7 @@ in the browser, then you use \f[C]5xxxxxxxx8\f[R] as the \f[C]root_folder_id\f[R] in the config. .SS Standard options .PP -Here are the standard options specific to pcloud (Pcloud). +Here are the Standard options specific to pcloud (Pcloud). .SS --pcloud-client-id .PP OAuth Client Id. @@ -43350,7 +48088,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to pcloud (Pcloud). +Here are the Advanced options specific to pcloud (Pcloud). .SS --pcloud-token .PP OAuth Access Token as a JSON blob. @@ -43456,6 +48194,40 @@ Original/US region EU region .RE .RE +.SS --pcloud-username +.PP +Your pcloud username. +.PP +This is only required when you want to use the cleanup command. +Due to a bug in the pcloud API the required API does not support OAuth +authentication so we have to rely on user password authentication for +it. +.PP +Properties: +.IP \[bu] 2 +Config: username +.IP \[bu] 2 +Env Var: RCLONE_PCLOUD_USERNAME +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --pcloud-password +.PP +Your pcloud password. +.PP +\f[B]NB\f[R] Input to this must be obscured - see rclone +obscure (https://rclone.org/commands/rclone_obscure/). +.PP +Properties: +.IP \[bu] 2 +Config: password +.IP \[bu] 2 +Env Var: RCLONE_PCLOUD_PASSWORD +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false .SH premiumize.me .PP Paths are specified as \f[C]remote:path\f[R] @@ -43598,7 +48370,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in JSON strings. .SS Standard options .PP -Here are the standard options specific to premiumizeme (premiumize.me). +Here are the Standard options specific to premiumizeme (premiumize.me). .SS --premiumizeme-api-key .PP API Key. @@ -43616,7 +48388,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to premiumizeme (premiumize.me). +Here are the Advanced options specific to premiumizeme (premiumize.me). .SS --premiumizeme-encoding .PP The encoding for the backend. @@ -43786,7 +48558,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in JSON strings. .SS Advanced options .PP -Here are the advanced options specific to putio (Put.io). +Here are the Advanced options specific to putio (Put.io). .SS --putio-encoding .PP The encoding for the backend. @@ -43803,6 +48575,16 @@ Env Var: RCLONE_PUTIO_ENCODING Type: MultiEncoder .IP \[bu] 2 Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot +.SS Limitations +.PP +put.io has rate limiting. +When you hit a limit, rclone automatically retries after waiting the +amount of time requested by the server. +.PP +If you want to avoid ever hitting these limits, you may use the +\f[C]--tpslimit\f[R] flag with a low number. +Note that the imposed limits may be different for different operations, +and may change over time. .SH Seafile .PP This is a backend for the Seafile (https://www.seafile.com/) storage @@ -44147,7 +48929,7 @@ Versions between 6.0 and 6.3 haven\[aq]t been tested and might not work properly. .SS Standard options .PP -Here are the standard options specific to seafile (seafile). +Here are the Standard options specific to seafile (seafile). .SS --seafile-url .PP URL of seafile host to connect to. @@ -44262,7 +49044,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to seafile (seafile). +Here are the Advanced options specific to seafile (seafile). .SS --seafile-create-library .PP Should rclone create a library if it doesn\[aq]t exist. @@ -44299,7 +49081,7 @@ Protocol (https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol). .PP The SFTP backend can be used with a number of different providers: .IP \[bu] 2 -C14 +Hetzner Storage Box .IP \[bu] 2 rsync.net .PP @@ -44319,7 +49101,11 @@ remote machine (i.e. .PP Note that some SFTP servers will need the leading / - Synology is a good example of this. -rsync.net, on the other hand, requires users to OMIT the leading /. +rsync.net and Hetzner, on the other hand, requires users to OMIT the +leading /. +.PP +Note that by default rclone will try to execute shell commands on the +server, see shell access considerations. .SS Configuration .PP Here is an example of making an SFTP configuration. @@ -44344,7 +49130,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / SSH/SFTP Connection +XX / SSH/SFTP \[rs] \[dq]sftp\[dq] [snip] Storage> sftp @@ -44618,6 +49404,140 @@ eval \[ga]ssh-agent -k\[ga] .fi .PP These commands can be used in scripts of course. +.SS Shell access +.PP +Some functionality of the SFTP backend relies on remote shell access, +and the possibility to execute commands. +This includes checksum, and in some cases also about. +The shell commands that must be executed may be different on different +type of shells, and also quoting/escaping of file path arguments +containing special characters may be different. +Rclone therefore needs to know what type of shell it is, and if shell +access is available at all. +.PP +Most servers run on some version of Unix, and then a basic Unix shell +can be assumed, without further distinction. +Windows 10, Server 2019, and later can also run a SSH server, which is a +port of OpenSSH (see official installation +guide (https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse)). +On a Windows server the shell handling is different: Although it can +also be set up to use a Unix type shell, e.g. +Cygwin bash, the default is to use Windows Command Prompt (cmd.exe), and +PowerShell is a recommended alternative. +All of these have bahave differently, which rclone must handle. +.PP +Rclone tries to auto-detect what type of shell is used on the server, +first time you access the SFTP remote. +If a remote shell session is successfully created, it will look for +indications that it is CMD or PowerShell, with fall-back to Unix if not +something else is detected. +If unable to even create a remote shell session, then shell command +execution will be disabled entirely. +The result is stored in the SFTP remote configuration, in option +\f[C]shell_type\f[R], so that the auto-detection only have to be +performed once. +If you manually set a value for this option before first run, the +auto-detection will be skipped, and if you set a different value later +this will override any existing. +Value \f[C]none\f[R] can be set to avoid any attempts at executing shell +commands, e.g. +if this is not allowed on the server. +.PP +When the server is rclone serve +sftp (https://rclone.org/commands/rclone_serve_sftp/), the rclone SFTP +remote will detect this as a Unix type shell - even if it is running on +Windows. +This server does not actually have a shell, but it accepts input +commands matching the specific ones that the SFTP backend relies on for +Unix shells, e.g. +\f[C]md5sum\f[R] and \f[C]df\f[R]. +Also it handles the string escape rules used for Unix shell. +Treating it as a Unix type shell from a SFTP remote will therefore +always be correct, and support all features. +.SS Shell access considerations +.PP +The shell type auto-detection logic, described above, means that by +default rclone will try to run a shell command the first time a new sftp +remote is accessed. +If you configure a sftp remote without a config file, e.g. +an on the fly (https://rclone.org/docs/#backend-path-to-dir%5D) remote, +rclone will have nowhere to store the result, and it will re-run the +command on every access. +To avoid this you should explicitely set the \f[C]shell_type\f[R] option +to the correct value, or to \f[C]none\f[R] if you want to prevent rclone +from executing any remote shell commands. +.PP +It is also important to note that, since the shell type decides how +quoting and escaping of file paths used as command-line arguments are +performed, configuring the wrong shell type may leave you exposed to +command injection exploits. +Make sure to confirm the auto-detected shell type, or explicitely set +the shell type you know is correct, or disable shell access until you +know. +.SS Checksum +.PP +SFTP does not natively support checksums (file hash), but rclone is able +to use checksumming if the same login has shell access, and can execute +remote commands. +If there is a command that can calculate compatible checksums on the +remote system, Rclone can then be configured to execute this whenever a +checksum is needed, and read back the results. +Currently MD5 and SHA-1 are supported. +.PP +Normally this requires an external utility being available on the +server. +By default rclone will try commands \f[C]md5sum\f[R], \f[C]md5\f[R] and +\f[C]rclone md5sum\f[R] for MD5 checksums, and the first one found +usable will be picked. +Same with \f[C]sha1sum\f[R], \f[C]sha1\f[R] and \f[C]rclone sha1sum\f[R] +commands for SHA-1 checksums. +These utilities normally need to be in the remote\[aq]s PATH to be +found. +.PP +In some cases the shell itself is capable of calculating checksums. +PowerShell is an example of such a shell. +If rclone detects that the remote shell is PowerShell, which means it +most probably is a Windows OpenSSH server, rclone will use a predefined +script block to produce the checksums when no external checksum commands +are found (see shell access). +This assumes PowerShell version 4.0 or newer. +.PP +The options \f[C]md5sum_command\f[R] and \f[C]sha1_command\f[R] can be +used to customize the command to be executed for calculation of +checksums. +You can for example set a specific path to where md5sum and sha1sum +executables are located, or use them to specify some other tools that +print checksums in compatible format. +The value can include command-line arguments, or even shell script +blocks as with PowerShell. +Rclone has subcommands +md5sum (https://rclone.org/commands/rclone_md5sum/) and +sha1sum (https://rclone.org/commands/rclone_sha1sum/) that use +compatible format, which means if you have an rclone executable on the +server it can be used. +As mentioned above, they will be automatically picked up if found in +PATH, but if not you can set something like +\f[C]/path/to/rclone md5sum\f[R] as the value of option +\f[C]md5sum_command\f[R] to make sure a specific executable is used. +.PP +Remote checksumming is recommended and enabled by default. +First time rclone is using a SFTP remote, if options +\f[C]md5sum_command\f[R] or \f[C]sha1_command\f[R] are not set, it will +check if any of the default commands for each of them, as described +above, can be used. +The result will be saved in the remote configuration, so next time it +will use the same. +Value \f[C]none\f[R] will be set if none of the default commands could +be used for a specific algorithm, and this algorithm will not be +supported by the remote. +.PP +Disabling the checksumming may be required if you are connecting to SFTP +servers which are not under your control, and to which the execution of +remote shell commands is prohibited. +Set the configuration option \f[C]disable_hashcheck\f[R] to +\f[C]true\f[R] to disable checksumming entirely, or set +\f[C]shell_type\f[R] to \f[C]none\f[R] to disable all functionality +based on remote shell command execution. .SS Modified time .PP Modified times are stored on the server to 1 second precision. @@ -44630,9 +49550,26 @@ mod_sftp). If you are using one of these servers, you can set the option \f[C]set_modtime = false\f[R] in your RClone backend configuration to disable this behaviour. +.SS About command +.PP +The \f[C]about\f[R] command returns the total space, free space, and +used space on the remote for the disk of the specified path on the +remote or, if not set, the disk of the root on the remote. +.PP +SFTP usually supports the +about (https://rclone.org/commands/rclone_about/) command, but it +depends on the server. +If the server implements the vendor-specific VFS statistics extension, +which is normally the case with OpenSSH instances, it will be used. +If not, but the same login has access to a Unix shell, where the +\f[C]df\f[R] command is available (e.g. +in the remote\[aq]s PATH), then this will be used instead. +If the server shell is PowerShell, probably with a Windows OpenSSH +server, rclone will use a built-in shell command (see shell access). +If none of the above is applicable, \f[C]about\f[R] will fail. .SS Standard options .PP -Here are the standard options specific to sftp (SSH/SFTP Connection). +Here are the Standard options specific to sftp (SSH/SFTP). .SS --sftp-host .PP SSH host to connect to. @@ -44850,7 +49787,7 @@ Type: bool Default: false .SS Advanced options .PP -Here are the advanced options specific to sftp (SSH/SFTP Connection). +Here are the Advanced options specific to sftp (SSH/SFTP). .SS --sftp-known-hosts-file .PP Optional path to known_hosts file. @@ -44897,12 +49834,13 @@ Type: bool Default: false .SS --sftp-path-override .PP -Override path used by SSH connection. +Override path used by SSH shell commands. .PP This allows checksum calculation when SFTP and SSH paths are different. This issue affects among others Synology NAS boxes. .PP -Shared folders can be found in directories representing volumes +E.g. +if shared folders can be found in directories representing volumes: .IP .nf \f[C] @@ -44910,7 +49848,8 @@ rclone sync /home/local/directory remote:/directory --sftp-path-override /volume \f[R] .fi .PP -Home directory can be found in a shared folder called \[dq]home\[dq] +E.g. +if home directory can be found in a shared folder called \[dq]home\[dq]: .IP .nf \f[C] @@ -44940,6 +49879,49 @@ Env Var: RCLONE_SFTP_SET_MODTIME Type: bool .IP \[bu] 2 Default: true +.SS --sftp-shell-type +.PP +The type of SSH shell on remote server, if any. +.PP +Leave blank for autodetect. +.PP +Properties: +.IP \[bu] 2 +Config: shell_type +.IP \[bu] 2 +Env Var: RCLONE_SFTP_SHELL_TYPE +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]none\[dq] +.RS 2 +.IP \[bu] 2 +No shell access +.RE +.IP \[bu] 2 +\[dq]unix\[dq] +.RS 2 +.IP \[bu] 2 +Unix shell +.RE +.IP \[bu] 2 +\[dq]powershell\[dq] +.RS 2 +.IP \[bu] 2 +PowerShell +.RE +.IP \[bu] 2 +\[dq]cmd\[dq] +.RS 2 +.IP \[bu] 2 +Windows Command Prompt +.RE +.RE .SS --sftp-md5sum-command .PP The command used to read md5 hashes. @@ -45099,28 +50081,92 @@ Env Var: RCLONE_SFTP_IDLE_TIMEOUT Type: Duration .IP \[bu] 2 Default: 1m0s -.SS Limitations +.SS --sftp-chunk-size .PP -SFTP supports checksums if the same login has shell access and -\f[C]md5sum\f[R] or \f[C]sha1sum\f[R] as well as \f[C]echo\f[R] are in -the remote\[aq]s PATH. -This remote checksumming (file hashing) is recommended and enabled by -default. -Disabling the checksumming may be required if you are connecting to SFTP -servers which are not under your control, and to which the execution of -remote commands is prohibited. -Set the configuration option \f[C]disable_hashcheck\f[R] to -\f[C]true\f[R] to disable checksumming. +Upload and download chunk size. .PP -SFTP also supports \f[C]about\f[R] if the same login has shell access -and \f[C]df\f[R] are in the remote\[aq]s PATH. -\f[C]about\f[R] will return the total space, free space, and used space -on the remote for the disk of the specified path on the remote or, if -not set, the disk of the root on the remote. -\f[C]about\f[R] will fail if it does not have shell access or if -\f[C]df\f[R] is not in the remote\[aq]s PATH. +This controls the maximum packet size used in the SFTP protocol. +The RFC limits this to 32768 bytes (32k), however a lot of servers +support larger sizes and setting it larger will increase transfer speed +dramatically on high latency links. .PP -Note that some SFTP servers (e.g. +Only use a setting higher than 32k if you always connect to the same +server or after sufficiently broad testing. +.PP +For example using the value of 252k with OpenSSH works well with its +maximum packet size of 256k. +.PP +If you get the error \[dq]failed to send packet header: EOF\[dq] when +copying a large file, try lowering this number. +.PP +Properties: +.IP \[bu] 2 +Config: chunk_size +.IP \[bu] 2 +Env Var: RCLONE_SFTP_CHUNK_SIZE +.IP \[bu] 2 +Type: SizeSuffix +.IP \[bu] 2 +Default: 32Ki +.SS --sftp-concurrency +.PP +The maximum number of outstanding requests for one file +.PP +This controls the maximum number of outstanding requests for one file. +Increasing it will increase throughput on high latency links at the cost +of using more memory. +.PP +Properties: +.IP \[bu] 2 +Config: concurrency +.IP \[bu] 2 +Env Var: RCLONE_SFTP_CONCURRENCY +.IP \[bu] 2 +Type: int +.IP \[bu] 2 +Default: 64 +.SS --sftp-set-env +.PP +Environment variables to pass to sftp and commands +.PP +Set environment variables in the form: +.IP +.nf +\f[C] +VAR=value +\f[R] +.fi +.PP +to be passed to the sftp client and to any commands run (eg md5sum). +.PP +Pass multiple variables space separated, eg +.IP +.nf +\f[C] +VAR1=value VAR2=value +\f[R] +.fi +.PP +and pass variables with spaces in in quotes, eg +.IP +.nf +\f[C] +\[dq]VAR3=value with space\[dq] \[dq]VAR4=value with space\[dq] VAR5=nospacehere +\f[R] +.fi +.PP +Properties: +.IP \[bu] 2 +Config: set_env +.IP \[bu] 2 +Env Var: RCLONE_SFTP_SET_ENV +.IP \[bu] 2 +Type: SpaceSepList +.IP \[bu] 2 +Default: +.SS Limitations +.PP +On some SFTP servers (e.g. Synology) the paths are different for SSH and SFTP so the hashes can\[aq]t be calculated properly. For them using \f[C]disable_hashcheck\f[R] is a good idea. @@ -45140,22 +50186,22 @@ issue (https://github.com/pkg/sftp/issues/156) is fixed. .PP Note that since SFTP isn\[aq]t HTTP based the following flags don\[aq]t work with it: \f[C]--dump-headers\f[R], \f[C]--dump-bodies\f[R], -\f[C]--dump-auth\f[R] +\f[C]--dump-auth\f[R]. .PP Note that \f[C]--timeout\f[R] and \f[C]--contimeout\f[R] are both supported. -.SS C14 -.PP -C14 is supported through the SFTP backend. -.PP -See C14\[aq]s -documentation (https://www.online.net/en/storage/c14-cold-storage) .SS rsync.net .PP rsync.net is supported through the SFTP backend. .PP See rsync.net\[aq]s documentation of rclone examples (https://www.rsync.net/products/rclone.html). +.SS Hetzner Storage Box +.PP +Hetzner Storage Boxes are supported through the SFTP backend on port 23. +.PP +See Hetzner\[aq]s documentation for +details (https://docs.hetzner.com/robot/storage-box/access/access-ssh-rsync-borg#rclone) .SH Storj .PP Storj (https://storj.io) is an encrypted, secure, and cost-effective @@ -45431,7 +50477,7 @@ y/e/d> y .fi .SS Standard options .PP -Here are the standard options specific to storj (Storj Decentralized +Here are the Standard options specific to storj (Storj Decentralized Cloud Storage). .SS --storj-provider .PP @@ -45741,7 +50787,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SS Known issues .PP @@ -45904,7 +50950,7 @@ the config parameter \f[C]hard_delete = true\f[R] if you would like files to be deleted straight away. .SS Standard options .PP -Here are the standard options specific to sugarsync (Sugarsync). +Here are the Standard options specific to sugarsync (Sugarsync). .SS --sugarsync-app-id .PP Sugarsync App ID. @@ -45966,7 +51012,7 @@ Type: bool Default: false .SS Advanced options .PP -Here are the advanced options specific to sugarsync (Sugarsync). +Here are the Advanced options specific to sugarsync (Sugarsync). .SS --sugarsync-refresh-token .PP Sugarsync refresh token. @@ -46081,7 +51127,7 @@ rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member of an rclone union remote. .PP See List of backends that do not support rclone -about (https://rclone.org/overview/#optional-features) See rclone +about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SH Tardigrade .PP @@ -46229,7 +51275,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in XML strings. .SS Standard options .PP -Here are the standard options specific to uptobox (Uptobox). +Here are the Standard options specific to uptobox (Uptobox). .SS --uptobox-access-token .PP Your access token. @@ -46247,7 +51293,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to uptobox (Uptobox). +Here are the Advanced options specific to uptobox (Uptobox). .SS --uptobox-encoding .PP The encoding for the backend. @@ -46645,7 +51691,7 @@ T} .TE .SS Standard options .PP -Here are the standard options specific to union (Union merges the +Here are the Standard options specific to union (Union merges the contents of several upstream fs). .SS --union-upstreams .PP @@ -46717,6 +51763,31 @@ Env Var: RCLONE_UNION_CACHE_TIME Type: int .IP \[bu] 2 Default: 120 +.SS Advanced options +.PP +Here are the Advanced options specific to union (Union merges the +contents of several upstream fs). +.SS --union-min-free-space +.PP +Minimum viable free space for lfs/eplfs policies. +.PP +If a remote has less than this much free space then it won\[aq]t be +considered for use in lfs or eplfs policies. +.PP +Properties: +.IP \[bu] 2 +Config: min_free_space +.IP \[bu] 2 +Env Var: RCLONE_UNION_MIN_FREE_SPACE +.IP \[bu] 2 +Type: SizeSuffix +.IP \[bu] 2 +Default: 1Gi +.SS Metadata +.PP +Any metadata supported by the underlying remote is read and written. +.PP +See the metadata (https://rclone.org/docs/#metadata) docs for more info. .SH WebDAV .PP Paths are specified as \f[C]remote:path\f[R] @@ -46752,7 +51823,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Webdav +XX / WebDAV \[rs] \[dq]webdav\[dq] [snip] Storage> webdav @@ -46761,7 +51832,7 @@ Choose a number from below, or type in your own value 1 / Connect to example.com \[rs] \[dq]https://example.com\[dq] url> https://example.com/remote.php/webdav/ -Name of the Webdav site/service/software you are using +Name of the WebDAV site/service/software you are using Choose a number from below, or type in your own value 1 / Nextcloud \[rs] \[dq]nextcloud\[dq] @@ -46842,7 +51913,7 @@ appear on all objects, or only on objects which had a hash uploaded with them. .SS Standard options .PP -Here are the standard options specific to webdav (Webdav). +Here are the Standard options specific to webdav (WebDAV). .SS --webdav-url .PP URL of http host to connect to. @@ -46861,7 +51932,7 @@ Type: string Required: true .SS --webdav-vendor .PP -Name of the Webdav site/service/software you are using. +Name of the WebDAV site/service/software you are using. .PP Properties: .IP \[bu] 2 @@ -46954,7 +52025,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to webdav (Webdav). +Here are the Advanced options specific to webdav (WebDAV). .SS --webdav-bearer-token-command .PP Command to run to get a bearer token. @@ -47348,7 +52419,7 @@ replaced (https://rclone.org/overview/#invalid-utf8), as they can\[aq]t be used in JSON strings. .SS Standard options .PP -Here are the standard options specific to yandex (Yandex Disk). +Here are the Standard options specific to yandex (Yandex Disk). .SS --yandex-client-id .PP OAuth Client Id. @@ -47381,7 +52452,7 @@ Type: string Required: false .SS Advanced options .PP -Here are the advanced options specific to yandex (Yandex Disk). +Here are the Advanced options specific to yandex (Yandex Disk). .SS --yandex-token .PP OAuth Access Token as a JSON blob. @@ -47623,7 +52694,7 @@ In addition most Unicode full-width characters are not supported at all and will be removed from filenames during upload. .SS Standard options .PP -Here are the standard options specific to zoho (Zoho). +Here are the Standard options specific to zoho (Zoho). .SS --zoho-client-id .PP OAuth Client Id. @@ -47693,6 +52764,18 @@ Europe India .RE .IP \[bu] 2 +\[dq]jp\[dq] +.RS 2 +.IP \[bu] 2 +Japan +.RE +.IP \[bu] 2 +\[dq]com.cn\[dq] +.RS 2 +.IP \[bu] 2 +China +.RE +.IP \[bu] 2 \[dq]com.au\[dq] .RS 2 .IP \[bu] 2 @@ -47701,7 +52784,7 @@ Australia .RE .SS Advanced options .PP -Here are the advanced options specific to zoho (Zoho). +Here are the Advanced options specific to zoho (Zoho). .SS --zoho-token .PP OAuth Access Token as a JSON blob. @@ -47761,6 +52844,21 @@ Env Var: RCLONE_ZOHO_ENCODING Type: MultiEncoder .IP \[bu] 2 Default: Del,Ctl,InvalidUtf8 +.SS Setting up your own client_id +.PP +For Zoho we advise you to set up your own client_id. +To do so you have to complete the following steps. +.IP "1." 3 +Log in to the Zoho API Console (https://api-console.zoho.com) +.IP "2." 3 +Create a new client of type \[dq]Server-based Application\[dq]. +The name and website don\[aq]t matter, but you must add the redirect URL +\f[C]http://localhost:53682/\f[R]. +.IP "3." 3 +Once the client is created, you can go to the settings tab and enable it +in other regions. +.PP +The client id and client secret can now be used with rclone. .SH Local Filesystem .PP Local paths are specified as normal filesystem paths, e.g. @@ -48440,7 +53538,7 @@ On systems where it isn\[aq]t supported (e.g. Windows) it will be ignored. .SS Advanced options .PP -Here are the advanced options specific to local (Local Disk). +Here are the Advanced options specific to local (Local Disk). .SS --local-nounc .PP Disable UNC (long path names) conversion on Windows. @@ -48451,9 +53549,9 @@ Config: nounc .IP \[bu] 2 Env Var: RCLONE_LOCAL_NOUNC .IP \[bu] 2 -Type: string +Type: bool .IP \[bu] 2 -Required: false +Default: false .IP \[bu] 2 Examples: .RS 2 @@ -48720,6 +53818,115 @@ Env Var: RCLONE_LOCAL_ENCODING Type: MultiEncoder .IP \[bu] 2 Default: Slash,Dot +.SS Metadata +.PP +Depending on which OS is in use the local backend may return only some +of the system metadata. +Setting system metadata is supported on all OSes but setting user +metadata is only supported on linux, freebsd, netbsd, macOS and Solaris. +It is \f[B]not\f[R] supported on Windows yet (see +pkg/attrs#47 (https://github.com/pkg/xattr/issues/47)). +.PP +User metadata is stored as extended attributes (which may not be +supported by all file systems) under the \[dq]user.*\[dq] prefix. +.PP +Here are the possible system metadata items for the local backend. +.PP +.TS +tab(@); +lw(11.1n) lw(11.1n) lw(11.1n) lw(16.6n) lw(20.3n). +T{ +Name +T}@T{ +Help +T}@T{ +Type +T}@T{ +Example +T}@T{ +Read Only +T} +_ +T{ +atime +T}@T{ +Time of last access +T}@T{ +RFC 3339 +T}@T{ +2006-01-02T15:04:05.999999999Z07:00 +T}@T{ +N +T} +T{ +btime +T}@T{ +Time of file birth (creation) +T}@T{ +RFC 3339 +T}@T{ +2006-01-02T15:04:05.999999999Z07:00 +T}@T{ +N +T} +T{ +gid +T}@T{ +Group ID of owner +T}@T{ +decimal number +T}@T{ +500 +T}@T{ +N +T} +T{ +mode +T}@T{ +File type and mode +T}@T{ +octal, unix style +T}@T{ +0100664 +T}@T{ +N +T} +T{ +mtime +T}@T{ +Time of last modification +T}@T{ +RFC 3339 +T}@T{ +2006-01-02T15:04:05.999999999Z07:00 +T}@T{ +N +T} +T{ +rdev +T}@T{ +Device ID (if special file) +T}@T{ +hexadecimal +T}@T{ +1abc +T}@T{ +N +T} +T{ +uid +T}@T{ +User ID of owner +T}@T{ +decimal number +T}@T{ +500 +T}@T{ +N +T} +.TE +.PP +See the metadata (https://rclone.org/docs/#metadata) docs for more info. .SS Backend commands .PP Here are the commands specific to the local backend. @@ -48734,9 +53941,8 @@ rclone backend COMMAND remote: .PP The help below will explain what arguments each command takes. .PP -See the \[dq]rclone backend\[dq] -command (https://rclone.org/commands/rclone_backend/) for more info on -how to pass options and arguments. +See the backend (https://rclone.org/commands/rclone_backend/) command +for more info on how to pass options and arguments. .PP These can be run on a running backend using the rc command backend/command (https://rclone.org/rc/#backend-command). @@ -48759,6 +53965,568 @@ Options: .IP \[bu] 2 \[dq]error\[dq]: return an error based on option value .SH Changelog +.SS v1.59.0 - 2022-07-09 +.PP +See commits (https://github.com/rclone/rclone/compare/v1.58.0...v1.59.0) +.IP \[bu] 2 +New backends +.RS 2 +.IP \[bu] 2 +Combine multiple remotes in one directory tree (Nick Craig-Wood) +.IP \[bu] 2 +Hidrive (https://rclone.org/hidrive/) (Ovidiu Victor Tatar) +.IP \[bu] 2 +Internet Archive (https://rclone.org/internetarchive/) (Lesmiscore +(Naoya Ozaki)) +.IP \[bu] 2 +New S3 providers +.RS 2 +.IP \[bu] 2 +ArvanCloud AOS (https://rclone.org/s3/#arvan-cloud) (ehsantdy) +.IP \[bu] 2 +Cloudflare R2 (https://rclone.org/s3/#cloudflare-r2) (Nick Craig-Wood) +.IP \[bu] 2 +Huawei OBS (https://rclone.org/s3/#huawei-obs) (m00594701) +.IP \[bu] 2 +IDrive e2 (https://rclone.org/s3/#idrive-e2) (vyloy) +.RE +.RE +.IP \[bu] 2 +New commands +.RS 2 +.IP \[bu] 2 +test makefile (https://rclone.org/commands/rclone_test_makefile/): +Create a single file for testing (Nick Craig-Wood) +.RE +.IP \[bu] 2 +New Features +.RS 2 +.IP \[bu] 2 +Metadata framework (https://rclone.org/docs/#metadata) to read and write +system and user metadata on backends (Nick Craig-Wood) +.RS 2 +.IP \[bu] 2 +Implemented initially for \f[C]local\f[R], \f[C]s3\f[R] and +\f[C]internetarchive\f[R] backends +.IP \[bu] 2 +\f[C]--metadata\f[R]/\f[C]-M\f[R] flag to control whether metadata is +copied +.IP \[bu] 2 +\f[C]--metadata-set\f[R] flag to specify metadata for uploads +.IP \[bu] 2 +Thanks to Manz Solutions (https://manz-solutions.at/) for sponsoring +this work. +.RE +.IP \[bu] 2 +build +.RS 2 +.IP \[bu] 2 +Update to go1.18 and make go1.16 the minimum required version (Nick +Craig-Wood) +.IP \[bu] 2 +Update android go build to 1.18.x and NDK to 23.1.7779620 (Nick +Craig-Wood) +.IP \[bu] 2 +All windows binaries now no longer CGO (Nick Craig-Wood) +.IP \[bu] 2 +Add \f[C]linux/arm/v6\f[R] to docker images (Nick Craig-Wood) +.IP \[bu] 2 +A huge number of fixes found with staticcheck (https://staticcheck.io/) +(albertony) +.IP \[bu] 2 +Configurable version suffix independent of version number (albertony) +.RE +.IP \[bu] 2 +check: Implement \f[C]--no-traverse\f[R] and +\f[C]--no-unicode-normalization\f[R] (Nick Craig-Wood) +.IP \[bu] 2 +config: Readability improvements (albertony) +.IP \[bu] 2 +copyurl: Add \f[C]--header-filename\f[R] to honor the HTTP header +filename directive (J-P Treen) +.IP \[bu] 2 +filter: Allow multiple \f[C]--exclude-if-present\f[R] flags (albertony) +.IP \[bu] 2 +fshttp: Add \f[C]--disable-http-keep-alives\f[R] to disable HTTP Keep +Alives (Nick Craig-Wood) +.IP \[bu] 2 +install.sh +.RS 2 +.IP \[bu] 2 +Set the modes on the files and/or directories on macOS (Michael C +Tiernan - MIT-Research Computing Project) +.IP \[bu] 2 +Pre verify sudo authorization \f[C]-v\f[R] before calling curl. +(Michael C Tiernan - MIT-Research Computing Project) +.RE +.IP \[bu] 2 +lib/encoder: Add Semicolon encoding (Nick Craig-Wood) +.IP \[bu] 2 +lsf: Add metadata support with \f[C]M\f[R] flag (Nick Craig-Wood) +.IP \[bu] 2 +lsjson: Add \f[C]--metadata\f[R]/\f[C]-M\f[R] flag (Nick Craig-Wood) +.IP \[bu] 2 +ncdu +.RS 2 +.IP \[bu] 2 +Implement multi selection (CrossR) +.IP \[bu] 2 +Replace termbox with tcell\[aq]s termbox wrapper (eNV25) +.IP \[bu] 2 +Display correct path in delete confirmation dialog (Roberto Ricci) +.RE +.IP \[bu] 2 +operations +.RS 2 +.IP \[bu] 2 +Speed up hash checking by aborting the other hash if first returns +nothing (Nick Craig-Wood) +.IP \[bu] 2 +Use correct src/dst in some log messages (zzr93) +.RE +.IP \[bu] 2 +rcat: Check checksums by default like copy does (Nick Craig-Wood) +.IP \[bu] 2 +selfupdate: Replace deprecated \f[C]x/crypto/openpgp\f[R] package with +\f[C]ProtonMail/go-crypto\f[R] (albertony) +.IP \[bu] 2 +serve ftp: Check \f[C]--passive-port\f[R] arguments are correct (Nick +Craig-Wood) +.IP \[bu] 2 +size: Warn about inaccurate results when objects with unknown size +(albertony) +.IP \[bu] 2 +sync: Overlap check is now filter-sensitive so \f[C]--backup-dir\f[R] +can be in the root provided it is filtered (Nick) +.IP \[bu] 2 +test info: Check file name lengths using 1,2,3,4 byte unicode characters +(Nick Craig-Wood) +.IP \[bu] 2 +test makefile(s): \f[C]--sparse\f[R], \f[C]--zero\f[R], +\f[C]--pattern\f[R], \f[C]--ascii\f[R], \f[C]--chargen\f[R] flags to +control file contents (Nick Craig-Wood) +.IP \[bu] 2 +Make sure we call the \f[C]Shutdown\f[R] method on backends (Martin +Czygan) +.RE +.IP \[bu] 2 +Bug Fixes +.RS 2 +.IP \[bu] 2 +accounting: Fix unknown length file transfers counting 3 transfers each +(buda) +.IP \[bu] 2 +ncdu: Fix issue where dir size is summed when file sizes are -1 +(albertony) +.IP \[bu] 2 +sync/copy/move +.RS 2 +.IP \[bu] 2 +Fix \f[C]--fast-list\f[R] \f[C]--create-empty-src-dirs\f[R] and +\f[C]--exclude\f[R] (Nick Craig-Wood) +.IP \[bu] 2 +Fix \f[C]--max-duration\f[R] and \f[C]--cutoff-mode soft\f[R] (Nick +Craig-Wood) +.RE +.IP \[bu] 2 +Fix fs cache unpin (Martin Czygan) +.IP \[bu] 2 +Set proper exit code for errors that are not low-level retried (e.g. +size/timestamp changing) (albertony) +.RE +.IP \[bu] 2 +Mount +.RS 2 +.IP \[bu] 2 +Support \f[C]windows/arm64\f[R] (may still be problems - see +#5828 (https://github.com/rclone/rclone/issues/5828)) (Nick Craig-Wood) +.IP \[bu] 2 +Log IO errors at ERROR level (Nick Craig-Wood) +.IP \[bu] 2 +Ignore \f[C]_netdev\f[R] mount argument (Hugal31) +.RE +.IP \[bu] 2 +VFS +.RS 2 +.IP \[bu] 2 +Add \f[C]--vfs-fast-fingerprint\f[R] for less accurate but faster +fingerprints (Nick Craig-Wood) +.IP \[bu] 2 +Add \f[C]--vfs-disk-space-total-size\f[R] option to manually set the +total disk space (Claudio Maradonna) +.IP \[bu] 2 +vfscache: Fix fatal error: sync: unlock of unlocked mutex error (Nick +Craig-Wood) +.RE +.IP \[bu] 2 +Local +.RS 2 +.IP \[bu] 2 +Fix parsing of \f[C]--local-nounc\f[R] flag (Nick Craig-Wood) +.IP \[bu] 2 +Add Metadata support (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Crypt +.RS 2 +.IP \[bu] 2 +Support metadata (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Azure Blob +.RS 2 +.IP \[bu] 2 +Calculate Chunksize/blocksize to stay below maxUploadParts (Leroy van +Logchem) +.IP \[bu] 2 +Use chunksize lib to determine chunksize dynamically (Derek Battams) +.IP \[bu] 2 +Case insensitive access tier (Rob Pickerill) +.IP \[bu] 2 +Allow remote emulator (azurite) (Lorenzo Maiorfi) +.RE +.IP \[bu] 2 +B2 +.RS 2 +.IP \[bu] 2 +Add \f[C]--b2-version-at\f[R] flag to show file versions at time +specified (SwazRGB) +.IP \[bu] 2 +Use chunksize lib to determine chunksize dynamically (Derek Battams) +.RE +.IP \[bu] 2 +Chunker +.RS 2 +.IP \[bu] 2 +Mark as not supporting metadata (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Compress +.RS 2 +.IP \[bu] 2 +Support metadata (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Drive +.RS 2 +.IP \[bu] 2 +Make \f[C]backend config -o config\f[R] add a combined +\f[C]AllDrives:\f[R] remote (Nick Craig-Wood) +.IP \[bu] 2 +Make \f[C]--drive-shared-with-me\f[R] work with shared drives (Nick +Craig-Wood) +.IP \[bu] 2 +Add \f[C]--drive-resource-key\f[R] for accessing link-shared files (Nick +Craig-Wood) +.IP \[bu] 2 +Add backend commands \f[C]exportformats\f[R] and \f[C]importformats\f[R] +for debugging (Nick Craig-Wood) +.IP \[bu] 2 +Fix 404 errors on copy/server side copy objects from public folder (Nick +Craig-Wood) +.IP \[bu] 2 +Update Internal OAuth consent screen docs (Phil Shackleton) +.IP \[bu] 2 +Moved \f[C]root_folder_id\f[R] to advanced section (Abhiraj) +.RE +.IP \[bu] 2 +Dropbox +.RS 2 +.IP \[bu] 2 +Migrate from deprecated api (m8rge) +.IP \[bu] 2 +Add logs to show when poll interval limits are exceeded (Nick +Craig-Wood) +.IP \[bu] 2 +Fix nil pointer exception on dropbox impersonate user not found (Nick +Craig-Wood) +.RE +.IP \[bu] 2 +Fichier +.RS 2 +.IP \[bu] 2 +Parse api error codes and them accordingly (buengese) +.RE +.IP \[bu] 2 +FTP +.RS 2 +.IP \[bu] 2 +Add support for \f[C]disable_utf8\f[R] option (Jason Zheng) +.IP \[bu] 2 +Revert to upstream \f[C]github.com/jlaffaye/ftp\f[R] from our fork (Nick +Craig-Wood) +.RE +.IP \[bu] 2 +Google Cloud Storage +.RS 2 +.IP \[bu] 2 +Add \f[C]--gcs-no-check-bucket\f[R] to minimise transactions and perms +(Nick Gooding) +.IP \[bu] 2 +Add \f[C]--gcs-decompress\f[R] flag to decompress gzip-encoded files +(Nick Craig-Wood) +.RS 2 +.IP \[bu] 2 +by default these will be downloaded compressed (which previously failed) +.RE +.RE +.IP \[bu] 2 +Hasher +.RS 2 +.IP \[bu] 2 +Support metadata (Nick Craig-Wood) +.RE +.IP \[bu] 2 +HTTP +.RS 2 +.IP \[bu] 2 +Fix missing response when using custom auth handler (albertony) +.RE +.IP \[bu] 2 +Jottacloud +.RS 2 +.IP \[bu] 2 +Add support for upload to custom device and mountpoint (albertony) +.IP \[bu] 2 +Always store username in config and use it to avoid initial API request +(albertony) +.IP \[bu] 2 +Fix issue with server-side copy when destination is in trash (albertony) +.IP \[bu] 2 +Fix listing output of remote with special characters (albertony) +.RE +.IP \[bu] 2 +Mailru +.RS 2 +.IP \[bu] 2 +Fix timeout by using int instead of time.Duration for keeping number of +seconds (albertony) +.RE +.IP \[bu] 2 +Mega +.RS 2 +.IP \[bu] 2 +Document using MEGAcmd to help with login failures (Art M. +Gallagher) +.RE +.IP \[bu] 2 +Onedrive +.RS 2 +.IP \[bu] 2 +Implement \f[C]--poll-interval\f[R] for onedrive (Hugo Laloge) +.IP \[bu] 2 +Add access scopes option (Sven Gerber) +.RE +.IP \[bu] 2 +Opendrive +.RS 2 +.IP \[bu] 2 +Resolve lag and truncate bugs (Scott Grimes) +.RE +.IP \[bu] 2 +Pcloud +.RS 2 +.IP \[bu] 2 +Fix about with no free space left (buengese) +.IP \[bu] 2 +Fix cleanup (buengese) +.RE +.IP \[bu] 2 +S3 +.RS 2 +.IP \[bu] 2 +Use PUT Object instead of presigned URLs to upload single part objects +(Nick Craig-Wood) +.IP \[bu] 2 +Backend restore command to skip non-GLACIER objects (Vincent Murphy) +.IP \[bu] 2 +Use chunksize lib to determine chunksize dynamically (Derek Battams) +.IP \[bu] 2 +Retry RequestTimeout errors (Nick Craig-Wood) +.IP \[bu] 2 +Implement reading and writing of metadata (Nick Craig-Wood) +.RE +.IP \[bu] 2 +SFTP +.RS 2 +.IP \[bu] 2 +Add support for about and hashsum on windows server (albertony) +.IP \[bu] 2 +Use vendor-specific VFS statistics extension for about if available +(albertony) +.IP \[bu] 2 +Add \f[C]--sftp-chunk-size\f[R] to control packets sizes for high +latency links (Nick Craig-Wood) +.IP \[bu] 2 +Add \f[C]--sftp-concurrency\f[R] to improve high latency transfers (Nick +Craig-Wood) +.IP \[bu] 2 +Add \f[C]--sftp-set-env\f[R] option to set environment variables (Nick +Craig-Wood) +.IP \[bu] 2 +Add Hetzner Storage Boxes to supported sftp backends (Anthrazz) +.RE +.IP \[bu] 2 +Storj +.RS 2 +.IP \[bu] 2 +Fix put which lead to the file being unreadable when using mount (Erik +van Velzen) +.RE +.IP \[bu] 2 +Union +.RS 2 +.IP \[bu] 2 +Add \f[C]min_free_space\f[R] option for \f[C]lfs\f[R]/\f[C]eplfs\f[R] +policies (Nick Craig-Wood) +.IP \[bu] 2 +Fix uploading files to union of all bucket based remotes (Nick +Craig-Wood) +.IP \[bu] 2 +Fix get free space for remotes which don\[aq]t support it (Nick +Craig-Wood) +.IP \[bu] 2 +Fix \f[C]eplus\f[R] policy to select correct entry for existing files +(Nick Craig-Wood) +.IP \[bu] 2 +Support metadata (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Uptobox +.RS 2 +.IP \[bu] 2 +Fix root path handling (buengese) +.RE +.IP \[bu] 2 +WebDAV +.RS 2 +.IP \[bu] 2 +Add SharePoint in other specific regions support (Noah Hsu) +.RE +.IP \[bu] 2 +Yandex +.RS 2 +.IP \[bu] 2 +Handle api error on server-side move (albertony) +.RE +.IP \[bu] 2 +Zoho +.RS 2 +.IP \[bu] 2 +Add Japan and China regions (buengese) +.RE +.SS v1.58.1 - 2022-04-29 +.PP +See commits (https://github.com/rclone/rclone/compare/v1.58.0...v1.58.1) +.IP \[bu] 2 +Bug Fixes +.RS 2 +.IP \[bu] 2 +build: Update github.com/billziss-gh to github.com/winfsp (Nick +Craig-Wood) +.IP \[bu] 2 +filter: Fix timezone of \f[C]--min-age\f[R]/\f[C]-max-age\f[R] from UTC +to local as documented (Nick Craig-Wood) +.IP \[bu] 2 +rc/js: Correct RC method names (S\[u01A1]n Tr\[u1EA7]n-Nguy\[u1EC5]n) +.IP \[bu] 2 +docs +.RS 2 +.IP \[bu] 2 +Fix some links to command pages (albertony) +.IP \[bu] 2 +Add \f[C]--multi-thread-streams\f[R] note to \f[C]--transfers\f[R]. +(Zsolt Ero) +.RE +.RE +.IP \[bu] 2 +Mount +.RS 2 +.IP \[bu] 2 +Fix \f[C]--devname\f[R] and fusermount: unknown option \[aq]fsname\[aq] +when mounting via rc (Nick Craig-Wood) +.RE +.IP \[bu] 2 +VFS +.RS 2 +.IP \[bu] 2 +Remove wording which suggests VFS is only for mounting (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Dropbox +.RS 2 +.IP \[bu] 2 +Fix retries of multipart uploads with incorrect_offset error (Nick +Craig-Wood) +.RE +.IP \[bu] 2 +Google Cloud Storage +.RS 2 +.IP \[bu] 2 +Use the s3 pacer to speed up transactions (Nick Craig-Wood) +.IP \[bu] 2 +pacer: Default the Google pacer to a burst of 100 to fix gcs pacing +(Nick Craig-Wood) +.RE +.IP \[bu] 2 +Jottacloud +.RS 2 +.IP \[bu] 2 +Fix scope in token request (albertony) +.RE +.IP \[bu] 2 +Netstorage +.RS 2 +.IP \[bu] 2 +Fix unescaped HTML in documentation (Nick Craig-Wood) +.IP \[bu] 2 +Make levels of headings consistent (Nick Craig-Wood) +.IP \[bu] 2 +Add support contacts to netstorage doc (Nil Alexandrov) +.RE +.IP \[bu] 2 +Onedrive +.RS 2 +.IP \[bu] 2 +Note that sharepoint also changes web files (.html, .aspx) (GH) +.RE +.IP \[bu] 2 +Putio +.RS 2 +.IP \[bu] 2 +Handle rate limit errors (Berkan Teber) +.IP \[bu] 2 +Fix multithread download and other ranged requests (rafma0) +.RE +.IP \[bu] 2 +S3 +.RS 2 +.IP \[bu] 2 +Add ChinaMobile EOS to provider list (GuoXingbin) +.IP \[bu] 2 +Sync providers in config description with providers (Nick Craig-Wood) +.RE +.IP \[bu] 2 +SFTP +.RS 2 +.IP \[bu] 2 +Fix OpenSSH 8.8+ RSA keys incompatibility (KARBOWSKI Piotr) +.IP \[bu] 2 +Note that Scaleway C14 is deprecating SFTP in favor of S3 (Adrien +Rey-Jarthon) +.RE +.IP \[bu] 2 +Storj +.RS 2 +.IP \[bu] 2 +Fix bucket creation on Move (Nick Craig-Wood) +.RE +.IP \[bu] 2 +WebDAV +.RS 2 +.IP \[bu] 2 +Don\[aq]t override Referer if user sets it (Nick Craig-Wood) +.RE .SS v1.58.0 - 2022-03-18 .PP See commits (https://github.com/rclone/rclone/compare/v1.57.0...v1.58.0) @@ -58802,7 +64570,7 @@ node running rclone would need to have lots of bandwidth. .PP The syncs would be incremental (on a file by file basis). .PP -Eg +e.g. .IP .nf \f[C] @@ -58912,7 +64680,7 @@ export NO_PROXY=$no_proxy \f[R] .fi .PP -Note that the ftp backend does not support \f[C]ftp_proxy\f[R] yet. +Note that the FTP backend does not support \f[C]ftp_proxy\f[R] yet. .SS Rclone gives x509: failed to load system roots and no roots provided error .PP This means that \f[C]rclone\f[R] can\[aq]t find the SSL root @@ -60206,6 +65974,109 @@ Vincent Murphy ctrl-q <34975747+ctrl-q@users.noreply.github.com> .IP \[bu] 2 Nil Alexandrov +.IP \[bu] 2 +GuoXingbin <101376330+guoxingbin@users.noreply.github.com> +.IP \[bu] 2 +Berkan Teber +.IP \[bu] 2 +Tobias Klauser +.IP \[bu] 2 +KARBOWSKI Piotr +.IP \[bu] 2 +GH +.IP \[bu] 2 +rafma0 +.IP \[bu] 2 +Adrien Rey-Jarthon +.IP \[bu] 2 +Nick Gooding <73336146+nickgooding@users.noreply.github.com> +.IP \[bu] 2 +Leroy van Logchem +.IP \[bu] 2 +Zsolt Ero +.IP \[bu] 2 +Lesmiscore +.IP \[bu] 2 +ehsantdy +.IP \[bu] 2 +SwazRGB <65694696+swazrgb@users.noreply.github.com> +.IP \[bu] 2 +Mateusz Puczyn\[u0301]ski +.IP \[bu] 2 +Michael C Tiernan - MIT-Research Computing Project +.IP \[bu] 2 +Kaspian <34658474+KaspianDev@users.noreply.github.com> +.IP \[bu] 2 +Werner +.IP \[bu] 2 +Hugal31 +.IP \[bu] 2 +Christian Galo <36752715+cgalo5758@users.noreply.github.com> +.IP \[bu] 2 +Erik van Velzen +.IP \[bu] 2 +Derek Battams +.IP \[bu] 2 +SimonLiu +.IP \[bu] 2 +Hugo Laloge +.IP \[bu] 2 +Mr-Kanister <68117355+Mr-Kanister@users.noreply.github.com> +.IP \[bu] 2 +Rob Pickerill +.IP \[bu] 2 +Andrey +.IP \[bu] 2 +Eric Wolf <19wolf@gmail.com> +.IP \[bu] 2 +Nick +.IP \[bu] 2 +Jason Zheng +.IP \[bu] 2 +Matthew Vernon +.IP \[bu] 2 +Noah Hsu +.IP \[bu] 2 +m00594701 +.IP \[bu] 2 +Art M. +Gallagher +.IP \[bu] 2 +Sven Gerber <49589423+svengerber@users.noreply.github.com> +.IP \[bu] 2 +CrossR +.IP \[bu] 2 +Maciej Radzikowski +.IP \[bu] 2 +Scott Grimes +.IP \[bu] 2 +Phil Shackleton <71221528+philshacks@users.noreply.github.com> +.IP \[bu] 2 +eNV25 +.IP \[bu] 2 +Caleb +.IP \[bu] 2 +J-P Treen +.IP \[bu] 2 +Martin Czygan <53705+miku@users.noreply.github.com> +.IP \[bu] 2 +buda +.IP \[bu] 2 +mirekphd <36706320+mirekphd@users.noreply.github.com> +.IP \[bu] 2 +vyloy +.IP \[bu] 2 +Anthrazz <25553648+Anthrazz@users.noreply.github.com> +.IP \[bu] 2 +zzr93 <34027824+zzr93@users.noreply.github.com> +.IP \[bu] 2 +Paul Norman +.IP \[bu] 2 +Lorenzo Maiorfi +.IP \[bu] 2 +Claudio Maradonna +.IP \[bu] 2 +Ovidiu Victor Tatar .SH Contact the rclone project .SS Forum .PP From f5be1d6b65daee33a811308fef6171721fc56f32 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 9 Jul 2022 20:43:17 +0100 Subject: [PATCH 173/560] Start v1.60.0-DEV development --- VERSION | 2 +- docs/layouts/partials/version.html | 2 +- fs/versiontag.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 2abfc1f86ca0a..9dd0095850226 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.59.0 +v1.60.0 diff --git a/docs/layouts/partials/version.html b/docs/layouts/partials/version.html index 28a862658954e..514c727161d3e 100644 --- a/docs/layouts/partials/version.html +++ b/docs/layouts/partials/version.html @@ -1 +1 @@ -v1.59.0 \ No newline at end of file +v1.60.0 \ No newline at end of file diff --git a/fs/versiontag.go b/fs/versiontag.go index 714e98978ce6f..a9bc5f0e7ad82 100644 --- a/fs/versiontag.go +++ b/fs/versiontag.go @@ -1,4 +1,4 @@ package fs // VersionTag of rclone -var VersionTag = "v1.59.0" +var VersionTag = "v1.60.0" From f753d7cd42ee262530fe99543cc81b9e741a7223 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 10 Jul 2022 15:34:48 +0100 Subject: [PATCH 174/560] combine: fix docs showing `remote=` instead of `upstreams=` See: https://forum.rclone.org/t/v1-59-combine-qs/31814 --- backend/drive/drive.go | 2 +- docs/content/combine.md | 2 +- docs/content/drive.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index baf2be267e839..4b909d404f4be 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -3299,7 +3299,7 @@ drives found and a combined drive. [AllDrives] type = combine - remote = "My Drive=My Drive:" "Test Drive=Test Drive:" + upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:" Adding this to the rclone config file will cause those team drives to be accessible with the aliases shown. Any illegal charactes will be diff --git a/docs/content/combine.md b/docs/content/combine.md index c3b5ed322a5d7..f5d3c2da3d043 100644 --- a/docs/content/combine.md +++ b/docs/content/combine.md @@ -116,7 +116,7 @@ This would produce something like this: [AllDrives] type = combine - remote = "My Drive=My Drive:" "Test Drive=Test Drive:" + upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:" If you then add that config to your config file (find it with `rclone config file`) then you can access all the shared drives in one place diff --git a/docs/content/drive.md b/docs/content/drive.md index a4fdf8ccb7f0f..7bb23ed595020 100644 --- a/docs/content/drive.md +++ b/docs/content/drive.md @@ -1332,7 +1332,7 @@ drives found and a combined drive. [AllDrives] type = combine - remote = "My Drive=My Drive:" "Test Drive=Test Drive:" + upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:" Adding this to the rclone config file will cause those team drives to be accessible with the aliases shown. Any illegal charactes will be From 9c6cfc1ff0412b75592067e9017bf36a0e555078 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 10 Jul 2022 15:40:30 +0100 Subject: [PATCH 175/560] combine: throw error if duplicate directory name is specified See: https://forum.rclone.org/t/v1-59-combine-qs/31814 --- backend/combine/combine.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/combine/combine.go b/backend/combine/combine.go index 36fdc69c6ea56..a65192647e296 100644 --- a/backend/combine/combine.go +++ b/backend/combine/combine.go @@ -206,9 +206,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs return err } mu.Lock() - f.upstreams[dir] = u + if _, found := f.upstreams[dir]; found { + err = fmt.Errorf("duplicate directory name %q", dir) + } else { + f.upstreams[dir] = u + } mu.Unlock() - return nil + return err }) } err = g.Wait() From 55bbff6346bb2a1ee3798844860a92fee7636918 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 4 Jul 2022 16:26:08 +0100 Subject: [PATCH 176/560] operations: add --server-side-across-configs global flag for any backend --- docs/content/docs.md | 12 ++ fs/config.go | 181 ++++++++++++++------------- fs/config/configflags/configflags.go | 1 + fs/operations/operations.go | 5 +- 4 files changed, 107 insertions(+), 92 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index a71728cc7e98a..cefa65c232e4d 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -1643,6 +1643,18 @@ This sets the interval between each retry specified by `--retries` The default is `0`. Use `0` to disable. +### --server-side-across-configs ### + +Allow server-side operations (e.g. copy or move) to work across +different configurations. + +This can be useful if you wish to do a server-side copy or move +between two remotes which use the same backend but are configured +differently. + +Note that this isn't enabled by default because it isn't easy for +rclone to tell if it will work between any two configurations. + ### --size-only ### Normally rclone will look at modification time and size of files to diff --git a/fs/config.go b/fs/config.go index a39779bae7b4d..b253a5c30feb4 100644 --- a/fs/config.go +++ b/fs/config.go @@ -45,96 +45,97 @@ var ( // ConfigInfo is filesystem config options type ConfigInfo struct { - LogLevel LogLevel - StatsLogLevel LogLevel - UseJSONLog bool - DryRun bool - Interactive bool - CheckSum bool - SizeOnly bool - IgnoreTimes bool - IgnoreExisting bool - IgnoreErrors bool - ModifyWindow time.Duration - Checkers int - Transfers int - ConnectTimeout time.Duration // Connect timeout - Timeout time.Duration // Data channel timeout - ExpectContinueTimeout time.Duration - Dump DumpFlags - InsecureSkipVerify bool // Skip server certificate verification - DeleteMode DeleteMode - MaxDelete int64 - TrackRenames bool // Track file renames. - TrackRenamesStrategy string // Comma separated list of strategies used to track renames - LowLevelRetries int - UpdateOlder bool // Skip files that are newer on the destination - NoGzip bool // Disable compression - MaxDepth int - IgnoreSize bool - IgnoreChecksum bool - IgnoreCaseSync bool - NoTraverse bool - CheckFirst bool - NoCheckDest bool - NoUnicodeNormalization bool - NoUpdateModTime bool - DataRateUnit string - CompareDest []string - CopyDest []string - BackupDir string - Suffix string - SuffixKeepExtension bool - UseListR bool - BufferSize SizeSuffix - BwLimit BwTimetable - BwLimitFile BwTimetable - TPSLimit float64 - TPSLimitBurst int - BindAddr net.IP - DisableFeatures []string - UserAgent string - Immutable bool - AutoConfirm bool - StreamingUploadCutoff SizeSuffix - StatsFileNameLength int - AskPassword bool - PasswordCommand SpaceSepList - UseServerModTime bool - MaxTransfer SizeSuffix - MaxDuration time.Duration - CutoffMode CutoffMode - MaxBacklog int - MaxStatsGroups int - StatsOneLine bool - StatsOneLineDate bool // If we want a date prefix at all - StatsOneLineDateFormat string // If we want to customize the prefix - ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred - Progress bool - ProgressTerminalTitle bool - Cookie bool - UseMmap bool - CaCert string // Client Side CA - ClientCert string // Client Side Cert - ClientKey string // Client Side Key - MultiThreadCutoff SizeSuffix - MultiThreadStreams int - MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags) - OrderBy string // instructions on how to order the transfer - UploadHeaders []*HTTPOption - DownloadHeaders []*HTTPOption - Headers []*HTTPOption - MetadataSet Metadata // extra metadata to write when uploading - RefreshTimes bool - NoConsole bool - TrafficClass uint8 - FsCacheExpireDuration time.Duration - FsCacheExpireInterval time.Duration - DisableHTTP2 bool - HumanReadable bool - KvLockTime time.Duration // maximum time to keep key-value database locked by process - DisableHTTPKeepAlives bool - Metadata bool + LogLevel LogLevel + StatsLogLevel LogLevel + UseJSONLog bool + DryRun bool + Interactive bool + CheckSum bool + SizeOnly bool + IgnoreTimes bool + IgnoreExisting bool + IgnoreErrors bool + ModifyWindow time.Duration + Checkers int + Transfers int + ConnectTimeout time.Duration // Connect timeout + Timeout time.Duration // Data channel timeout + ExpectContinueTimeout time.Duration + Dump DumpFlags + InsecureSkipVerify bool // Skip server certificate verification + DeleteMode DeleteMode + MaxDelete int64 + TrackRenames bool // Track file renames. + TrackRenamesStrategy string // Comma separated list of strategies used to track renames + LowLevelRetries int + UpdateOlder bool // Skip files that are newer on the destination + NoGzip bool // Disable compression + MaxDepth int + IgnoreSize bool + IgnoreChecksum bool + IgnoreCaseSync bool + NoTraverse bool + CheckFirst bool + NoCheckDest bool + NoUnicodeNormalization bool + NoUpdateModTime bool + DataRateUnit string + CompareDest []string + CopyDest []string + BackupDir string + Suffix string + SuffixKeepExtension bool + UseListR bool + BufferSize SizeSuffix + BwLimit BwTimetable + BwLimitFile BwTimetable + TPSLimit float64 + TPSLimitBurst int + BindAddr net.IP + DisableFeatures []string + UserAgent string + Immutable bool + AutoConfirm bool + StreamingUploadCutoff SizeSuffix + StatsFileNameLength int + AskPassword bool + PasswordCommand SpaceSepList + UseServerModTime bool + MaxTransfer SizeSuffix + MaxDuration time.Duration + CutoffMode CutoffMode + MaxBacklog int + MaxStatsGroups int + StatsOneLine bool + StatsOneLineDate bool // If we want a date prefix at all + StatsOneLineDateFormat string // If we want to customize the prefix + ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred + Progress bool + ProgressTerminalTitle bool + Cookie bool + UseMmap bool + CaCert string // Client Side CA + ClientCert string // Client Side Cert + ClientKey string // Client Side Key + MultiThreadCutoff SizeSuffix + MultiThreadStreams int + MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags) + OrderBy string // instructions on how to order the transfer + UploadHeaders []*HTTPOption + DownloadHeaders []*HTTPOption + Headers []*HTTPOption + MetadataSet Metadata // extra metadata to write when uploading + RefreshTimes bool + NoConsole bool + TrafficClass uint8 + FsCacheExpireDuration time.Duration + FsCacheExpireInterval time.Duration + DisableHTTP2 bool + HumanReadable bool + KvLockTime time.Duration // maximum time to keep key-value database locked by process + DisableHTTPKeepAlives bool + Metadata bool + ServerSideAcrossConfigs bool } // NewConfig creates a new config with everything set to the default diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 61aa65f91fe97..42b2b4d05ec75 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -141,6 +141,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { flags.DurationVarP(flagSet, &ci.KvLockTime, "kv-lock-time", "", ci.KvLockTime, "Maximum time to keep key-value database locked by process") flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.") flags.BoolVarP(flagSet, &ci.Metadata, "metadata", "M", ci.Metadata, "If set, preserve metadata when copying objects") + flags.BoolVarP(flagSet, &ci.ServerSideAcrossConfigs, "server-side-across-configs", "", ci.ServerSideAcrossConfigs, "Allow server-side operations (e.g. copy) to work across different configs") } // ParseHeaders converts the strings passed in via the header flags into HTTPOptions diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 0b0f9f75e01d9..fc275c8944c1d 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -424,7 +424,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj return nil, accounting.ErrorMaxTransferLimitReachedGraceful } } - if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && f.Features().ServerSideAcrossConfigs)) { + if doCopy := f.Features().Copy; doCopy != nil && (SameConfig(src.Fs(), f) || (SameRemoteType(src.Fs(), f) && (f.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) { in := tr.Account(ctx, nil) // account the transfer in.ServerSideCopyStart() newDst, err = doCopy(ctx, src, remote) @@ -604,6 +604,7 @@ func SameObject(src, dst fs.Object) bool { // It returns the destination object if possible. Note that this may // be nil. func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) { + ci := fs.GetConfig(ctx) tr := accounting.Stats(ctx).NewCheckingTransfer(src) defer func() { if err == nil { @@ -618,7 +619,7 @@ func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs. return newDst, nil } // See if we have Move available - if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && fdst.Features().ServerSideAcrossConfigs)) { + if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) { // Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive) if dst != nil && !SameObject(src, dst) { err = DeleteFile(ctx, dst) From 0ee0812a2bea94295318e962bb4bbd0877b035bd Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 9 Jul 2022 21:13:06 +0100 Subject: [PATCH 177/560] union: fix duplicated files when using directories with leading / See: https://forum.rclone.org/t/union-backend-copying-to-all-remotes-while-it-shouldnt/31781 --- backend/union/union.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/union/union.go b/backend/union/union.go index c93fc06497927..fc78e20c3d869 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -834,6 +834,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e } } + root = strings.Trim(root, "/") upstreams := make([]*upstream.Fs, len(opt.Upstreams)) errs := Errors(make([]error, len(opt.Upstreams))) multithread(len(opt.Upstreams), func(i int) { From b310490fa5976b0f3e7b3ee9e323f88764687a5b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 9 Jul 2022 21:17:47 +0100 Subject: [PATCH 178/560] union: fix multiple files being uploaded when roots don't exist See: https://forum.rclone.org/t/union-backend-copying-to-all-remotes-while-it-shouldnt/31781 --- backend/union/union.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/union/union.go b/backend/union/union.go index fc78e20c3d869..0e894a50b063c 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -169,7 +169,11 @@ func (f *Fs) mkdir(ctx context.Context, dir string) ([]*upstream.Fs, error) { if err != nil { return nil, err } - return upstreams, nil + // If created roots then choose one + if dir == "" { + upstreams, err = f.create(ctx, dir) + } + return upstreams, err } // Mkdir makes the root directory of the Fs object From 67fd60275a6187c41c4944545c2145cd28bc330d Mon Sep 17 00:00:00 2001 From: r-ricci Date: Mon, 11 Jul 2022 18:34:06 +0100 Subject: [PATCH 179/560] union: fix panic due to misalignment of struct field in 32 bit architectures `FS.cacheExpiry` is accessed through sync/atomic. According to the documentation, "On ARM, 386, and 32-bit MIPS, it is the caller's responsibility to arrange for 64-bit alignment of 64-bit words accessed atomically. The first word in a variable or in an allocated struct, array, or slice can be relied upon to be 64-bit aligned." Before commit 1d2fe0d8564bc679ece166c24b24e6fe7dc1455c this field was aligned, but then a new field was added to the structure, causing the test suite to panic on linux/386. No other field is used with sync/atomic, so `cacheExpiry` can just be placed at the beginning of the stuct to ensure it is always aligned. --- backend/union/upstream/upstream.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go index 35b477f7930ab..7b9f44fd2a39d 100644 --- a/backend/union/upstream/upstream.go +++ b/backend/union/upstream/upstream.go @@ -24,6 +24,10 @@ var ( // Fs is a wrap of any fs and its configs type Fs struct { + // In order to ensure memory alignment on 32-bit architectures + // when this field is accessed through sync/atomic functions, + // it must be the first entry in the struct + cacheExpiry int64 // usage cache expiry time fs.Fs RootFs fs.Fs RootPath string @@ -32,7 +36,6 @@ type Fs struct { creatable bool usage *fs.Usage // Cache the usage cacheTime time.Duration // cache duration - cacheExpiry int64 // usage cache expiry time cacheMutex sync.RWMutex cacheOnce sync.Once cacheUpdate bool // if the cache is updating From 8c19b355a5ae8949821394865f6d13c6069a3d53 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 13 Jul 2022 12:29:32 +0200 Subject: [PATCH 180/560] docs: fix links to mount command from install docs --- docs/content/install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/install.md b/docs/content/install.md index 07d343d38cb25..f6fc8a55fa176 100644 --- a/docs/content/install.md +++ b/docs/content/install.md @@ -318,7 +318,7 @@ such as a regular [sync](https://rclone.org/commands/rclone_sync/), you will pro to configure your rclone command in your operating system's scheduler. If you need to expose *service*-like features, such as [remote control](https://rclone.org/rc/), [GUI](https://rclone.org/gui/), [serve](https://rclone.org/commands/rclone_serve/) -or [mount](https://rclone.org/commands/rclone_move/), you will often want an rclone +or [mount](https://rclone.org/commands/rclone_mount/), you will often want an rclone command always running in the background, and configuring it to run in a service infrastructure may be a better option. Below are some alternatives on how to achieve this on different operating systems. @@ -351,7 +351,7 @@ c:\rclone\rclone.exe sync c:\files remote:/files --no-console --log-file c:\rclo #### User account -As mentioned in the [mount](https://rclone.org/commands/rclone_move/) documentation, +As mentioned in the [mount](https://rclone.org/commands/rclone_mount/) documentation, mounted drives created as Administrator are not visible to other accounts, not even the account that was elevated as Administrator. By running the mount command as the built-in `SYSTEM` user account, it will create drives accessible for everyone on From 11be920e90ec2c87e1db96d3c5534963c1fa6118 Mon Sep 17 00:00:00 2001 From: Nick Naumann Date: Mon, 11 Jul 2022 23:26:50 +0200 Subject: [PATCH 181/560] sync: add filter-sensitivity to --backup-dir option The old Overlapping function and corresponding tests have been removed, as it has been completely replaced by the OverlappingFilterCheck function. --- fs/operations/operations.go | 15 ++------------- fs/operations/operations_test.go | 29 ----------------------------- 2 files changed, 2 insertions(+), 42 deletions(-) diff --git a/fs/operations/operations.go b/fs/operations/operations.go index fc275c8944c1d..a02aead24c99a 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -815,17 +815,6 @@ func fixRoot(f fs.Info) string { return s } -// Overlapping returns true if fdst and fsrc point to the same -// underlying Fs and they overlap. -func Overlapping(fdst, fsrc fs.Info) bool { - if !SameConfig(fdst, fsrc) { - return false - } - fdstRoot := fixRoot(fdst) - fsrcRoot := fixRoot(fsrc) - return strings.HasPrefix(fdstRoot, fsrcRoot) || strings.HasPrefix(fsrcRoot, fdstRoot) -} - // OverlappingFilterCheck returns true if fdst and fsrc point to the same // underlying Fs and they overlap without fdst being excluded by any filter rule. func OverlappingFilterCheck(ctx context.Context, fdst fs.Fs, fsrc fs.Fs) bool { @@ -1849,10 +1838,10 @@ func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string) return nil, fserrors.FatalError(errors.New("parameter to --backup-dir has to be on the same remote as destination")) } if srcFileName == "" { - if Overlapping(fdst, backupDir) { + if OverlappingFilterCheck(ctx, backupDir, fdst) { return nil, fserrors.FatalError(errors.New("destination and parameter to --backup-dir mustn't overlap")) } - if Overlapping(fsrc, backupDir) { + if OverlappingFilterCheck(ctx, backupDir, fsrc) { return nil, fserrors.FatalError(errors.New("source and parameter to --backup-dir mustn't overlap")) } } else { diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go index 34b88d6d14c03..20fcc61a025ad 100644 --- a/fs/operations/operations_test.go +++ b/fs/operations/operations_test.go @@ -1243,35 +1243,6 @@ func TestSame(t *testing.T) { } } -func TestOverlapping(t *testing.T) { - a := &testFsInfo{name: "name", root: "root"} - slash := string(os.PathSeparator) // native path separator - for _, test := range []struct { - name string - root string - expected bool - }{ - {"name", "root", true}, - {"namey", "root", false}, - {"name", "rooty", false}, - {"namey", "rooty", false}, - {"name", "roo", false}, - {"name", "root/toot", true}, - {"name", "root/toot/", true}, - {"name", "root" + slash + "toot", true}, - {"name", "root" + slash + "toot" + slash, true}, - {"name", "", true}, - {"name", "/", true}, - } { - b := &testFsInfo{name: test.name, root: test.root} - what := fmt.Sprintf("(%q,%q) vs (%q,%q)", a.name, a.root, b.name, b.root) - actual := operations.Overlapping(a, b) - assert.Equal(t, test.expected, actual, what) - actual = operations.Overlapping(b, a) - assert.Equal(t, test.expected, actual, what) - } -} - // testFs is for unit testing fs.Fs type testFs struct { testFsInfo From 8c02fe7b89a284156f2ebf04901d78769b6f9f43 Mon Sep 17 00:00:00 2001 From: Nick Naumann Date: Mon, 11 Jul 2022 23:31:30 +0200 Subject: [PATCH 182/560] sync: update docs and error messages to reflect fixes to overlap checks --- cmd/sync/sync.go | 5 +++++ docs/content/docs.md | 3 ++- fs/fs.go | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index c0c1885ba5d5b..be233cdd941ad 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -49,6 +49,11 @@ extended explanation in the [copy](/commands/rclone_copy/) command if unsure. If dest:path doesn't exist, it is created and the source:path contents go there. +It is not possible to sync overlapping remotes. However, you may exclude +the destination from the sync with a filter rule or by putting an +exclude-if-present file inside the destination directory and sync to a +destination that is inside the source directory. + **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics **Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors. diff --git a/docs/content/docs.md b/docs/content/docs.md index cefa65c232e4d..ed58947269d00 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -582,7 +582,8 @@ been added) in DIR, then it will be overwritten. The remote in use must support server-side move or copy and you must use the same remote as the destination of the sync. The backup -directory must not overlap the destination directory. +directory must not overlap the destination directory without it being +excluded by a filter rule. For example diff --git a/fs/fs.go b/fs/fs.go index 65d9c17d4b945..fe68517c4f31e 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -40,7 +40,7 @@ var ( ErrorNotAFile = errors.New("is not a regular file") ErrorNotDeleting = errors.New("not deleting files as there were IO errors") ErrorNotDeletingDirs = errors.New("not deleting directories as there were IO errors") - ErrorOverlapping = errors.New("can't sync or move files on overlapping remotes") + ErrorOverlapping = errors.New("can't sync or move files on overlapping remotes (try excluding the destination with a filter rule)") ErrorDirectoryNotEmpty = errors.New("directory not empty") ErrorImmutableModified = errors.New("immutable file modified") ErrorPermissionDenied = errors.New("permission denied") From 4a4379b3128de7ace692120e1c5ae84754d42047 Mon Sep 17 00:00:00 2001 From: Evan Spensley Date: Sun, 10 Jul 2022 10:42:04 -0400 Subject: [PATCH 183/560] jobs: add ability to stop group Adds new rc call to stop all running jobs in a group. Fixes #5561 --- fs/rc/jobs/job.go | 31 +++++++++++++++++++++++++++++++ fs/rc/jobs/job_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/fs/rc/jobs/job.go b/fs/rc/jobs/job.go index e452fe241b647..cb3aaa833b371 100644 --- a/fs/rc/jobs/job.go +++ b/fs/rc/jobs/job.go @@ -406,3 +406,34 @@ func rcJobStop(ctx context.Context, in rc.Params) (out rc.Params, err error) { job.Stop() return out, nil } + +func init() { + rc.Add(rc.Call{ + Path: "job/stopgroup", + Fn: rcGroupStop, + Title: "Stop all running jobs in a group", + Help: `Parameters: + +- group - name of the group (string). +`, + }) +} + +// Stops all running jobs in a group +func rcGroupStop(ctx context.Context, in rc.Params) (out rc.Params, err error) { + group, err := in.GetString("group") + if err != nil { + return nil, err + } + running.mu.RLock() + defer running.mu.RUnlock() + for _, job := range running.jobs { + if job.Group == group { + job.mu.Lock() + job.Stop() + job.mu.Unlock() + } + } + out = make(rc.Params) + return out, nil +} diff --git a/fs/rc/jobs/job_test.go b/fs/rc/jobs/job_test.go index cb92e602aeaa8..7552465def22d 100644 --- a/fs/rc/jobs/job_test.go +++ b/fs/rc/jobs/job_test.go @@ -452,6 +452,48 @@ func TestRcSyncJobStop(t *testing.T) { assert.Equal(t, false, out["success"]) } +func TestRcJobStopGroup(t *testing.T) { + ctx := context.Background() + jobID = 0 + _, _, err := NewJob(ctx, ctxFn, rc.Params{ + "_async": true, + "_group": "myparty", + }) + require.NoError(t, err) + _, _, err = NewJob(ctx, ctxFn, rc.Params{ + "_async": true, + "_group": "myparty", + }) + require.NoError(t, err) + + call := rc.Calls.Get("job/stopgroup") + assert.NotNil(t, call) + in := rc.Params{"group": "myparty"} + out, err := call.Fn(context.Background(), in) + require.NoError(t, err) + require.Empty(t, out) + + in = rc.Params{} + _, err = call.Fn(context.Background(), in) + require.Error(t, err) + assert.Contains(t, err.Error(), "Didn't find key") + + time.Sleep(10 * time.Millisecond) + + call = rc.Calls.Get("job/status") + assert.NotNil(t, call) + for i := 1; i <= 2; i++ { + in = rc.Params{"jobid": i} + out, err = call.Fn(context.Background(), in) + require.NoError(t, err) + require.NotNil(t, out) + assert.Equal(t, "myparty", out["group"]) + assert.Equal(t, "context canceled", out["error"]) + assert.Equal(t, true, out["finished"]) + assert.Equal(t, false, out["success"]) + } +} + func TestOnFinish(t *testing.T) { jobID = 0 done := make(chan struct{}) From f46718887660db0fba3082f15daaec8c5c367126 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 14 Jul 2022 20:12:53 +0100 Subject: [PATCH 184/560] Add Evan Spensley to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 0e8e998691095..f60c63a72ea41 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -626,3 +626,4 @@ put them back in again.` >}} * Lorenzo Maiorfi * Claudio Maradonna * Ovidiu Victor Tatar + * Evan Spensley From 140af43c26eadef09bc04cbe8de55ea71d3a2f5e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 13 Jul 2022 15:59:24 +0100 Subject: [PATCH 185/560] build: add 32 bit test runner to avoid problems like #6311 --- .github/workflows/build.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 784a236ccf9c8..4f719794f45d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.16', 'go1.17'] + job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.16', 'go1.17'] include: - job_name: linux @@ -39,6 +39,13 @@ jobs: librclonetest: true deploy: true + - job_name: linux_386 + os: ubuntu-latest + go: '1.18.x' + goarch: 386 + gotags: cmount + quicktest: true + - job_name: mac_amd64 os: macos-11 go: '1.18.x' From 50f053cadaa9d487768fef759829356235f0b35e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 12 Jul 2022 12:13:58 +0100 Subject: [PATCH 186/560] dropbox: fix hang on quit with --dropbox-batch-mode off This problem was created by the fact that we are much more diligent about calling Shutdown now, and the dropbox backend had a hang if the batch mode was "off" in the Shutdown method. See: https://forum.rclone.org/t/dropbox-lsjson-in-1-59-stuck-on-commiting-upload/31853 --- backend/dropbox/batcher.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/dropbox/batcher.go b/backend/dropbox/batcher.go index ac87c71e0d4d8..874bb93d945b4 100644 --- a/backend/dropbox/batcher.go +++ b/backend/dropbox/batcher.go @@ -304,6 +304,9 @@ outer: // // Can be called from atexit handler func (b *batcher) Shutdown() { + if !b.Batching() { + return + } b.shutOnce.Do(func() { atexit.Unregister(b.atexit) fs.Infof(b.f, "Commiting uploads - please wait...") From 049674aeabf234567b76a4c2a42f60e11b1cbc7f Mon Sep 17 00:00:00 2001 From: Lesmiscore Date: Sat, 16 Jul 2022 12:11:09 +0900 Subject: [PATCH 187/560] backend/internetarchive: ignore checksums for files using the different method --- backend/internetarchive/internetarchive.go | 92 ++++++++++++++-------- docs/content/internetarchive.md | 27 ++++--- 2 files changed, 71 insertions(+), 48 deletions(-) diff --git a/backend/internetarchive/internetarchive.go b/backend/internetarchive/internetarchive.go index 9d4dd36dba708..b7a0676b5d201 100644 --- a/backend/internetarchive/internetarchive.go +++ b/backend/internetarchive/internetarchive.go @@ -42,54 +42,70 @@ func init() { MetadataInfo: &fs.MetadataInfo{ System: map[string]fs.MetadataHelp{ "name": { - Help: "Full file path, without the bucket part", - Type: "filename", - Example: "backend/internetarchive/internetarchive.go", + Help: "Full file path, without the bucket part", + Type: "filename", + Example: "backend/internetarchive/internetarchive.go", + ReadOnly: true, }, "source": { - Help: "The source of the file", - Type: "string", - Example: "original", + Help: "The source of the file", + Type: "string", + Example: "original", + ReadOnly: true, }, "mtime": { - Help: "Time of last modification, managed by Rclone", - Type: "RFC 3339", - Example: "2006-01-02T15:04:05.999999999Z", + Help: "Time of last modification, managed by Rclone", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05.999999999Z", + ReadOnly: true, }, "size": { - Help: "File size in bytes", - Type: "decimal number", - Example: "123456", + Help: "File size in bytes", + Type: "decimal number", + Example: "123456", + ReadOnly: true, }, "md5": { - Help: "MD5 hash calculated by Internet Archive", - Type: "string", - Example: "01234567012345670123456701234567", + Help: "MD5 hash calculated by Internet Archive", + Type: "string", + Example: "01234567012345670123456701234567", + ReadOnly: true, }, "crc32": { - Help: "CRC32 calculated by Internet Archive", - Type: "string", - Example: "01234567", + Help: "CRC32 calculated by Internet Archive", + Type: "string", + Example: "01234567", + ReadOnly: true, }, "sha1": { - Help: "SHA1 hash calculated by Internet Archive", - Type: "string", - Example: "0123456701234567012345670123456701234567", + Help: "SHA1 hash calculated by Internet Archive", + Type: "string", + Example: "0123456701234567012345670123456701234567", + ReadOnly: true, }, "format": { - Help: "Name of format identified by Internet Archive", - Type: "string", - Example: "Comma-Separated Values", + Help: "Name of format identified by Internet Archive", + Type: "string", + Example: "Comma-Separated Values", + ReadOnly: true, }, "old_version": { - Help: "Whether the file was replaced and moved by keep-old-version flag", - Type: "boolean", - Example: "true", + Help: "Whether the file was replaced and moved by keep-old-version flag", + Type: "boolean", + Example: "true", + ReadOnly: true, }, "viruscheck": { - Help: "The last time viruscheck process was run for the file (?)", - Type: "unixtime", - Example: "1654191352", + Help: "The last time viruscheck process was run for the file (?)", + Type: "unixtime", + Example: "1654191352", + ReadOnly: true, + }, + "summation": { + Help: "Check https://forum.rclone.org/t/31922 for how it is used", + Type: "string", + Example: "md5", + ReadOnly: true, }, "rclone-ia-mtime": { @@ -173,7 +189,7 @@ var roMetadataKey = map[string]interface{}{ // do not add mtime here, it's a documented exception "name": nil, "source": nil, "size": nil, "md5": nil, "crc32": nil, "sha1": nil, "format": nil, "old_version": nil, - "viruscheck": nil, + "viruscheck": nil, "summation": nil, } // Options defines the configuration for this backend @@ -222,6 +238,7 @@ type IAFile struct { Md5 string `json:"md5"` Crc32 string `json:"crc32"` Sha1 string `json:"sha1"` + Summation string `json:"summation"` rawData json.RawMessage } @@ -1135,16 +1152,21 @@ func (f *Fs) waitDelete(ctx context.Context, bucket, bucketPath string) (err err } func makeValidObject(f *Fs, remote string, file IAFile, mtime time.Time, size int64) *Object { - return &Object{ + ret := &Object{ fs: f, remote: remote, modTime: mtime, size: size, - md5: file.Md5, - crc32: file.Crc32, - sha1: file.Sha1, rawData: file.rawData, } + // hashes from _files.xml (where summation != "") is different from one in other files + // https://forum.rclone.org/t/internet-archive-md5-tag-in-id-files-xml-interpreted-incorrectly/31922 + if file.Summation == "" { + ret.md5 = file.Md5 + ret.crc32 = file.Crc32 + ret.sha1 = file.Sha1 + } + return ret } func makeValidObject2(f *Fs, file IAFile, bucket string) *Object { diff --git a/docs/content/internetarchive.md b/docs/content/internetarchive.md index 1bdb05962713d..ae3393d80f2c0 100644 --- a/docs/content/internetarchive.md +++ b/docs/content/internetarchive.md @@ -12,11 +12,10 @@ Refer to [IAS3 API documentation](https://archive.org/services/docs/api/ias3.htm Paths are specified as `remote:bucket` (or `remote:` for the `lsd` command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`. -Once you have made a remote (see the provider specific section above) -you can use it like this: - Unlike S3, listing up all items uploaded by you isn't supported. +Once you have made a remote, you can use it like this: + Make a new item rclone mkdir remote:item @@ -53,6 +52,7 @@ The following are reserved by Internet Archive: - `format` - `old_version` - `viruscheck` +- `summation` Trying to set values to these keys is ignored with a warning. Only setting `mtime` is an exception. Doing so make it the identical behavior as setting ModTime. @@ -258,19 +258,20 @@ Here are the possible system metadata items for the internetarchive backend. | Name | Help | Type | Example | Read Only | |------|------|------|---------|-----------| -| crc32 | CRC32 calculated by Internet Archive | string | 01234567 | N | -| format | Name of format identified by Internet Archive | string | Comma-Separated Values | N | -| md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | N | -| mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | -| name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | N | -| old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | N | +| crc32 | CRC32 calculated by Internet Archive | string | 01234567 | **Y** | +| format | Name of format identified by Internet Archive | string | Comma-Separated Values | **Y** | +| md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | **Y** | +| mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | **Y** | +| name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | **Y** | +| old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | **Y** | | rclone-ia-mtime | Time of last modification, managed by Internet Archive | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | | rclone-mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N | | rclone-update-track | Random value used by Rclone for tracking changes inside Internet Archive | string | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | N | -| sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | N | -| size | File size in bytes | decimal number | 123456 | N | -| source | The source of the file | string | original | N | -| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | N | +| sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | **Y** | +| size | File size in bytes | decimal number | 123456 | **Y** | +| source | The source of the file | string | original | **Y** | +| summation | Check https://forum.rclone.org/t/31922 for how it is used | string | md5 | **Y** | +| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | **Y** | See the [metadata](/docs/#metadata) docs for more info. From 03d0f331f7dcea7bc610e96ac887e5912764fb66 Mon Sep 17 00:00:00 2001 From: Yen Hu <61753151+0x59656e@users.noreply.github.com> Date: Mon, 18 Jul 2022 00:07:23 +0800 Subject: [PATCH 188/560] onedrive: rename Onedrive(cn) 21Vianet to Vnet Group The old site had shown a redirect page to the new one since 2021-4-21. https://www.21vianet.com The official site had renamed to Vnet Group also. https://www.vnet.com/en/about --- backend/onedrive/onedrive.go | 6 +++--- docs/content/onedrive.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 4e899addc8601..b209b18f473b8 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -118,7 +118,7 @@ func init() { Help: "Microsoft Cloud Germany", }, { Value: regionCN, - Help: "Azure and Office 365 operated by 21Vianet in China", + Help: "Azure and Office 365 operated by Vnet Group in China", }, }, }, { @@ -2184,7 +2184,7 @@ func (o *Object) ID() string { * 3. To avoid region-related issues, please don't manually build rest.Opts from scratch. * Instead, use these helper function, and customize the URL afterwards if needed. * - * currently, the 21ViaNet's API differs in the following places: + * currently, the Vnet Group's API differs in the following places: * - https://{Endpoint}/drives/{driveID}/items/{leaf}:/{route} * - this API doesn't work (gives invalid request) * - can be replaced with the following API: @@ -2233,7 +2233,7 @@ func escapeSingleQuote(str string) string { // newOptsCallWithIDPath build the rest.Opts structure with *a normalizedID (driveID#fileID, or simply fileID) and leaf* // using url template https://{Endpoint}/drives/{driveID}/items/{leaf}:/{route} (for international OneDrive) // or https://{Endpoint}/drives/{driveID}/items/children('{leaf}')/{route} -// and https://{Endpoint}/drives/{driveID}/items/children('@a1')/{route}?@a1=URLEncode("'{leaf}'") (for 21ViaNet) +// and https://{Endpoint}/drives/{driveID}/items/children('@a1')/{route}?@a1=URLEncode("'{leaf}'") (for Vnet Group) // if isPath is false, this function will only work when the leaf is "" or a child name (i.e. it doesn't accept multi-level leaf) // if isPath is true, multi-level leaf like a/b/c can be passed func (f *Fs) newOptsCallWithIDPath(normalizedID string, leaf string, isPath bool, method string, route string) (opts rest.Opts, ok bool) { diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index b3141578753c5..13242910ff8c9 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -263,7 +263,7 @@ Properties: - "de" - Microsoft Cloud Germany - "cn" - - Azure and Office 365 operated by 21Vianet in China + - Azure and Office 365 operated by Vnet Group in China ### Advanced options From a047d30eca9b1570822db3ab7705a5606d15e5dd Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 18 Jul 2022 23:38:39 +0100 Subject: [PATCH 189/560] Add Yen Hu to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index f60c63a72ea41..18e1acddfbaf4 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -627,3 +627,4 @@ put them back in again.` >}} * Claudio Maradonna * Ovidiu Victor Tatar * Evan Spensley + * Yen Hu <61753151+0x59656e@users.noreply.github.com> From 440d0cd1797788c31d2c722f54c85c6f898a2441 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 17 Jul 2022 13:40:45 +0100 Subject: [PATCH 190/560] s3: fix --s3-no-head panic: reflect: Elem of invalid type s3.PutObjectInput In 22abd785eba35ca2 s3: implement reading and writing of metadata #111 The reading information of objects was refactored to use the s3.HeadObjectOutput structure. Unfortunately the code branch with `--s3-no-head` was not tested otherwise this panic would have been discovered. This shows that this is path is not integration tested, so this adds a new integration test. Fixes #6322 --- backend/s3/s3.go | 2 +- backend/s3/s3_internal_test.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 448b7c08f00b4..753e800f0e7f4 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -4570,7 +4570,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // uploaded properly. If size < 0 then we need to do the HEAD. if o.fs.opt.NoHead && size >= 0 { var head s3.HeadObjectOutput - structs.SetFrom(&head, req) + structs.SetFrom(&head, &req) head.ETag = &md5sumHex // doesn't matter quotes are misssing head.ContentLength = &size // If we have done a single part PUT request then we can read these diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go index 2568968d6700d..fcaf8915f2ca6 100644 --- a/backend/s3/s3_internal_test.go +++ b/backend/s3/s3_internal_test.go @@ -67,8 +67,26 @@ func (f *Fs) InternalTestMetadata(t *testing.T) { } } +func (f *Fs) InternalTestNoHead(t *testing.T) { + ctx := context.Background() + // Set NoHead for this test + f.opt.NoHead = true + defer func() { + f.opt.NoHead = false + }() + contents := random.String(1000) + item := fstest.NewItem("test-no-head", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) + obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) + defer func() { + assert.NoError(t, obj.Remove(ctx)) + }() + // PutTestcontests checks the received object + +} + func (f *Fs) InternalTest(t *testing.T) { t.Run("Metadata", f.InternalTestMetadata) + t.Run("NoHead", f.InternalTestNoHead) } var _ fstests.InternalTester = (*Fs)(nil) From 9b76434ad5730a55d8cfbaaf19410067485ee575 Mon Sep 17 00:00:00 2001 From: Steve Kowalik Date: Wed, 20 Jul 2022 19:37:34 +1000 Subject: [PATCH 191/560] drive: make --drive-stop-on-upload-limit obey quota exceeded error Extend the shouldRetry function by also checking for the quotaExceeded reason, and since this function appeared to be untested, add a test case for the existing errors and this new one. Fixes #615 --- backend/drive/drive.go | 3 ++ backend/drive/drive_internal_test.go | 57 ++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 4b909d404f4be..d2f92d6920134 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -758,6 +758,9 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) { } else if f.opt.StopOnDownloadLimit && reason == "downloadQuotaExceeded" { fs.Errorf(f, "Received download limit error: %v", err) return false, fserrors.FatalError(err) + } else if f.opt.StopOnUploadLimit && reason == "quotaExceeded" { + fs.Errorf(f, "Received upload limit error: %v", err) + return false, fserrors.FatalError(err) } else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" { fs.Errorf(f, "Received Shared Drive file limit error: %v", err) return false, fserrors.FatalError(err) diff --git a/backend/drive/drive_internal_test.go b/backend/drive/drive_internal_test.go index 8df52716c3a49..22a84b0bb5f3f 100644 --- a/backend/drive/drive_internal_test.go +++ b/backend/drive/drive_internal_test.go @@ -19,6 +19,7 @@ import ( _ "github.com/rclone/rclone/backend/local" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/sync" @@ -28,6 +29,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/api/drive/v3" + "google.golang.org/api/googleapi" ) func TestDriveScopes(t *testing.T) { @@ -190,6 +192,60 @@ func TestExtensionsForImportFormats(t *testing.T) { } } +func (f *Fs) InternalTestShouldRetry(t *testing.T) { + ctx := context.Background() + gatewayTimeout := googleapi.Error{ + Code: 503, + } + timeoutRetry, timeoutError := f.shouldRetry(ctx, &gatewayTimeout) + assert.True(t, timeoutRetry) + assert.Equal(t, &gatewayTimeout, timeoutError) + generic403 := googleapi.Error{ + Code: 403, + } + rLEItem := googleapi.ErrorItem{ + Reason: "rateLimitExceeded", + Message: "User rate limit exceeded.", + } + generic403.Errors = append(generic403.Errors, rLEItem) + oldStopUpload := f.opt.StopOnUploadLimit + oldStopDownload := f.opt.StopOnDownloadLimit + f.opt.StopOnUploadLimit = true + f.opt.StopOnDownloadLimit = true + defer func() { + f.opt.StopOnUploadLimit = oldStopUpload + f.opt.StopOnDownloadLimit = oldStopDownload + }() + expectedRLError := fserrors.FatalError(&generic403) + rateLimitRetry, rateLimitErr := f.shouldRetry(ctx, &generic403) + assert.False(t, rateLimitRetry) + assert.Equal(t, rateLimitErr, expectedRLError) + dQEItem := googleapi.ErrorItem{ + Reason: "downloadQuotaExceeded", + } + generic403.Errors[0] = dQEItem + expectedDQError := fserrors.FatalError(&generic403) + downloadQuotaRetry, downloadQuotaError := f.shouldRetry(ctx, &generic403) + assert.False(t, downloadQuotaRetry) + assert.Equal(t, downloadQuotaError, expectedDQError) + tDFLEItem := googleapi.ErrorItem{ + Reason: "teamDriveFileLimitExceeded", + } + generic403.Errors[0] = tDFLEItem + expectedTDFLError := fserrors.FatalError(&generic403) + teamDriveFileLimitRetry, teamDriveFileLimitError := f.shouldRetry(ctx, &generic403) + assert.False(t, teamDriveFileLimitRetry) + assert.Equal(t, teamDriveFileLimitError, expectedTDFLError) + qEItem := googleapi.ErrorItem{ + Reason: "quotaExceeded", + } + generic403.Errors[0] = qEItem + expectedQuotaError := fserrors.FatalError(&generic403) + quotaExceededRetry, quotaExceededError := f.shouldRetry(ctx, &generic403) + assert.False(t, quotaExceededRetry) + assert.Equal(t, quotaExceededError, expectedQuotaError) +} + func (f *Fs) InternalTestDocumentImport(t *testing.T) { oldAllow := f.opt.AllowImportNameChange f.opt.AllowImportNameChange = true @@ -545,6 +601,7 @@ func (f *Fs) InternalTest(t *testing.T) { t.Run("UnTrash", f.InternalTestUnTrash) t.Run("CopyID", f.InternalTestCopyID) t.Run("AgeQuery", f.InternalTestAgeQuery) + t.Run("ShouldRetry", f.InternalTestShouldRetry) } var _ fstests.InternalTester = (*Fs)(nil) From f1166757ba3cc0af744344f0010b1e7ed5383619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20Gonzalez=20Mu=C3=B1oz?= Date: Wed, 20 Jul 2022 18:20:12 +0200 Subject: [PATCH 192/560] librclone: add PHP bindings and test program --- librclone/README.md | 8 ++++++ librclone/php/rclone.php | 53 ++++++++++++++++++++++++++++++++++++++ librclone/php/test.php | 55 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 librclone/php/rclone.php create mode 100644 librclone/php/test.php diff --git a/librclone/README.md b/librclone/README.md index 77d3a86c54f8d..45d5228283077 100644 --- a/librclone/README.md +++ b/librclone/README.md @@ -217,6 +217,14 @@ This needs expanding and submitting to pypi... Rust bindings are available in the `librclone` crate: https://crates.io/crates/librclone +## PHP + +The `php` subdirectory contains how to use the C library librclone in php through foreign +function interface (FFI). + +Useful docs: +- [PHP / FFI](https://www.php.net/manual/en/book.ffi.php) + ## TODO - Async jobs must currently be cancelled manually at the moment - RcloneFinalize doesn't do it. diff --git a/librclone/php/rclone.php b/librclone/php/rclone.php new file mode 100644 index 0000000000000..485e27dc51de6 --- /dev/null +++ b/librclone/php/rclone.php @@ -0,0 +1,53 @@ +rpc( "config/listremotes", "{}" ); + +When finished, close it + + $rc->close(); +*/ + +class Rclone { + + protected $rclone; + private $out; + + public function __construct( $libshared ) + { + $this->rclone = \FFI::cdef(" + struct RcloneRPCResult { + char* Output; + int Status; + }; + extern void RcloneInitialize(); + extern void RcloneFinalize(); + extern struct RcloneRPCResult RcloneRPC(char* method, char* input); + extern void RcloneFreeString(char* str); + ", $libshared); + $this->rclone->RcloneInitialize(); + } + + public function rpc( $method, $input ): array + { + $this->out = $this->rclone->RcloneRPC( $method, $input ); + $response = [ + 'output' => \FFI::string( $this->out->Output ), + 'status' => $this->out->Status + ]; + $this->rclone->RcloneFreeString( $this->out->Output ); + return $response; + } + + public function close( ): void + { + $this->rclone->RcloneFinalize(); + } +} diff --git a/librclone/php/test.php b/librclone/php/test.php new file mode 100644 index 0000000000000..0de0f94adb552 --- /dev/null +++ b/librclone/php/test.php @@ -0,0 +1,55 @@ +rpc( "config/listremotes", "{}" ); +print_r( $response ); + +$response = $rc->rpc("operations/mkdir", + json_encode( [ + 'fs' => REMOTE, + 'remote'=> FOLDER + ])); +print_r( $response ); + +$response = $rc->rpc("operations/list", + json_encode( [ + 'fs' => REMOTE, + 'remote'=> '' + ])); +print_r( $response ); + +file_put_contents("./" . FILE, "Success!!!"); +$response = $rc->rpc("operations/copyfile", + json_encode( [ + 'srcFs' => getcwd(), + 'srcRemote'=> FILE, + 'dstFs' => REMOTE . FOLDER, + 'dstRemote' => FILE + ])); +print_r( $response ); + +$response = $rc->rpc("operations/list", + json_encode( [ + 'fs' => REMOTE . FOLDER, + 'remote'=> '' + ])); +print_r( $response ); +if ( $response['output'] ) { + $array = @json_decode( $response['output'], true ); + if ( $response['status'] == 200 && $array['list'] ?? 0 ) { + $valid = $array['list'][0]['Name'] == FILE ? "SUCCESS" : "FAIL"; + print_r("The test seems: " . $valid . "\n"); + } +} + +$rc->close(); From f0396070ebc24762c2f90a91835aa90367aa2872 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:15:50 +0200 Subject: [PATCH 193/560] sftp: fix issue with WS_FTP by working around failing RealPath --- backend/sftp/sftp.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index bb5add94ddc7c..6a4259542096a 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -935,11 +935,22 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m // It appears that WS FTP doesn't like relative paths, // and the openssh sftp tool also uses absolute paths. if !path.IsAbs(f.root) { - path, err := c.sftpClient.RealPath(f.root) + // Trying RealPath first, to perform proper server-side canonicalize. + // It may fail (SSH_FX_FAILURE reported on WS FTP) and will then resort + // to simple path join with current directory from Getwd (which can work + // on WS FTP, even though it is also based on RealPath). + absRoot, err := c.sftpClient.RealPath(f.root) if err != nil { - fs.Debugf(f, "Failed to resolve path - using relative paths: %v", err) + fs.Debugf(f, "Failed to resolve path using RealPath: %v", err) + cwd, err := c.sftpClient.Getwd() + if err != nil { + fs.Debugf(f, "Failed to to read current directory - using relative paths: %v", err) + } else { + f.absRoot = path.Join(cwd, f.root) + fs.Debugf(f, "Relative path joined with current directory to get absolute path %q", f.absRoot) + } } else { - f.absRoot = path + f.absRoot = absRoot fs.Debugf(f, "Relative path resolved to %q", f.absRoot) } } From 695736d1e4797d88004f23fd5017c9e38cec3532 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 20 Jul 2022 23:06:45 +0100 Subject: [PATCH 194/560] Add Steve Kowalik to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 18e1acddfbaf4..011771cf03463 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -628,3 +628,4 @@ put them back in again.` >}} * Ovidiu Victor Tatar * Evan Spensley * Yen Hu <61753151+0x59656e@users.noreply.github.com> + * Steve Kowalik From fb60aeddaeffdc422eb35150f280844f2ba6a14c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 20 Jul 2022 23:06:45 +0100 Subject: [PATCH 195/560] =?UTF-8?q?Add=20Jordi=20Gonzalez=20Mu=C3=B1oz=20t?= =?UTF-8?q?o=20contributors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 011771cf03463..088bda7422ebf 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -629,3 +629,4 @@ put them back in again.` >}} * Evan Spensley * Yen Hu <61753151+0x59656e@users.noreply.github.com> * Steve Kowalik + * Jordi Gonzalez Muñoz From 7a24c173f6669172d845221c7e37e5824fa13fb7 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 20 Jul 2022 23:06:35 +0100 Subject: [PATCH 196/560] build: disable revive linter pending a fix in golangci-lint The revive linter got extremely slow in golangci-lint 1.47.1 causing the CI to time out. Disable for the time being until it is fixed. See: https://github.com/golangci/golangci-lint/issues/2997 --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 67b3edda9afa1..48fbe05b7e079 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -5,7 +5,7 @@ linters: - deadcode - errcheck - goimports - - revive + #- revive - ineffassign - structcheck - varcheck From 2f461f13e3efe603bdd8fb4057f3e8242a59956f Mon Sep 17 00:00:00 2001 From: Lesmiscore Date: Fri, 22 Jul 2022 12:52:48 +0900 Subject: [PATCH 197/560] internetarchive: handle hash symbol in the middle of filename --- backend/internetarchive/internetarchive.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/internetarchive/internetarchive.go b/backend/internetarchive/internetarchive.go index b7a0676b5d201..db553eda19e43 100644 --- a/backend/internetarchive/internetarchive.go +++ b/backend/internetarchive/internetarchive.go @@ -572,7 +572,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, return "", err } bucket, bucketPath := f.split(remote) - return path.Join(f.opt.FrontEndpoint, "/download/", bucket, bucketPath), nil + return path.Join(f.opt.FrontEndpoint, "/download/", bucket, quotePath(bucketPath)), nil } // Copy src to this remote using server-side copy operations. @@ -760,7 +760,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // make a GET request to (frontend)/download/:item/:path opts := rest.Opts{ Method: "GET", - Path: path.Join("/download/", o.fs.root, o.fs.opt.Enc.FromStandardPath(o.remote)), + Path: path.Join("/download/", o.fs.root, quotePath(o.fs.opt.Enc.FromStandardPath(o.remote))), Options: optionsFixed, } err = o.fs.pacer.Call(func() (bool, error) { From 9f40cb114a2c751c5821205c25fb373cc6dee1a8 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 12 Jul 2022 21:10:04 +0100 Subject: [PATCH 198/560] Revert "jottacloud: always store username in config and use it to avoid initial api request" This reverts commit 9dbed02329456c71d2052a6365041deda94bc020. See: #6309 --- backend/jottacloud/jottacloud.go | 53 ++++++++++++-------------------- 1 file changed, 19 insertions(+), 34 deletions(-) diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 5998a04a2780e..735b5181fb080 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -152,7 +152,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf m.Set(configClientSecret, "") srv := rest.NewClient(fshttp.NewClient(ctx)) - token, tokenEndpoint, username, err := doTokenAuth(ctx, srv, loginToken) + token, tokenEndpoint, err := doTokenAuth(ctx, srv, loginToken) if err != nil { return nil, fmt.Errorf("failed to get oauth token: %w", err) } @@ -161,7 +161,6 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf if err != nil { return nil, fmt.Errorf("error while saving token: %w", err) } - m.Set(configUsername, username) return fs.ConfigGoto("choose_device") case "legacy": // configure a jottacloud backend using legacy authentication m.Set("configVersion", fmt.Sprint(legacyConfigVersion)) @@ -272,30 +271,22 @@ sync or the backup section, for example, you must choose yes.`) if config.Result != "true" { m.Set(configDevice, "") m.Set(configMountpoint, "") - } - username, userOk := m.Get(configUsername) - if userOk && config.Result != "true" { return fs.ConfigGoto("end") } oAuthClient, _, err := getOAuthClient(ctx, name, m) if err != nil { return nil, err } - if !userOk { - apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL) - cust, err := getCustomerInfo(ctx, apiSrv) - if err != nil { - return nil, err - } - username = cust.Username - m.Set(configUsername, username) - if config.Result != "true" { - return fs.ConfigGoto("end") - } + jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL) + apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL) + + cust, err := getCustomerInfo(ctx, apiSrv) + if err != nil { + return nil, err } + m.Set(configUsername, cust.Username) - jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL) - acc, err := getDriveInfo(ctx, jfsSrv, username) + acc, err := getDriveInfo(ctx, jfsSrv, cust.Username) if err != nil { return nil, err } @@ -591,10 +582,10 @@ func doLegacyAuth(ctx context.Context, srv *rest.Client, oauthConfig *oauth2.Con } // doTokenAuth runs the actual token request for V2 authentication -func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 string) (token oauth2.Token, tokenEndpoint string, username string, err error) { +func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 string) (token oauth2.Token, tokenEndpoint string, err error) { loginTokenBytes, err := base64.RawURLEncoding.DecodeString(loginTokenBase64) if err != nil { - return token, "", "", err + return token, "", err } // decode login token @@ -602,7 +593,7 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri decoder := json.NewDecoder(bytes.NewReader(loginTokenBytes)) err = decoder.Decode(&loginToken) if err != nil { - return token, "", "", err + return token, "", err } // retrieve endpoint urls @@ -613,7 +604,7 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri var wellKnown api.WellKnown _, err = apiSrv.CallJSON(ctx, &opts, nil, &wellKnown) if err != nil { - return token, "", "", err + return token, "", err } // prepare out token request with username and password @@ -635,14 +626,14 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri var jsonToken api.TokenJSON _, err = apiSrv.CallJSON(ctx, &opts, nil, &jsonToken) if err != nil { - return token, "", "", err + return token, "", err } token.AccessToken = jsonToken.AccessToken token.RefreshToken = jsonToken.RefreshToken token.TokenType = jsonToken.TokenType token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second) - return token, wellKnown.TokenEndpoint, loginToken.Username, err + return token, wellKnown.TokenEndpoint, err } // getCustomerInfo queries general information about the account @@ -944,17 +935,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e return err }) - user, userOk := m.Get(configUsername) - if userOk { - f.user = user - } else { - fs.Infof(nil, "Username not found in config and must be looked up, reconfigure to avoid the extra request") - cust, err := getCustomerInfo(ctx, f.apiSrv) - if err != nil { - return nil, err - } - f.user = cust.Username + cust, err := getCustomerInfo(ctx, f.apiSrv) + if err != nil { + return nil, err } + f.user = cust.Username f.setEndpoints() if root != "" && !rootIsDir { From 72227a01519f4cc9b0a1ecd390785c9d1d09d5d0 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 15 Jul 2022 10:01:06 +0200 Subject: [PATCH 199/560] jottacloud: do not store username in config when using standard auth Previously, with standard auth, the username would be stored in config - but only after entering the non-standard device/mountpoint sequence during config (a feature introduced with #5926). Regardless of that, rclone always requests the username from the api at startup (NewFS). In #6270 (commit 9dbed02329456c71d2052a6365041deda94bc020) this was changed to always store username in config (consistency), and then also use it to avoid the repeated customer info request in NewFs (performance). But, as reported in #6309, it did not work with legacy auth, where user enters username manually, if user entered an email address instead of the internal username required for api requests. This change was therefore recently reverted. The current commit takes another step back to not store the username in config during the non-standard device/mountpoint config sequence (consistentcy). The username will now only be stored in config when using legacy auth, where it is an input parameter. --- backend/jottacloud/jottacloud.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 735b5181fb080..3bdc98ec6523c 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -284,7 +284,6 @@ sync or the backup section, for example, you must choose yes.`) if err != nil { return nil, err } - m.Set(configUsername, cust.Username) acc, err := getDriveInfo(ctx, jfsSrv, cust.Username) if err != nil { @@ -317,10 +316,14 @@ a new by entering a unique name.`, defaultDevice) return nil, err } jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL) + apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL) - username, _ := m.Get(configUsername) + cust, err := getCustomerInfo(ctx, apiSrv) + if err != nil { + return nil, err + } - acc, err := getDriveInfo(ctx, jfsSrv, username) + acc, err := getDriveInfo(ctx, jfsSrv, cust.Username) if err != nil { return nil, err } @@ -335,7 +338,7 @@ a new by entering a unique name.`, defaultDevice) var dev *api.JottaDevice if isNew { fs.Debugf(nil, "Creating new device: %s", device) - dev, err = createDevice(ctx, jfsSrv, path.Join(username, device)) + dev, err = createDevice(ctx, jfsSrv, path.Join(cust.Username, device)) if err != nil { return nil, err } @@ -343,7 +346,7 @@ a new by entering a unique name.`, defaultDevice) m.Set(configDevice, device) if !isNew { - dev, err = getDeviceInfo(ctx, jfsSrv, path.Join(username, device)) + dev, err = getDeviceInfo(ctx, jfsSrv, path.Join(cust.Username, device)) if err != nil { return nil, err } @@ -373,11 +376,16 @@ You may create a new by entering a unique name.`, device) return nil, err } jfsSrv := rest.NewClient(oAuthClient).SetRoot(jfsURL) + apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL) + + cust, err := getCustomerInfo(ctx, apiSrv) + if err != nil { + return nil, err + } - username, _ := m.Get(configUsername) device, _ := m.Get(configDevice) - dev, err := getDeviceInfo(ctx, jfsSrv, path.Join(username, device)) + dev, err := getDeviceInfo(ctx, jfsSrv, path.Join(cust.Username, device)) if err != nil { return nil, err } @@ -395,7 +403,7 @@ You may create a new by entering a unique name.`, device) return nil, fmt.Errorf("custom mountpoints not supported on built-in %s device: %w", defaultDevice, err) } fs.Debugf(nil, "Creating new mountpoint: %s", mountpoint) - _, err := createMountPoint(ctx, jfsSrv, path.Join(username, device, mountpoint)) + _, err := createMountPoint(ctx, jfsSrv, path.Join(cust.Username, device, mountpoint)) if err != nil { return nil, err } From 3a8e52de74e81f499f44a47a937552378d5bc83b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 29 Jul 2022 17:40:05 +0100 Subject: [PATCH 200/560] dropbox: fix infinite loop on uploading a corrupted file Before this change, if rclone attempted to upload a file which read more bytes than the size it declared then the uploader would enter an infinite loop. See: https://forum.rclone.org/t/transfer-percentages-100-again/32109 --- backend/dropbox/dropbox.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index bcee49821d9af..100044862440c 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -1697,6 +1697,9 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f if size > 0 { // if size is known, check if next chunk is final appendArg.Close = uint64(size)-in.BytesRead() <= uint64(chunkSize) + if in.BytesRead() > uint64(size) { + return nil, fmt.Errorf("expected %d bytes in input, but have read %d so far", size, in.BytesRead()) + } } else { // if size is unknown, upload as long as we can read full chunks from the reader appendArg.Close = in.BytesRead()-cursor.Offset < uint64(chunkSize) From 9d3958bd0b0f26ee597cb732761d705913bacdb1 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 3 Aug 2022 10:11:16 +0100 Subject: [PATCH 201/560] build: fix formatting after golangci-lint update --- librclone/librclone.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/librclone/librclone.go b/librclone/librclone.go index 5759acef566d4..f90a7e4130240 100644 --- a/librclone/librclone.go +++ b/librclone/librclone.go @@ -35,12 +35,12 @@ import ( "github.com/rclone/rclone/librclone/librclone" _ "github.com/rclone/rclone/backend/all" // import all backends + _ "github.com/rclone/rclone/cmd/cmount" // import cmount + _ "github.com/rclone/rclone/cmd/mount" // import mount + _ "github.com/rclone/rclone/cmd/mount2" // import mount2 _ "github.com/rclone/rclone/fs/operations" // import operations/* rc commands _ "github.com/rclone/rclone/fs/sync" // import sync/* _ "github.com/rclone/rclone/lib/plugin" // import plugins - _ "github.com/rclone/rclone/cmd/mount" // import mount - _ "github.com/rclone/rclone/cmd/mount2" // import mount2 - _ "github.com/rclone/rclone/cmd/cmount" // import cmount ) // RcloneInitialize initializes rclone as a library From fe84cca1ad1d018a2a3b2b356689f64b2d08f3ad Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 3 Aug 2022 13:14:51 +0100 Subject: [PATCH 202/560] Revert "build: disable revive linter pending a fix in golangci-lint" This reverts commit 7a24c173f6669172d845221c7e37e5824fa13fb7. --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 48fbe05b7e079..67b3edda9afa1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -5,7 +5,7 @@ linters: - deadcode - errcheck - goimports - #- revive + - revive - ineffassign - structcheck - varcheck From 639624184deea156387d616f26bcce9577672dd0 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 3 Aug 2022 15:03:15 +0100 Subject: [PATCH 203/560] build: fix android build after GitHub actions change Before this change the android build started failing with gomobile: ANDROID_NDK_HOME specifies /usr/local/lib/android/sdk/ndk/25.0.8775105 which is unusable: unsupported API version 16 (not in 19..33) This was caused by a change to github actions, but is ultimately due to an issue in gomobile with the newest version of the SDK. This change fixes the problem by declaring a minimum API version of 21 and using version 21 compilers to build everything and using the default NDK in github actions. See: https://github.com/actions/virtual-environments/issues/5930 See: https://github.com/lightningnetwork/lnd/issues/6651 --- .github/workflows/build.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f719794f45d1..7d18b0a12bb95 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -252,10 +252,6 @@ jobs: with: go-version: 1.18.x - # Upgrade together with Go version. Using a GitHub-provided version saves around 2 minutes. - - name: Force NDK version - run: echo "y" | sudo ${ANDROID_HOME}/tools/bin/sdkmanager --install "ndk;23.1.7779620" | grep -v = || true - - name: Go module cache uses: actions/cache@v2 with: @@ -278,27 +274,29 @@ jobs: go install golang.org/x/mobile/cmd/gobind@latest go install golang.org/x/mobile/cmd/gomobile@latest env PATH=$PATH:~/go/bin gomobile init + echo "RCLONE_NDK_VERSION=21" >> $GITHUB_ENV - name: arm-v7a gomobile build - run: env PATH=$PATH:~/go/bin gomobile bind -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile + run: env PATH=$PATH:~/go/bin gomobile bind -androidapi ${RCLONE_NDK_VERSION} -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile - name: arm-v7a Set environment variables shell: bash run: | - echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi16-clang)" >> $GITHUB_ENV + echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV echo 'GOOS=android' >> $GITHUB_ENV echo 'GOARCH=arm' >> $GITHUB_ENV echo 'GOARM=7' >> $GITHUB_ENV echo 'CGO_ENABLED=1' >> $GITHUB_ENV echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV + - name: arm-v7a build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-armv7a . + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv7a . - name: arm64-v8a Set environment variables shell: bash run: | - echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang)" >> $GITHUB_ENV + echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV echo 'GOOS=android' >> $GITHUB_ENV echo 'GOARCH=arm64' >> $GITHUB_ENV @@ -306,12 +304,12 @@ jobs: echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV - name: arm64-v8a build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-armv8a . + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv8a . - name: x86 Set environment variables shell: bash run: | - echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android16-clang)" >> $GITHUB_ENV + echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV echo 'GOOS=android' >> $GITHUB_ENV echo 'GOARCH=386' >> $GITHUB_ENV @@ -319,12 +317,12 @@ jobs: echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV - name: x86 build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-x86 . + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x86 . - name: x64 Set environment variables shell: bash run: | - echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android21-clang)" >> $GITHUB_ENV + echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV echo 'GOOS=android' >> $GITHUB_ENV echo 'GOARCH=amd64' >> $GITHUB_ENV @@ -332,7 +330,7 @@ jobs: echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV - name: x64 build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-x64 . + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x64 . - name: Upload artifacts run: | From a875320e37a6eb349899f4f6058ed0c692d92a60 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 1 Aug 2022 17:51:46 +0100 Subject: [PATCH 204/560] sync,operations: optimise --copy-dest and --compare-dest Before this change --compare-dest and --copy-dest would check to see if the compare/copy object existed first, before seeing if the destination object was present. This is inefficient, because in most --copy-dest syncs the destination will be present and the compare/copy object need never be tested. --compare-dest syncs may also be speeded up if they are done to the same directory repeatedly. This fixes the problem by re-arranging the logic so if the transfer is not needed then the compare/copy object is never tested. See: https://forum.rclone.org/t/union-with-copy-dest-enabled-is-slower-than-expected/32172 --- fs/operations/operations.go | 14 ++++++++++---- fs/sync/sync.go | 14 ++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/fs/operations/operations.go b/fs/operations/operations.go index a02aead24c99a..081ceb7a7efc5 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1952,11 +1952,17 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str return err } } - NoNeedTransfer, err := CompareOrCopyDest(ctx, fdst, dstObj, srcObj, copyDestDir, backupDir) - if err != nil { - return err + needTransfer := NeedTransfer(ctx, dstObj, srcObj) + if needTransfer { + NoNeedTransfer, err := CompareOrCopyDest(ctx, fdst, dstObj, srcObj, copyDestDir, backupDir) + if err != nil { + return err + } + if NoNeedTransfer { + needTransfer = false + } } - if !NoNeedTransfer && NeedTransfer(ctx, dstObj, srcObj) { + if needTransfer { // If destination already exists, then we must move it into --backup-dir if required if dstObj != nil && backupDir != nil { err = MoveBackupDir(ctx, backupDir, dstObj) diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 052b867e8d60c..2c671cf72b802 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -331,11 +331,17 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W tr := accounting.Stats(s.ctx).NewCheckingTransfer(src) // Check to see if can store this if src.Storable() { - NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, pair.Dst, pair.Src, s.compareCopyDest, s.backupDir) - if err != nil { - s.processError(err) + needTransfer := operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) + if needTransfer { + NoNeedTransfer, err := operations.CompareOrCopyDest(s.ctx, s.fdst, pair.Dst, pair.Src, s.compareCopyDest, s.backupDir) + if err != nil { + s.processError(err) + } + if NoNeedTransfer { + needTransfer = false + } } - if !NoNeedTransfer && operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) { + if needTransfer { // If files are treated as immutable, fail if destination exists and does not match if s.ci.Immutable && pair.Dst != nil { err := fs.CountError(fserrors.NoRetryError(fs.ErrorImmutableModified)) From ba60984f33676fac258cbebd3088a0c73b3bb717 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 2 Aug 2022 18:47:43 +0100 Subject: [PATCH 205/560] build: update to go1.19 and make go1.17 the minimum required version --- .github/workflows/build.yml | 24 ++++---- cmd/serve/restic/cache.go | 3 - cmd/serve/restic/cache_test.go | 3 - cmd/serve/restic/restic.go | 4 -- cmd/serve/restic/restic_appendonly_test.go | 3 - cmd/serve/restic/restic_privaterepos_test.go | 3 - cmd/serve/restic/restic_test.go | 3 - cmd/serve/restic/restic_unsupported.go | 14 ----- cmd/serve/restic/restic_utils_test.go | 3 - cmd/serve/restic/stdio_conn.go | 3 - docs/content/install.md | 2 +- fs/versioncheck.go | 8 +-- go.mod | 59 +++++++++++++++++++- 13 files changed, 75 insertions(+), 57 deletions(-) delete mode 100644 cmd/serve/restic/restic_unsupported.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7d18b0a12bb95..9e44390878b81 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,12 +25,12 @@ jobs: strategy: fail-fast: false matrix: - job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.16', 'go1.17'] + job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.17', 'go1.18'] include: - job_name: linux os: ubuntu-latest - go: '1.18.x' + go: '1.19.x' gotags: cmount build_flags: '-include "^linux/"' check: true @@ -41,14 +41,14 @@ jobs: - job_name: linux_386 os: ubuntu-latest - go: '1.18.x' + go: '1.19.x' goarch: 386 gotags: cmount quicktest: true - job_name: mac_amd64 os: macos-11 - go: '1.18.x' + go: '1.19.x' gotags: 'cmount' build_flags: '-include "^darwin/amd64" -cgo' quicktest: true @@ -57,14 +57,14 @@ jobs: - job_name: mac_arm64 os: macos-11 - go: '1.18.x' + go: '1.19.x' gotags: 'cmount' build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib' deploy: true - job_name: windows os: windows-latest - go: '1.18.x' + go: '1.19.x' gotags: cmount cgo: '0' build_flags: '-include "^windows/"' @@ -74,20 +74,20 @@ jobs: - job_name: other_os os: ubuntu-latest - go: '1.18.x' + go: '1.19.x' build_flags: '-exclude "^(windows/|darwin/|linux/)"' compile_all: true deploy: true - - job_name: go1.16 + - job_name: go1.17 os: ubuntu-latest - go: '1.16.x' + go: '1.17.x' quicktest: true racequicktest: true - - job_name: go1.17 + - job_name: go1.18 os: ubuntu-latest - go: '1.17.x' + go: '1.18.x' quicktest: true racequicktest: true @@ -250,7 +250,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v1 with: - go-version: 1.18.x + go-version: 1.19.x - name: Go module cache uses: actions/cache@v2 diff --git a/cmd/serve/restic/cache.go b/cmd/serve/restic/cache.go index 40782358118c2..f7f376c8236a5 100644 --- a/cmd/serve/restic/cache.go +++ b/cmd/serve/restic/cache.go @@ -1,6 +1,3 @@ -//go:build go1.17 -// +build go1.17 - package restic import ( diff --git a/cmd/serve/restic/cache_test.go b/cmd/serve/restic/cache_test.go index 290768aba9244..05687ad84ca30 100644 --- a/cmd/serve/restic/cache_test.go +++ b/cmd/serve/restic/cache_test.go @@ -1,6 +1,3 @@ -//go:build go1.17 -// +build go1.17 - package restic import ( diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go index 7139b084b8e5d..a0a521bb678b3 100644 --- a/cmd/serve/restic/restic.go +++ b/cmd/serve/restic/restic.go @@ -1,8 +1,4 @@ // Package restic serves a remote suitable for use with restic - -//go:build go1.17 -// +build go1.17 - package restic import ( diff --git a/cmd/serve/restic/restic_appendonly_test.go b/cmd/serve/restic/restic_appendonly_test.go index c190da3294392..b3562db9edcd5 100644 --- a/cmd/serve/restic/restic_appendonly_test.go +++ b/cmd/serve/restic/restic_appendonly_test.go @@ -1,6 +1,3 @@ -//go:build go1.17 -// +build go1.17 - package restic import ( diff --git a/cmd/serve/restic/restic_privaterepos_test.go b/cmd/serve/restic/restic_privaterepos_test.go index fd59e181df765..4faae363cc2f3 100644 --- a/cmd/serve/restic/restic_privaterepos_test.go +++ b/cmd/serve/restic/restic_privaterepos_test.go @@ -1,6 +1,3 @@ -//go:build go1.17 -// +build go1.17 - package restic import ( diff --git a/cmd/serve/restic/restic_test.go b/cmd/serve/restic/restic_test.go index 8e8cccf672886..d9f343ca55a4f 100644 --- a/cmd/serve/restic/restic_test.go +++ b/cmd/serve/restic/restic_test.go @@ -1,9 +1,6 @@ // Serve restic tests set up a server and run the integration tests // for restic against it. -//go:build go1.17 -// +build go1.17 - package restic import ( diff --git a/cmd/serve/restic/restic_unsupported.go b/cmd/serve/restic/restic_unsupported.go deleted file mode 100644 index a270482a3e3e7..0000000000000 --- a/cmd/serve/restic/restic_unsupported.go +++ /dev/null @@ -1,14 +0,0 @@ -// Build for restic for unsupported platforms to stop go complaining -// about "no buildable Go source files " - -//go:build !go1.17 -// +build !go1.17 - -package restic - -import ( - "github.com/spf13/cobra" -) - -// Command definition for cobra -var Command *cobra.Command diff --git a/cmd/serve/restic/restic_utils_test.go b/cmd/serve/restic/restic_utils_test.go index 17bbfa2d95469..0721f7c0f3adb 100644 --- a/cmd/serve/restic/restic_utils_test.go +++ b/cmd/serve/restic/restic_utils_test.go @@ -1,6 +1,3 @@ -//go:build go1.17 -// +build go1.17 - package restic import ( diff --git a/cmd/serve/restic/stdio_conn.go b/cmd/serve/restic/stdio_conn.go index 3e5160bad948e..3b411eb0eb9eb 100644 --- a/cmd/serve/restic/stdio_conn.go +++ b/cmd/serve/restic/stdio_conn.go @@ -1,6 +1,3 @@ -//go:build go1.17 -// +build go1.17 - package restic import ( diff --git a/docs/content/install.md b/docs/content/install.md index f6fc8a55fa176..714794f3d5344 100644 --- a/docs/content/install.md +++ b/docs/content/install.md @@ -191,7 +191,7 @@ kill %1 ## Install from source Make sure you have git and [Go](https://golang.org/) installed. -Go version 1.16 or newer is required, latest release is recommended. +Go version 1.17 or newer is required, latest release is recommended. You can get it from your package manager, or download it from [golang.org/dl](https://golang.org/dl/). Then you can run the following: diff --git a/fs/versioncheck.go b/fs/versioncheck.go index e7b9f7e1ce01d..9820eccadbc73 100644 --- a/fs/versioncheck.go +++ b/fs/versioncheck.go @@ -1,8 +1,8 @@ -//go:build !go1.16 -// +build !go1.16 +//go:build !go1.17 +// +build !go1.17 package fs -// Upgrade to Go version 1.16 to compile rclone - latest stable go +// Upgrade to Go version 1.17 to compile rclone - latest stable go // compiler recommended. -func init() { Go_version_1_16_required_for_compilation() } +func init() { Go_version_1_17_required_for_compilation() } diff --git a/go.mod b/go.mod index 8f301a4df5ce3..2ff7f3f13ec0c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/rclone/rclone -go 1.16 +go 1.17 require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 @@ -70,6 +70,63 @@ require ( storj.io/uplink v1.9.0 ) +require ( + cloud.google.com/go/compute v1.6.1 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/anacrolix/log v0.10.1-0.20220126091220-5c1b6f3af59c // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/calebcase/tmpfile v1.0.3 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gdamore/encoding v1.0.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/googleapis/gax-go/v2 v2.4.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-uuid v1.0.2 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jcmturner/aescts/v2 v2.0.0 // indirect + github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect + github.com/jcmturner/gofork v1.0.0 // indirect + github.com/jcmturner/goidentity/v6 v6.0.1 // indirect + github.com/jcmturner/rpc/v2 v2.0.3 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mattn/go-ieproxy v0.0.1 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/onsi/gomega v1.13.0 // indirect + github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/spacemonkeygo/monkit/v3 v3.0.17 // indirect + github.com/tklauser/go-sysconf v0.3.10 // indirect + github.com/tklauser/numcpus v0.4.0 // indirect + github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect + github.com/zeebo/errs v1.3.0 // indirect + go.opencensus.io v0.23.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 // indirect + google.golang.org/grpc v1.47.0 // indirect + google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + storj.io/common v0.0.0-20220414110316-a5cb7172d6bf // indirect + storj.io/drpc v0.0.30 // indirect +) + require ( github.com/Microsoft/go-winio v0.5.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20220623141421-5afb4c282135 From 8125b1cf085fc10d325cdff752b84511fc8c7886 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 3 Aug 2022 17:39:55 +0100 Subject: [PATCH 206/560] build: update to v3 of golangci-lint action --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e44390878b81..c7c76dee1ed6b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -229,7 +229,7 @@ jobs: uses: actions/checkout@v2 - name: Code quality test - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v3 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version version: latest From 2170376d1be915a24cc94046e30017ce47cfabc6 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 3 Aug 2022 12:07:52 +0100 Subject: [PATCH 207/560] build: lint with go1.18 until golangci-lint is updated See: https://github.com/golangci/golangci-lint/pull/3037 --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c7c76dee1ed6b..ae8d40ad2fcd4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -227,6 +227,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + # FIXME temporary until golangci-lint supports go1.19 + go-version: 1.18.x - name: Code quality test uses: golangci/golangci-lint-action@v3 From 821e084f28d07d4c4327ccee6a62a02c2fbf9afd Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 3 Aug 2022 17:15:59 +0100 Subject: [PATCH 208/560] combine: fix errors with backends shutting down while in use Before this patch backends could be shutdown when they fell out of the cache when they were in use with combine. This was particularly noticeable with the dropbox backend which gave this error when uploading files after the backend was Shutdown. Failed to copy: upload failed: batcher is shutting down This patch gets the combine remote to pin them until it is finished. See: https://forum.rclone.org/t/rclone-combine-upload-failed-batcher-is-shutting-down/32168 --- backend/combine/combine.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/combine/combine.go b/backend/combine/combine.go index a65192647e296..bae4bd8a1c822 100644 --- a/backend/combine/combine.go +++ b/backend/combine/combine.go @@ -145,6 +145,7 @@ func (f *Fs) newUpstream(ctx context.Context, dir, remote string) (*upstream, er dir: dir, pathAdjustment: newAdjustment(f.root, dir), } + cache.PinUntilFinalized(u.f, u) return u, nil } From e749bc58f4e54833fc860c682040f465794d377d Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 28 Jul 2022 17:35:14 +0100 Subject: [PATCH 209/560] vfs: reduce memory use by embedding sync.Cond --- vfs/read.go | 6 +++--- vfs/vfscache/cache.go | 4 ++-- vfs/vfscache/item.go | 4 ++-- vfs/write.go | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/vfs/read.go b/vfs/read.go index 88bcabfce2109..4a5658eada14f 100644 --- a/vfs/read.go +++ b/vfs/read.go @@ -20,7 +20,7 @@ type ReadFileHandle struct { baseHandle done func(ctx context.Context, err error) mu sync.Mutex - cond *sync.Cond // cond lock for out of sequence reads + cond sync.Cond // cond lock for out of sequence reads closed bool // set if handle has been closed r *accounting.Account readCalled bool // set if read has been called @@ -63,7 +63,7 @@ func newReadFileHandle(f *File) (*ReadFileHandle, error) { size: nonNegative(o.Size()), sizeUnknown: o.Size() < 0, } - fh.cond = sync.NewCond(&fh.mu) + fh.cond = sync.Cond{L: &fh.mu} return fh, nil } @@ -267,7 +267,7 @@ func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) { maxBuf = len(p) } if gap := off - fh.offset; gap > 0 && gap < int64(8*maxBuf) { - waitSequential("read", fh.remote, fh.cond, fh.file.VFS().Opt.ReadWait, &fh.offset, off) + waitSequential("read", fh.remote, &fh.cond, fh.file.VFS().Opt.ReadWait, &fh.offset, off) } doSeek := off != fh.offset if doSeek && fh.noSeek { diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go index 9c7feb646fb8b..8e5209a795703 100644 --- a/vfs/vfscache/cache.go +++ b/vfs/vfscache/cache.go @@ -52,7 +52,7 @@ type Cache struct { avFn AddVirtualFn // if set, can be called to add dir entries mu sync.Mutex // protects the following variables - cond *sync.Cond // cond lock for synchronous cache cleaning + cond sync.Cond // cond lock for synchronous cache cleaning item map[string]*Item // files/directories in the cache errItems map[string]error // items in error state used int64 // total size of files in the cache @@ -139,7 +139,7 @@ func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options, avFn AddVir // Create a channel for cleaner to be kicked upon out of space con c.kick = make(chan struct{}, 1) - c.cond = sync.NewCond(&c.mu) + c.cond = sync.Cond{L: &c.mu} go c.cleaner(ctx) diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go index 2c81458709025..883bc9bde5e4c 100644 --- a/vfs/vfscache/item.go +++ b/vfs/vfscache/item.go @@ -57,7 +57,7 @@ type Item struct { // read only c *Cache // cache this is part of mu sync.Mutex // protect the variables - cond *sync.Cond // synchronize with cache cleaner + cond sync.Cond // synchronize with cache cleaner name string // name in the VFS opens int // number of times file is open downloaders *downloaders.Downloaders // a record of the downloaders in action - may be nil @@ -138,7 +138,7 @@ func newItem(c *Cache, name string) (item *Item) { ATime: now, }, } - item.cond = sync.NewCond(&item.mu) + item.cond = sync.Cond{L: &item.mu} // check the cache file exists osPath := c.toOSPath(name) fi, statErr := os.Stat(osPath) diff --git a/vfs/write.go b/vfs/write.go index 24595e0419ccb..25fa4fbf6851f 100644 --- a/vfs/write.go +++ b/vfs/write.go @@ -15,7 +15,7 @@ import ( type WriteFileHandle struct { baseHandle mu sync.Mutex - cond *sync.Cond // cond lock for out of sequence writes + cond sync.Cond // cond lock for out of sequence writes closed bool // set if handle has been closed remote string pipeWriter *io.PipeWriter @@ -43,7 +43,7 @@ func newWriteFileHandle(d *Dir, f *File, remote string, flags int) (*WriteFileHa result: make(chan error, 1), file: f, } - fh.cond = sync.NewCond(&fh.mu) + fh.cond = sync.Cond{L: &fh.mu} fh.file.addWriter(fh) return fh, nil } @@ -130,7 +130,7 @@ func (fh *WriteFileHandle) writeAt(p []byte, off int64) (n int, err error) { return 0, ECLOSED } if fh.offset != off { - waitSequential("write", fh.remote, fh.cond, fh.file.VFS().Opt.WriteWait, &fh.offset, off) + waitSequential("write", fh.remote, &fh.cond, fh.file.VFS().Opt.WriteWait, &fh.offset, off) } if fh.offset != off { fs.Errorf(fh.remote, "WriteFileHandle.Write: can't seek in file without --vfs-cache-mode >= writes") From a07d376fb111944cdd74521446caa6f3ba43b0d4 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 28 Jul 2022 18:14:04 +0100 Subject: [PATCH 210/560] vfs: reduce memory usage by re-ordering commonly used structures --- vfs/file.go | 8 ++++---- vfs/read.go | 10 +++++----- vfs/vfscache/item.go | 2 +- vfs/write.go | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/vfs/file.go b/vfs/file.go index 2d3199f137dc2..4431c6cc5e5a3 100644 --- a/vfs/file.go +++ b/vfs/file.go @@ -39,19 +39,19 @@ type File struct { inode uint64 // inode number - read only size int64 // size of file - read and written with atomic int64 - must be 64 bit aligned + muRW sync.Mutex // synchronize RWFileHandle.openPending(), RWFileHandle.close() and File.Remove + mu sync.RWMutex // protects the following d *Dir // parent directory dPath string // path of parent directory. NB dir rename means all Files are flushed o fs.Object // NB o may be nil if file is being written leaf string // leaf name of the object writers []Handle // writers for this file - nwriters int32 // len(writers) which is read/updated with atomic pendingModTime time.Time // will be applied once o becomes available, i.e. after file was written pendingRenameFun func(ctx context.Context) error // will be run/renamed after all writers close - appendMode bool // file was opened with O_APPEND sys atomic.Value // user defined info to be attached here - - muRW sync.Mutex // synchronize RWFileHandle.openPending(), RWFileHandle.close() and File.Remove + nwriters int32 // len(writers) which is read/updated with atomic + appendMode bool // file was opened with O_APPEND } // newFile creates a new File diff --git a/vfs/read.go b/vfs/read.go index 4a5658eada14f..bfe4b2d5cd49b 100644 --- a/vfs/read.go +++ b/vfs/read.go @@ -21,18 +21,18 @@ type ReadFileHandle struct { done func(ctx context.Context, err error) mu sync.Mutex cond sync.Cond // cond lock for out of sequence reads - closed bool // set if handle has been closed r *accounting.Account - readCalled bool // set if read has been called size int64 // size of the object (0 for unknown length) offset int64 // offset of read of o roffset int64 // offset of Read() calls - noSeek bool - sizeUnknown bool // set if size of source is not known file *File hash *hash.MultiHasher - opened bool remote string + closed bool // set if handle has been closed + readCalled bool // set if read has been called + noSeek bool + sizeUnknown bool // set if size of source is not known + opened bool } // Check interfaces diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go index 883bc9bde5e4c..5d606f21e980f 100644 --- a/vfs/vfscache/item.go +++ b/vfs/vfscache/item.go @@ -63,10 +63,10 @@ type Item struct { downloaders *downloaders.Downloaders // a record of the downloaders in action - may be nil o fs.Object // object we are caching - may be nil fd *os.File // handle we are using to read and write to the file - modified bool // set if the file has been modified since the last Open info Info // info about the file to persist to backing store writeBackID writeback.Handle // id of any writebacks in progress pendingAccesses int // number of threads - cache reset not allowed if not zero + modified bool // set if the file has been modified since the last Open beingReset bool // cache cleaner is resetting the cache file, access not allowed } diff --git a/vfs/write.go b/vfs/write.go index 25fa4fbf6851f..4556ebe3e2a56 100644 --- a/vfs/write.go +++ b/vfs/write.go @@ -16,16 +16,16 @@ type WriteFileHandle struct { baseHandle mu sync.Mutex cond sync.Cond // cond lock for out of sequence writes - closed bool // set if handle has been closed remote string pipeWriter *io.PipeWriter o fs.Object result chan error file *File - writeCalled bool // set the first time Write() is called offset int64 - opened bool flags int + closed bool // set if handle has been closed + writeCalled bool // set the first time Write() is called + opened bool truncated bool } From 2a817e21cb3b047a3b8c9909b072892506763bc1 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 28 Jul 2022 17:43:05 +0100 Subject: [PATCH 211/560] vfs: fix excess CPU used by VFS cache cleaner looping Before this change the VFS cache cleaner would loop indefinitely while the cache was above quota. This used up all the CPU. This fix prevents the cache cleaner from looping. It will be kicked on ENOSPACE and run in its scheduled time otherwise so this should be sufficient. See: https://forum.rclone.org/t/vfs-keeps-checking-same-files/32120 --- vfs/vfscache/cache.go | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go index 8e5209a795703..9c0ec0bf0fb2c 100644 --- a/vfs/vfscache/cache.go +++ b/vfs/vfscache/cache.go @@ -739,27 +739,17 @@ func (c *Cache) clean(kicked bool) { oldItems, oldUsed := len(c.item), fs.SizeSuffix(c.used) c.mu.Unlock() - // loop cleaning the cache until we reach below cache quota - for { - // Remove any files that are over age - c.purgeOld(c.opt.CacheMaxAge) - - if int64(c.opt.CacheMaxSize) <= 0 { - break - } + // Remove any files that are over age + c.purgeOld(c.opt.CacheMaxAge) - // Now remove files not in use until cache size is below quota starting from the - // oldest first + // If have a maximum cache size... + if int64(c.opt.CacheMaxSize) > 0 { + // Remove files not in use until cache size is below quota starting from the oldest first c.purgeOverQuota(int64(c.opt.CacheMaxSize)) // Remove cache files that are not dirty if we are still above the max cache size c.purgeClean(int64(c.opt.CacheMaxSize)) c.retryFailedResets() - - used := c.updateUsed() - if used <= int64(c.opt.CacheMaxSize) && len(c.errItems) == 0 { - break - } } // Was kicked? From f49be033c6f03a858639353c949447f04017d7b0 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 25 Jul 2022 10:52:49 +0100 Subject: [PATCH 212/560] mega: Fix nil pointer exception when bad node received Fixes: #6336 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2ff7f3f13ec0c..9e9e8c6ef5d59 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.2 - github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8 + github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf github.com/winfsp/cgofuse v1.5.1-0.20220421173602-ce7e5a65cac7 github.com/xanzy/ssh-agent v0.3.1 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a diff --git a/go.sum b/go.sum index 008e5a7f9652c..f00338169624c 100644 --- a/go.sum +++ b/go.sum @@ -593,6 +593,8 @@ github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8 h1:IGJQmLBLYBdAknj21W3JsVof0yjEXfy1Q0K3YZebDOg= github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0= +github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf h1:Y43S3e9P1NPs/QF4R5/SdlXj2d31540hP4Gk8VKNvDg= +github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf/go.mod h1:c+cGNU1qi9bO7ZF4IRMYk+KaZTNiQ/gQrSbyMmGFq1Q= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= From 918bd6d3c3b28606e0208ee657b25274e4a06a18 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 17 Jul 2022 12:47:40 +0100 Subject: [PATCH 213/560] dropox: fix ChangeNotify was unable to decrypt errors Before this fix, the dropbox backend wasn't decoding the file names received in changenotify events into rclone standard format. This meant that changenotify events for filenames which had encoded characters were failing to be decrypted properly if wrapped in crypt. See: https://forum.rclone.org/t/rclone-vfs-cache-says-file-name-too-long/31535 --- backend/dropbox/dropbox.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 100044862440c..ca026e8c91176 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -1435,7 +1435,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs. } if entryPath != "" { - notifyFunc(entryPath, entryType) + notifyFunc(f.opt.Enc.ToStandardPath(entryPath), entryType) } } if !changeList.HasMore { From 876f791ecdf6a7801a74492e8a6b9050b9ffe8a9 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 5 Aug 2022 16:04:54 +0100 Subject: [PATCH 214/560] Revert "build: lint with go1.18 until golangci-lint is updated" This reverts commit 2170376d1be915a24cc94046e30017ce47cfabc6. --- .github/workflows/build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae8d40ad2fcd4..c7c76dee1ed6b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -227,9 +227,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - with: - # FIXME temporary until golangci-lint supports go1.19 - go-version: 1.18.x - name: Code quality test uses: golangci/golangci-lint-action@v3 From 6fd9e3d7170551b8272e08a206ae5d8fbf1ecebf Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 5 Aug 2022 16:35:41 +0100 Subject: [PATCH 215/560] build: reformat comments to pass go1.19 vet See: https://go.dev/doc/go1.19#go-doc --- backend/amazonclouddrive/amazonclouddrive.go | 13 ++-- backend/azureblob/azureblob.go | 26 +++---- backend/b2/b2.go | 52 +++++++------- backend/box/box.go | 21 +++--- backend/chunker/chunker.go | 18 ++--- backend/combine/combine.go | 8 +-- backend/compress/compress.go | 8 +-- backend/crypt/cipher.go | 12 ++-- backend/crypt/crypt.go | 8 +-- backend/drive/drive.go | 13 ++-- backend/dropbox/dropbox.go | 12 ++-- backend/filefabric/filefabric.go | 17 +++-- backend/ftp/ftp.go | 13 ++-- .../googlecloudstorage/googlecloudstorage.go | 8 +-- backend/googlephotos/googlephotos.go | 2 +- backend/googlephotos/pattern.go | 2 +- backend/hdfs/fs.go | 4 +- backend/internetarchive/internetarchive.go | 4 +- backend/jottacloud/jottacloud.go | 12 ++-- backend/local/local.go | 4 +- backend/local/setbtime_windows.go | 2 +- backend/mailru/mailru.go | 7 +- backend/mega/mega.go | 15 ++-- backend/memory/memory.go | 6 +- backend/onedrive/api/types.go | 6 +- backend/onedrive/onedrive.go | 15 ++-- backend/opendrive/opendrive.go | 13 ++-- backend/pcloud/pcloud.go | 15 ++-- backend/premiumizeme/premiumizeme.go | 15 ++-- backend/putio/fs.go | 10 +-- backend/putio/object.go | 2 +- backend/qingstor/qingstor.go | 6 +- backend/s3/s3.go | 8 +-- backend/seafile/seafile.go | 8 +-- backend/sharefile/sharefile.go | 19 +++--- backend/storj/fs.go | 4 +- backend/sugarsync/sugarsync.go | 19 +++--- backend/swift/swift.go | 18 ++--- backend/union/union.go | 8 +-- backend/uptobox/uptobox.go | 2 +- backend/webdav/api/types.go | 68 ++++++++++--------- backend/webdav/webdav.go | 16 ++--- backend/yandex/api/types.go | 4 +- backend/yandex/yandex.go | 14 ++-- backend/zoho/zoho.go | 17 +++-- cmd/bisync/bisync_test.go | 3 +- cmd/bisync/listing.go | 4 +- cmd/cmount/mount_brew.go | 2 +- cmd/cmount/mountpoint_windows.go | 2 +- cmd/serve/docker/options.go | 20 +++--- cmd/serve/ftp/ftp.go | 39 +++++------ cmdtest/cmdtest.go | 1 - fs/bwtimetable.go | 2 +- fs/chunksize/chunksize.go | 12 ++-- fs/config/authorize.go | 6 +- fs/filter/filter.go | 4 +- fs/log.go | 16 ++--- fs/mount_helper.go | 4 +- fs/open_options.go | 8 +-- fs/operations/operations.go | 8 +-- fs/operations/rc_test.go | 3 +- fs/operations/reopen.go | 2 +- fs/rc/params.go | 1 - fs/rc/webgui/plugins.go | 4 +- fs/registry.go | 14 ++-- fs/sync/sync.go | 11 +-- fs/walk/walk.go | 2 +- fstest/fstests/fstests.go | 2 +- fstest/test_all/test_all.go | 1 - lib/atexit/atexit.go | 2 +- lib/bucket/bucket.go | 2 +- lib/dircache/dircache.go | 10 +-- lib/encoder/os_windows.go | 22 +++--- lib/encoder/standard.go | 5 +- lib/errors/errors.go | 9 +-- lib/oauthutil/oauthutil.go | 10 +-- lib/pacer/pacer.go | 2 +- lib/plugin/package.go | 2 +- lib/rest/rest.go | 6 +- librclone/gomobile/gomobile.go | 4 +- librclone/librclone.go | 16 ++--- librclone/librclone/librclone.go | 2 +- vfs/dir.go | 2 +- vfs/file.go | 16 ++--- vfs/help.go | 3 +- vfs/read.go | 2 +- vfs/read_write.go | 2 +- vfs/vfscache/cache.go | 2 +- vfs/vfscache/downloaders/downloaders.go | 4 +- vfs/vfscache/downloaders/downloaders_test.go | 2 +- vfs/vfscache/item.go | 8 +-- vfs/vfscache/writeback/writeback.go | 2 +- vfs/vfstest/submount.go | 7 +- 93 files changed, 444 insertions(+), 443 deletions(-) diff --git a/backend/amazonclouddrive/amazonclouddrive.go b/backend/amazonclouddrive/amazonclouddrive.go index e5f5512ed2794..1bc6bcb8453c7 100644 --- a/backend/amazonclouddrive/amazonclouddrive.go +++ b/backend/amazonclouddrive/amazonclouddrive.go @@ -556,9 +556,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // // This is a workaround for Amazon sometimes returning // -// * 408 REQUEST_TIMEOUT -// * 504 GATEWAY_TIMEOUT -// * 500 Internal server error +// - 408 REQUEST_TIMEOUT +// - 504 GATEWAY_TIMEOUT +// - 500 Internal server error // // At the end of large uploads. The speculation is that the timeout // is waiting for the sha1 hashing to complete and the file may well @@ -626,7 +626,7 @@ func (f *Fs) checkUpload(ctx context.Context, resp *http.Response, in io.Reader, // Put the object into the container // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -685,9 +685,9 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1002,7 +1002,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 5597ed21fd6c1..0814b0168041d 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -1115,7 +1115,7 @@ func (f *Fs) listContainersToFn(fn listContainerFn) error { // Put the object into the container // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -1247,9 +1247,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1356,11 +1356,12 @@ func (o *Object) setMetadata(metadata azblob.Metadata) { // decodeMetaDataFromPropertiesResponse sets the metadata from the data passed in // // Sets -// o.id -// o.modTime -// o.size -// o.md5 -// o.meta +// +// o.id +// o.modTime +// o.size +// o.md5 +// o.meta func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetPropertiesResponse) (err error) { metadata := info.NewMetadata() size := info.ContentLength() @@ -1443,10 +1444,11 @@ func (o *Object) clearMetaData() { // readMetaData gets the metadata if it hasn't already been fetched // // Sets -// o.id -// o.modTime -// o.size -// o.md5 +// +// o.id +// o.modTime +// o.size +// o.md5 func (o *Object) readMetaData() (err error) { container, _ := o.split() if !o.fs.containerOK(container) { diff --git a/backend/b2/b2.go b/backend/b2/b2.go index 322aa50a56efd..6fd77b6a0c30c 100644 --- a/backend/b2/b2.go +++ b/backend/b2/b2.go @@ -656,15 +656,15 @@ var errEndList = errors.New("end list") // // (bucket, directory) is the starting directory // -// If prefix is set then it is removed from all file names +// If prefix is set then it is removed from all file names. // // If addBucket is set then it adds the bucket to the start of the -// remotes generated +// remotes generated. // -// If recurse is set the function will recursively list +// If recurse is set the function will recursively list. // // If limit is > 0 then it limits to that many files (must be less -// than 1000) +// than 1000). // // If hidden is set then it will list the hidden (deleted) files too. // @@ -1025,7 +1025,7 @@ func (f *Fs) clearBucketID(bucket string) { // Put the object into the bucket // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -1334,9 +1334,9 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo * // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1478,7 +1478,7 @@ func (o *Object) Size() int64 { // Clean the SHA1 // -// Make sure it is lower case +// Make sure it is lower case. // // Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html // Some tools (e.g. Cyberduck) use this @@ -1490,10 +1490,11 @@ func cleanSHA1(sha1 string) string { // decodeMetaDataRaw sets the metadata from the data passed in // // Sets -// o.id -// o.modTime -// o.size -// o.sha1 +// +// o.id +// o.modTime +// o.size +// o.sha1 func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp api.Timestamp, Info map[string]string, mimeType string) (err error) { o.id = ID o.sha1 = SHA1 @@ -1512,10 +1513,11 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp // decodeMetaData sets the metadata in the object from an api.File // // Sets -// o.id -// o.modTime -// o.size -// o.sha1 +// +// o.id +// o.modTime +// o.size +// o.sha1 func (o *Object) decodeMetaData(info *api.File) (err error) { return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType) } @@ -1523,10 +1525,11 @@ func (o *Object) decodeMetaData(info *api.File) (err error) { // decodeMetaDataFileInfo sets the metadata in the object from an api.FileInfo // // Sets -// o.id -// o.modTime -// o.size -// o.sha1 +// +// o.id +// o.modTime +// o.size +// o.sha1 func (o *Object) decodeMetaDataFileInfo(info *api.FileInfo) (err error) { return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType) } @@ -1584,10 +1587,11 @@ func (o *Object) getMetaData(ctx context.Context) (info *api.File, err error) { // readMetaData gets the metadata if it hasn't already been fetched // // Sets -// o.id -// o.modTime -// o.size -// o.sha1 +// +// o.id +// o.modTime +// o.size +// o.sha1 func (o *Object) readMetaData(ctx context.Context) (err error) { if o.id != "" { return nil diff --git a/backend/box/box.go b/backend/box/box.go index 55d8a40303c08..93b2cd0ccb506 100644 --- a/backend/box/box.go +++ b/backend/box/box.go @@ -692,7 +692,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Creates from the parameters passed in a half finished Object which // must have setMetaData called on it // -// Returns the object, leaf, directoryID and error +// Returns the object, leaf, directoryID and error. // // Used to create new objects func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { @@ -752,7 +752,7 @@ func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -792,9 +792,9 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt // PutUnchecked the object into the container // -// This will produce an error if the object already exists +// This will produce an error if the object already exists. // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -877,9 +877,9 @@ func (f *Fs) Precision() time.Duration { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -995,9 +995,9 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1235,7 +1235,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { @@ -1346,9 +1345,9 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str // Update the object with the contents of the io.Reader, modTime and size // -// If existing is set then it updates the object rather than creating a new one +// If existing is set then it updates the object rather than creating a new one. // -// The new object may have been created if an error is returned +// The new object may have been created if an error is returned. func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { if o.fs.tokenRenewer != nil { o.fs.tokenRenewer.Start() diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.go index bfb3e3e65ef5d..4ed8ec9e87294 100644 --- a/backend/chunker/chunker.go +++ b/backend/chunker/chunker.go @@ -32,7 +32,6 @@ import ( "github.com/rclone/rclone/fs/operations" ) -// // Chunker's composite files have one or more chunks // and optional metadata object. If it's present, // meta object is named after the original file. @@ -79,7 +78,6 @@ import ( // Metadata format v1 does not define any control chunk types, // they are currently ignored aka reserved. // In future they can be used to implement resumable uploads etc. -// const ( ctrlTypeRegStr = `[a-z][a-z0-9]{2,6}` tempSuffixFormat = `_%04s` @@ -542,7 +540,6 @@ func (f *Fs) setChunkNameFormat(pattern string) error { // // xactID is a transaction identifier. Empty xactID denotes active chunk, // otherwise temporary chunk name is produced. -// func (f *Fs) makeChunkName(filePath string, chunkNo int, ctrlType, xactID string) string { dir, parentName := path.Split(filePath) var name, tempSuffix string @@ -708,7 +705,6 @@ func (f *Fs) newXactID(ctx context.Context, filePath string) (xactID string, err // directory together with dead chunks. // In future a flag named like `--chunker-list-hidden` may be added to // rclone that will tell List to reveal hidden chunks. -// func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { entries, err = f.base.List(ctx, dir) if err != nil { @@ -868,7 +864,6 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP // Note that chunker prefers analyzing file names rather than reading // the content of meta object assuming that directory scans are fast // but opening even a small file can be slow on some backends. -// func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { return f.scanObject(ctx, remote, false) } @@ -1586,7 +1581,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error { // This command will chain to `purge` from wrapped remote. // As a result it removes not only composite chunker files with their // active chunks but also all hidden temporary chunks in the directory. -// func (f *Fs) Purge(ctx context.Context, dir string) error { do := f.base.Features().Purge if do == nil { @@ -1628,7 +1622,6 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // Unsupported control chunks will get re-picked by a more recent // rclone version with unexpected results. This can be helped by // the `delete hidden` flag above or at least the user has been warned. -// func (o *Object) Remove(ctx context.Context) (err error) { if err := o.f.forbidChunk(o, o.Remote()); err != nil { // operations.Move can still call Remove if chunker's Move refuses @@ -1804,9 +1797,9 @@ func (f *Fs) okForServerSide(ctx context.Context, src fs.Object, opName string) // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1825,9 +1818,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -2125,7 +2118,6 @@ func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error { // file, then tries to read it from metadata. This in theory // handles the unusual case when a small file has been tampered // on the level of wrapped remote but chunker is unaware of that. -// func (o *Object) Hash(ctx context.Context, hashType hash.Type) (string, error) { if err := o.readMetadata(ctx); err != nil { return "", err // valid metadata is required to get hash, abort @@ -2414,7 +2406,6 @@ type metaSimpleJSON struct { // - for files larger than chunk size // - if file contents can be mistaken as meta object // - if consistent hashing is On but wrapped remote can't provide given hash -// func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1, xactID string) ([]byte, error) { version := metadataVersion if xactID == "" && version == 2 { @@ -2447,7 +2438,6 @@ func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1, // New format will have a higher version number and cannot be correctly // handled by current implementation. // The version check below will then explicitly ask user to upgrade rclone. -// func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte) (info *ObjectInfo, madeByChunker bool, err error) { // Be strict about JSON format // to reduce possibility that a random small file resembles metadata. diff --git a/backend/combine/combine.go b/backend/combine/combine.go index bae4bd8a1c822..dab80a82b5b30 100644 --- a/backend/combine/combine.go +++ b/backend/combine/combine.go @@ -457,9 +457,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -491,9 +491,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/compress/compress.go b/backend/compress/compress.go index 9033b38ff4f11..b53e725b8a8b8 100644 --- a/backend/compress/compress.go +++ b/backend/compress/compress.go @@ -785,9 +785,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // Copy src to this remote using server side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -835,9 +835,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // Move src to this remote using server side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/crypt/cipher.go b/backend/crypt/cipher.go index 1b265ad19e918..10485776b0e3e 100644 --- a/backend/crypt/cipher.go +++ b/backend/crypt/cipher.go @@ -127,8 +127,8 @@ type fileNameEncoding interface { // RFC4648 // // The standard encoding is modified in two ways -// * it becomes lower case (no-one likes upper case filenames!) -// * we strip the padding character `=` +// - it becomes lower case (no-one likes upper case filenames!) +// - we strip the padding character `=` type caseInsensitiveBase32Encoding struct{} // EncodeToString encodes a strign using the modified version of @@ -244,7 +244,7 @@ func (c *Cipher) putBlock(buf []byte) { // encryptSegment encrypts a path segment // -// This uses EME with AES +// This uses EME with AES. // // EME (ECB-Mix-ECB) is a wide-block encryption mode presented in the // 2003 paper "A Parallelizable Enciphering Mode" by Halevi and @@ -254,8 +254,8 @@ func (c *Cipher) putBlock(buf []byte) { // same filename must encrypt to the same thing. // // This means that -// * filenames with the same name will encrypt the same -// * filenames which start the same won't have a common prefix +// - filenames with the same name will encrypt the same +// - filenames which start the same won't have a common prefix func (c *Cipher) encryptSegment(plaintext string) string { if plaintext == "" { return "" @@ -1085,7 +1085,7 @@ func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) { // DecryptDataSeek decrypts the data stream from offset // -// The open function must return a ReadCloser opened to the offset supplied +// The open function must return a ReadCloser opened to the offset supplied. // // You must use this form of DecryptData if you might want to Seek the file handle func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) { diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go index 6f599cafb2c90..bae2c97999afb 100644 --- a/backend/crypt/crypt.go +++ b/backend/crypt/crypt.go @@ -507,9 +507,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -532,9 +532,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/drive/drive.go b/backend/drive/drive.go index d2f92d6920134..b4f310eafb172 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -2180,7 +2180,7 @@ func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Tim // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -2414,9 +2414,9 @@ func (f *Fs) Precision() time.Duration { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -2649,9 +2649,9 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -3569,7 +3569,6 @@ func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) ( // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *baseObject) ModTime(ctx context.Context) time.Time { @@ -3826,7 +3825,7 @@ func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadM // Update the already existing object // -// Copy the reader into the object updating modTime and size +// Copy the reader into the object updating modTime and size. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index ca026e8c91176..42686394d62ef 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -925,7 +925,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -1044,9 +1044,9 @@ func (f *Fs) Precision() time.Duration { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1105,9 +1105,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) (err error) { // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1763,7 +1763,7 @@ func checkPathLength(name string) (err error) { // Update the already existing object // -// Copy the reader into the object updating modTime and size +// Copy the reader into the object updating modTime and size. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { diff --git a/backend/filefabric/filefabric.go b/backend/filefabric/filefabric.go index a70e1f7aea2c3..fa3e7a178f226 100644 --- a/backend/filefabric/filefabric.go +++ b/backend/filefabric/filefabric.go @@ -373,7 +373,7 @@ type params map[string]interface{} // rpc calls the rpc.php method of the SME file fabric // -// This is an entry point to all the method calls +// This is an entry point to all the method calls. // // If result is nil then resp.Body will need closing func (f *Fs) rpc(ctx context.Context, function string, p params, result api.OKError, options []fs.OpenOption) (resp *http.Response, err error) { @@ -678,7 +678,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Creates from the parameters passed in a half finished Object which // must have setMetaData called on it // -// Returns the object, leaf, directoryID and error +// Returns the object, leaf, directoryID and error. // // Used to create new objects func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { @@ -697,7 +697,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -783,9 +783,9 @@ func (f *Fs) Precision() time.Duration { // Copy src to this remote using server side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -956,9 +956,9 @@ func (f *Fs) move(ctx context.Context, isDir bool, id, oldLeaf, newLeaf, oldDire // Move src to this remote using server side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1135,7 +1135,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { @@ -1201,7 +1200,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // Update the object with the contents of the io.Reader, modTime and size // -// If existing is set then it updates the object rather than creating a new one +// If existing is set then it updates the object rather than creating a new one. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index d92678f08a71b..24916d1613dac 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -769,11 +769,12 @@ func (f *Fs) Hashes() hash.Set { // Precision shows whether modified time is supported or not depending on the // FTP server capabilities, namely whether FTP server: -// - accepts the MDTM command to get file time (fGetTime) -// or supports MLSD returning precise file time in the list (fLstTime) -// - accepts the MFMT command to set file time (fSetTime) -// or non-standard form of the MDTM command (fSetTime, too) -// used by VsFtpd for the same purpose (WritingMDTM) +// - accepts the MDTM command to get file time (fGetTime) +// or supports MLSD returning precise file time in the list (fLstTime) +// - accepts the MFMT command to set file time (fSetTime) +// or non-standard form of the MDTM command (fSetTime, too) +// used by VsFtpd for the same purpose (WritingMDTM) +// // See "mdtm_write" in https://security.appspot.com/vsftpd/vsftpd_conf.html func (f *Fs) Precision() time.Duration { if (f.fGetTime || f.fLstTime) && f.fSetTime { @@ -1149,7 +1150,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read // Update the already existing object // -// Copy the reader into the object updating modTime and size +// Copy the reader into the object updating modTime and size. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index b4d355021b053..0e6a2b4ed7136 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -580,7 +580,7 @@ type listFn func(remote string, object *storage.Object, isDirectory bool) error // // dir is the starting directory, "" for root // -// Set recurse to read sub directories +// Set recurse to read sub directories. // // The remote has prefix removed from it and if addBucket is set // then it adds the bucket to the start. @@ -798,7 +798,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( // Put the object into the bucket // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -900,9 +900,9 @@ func (f *Fs) Precision() time.Duration { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/googlephotos/googlephotos.go b/backend/googlephotos/googlephotos.go index 4e16d81d39e19..238d6c783a3dc 100644 --- a/backend/googlephotos/googlephotos.go +++ b/backend/googlephotos/googlephotos.go @@ -661,7 +661,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Put the object into the bucket // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { diff --git a/backend/googlephotos/pattern.go b/backend/googlephotos/pattern.go index e2cbcbf724869..fc84f4a0e790e 100644 --- a/backend/googlephotos/pattern.go +++ b/backend/googlephotos/pattern.go @@ -315,7 +315,7 @@ func yearMonthDayFilter(ctx context.Context, f lister, match []string) (sf api.S // featureFilter creates a filter for the Feature enum // -// The API only supports one feature, FAVORITES, so hardcode that feature +// The API only supports one feature, FAVORITES, so hardcode that feature. // // https://developers.google.com/photos/library/reference/rest/v1/mediaItems/search#FeatureFilter func featureFilter(ctx context.Context, f lister, match []string) (sf api.SearchFilter) { diff --git a/backend/hdfs/fs.go b/backend/hdfs/fs.go index a68372db69aaf..7d48ffed60ea5 100644 --- a/backend/hdfs/fs.go +++ b/backend/hdfs/fs.go @@ -265,9 +265,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/internetarchive/internetarchive.go b/backend/internetarchive/internetarchive.go index db553eda19e43..b3d4a55cb302d 100644 --- a/backend/internetarchive/internetarchive.go +++ b/backend/internetarchive/internetarchive.go @@ -577,9 +577,9 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 3bdc98ec6523c..0044c7f79d1e2 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -1247,7 +1247,7 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -1397,9 +1397,9 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1433,9 +1433,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1829,7 +1829,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, // Update the object with the contents of the io.Reader, modTime and size // -// If existing is set then it updates the object rather than creating a new one +// If existing is set then it updates the object rather than creating a new one. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { diff --git a/backend/local/local.go b/backend/local/local.go index 3a83bd2623442..c625b426042b4 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -695,9 +695,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/local/setbtime_windows.go b/backend/local/setbtime_windows.go index 3734c2e7b5584..2b8fd98d5e1b1 100644 --- a/backend/local/setbtime_windows.go +++ b/backend/local/setbtime_windows.go @@ -5,8 +5,8 @@ package local import ( "os" - "time" "syscall" + "time" ) const haveSetBTime = true diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go index a9a9bab2dc38d..bd527e8e23b6c 100644 --- a/backend/mailru/mailru.go +++ b/backend/mailru/mailru.go @@ -630,9 +630,10 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt // itemToEntry converts API item to rclone directory entry // The dirSize return value is: -// <0 - for a file or in case of error -// =0 - for an empty directory -// >0 - for a non-empty directory +// +// <0 - for a file or in case of error +// =0 - for an empty directory +// >0 - for a non-empty directory func (f *Fs) itemToDirEntry(ctx context.Context, item *api.ListItem) (entry fs.DirEntry, dirSize int, err error) { remote, err := f.relPath(f.opt.Enc.ToStandardPath(item.Home)) if err != nil { diff --git a/backend/mega/mega.go b/backend/mega/mega.go index 7e154fe2d99f9..bb038506be834 100644 --- a/backend/mega/mega.go +++ b/backend/mega/mega.go @@ -118,7 +118,7 @@ type Fs struct { // Object describes a mega object // -// Will definitely have info but maybe not meta +// Will definitely have info but maybe not meta. // // Normally rclone would just store an ID here but go-mega and mega.nz // expect you to build an entire tree of all the objects in memory. @@ -536,7 +536,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Creates from the parameters passed in a half finished Object which // must have setMetaData called on it // -// Returns the dirNode, object, leaf and error +// Returns the dirNode, object, leaf and error. // // Used to create new objects func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) { @@ -554,7 +554,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned // PutUnchecked uploads the object @@ -576,7 +576,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . // PutUnchecked the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned // PutUnchecked uploads the object @@ -749,9 +749,9 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -979,7 +979,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { @@ -1115,7 +1114,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // Update the object with the contents of the io.Reader, modTime and size // -// If existing is set then it updates the object rather than creating a new one +// If existing is set then it updates the object rather than creating a new one. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { diff --git a/backend/memory/memory.go b/backend/memory/memory.go index 9675abcd38fc9..838a060d59c53 100644 --- a/backend/memory/memory.go +++ b/backend/memory/memory.go @@ -418,7 +418,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( // Put the object into the bucket // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -463,9 +463,9 @@ func (f *Fs) Precision() time.Duration { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/onedrive/api/types.go b/backend/onedrive/api/types.go index 236c1c68218b4..12dca863b0281 100644 --- a/backend/onedrive/api/types.go +++ b/backend/onedrive/api/types.go @@ -250,8 +250,8 @@ type MoveItemRequest struct { FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo,omitempty"` // File system information on client. Read-write. } -//CreateShareLinkRequest is the request to create a sharing link -//Always Type:view and Scope:anonymous for public sharing +// CreateShareLinkRequest is the request to create a sharing link +// Always Type:view and Scope:anonymous for public sharing type CreateShareLinkRequest struct { Type string `json:"type"` // Link type in View, Edit or Embed Scope string `json:"scope,omitempty"` // Scope in anonymous, organization @@ -259,7 +259,7 @@ type CreateShareLinkRequest struct { Expiry *time.Time `json:"expirationDateTime,omitempty"` // A String with format of yyyy-MM-ddTHH:mm:ssZ of DateTime indicates the expiration time of the permission. } -//CreateShareLinkResponse is the response from CreateShareLinkRequest +// CreateShareLinkResponse is the response from CreateShareLinkRequest type CreateShareLinkResponse struct { ID string `json:"id"` Roles []string `json:"roles"` diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index b209b18f473b8..c0674c9c596c5 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -727,7 +727,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err // "shared with me" folders in OneDrive Personal (See #2536, #2778) // This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480 // -// If `relPath` == '', do not append the slash (See #3664) +// If `relPath` == ”, do not append the slash (See #3664) func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) { opts, _ := f.newOptsCallWithIDPath(normalizedID, relPath, true, "GET", "") @@ -1137,7 +1137,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Creates from the parameters passed in a half finished Object which // must have setMetaData called on it // -// Returns the object, leaf, directoryID and error +// Returns the object, leaf, directoryID and error. // // Used to create new objects func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { @@ -1156,7 +1156,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, // Put the object into the container // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -1280,9 +1280,9 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1387,9 +1387,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1843,7 +1843,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.go index ac1a1c01df8ab..35e1871c31c2e 100644 --- a/backend/opendrive/opendrive.go +++ b/backend/opendrive/opendrive.go @@ -340,9 +340,9 @@ func (f *Fs) Precision() time.Duration { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -404,9 +404,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -562,7 +562,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { // Creates from the parameters passed in a half finished Object which // must have setMetaData called on it // -// Returns the object, leaf, directoryID and error +// Returns the object, leaf, directoryID and error. // // Used to create new objects func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { @@ -599,7 +599,7 @@ func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *Fold // Put the object into the bucket // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -829,7 +829,6 @@ func (o *Object) Size() int64 { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index c558e4495a066..726d9ee8bd3fc 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -588,7 +588,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( // Creates from the parameters passed in a half finished Object which // must have setMetaData called on it // -// Returns the object, leaf, directoryID and error +// Returns the object, leaf, directoryID and error. // // Used to create new objects func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { @@ -607,7 +607,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, // Put the object into the container // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -681,9 +681,9 @@ func (f *Fs) Precision() time.Duration { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -766,9 +766,9 @@ func (f *Fs) CleanUp(ctx context.Context) error { // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1073,7 +1073,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { @@ -1152,7 +1151,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // Update the object with the contents of the io.Reader, modTime and size // -// If existing is set then it updates the object rather than creating a new one +// If existing is set then it updates the object rather than creating a new one. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { diff --git a/backend/premiumizeme/premiumizeme.go b/backend/premiumizeme/premiumizeme.go index 3b02a3ac3176e..8f3aa1ce8c377 100644 --- a/backend/premiumizeme/premiumizeme.go +++ b/backend/premiumizeme/premiumizeme.go @@ -493,7 +493,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Creates from the parameters passed in a half finished Object which // must have setMetaData called on it // -// Returns the object, leaf, directoryID and error +// Returns the object, leaf, directoryID and error. // // Used to create new objects func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { @@ -512,7 +512,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -530,9 +530,9 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . // PutUnchecked the object into the container // -// This will produce an error if the object already exists +// This will produce an error if the object already exists. // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -694,9 +694,9 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -870,7 +870,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { @@ -917,7 +916,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // Update the object with the contents of the io.Reader, modTime and size // -// If existing is set then it updates the object rather than creating a new one +// If existing is set then it updates the object rather than creating a new one. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { diff --git a/backend/putio/fs.go b/backend/putio/fs.go index 3e2855b7daaa6..cbcda547f7475 100644 --- a/backend/putio/fs.go +++ b/backend/putio/fs.go @@ -230,7 +230,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) { @@ -523,9 +523,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) (err error) { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -562,9 +562,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/putio/object.go b/backend/putio/object.go index 28caa5d1e4a55..bd03982caf7c8 100644 --- a/backend/putio/object.go +++ b/backend/putio/object.go @@ -261,7 +261,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // Update the already existing object // -// Copy the reader into the object updating modTime and size +// Copy the reader into the object updating modTime and size. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go index 9e9b0c6cc9b0a..cd4880358c725 100644 --- a/backend/qingstor/qingstor.go +++ b/backend/qingstor/qingstor.go @@ -431,9 +431,9 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -477,7 +477,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { // Return an Object from a path // -//If it can't be found it returns the error ErrorObjectNotFound. +// If it can't be found it returns the error ErrorObjectNotFound. func (f *Fs) newObjectWithInfo(remote string, info *qs.KeyType) (fs.Object, error) { o := &Object{ fs: f, diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 753e800f0e7f4..35745cbd5e1cf 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2179,7 +2179,7 @@ var retryErrorCodes = []int{ 503, // Service Unavailable/Slow Down - "Reduce your request rate" } -//S3 is pretty resilient, and the built in retry handling is probably sufficient +// S3 is pretty resilient, and the built in retry handling is probably sufficient // as it should notice closed connections and timeouts which are the most likely // sort of failure modes func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) { @@ -2671,7 +2671,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // Return an Object from a path // -//If it can't be found it returns the error ErrorObjectNotFound. +// If it can't be found it returns the error ErrorObjectNotFound. func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Object) (fs.Object, error) { o := &Object{ fs: f, @@ -3336,9 +3336,9 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/seafile/seafile.go b/backend/seafile/seafile.go index ffae61a5ae1dc..00457676c457f 100644 --- a/backend/seafile/seafile.go +++ b/backend/seafile/seafile.go @@ -671,9 +671,9 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) e // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // If it isn't possible then return fs.ErrorCantCopy func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { @@ -722,9 +722,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // If it isn't possible then return fs.ErrorCantMove func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { diff --git a/backend/sharefile/sharefile.go b/backend/sharefile/sharefile.go index e6add47c2e6ac..acb5d65c85b2c 100644 --- a/backend/sharefile/sharefile.go +++ b/backend/sharefile/sharefile.go @@ -741,7 +741,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Creates from the parameters passed in a half finished Object which // must have setMetaData called on it // -// Returns the object, leaf, directoryID and error +// Returns the object, leaf, directoryID and error. // // Used to create new objects func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { @@ -760,7 +760,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -783,9 +783,9 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt // PutUnchecked the object into the container // -// This will produce an error if the object already exists +// This will produce an error if the object already exists. // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -973,9 +973,9 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1043,9 +1043,9 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1256,7 +1256,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { @@ -1324,7 +1323,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // Update the object with the contents of the io.Reader, modTime and size // -// If existing is set then it updates the object rather than creating a new one +// If existing is set then it updates the object rather than creating a new one. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { diff --git a/backend/storj/fs.go b/backend/storj/fs.go index 95146edbfb34b..b39e0b25d8e52 100644 --- a/backend/storj/fs.go +++ b/backend/storj/fs.go @@ -683,9 +683,9 @@ func newPrefix(prefix string) string { // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/sugarsync/sugarsync.go b/backend/sugarsync/sugarsync.go index 2041bdc05c9f8..7078de975fdf0 100644 --- a/backend/sugarsync/sugarsync.go +++ b/backend/sugarsync/sugarsync.go @@ -713,7 +713,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // Creates from the parameters passed in a half finished Object which // must have setMetaData called on it // -// Returns the object, leaf, directoryID and error +// Returns the object, leaf, directoryID and error. // // Used to create new objects func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { @@ -732,7 +732,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -755,9 +755,9 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt // PutUnchecked the object into the container // -// This will produce an error if the object already exists +// This will produce an error if the object already exists. // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -852,9 +852,9 @@ func (f *Fs) Precision() time.Duration { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -985,9 +985,9 @@ func (f *Fs) moveDir(ctx context.Context, id, leaf, directoryID string) (err err // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1156,7 +1156,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { @@ -1229,7 +1228,7 @@ func (f *Fs) createFile(ctx context.Context, pathID, leaf, mimeType string) (new // Update the object with the contents of the io.Reader, modTime and size // -// If existing is set then it updates the object rather than creating a new one +// If existing is set then it updates the object rather than creating a new one. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { diff --git a/backend/swift/swift.go b/backend/swift/swift.go index e4bae345e8be6..bea898bda2d9c 100644 --- a/backend/swift/swift.go +++ b/backend/swift/swift.go @@ -790,7 +790,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { // Put the object into the container // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -902,9 +902,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1019,7 +1019,7 @@ func copyLargeObject(ctx context.Context, f *Fs, src *Object, dstContainer strin return err } -//remove copied segments when copy process failed +// remove copied segments when copy process failed func handleCopyFail(ctx context.Context, f *Fs, segmentsContainer string, segments []string, err error) { fs.Debugf(f, "handle copy segment fail") if err == nil { @@ -1140,10 +1140,11 @@ func (o *Object) Size() int64 { // decodeMetaData sets the metadata in the object from a swift.Object // // Sets -// o.lastModified -// o.size -// o.md5 -// o.contentType +// +// o.lastModified +// o.size +// o.md5 +// o.contentType func (o *Object) decodeMetaData(info *swift.Object) (err error) { o.lastModified = info.LastModified o.size = info.Bytes @@ -1184,7 +1185,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { diff --git a/backend/union/union.go b/backend/union/union.go index 0e894a50b063c..8bcbf66c940c1 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -213,9 +213,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -253,9 +253,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // diff --git a/backend/uptobox/uptobox.go b/backend/uptobox/uptobox.go index 88faefce416f7..6bc5bd8235d93 100644 --- a/backend/uptobox/uptobox.go +++ b/backend/uptobox/uptobox.go @@ -991,7 +991,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // Update the already existing object // -// Copy the reader into the object updating modTime and size +// Copy the reader into the object updating modTime and size. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { diff --git a/backend/webdav/api/types.go b/backend/webdav/api/types.go index 44bdc7e27f031..7d36f058ab5ed 100644 --- a/backend/webdav/api/types.go +++ b/backend/webdav/api/types.go @@ -37,27 +37,29 @@ type Response struct { // This is a lazy way of decoding the multiple in the // response. // -// The response might look like this +// The response might look like this. // // -// /remote.php/webdav/Nextcloud%20Manual.pdf -// -// -// Tue, 19 Dec 2017 22:02:36 GMT -// 4143665 -// -// "048d7be4437ff7deeae94db50ff3e209" -// application/pdf -// -// HTTP/1.1 200 OK -// -// -// -// -// -// -// HTTP/1.1 404 Not Found -// +// +// /remote.php/webdav/Nextcloud%20Manual.pdf +// +// +// Tue, 19 Dec 2017 22:02:36 GMT +// 4143665 +// +// "048d7be4437ff7deeae94db50ff3e209" +// application/pdf +// +// HTTP/1.1 200 OK +// +// +// +// +// +// +// HTTP/1.1 404 Not Found +// +// // // // So we elide the array of and within that the array of @@ -127,8 +129,10 @@ type PropValue struct { // Error is used to describe webdav errors // // -// Sabre\DAV\Exception\NotFound -// File with name Photo could not be located +// +// Sabre\DAV\Exception\NotFound +// File with name Photo could not be located +// // type Error struct { Exception string `xml:"exception,omitempty"` @@ -214,16 +218,18 @@ func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { // Quota is used to read the bytes used and available // // -// -// /remote.php/webdav/ -// -// -// -3 -// 376461895 -// -// HTTP/1.1 200 OK -// -// +// +// +// /remote.php/webdav/ +// +// +// -3 +// 376461895 +// +// HTTP/1.1 200 OK +// +// +// // type Quota struct { Available string `xml:"DAV: response>propstat>prop>quota-available-bytes"` diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go index d663ac0bfdd4d..ff5595f1b5159 100644 --- a/backend/webdav/webdav.go +++ b/backend/webdav/webdav.go @@ -800,7 +800,7 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -975,9 +975,9 @@ func (f *Fs) Precision() time.Duration { // Copy or Move src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1029,9 +1029,9 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1051,9 +1051,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1291,7 +1291,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // Update the object with the contents of the io.Reader, modTime and size // -// If existing is set then it updates the object rather than creating a new one +// If existing is set then it updates the object rather than creating a new one. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { diff --git a/backend/yandex/api/types.go b/backend/yandex/api/types.go index 7e346b1ffe153..78d4c6850bfca 100644 --- a/backend/yandex/api/types.go +++ b/backend/yandex/api/types.go @@ -20,7 +20,7 @@ type ResourceInfoRequestOptions struct { Fields []string } -//ResourceInfoResponse struct is returned by the API for metadata requests. +// ResourceInfoResponse struct is returned by the API for metadata requests. type ResourceInfoResponse struct { PublicKey string `json:"public_key"` Name string `json:"name"` @@ -61,7 +61,7 @@ type AsyncStatus struct { Status string `json:"status"` } -//CustomPropertyResponse struct we send and is returned by the API for CustomProperty request. +// CustomPropertyResponse struct we send and is returned by the API for CustomProperty request. type CustomPropertyResponse struct { CustomProperties map[string]interface{} `json:"custom_properties"` } diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.go index 7f0ddc5d8d4df..3d5bf53cc0c67 100644 --- a/backend/yandex/yandex.go +++ b/backend/yandex/yandex.go @@ -30,7 +30,7 @@ import ( "golang.org/x/oauth2" ) -//oAuth +// oAuth const ( rcloneClientID = "ac39b43b9eba4cae8ffb788c06d816a8" rcloneEncryptedClientSecret = "EfyyNZ3YUEwXM5yAhi72G9YwKn2mkFrYwJNS7cY0TJAhFlX9K-uJFbGlpO-RYjrJ" @@ -442,7 +442,7 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -693,9 +693,9 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dst string, overwrite // Copy src to this remote using server-side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -723,9 +723,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // Move src to this remote using server-side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1109,7 +1109,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT // Update the already existing object // -// Copy the reader into the object updating modTime and size +// Copy the reader into the object updating modTime and size. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.go index 6175be3c5f327..167fa03ea7551 100644 --- a/backend/zoho/zoho.go +++ b/backend/zoho/zoho.go @@ -644,7 +644,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, size int64, modTim // Put the object // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -714,9 +714,9 @@ func (f *Fs) upload(ctx context.Context, name string, parent string, size int64, // PutUnchecked the object into the container // -// This will produce an error if the object already exists +// This will produce an error if the object already exists. // -// Copy the reader in to the new object which is returned +// Copy the reader in to the new object which is returned. // // The new object may have been created if an error is returned func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { @@ -857,9 +857,9 @@ func (f *Fs) rename(ctx context.Context, id, name string) (item *api.Item, err e // Copy src to this remote using server side copy operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -960,9 +960,9 @@ func (f *Fs) move(ctx context.Context, srcID, parentID string) (item *api.Item, // Move src to this remote using server side move operations. // -// This is stored with the remote path given +// This is stored with the remote path given. // -// It returns the destination Object and a possible error +// It returns the destination Object and a possible error. // // Will only be called if src.Fs().Name() == f.Name() // @@ -1152,7 +1152,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) { // ModTime returns the modification time of the object // -// // It attempts to read the objects mtime and if that isn't present the // LastModified returned in the http headers func (o *Object) ModTime(ctx context.Context) time.Time { @@ -1236,7 +1235,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read // Update the object with the contents of the io.Reader, modTime and size // -// If existing is set then it updates the object rather than creating a new one +// If existing is set then it updates the object rather than creating a new one. // // The new object may have been created if an error is returned func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go index 52417b49005fd..758d786eacf06 100644 --- a/cmd/bisync/bisync_test.go +++ b/cmd/bisync/bisync_test.go @@ -56,7 +56,8 @@ const ( // logReplacements make modern test logs comparable with golden dir. // It is a string slice of even length with this structure: -// {`matching regular expression`, "mangled result string", ...} +// +// {`matching regular expression`, "mangled result string", ...} var logReplacements = []string{ // skip syslog facility markers `^(<[1-9]>)(INFO |ERROR |NOTICE|DEBUG ):(.*)$`, "$2:$3", diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.go index 610823c64a142..066b927e81843 100644 --- a/cmd/bisync/listing.go +++ b/cmd/bisync/listing.go @@ -24,8 +24,8 @@ const ListingHeader = "# bisync listing v1 from" // lineRegex and lineFormat define listing line format // -// flags <- size -> <- hash -> id <------------ modtime -----------> "<----- remote" -// - 3009805 md5:xxxxxx - 2006-01-02T15:04:05.000000000-0700 "12 - Wait.mp3" +// flags <- size -> <- hash -> id <------------ modtime -----------> "<----- remote" +// - 3009805 md5:xxxxxx - 2006-01-02T15:04:05.000000000-0700 "12 - Wait.mp3" // // flags: "-" for a file and "d" for a directory (reserved) // hash: "type:value" or "-" (example: "md5:378840336ab14afa9c6b8d887e68a340") diff --git a/cmd/cmount/mount_brew.go b/cmd/cmount/mount_brew.go index 26531ff7a7aa9..ec1b97881e277 100644 --- a/cmd/cmount/mount_brew.go +++ b/cmd/cmount/mount_brew.go @@ -8,7 +8,7 @@ package cmount import ( "errors" - + "github.com/rclone/rclone/cmd/mountlib" "github.com/rclone/rclone/vfs" ) diff --git a/cmd/cmount/mountpoint_windows.go b/cmd/cmount/mountpoint_windows.go index c58dbc0f3617e..5cd0aa1d06888 100644 --- a/cmd/cmount/mountpoint_windows.go +++ b/cmd/cmount/mountpoint_windows.go @@ -4,9 +4,9 @@ package cmount import ( + "errors" "fmt" "os" - "errors" "path/filepath" "regexp" diff --git a/cmd/serve/docker/options.go b/cmd/serve/docker/options.go index 3b6fb07f1fe2b..9026659796e58 100644 --- a/cmd/serve/docker/options.go +++ b/cmd/serve/docker/options.go @@ -19,16 +19,16 @@ import ( // applyOptions configures volume from request options. // // There are 5 special options: -// - "remote" aka "fs" determines existing remote from config file -// with a path or on-the-fly remote using the ":backend:" syntax. -// It is usually named "remote" in documentation but can be aliased as -// "fs" to avoid confusion with the "remote" option of some backends. -// - "type" is equivalent to the ":backend:" syntax (optional). -// - "path" provides explicit on-remote path for "type" (optional). -// - "mount-type" can be "mount", "cmount" or "mount2", defaults to -// first found (optional). -// - "persist" is reserved for future to create remotes persisted -// in rclone.conf similar to rcd (optional). +// - "remote" aka "fs" determines existing remote from config file +// with a path or on-the-fly remote using the ":backend:" syntax. +// It is usually named "remote" in documentation but can be aliased as +// "fs" to avoid confusion with the "remote" option of some backends. +// - "type" is equivalent to the ":backend:" syntax (optional). +// - "path" provides explicit on-remote path for "type" (optional). +// - "mount-type" can be "mount", "cmount" or "mount2", defaults to +// first found (optional). +// - "persist" is reserved for future to create remotes persisted +// in rclone.conf similar to rcd (optional). // // Unlike rcd we use the flat naming scheme for mount, vfs and backend // options without substructures. Dashes, underscores and mixed case diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go index 7ed96e3c0e385..11753e42fbf1d 100644 --- a/cmd/serve/ftp/ftp.go +++ b/cmd/serve/ftp/ftp.go @@ -185,26 +185,27 @@ func (s *server) serve() error { } // close stops the ftp server +// //lint:ignore U1000 unused when not building linux func (s *server) close() error { fs.Logf(s.f, "Stopping FTP on %s", s.srv.Hostname+":"+strconv.Itoa(s.srv.Port)) return s.srv.Shutdown() } -//Logger ftp logger output formatted message +// Logger ftp logger output formatted message type Logger struct{} -//Print log simple text message +// Print log simple text message func (l *Logger) Print(sessionID string, message interface{}) { fs.Infof(sessionID, "%s", message) } -//Printf log formatted text message +// Printf log formatted text message func (l *Logger) Printf(sessionID string, format string, v ...interface{}) { fs.Infof(sessionID, format, v...) } -//PrintCommand log formatted command execution +// PrintCommand log formatted command execution func (l *Logger) PrintCommand(sessionID string, command string, params string) { if command == "PASS" { fs.Infof(sessionID, "> PASS ****") @@ -213,7 +214,7 @@ func (l *Logger) PrintCommand(sessionID string, command string, params string) { } } -//PrintResponse log responses +// PrintResponse log responses func (l *Logger) PrintResponse(sessionID string, code int, message string) { fs.Infof(sessionID, "< %d %s", code, message) } @@ -237,7 +238,7 @@ func (s *server) NewDriver() (ftp.Driver, error) { return d, nil } -//Driver implementation of ftp server +// Driver implementation of ftp server type Driver struct { s *server vfs *vfs.VFS @@ -265,7 +266,7 @@ func (d *Driver) CheckPasswd(user, pass string) (ok bool, err error) { return true, nil } -//Stat get information on file or folder +// Stat get information on file or folder func (d *Driver) Stat(path string) (fi ftp.FileInfo, err error) { defer log.Trace(path, "")("fi=%+v, err = %v", &fi, &err) n, err := d.vfs.Stat(path) @@ -275,7 +276,7 @@ func (d *Driver) Stat(path string) (fi ftp.FileInfo, err error) { return &FileInfo{n, n.Mode(), d.vfs.Opt.UID, d.vfs.Opt.GID}, err } -//ChangeDir move current folder +// ChangeDir move current folder func (d *Driver) ChangeDir(path string) (err error) { d.lock.Lock() defer d.lock.Unlock() @@ -290,7 +291,7 @@ func (d *Driver) ChangeDir(path string) (err error) { return nil } -//ListDir list content of a folder +// ListDir list content of a folder func (d *Driver) ListDir(path string, callback func(ftp.FileInfo) error) (err error) { d.lock.Lock() defer d.lock.Unlock() @@ -326,7 +327,7 @@ func (d *Driver) ListDir(path string, callback func(ftp.FileInfo) error) (err er return nil } -//DeleteDir delete a folder and his content +// DeleteDir delete a folder and his content func (d *Driver) DeleteDir(path string) (err error) { d.lock.Lock() defer d.lock.Unlock() @@ -345,7 +346,7 @@ func (d *Driver) DeleteDir(path string) (err error) { return nil } -//DeleteFile delete a file +// DeleteFile delete a file func (d *Driver) DeleteFile(path string) (err error) { d.lock.Lock() defer d.lock.Unlock() @@ -364,7 +365,7 @@ func (d *Driver) DeleteFile(path string) (err error) { return nil } -//Rename rename a file or folder +// Rename rename a file or folder func (d *Driver) Rename(oldName, newName string) (err error) { d.lock.Lock() defer d.lock.Unlock() @@ -372,7 +373,7 @@ func (d *Driver) Rename(oldName, newName string) (err error) { return d.vfs.Rename(oldName, newName) } -//MakeDir create a folder +// MakeDir create a folder func (d *Driver) MakeDir(path string) (err error) { d.lock.Lock() defer d.lock.Unlock() @@ -385,7 +386,7 @@ func (d *Driver) MakeDir(path string) (err error) { return err } -//GetFile download a file +// GetFile download a file func (d *Driver) GetFile(path string, offset int64) (size int64, fr io.ReadCloser, err error) { d.lock.Lock() defer d.lock.Unlock() @@ -417,7 +418,7 @@ func (d *Driver) GetFile(path string, offset int64) (size int64, fr io.ReadClose return node.Size(), handle, nil } -//PutFile upload a file +// PutFile upload a file func (d *Driver) PutFile(path string, data io.Reader, appendData bool) (n int64, err error) { d.lock.Lock() defer d.lock.Unlock() @@ -479,7 +480,7 @@ func (d *Driver) PutFile(path string, data io.Reader, appendData bool) (n int64, return bytes, nil } -//FileInfo struct to hold file info for ftp server +// FileInfo struct to hold file info for ftp server type FileInfo struct { os.FileInfo @@ -488,12 +489,12 @@ type FileInfo struct { group uint32 } -//Mode return mode of file. +// Mode return mode of file. func (f *FileInfo) Mode() os.FileMode { return f.mode } -//Owner return owner of file. Try to find the username if possible +// Owner return owner of file. Try to find the username if possible func (f *FileInfo) Owner() string { str := fmt.Sprint(f.owner) u, err := user.LookupId(str) @@ -503,7 +504,7 @@ func (f *FileInfo) Owner() string { return u.Username } -//Group return group of file. Try to find the group name if possible +// Group return group of file. Try to find the group name if possible func (f *FileInfo) Group() string { str := fmt.Sprint(f.group) g, err := user.LookupGroupId(str) diff --git a/cmdtest/cmdtest.go b/cmdtest/cmdtest.go index 663178a1d5712..06a87ebab5130 100644 --- a/cmdtest/cmdtest.go +++ b/cmdtest/cmdtest.go @@ -2,7 +2,6 @@ // // The interface is used to perform end-to-end test of // commands, flags, environment variables etc. -// package cmdtest // The rest of this file is a 1:1 copy from rclone.go diff --git a/fs/bwtimetable.go b/fs/bwtimetable.go index 5cf76ee264fd8..3361fd61fd307 100644 --- a/fs/bwtimetable.go +++ b/fs/bwtimetable.go @@ -208,7 +208,7 @@ func (x *BwTimetable) Set(s string) error { return nil } -// Difference in minutes between lateDayOfWeekHHMM and earlyDayOfWeekHHMM +// Difference in minutes between lateDayOfWeekHHMM and earlyDayOfWeekHHMM func timeDiff(lateDayOfWeekHHMM int, earlyDayOfWeekHHMM int) int { lateTimeMinutes := (lateDayOfWeekHHMM / 10000) * 24 * 60 diff --git a/fs/chunksize/chunksize.go b/fs/chunksize/chunksize.go index ab171f07d1835..5ecf1d5f38ccc 100644 --- a/fs/chunksize/chunksize.go +++ b/fs/chunksize/chunksize.go @@ -6,14 +6,14 @@ import ( ) /* - Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi +Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi - For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest - possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption - that some backends may only allow integer type parameters when specifying the chunk size. +For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest +possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption +that some backends may only allow integer type parameters when specifying the chunk size. - Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the - smallest chunk size necessary to allow the upload to proceed. +Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the +smallest chunk size necessary to allow the upload to proceed. */ func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix { fileSize := fs.SizeSuffix(objInfo.Size()) diff --git a/fs/config/authorize.go b/fs/config/authorize.go index fbfea2e0cd80d..99aa2c9475042 100644 --- a/fs/config/authorize.go +++ b/fs/config/authorize.go @@ -12,9 +12,9 @@ import ( // // It expects 1, 2 or 3 arguments // -// rclone authorize "fs name" -// rclone authorize "fs name" "base64 encoded JSON blob" -// rclone authorize "fs name" "client id" "client secret" +// rclone authorize "fs name" +// rclone authorize "fs name" "base64 encoded JSON blob" +// rclone authorize "fs name" "client id" "client secret" func Authorize(ctx context.Context, args []string, noAutoBrowser bool) error { ctx = suppressConfirm(ctx) ctx = fs.ConfigOAuthOnly(ctx) diff --git a/fs/filter/filter.go b/fs/filter/filter.go index 741580fffec9e..6f4ba9d5ad864 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -301,9 +301,9 @@ func (f *Filter) Add(Include bool, glob string) error { // // These are // -// + glob // - glob -// ! +// - glob +// ! // // '+' includes the glob, '-' excludes it and '!' resets the filter list // diff --git a/fs/log.go b/fs/log.go index 8328b9106e01d..3f6fe21a74ce6 100644 --- a/fs/log.go +++ b/fs/log.go @@ -15,14 +15,14 @@ type LogLevel byte // Log levels. These are the syslog levels of which we only use a // subset. // -// LOG_EMERG system is unusable -// LOG_ALERT action must be taken immediately -// LOG_CRIT critical conditions -// LOG_ERR error conditions -// LOG_WARNING warning conditions -// LOG_NOTICE normal, but significant, condition -// LOG_INFO informational message -// LOG_DEBUG debug-level message +// LOG_EMERG system is unusable +// LOG_ALERT action must be taken immediately +// LOG_CRIT critical conditions +// LOG_ERR error conditions +// LOG_WARNING warning conditions +// LOG_NOTICE normal, but significant, condition +// LOG_INFO informational message +// LOG_DEBUG debug-level message const ( LogLevelEmergency LogLevel = iota LogLevelAlert diff --git a/fs/mount_helper.go b/fs/mount_helper.go index f45b85895475a..5239ff9a584be 100644 --- a/fs/mount_helper.go +++ b/fs/mount_helper.go @@ -169,7 +169,9 @@ func convertMountHelperArgs(origArgs []string) ([]string, error) { // parseHelperOptionString deconstructs the -o value into slice of options // in a way similar to connection strings. // Example: -// param1=value,param2="qvalue",param3='item1,item2',param4="a ""b"" 'c'" +// +// param1=value,param2="qvalue",param3='item1,item2',param4="a ""b"" 'c'" +// // An error may be returned if the remote name has invalid characters // or the parameters are invalid or the path is empty. // diff --git a/fs/open_options.go b/fs/open_options.go index 57daa132ab2c5..5dc3e32d47178 100644 --- a/fs/open_options.go +++ b/fs/open_options.go @@ -40,10 +40,10 @@ type OpenOption interface { // // Examples: // -// RangeOption{Start: 0, End: 99} - fetch the first 100 bytes -// RangeOption{Start: 100, End: 199} - fetch the second 100 bytes -// RangeOption{Start: 100, End: -1} - fetch bytes from offset 100 to the end -// RangeOption{Start: -1, End: 100} - fetch the last 100 bytes +// RangeOption{Start: 0, End: 99} - fetch the first 100 bytes +// RangeOption{Start: 100, End: 199} - fetch the second 100 bytes +// RangeOption{Start: 100, End: -1} - fetch bytes from offset 100 to the end +// RangeOption{Start: -1, End: 100} - fetch the last 100 bytes // // A RangeOption implements a single byte-range-spec from // https://tools.ietf.org/html/rfc7233#section-2.1 diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 081ceb7a7efc5..2bfddd07d3c7e 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -44,7 +44,7 @@ import ( // CheckHashes checks the two files to see if they have common // known hash types and compares them // -// Returns +// Returns. // // equal - which is equality of the hashes // @@ -911,7 +911,7 @@ var SyncPrintf = func(format string, a ...interface{}) { // Synchronized fmt.Fprintf // -// Ignores errors from Fprintf +// Ignores errors from Fprintf. // // Updated to print to terminal if no writer is defined // This special behavior is used to allow easier replacement of the print to terminal code by progress @@ -981,7 +981,7 @@ func CountStringField(count int64, humanReadable bool, rawWidth int) string { // List the Fs to the supplied writer // -// Shows size and path - obeys includes and excludes +// Shows size and path - obeys includes and excludes. // // Lists in parallel which may get them out of order func List(ctx context.Context, f fs.Fs, w io.Writer) error { @@ -993,7 +993,7 @@ func List(ctx context.Context, f fs.Fs, w io.Writer) error { // ListLong lists the Fs to the supplied writer // -// Shows size, mod time and path - obeys includes and excludes +// Shows size, mod time and path - obeys includes and excludes. // // Lists in parallel which may get them out of order func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error { diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go index 8b4cf1bb6c865..16b3267f4357f 100644 --- a/fs/operations/rc_test.go +++ b/fs/operations/rc_test.go @@ -499,8 +499,7 @@ func TestRcFsInfo(t *testing.T) { } -//operations/uploadfile : Tests if upload file succeeds -// +// operations/uploadfile : Tests if upload file succeeds func TestUploadFile(t *testing.T) { r, call := rcNewRun(t, "operations/uploadfile") defer r.Finalise() diff --git a/fs/operations/reopen.go b/fs/operations/reopen.go index e0f38cfcb1bf6..9bfb1508f61f3 100644 --- a/fs/operations/reopen.go +++ b/fs/operations/reopen.go @@ -31,7 +31,7 @@ var ( // NewReOpen makes a handle which will reopen itself and seek to where it was on errors // -// If hashOption is set this will be applied when reading from the start +// If hashOption is set this will be applied when reading from the start. // // If rangeOption is set then this will applied when reading from the // start, and updated on retries. diff --git a/fs/rc/params.go b/fs/rc/params.go index a2eabb6f42740..b6d5545f1f2bc 100644 --- a/fs/rc/params.go +++ b/fs/rc/params.go @@ -48,7 +48,6 @@ func NotErrParamNotFound(err error) bool { // ErrParamInvalid - this is returned from the Get* functions if the // parameter is invalid. // -// // Returning an error of this type from an rc.Func will cause the http // method to return http.StatusBadRequest type ErrParamInvalid struct { diff --git a/fs/rc/webgui/plugins.go b/fs/rc/webgui/plugins.go index 8f212be582e5e..a6730aacbe216 100644 --- a/fs/rc/webgui/plugins.go +++ b/fs/rc/webgui/plugins.go @@ -227,12 +227,12 @@ func (p *Plugins) GetPluginByName(name string) (out *PackageJSON, err error) { } // getAuthorRepoBranchGithub gives author, repoName and branch from a github.com url +// // url examples: // https://github.com/rclone/rclone-webui-react/ // http://github.com/rclone/rclone-webui-react // https://github.com/rclone/rclone-webui-react/tree/caman-js -// github.com/rclone/rclone-webui-react -// +// github.com/rclone/rclone-webui-react func getAuthorRepoBranchGithub(url string) (author string, repoName string, branch string, err error) { repoURL := url repoURL = strings.Replace(repoURL, "https://", "", 1) diff --git a/fs/registry.go b/fs/registry.go index ca2c3d976f05c..33dba915b6aa1 100644 --- a/fs/registry.go +++ b/fs/registry.go @@ -130,16 +130,16 @@ const ( // Option is describes an option for the config wizard // -// This also describes command line options and environment variables +// This also describes command line options and environment variables. // // To create a multiple-choice option, specify the possible values // in the Examples property. Whether the option's value is required // to be one of these depends on other properties: -// - Default is to allow any value, either from specified examples, -// or any other value. To restrict exclusively to the specified -// examples, also set Exclusive=true. -// - If empty string should not be allowed then set Required=true, -// and do not set Default. +// - Default is to allow any value, either from specified examples, +// or any other value. To restrict exclusively to the specified +// examples, also set Exclusive=true. +// - If empty string should not be allowed then set Required=true, +// and do not set Default. type Option struct { Name string // name of the option in snake_case Help string // help, start with a single sentence on a single line that will be extracted for command line help @@ -292,7 +292,7 @@ func Find(name string) (*RegInfo, error) { // MustFind looks for an Info object for the type name passed in // -// Services are looked up in the config file +// Services are looked up in the config file. // // Exits with a fatal error if not found func MustFind(name string) *RegInfo { diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 2c671cf72b802..9ef36da105abe 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -301,9 +301,10 @@ func (s *syncCopyMove) processError(err error) { } // Returns the current error (if any) in the order of precedence -// fatalErr -// normal error -// noRetryErr +// +// fatalErr +// normal error +// noRetryErr func (s *syncCopyMove) currentError() error { s.errorMu.Lock() defer s.errorMu.Unlock() @@ -837,7 +838,7 @@ var errorMaxDurationReached = fserrors.FatalError(errors.New("max transfer durat // // If Delete is true then it deletes any files in fdst that aren't in fsrc // -// If DoMove is true then files will be moved instead of copied +// If DoMove is true then files will be moved instead of copied. // // dir is the start directory, "" for root func (s *syncCopyMove) run() error { @@ -1083,7 +1084,7 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse // // If Delete is true then it deletes any files in fdst that aren't in fsrc // -// If DoMove is true then files will be moved instead of copied +// If DoMove is true then files will be moved instead of copied. // // dir is the start directory, "" for root func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error { diff --git a/fs/walk/walk.go b/fs/walk/walk.go index b6ca99c325e52..f63350a477393 100644 --- a/fs/walk/walk.go +++ b/fs/walk/walk.go @@ -50,7 +50,7 @@ type Func func(path string, entries fs.DirEntries, err error) error // Note that fn will not be called concurrently whereas the directory // listing will proceed concurrently. // -// Parent directories are always listed before their children +// Parent directories are always listed before their children. // // This is implemented by WalkR if Config.UseListR is true // and f supports it and level > 1, or WalkN otherwise. diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 97e44ea1a298e..5bf90d8400f57 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -189,7 +189,7 @@ var ( // PutTestContentsMetadata puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove // -// It uploads the object with the mimeType and metadata passed in if set +// It uploads the object with the mimeType and metadata passed in if set. // // It returns the object which will have been checked if check is set func PutTestContentsMetadata(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item, contents string, check bool, mimeType string, metadata fs.Metadata) fs.Object { diff --git a/fstest/test_all/test_all.go b/fstest/test_all/test_all.go index 2a3b9421d5a16..1bf0311e01cf4 100644 --- a/fstest/test_all/test_all.go +++ b/fstest/test_all/test_all.go @@ -2,7 +2,6 @@ // need integration testing. // // See the `test` target in the Makefile. -// package main /* FIXME diff --git a/lib/atexit/atexit.go b/lib/atexit/atexit.go index bcc851847fb73..363d3267b6b2c 100644 --- a/lib/atexit/atexit.go +++ b/lib/atexit/atexit.go @@ -111,7 +111,7 @@ func Run() { // // It should be used in a defer statement normally so // -// defer OnError(&err, cancelFunc)() +// defer OnError(&err, cancelFunc)() // // So cancelFunc will be run if the function exits with an error or // at exit. diff --git a/lib/bucket/bucket.go b/lib/bucket/bucket.go index f778871290788..13217f0dae8b4 100644 --- a/lib/bucket/bucket.go +++ b/lib/bucket/bucket.go @@ -120,7 +120,7 @@ func (c *Cache) Create(bucket string, create CreateFn, exists ExistsFn) (err err // Remove the bucket with f if it exists // -// If f returns an error we assume the bucket was not removed +// If f returns an error we assume the bucket was not removed. // // If the bucket has already been deleted it returns ErrAlreadyDeleted func (c *Cache) Remove(bucket string, f func() error) error { diff --git a/lib/dircache/dircache.go b/lib/dircache/dircache.go index 05766335de444..920215e08b629 100644 --- a/lib/dircache/dircache.go +++ b/lib/dircache/dircache.go @@ -192,7 +192,7 @@ func SplitPath(path string) (directory, leaf string) { // // Path shouldn't start or end with a / // -// If create is set it will make the directory if not found +// If create is set it will make the directory if not found. // // It will call FindRoot if it hasn't been called already func (dc *DirCache) FindDir(ctx context.Context, path string, create bool) (pathID string, err error) { @@ -282,7 +282,7 @@ func (dc *DirCache) FindPath(ctx context.Context, path string, create bool) (lea // If successful this changes the root of the cache from the true root // to the root specified by the path passed into New. // -// Resets the root directory +// Resets the root directory. // // If create is set it will make the directory if not found func (dc *DirCache) FindRoot(ctx context.Context, create bool) error { @@ -293,9 +293,9 @@ func (dc *DirCache) FindRoot(ctx context.Context, create bool) error { // _findRoot finds the root directory if not already found // -// Resets the root directory +// Resets the root directory. // -// If create is set it will make the directory if not found +// If create is set it will make the directory if not found. // // Call with mu held func (dc *DirCache) _findRoot(ctx context.Context, create bool) error { @@ -390,7 +390,7 @@ func (dc *DirCache) ResetRoot() { // It does all the checking, creates intermediate directories and // returns leafs and IDs ready for the move. // -// This returns +// This returns: // // - srcID - ID of the source directory // - srcDirectoryID - ID of the parent of the source directory diff --git a/lib/encoder/os_windows.go b/lib/encoder/os_windows.go index 87b47b6465327..7e3c4dd87f506 100644 --- a/lib/encoder/os_windows.go +++ b/lib/encoder/os_windows.go @@ -6,19 +6,21 @@ package encoder // OS is the encoding used by the local backend for windows platforms // // List of replaced characters: -// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN -// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN -// : (colon) -> '：' // FULLWIDTH COLON -// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK -// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS -// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE -// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK -// * (asterisk) -> '＊' // FULLWIDTH ASTERISK +// +// < (less than) -> '＜' // FULLWIDTH LESS-THAN SIGN +// > (greater than) -> '＞' // FULLWIDTH GREATER-THAN SIGN +// : (colon) -> '：' // FULLWIDTH COLON +// " (double quote) -> '＂' // FULLWIDTH QUOTATION MARK +// \ (backslash) -> '＼' // FULLWIDTH REVERSE SOLIDUS +// | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE +// ? (question mark) -> '？' // FULLWIDTH QUESTION MARK +// * (asterisk) -> '＊' // FULLWIDTH ASTERISK // // Additionally names can't end with a period (.) or space ( ). // List of replaced characters: -// . (period) -> '．' // FULLWIDTH FULL STOP -// (space) -> '␠' // SYMBOL FOR SPACE +// +// . (period) -> '．' // FULLWIDTH FULL STOP +// (space) -> '␠' // SYMBOL FOR SPACE // // Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16. // diff --git a/lib/encoder/standard.go b/lib/encoder/standard.go index 439437c7932c1..94180de1a744b 100644 --- a/lib/encoder/standard.go +++ b/lib/encoder/standard.go @@ -3,8 +3,9 @@ package encoder // Standard defines the encoding that is used for paths in- and output by rclone. // // List of replaced characters: -// (0x00) -> '␀' // SYMBOL FOR NULL -// / (slash) -> '／' // FULLWIDTH SOLIDUS +// +// (0x00) -> '␀' // SYMBOL FOR NULL +// / (slash) -> '／' // FULLWIDTH SOLIDUS const Standard = (EncodeZero | EncodeSlash | EncodeCtl | diff --git a/lib/errors/errors.go b/lib/errors/errors.go index 2d9c6b93fa2f4..68238521eb429 100644 --- a/lib/errors/errors.go +++ b/lib/errors/errors.go @@ -14,10 +14,11 @@ type WalkFunc func(error) bool // is stopped and no further calls will be made. // // The next error in the chain is determined by the following rules: -// the return value of this method is used. -// - If the current error has a `Unwrap() error` method (golang.org/x/xerrors), -// the return value of this method is used. -// - Common errors in the Go runtime that contain an Err field will use this value. +// +// the return value of this method is used. +// - If the current error has a `Unwrap() error` method (golang.org/x/xerrors), +// the return value of this method is used. +// - Common errors in the Go runtime that contain an Err field will use this value. func Walk(err error, f WalkFunc) { for prev := err; err != nil; prev = err { if f(err) { diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go index 2033ac5f55184..350fe1671aded 100644 --- a/lib/oauthutil/oauthutil.go +++ b/lib/oauthutil/oauthutil.go @@ -433,10 +433,10 @@ func ConfigOut(state string, oAuth *Options) (*fs.ConfigOut, error) { // // This is called with a state which has pushed on it // -// state prefixed with "*oauth" -// state for oauth to return to -// state that returned the OAuth when we wish to recall it -// value that returned the OAuth +// state prefixed with "*oauth" +// state for oauth to return to +// state that returned the OAuth when we wish to recall it +// value that returned the OAuth func ConfigOAuth(ctx context.Context, name string, m configmap.Mapper, ri *fs.RegInfo, in fs.ConfigIn) (*fs.ConfigOut, error) { stateParams, state := fs.StatePop(in.State) @@ -624,7 +624,7 @@ func fixRedirect(oauthConfig *oauth2.Config) *oauth2.Config { // configSetup does the initial creation of the token // -// If opt is nil it will use the default Options +// If opt is nil it will use the default Options. // // It will run an internal webserver to receive the results func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauthConfig *oauth2.Config, opt *Options) (string, error) { diff --git a/lib/pacer/pacer.go b/lib/pacer/pacer.go index 0257d70dad1e7..4b1cbb37b022e 100644 --- a/lib/pacer/pacer.go +++ b/lib/pacer/pacer.go @@ -148,7 +148,7 @@ func (p *Pacer) ModifyCalculator(f func(Calculator)) { // Start a call to the API // -// This must be called as a pair with endCall +// This must be called as a pair with endCall. // // This waits for the pacer token func (p *Pacer) beginCall() { diff --git a/lib/plugin/package.go b/lib/plugin/package.go index 72598680f0215..d962d713ed08b 100644 --- a/lib/plugin/package.go +++ b/lib/plugin/package.go @@ -7,7 +7,7 @@ // To create a plugin, write the backend package like it was in-tree // but set the package name to "main". Then, build the plugin with // -// go build -buildmode=plugin -o librcloneplugin_NAME.so +// go build -buildmode=plugin -o librcloneplugin_NAME.so // // where NAME equals the plugin's fs.RegInfo.Name. package plugin diff --git a/lib/rest/rest.go b/lib/rest/rest.go index adf10925d1eab..69bb1deff7ede 100644 --- a/lib/rest/rest.go +++ b/lib/rest/rest.go @@ -407,7 +407,7 @@ func MultipartUpload(ctx context.Context, in io.Reader, params url.Values, conte // CallJSON runs Call and decodes the body as a JSON object into response (if not nil) // -// If request is not nil then it will be JSON encoded as the body of the request +// If request is not nil then it will be JSON encoded as the body of the request. // // If response is not nil then the response will be JSON decoded into // it and resp.Body will be closed. @@ -430,7 +430,7 @@ func (api *Client) CallJSON(ctx context.Context, opts *Opts, request interface{} // CallXML runs Call and decodes the body as an XML object into response (if not nil) // -// If request is not nil then it will be XML encoded as the body of the request +// If request is not nil then it will be XML encoded as the body of the request. // // If response is not nil then the response will be XML decoded into // it and resp.Body will be closed. @@ -438,7 +438,7 @@ func (api *Client) CallJSON(ctx context.Context, opts *Opts, request interface{} // If response is nil then the resp.Body will be closed only if // opts.NoResponse is set. // -// See CallJSON for a description of MultipartParams and related opts +// See CallJSON for a description of MultipartParams and related opts. // // It will return resp if at all possible, even if err is set func (api *Client) CallXML(ctx context.Context, opts *Opts, request interface{}, response interface{}) (resp *http.Response, err error) { diff --git a/librclone/gomobile/gomobile.go b/librclone/gomobile/gomobile.go index cc899e43c0239..ff3f451c2c6c9 100644 --- a/librclone/gomobile/gomobile.go +++ b/librclone/gomobile/gomobile.go @@ -22,8 +22,8 @@ func RcloneFinalize() { // RcloneRPCResult is returned from RcloneRPC // -// Output will be returned as a serialized JSON object -// Status is a HTTP status return (200=OK anything else fail) +// Output will be returned as a serialized JSON object +// Status is a HTTP status return (200=OK anything else fail) type RcloneRPCResult struct { Output string Status int diff --git a/librclone/librclone.go b/librclone/librclone.go index f90a7e4130240..ef92b6d8a1114 100644 --- a/librclone/librclone.go +++ b/librclone/librclone.go @@ -7,11 +7,11 @@ // // Build a shared library like this: // -// go build --buildmode=c-shared -o librclone.so github.com/rclone/rclone/librclone +// go build --buildmode=c-shared -o librclone.so github.com/rclone/rclone/librclone // // Build a static library like this: // -// go build --buildmode=c-archive -o librclone.a github.com/rclone/rclone/librclone +// go build --buildmode=c-archive -o librclone.a github.com/rclone/rclone/librclone // // Both the above commands will also generate `librclone.h` which should // be `#include`d in `C` programs wishing to use the library. @@ -59,8 +59,8 @@ func RcloneFinalize() { // RcloneRPCResult is returned from RcloneRPC // -// Output will be returned as a serialized JSON object -// Status is a HTTP status return (200=OK anything else fail) +// Output will be returned as a serialized JSON object +// Status is a HTTP status return (200=OK anything else fail) type RcloneRPCResult struct { //nolint:deadcode Output *C.char Status C.int @@ -70,10 +70,10 @@ type RcloneRPCResult struct { //nolint:deadcode // and the output is (output, status). This is an exported interface // to the rclone API as described in https://rclone.org/rc/ // -// method is a string, eg "operations/list" -// input should be a string with a serialized JSON object -// result.Output will be returned as a string with a serialized JSON object -// result.Status is a HTTP status return (200=OK anything else fail) +// method is a string, eg "operations/list" +// input should be a string with a serialized JSON object +// result.Output will be returned as a string with a serialized JSON object +// result.Status is a HTTP status return (200=OK anything else fail) // // All strings are UTF-8 encoded, on all platforms. // diff --git a/librclone/librclone/librclone.go b/librclone/librclone/librclone.go index 58e10f83bdb5f..5b29e4c5975de 100644 --- a/librclone/librclone/librclone.go +++ b/librclone/librclone/librclone.go @@ -72,7 +72,7 @@ func writeError(path string, in rc.Params, err error, status int) (string, int) // RPC runs a transaction over the RC // -// Calling an rc function using JSON to input parameters and output the resulted JSON +// Calling an rc function using JSON to input parameters and output the resulted JSON. // // operations/uploadfile and core/command are not supported as they need request or response object // modified from handlePost in rcserver.go diff --git a/vfs/dir.go b/vfs/dir.go index 004595b77435b..7f68305821fa2 100644 --- a/vfs/dir.go +++ b/vfs/dir.go @@ -555,7 +555,7 @@ func (d *Dir) _newManageVirtuals() manageVirtuals { // This should be called for every entry added to the directory // -// It returns true if this entry should be skipped +// It returns true if this entry should be skipped. // // must be called with the Dir lock held func (mv manageVirtuals) add(d *Dir, name string) bool { diff --git a/vfs/file.go b/vfs/file.go index 4431c6cc5e5a3..4e59aee747804 100644 --- a/vfs/file.go +++ b/vfs/file.go @@ -645,15 +645,15 @@ func (f *File) Fs() fs.Fs { // Open a file according to the flags provided // -// O_RDONLY open the file read-only. -// O_WRONLY open the file write-only. -// O_RDWR open the file read-write. +// O_RDONLY open the file read-only. +// O_WRONLY open the file write-only. +// O_RDWR open the file read-write. // -// O_APPEND append data to the file when writing. -// O_CREATE create a new file if none exists. -// O_EXCL used with O_CREATE, file must not exist -// O_SYNC open for synchronous I/O. -// O_TRUNC if possible, truncate file when opened +// O_APPEND append data to the file when writing. +// O_CREATE create a new file if none exists. +// O_EXCL used with O_CREATE, file must not exist +// O_SYNC open for synchronous I/O. +// O_TRUNC if possible, truncate file when opened // // We ignore O_SYNC and O_EXCL func (f *File) Open(flags int) (fd Handle, err error) { diff --git a/vfs/help.go b/vfs/help.go index 3bb809ca8b010..84ad2dc9d7afb 100644 --- a/vfs/help.go +++ b/vfs/help.go @@ -7,7 +7,8 @@ import ( // Help contains text describing file and directory caching to add to // the command help. // Warning: "!" (sic) will be replaced by backticks below, -// but the pipe character "|" can be used as is. +// +// but the pipe character "|" can be used as is. var Help = strings.ReplaceAll(` ### VFS - Virtual File System diff --git a/vfs/read.go b/vfs/read.go index bfe4b2d5cd49b..53d621a3c2529 100644 --- a/vfs/read.go +++ b/vfs/read.go @@ -215,7 +215,7 @@ func (fh *ReadFileHandle) ReadAt(p []byte, off int64) (n int, err error) { // This waits for *poff to equal off or aborts after the timeout. // -// Waits here potentially affect all seeks so need to keep them short +// Waits here potentially affect all seeks so need to keep them short. // // Call with fh.mu Locked func waitSequential(what string, remote string, cond *sync.Cond, maxWait time.Duration, poff *int64, off int64) { diff --git a/vfs/read_write.go b/vfs/read_write.go index a66ea051f4301..ead0f6b23ba43 100644 --- a/vfs/read_write.go +++ b/vfs/read_write.go @@ -139,7 +139,7 @@ func (fh *RWFileHandle) updateSize() { // close the file handle returning EBADF if it has been // closed already. // -// Must be called with fh.mu held +// Must be called with fh.mu held. // // Note that we leave the file around in the cache on error conditions // to give the user a chance to recover it. diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go index 9c0ec0bf0fb2c..1649503881366 100644 --- a/vfs/vfscache/cache.go +++ b/vfs/vfscache/cache.go @@ -339,7 +339,7 @@ func (c *Cache) get(name string) (item *Item, found bool) { // Item gets a cache item for name // -// To use it item.Open will need to be called +// To use it item.Open will need to be called. // // name should be a remote path not an osPath func (c *Cache) Item(name string) (item *Item) { diff --git a/vfs/vfscache/downloaders/downloaders.go b/vfs/vfscache/downloaders/downloaders.go index 42ca677fd8931..46018c7e1988e 100644 --- a/vfs/vfscache/downloaders/downloaders.go +++ b/vfs/vfscache/downloaders/downloaders.go @@ -137,8 +137,8 @@ func New(item Item, opt *vfscommon.Options, remote string, src fs.Object) (dls * // // It should be called with // -// n bytes downloaded -// err is error from download +// n bytes downloaded +// err is error from download // // call with lock held func (dls *Downloaders) _countErrors(n int64, err error) { diff --git a/vfs/vfscache/downloaders/downloaders_test.go b/vfs/vfscache/downloaders/downloaders_test.go index 462a06a61c81e..fdbd942dcdbbd 100644 --- a/vfs/vfscache/downloaders/downloaders_test.go +++ b/vfs/vfscache/downloaders/downloaders_test.go @@ -53,7 +53,7 @@ func (item *testItem) FindMissing(r ranges.Range) (outr ranges.Range) { // WriteAtNoOverwrite writes b to the file, but will not overwrite // already present ranges. // -// This is used by the downloader to write bytes to the file +// This is used by the downloader to write bytes to the file. // // It returns n the total bytes processed and skipped the number of // bytes which were processed but not actually written to the file. diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go index 5d606f21e980f..36355d98133d2 100644 --- a/vfs/vfscache/item.go +++ b/vfs/vfscache/item.go @@ -286,7 +286,7 @@ func (item *Item) _truncate(size int64) (err error) { // Truncate the item to the current size, creating if necessary // -// This does not mark the object as dirty +// This does not mark the object as dirty. // // call with the lock held func (item *Item) _truncateToCurrentSize() (err error) { @@ -725,7 +725,7 @@ func (item *Item) Close(storeFn StoreFn) (err error) { // reload is called with valid items recovered from a cache reload. // -// If they are dirty then it makes sure they get uploaded +// If they are dirty then it makes sure they get uploaded. // // it is called before the cache has started so opens will be 0 and // metaDirty will be false. @@ -766,7 +766,7 @@ func (item *Item) reload(ctx context.Context) error { // If we have local modifications then they take precedence // over a change in the remote // -// It ensures the file is the correct size for the object +// It ensures the file is the correct size for the object. // // call with lock held func (item *Item) _checkObject(o fs.Object) error { @@ -1305,7 +1305,7 @@ func (item *Item) WriteAt(b []byte, off int64) (n int, err error) { // WriteAtNoOverwrite writes b to the file, but will not overwrite // already present ranges. // -// This is used by the downloader to write bytes to the file +// This is used by the downloader to write bytes to the file. // // It returns n the total bytes processed and skipped the number of // bytes which were processed but not actually written to the file. diff --git a/vfs/vfscache/writeback/writeback.go b/vfs/vfscache/writeback/writeback.go index 5f63f5461abd7..b4ab6d39acf52 100644 --- a/vfs/vfscache/writeback/writeback.go +++ b/vfs/vfscache/writeback/writeback.go @@ -252,7 +252,7 @@ func (wb *WriteBack) SetID(pid *Handle) { // If id is 0 then a new item will always be created and the new // Handle will be returned. // -// Use SetID to create Handles in advance of calling Add +// Use SetID to create Handles in advance of calling Add. // // If modified is false then it it doesn't cancel a pending upload if // there is one as there is no need. diff --git a/vfs/vfstest/submount.go b/vfs/vfstest/submount.go index e405dd847658b..7c28de8a9d38a 100644 --- a/vfs/vfstest/submount.go +++ b/vfs/vfstest/submount.go @@ -181,9 +181,12 @@ func startMount(mountFn mountlib.MountFn, useVFS bool, opts string) { // line to send to stdout with an exit flag. // // The format of the lines is -// command \t parameter (optional) +// +// command \t parameter (optional) +// // The response should be -// OK|ERR \t result (optional) +// +// OK|ERR \t result (optional) func doMountCommand(vfs *vfs.VFS, rx string) (tx string, exit bool) { command := strings.Split(rx, "\t") // log.Printf("doMountCommand: %q received", command) From 337b43e7e40991b681a44d9a008d9c397c8d67c3 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 26 Jul 2022 16:50:32 +0100 Subject: [PATCH 216/560] fstests: make ReadObject publically accessible --- fstest/fstests/fstests.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 5bf90d8400f57..4d3698c750946 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -294,8 +294,8 @@ func TestPutLarge(ctx context.Context, t *testing.T, f fs.Fs, file *fstest.Item) require.NoError(t, obj.Remove(ctx)) } -// read the contents of an object as a string -func readObject(ctx context.Context, t *testing.T, obj fs.Object, limit int64, options ...fs.OpenOption) string { +// ReadObject reads the contents of an object as a string +func ReadObject(ctx context.Context, t *testing.T, obj fs.Object, limit int64, options ...fs.OpenOption) string { what := fmt.Sprintf("readObject(%q) limit=%d, options=%+v", obj, limit, options) in, err := obj.Open(ctx, options...) require.NoError(t, err, what) @@ -797,7 +797,7 @@ func Run(t *testing.T, opt *Opt) { assert.NoError(t, out.Close()) obj := findObject(ctx, t, f, path) - assert.Equal(t, "abcdefghi", readObject(ctx, t, obj, -1), "contents of file differ") + assert.Equal(t, "abcdefghi", ReadObject(ctx, t, obj, -1), "contents of file differ") assert.NoError(t, obj.Remove(ctx)) assert.NoError(t, f.Rmdir(ctx, "writer-at-subdir")) @@ -1490,14 +1490,14 @@ func Run(t *testing.T, opt *Opt) { t.Run("ObjectOpen", func(t *testing.T) { skipIfNotOk(t) obj := findObject(ctx, t, f, file1.Path) - assert.Equal(t, file1Contents, readObject(ctx, t, obj, -1), "contents of file1 differ") + assert.Equal(t, file1Contents, ReadObject(ctx, t, obj, -1), "contents of file1 differ") }) // TestObjectOpenSeek tests that Open works with SeekOption t.Run("ObjectOpenSeek", func(t *testing.T) { skipIfNotOk(t) obj := findObject(ctx, t, f, file1.Path) - assert.Equal(t, file1Contents[50:], readObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek") + assert.Equal(t, file1Contents[50:], ReadObject(ctx, t, obj, -1, &fs.SeekOption{Offset: 50}), "contents of file1 differ after seek") }) // TestObjectOpenRange tests that Open works with RangeOption @@ -1516,7 +1516,7 @@ func Run(t *testing.T, opt *Opt) { {fs.RangeOption{Start: -1, End: 20}, 80, 100}, // if start is omitted this means get the final bytes // {fs.RangeOption{Start: -1, End: -1}, 0, 100}, - this seems to work but the RFC doesn't define it } { - got := readObject(ctx, t, obj, -1, &test.ro) + got := ReadObject(ctx, t, obj, -1, &test.ro) foundAt := strings.Index(file1Contents, got) help := fmt.Sprintf("%#v failed want [%d:%d] got [%d:%d]", test.ro, test.wantStart, test.wantEnd, foundAt, foundAt+len(got)) assert.Equal(t, file1Contents[test.wantStart:test.wantEnd], got, help) @@ -1527,7 +1527,7 @@ func Run(t *testing.T, opt *Opt) { t.Run("ObjectPartialRead", func(t *testing.T) { skipIfNotOk(t) obj := findObject(ctx, t, f, file1.Path) - assert.Equal(t, file1Contents[:50], readObject(ctx, t, obj, 50), "contents of file1 differ after limited read") + assert.Equal(t, file1Contents[:50], ReadObject(ctx, t, obj, 50), "contents of file1 differ after limited read") }) // TestObjectUpdate tests that Update works @@ -1553,7 +1553,7 @@ func Run(t *testing.T, opt *Opt) { file1.Check(t, obj, f.Precision()) // check contents correct - assert.Equal(t, contents, readObject(ctx, t, obj, -1), "contents of updated file1 differ") + assert.Equal(t, contents, ReadObject(ctx, t, obj, -1), "contents of updated file1 differ") file1Contents = contents }) From 77e3512714d2d90ac8f44de6961d67d3bcd40c52 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 26 Jul 2022 16:50:55 +0100 Subject: [PATCH 217/560] fstests: Make InternalTestFiles so the internal tests know the current state --- fstest/fstests/fstests.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 4d3698c750946..28e02a44b2cc6 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -362,6 +362,9 @@ func removeConfigID(s string) string { return s } +// InternalTestFiles is the state of the remote at the moment the internal tests are called +var InternalTestFiles []fstest.Item + // Run runs the basic integration tests for a remote using the options passed in. // // They are structured in a hierarchical way so that dependencies for the tests can be created. @@ -1813,6 +1816,9 @@ func Run(t *testing.T, opt *Opt) { } }) + // State of remote at the moment the internal tests are called + InternalTestFiles = []fstest.Item{file1, file2} + // TestObjectRemove tests Remove t.Run("ObjectRemove", func(t *testing.T) { skipIfNotOk(t) @@ -1822,6 +1828,8 @@ func Run(t *testing.T, opt *Opt) { require.NoError(t, err) // check listing without modtime as TestPublicLink may change the modtime fstest.CheckListingWithPrecision(t, f, []fstest.Item{file2}, nil, fs.ModTimeNotSupported) + // Show the internal tests file2 is gone + InternalTestFiles = []fstest.Item{file2} }) // TestAbout tests the About optional interface From fa49971d4982497f36280ff3cdb8fe9195285a75 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 26 Jul 2022 17:53:35 +0100 Subject: [PATCH 218/560] docs: move time/duration option docs to the main docs --- docs/content/docs.md | 32 ++++++++++++++++++++++++++++---- docs/content/filtering.md | 26 ++++++++------------------ 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index ed58947269d00..c031a94bebdca 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -560,10 +560,34 @@ it to `false`. It is also possible to specify `--boolean=false` or parsed as `--boolean` and the `false` is parsed as an extra command line argument for rclone. -Options which use TIME use the go time parser. A duration string is a -possibly signed sequence of decimal numbers, each with optional -fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid -time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". +### Time or duration options {#time-option} + +TIME or DURATION options can be specified as a duration string or a +time string. + +A duration string is a possibly signed sequence of decimal numbers, +each with optional fraction and a unit suffix, such as "300ms", +"-1.5h" or "2h45m". Default units are seconds or the following +abbreviations are valid: + + * `ms` - Milliseconds + * `s` - Seconds + * `m` - Minutes + * `h` - Hours + * `d` - Days + * `w` - Weeks + * `M` - Months + * `y` - Years + +These can also be specified as an absolute time in the following +formats: + +- RFC3339 - e.g. `2006-01-02T15:04:05Z` or `2006-01-02T15:04:05+07:00` +- ISO8601 Date and time, local timezone - `2006-01-02T15:04:05` +- ISO8601 Date and time, local timezone - `2006-01-02 15:04:05` +- ISO8601 Date - `2006-01-02` (YYYY-MM-DD) + +### Size options {#size-option} Options which use SIZE use KiB (multiples of 1024 bytes) by default. However, a suffix of `B` for Byte, `K` for KiB, `M` for MiB, diff --git a/docs/content/filtering.md b/docs/content/filtering.md index 8fc3bf109978d..0b7d3d99c49e6 100644 --- a/docs/content/filtering.md +++ b/docs/content/filtering.md @@ -677,6 +677,8 @@ Default units are `KiB` but abbreviations `K`, `M`, `G`, `T` or `P` are valid. E.g. `rclone ls remote: --min-size 50k` lists files on `remote:` of 50 KiB size or larger. +See [the size option docs](/docs/#size-option) for more info. + ### `--max-size` - Don't transfer any file larger than this Controls the maximum size file within the scope of an rclone command. @@ -685,33 +687,19 @@ Default units are `KiB` but abbreviations `K`, `M`, `G`, `T` or `P` are valid. E.g. `rclone ls remote: --max-size 1G` lists files on `remote:` of 1 GiB size or smaller. +See [the size option docs](/docs/#size-option) for more info. + ### `--max-age` - Don't transfer any file older than this Controls the maximum age of files within the scope of an rclone command. -Default units are seconds or the following abbreviations are valid: - - * `ms` - Milliseconds - * `s` - Seconds - * `m` - Minutes - * `h` - Hours - * `d` - Days - * `w` - Weeks - * `M` - Months - * `y` - Years - -`--max-age` can also be specified as an absolute time in the following -formats: - -- RFC3339 - e.g. `2006-01-02T15:04:05Z` or `2006-01-02T15:04:05+07:00` -- ISO8601 Date and time, local timezone - `2006-01-02T15:04:05` -- ISO8601 Date and time, local timezone - `2006-01-02 15:04:05` -- ISO8601 Date - `2006-01-02` (YYYY-MM-DD) `--max-age` applies only to files and not to directories. E.g. `rclone ls remote: --max-age 2d` lists files on `remote:` of 2 days old or less. +See [the time option docs](/docs/#time-option) for valid formats. + ### `--min-age` - Don't transfer any file younger than this Controls the minimum age of files within the scope of an rclone command. @@ -722,6 +710,8 @@ Controls the minimum age of files within the scope of an rclone command. E.g. `rclone ls remote: --min-age 2d` lists files on `remote:` of 2 days old or more. +See [the time option docs](/docs/#time-option) for valid formats. + ## Other flags ### `--delete-excluded` - Delete files on dest excluded from sync From 724391806943cf420bddd4d8e70a26dd0b96b647 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 25 Jul 2022 16:05:17 +0100 Subject: [PATCH 219/560] s3: implement backend versioning command to get/set bucket versioning --- backend/s3/s3.go | 82 ++++++++++++++++++++++++++++++++++++++++++++++ docs/content/s3.md | 18 ++++++++++ 2 files changed, 100 insertions(+) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 35745cbd5e1cf..4d196719ac2dd 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2119,6 +2119,8 @@ type Fs struct { srvRest *rest.Client // the rest connection to the server pool *pool.Pool // memory pool etagIsNotMD5 bool // if set ETags are not MD5s + versioningMu sync.Mutex + versioning fs.Tristate // if set bucket is using versions } // Object describes a s3 object @@ -3497,6 +3499,20 @@ Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc. Opts: map[string]string{ "max-age": "Max age of upload to delete", }, +}, { + Name: "versioning", + Short: "Set/get versioning support for a bucket.", + Long: `This command sets versioning support if a parameter is +passed and then returns the current versioning status for the bucket +supplied. + + rclone backend versioning s3:bucket # read status only + rclone backend versioning s3:bucket Enabled + rclone backend versioning s3:bucket Suspended + +It may return "Enabled", "Suspended" or "Unversioned". Note that once versioning +has been enabled the status can't be set back to "Unversioned". +`, }} // Command the backend to run a named command @@ -3586,6 +3602,8 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str } } return nil, f.cleanUp(ctx, maxAge) + case "versioning": + return f.setGetVersioning(ctx, arg...) default: return nil, fs.ErrorCommandNotFound } @@ -3701,6 +3719,70 @@ func (f *Fs) cleanUp(ctx context.Context, maxAge time.Duration) (err error) { return err } +// Read whether the bucket is versioned or not +func (f *Fs) isVersioned(ctx context.Context) bool { + f.versioningMu.Lock() + defer f.versioningMu.Unlock() + if !f.versioning.Valid { + _, _ = f.setGetVersioning(ctx) + fs.Debugf(f, "bucket is versioned: %v", f.versioning.Value) + } + return f.versioning.Value +} + +// Set or get bucket versioning. +// +// Pass no arguments to get, or pass "Enabled" or "Suspended" +// +// Updates f.versioning +func (f *Fs) setGetVersioning(ctx context.Context, arg ...string) (status string, err error) { + if len(arg) > 1 { + return "", errors.New("too many arguments") + } + if f.rootBucket == "" { + return "", errors.New("need a bucket") + } + if len(arg) == 1 { + var versioning = s3.VersioningConfiguration{ + Status: aws.String(arg[0]), + } + // Disabled is indicated by the parameter missing + if *versioning.Status == "Disabled" { + versioning.Status = aws.String("") + } + req := s3.PutBucketVersioningInput{ + Bucket: &f.rootBucket, + VersioningConfiguration: &versioning, + } + err := f.pacer.Call(func() (bool, error) { + _, err = f.c.PutBucketVersioningWithContext(ctx, &req) + return f.shouldRetry(ctx, err) + }) + if err != nil { + return "", err + } + } + req := s3.GetBucketVersioningInput{ + Bucket: &f.rootBucket, + } + var resp *s3.GetBucketVersioningOutput + err = f.pacer.Call(func() (bool, error) { + resp, err = f.c.GetBucketVersioningWithContext(ctx, &req) + return f.shouldRetry(ctx, err) + }) + f.versioning.Valid = true + f.versioning.Value = false + if err != nil { + fs.Errorf(f, "Failed to read versioning status, assuming unversioned: %v", err) + return "", err + } + if resp.Status == nil { + return "Unversioned", err + } + f.versioning.Value = true + return *resp.Status, err +} + // CleanUp removes all pending multipart uploads older than 24 hours func (f *Fs) CleanUp(ctx context.Context) (err error) { return f.cleanUp(ctx, 24*time.Hour) diff --git a/docs/content/s3.md b/docs/content/s3.md index e68b2f52ac32e..6462dcf0dac5c 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -2704,6 +2704,24 @@ Options: - "max-age": Max age of upload to delete +### versioning + +Set/get versioning support for a bucket. + + rclone backend versioning remote: [options] [+] + +This command sets versioning support if a parameter is +passed and then returns the current versioning status for the bucket +supplied. + + rclone backend versioning s3:bucket # read status only + rclone backend versioning s3:bucket Enabled + rclone backend versioning s3:bucket Suspended + +It may return "Enabled", "Suspended" or "Unversioned". Note that once versioning +has been enabled the status can't be set back to "Unversioned". + + {{< rem autogenerated options stop >}} ### Anonymous access to public buckets From a59fa2977d7e5b3747bf942c7fc5abe366cbcd72 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 25 Jul 2022 15:56:17 +0100 Subject: [PATCH 220/560] s3: factor different listing versions into separate objects --- backend/s3/s3.go | 166 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 112 insertions(+), 54 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 4d196719ac2dd..f6fc6e956f205 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2750,6 +2750,100 @@ func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error { return nil } +// Common interface for bucket listers +type bucketLister interface { + List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) + URLEncodeListings(bool) +} + +// V1 bucket lister +type v1List struct { + f *Fs + req s3.ListObjectsInput +} + +// Create a new V1 bucket lister +func (f *Fs) newV1List(req *s3.ListObjectsV2Input) bucketLister { + l := &v1List{ + f: f, + } + // Convert v2 req into v1 req + structs.SetFrom(&l.req, req) + return l +} + +// List a bucket with V1 listing +func (ls *v1List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { + respv1, err := ls.f.c.ListObjectsWithContext(ctx, &ls.req) + if err != nil { + return nil, nil, err + } + + // Set up the request for next time + ls.req.Marker = respv1.NextMarker + if aws.BoolValue(respv1.IsTruncated) && ls.req.Marker == nil { + if len(respv1.Contents) == 0 { + return nil, nil, errors.New("s3 protocol error: received listing v1 with IsTruncated set, no NextMarker and no Contents") + } + // Use the last Key received if no NextMarker and isTruncated + ls.req.Marker = respv1.Contents[len(respv1.Contents)-1].Key + + } + + // If we are URL encoding then must decode the marker + if ls.req.Marker != nil && ls.req.EncodingType != nil { + *ls.req.Marker, err = url.QueryUnescape(*ls.req.Marker) + if err != nil { + return nil, nil, fmt.Errorf("failed to URL decode Marker %q: %w", *ls.req.Marker, err) + } + } + + // convert v1 resp into v2 resp + resp = new(s3.ListObjectsV2Output) + structs.SetFrom(resp, respv1) + + return resp, nil, nil +} + +// URL Encode the listings +func (ls *v1List) URLEncodeListings(encode bool) { + if encode { + ls.req.EncodingType = aws.String(s3.EncodingTypeUrl) + } else { + ls.req.EncodingType = nil + } +} + +// V2 bucket lister +type v2List struct { + f *Fs + req s3.ListObjectsV2Input +} + +// Create a new V2 bucket lister +func (f *Fs) newV2List(req *s3.ListObjectsV2Input) bucketLister { + return &v2List{ + f: f, + req: *req, + } +} + +// Do a V2 listing +func (ls *v2List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { + resp, err = ls.f.c.ListObjectsV2WithContext(ctx, &ls.req) + ls.req.ContinuationToken = resp.NextContinuationToken + return resp, nil, err +} + +// URL Encode the listings +func (ls *v2List) URLEncodeListings(encode bool) { + if encode { + ls.req.EncodingType = aws.String(s3.EncodingTypeUrl) + } else { + ls.req.EncodingType = nil + } +} + // listFn is called from list to handle an object. type listFn func(remote string, object *s3.Object, isDirectory bool) error @@ -2760,7 +2854,6 @@ type listFn func(remote string, object *s3.Object, isDirectory bool) error // // Set recurse to read sub directories func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, fn listFn) error { - v1 := f.opt.ListVersion == 1 if prefix != "" { prefix += "/" } @@ -2771,7 +2864,6 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck if !recurse { delimiter = "/" } - var continuationToken, startAfter *string // URL encode the listings so we can use control characters in object names // See: https://github.com/aws/aws-sdk-go/issues/1914 // @@ -2788,51 +2880,34 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck // So we enable only on providers we know supports it properly, all others can retry when a // XML Syntax error is detected. urlEncodeListings := f.opt.ListURLEncode.Value + req := s3.ListObjectsV2Input{ + Bucket: &bucket, + Delimiter: &delimiter, + Prefix: &directory, + MaxKeys: &f.opt.ListChunk, + } + if f.opt.RequesterPays { + req.RequestPayer = aws.String(s3.RequestPayerRequester) + } + var listBucket bucketLister + switch { + case f.opt.ListVersion == 1: + listBucket = f.newV1List(&req) + default: + listBucket = f.newV2List(&req) + } for { - // FIXME need to implement ALL loop - req := s3.ListObjectsV2Input{ - Bucket: &bucket, - ContinuationToken: continuationToken, - Delimiter: &delimiter, - Prefix: &directory, - MaxKeys: &f.opt.ListChunk, - StartAfter: startAfter, - } - if urlEncodeListings { - req.EncodingType = aws.String(s3.EncodingTypeUrl) - } - if f.opt.RequesterPays { - req.RequestPayer = aws.String(s3.RequestPayerRequester) - } var resp *s3.ListObjectsV2Output var err error err = f.pacer.Call(func() (bool, error) { - if v1 { - // Convert v2 req into v1 req - var reqv1 s3.ListObjectsInput - structs.SetFrom(&reqv1, &req) - reqv1.Marker = continuationToken - if startAfter != nil { - reqv1.Marker = startAfter - } - var respv1 *s3.ListObjectsOutput - respv1, err = f.c.ListObjectsWithContext(ctx, &reqv1) - if err == nil && respv1 != nil { - // convert v1 resp into v2 resp - resp = new(s3.ListObjectsV2Output) - structs.SetFrom(resp, respv1) - resp.NextContinuationToken = respv1.NextMarker - } - } else { - resp, err = f.c.ListObjectsV2WithContext(ctx, &req) - } + listBucket.URLEncodeListings(urlEncodeListings) + resp, _, err = listBucket.List(ctx) if err != nil && !urlEncodeListings { if awsErr, ok := err.(awserr.RequestFailure); ok { if origErr := awsErr.OrigErr(); origErr != nil { if _, ok := origErr.(*xml.SyntaxError); ok { // Retry the listing with URL encoding as there were characters that XML can't encode urlEncodeListings = true - req.EncodingType = aws.String(s3.EncodingTypeUrl) fs.Debugf(f, "Retrying listing because of characters which can't be XML encoded") return true, err } @@ -2921,23 +2996,6 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck if !aws.BoolValue(resp.IsTruncated) { break } - // Use NextContinuationToken if set, otherwise use last Key for StartAfter - if resp.NextContinuationToken == nil || *resp.NextContinuationToken == "" { - if len(resp.Contents) == 0 { - return errors.New("s3 protocol error: received listing with IsTruncated set, no NextContinuationToken/NextMarker and no Contents") - } - continuationToken = nil - startAfter = resp.Contents[len(resp.Contents)-1].Key - } else { - continuationToken = resp.NextContinuationToken - startAfter = nil - } - if startAfter != nil && urlEncodeListings { - *startAfter, err = url.QueryUnescape(*startAfter) - if err != nil { - return fmt.Errorf("failed to URL decode StartAfter/NextMarker %q: %w", *continuationToken, err) - } - } } return nil } From 0ae171416f393ff67354608091bc177606f8d72a Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 25 Jul 2022 16:06:15 +0100 Subject: [PATCH 221/560] s3: implement --s3-versions flag - See #1776 --- backend/s3/s3.go | 304 +++++++++++++++++++++++++++------ backend/s3/s3_internal_test.go | 77 +++++++++ docs/content/s3.md | 73 ++++++++ 3 files changed, 402 insertions(+), 52 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index f6fc6e956f205..88c4e3bf7f541 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -54,6 +54,7 @@ import ( "github.com/rclone/rclone/lib/readers" "github.com/rclone/rclone/lib/rest" "github.com/rclone/rclone/lib/structs" + "github.com/rclone/rclone/lib/version" "golang.org/x/sync/errgroup" ) @@ -1982,6 +1983,11 @@ circumstances or for testing. `, Default: false, Advanced: true, + }, { + Name: "versions", + Help: "Include old versions in directory listings.", + Default: false, + Advanced: true, }, }}) } @@ -2099,6 +2105,7 @@ type Options struct { DownloadURL string `config:"download_url"` UseMultipartEtag fs.Tristate `config:"use_multipart_etag"` UsePresignedRequest bool `config:"use_presigned_request"` + Versions bool `config:"versions"` } // Fs represents a remote s3 server @@ -2136,6 +2143,7 @@ type Object struct { lastModified time.Time // Last modified meta map[string]string // The object metadata if known - may be nil - with lower case keys mimeType string // MimeType of object - may be "" + versionID *string // If present this points to an object version // Metadata as pointers to strings as they often won't be present storageClass *string // e.g. GLACIER @@ -2237,7 +2245,17 @@ func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) { // split returns bucket and bucketPath from the object func (o *Object) split() (bucket, bucketPath string) { - return o.fs.split(o.remote) + bucket, bucketPath = o.fs.split(o.remote) + // If there is an object version, then the path may have a + // version suffix, if so remove it. + // + // If we are unlucky enough to have a file name with a valid + // version path where this wasn't required (eg using + // --s3-version-at) then this will go wrong. + if o.versionID != nil { + _, bucketPath = version.Remove(bucketPath) + } + return bucket, bucketPath } // getClient makes an http client according to the options @@ -2671,14 +2689,58 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e return f, nil } +// getMetaDataListing gets the metadata from the object unconditionally from the listing +// +// This is needed to find versioned objects from their paths. +// +// It may return info == nil and err == nil if a HEAD would be more appropriate +func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s3.Object, versionID *string, err error) { + bucket, bucketPath := f.split(wantRemote) + timestamp, bucketPath := version.Remove(bucketPath) + + // If the path had no version string return no info, to force caller to look it up + if timestamp.IsZero() { + return nil, nil, nil + } + + err = f.list(ctx, bucket, bucketPath, "", false, true, f.opt.Versions, false, true, func(gotRemote string, object *s3.Object, objectVersionID *string, isDirectory bool) error { + if isDirectory { + return nil + } + if wantRemote != gotRemote { + return nil + } + info = object + versionID = objectVersionID + return errEndList // read only 1 item + }) + if err != nil { + if err == fs.ErrorDirNotFound { + return nil, nil, fs.ErrorObjectNotFound + } + return nil, nil, err + } + if info == nil { + return nil, nil, fs.ErrorObjectNotFound + } + return info, versionID, nil +} + // Return an Object from a path // // If it can't be found it returns the error ErrorObjectNotFound. -func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Object) (fs.Object, error) { +func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Object, versionID *string) (obj fs.Object, err error) { o := &Object{ fs: f, remote: remote, } + if info == nil && f.opt.Versions && version.Match(remote) { + // If versions, have to read the listing to find the version ID + info, versionID, err = f.getMetaDataListing(ctx, remote) + if err != nil { + return nil, err + } + } if info != nil { // Set info but not meta if info.LastModified == nil { @@ -2690,6 +2752,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje o.setMD5FromEtag(aws.StringValue(info.ETag)) o.bytes = aws.Int64Value(info.Size) o.storageClass = info.StorageClass + o.versionID = versionID } else if !o.fs.opt.NoHeadObject { err := o.readMetaData(ctx) // reads info and meta, returning an error if err != nil { @@ -2702,7 +2765,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje // NewObject finds the Object at remote. If it can't be found // it returns the error fs.ErrorObjectNotFound. func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { - return f.newObjectWithInfo(ctx, remote, nil) + return f.newObjectWithInfo(ctx, remote, nil, nil) } // Gets the bucket location @@ -2752,7 +2815,7 @@ func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error { // Common interface for bucket listers type bucketLister interface { - List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) + List(ctx context.Context, hidden bool) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) URLEncodeListings(bool) } @@ -2773,7 +2836,7 @@ func (f *Fs) newV1List(req *s3.ListObjectsV2Input) bucketLister { } // List a bucket with V1 listing -func (ls *v1List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { +func (ls *v1List) List(ctx context.Context, hidden bool) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { respv1, err := ls.f.c.ListObjectsWithContext(ctx, &ls.req) if err != nil { return nil, nil, err @@ -2829,7 +2892,7 @@ func (f *Fs) newV2List(req *s3.ListObjectsV2Input) bucketLister { } // Do a V2 listing -func (ls *v2List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { +func (ls *v2List) List(ctx context.Context, hidden bool) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { resp, err = ls.f.c.ListObjectsV2WithContext(ctx, &ls.req) ls.req.ContinuationToken = resp.NextContinuationToken return resp, nil, err @@ -2844,8 +2907,104 @@ func (ls *v2List) URLEncodeListings(encode bool) { } } +// Versions bucket lister +type versionsList struct { + f *Fs + req s3.ListObjectVersionsInput +} + +// Create a new Versions bucket lister +func (f *Fs) newVersionsList(req *s3.ListObjectsV2Input) bucketLister { + l := &versionsList{ + f: f, + } + // Convert v2 req into withVersions req + structs.SetFrom(&l.req, req) + return l +} + +// Any s3.Objects with this as their size are delete markers +var isDeleteMarker = new(int64) + +// List a bucket with versions +func (ls *versionsList) List(ctx context.Context, hidden bool) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { + respVersions, err := ls.f.c.ListObjectVersionsWithContext(ctx, &ls.req) + if err != nil { + return nil, nil, err + } + + // Set up the request for next time + ls.req.KeyMarker = respVersions.NextKeyMarker + ls.req.VersionIdMarker = respVersions.NextVersionIdMarker + + // If we are URL encoding then must decode the marker + if ls.req.KeyMarker != nil && ls.req.EncodingType != nil { + *ls.req.KeyMarker, err = url.QueryUnescape(*ls.req.KeyMarker) + if err != nil { + return nil, nil, fmt.Errorf("failed to URL decode KeyMarker %q: %w", *ls.req.KeyMarker, err) + } + } + + // convert Versions resp into v2 resp + resp = new(s3.ListObjectsV2Output) + structs.SetFrom(resp, respVersions) + + // Convert the Versions and the DeleteMarkers into an array of s3.Object + // + // These are returned in the order that they are stored with the most recent first. + // With the annoyance that the Versions and DeleteMarkers are split into two + objs := make([]*s3.Object, 0, len(respVersions.Versions)) + for _, objVersion := range respVersions.Versions { + var obj = new(s3.Object) + structs.SetFrom(obj, objVersion) + // Adjust the file names + if !aws.BoolValue(objVersion.IsLatest) { + if obj.Key != nil && objVersion.LastModified != nil { + *obj.Key = version.Add(*obj.Key, *objVersion.LastModified) + } + } + objs = append(objs, obj) + versionIDs = append(versionIDs, objVersion.VersionId) + } + + // If hidden is set, put the delete markers in too, but set + // their sizes to a sentinel delete marker size + if hidden { + for _, deleteMarker := range respVersions.DeleteMarkers { + var obj = new(s3.Object) + structs.SetFrom(obj, deleteMarker) + obj.Size = isDeleteMarker + // Adjust the file names + if !aws.BoolValue(deleteMarker.IsLatest) { + if obj.Key != nil && deleteMarker.LastModified != nil { + *obj.Key = version.Add(*obj.Key, *deleteMarker.LastModified) + } + } + objs = append(objs, obj) + versionIDs = append(versionIDs, deleteMarker.VersionId) + + } + } + + resp.Contents = objs + return resp, versionIDs, nil +} + +// URL Encode the listings +func (ls *versionsList) URLEncodeListings(encode bool) { + if encode { + ls.req.EncodingType = aws.String(s3.EncodingTypeUrl) + } else { + ls.req.EncodingType = nil + } +} + // listFn is called from list to handle an object. -type listFn func(remote string, object *s3.Object, isDirectory bool) error +type listFn func(remote string, object *s3.Object, versionID *string, isDirectory bool) error + +// errEndList is a sentinel used to end the list iteration now. +// listFn should return it to end the iteration with no errors. +var errEndList = errors.New("end list") // list lists the objects into the function supplied from // the bucket and directory supplied. The remote has prefix @@ -2853,12 +3012,16 @@ type listFn func(remote string, object *s3.Object, isDirectory bool) error // bucket to the start. // // Set recurse to read sub directories -func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, fn listFn) error { - if prefix != "" { - prefix += "/" - } - if directory != "" { - directory += "/" +// +// if findFile is set it will look for files called (bucket, directory) +func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, withVersions bool, hidden bool, findFile bool, fn listFn) error { + if !findFile { + if prefix != "" { + prefix += "/" + } + if directory != "" { + directory += "/" + } } delimiter := "" if !recurse { @@ -2891,6 +3054,8 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck } var listBucket bucketLister switch { + case withVersions: + listBucket = f.newVersionsList(&req) case f.opt.ListVersion == 1: listBucket = f.newV1List(&req) default: @@ -2899,9 +3064,10 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck for { var resp *s3.ListObjectsV2Output var err error + var versionIDs []*string err = f.pacer.Call(func() (bool, error) { listBucket.URLEncodeListings(urlEncodeListings) - resp, _, err = listBucket.List(ctx) + resp, versionIDs, err = listBucket.List(ctx, hidden) if err != nil && !urlEncodeListings { if awsErr, ok := err.(awserr.RequestFailure); ok { if origErr := awsErr.OrigErr(); origErr != nil { @@ -2959,13 +3125,16 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck remote = path.Join(bucket, remote) } remote = strings.TrimSuffix(remote, "/") - err = fn(remote, &s3.Object{Key: &remote}, true) + err = fn(remote, &s3.Object{Key: &remote}, nil, true) if err != nil { + if err == errEndList { + return nil + } return err } } } - for _, object := range resp.Contents { + for i, object := range resp.Contents { remote := aws.StringValue(object.Key) if urlEncodeListings { remote, err = url.QueryUnescape(remote) @@ -2988,8 +3157,15 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck if isDirectory && object.Size != nil && *object.Size == 0 { continue // skip directory marker } - err = fn(remote, object, false) + if versionIDs != nil { + err = fn(remote, object, versionIDs[i], false) + } else { + err = fn(remote, object, nil, false) + } if err != nil { + if err == errEndList { + return nil + } return err } } @@ -3001,7 +3177,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck } // Convert a list item into a DirEntry -func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *s3.Object, isDirectory bool) (fs.DirEntry, error) { +func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *s3.Object, versionID *string, isDirectory bool) (fs.DirEntry, error) { if isDirectory { size := int64(0) if object.Size != nil { @@ -3010,7 +3186,7 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *s3.Objec d := fs.NewDir(remote, time.Time{}).SetSize(size) return d, nil } - o, err := f.newObjectWithInfo(ctx, remote, object) + o, err := f.newObjectWithInfo(ctx, remote, object, versionID) if err != nil { return nil, err } @@ -3020,8 +3196,8 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *s3.Objec // listDir lists files and directories to out func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool) (entries fs.DirEntries, err error) { // List the objects and directories - err = f.list(ctx, bucket, directory, prefix, addBucket, false, func(remote string, object *s3.Object, isDirectory bool) error { - entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory) + err = f.list(ctx, bucket, directory, prefix, addBucket, false, f.opt.Versions, false, false, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { + entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory) if err != nil { return err } @@ -3098,8 +3274,8 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( bucket, directory := f.split(dir) list := walk.NewListRHelper(callback) listR := func(bucket, directory, prefix string, addBucket bool) error { - return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, object *s3.Object, isDirectory bool) error { - entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory) + return f.list(ctx, bucket, directory, prefix, addBucket, true, f.opt.Versions, false, false, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { + entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory) if err != nil { return err } @@ -3254,6 +3430,9 @@ func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPa req.ACL = &f.opt.ACL req.Key = &dstPath source := pathEscape(path.Join(srcBucket, srcPath)) + if src.versionID != nil { + source += fmt.Sprintf("?versionId=%s", *src.versionID) + } req.CopySource = &source if f.opt.RequesterPays { req.RequestPayer = aws.String(s3.RequestPayerRequester) @@ -3448,17 +3627,20 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, if strings.HasSuffix(remote, "/") { return "", fs.ErrorCantShareDirectories } - if _, err := f.NewObject(ctx, remote); err != nil { + obj, err := f.NewObject(ctx, remote) + if err != nil { return "", err } + o := obj.(*Object) if expire > maxExpireDuration { fs.Logf(f, "Public Link: Reducing expiry to %v as %v is greater than the max time allowed", maxExpireDuration, expire) expire = maxExpireDuration } - bucket, bucketPath := f.split(remote) + bucket, bucketPath := o.split() httpReq, _ := f.c.GetObjectRequest(&s3.GetObjectInput{ - Bucket: &bucket, - Key: &bucketPath, + Bucket: &bucket, + Key: &bucketPath, + VersionId: o.versionID, }) return httpReq.Presign(time.Duration(expire)) @@ -3637,6 +3819,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str reqCopy := req reqCopy.Bucket = &bucket reqCopy.Key = &bucketPath + reqCopy.VersionId = o.versionID err = f.pacer.Call(func() (bool, error) { _, err = f.c.RestoreObject(&reqCopy) return f.shouldRetry(ctx, err) @@ -3910,8 +4093,9 @@ func (o *Object) Size() int64 { func (o *Object) headObject(ctx context.Context) (resp *s3.HeadObjectOutput, err error) { bucket, bucketPath := o.split() req := s3.HeadObjectInput{ - Bucket: &bucket, - Key: &bucketPath, + Bucket: &bucket, + Key: &bucketPath, + VersionId: o.versionID, } if o.fs.opt.RequesterPays { req.RequestPayer = aws.String(s3.RequestPayerRequester) @@ -4151,8 +4335,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } req := s3.GetObjectInput{ - Bucket: &bucket, - Key: &bucketPath, + Bucket: &bucket, + Key: &bucketPath, + VersionId: o.versionID, } if o.fs.opt.RequesterPays { req.RequestPayer = aws.String(s3.RequestPayerRequester) @@ -4222,7 +4407,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read var warnStreamUpload sync.Once -func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, err error) { +func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, versionID *string, err error) { f := o.fs // make concurrency machinery @@ -4265,7 +4450,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si return f.shouldRetry(ctx, err) }) if err != nil { - return etag, fmt.Errorf("multipart upload failed to initialise: %w", err) + return etag, nil, fmt.Errorf("multipart upload failed to initialise: %w", err) } uid := cout.UploadId @@ -4338,7 +4523,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si finished = true } else if err != nil { free() - return etag, fmt.Errorf("multipart upload failed to read source: %w", err) + return etag, nil, fmt.Errorf("multipart upload failed to read source: %w", err) } buf = buf[:n] @@ -4393,7 +4578,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si } err = g.Wait() if err != nil { - return etag, err + return etag, nil, err } // sort the completed parts by part number @@ -4401,8 +4586,9 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si return *parts[i].PartNumber < *parts[j].PartNumber }) + var resp *s3.CompleteMultipartUploadOutput err = f.pacer.Call(func() (bool, error) { - _, err := f.c.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{ + resp, err = f.c.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{ Bucket: req.Bucket, Key: req.Key, MultipartUpload: &s3.CompletedMultipartUpload{ @@ -4414,11 +4600,14 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si return f.shouldRetry(ctx, err) }) if err != nil { - return etag, fmt.Errorf("multipart upload failed to finalise: %w", err) + return etag, nil, fmt.Errorf("multipart upload failed to finalise: %w", err) } hashOfHashes := md5.Sum(md5s) etag = fmt.Sprintf("%s-%d", hex.EncodeToString(hashOfHashes[:]), len(parts)) - return etag, nil + if resp != nil { + versionID = resp.VersionId + } + return etag, versionID, nil } // unWrapAwsError unwraps AWS errors, looking for a non AWS error @@ -4445,7 +4634,7 @@ func unWrapAwsError(err error) (found bool, outErr error) { } // Upload a single part using PutObject -func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) { +func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, versionID *string, err error) { r, resp := o.fs.c.PutObjectRequest(req) if req.ContentLength != nil && *req.ContentLength == 0 { // Can't upload zero length files like this for some reason @@ -4472,15 +4661,18 @@ func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjec err = newErr } } - return etag, lastModified, err + return etag, lastModified, nil, err } lastModified = time.Now() - etag = aws.StringValue(resp.ETag) - return etag, lastModified, nil + if resp != nil { + etag = aws.StringValue(resp.ETag) + versionID = resp.VersionId + } + return etag, lastModified, versionID, nil } // Upload a single part using a presigned request -func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) { +func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, versionID *string, err error) { // Create the request putObj, _ := o.fs.c.PutObjectRequest(req) @@ -4490,7 +4682,7 @@ func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.P // PutObject so we used this work-around. url, headers, err := putObj.PresignRequest(15 * time.Minute) if err != nil { - return etag, lastModified, fmt.Errorf("s3 upload: sign request: %w", err) + return etag, lastModified, nil, fmt.Errorf("s3 upload: sign request: %w", err) } if o.fs.opt.V2Auth && headers == nil { @@ -4505,7 +4697,7 @@ func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.P // create the vanilla http request httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in) if err != nil { - return etag, lastModified, fmt.Errorf("s3 upload: new request: %w", err) + return etag, lastModified, nil, fmt.Errorf("s3 upload: new request: %w", err) } // set the headers we signed and the length @@ -4530,15 +4722,19 @@ func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.P return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err }) if err != nil { - return etag, lastModified, err + return etag, lastModified, nil, err } if resp != nil { if date, err := http.ParseTime(resp.Header.Get("Date")); err != nil { lastModified = date } etag = resp.Header.Get("Etag") + vID := resp.Header.Get("x-amz-version-id") + if vID != "" { + versionID = &vID + } } - return etag, lastModified, nil + return etag, lastModified, versionID, nil } // Update the Object from in with modTime and size @@ -4692,18 +4888,20 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op var wantETag string // Multipart upload Etag to check var gotEtag string // Etag we got from the upload var lastModified time.Time // Time we got from the upload + var versionID *string // versionID we got from the upload if multipart { - wantETag, err = o.uploadMultipart(ctx, &req, size, in) + wantETag, versionID, err = o.uploadMultipart(ctx, &req, size, in) } else { if o.fs.opt.UsePresignedRequest { - gotEtag, lastModified, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in) + gotEtag, lastModified, versionID, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in) } else { - gotEtag, lastModified, err = o.uploadSinglepartPutObject(ctx, &req, size, in) + gotEtag, lastModified, versionID, err = o.uploadSinglepartPutObject(ctx, &req, size, in) } } if err != nil { return err } + o.versionID = versionID // User requested we don't HEAD the object after uploading it // so make up the object as best we can assuming it got @@ -4721,6 +4919,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op lastModified = time.Now() } head.LastModified = &lastModified + head.VersionId = versionID o.setMetaData(&head) return nil } @@ -4746,8 +4945,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op func (o *Object) Remove(ctx context.Context) error { bucket, bucketPath := o.split() req := s3.DeleteObjectInput{ - Bucket: &bucket, - Key: &bucketPath, + Bucket: &bucket, + Key: &bucketPath, + VersionId: o.versionID, } if o.fs.opt.RequesterPays { req.RequestPayer = aws.String(s3.RequestPayerRequester) diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go index fcaf8915f2ca6..f3b881fc5e0a2 100644 --- a/backend/s3/s3_internal_test.go +++ b/backend/s3/s3_internal_test.go @@ -12,6 +12,7 @@ import ( "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/fstests" "github.com/rclone/rclone/lib/random" + "github.com/rclone/rclone/lib/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -84,9 +85,85 @@ func (f *Fs) InternalTestNoHead(t *testing.T) { } +func (f *Fs) InternalTestVersions(t *testing.T) { + ctx := context.Background() + + // Enable versioning for this bucket during this test + _, err := f.setGetVersioning(ctx, "Enabled") + if err != nil { + t.Skipf("Couldn't enable versioning: %v", err) + } + defer func() { + // Disable versioning for this bucket + _, err := f.setGetVersioning(ctx, "Suspended") + assert.NoError(t, err) + }() + + // Create an object + const fileName = "test-versions.txt" + contents := random.String(100) + item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) + obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) + defer func() { + assert.NoError(t, obj.Remove(ctx)) + }() + + // Remove it + assert.NoError(t, obj.Remove(ctx)) + + // And create it with different size and contents + newContents := random.String(101) + newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z")) + _ = fstests.PutTestContents(ctx, t, f, &newItem, newContents, true) + + // Add the expected version suffix to the old version + item.Path = version.Add(item.Path, obj.(*Object).lastModified) + + t.Run("S3Version", func(t *testing.T) { + // Set --s3-versions for this test + f.opt.Versions = true + defer func() { + f.opt.Versions = false + }() + + // Check listing + items := append([]fstest.Item{item, newItem}, fstests.InternalTestFiles...) + fstest.CheckListing(t, f, items) + + // Read the contents + entries, err := f.List(ctx, "") + require.NoError(t, err) + tests := 0 + for _, entry := range entries { + switch entry.Remote() { + case newItem.Path: + t.Run("ReadCurrent", func(t *testing.T) { + assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) + }) + tests++ + case item.Path: + t.Run("ReadVersion", func(t *testing.T) { + assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) + }) + tests++ + } + } + assert.Equal(t, 2, tests) + + // Check we can read the object with a version suffix + t.Run("NewObject", func(t *testing.T) { + o, err := f.NewObject(ctx, item.Path) + require.NoError(t, err) + require.NotNil(t, o) + assert.Equal(t, int64(100), o.Size(), o.Remote()) + }) + }) +} + func (f *Fs) InternalTest(t *testing.T) { t.Run("Metadata", f.InternalTestMetadata) t.Run("NoHead", f.InternalTestNoHead) + t.Run("Versions", f.InternalTestVersions) } var _ fstests.InternalTester = (*Fs)(nil) diff --git a/docs/content/s3.md b/docs/content/s3.md index 6462dcf0dac5c..e2089b1d74c10 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -384,6 +384,68 @@ This will mean that these objects do not have an MD5 checksum. Note that reading this from the object takes an additional `HEAD` request as the metadata isn't returned in object listings. +### Versions + +When bucket versioning is enabled (this can be done with rclone with +the [`rclone backend versioning`](#versioning) command) when rclone +uploads a new version of a file it creates a +[new version of it](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html) +Likewise when you delete a file, the old version will be marked hidden +and still be available. + +Old versions of files, where available, are visible using the +`--s3-versions` flag. + +If you wish to remove all the old versions then you can use the +[`rclone backend cleanup-hidden remote:bucket`](#cleanup-hidden) +command which will delete all the old hidden versions of files, +leaving the current ones intact. You can also supply a path and only +old versions under that path will be deleted, e.g. +`rclone backend cleanup-hidden remote:bucket/path/to/stuff`. + +When you `purge` a bucket, the current and the old versions will be +deleted then the bucket will be deleted. + +However `delete` will cause the current versions of the files to +become hidden old versions. + +Here is a session showing the listing and retrieval of an old +version followed by a `cleanup` of the old versions. + +Show current version and all the versions with `--s3-versions` flag. + +``` +$ rclone -q ls s3:cleanup-test + 9 one.txt + +$ rclone -q --s3-versions ls s3:cleanup-test + 9 one.txt + 8 one-v2016-07-04-141032-000.txt + 16 one-v2016-07-04-141003-000.txt + 15 one-v2016-07-02-155621-000.txt +``` + +Retrieve an old version + +``` +$ rclone -q --s3-versions copy s3:cleanup-test/one-v2016-07-04-141003-000.txt /tmp + +$ ls -l /tmp/one-v2016-07-04-141003-000.txt +-rw-rw-r-- 1 ncw ncw 16 Jul 2 17:46 /tmp/one-v2016-07-04-141003-000.txt +``` + +Clean up all the old versions and show that they've gone. + +``` +$ rclone -q backend cleanup-hidden s3:cleanup-test + +$ rclone -q ls s3:cleanup-test + 9 one.txt + +$ rclone -q --s3-versions ls s3:cleanup-test + 9 one.txt +``` + ### Cleanup If you run `rclone cleanup s3:bucket` then it will remove all pending @@ -2562,6 +2624,17 @@ Properties: - Type: bool - Default: false +#### --s3-versions + +Include old versions in directory listings. + +Properties: + +- Config: versions +- Env Var: RCLONE_S3_VERSIONS +- Type: bool +- Default: false + ### Metadata User metadata is stored as x-amz-meta- keys. S3 metadata keys are case insensitive and are always returned in lower case. From 81d242473a8530f2500770c9fbabdacfbbd051f4 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 26 Jul 2022 15:03:32 +0100 Subject: [PATCH 222/560] s3: implement Purge to purge versions and `backend cleanup-hidden` --- backend/s3/s3.go | 100 +++++++++++++++++++++++++++++++++ backend/s3/s3_internal_test.go | 14 +++++ docs/content/s3.md | 15 +++++ 3 files changed, 129 insertions(+) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 88c4e3bf7f541..c77ae0a0fae3d 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -37,6 +37,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/ncw/swift/v2" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/chunksize" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" @@ -3739,6 +3740,17 @@ Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc. Opts: map[string]string{ "max-age": "Max age of upload to delete", }, +}, { + Name: "cleanup-hidden", + Short: "Remove old versions of files.", + Long: `This command removes any old hidden versions of files +on a versions enabled bucket. + +Note that you can use -i/--dry-run with this command to see what it +would do. + + rclone backend cleanup-hidden s3:bucket/path/to/dir +`, }, { Name: "versioning", Short: "Set/get versioning support for a bucket.", @@ -3843,6 +3855,8 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str } } return nil, f.cleanUp(ctx, maxAge) + case "cleanup-hidden": + return nil, f.CleanUpHidden(ctx) case "versioning": return f.setGetVersioning(ctx, arg...) default: @@ -4029,6 +4043,91 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) { return f.cleanUp(ctx, 24*time.Hour) } +// purge deletes all the files and directories +// +// if oldOnly is true then it deletes only non current files. +// +// Implemented here so we can make sure we delete old versions. +func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error { + bucket, directory := f.split(dir) + if bucket == "" { + return errors.New("can't purge from root") + } + versioned := f.isVersioned(ctx) + if !versioned && oldOnly { + fs.Infof(f, "bucket is not versioned so not removing old versions") + return nil + } + var errReturn error + var checkErrMutex sync.Mutex + var checkErr = func(err error) { + if err == nil { + return + } + checkErrMutex.Lock() + defer checkErrMutex.Unlock() + if errReturn == nil { + errReturn = err + } + } + + // Delete Config.Transfers in parallel + delChan := make(fs.ObjectsChan, f.ci.Transfers) + delErr := make(chan error, 1) + go func() { + delErr <- operations.DeleteFiles(ctx, delChan) + }() + checkErr(f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", true, versioned, true, false, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { + if isDirectory { + return nil + } + oi, err := f.newObjectWithInfo(ctx, remote, object, versionID) + if err != nil { + fs.Errorf(object, "Can't create object %+v", err) + return nil + } + tr := accounting.Stats(ctx).NewCheckingTransfer(oi) + // Work out whether the file is the current version or not + isCurrentVersion := !versioned || !version.Match(remote) + fs.Debugf(nil, "%q version %v", remote, version.Match(remote)) + if oldOnly && isCurrentVersion { + // Check current version of the file + if object.Size == isDeleteMarker { + fs.Debugf(remote, "Deleting current version (id %q) as it is a delete marker", aws.StringValue(versionID)) + delChan <- oi + } else { + fs.Debugf(remote, "Not deleting current version %q", aws.StringValue(versionID)) + } + } else { + if object.Size == isDeleteMarker { + fs.Debugf(remote, "Deleting delete marker (id %q)", aws.StringValue(versionID)) + } else { + fs.Debugf(remote, "Deleting (id %q)", aws.StringValue(versionID)) + } + delChan <- oi + } + tr.Done(ctx, nil) + return nil + })) + close(delChan) + checkErr(<-delErr) + + if !oldOnly { + checkErr(f.Rmdir(ctx, dir)) + } + return errReturn +} + +// Purge deletes all the files and directories including the old versions. +func (f *Fs) Purge(ctx context.Context, dir string) error { + return f.purge(ctx, dir, false) +} + +// CleanUpHidden deletes all the hidden files. +func (f *Fs) CleanUpHidden(ctx context.Context) error { + return f.purge(ctx, "", true) +} + // ------------------------------------------------------------ // Fs returns the parent Fs @@ -5042,6 +5141,7 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) // Check the interfaces are satisfied var ( _ fs.Fs = &Fs{} + _ fs.Purger = &Fs{} _ fs.Copier = &Fs{} _ fs.PutStreamer = &Fs{} _ fs.ListRer = &Fs{} diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go index f3b881fc5e0a2..7f26f919e39a5 100644 --- a/backend/s3/s3_internal_test.go +++ b/backend/s3/s3_internal_test.go @@ -158,6 +158,20 @@ func (f *Fs) InternalTestVersions(t *testing.T) { assert.Equal(t, int64(100), o.Size(), o.Remote()) }) }) + + t.Run("Cleanup", func(t *testing.T) { + require.NoError(t, f.CleanUpHidden(ctx)) + items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...) + fstest.CheckListing(t, f, items) + // Set --s3-versions for this test + f.opt.Versions = true + defer func() { + f.opt.Versions = false + }() + fstest.CheckListing(t, f, items) + }) + + // Purge gets tested later } func (f *Fs) InternalTest(t *testing.T) { diff --git a/docs/content/s3.md b/docs/content/s3.md index e2089b1d74c10..d1b99c26c4bac 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -2777,6 +2777,21 @@ Options: - "max-age": Max age of upload to delete +### cleanup-hidden + +Remove old versions of files. + + rclone backend cleanup-hidden remote: [options] [+] + +This command removes any old hidden versions of files +on a versions enabled bucket. + +Note that you can use -i/--dry-run with this command to see what it +would do. + + rclone backend cleanup-hidden s3:bucket/path/to/dir + + ### versioning Set/get versioning support for a bucket. From 1542a979f9a7e29214ec44b9fa5729c3208528b3 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 27 Jul 2022 10:46:28 +0100 Subject: [PATCH 223/560] s3: refactor f.list() to take an options struct as it had too many parameters --- backend/s3/s3.go | 99 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index c77ae0a0fae3d..65ad533e53878 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2704,7 +2704,13 @@ func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s return nil, nil, nil } - err = f.list(ctx, bucket, bucketPath, "", false, true, f.opt.Versions, false, true, func(gotRemote string, object *s3.Object, objectVersionID *string, isDirectory bool) error { + err = f.list(ctx, listOpt{ + bucket: bucket, + directory: bucketPath, + recurse: true, + withVersions: f.opt.Versions, + findFile: true, + }, func(gotRemote string, object *s3.Object, objectVersionID *string, isDirectory bool) error { if isDirectory { return nil } @@ -3007,25 +3013,31 @@ type listFn func(remote string, object *s3.Object, versionID *string, isDirector // listFn should return it to end the iteration with no errors. var errEndList = errors.New("end list") -// list lists the objects into the function supplied from -// the bucket and directory supplied. The remote has prefix -// removed from it and if addBucket is set then it adds the -// bucket to the start. -// -// Set recurse to read sub directories -// -// if findFile is set it will look for files called (bucket, directory) -func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, withVersions bool, hidden bool, findFile bool, fn listFn) error { - if !findFile { - if prefix != "" { - prefix += "/" +// list options +type listOpt struct { + bucket string // bucket to list + directory string // directory with bucket + prefix string // prefix to remove from listing + addBucket bool // if set, the bucket is added to the start of the remote + recurse bool // if set, recurse to read sub directories + withVersions bool // if set, versions are produced + hidden bool // if set, return delete markers as objects with size == isDeleteMarker + findFile bool // if set, it will look for files called (bucket, directory) +} + +// list lists the objects into the function supplied with the opt +// supplied. +func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error { + if !opt.findFile { + if opt.prefix != "" { + opt.prefix += "/" } - if directory != "" { - directory += "/" + if opt.directory != "" { + opt.directory += "/" } } delimiter := "" - if !recurse { + if !opt.recurse { delimiter = "/" } // URL encode the listings so we can use control characters in object names @@ -3045,9 +3057,9 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck // XML Syntax error is detected. urlEncodeListings := f.opt.ListURLEncode.Value req := s3.ListObjectsV2Input{ - Bucket: &bucket, + Bucket: &opt.bucket, Delimiter: &delimiter, - Prefix: &directory, + Prefix: &opt.directory, MaxKeys: &f.opt.ListChunk, } if f.opt.RequesterPays { @@ -3055,7 +3067,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck } var listBucket bucketLister switch { - case withVersions: + case opt.withVersions: listBucket = f.newVersionsList(&req) case f.opt.ListVersion == 1: listBucket = f.newV1List(&req) @@ -3068,7 +3080,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck var versionIDs []*string err = f.pacer.Call(func() (bool, error) { listBucket.URLEncodeListings(urlEncodeListings) - resp, versionIDs, err = listBucket.List(ctx, hidden) + resp, versionIDs, err = listBucket.List(ctx, opt.hidden) if err != nil && !urlEncodeListings { if awsErr, ok := err.(awserr.RequestFailure); ok { if origErr := awsErr.OrigErr(); origErr != nil { @@ -3095,14 +3107,14 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck if reqErr, ok := err.(awserr.RequestFailure); ok { // 301 if wrong region for bucket if reqErr.StatusCode() == http.StatusMovedPermanently { - fs.Errorf(f, "Can't change region for bucket %q with no bucket specified", bucket) + fs.Errorf(f, "Can't change region for bucket %q with no bucket specified", opt.bucket) return nil } } } return err } - if !recurse { + if !opt.recurse { for _, commonPrefix := range resp.CommonPrefixes { if commonPrefix.Prefix == nil { fs.Logf(f, "Nil common prefix received") @@ -3117,13 +3129,13 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck } } remote = f.opt.Enc.ToStandardPath(remote) - if !strings.HasPrefix(remote, prefix) { + if !strings.HasPrefix(remote, opt.prefix) { fs.Logf(f, "Odd name received %q", remote) continue } - remote = remote[len(prefix):] - if addBucket { - remote = path.Join(bucket, remote) + remote = remote[len(opt.prefix):] + if opt.addBucket { + remote = path.Join(opt.bucket, remote) } remote = strings.TrimSuffix(remote, "/") err = fn(remote, &s3.Object{Key: &remote}, nil, true) @@ -3145,14 +3157,14 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck } } remote = f.opt.Enc.ToStandardPath(remote) - if !strings.HasPrefix(remote, prefix) { + if !strings.HasPrefix(remote, opt.prefix) { fs.Logf(f, "Odd name received %q", remote) continue } - remote = remote[len(prefix):] + remote = remote[len(opt.prefix):] isDirectory := remote == "" || strings.HasSuffix(remote, "/") - if addBucket { - remote = path.Join(bucket, remote) + if opt.addBucket { + remote = path.Join(opt.bucket, remote) } // is this a directory marker? if isDirectory && object.Size != nil && *object.Size == 0 { @@ -3197,7 +3209,13 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *s3.Objec // listDir lists files and directories to out func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool) (entries fs.DirEntries, err error) { // List the objects and directories - err = f.list(ctx, bucket, directory, prefix, addBucket, false, f.opt.Versions, false, false, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { + err = f.list(ctx, listOpt{ + bucket: bucket, + directory: directory, + prefix: prefix, + addBucket: addBucket, + withVersions: f.opt.Versions, + }, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory) if err != nil { return err @@ -3275,7 +3293,14 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( bucket, directory := f.split(dir) list := walk.NewListRHelper(callback) listR := func(bucket, directory, prefix string, addBucket bool) error { - return f.list(ctx, bucket, directory, prefix, addBucket, true, f.opt.Versions, false, false, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { + return f.list(ctx, listOpt{ + bucket: bucket, + directory: directory, + prefix: prefix, + addBucket: addBucket, + recurse: true, + withVersions: f.opt.Versions, + }, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory) if err != nil { return err @@ -4077,7 +4102,15 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error { go func() { delErr <- operations.DeleteFiles(ctx, delChan) }() - checkErr(f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", true, versioned, true, false, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { + checkErr(f.list(ctx, listOpt{ + bucket: bucket, + directory: directory, + prefix: f.rootDirectory, + addBucket: f.rootBucket == "", + recurse: true, + withVersions: versioned, + hidden: true, + }, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { if isDirectory { return nil } From 4344a3e2ea8e78127371a5feb476344171898950 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 26 Jul 2022 17:58:57 +0100 Subject: [PATCH 224/560] s3: implement --s3-version-at flag - Fixes #1776 --- backend/s3/s3.go | 199 +++++++++++++++++++++++-------- backend/s3/s3_internal_test.go | 206 ++++++++++++++++++++++++++++++--- docs/content/s3.md | 28 ++++- 3 files changed, 372 insertions(+), 61 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 65ad533e53878..4f7dd8aeef0d7 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1989,6 +1989,20 @@ circumstances or for testing. Help: "Include old versions in directory listings.", Default: false, Advanced: true, + }, { + Name: "version_at", + Help: `Show file versions as they were at the specified time. + +The parameter should be a date, "2006-01-02", datetime "2006-01-02 +15:04:05" or a duration for that long ago, eg "100d" or "1h". + +Note that when using this no file write operations are permitted, +so you can't upload files or delete them. + +See [the time option docs](/docs/#time-option) for valid formats. +`, + Default: fs.Time{}, + Advanced: true, }, }}) } @@ -2011,6 +2025,11 @@ const ( maxExpireDuration = fs.Duration(7 * 24 * time.Hour) // max expiry is 1 week ) +// globals +var ( + errNotWithVersionAt = errors.New("can't modify or delete files in --s3-version-at mode") +) + // system metadata keys which this backend owns var systemMetadataInfo = map[string]fs.MetadataHelp{ "cache-control": { @@ -2107,6 +2126,7 @@ type Options struct { UseMultipartEtag fs.Tristate `config:"use_multipart_etag"` UsePresignedRequest bool `config:"use_presigned_request"` Versions bool `config:"versions"` + VersionAt fs.Time `config:"version_at"` } // Fs represents a remote s3 server @@ -2599,6 +2619,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e if err != nil { return nil, fmt.Errorf("s3: upload cutoff: %w", err) } + if opt.Versions && opt.VersionAt.IsSet() { + return nil, errors.New("s3: cant use --s3-versions and --s3-version-at at the same time") + } if opt.ACL == "" { opt.ACL = "private" } @@ -2697,11 +2720,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // It may return info == nil and err == nil if a HEAD would be more appropriate func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s3.Object, versionID *string, err error) { bucket, bucketPath := f.split(wantRemote) - timestamp, bucketPath := version.Remove(bucketPath) - // If the path had no version string return no info, to force caller to look it up - if timestamp.IsZero() { - return nil, nil, nil + // Strip the version string off if using versions + if f.opt.Versions { + var timestamp time.Time + timestamp, bucketPath = version.Remove(bucketPath) + // If the path had no version string return no info, to force caller to look it up + if timestamp.IsZero() { + return nil, nil, nil + } } err = f.list(ctx, listOpt{ @@ -2710,6 +2737,7 @@ func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s recurse: true, withVersions: f.opt.Versions, findFile: true, + versionAt: f.opt.VersionAt, }, func(gotRemote string, object *s3.Object, objectVersionID *string, isDirectory bool) error { if isDirectory { return nil @@ -2741,8 +2769,8 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje fs: f, remote: remote, } - if info == nil && f.opt.Versions && version.Match(remote) { - // If versions, have to read the listing to find the version ID + if info == nil && ((f.opt.Versions && version.Match(remote)) || f.opt.VersionAt.IsSet()) { + // If versions, have to read the listing to find the correct version ID info, versionID, err = f.getMetaDataListing(ctx, remote) if err != nil { return nil, err @@ -2822,7 +2850,7 @@ func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error { // Common interface for bucket listers type bucketLister interface { - List(ctx context.Context, hidden bool) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) + List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) URLEncodeListings(bool) } @@ -2843,7 +2871,7 @@ func (f *Fs) newV1List(req *s3.ListObjectsV2Input) bucketLister { } // List a bucket with V1 listing -func (ls *v1List) List(ctx context.Context, hidden bool) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { +func (ls *v1List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { respv1, err := ls.f.c.ListObjectsWithContext(ctx, &ls.req) if err != nil { return nil, nil, err @@ -2899,7 +2927,7 @@ func (f *Fs) newV2List(req *s3.ListObjectsV2Input) bucketLister { } // Do a V2 listing -func (ls *v2List) List(ctx context.Context, hidden bool) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { +func (ls *v2List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { resp, err = ls.f.c.ListObjectsV2WithContext(ctx, &ls.req) ls.req.ContinuationToken = resp.NextContinuationToken return resp, nil, err @@ -2916,25 +2944,88 @@ func (ls *v2List) URLEncodeListings(encode bool) { // Versions bucket lister type versionsList struct { - f *Fs - req s3.ListObjectVersionsInput + f *Fs + req s3.ListObjectVersionsInput + versionAt time.Time // set if we want only versions before this + usingVersionAt bool // set if we need to use versionAt + hidden bool // set to see hidden versions + lastKeySent string // last Key sent to the receiving function } // Create a new Versions bucket lister -func (f *Fs) newVersionsList(req *s3.ListObjectsV2Input) bucketLister { +func (f *Fs) newVersionsList(req *s3.ListObjectsV2Input, hidden bool, versionAt time.Time) bucketLister { l := &versionsList{ - f: f, + f: f, + versionAt: versionAt, + usingVersionAt: !versionAt.IsZero(), + hidden: hidden, } // Convert v2 req into withVersions req structs.SetFrom(&l.req, req) return l } -// Any s3.Objects with this as their size are delete markers +// Any s3.Object or s3.ObjectVersion with this as their Size are delete markers var isDeleteMarker = new(int64) +// Compare two s3.ObjectVersions, sorted alphabetically by key with +// the newest first if the Keys match or the one with IsLatest set if +// everything matches. +func versionLess(a, b *s3.ObjectVersion) bool { + if a == nil || a.Key == nil || a.LastModified == nil { + return true + } + if b == nil || b.Key == nil || b.LastModified == nil { + return false + } + if *a.Key < *b.Key { + return true + } + if *a.Key > *b.Key { + return false + } + dt := (*a.LastModified).Sub(*b.LastModified) + if dt > 0 { + return true + } + if dt < 0 { + return false + } + if aws.BoolValue(a.IsLatest) { + return true + } + return false +} + +// Merge the DeleteMarkers into the Versions. +// +// These are delivered by S3 sorted by key then by LastUpdated +// newest first but annoyingly the SDK splits them up into two +// so we need to merge them back again +// +// We do this by converting the s3.DeleteEntry into +// s3.ObjectVersion with Size = isDeleteMarker to tell them apart +// +// We then merge them back into the Versions in the correct order +func mergeDeleteMarkers(oldVersions []*s3.ObjectVersion, deleteMarkers []*s3.DeleteMarkerEntry) (newVersions []*s3.ObjectVersion) { + newVersions = make([]*s3.ObjectVersion, 0, len(oldVersions)+len(deleteMarkers)) + for _, deleteMarker := range deleteMarkers { + var obj = new(s3.ObjectVersion) + structs.SetFrom(obj, deleteMarker) + obj.Size = isDeleteMarker + for len(oldVersions) > 0 && versionLess(oldVersions[0], obj) { + newVersions = append(newVersions, oldVersions[0]) + oldVersions = oldVersions[1:] + } + newVersions = append(newVersions, obj) + } + // Merge any remaining versions + newVersions = append(newVersions, oldVersions...) + return newVersions +} + // List a bucket with versions -func (ls *versionsList) List(ctx context.Context, hidden bool) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { +func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) { respVersions, err := ls.f.c.ListObjectVersionsWithContext(ctx, &ls.req) if err != nil { return nil, nil, err @@ -2956,16 +3047,36 @@ func (ls *versionsList) List(ctx context.Context, hidden bool) (resp *s3.ListObj resp = new(s3.ListObjectsV2Output) structs.SetFrom(resp, respVersions) + // Merge in delete Markers as s3.ObjectVersion if we need them + if ls.hidden || ls.usingVersionAt { + respVersions.Versions = mergeDeleteMarkers(respVersions.Versions, respVersions.DeleteMarkers) + } + // Convert the Versions and the DeleteMarkers into an array of s3.Object // // These are returned in the order that they are stored with the most recent first. // With the annoyance that the Versions and DeleteMarkers are split into two objs := make([]*s3.Object, 0, len(respVersions.Versions)) for _, objVersion := range respVersions.Versions { + if ls.usingVersionAt { + if objVersion.LastModified.After(ls.versionAt) { + // Ignore versions that were created after the specified time + continue + } + if *objVersion.Key == ls.lastKeySent { + // Ignore versions before the already returned version + continue + } + } + ls.lastKeySent = *objVersion.Key + // Don't send delete markers if we don't want hidden things + if !ls.hidden && objVersion.Size == isDeleteMarker { + continue + } var obj = new(s3.Object) structs.SetFrom(obj, objVersion) // Adjust the file names - if !aws.BoolValue(objVersion.IsLatest) { + if !ls.usingVersionAt && !aws.BoolValue(objVersion.IsLatest) { if obj.Key != nil && objVersion.LastModified != nil { *obj.Key = version.Add(*obj.Key, *objVersion.LastModified) } @@ -2974,25 +3085,6 @@ func (ls *versionsList) List(ctx context.Context, hidden bool) (resp *s3.ListObj versionIDs = append(versionIDs, objVersion.VersionId) } - // If hidden is set, put the delete markers in too, but set - // their sizes to a sentinel delete marker size - if hidden { - for _, deleteMarker := range respVersions.DeleteMarkers { - var obj = new(s3.Object) - structs.SetFrom(obj, deleteMarker) - obj.Size = isDeleteMarker - // Adjust the file names - if !aws.BoolValue(deleteMarker.IsLatest) { - if obj.Key != nil && deleteMarker.LastModified != nil { - *obj.Key = version.Add(*obj.Key, *deleteMarker.LastModified) - } - } - objs = append(objs, obj) - versionIDs = append(versionIDs, deleteMarker.VersionId) - - } - } - resp.Contents = objs return resp, versionIDs, nil } @@ -3015,14 +3107,15 @@ var errEndList = errors.New("end list") // list options type listOpt struct { - bucket string // bucket to list - directory string // directory with bucket - prefix string // prefix to remove from listing - addBucket bool // if set, the bucket is added to the start of the remote - recurse bool // if set, recurse to read sub directories - withVersions bool // if set, versions are produced - hidden bool // if set, return delete markers as objects with size == isDeleteMarker - findFile bool // if set, it will look for files called (bucket, directory) + bucket string // bucket to list + directory string // directory with bucket + prefix string // prefix to remove from listing + addBucket bool // if set, the bucket is added to the start of the remote + recurse bool // if set, recurse to read sub directories + withVersions bool // if set, versions are produced + hidden bool // if set, return delete markers as objects with size == isDeleteMarker + findFile bool // if set, it will look for files called (bucket, directory) + versionAt fs.Time // if set only show versions <= this time } // list lists the objects into the function supplied with the opt @@ -3067,8 +3160,8 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error { } var listBucket bucketLister switch { - case opt.withVersions: - listBucket = f.newVersionsList(&req) + case opt.withVersions || opt.versionAt.IsSet(): + listBucket = f.newVersionsList(&req, opt.hidden, time.Time(opt.versionAt)) case f.opt.ListVersion == 1: listBucket = f.newV1List(&req) default: @@ -3080,7 +3173,7 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error { var versionIDs []*string err = f.pacer.Call(func() (bool, error) { listBucket.URLEncodeListings(urlEncodeListings) - resp, versionIDs, err = listBucket.List(ctx, opt.hidden) + resp, versionIDs, err = listBucket.List(ctx) if err != nil && !urlEncodeListings { if awsErr, ok := err.(awserr.RequestFailure); ok { if origErr := awsErr.OrigErr(); origErr != nil { @@ -3215,6 +3308,7 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB prefix: prefix, addBucket: addBucket, withVersions: f.opt.Versions, + versionAt: f.opt.VersionAt, }, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory) if err != nil { @@ -3300,6 +3394,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) ( addBucket: addBucket, recurse: true, withVersions: f.opt.Versions, + versionAt: f.opt.VersionAt, }, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error { entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory) if err != nil { @@ -3609,6 +3704,9 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst // // If it isn't possible then return fs.ErrorCantCopy func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + if f.opt.VersionAt.IsSet() { + return nil, errNotWithVersionAt + } dstBucket, dstPath := f.split(remote) err := f.makeBucket(ctx, dstBucket) if err != nil { @@ -4074,6 +4172,9 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) { // // Implemented here so we can make sure we delete old versions. func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error { + if f.opt.VersionAt.IsSet() { + return errNotWithVersionAt + } bucket, directory := f.split(dir) if bucket == "" { return errors.New("can't purge from root") @@ -4871,6 +4972,9 @@ func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.P // Update the Object from in with modTime and size func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + if o.fs.opt.VersionAt.IsSet() { + return errNotWithVersionAt + } bucket, bucketPath := o.split() err := o.fs.makeBucket(ctx, bucket) if err != nil { @@ -5075,6 +5179,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // Remove an object func (o *Object) Remove(ctx context.Context) error { + if o.fs.opt.VersionAt.IsSet() { + return errNotWithVersionAt + } bucket, bucketPath := o.split() req := s3.DeleteObjectInput{ Bucket: &bucket, diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go index 7f26f919e39a5..948b5242eaae5 100644 --- a/backend/s3/s3_internal_test.go +++ b/backend/s3/s3_internal_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/fstests" @@ -85,6 +87,117 @@ func (f *Fs) InternalTestNoHead(t *testing.T) { } +func TestVersionLess(t *testing.T) { + key1 := "key1" + key2 := "key2" + t1 := fstest.Time("2022-01-21T12:00:00+01:00") + t2 := fstest.Time("2022-01-21T12:00:01+01:00") + for n, test := range []struct { + a, b *s3.ObjectVersion + want bool + }{ + {a: nil, b: nil, want: true}, + {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: nil, want: false}, + {a: nil, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true}, + {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false}, + {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t2}, want: false}, + {a: &s3.ObjectVersion{Key: &key1, LastModified: &t2}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true}, + {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key2, LastModified: &t1}, want: true}, + {a: &s3.ObjectVersion{Key: &key2, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false}, + {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false}, + {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true}, + {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, want: false}, + } { + got := versionLess(test.a, test.b) + assert.Equal(t, test.want, got, fmt.Sprintf("%d: %+v", n, test)) + } +} + +func TestMergeDeleteMarkers(t *testing.T) { + key1 := "key1" + key2 := "key2" + t1 := fstest.Time("2022-01-21T12:00:00+01:00") + t2 := fstest.Time("2022-01-21T12:00:01+01:00") + for n, test := range []struct { + versions []*s3.ObjectVersion + markers []*s3.DeleteMarkerEntry + want []*s3.ObjectVersion + }{ + { + versions: []*s3.ObjectVersion{}, + markers: []*s3.DeleteMarkerEntry{}, + want: []*s3.ObjectVersion{}, + }, + { + versions: []*s3.ObjectVersion{ + { + Key: &key1, + LastModified: &t1, + }, + }, + markers: []*s3.DeleteMarkerEntry{}, + want: []*s3.ObjectVersion{ + { + Key: &key1, + LastModified: &t1, + }, + }, + }, + { + versions: []*s3.ObjectVersion{}, + markers: []*s3.DeleteMarkerEntry{ + { + Key: &key1, + LastModified: &t1, + }, + }, + want: []*s3.ObjectVersion{ + { + Key: &key1, + LastModified: &t1, + Size: isDeleteMarker, + }, + }, + }, + { + versions: []*s3.ObjectVersion{ + { + Key: &key1, + LastModified: &t2, + }, + { + Key: &key2, + LastModified: &t2, + }, + }, + markers: []*s3.DeleteMarkerEntry{ + { + Key: &key1, + LastModified: &t1, + }, + }, + want: []*s3.ObjectVersion{ + { + Key: &key1, + LastModified: &t2, + }, + { + Key: &key1, + LastModified: &t1, + Size: isDeleteMarker, + }, + { + Key: &key2, + LastModified: &t2, + }, + }, + }, + } { + got := mergeDeleteMarkers(test.versions, test.markers) + assert.Equal(t, test.want, got, fmt.Sprintf("%d: %+v", n, test)) + } +} + func (f *Fs) InternalTestVersions(t *testing.T) { ctx := context.Background() @@ -99,6 +212,10 @@ func (f *Fs) InternalTestVersions(t *testing.T) { assert.NoError(t, err) }() + // Small pause to make the LastModified different since AWS + // only seems to track them to 1 second granularity + time.Sleep(2 * time.Second) + // Create an object const fileName = "test-versions.txt" contents := random.String(100) @@ -108,57 +225,118 @@ func (f *Fs) InternalTestVersions(t *testing.T) { assert.NoError(t, obj.Remove(ctx)) }() + // Small pause + time.Sleep(2 * time.Second) + // Remove it assert.NoError(t, obj.Remove(ctx)) + // Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity + time.Sleep(2 * time.Second) + // And create it with different size and contents newContents := random.String(101) newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z")) - _ = fstests.PutTestContents(ctx, t, f, &newItem, newContents, true) + newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true) - // Add the expected version suffix to the old version - item.Path = version.Add(item.Path, obj.(*Object).lastModified) - - t.Run("S3Version", func(t *testing.T) { + t.Run("Versions", func(t *testing.T) { // Set --s3-versions for this test f.opt.Versions = true defer func() { f.opt.Versions = false }() - // Check listing - items := append([]fstest.Item{item, newItem}, fstests.InternalTestFiles...) - fstest.CheckListing(t, f, items) - // Read the contents entries, err := f.List(ctx, "") require.NoError(t, err) tests := 0 + var fileNameVersion string for _, entry := range entries { - switch entry.Remote() { - case newItem.Path: + remote := entry.Remote() + if remote == fileName { t.Run("ReadCurrent", func(t *testing.T) { assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) }) tests++ - case item.Path: + } else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName { t.Run("ReadVersion", func(t *testing.T) { assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) }) + assert.WithinDuration(t, obj.(*Object).lastModified, versionTime, time.Second, "object time must be with 1 second of version time") + fileNameVersion = remote tests++ } } - assert.Equal(t, 2, tests) + assert.Equal(t, 2, tests, "object missing from listing") // Check we can read the object with a version suffix t.Run("NewObject", func(t *testing.T) { - o, err := f.NewObject(ctx, item.Path) + o, err := f.NewObject(ctx, fileNameVersion) require.NoError(t, err) require.NotNil(t, o) assert.Equal(t, int64(100), o.Size(), o.Remote()) }) }) + t.Run("VersionAt", func(t *testing.T) { + // We set --s3-version-at for this test so make sure we reset it at the end + defer func() { + f.opt.VersionAt = fs.Time{} + }() + + var ( + firstObjectTime = obj.(*Object).lastModified + secondObjectTime = newObj.(*Object).lastModified + ) + + for _, test := range []struct { + what string + at time.Time + want []fstest.Item + wantErr error + wantSize int64 + }{ + { + what: "Before", + at: firstObjectTime.Add(-time.Second), + want: fstests.InternalTestFiles, + wantErr: fs.ErrorObjectNotFound, + }, + { + what: "AfterOne", + at: firstObjectTime.Add(time.Second), + want: append([]fstest.Item{item}, fstests.InternalTestFiles...), + wantSize: 100, + }, + { + what: "AfterDelete", + at: secondObjectTime.Add(-time.Second), + want: fstests.InternalTestFiles, + wantErr: fs.ErrorObjectNotFound, + }, + { + what: "AfterTwo", + at: secondObjectTime.Add(time.Second), + want: append([]fstest.Item{newItem}, fstests.InternalTestFiles...), + wantSize: 101, + }, + } { + t.Run(test.what, func(t *testing.T) { + f.opt.VersionAt = fs.Time(test.at) + t.Run("List", func(t *testing.T) { + fstest.CheckListing(t, f, test.want) + }) + t.Run("NewObject", func(t *testing.T) { + gotObj, gotErr := f.NewObject(ctx, fileName) + assert.Equal(t, test.wantErr, gotErr) + if gotErr == nil { + assert.Equal(t, test.wantSize, gotObj.Size()) + } + }) + }) + } + }) + t.Run("Cleanup", func(t *testing.T) { require.NoError(t, f.CleanUpHidden(ctx)) items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...) diff --git a/docs/content/s3.md b/docs/content/s3.md index d1b99c26c4bac..dc5fd6d474ad5 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -394,7 +394,13 @@ Likewise when you delete a file, the old version will be marked hidden and still be available. Old versions of files, where available, are visible using the -`--s3-versions` flag. +[`--s3-versions`](#s3-versions) flag. + +It is also possible to view a bucket as it was at a certain point in +time, using the [`--s3-version-at`](#s3-version-at) flag. This will +show the file versions as they were at that time, showing files that +have been deleted afterwards, and hiding files that were created +since. If you wish to remove all the old versions then you can use the [`rclone backend cleanup-hidden remote:bucket`](#cleanup-hidden) @@ -2635,6 +2641,26 @@ Properties: - Type: bool - Default: false +#### --s3-version-at + +Show file versions as they were at the specified time. + +The parameter should be a date, "2006-01-02", datetime "2006-01-02 +15:04:05" or a duration for that long ago, eg "100d" or "1h". + +Note that when using this no file write operations are permitted, +so you can't upload files or delete them. + +See [the time option docs](/docs/#time-option) for valid formats. + + +Properties: + +- Config: version_at +- Env Var: RCLONE_S3_VERSION_AT +- Type: Time +- Default: off + ### Metadata User metadata is stored as x-amz-meta- keys. S3 metadata keys are case insensitive and are always returned in lower case. From 4b981100db7196fe221e8daa7e77a9dce1a171d7 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 28 Jul 2022 11:49:19 +0100 Subject: [PATCH 225/560] s3: refactor to use generated code instead of reflection to copy structs --- backend/s3/gen_setfrom.go | 101 ++++++++++++++++ backend/s3/s3.go | 39 ++++--- backend/s3/setfrom.go | 239 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 366 insertions(+), 13 deletions(-) create mode 100644 backend/s3/gen_setfrom.go create mode 100644 backend/s3/setfrom.go diff --git a/backend/s3/gen_setfrom.go b/backend/s3/gen_setfrom.go new file mode 100644 index 0000000000000..b3e0247f245cd --- /dev/null +++ b/backend/s3/gen_setfrom.go @@ -0,0 +1,101 @@ +// Generate boilerplate code for setting similar structs from each other + +//go:build ignore +// +build ignore + +package main + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "reflect" + "strings" + + "github.com/aws/aws-sdk-go/service/s3" +) + +// flags +var ( + outputFile = flag.String("o", "", "Output file name, stdout if unset") +) + +// globals +var ( + out io.Writer = os.Stdout +) + +// genSetFrom generates code to set the public members of a from b +// +// a and b should be pointers to structs +// +// a can be a different type from b +// +// Only the Fields which have the same name and assignable type on a +// and b will be set. +// +// This is useful for copying between almost identical structures that +// are frequently present in auto-generated code for cloud storage +// interfaces. +func genSetFrom(a, b interface{}) { + name := fmt.Sprintf("setFrom_%T_%T", a, b) + name = strings.Replace(name, ".", "", -1) + name = strings.Replace(name, "*", "", -1) + fmt.Fprintf(out, "\n// %s copies matching elements from a to b\n", name) + fmt.Fprintf(out, "func %s(a %T, b %T) {\n", name, a, b) + ta := reflect.TypeOf(a).Elem() + tb := reflect.TypeOf(b).Elem() + va := reflect.ValueOf(a).Elem() + vb := reflect.ValueOf(b).Elem() + for i := 0; i < tb.NumField(); i++ { + bField := vb.Field(i) + tbField := tb.Field(i) + name := tbField.Name + aField := va.FieldByName(name) + taField, found := ta.FieldByName(name) + if found && aField.IsValid() && bField.IsValid() && aField.CanSet() && tbField.Type.AssignableTo(taField.Type) { + fmt.Fprintf(out, "\ta.%s = b.%s\n", name, name) + } + } + fmt.Fprintf(out, "}\n") +} + +func main() { + flag.Parse() + + if *outputFile != "" { + fd, err := os.Create(*outputFile) + if err != nil { + log.Fatal(err) + } + defer func() { + err := fd.Close() + if err != nil { + log.Fatal(err) + } + }() + out = fd + } + + fmt.Fprintf(out, `// Code generated by "go run gen_setfrom.go"; DO NOT EDIT. + +package s3 + +import "github.com/aws/aws-sdk-go/service/s3" +`) + + genSetFrom(new(s3.ListObjectsInput), new(s3.ListObjectsV2Input)) + genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectsOutput)) + genSetFrom(new(s3.ListObjectVersionsInput), new(s3.ListObjectsV2Input)) + genSetFrom(new(s3.ObjectVersion), new(s3.DeleteMarkerEntry)) + genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectVersionsOutput)) + genSetFrom(new(s3.Object), new(s3.ObjectVersion)) + genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.HeadObjectOutput)) + genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.CopyObjectInput)) + genSetFrom(new(s3.UploadPartCopyInput), new(s3.CopyObjectInput)) + genSetFrom(new(s3.HeadObjectOutput), new(s3.GetObjectOutput)) + genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.PutObjectInput)) + genSetFrom(new(s3.HeadObjectOutput), new(s3.PutObjectInput)) +} diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 4f7dd8aeef0d7..78f23b3ed953d 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1,6 +1,8 @@ // Package s3 provides an interface to Amazon S3 oject storage package s3 +//go:generate go run gen_setfrom.go -o setfrom.go + import ( "bytes" "context" @@ -54,7 +56,6 @@ import ( "github.com/rclone/rclone/lib/pool" "github.com/rclone/rclone/lib/readers" "github.com/rclone/rclone/lib/rest" - "github.com/rclone/rclone/lib/structs" "github.com/rclone/rclone/lib/version" "golang.org/x/sync/errgroup" ) @@ -2866,7 +2867,8 @@ func (f *Fs) newV1List(req *s3.ListObjectsV2Input) bucketLister { f: f, } // Convert v2 req into v1 req - structs.SetFrom(&l.req, req) + //structs.SetFrom(&l.req, req) + setFrom_s3ListObjectsInput_s3ListObjectsV2Input(&l.req, req) return l } @@ -2898,7 +2900,8 @@ func (ls *v1List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versi // convert v1 resp into v2 resp resp = new(s3.ListObjectsV2Output) - structs.SetFrom(resp, respv1) + //structs.SetFrom(resp, respv1) + setFrom_s3ListObjectsV2Output_s3ListObjectsOutput(resp, respv1) return resp, nil, nil } @@ -2961,7 +2964,8 @@ func (f *Fs) newVersionsList(req *s3.ListObjectsV2Input, hidden bool, versionAt hidden: hidden, } // Convert v2 req into withVersions req - structs.SetFrom(&l.req, req) + //structs.SetFrom(&l.req, req) + setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input(&l.req, req) return l } @@ -3011,7 +3015,8 @@ func mergeDeleteMarkers(oldVersions []*s3.ObjectVersion, deleteMarkers []*s3.Del newVersions = make([]*s3.ObjectVersion, 0, len(oldVersions)+len(deleteMarkers)) for _, deleteMarker := range deleteMarkers { var obj = new(s3.ObjectVersion) - structs.SetFrom(obj, deleteMarker) + //structs.SetFrom(obj, deleteMarker) + setFrom_s3ObjectVersion_s3DeleteMarkerEntry(obj, deleteMarker) obj.Size = isDeleteMarker for len(oldVersions) > 0 && versionLess(oldVersions[0], obj) { newVersions = append(newVersions, oldVersions[0]) @@ -3045,7 +3050,8 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output, // convert Versions resp into v2 resp resp = new(s3.ListObjectsV2Output) - structs.SetFrom(resp, respVersions) + //structs.SetFrom(resp, respVersions) + setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(resp, respVersions) // Merge in delete Markers as s3.ObjectVersion if we need them if ls.hidden || ls.usingVersionAt { @@ -3074,7 +3080,8 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output, continue } var obj = new(s3.Object) - structs.SetFrom(obj, objVersion) + //structs.SetFrom(obj, objVersion) + setFrom_s3Object_s3ObjectVersion(obj, objVersion) // Adjust the file names if !ls.usingVersionAt && !aws.BoolValue(objVersion.IsLatest) { if obj.Key != nil && objVersion.LastModified != nil { @@ -3611,7 +3618,8 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst req := &s3.CreateMultipartUploadInput{} // Fill in the request from the head info - structs.SetFrom(req, info) + //structs.SetFrom(req, info) + setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(req, info) // If copy metadata was set then set the Metadata to that read // from the head request @@ -3620,7 +3628,8 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst } // Overwrite any from the copyReq - structs.SetFrom(req, copyReq) + //structs.SetFrom(req, copyReq) + setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(req, copyReq) req.Bucket = &dstBucket req.Key = &dstPath @@ -3660,7 +3669,8 @@ func (f *Fs) copyMultipart(ctx context.Context, copyReq *s3.CopyObjectInput, dst if err := f.pacer.Call(func() (bool, error) { partNum := partNum uploadPartReq := &s3.UploadPartCopyInput{} - structs.SetFrom(uploadPartReq, copyReq) + //structs.SetFrom(uploadPartReq, copyReq) + setFrom_s3UploadPartCopyInput_s3CopyObjectInput(uploadPartReq, copyReq) uploadPartReq.Bucket = &dstBucket uploadPartReq.Key = &dstPath uploadPartReq.PartNumber = &partNum @@ -4632,7 +4642,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } } var head s3.HeadObjectOutput - structs.SetFrom(&head, resp) + //structs.SetFrom(&head, resp) + setFrom_s3HeadObjectOutput_s3GetObjectOutput(&head, resp) head.ContentLength = size o.setMetaData(&head) return resp.Body, nil @@ -4675,7 +4686,8 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si memPool := f.getMemoryPool(int64(partSize)) var mReq s3.CreateMultipartUploadInput - structs.SetFrom(&mReq, req) + //structs.SetFrom(&mReq, req) + setFrom_s3CreateMultipartUploadInput_s3PutObjectInput(&mReq, req) var cout *s3.CreateMultipartUploadOutput err = f.pacer.Call(func() (bool, error) { var err error @@ -5144,7 +5156,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // uploaded properly. If size < 0 then we need to do the HEAD. if o.fs.opt.NoHead && size >= 0 { var head s3.HeadObjectOutput - structs.SetFrom(&head, &req) + //structs.SetFrom(&head, &req) + setFrom_s3HeadObjectOutput_s3PutObjectInput(&head, &req) head.ETag = &md5sumHex // doesn't matter quotes are misssing head.ContentLength = &size // If we have done a single part PUT request then we can read these diff --git a/backend/s3/setfrom.go b/backend/s3/setfrom.go new file mode 100644 index 0000000000000..1f7000b241949 --- /dev/null +++ b/backend/s3/setfrom.go @@ -0,0 +1,239 @@ +// Code generated by "go run gen_setfrom.go"; DO NOT EDIT. + +package s3 + +import "github.com/aws/aws-sdk-go/service/s3" + +// setFrom_s3ListObjectsInput_s3ListObjectsV2Input copies matching elements from a to b +func setFrom_s3ListObjectsInput_s3ListObjectsV2Input(a *s3.ListObjectsInput, b *s3.ListObjectsV2Input) { + a.Bucket = b.Bucket + a.Delimiter = b.Delimiter + a.EncodingType = b.EncodingType + a.ExpectedBucketOwner = b.ExpectedBucketOwner + a.MaxKeys = b.MaxKeys + a.Prefix = b.Prefix + a.RequestPayer = b.RequestPayer +} + +// setFrom_s3ListObjectsV2Output_s3ListObjectsOutput copies matching elements from a to b +func setFrom_s3ListObjectsV2Output_s3ListObjectsOutput(a *s3.ListObjectsV2Output, b *s3.ListObjectsOutput) { + a.CommonPrefixes = b.CommonPrefixes + a.Contents = b.Contents + a.Delimiter = b.Delimiter + a.EncodingType = b.EncodingType + a.IsTruncated = b.IsTruncated + a.MaxKeys = b.MaxKeys + a.Name = b.Name + a.Prefix = b.Prefix +} + +// setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input copies matching elements from a to b +func setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input(a *s3.ListObjectVersionsInput, b *s3.ListObjectsV2Input) { + a.Bucket = b.Bucket + a.Delimiter = b.Delimiter + a.EncodingType = b.EncodingType + a.ExpectedBucketOwner = b.ExpectedBucketOwner + a.MaxKeys = b.MaxKeys + a.Prefix = b.Prefix +} + +// setFrom_s3ObjectVersion_s3DeleteMarkerEntry copies matching elements from a to b +func setFrom_s3ObjectVersion_s3DeleteMarkerEntry(a *s3.ObjectVersion, b *s3.DeleteMarkerEntry) { + a.IsLatest = b.IsLatest + a.Key = b.Key + a.LastModified = b.LastModified + a.Owner = b.Owner + a.VersionId = b.VersionId +} + +// setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput copies matching elements from a to b +func setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(a *s3.ListObjectsV2Output, b *s3.ListObjectVersionsOutput) { + a.CommonPrefixes = b.CommonPrefixes + a.Delimiter = b.Delimiter + a.EncodingType = b.EncodingType + a.IsTruncated = b.IsTruncated + a.MaxKeys = b.MaxKeys + a.Name = b.Name + a.Prefix = b.Prefix +} + +// setFrom_s3Object_s3ObjectVersion copies matching elements from a to b +func setFrom_s3Object_s3ObjectVersion(a *s3.Object, b *s3.ObjectVersion) { + a.ChecksumAlgorithm = b.ChecksumAlgorithm + a.ETag = b.ETag + a.Key = b.Key + a.LastModified = b.LastModified + a.Owner = b.Owner + a.Size = b.Size + a.StorageClass = b.StorageClass +} + +// setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput copies matching elements from a to b +func setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(a *s3.CreateMultipartUploadInput, b *s3.HeadObjectOutput) { + a.BucketKeyEnabled = b.BucketKeyEnabled + a.CacheControl = b.CacheControl + a.ContentDisposition = b.ContentDisposition + a.ContentEncoding = b.ContentEncoding + a.ContentLanguage = b.ContentLanguage + a.ContentType = b.ContentType + a.Metadata = b.Metadata + a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus + a.ObjectLockMode = b.ObjectLockMode + a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 + a.SSEKMSKeyId = b.SSEKMSKeyId + a.ServerSideEncryption = b.ServerSideEncryption + a.StorageClass = b.StorageClass + a.WebsiteRedirectLocation = b.WebsiteRedirectLocation +} + +// setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput copies matching elements from a to b +func setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(a *s3.CreateMultipartUploadInput, b *s3.CopyObjectInput) { + a.ACL = b.ACL + a.Bucket = b.Bucket + a.BucketKeyEnabled = b.BucketKeyEnabled + a.CacheControl = b.CacheControl + a.ChecksumAlgorithm = b.ChecksumAlgorithm + a.ContentDisposition = b.ContentDisposition + a.ContentEncoding = b.ContentEncoding + a.ContentLanguage = b.ContentLanguage + a.ContentType = b.ContentType + a.ExpectedBucketOwner = b.ExpectedBucketOwner + a.Expires = b.Expires + a.GrantFullControl = b.GrantFullControl + a.GrantRead = b.GrantRead + a.GrantReadACP = b.GrantReadACP + a.GrantWriteACP = b.GrantWriteACP + a.Key = b.Key + a.Metadata = b.Metadata + a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus + a.ObjectLockMode = b.ObjectLockMode + a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate + a.RequestPayer = b.RequestPayer + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKey = b.SSECustomerKey + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 + a.SSEKMSEncryptionContext = b.SSEKMSEncryptionContext + a.SSEKMSKeyId = b.SSEKMSKeyId + a.ServerSideEncryption = b.ServerSideEncryption + a.StorageClass = b.StorageClass + a.Tagging = b.Tagging + a.WebsiteRedirectLocation = b.WebsiteRedirectLocation +} + +// setFrom_s3UploadPartCopyInput_s3CopyObjectInput copies matching elements from a to b +func setFrom_s3UploadPartCopyInput_s3CopyObjectInput(a *s3.UploadPartCopyInput, b *s3.CopyObjectInput) { + a.Bucket = b.Bucket + a.CopySource = b.CopySource + a.CopySourceIfMatch = b.CopySourceIfMatch + a.CopySourceIfModifiedSince = b.CopySourceIfModifiedSince + a.CopySourceIfNoneMatch = b.CopySourceIfNoneMatch + a.CopySourceIfUnmodifiedSince = b.CopySourceIfUnmodifiedSince + a.CopySourceSSECustomerAlgorithm = b.CopySourceSSECustomerAlgorithm + a.CopySourceSSECustomerKey = b.CopySourceSSECustomerKey + a.CopySourceSSECustomerKeyMD5 = b.CopySourceSSECustomerKeyMD5 + a.ExpectedBucketOwner = b.ExpectedBucketOwner + a.ExpectedSourceBucketOwner = b.ExpectedSourceBucketOwner + a.Key = b.Key + a.RequestPayer = b.RequestPayer + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKey = b.SSECustomerKey + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 +} + +// setFrom_s3HeadObjectOutput_s3GetObjectOutput copies matching elements from a to b +func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.GetObjectOutput) { + a.AcceptRanges = b.AcceptRanges + a.BucketKeyEnabled = b.BucketKeyEnabled + a.CacheControl = b.CacheControl + a.ChecksumCRC32 = b.ChecksumCRC32 + a.ChecksumCRC32C = b.ChecksumCRC32C + a.ChecksumSHA1 = b.ChecksumSHA1 + a.ChecksumSHA256 = b.ChecksumSHA256 + a.ContentDisposition = b.ContentDisposition + a.ContentEncoding = b.ContentEncoding + a.ContentLanguage = b.ContentLanguage + a.ContentLength = b.ContentLength + a.ContentType = b.ContentType + a.DeleteMarker = b.DeleteMarker + a.ETag = b.ETag + a.Expiration = b.Expiration + a.Expires = b.Expires + a.LastModified = b.LastModified + a.Metadata = b.Metadata + a.MissingMeta = b.MissingMeta + a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus + a.ObjectLockMode = b.ObjectLockMode + a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate + a.PartsCount = b.PartsCount + a.ReplicationStatus = b.ReplicationStatus + a.RequestCharged = b.RequestCharged + a.Restore = b.Restore + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 + a.SSEKMSKeyId = b.SSEKMSKeyId + a.ServerSideEncryption = b.ServerSideEncryption + a.StorageClass = b.StorageClass + a.VersionId = b.VersionId + a.WebsiteRedirectLocation = b.WebsiteRedirectLocation +} + +// setFrom_s3CreateMultipartUploadInput_s3PutObjectInput copies matching elements from a to b +func setFrom_s3CreateMultipartUploadInput_s3PutObjectInput(a *s3.CreateMultipartUploadInput, b *s3.PutObjectInput) { + a.ACL = b.ACL + a.Bucket = b.Bucket + a.BucketKeyEnabled = b.BucketKeyEnabled + a.CacheControl = b.CacheControl + a.ChecksumAlgorithm = b.ChecksumAlgorithm + a.ContentDisposition = b.ContentDisposition + a.ContentEncoding = b.ContentEncoding + a.ContentLanguage = b.ContentLanguage + a.ContentType = b.ContentType + a.ExpectedBucketOwner = b.ExpectedBucketOwner + a.Expires = b.Expires + a.GrantFullControl = b.GrantFullControl + a.GrantRead = b.GrantRead + a.GrantReadACP = b.GrantReadACP + a.GrantWriteACP = b.GrantWriteACP + a.Key = b.Key + a.Metadata = b.Metadata + a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus + a.ObjectLockMode = b.ObjectLockMode + a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate + a.RequestPayer = b.RequestPayer + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKey = b.SSECustomerKey + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 + a.SSEKMSEncryptionContext = b.SSEKMSEncryptionContext + a.SSEKMSKeyId = b.SSEKMSKeyId + a.ServerSideEncryption = b.ServerSideEncryption + a.StorageClass = b.StorageClass + a.Tagging = b.Tagging + a.WebsiteRedirectLocation = b.WebsiteRedirectLocation +} + +// setFrom_s3HeadObjectOutput_s3PutObjectInput copies matching elements from a to b +func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.PutObjectInput) { + a.BucketKeyEnabled = b.BucketKeyEnabled + a.CacheControl = b.CacheControl + a.ChecksumCRC32 = b.ChecksumCRC32 + a.ChecksumCRC32C = b.ChecksumCRC32C + a.ChecksumSHA1 = b.ChecksumSHA1 + a.ChecksumSHA256 = b.ChecksumSHA256 + a.ContentDisposition = b.ContentDisposition + a.ContentEncoding = b.ContentEncoding + a.ContentLanguage = b.ContentLanguage + a.ContentLength = b.ContentLength + a.ContentType = b.ContentType + a.Metadata = b.Metadata + a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus + a.ObjectLockMode = b.ObjectLockMode + a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate + a.SSECustomerAlgorithm = b.SSECustomerAlgorithm + a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5 + a.SSEKMSKeyId = b.SSEKMSKeyId + a.ServerSideEncryption = b.ServerSideEncryption + a.StorageClass = b.StorageClass + a.WebsiteRedirectLocation = b.WebsiteRedirectLocation +} From 1f5e7ce598e1d22ecd9d6c69b50106e80d8aa620 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 29 Jul 2022 16:39:02 +0100 Subject: [PATCH 226/560] lib/readers: add GzipReader --- lib/readers/gzip.go | 42 +++++++++++++++++++++++++++++++++++ lib/readers/gzip_test.go | 47 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 lib/readers/gzip.go create mode 100644 lib/readers/gzip_test.go diff --git a/lib/readers/gzip.go b/lib/readers/gzip.go new file mode 100644 index 0000000000000..e018f288090a1 --- /dev/null +++ b/lib/readers/gzip.go @@ -0,0 +1,42 @@ +package readers + +import ( + "compress/gzip" + "io" +) + +// gzipReader wraps a *gzip.Reader so it closes the underlying stream +// which the gzip library doesn't. +type gzipReader struct { + *gzip.Reader + in io.ReadCloser +} + +// NewGzipReader returns an io.ReadCloser which will read the stream +// and close it when Close is called. +// +// Unfortunately gz.Reader does not close the underlying stream so we +// can't use that directly. +func NewGzipReader(in io.ReadCloser) (io.ReadCloser, error) { + zr, err := gzip.NewReader(in) + if err != nil { + return nil, err + } + return &gzipReader{ + Reader: zr, + in: in, + }, nil +} + +// Close the underlying stream and the gzip reader +func (gz *gzipReader) Close() error { + zrErr := gz.Reader.Close() + inErr := gz.in.Close() + if inErr != nil { + return inErr + } + if zrErr != nil { + return zrErr + } + return nil +} diff --git a/lib/readers/gzip_test.go b/lib/readers/gzip_test.go new file mode 100644 index 0000000000000..799e52f6e6122 --- /dev/null +++ b/lib/readers/gzip_test.go @@ -0,0 +1,47 @@ +package readers + +import ( + "bytes" + "compress/gzip" + "io" + "testing" + + "github.com/rclone/rclone/lib/random" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type checkClose struct { + io.Reader + closed bool +} + +func (cc *checkClose) Close() error { + cc.closed = true + return nil +} + +func TestGzipReader(t *testing.T) { + // Create some compressed data + data := random.String(1000) + var out bytes.Buffer + zw := gzip.NewWriter(&out) + _, err := io.Copy(zw, bytes.NewBufferString(data)) + require.NoError(t, err) + require.NoError(t, zw.Close()) + gzData := out.Bytes() + + // Check we can decompress it + cc := &checkClose{Reader: bytes.NewBuffer(gzData)} + var decompressed bytes.Buffer + zr, err := NewGzipReader(cc) + require.NoError(t, err) + _, err = io.Copy(&decompressed, zr) + require.NoError(t, err) + assert.Equal(t, data, string(decompressed.Bytes())) + + // Check the underlying close gets called + assert.False(t, cc.closed) + require.NoError(t, zr.Close()) + assert.True(t, cc.closed) +} From ebe86c6cec4b699397dc8c9d3e2a3b28c48fa3ab Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 29 Jul 2022 17:01:59 +0100 Subject: [PATCH 227/560] s3: add --s3-decompress flag to download gzip-encoded files Before this change, if an object compressed with "Content-Encoding: gzip" was downloaded, a length and hash mismatch would occur since the go runtime automatically decompressed the object on download. If --s3-decompress is set, this change erases the length and hash on compressed objects so they can be downloaded successfully, at the cost of not being able to check the length or the hash of the downloaded object. If --s3-decompress is not set the compressed files will be downloaded as-is providing compressed objects with intact size and hash information. See #2658 --- backend/s3/s3.go | 77 ++++++++++++++++++++++++++-------- backend/s3/s3_internal_test.go | 35 +++++++++++++++- 2 files changed, 93 insertions(+), 19 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 78f23b3ed953d..f0a02b391abcc 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2004,6 +2004,19 @@ See [the time option docs](/docs/#time-option) for valid formats. `, Default: fs.Time{}, Advanced: true, + }, { + Name: "decompress", + Help: `If set this will decompress gzip encoded objects. + +It is possible to upload objects to S3 with "Content-Encoding: gzip" +set. Normally rclone will download these files files as compressed objects. + +If this flag is set then rclone will decompress these files with +"Content-Encoding: gzip" as they are received. This means that rclone +can't check the size and hash but the file contents will be decompressed. +`, + Advanced: true, + Default: false, }, }}) } @@ -2128,28 +2141,30 @@ type Options struct { UsePresignedRequest bool `config:"use_presigned_request"` Versions bool `config:"versions"` VersionAt fs.Time `config:"version_at"` + Decompress bool `config:"decompress"` } // Fs represents a remote s3 server type Fs struct { - name string // the name of the remote - root string // root of the bucket - ignore all objects above this - opt Options // parsed options - ci *fs.ConfigInfo // global config - ctx context.Context // global context for reading config - features *fs.Features // optional features - c *s3.S3 // the connection to the s3 server - ses *session.Session // the s3 session - rootBucket string // bucket part of root (if any) - rootDirectory string // directory part of root (if any) - cache *bucket.Cache // cache for bucket creation status - pacer *fs.Pacer // To pace the API calls - srv *http.Client // a plain http client - srvRest *rest.Client // the rest connection to the server - pool *pool.Pool // memory pool - etagIsNotMD5 bool // if set ETags are not MD5s - versioningMu sync.Mutex - versioning fs.Tristate // if set bucket is using versions + name string // the name of the remote + root string // root of the bucket - ignore all objects above this + opt Options // parsed options + ci *fs.ConfigInfo // global config + ctx context.Context // global context for reading config + features *fs.Features // optional features + c *s3.S3 // the connection to the s3 server + ses *session.Session // the s3 session + rootBucket string // bucket part of root (if any) + rootDirectory string // directory part of root (if any) + cache *bucket.Cache // cache for bucket creation status + pacer *fs.Pacer // To pace the API calls + srv *http.Client // a plain http client + srvRest *rest.Client // the rest connection to the server + pool *pool.Pool // memory pool + etagIsNotMD5 bool // if set ETags are not MD5s + versioningMu sync.Mutex + versioning fs.Tristate // if set bucket is using versions + warnCompressed sync.Once // warn once about compressed files } // Object describes a s3 object @@ -4318,6 +4333,10 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { if t != hash.MD5 { return "", hash.ErrUnsupported } + // If decompressing, erase the hash + if o.bytes < 0 { + return "", nil + } // If we haven't got an MD5, then check the metadata if o.md5 == "" { err := o.readMetaData(ctx) @@ -4439,6 +4458,12 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) { o.contentDisposition = resp.ContentDisposition o.contentEncoding = resp.ContentEncoding o.contentLanguage = resp.ContentLanguage + + // If decompressing then size and md5sum are unknown + if o.fs.opt.Decompress && aws.StringValue(o.contentEncoding) == "gzip" { + o.bytes = -1 + o.md5 = "" + } } // ModTime returns the modification time of the object @@ -4596,6 +4621,11 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } httpReq, resp := o.fs.c.GetObjectRequest(&req) fs.FixRangeOption(options, o.bytes) + + // Override the automatic decompression in the transport to + // download compressed files as-is + httpReq.HTTPRequest.Header.Set("Accept-Encoding", "gzip") + for _, option := range options { switch option.(type) { case *fs.RangeOption, *fs.SeekOption: @@ -4646,6 +4676,17 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read setFrom_s3HeadObjectOutput_s3GetObjectOutput(&head, resp) head.ContentLength = size o.setMetaData(&head) + + // Decompress body if necessary + if aws.StringValue(resp.ContentEncoding) == "gzip" { + if o.fs.opt.Decompress { + return readers.NewGzipReader(resp.Body) + } + o.fs.warnCompressed.Do(func() { + fs.Logf(o, "Not decompressing 'Content-Encoding: gzip' compressed file. Use --s3-decompress to override") + }) + } + return resp.Body, nil } diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go index 948b5242eaae5..be12cc42db80e 100644 --- a/backend/s3/s3_internal_test.go +++ b/backend/s3/s3_internal_test.go @@ -4,6 +4,7 @@ import ( "bytes" "compress/gzip" "context" + "crypto/md5" "fmt" "testing" "time" @@ -11,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/fstests" "github.com/rclone/rclone/lib/random" @@ -29,9 +31,15 @@ func gz(t *testing.T, s string) string { return buf.String() } +func md5sum(t *testing.T, s string) string { + hash := md5.Sum([]byte(s)) + return fmt.Sprintf("%x", hash) +} + func (f *Fs) InternalTestMetadata(t *testing.T) { ctx := context.Background() - contents := gz(t, random.String(1000)) + original := random.String(1000) + contents := gz(t, original) item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) btime := time.Now() @@ -68,6 +76,31 @@ func (f *Fs) InternalTestMetadata(t *testing.T) { assert.Equal(t, v, got, k) } } + + t.Run("GzipEncoding", func(t *testing.T) { + // Test that the gziped file we uploaded can be + // downloaded with and without decompression + checkDownload := func(wantContents string, wantSize int64, wantHash string) { + gotContents := fstests.ReadObject(ctx, t, o, -1) + assert.Equal(t, wantContents, gotContents) + assert.Equal(t, wantSize, o.Size()) + gotHash, err := o.Hash(ctx, hash.MD5) + require.NoError(t, err) + assert.Equal(t, wantHash, gotHash) + } + + t.Run("NoDecompress", func(t *testing.T) { + checkDownload(contents, int64(len(contents)), md5sum(t, contents)) + }) + t.Run("Decompress", func(t *testing.T) { + f.opt.Decompress = true + defer func() { + f.opt.Decompress = false + }() + checkDownload(original, -1, "") + }) + + }) } func (f *Fs) InternalTestNoHead(t *testing.T) { From 16039b350da44ebb9e8e5d5e5f4b07144d207daf Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 5 Aug 2022 15:23:44 +0100 Subject: [PATCH 228/560] fs: fix parsing of times and durations of the form "YYYY-MM-DD HH:MM:SS" Before this fix, the parsing code gave an error like this parsing "2022-08-02 07:00:00" as fs.Time failed: expected newline This was due to the Scan call failing to read all the data. This patch fixes that, and redoes the tests --- fs/parseduration.go | 4 ++-- fs/parseduration_test.go | 27 ++++++++++++++++++++++----- fs/parsetime.go | 2 +- fs/parsetime_test.go | 26 +++++++++++++++++--------- 4 files changed, 42 insertions(+), 17 deletions(-) diff --git a/fs/parseduration.go b/fs/parseduration.go index 23e7494463a6b..fe7d099869d58 100644 --- a/fs/parseduration.go +++ b/fs/parseduration.go @@ -126,7 +126,7 @@ func parseDurationFromNow(age string, getNow func() time.Time) (d time.Duration, // ParseDuration parses a duration string. Accept ms|s|m|h|d|w|M|y suffixes. Defaults to second if not provided func ParseDuration(age string) (time.Duration, error) { - return parseDurationFromNow(age, time.Now) + return parseDurationFromNow(age, timeNowFunc) } // ReadableString parses d into a human-readable duration. @@ -216,7 +216,7 @@ func (d *Duration) UnmarshalJSON(in []byte) error { // Scan implements the fmt.Scanner interface func (d *Duration) Scan(s fmt.ScanState, ch rune) error { - token, err := s.Token(true, nil) + token, err := s.Token(true, func(rune) bool { return true }) if err != nil { return err } diff --git a/fs/parseduration_test.go b/fs/parseduration_test.go index f0efba4bdafb7..4a5dbb16c05f4 100644 --- a/fs/parseduration_test.go +++ b/fs/parseduration_test.go @@ -145,11 +145,28 @@ func TestDurationReadableString(t *testing.T) { } func TestDurationScan(t *testing.T) { - var v Duration - n, err := fmt.Sscan(" 17m ", &v) - require.NoError(t, err) - assert.Equal(t, 1, n) - assert.Equal(t, Duration(17*60*time.Second), v) + now := time.Date(2020, 9, 5, 8, 15, 5, 250, time.UTC) + oldTimeNowFunc := timeNowFunc + timeNowFunc = func() time.Time { return now } + defer func() { timeNowFunc = oldTimeNowFunc }() + + for _, test := range []struct { + in string + want Duration + }{ + {"17m", Duration(17 * time.Minute)}, + {"-12h", Duration(-12 * time.Hour)}, + {"0", Duration(0)}, + {"off", DurationOff}, + {"2022-03-26T17:48:19Z", Duration(now.Sub(time.Date(2022, 03, 26, 17, 48, 19, 0, time.UTC)))}, + {"2022-03-26 17:48:19", Duration(now.Sub(time.Date(2022, 03, 26, 17, 48, 19, 0, time.Local)))}, + } { + var got Duration + n, err := fmt.Sscan(test.in, &got) + require.NoError(t, err) + assert.Equal(t, 1, n) + assert.Equal(t, test.want, got) + } } func TestParseUnmarshalJSON(t *testing.T) { diff --git a/fs/parsetime.go b/fs/parsetime.go index 145cfa0dec23b..7e762451da719 100644 --- a/fs/parsetime.go +++ b/fs/parsetime.go @@ -83,7 +83,7 @@ func (t *Time) UnmarshalJSON(in []byte) error { // Scan implements the fmt.Scanner interface func (t *Time) Scan(s fmt.ScanState, ch rune) error { - token, err := s.Token(true, nil) + token, err := s.Token(true, func(rune) bool { return true }) if err != nil { return err } diff --git a/fs/parsetime_test.go b/fs/parsetime_test.go index 5784b971a3103..21d34c1eed945 100644 --- a/fs/parsetime_test.go +++ b/fs/parsetime_test.go @@ -93,15 +93,23 @@ func TestTimeScan(t *testing.T) { timeNowFunc = func() time.Time { return now } defer func() { timeNowFunc = oldTimeNowFunc }() - var v1, v2, v3, v4, v5 Time - n, err := fmt.Sscan(" 17m -12h 0 off 2022-03-26T17:48:19Z ", &v1, &v2, &v3, &v4, &v5) - require.NoError(t, err) - assert.Equal(t, 5, n) - assert.Equal(t, Time(now.Add(-17*time.Minute)), v1) - assert.Equal(t, Time(now.Add(12*time.Hour)), v2) - assert.Equal(t, Time(now), v3) - assert.Equal(t, Time(time.Time{}), v4) - assert.Equal(t, Time(time.Date(2022, 03, 26, 17, 48, 19, 0, time.UTC)), v5) + for _, test := range []struct { + in string + want Time + }{ + {"17m", Time(now.Add(-17 * time.Minute))}, + {"-12h", Time(now.Add(12 * time.Hour))}, + {"0", Time(now)}, + {"off", Time(time.Time{})}, + {"2022-03-26T17:48:19Z", Time(time.Date(2022, 03, 26, 17, 48, 19, 0, time.UTC))}, + {"2022-03-26 17:48:19", Time(time.Date(2022, 03, 26, 17, 48, 19, 0, time.Local))}, + } { + var got Time + n, err := fmt.Sscan(test.in, &got) + require.NoError(t, err) + assert.Equal(t, 1, n) + assert.Equal(t, test.want, got) + } } func TestParseTimeUnmarshalJSON(t *testing.T) { From 98fd00a655cada0bda53b7be3d54dd656a8ab21c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 4 Aug 2022 16:11:22 +0100 Subject: [PATCH 229/560] serve sftp: fix checksum detection - Fixes #6351 Before this change, rclone serve sftp operating with a new rclone after the md5sum/sha1sum detection was reworked to just run a plain `md5sum`/`sha1sum` command in 3ea82032e796d1f1 sftp: support md5/sha1 with rsync.net #3254 Failed to signal to the remote that md5sum/sha1sum wasn't supported as in 71e172a139f00291 serve/sftp: support empty "md5sum" and "sha1sum" commands We unconditionally return good hashes even if the remote being served doesn't support the hash type in question. This fix checks the hash type is supported and returns an error MD5 hash not supported When the backend is first contacted this will cause the sftp backend to detect that the hash type isn't available. Unfortunately this may have cached the wrong state so editing or remaking the config may be necessary to fix it. --- cmd/serve/sftp/connection.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/serve/sftp/connection.go b/cmd/serve/sftp/connection.go index d8be2bce5831b..d9eaf855debb8 100644 --- a/cmd/serve/sftp/connection.go +++ b/cmd/serve/sftp/connection.go @@ -101,6 +101,9 @@ func (c *conn) execCommand(ctx context.Context, out io.Writer, command string) ( if binary == "sha1sum" { ht = hash.SHA1 } + if !c.vfs.Fs().Hashes().Contains(ht) { + return fmt.Errorf("%v hash not supported", ht) + } var hashSum string if args == "" { // empty hash for no input From 49bb640bae0aac7367d43bf9f16c7e3ead8a62ab Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 20 Jul 2022 10:49:00 +0100 Subject: [PATCH 230/560] accounting: fix panic in core/stats-reset with unknown group - fixes #6327 This also adds tests for the rc commands for stats groups --- fs/accounting/stats_groups.go | 4 ++ fs/accounting/stats_groups_test.go | 85 ++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.go index 28548cfc9030a..7906df99842f4 100644 --- a/fs/accounting/stats_groups.go +++ b/fs/accounting/stats_groups.go @@ -2,6 +2,7 @@ package accounting import ( "context" + "fmt" "sync" "github.com/rclone/rclone/fs/rc" @@ -190,6 +191,9 @@ func rcResetStats(ctx context.Context, in rc.Params) (rc.Params, error) { if group != "" { stats := groups.get(group) + if stats == nil { + return rc.Params{}, fmt.Errorf("group %q not found", group) + } stats.ResetErrors() stats.ResetCounters() } else { diff --git a/fs/accounting/stats_groups_test.go b/fs/accounting/stats_groups_test.go index c111a08d80dc3..ed04d134ed511 100644 --- a/fs/accounting/stats_groups_test.go +++ b/fs/accounting/stats_groups_test.go @@ -7,8 +7,10 @@ import ( "testing" "time" + "github.com/rclone/rclone/fs/rc" "github.com/rclone/rclone/fstest/testy" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestStatsGroupOperations(t *testing.T) { @@ -117,6 +119,89 @@ func TestStatsGroupOperations(t *testing.T) { t.Errorf("HeapObjects = %d, expected %d", end.HeapObjects, start.HeapObjects) } }) + + testGroupStatsInfo := NewStatsGroup(ctx, "test-group") + testGroupStatsInfo.Deletes(1) + GlobalStats().Deletes(41) + + t.Run("core/group-list", func(t *testing.T) { + call := rc.Calls.Get("core/group-list") + require.NotNil(t, call) + got, err := call.Fn(ctx, rc.Params{}) + require.NoError(t, err) + require.Equal(t, rc.Params{ + "groups": []string{ + "test-group", + }, + }, got) + }) + + t.Run("core/stats", func(t *testing.T) { + call := rc.Calls.Get("core/stats") + require.NotNil(t, call) + gotNoGroup, err := call.Fn(ctx, rc.Params{}) + require.NoError(t, err) + gotGroup, err := call.Fn(ctx, rc.Params{"group": "test-group"}) + require.NoError(t, err) + assert.Equal(t, int64(42), gotNoGroup["deletes"]) + assert.Equal(t, int64(1), gotGroup["deletes"]) + }) + + t.Run("core/transferred", func(t *testing.T) { + call := rc.Calls.Get("core/transferred") + require.NotNil(t, call) + gotNoGroup, err := call.Fn(ctx, rc.Params{}) + require.NoError(t, err) + gotGroup, err := call.Fn(ctx, rc.Params{"group": "test-group"}) + require.NoError(t, err) + assert.Equal(t, rc.Params{ + "transferred": []TransferSnapshot{}, + }, gotNoGroup) + assert.Equal(t, rc.Params{ + "transferred": []TransferSnapshot{}, + }, gotGroup) + }) + + t.Run("core/stats-reset", func(t *testing.T) { + call := rc.Calls.Get("core/stats-reset") + require.NotNil(t, call) + + assert.Equal(t, int64(41), GlobalStats().deletes) + assert.Equal(t, int64(1), testGroupStatsInfo.deletes) + + _, err := call.Fn(ctx, rc.Params{"group": "test-group"}) + require.NoError(t, err) + + assert.Equal(t, int64(41), GlobalStats().deletes) + assert.Equal(t, int64(0), testGroupStatsInfo.deletes) + + _, err = call.Fn(ctx, rc.Params{}) + require.NoError(t, err) + + assert.Equal(t, int64(0), GlobalStats().deletes) + assert.Equal(t, int64(0), testGroupStatsInfo.deletes) + + _, err = call.Fn(ctx, rc.Params{"group": "not-found"}) + require.ErrorContains(t, err, `group "not-found" not found`) + + }) + + testGroupStatsInfo = NewStatsGroup(ctx, "test-group") + + t.Run("core/stats-delete", func(t *testing.T) { + call := rc.Calls.Get("core/stats-delete") + require.NotNil(t, call) + + assert.Equal(t, []string{"test-group"}, groups.names()) + + _, err := call.Fn(ctx, rc.Params{"group": "test-group"}) + require.NoError(t, err) + + assert.Equal(t, []string{}, groups.names()) + + _, err = call.Fn(ctx, rc.Params{"group": "not-found"}) + require.NoError(t, err) + }) } func percentDiff(start, end uint64) uint64 { From df513ca90a9dfff9f7dca9cc38e6ddc832afb597 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 5 Aug 2022 17:43:53 +0100 Subject: [PATCH 231/560] build: update dependencies --- go.mod | 62 +++++++++++------------ go.sum | 152 ++++++++++++++++++++++++++++++++++----------------------- 2 files changed, 123 insertions(+), 91 deletions(-) diff --git a/go.mod b/go.mod index 9e9e8c6ef5d59..31072dbff8dc2 100644 --- a/go.mod +++ b/go.mod @@ -6,32 +6,32 @@ require ( bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 - github.com/Azure/go-autorest/autorest/adal v0.9.20 - github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e + github.com/Azure/go-autorest/autorest/adal v0.9.21 + github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5 github.com/Unknwon/goconfig v1.0.0 github.com/a8m/tree v0.0.0-20210414114729-ce3525c5c2ef github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 github.com/abbot/go-http-auth v0.4.0 - github.com/anacrolix/dms v1.4.0 + github.com/anacrolix/dms v1.5.0 github.com/artyom/mtab v1.0.0 github.com/atotto/clipboard v0.1.4 - github.com/aws/aws-sdk-go v1.44.29 + github.com/aws/aws-sdk-go v1.44.70 github.com/buengese/sgzip v0.1.1 github.com/colinmarc/hdfs/v2 v2.3.0 github.com/coreos/go-semver v0.3.0 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00 github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.4 - github.com/gabriel-vasile/mimetype v1.4.0 - github.com/gdamore/tcell/v2 v2.5.1 + github.com/gabriel-vasile/mimetype v1.4.1 + github.com/gdamore/tcell/v2 v2.5.2 github.com/go-chi/chi/v5 v5.0.7 github.com/google/uuid v1.3.0 github.com/hanwen/go-fuse/v2 v2.1.0 github.com/iguanesolutions/go-systemd/v5 v5.1.0 - github.com/jcmturner/gokrb5/v8 v8.4.2 + github.com/jcmturner/gokrb5/v8 v8.4.3 github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 - github.com/klauspost/compress v1.15.6 + github.com/klauspost/compress v1.15.9 github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348 github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a github.com/mattn/go-colorable v0.1.12 @@ -40,17 +40,17 @@ require ( github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 github.com/ncw/swift/v2 v2.0.1 github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/pkg/sftp v1.13.5-0.20211228200725-31aac3e1878d + github.com/pkg/sftp v1.13.5 github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.12.2 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 github.com/rfjakob/eme v1.1.2 - github.com/shirou/gopsutil/v3 v3.22.5 - github.com/sirupsen/logrus v1.8.1 + github.com/shirou/gopsutil/v3 v3.22.7 + github.com/sirupsen/logrus v1.9.0 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 - github.com/spf13/cobra v1.4.0 + github.com/spf13/cobra v1.5.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.7.2 + github.com/stretchr/testify v1.8.0 github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf github.com/winfsp/cgofuse v1.5.1-0.20220421173602-ce7e5a65cac7 github.com/xanzy/ssh-agent v0.3.1 @@ -58,42 +58,44 @@ require ( github.com/yunify/qingstor-sdk-go/v3 v3.2.0 go.etcd.io/bbolt v1.3.6 goftp.io/server v0.4.1 - golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e - golang.org/x/net v0.0.0-20220607020251-c690dde0001d - golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb - golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a + golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa + golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 + golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 + golang.org/x/sys v0.0.0-20220804214406-8e32c043e418 golang.org/x/text v0.3.7 - golang.org/x/time v0.0.0-20220411224347-583f2d630306 - google.golang.org/api v0.83.0 + golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 + google.golang.org/api v0.91.0 gopkg.in/yaml.v2 v2.4.0 storj.io/uplink v1.9.0 ) require ( - cloud.google.com/go/compute v1.6.1 // indirect + cloud.google.com/go/compute v1.7.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/anacrolix/log v0.10.1-0.20220126091220-5c1b6f3af59c // indirect + github.com/anacrolix/log v0.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/calebcase/tmpfile v1.0.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/cloudflare/circl v1.1.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gdamore/encoding v1.0.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect github.com/googleapis/gax-go/v2 v2.4.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-uuid v1.0.2 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect - github.com/jcmturner/gofork v1.0.0 // indirect + github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/goidentity/v6 v6.0.1 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -119,7 +121,7 @@ require ( github.com/zeebo/errs v1.3.0 // indirect go.opencensus.io v0.23.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 // indirect + google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -129,12 +131,12 @@ require ( require ( github.com/Microsoft/go-winio v0.5.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20220623141421-5afb4c282135 + github.com/ProtonMail/go-crypto v0.0.0-20220730123233-d6ffb7692adf github.com/golang-jwt/jwt/v4 v4.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af + github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff github.com/pkg/xattr v0.4.7 - golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd - golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 + golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 + golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 ) diff --git a/go.sum b/go.sum index f00338169624c..d09ac3566dff5 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,9 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0 h1:DAq3r8y4mDgyB/ZPJ9v/5VJNqjgJAxTn6ZYLlUywOu8= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -42,10 +43,12 @@ cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTB cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1 h1:2sMmt8prCn7DPaG4Pmh0N3Inmc8cT8ae5k1M6VJ9Wqc= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -55,6 +58,7 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= @@ -68,8 +72,8 @@ github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9B github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg= -github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= +github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= @@ -78,8 +82,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e h1:ZU22z/2YRFLyf/P4ZwUYSdNCWsMEI0VeyrFoI2rAhJQ= -github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU= +github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= @@ -89,8 +93,8 @@ github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpz github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/ProtonMail/go-crypto v0.0.0-20220623141421-5afb4c282135 h1:xDc/cFH/hwyr9KyWc0sm26lpsscqtfZBvU8NpRLHwJ0= -github.com/ProtonMail/go-crypto v0.0.0-20220623141421-5afb4c282135/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/ProtonMail/go-crypto v0.0.0-20220730123233-d6ffb7692adf h1:aFFtnGZ6/2Qlvx80yxA2fFSYDQWTFjtKozQKB36A3/A= +github.com/ProtonMail/go-crypto v0.0.0-20220730123233-d6ffb7692adf/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= @@ -105,13 +109,13 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/anacrolix/dms v1.4.0 h1:RRD/P46p6cJKVGEoKnndNUV2ttwMMygpHsq38eQVMB8= -github.com/anacrolix/dms v1.4.0/go.mod h1:cpveZyOGE7M14mQKnd5T9AsUNQMRlWC0J7PgmGMBjmM= +github.com/anacrolix/dms v1.5.0 h1:2WWI++sNL3Jv1UtrlVzw2KvEcOO3yGX6LMR8UwMj6/Q= +github.com/anacrolix/dms v1.5.0/go.mod h1:5fAMpBcPFG4WQFh91zhf2E7/KYZ3/WmmRAf/WMoL0Q0= github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= github.com/anacrolix/ffprobe v1.0.0/go.mod h1:BIw+Bjol6CWjm/CRWrVLk2Vy+UYlkgmBZ05vpSYqZPw= -github.com/anacrolix/log v0.10.1-0.20220126091220-5c1b6f3af59c h1:ol0pMH2ony7/tMROtlycuy+WGkShsTFGud5GCUkw3yU= -github.com/anacrolix/log v0.10.1-0.20220126091220-5c1b6f3af59c/go.mod h1:D4+CvN8SnruK6zIFS/xPoRJmtvtnxs+CSfDQ+BFxZ68= +github.com/anacrolix/log v0.13.1 h1:BmVwTdxHd5VcNrLylgKwph4P4wf+5VvPgOK4yi91fTY= +github.com/anacrolix/log v0.13.1/go.mod h1:D4+CvN8SnruK6zIFS/xPoRJmtvtnxs+CSfDQ+BFxZ68= github.com/anacrolix/missinggo v1.1.0/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo= github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= @@ -120,8 +124,8 @@ github.com/artyom/mtab v1.0.0 h1:r7OSVo5Jeqi8+LotZ0rT2kzfPIBp9KCpEJP8RQqGmSE= github.com/artyom/mtab v1.0.0/go.mod h1:EHpkp5OmPfS1yZX+/DFTztlJ9di5UzdDLX1/XzWPXw8= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/aws/aws-sdk-go v1.44.29 h1:53YWlelsMiYmGxuTRpAq7Xp+pE+0esAVqNFiNyekU+A= -github.com/aws/aws-sdk-go v1.44.29/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.70 h1:wrwAbqJqf+ncEK1F/bXTYpgO6zXIgQXi/2ppBgmYI9g= +github.com/aws/aws-sdk-go v1.44.70/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -131,6 +135,7 @@ github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2w github.com/buengese/sgzip v0.1.1 h1:ry+T8l1mlmiWEsDrH/YHZnCVWD2S3im1KLsyO+8ZmTU= github.com/buengese/sgzip v0.1.1/go.mod h1:i5ZiXGF3fhV7gL1xaRRL1nDnmpNj0X061FQzOS8VMas= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo= github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -144,6 +149,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -160,8 +167,8 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -191,12 +198,12 @@ github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro= -github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= +github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= +github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= -github.com/gdamore/tcell/v2 v2.5.1 h1:zc3LPdpK184lBW7syF2a5C6MV827KmErk9jGVnmsl/I= -github.com/gdamore/tcell/v2 v2.5.1/go.mod h1:wSkrPaXoiIWZqW/g7Px4xc79di6FTcpB8tvaKJ6uGBo= +github.com/gdamore/tcell/v2 v2.5.2 h1:tKzG29kO9p2V++3oBY2W9zUjYu7IK1MENFeY/BzJSVY= +github.com/gdamore/tcell/v2 v2.5.2/go.mod h1:wSkrPaXoiIWZqW/g7Px4xc79di6FTcpB8tvaKJ6uGBo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -308,6 +315,9 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= @@ -319,6 +329,7 @@ github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/Oth github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk= github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -337,8 +348,9 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -354,18 +366,20 @@ github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFK github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= -github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA= github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= +github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE/Tq8= +github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= -github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af h1:sh8vAWJ+vr9izhkDAMS3JRGDIjj0tNVwxfwd+2U2xMo= -github.com/jlaffaye/ftp v0.0.0-20220524001917-dfa1e758f3af/go.mod h1:oZaomI+9/et52UBjvNU9LCIqmgt816+7ljXCx0EIPzo= +github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff h1:tN6UCYCBFNrPwvKf4RP9cIhGo6GcZ/IQTN8nqD7eCok= +github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -386,8 +400,8 @@ github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= -github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348 h1:Lrn8srO9JDBCf2iPjqy62stl49UDwoOxZ9/NGVi+fnk= @@ -478,8 +492,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.5-0.20211228200725-31aac3e1878d h1:7cHNeARnMq3icpbMdvyUELykWM4zOj5NRhH2Y3sfgBc= -github.com/pkg/sftp v1.13.5-0.20211228200725-31aac3e1878d/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= +github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go= +github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= github.com/pkg/xattr v0.4.7 h1:XoA3KzmFvyPlH4RwX5eMcgtzcaGBaSvgt3IoFQfbrmQ= github.com/pkg/xattr v0.4.7/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -529,8 +543,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil/v3 v3.22.5 h1:atX36I/IXgFiB81687vSiBI5zrMsxcIBkP9cQMJQoJA= -github.com/shirou/gopsutil/v3 v3.22.5/go.mod h1:so9G9VzeHt/hsd0YwqprnjHnfARAUktauykSbr+y2gA= +github.com/shirou/gopsutil/v3 v3.22.7 h1:flKnuCMfUUrO+oAvwAd6GKZgnPzr098VA/UJ14nhJd4= +github.com/shirou/gopsutil/v3 v3.22.7/go.mod h1:s648gW4IywYzUfE/KjXxUsqrqx/T2xO5VqOXxONeRfI= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -557,8 +571,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -573,13 +587,14 @@ github.com/spacemonkeygo/monkit/v3 v3.0.17 h1:rqIuLhRUr2UtS3WNVbPY/BwvjlwKVvSOVY github.com/spacemonkeygo/monkit/v3 v3.0.17/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4= github.com/spacemonkeygo/monotime v0.0.0-20180824235756-e3f48a95f98a/go.mod h1:ul4bvvnCOPZgq8w0nTkSmWVg/hauVpFS97Am1YM1XXo= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -589,10 +604,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8 h1:IGJQmLBLYBdAknj21W3JsVof0yjEXfy1Q0K3YZebDOg= -github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf h1:Y43S3e9P1NPs/QF4R5/SdlXj2d31540hP4Gk8VKNvDg= github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf/go.mod h1:c+cGNU1qi9bO7ZF4IRMYk+KaZTNiQ/gQrSbyMmGFq1Q= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= @@ -664,13 +677,12 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -699,8 +711,8 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd h1:x1GptNaTtxPAlTVIAJk61fuXg0y17h09DTxyb+VNC/k= -golang.org/x/mobile v0.0.0-20220518205345-8578da9835fd/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= +golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 h1:3vUV5x5+3LfQbgk7paCM6INOaJG9xXQbn79xoNkwfIk= +golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -756,19 +768,21 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= +golang.org/x/net v0.0.0-20220524220425-1d687d428aca/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 h1:N9Vc/rorQUDes6B9CNdIxAn5jODGj2wzfrei2x4wNj4= +golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -790,9 +804,10 @@ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g= golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c h1:q3gFqPqH7NVofKo3c3yETAP//pPI+G5mvB7qqj1Y5kY= +golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -805,8 +820,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -879,7 +895,6 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -892,13 +907,20 @@ golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220804214406-8e32c043e418 h1:9vYwv7OjYaky/tlAeD7C4oC9EsPTlaFl1H2jS++V+ME= +golang.org/x/sys v0.0.0-20220804214406-8e32c043e418/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -913,8 +935,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= -golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ= +golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -979,8 +1001,9 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= @@ -1021,8 +1044,10 @@ google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.83.0 h1:pMvST+6v+46Gabac4zlJlalxZjCeRcepwg2EdBU+nCc= -google.golang.org/api v0.83.0/go.mod h1:CNywQoj/AfhTw26ZWAa6LwOv+6WFxHmeLPZq2uncLZk= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.91.0 h1:731+JzuwaJoZXRQGmPoBiV+SrsAfUaIkdMCWTcQNPyA= +google.golang.org/api v0.91.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1076,6 +1101,7 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= @@ -1110,8 +1136,12 @@ google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 h1:qRu95HZ148xXw+XeZ3dvqe85PxH4X8+jIo0iRPKcEnM= -google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f h1:hJ/Y5SqPXbarffmAsApliUlcvMU+wScNGfyop4bZm8o= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= From 5a6d233924abd3796ec3d9d7fe3c4993fcc708a3 Mon Sep 17 00:00:00 2001 From: Joram Schrijver Date: Sat, 6 Aug 2022 02:59:00 +0200 Subject: [PATCH 232/560] dlna: fix SOAP action header parsing - fixes #6354 Changes in github.com/anacrolix/dms changed upnp.ServiceURN to include a namespace identifier. This identifier was previously hardcoded, but is now parsed out of the URN. The old SOAP action header parsing logic was duplicated in rclone and did not handle this field. Resulting responses included a URN with an empty namespace identifier, breaking clients. --- cmd/serve/dlna/dlna.go | 2 +- cmd/serve/dlna/dlna_test.go | 2 ++ cmd/serve/dlna/dlna_util.go | 34 ---------------------------------- 3 files changed, 3 insertions(+), 35 deletions(-) diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go index 0c9d2fbcaab51..fc802226c3e9d 100644 --- a/cmd/serve/dlna/dlna.go +++ b/cmd/serve/dlna/dlna.go @@ -186,7 +186,7 @@ func (s *server) rootDescHandler(w http.ResponseWriter, r *http.Request) { // Handle a service control HTTP request. func (s *server) serviceControlHandler(w http.ResponseWriter, r *http.Request) { soapActionString := r.Header.Get("SOAPACTION") - soapAction, err := parseActionHTTPHeader(soapActionString) + soapAction, err := upnp.ParseActionHTTPHeader(soapActionString) if err != nil { serveError(s, w, "Could not parse SOAPACTION header", err) return diff --git a/cmd/serve/dlna/dlna_test.go b/cmd/serve/dlna/dlna_test.go index acc1d0ffe8564..8327ee0545a97 100644 --- a/cmd/serve/dlna/dlna_test.go +++ b/cmd/serve/dlna/dlna_test.go @@ -119,6 +119,8 @@ func TestContentDirectoryBrowseMetadata(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) body, err := ioutil.ReadAll(resp.Body) require.NoError(t, err) + // should contain an appropriate URN + require.Contains(t, string(body), "urn:schemas-upnp-org:service:ContentDirectory:1") // expect a element require.Contains(t, string(body), html.EscapeString(" Date: Thu, 4 Aug 2022 16:31:16 +0200 Subject: [PATCH 233/560] serve sftp: document legacy code for checksum detection See #6351 --- cmd/serve/sftp/connection.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/serve/sftp/connection.go b/cmd/serve/sftp/connection.go index d9eaf855debb8..42564c151d8c7 100644 --- a/cmd/serve/sftp/connection.go +++ b/cmd/serve/sftp/connection.go @@ -135,7 +135,13 @@ func (c *conn) execCommand(ctx context.Context, out io.Writer, command string) ( return fmt.Errorf("send output failed: %w", err) } case "echo": - // special cases for rclone command detection + // Special cases for legacy rclone command detection. + // Before rclone v1.49.0 the sftp backend used "echo 'abc' | md5sum" when + // detecting hash support, but was then changed to instead just execute + // md5sum/sha1sum (without arguments), which is handled above. The following + // code is therefore only necessary to support rclone versions older than + // v1.49.0 using a sftp remote connected to a rclone serve sftp instance + // running a newer version of rclone (e.g. latest). switch args { case "'abc' | md5sum": if c.vfs.Fs().Hashes().Contains(hash.MD5) { From fe801b8fefd13fdba865abb52d6ac5218f74709e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 8 Aug 2022 19:01:03 +0100 Subject: [PATCH 234/560] Add Joram Schrijver to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 088bda7422ebf..fb8d03970e582 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -630,3 +630,4 @@ put them back in again.` >}} * Yen Hu <61753151+0x59656e@users.noreply.github.com> * Steve Kowalik * Jordi Gonzalez Muñoz + * Joram Schrijver From 9f33eb2e658aab4161712ee69f86feafd30b0887 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 8 Aug 2022 19:00:56 +0100 Subject: [PATCH 235/560] Changelog updates from Version 1.59.1 --- docs/content/changelog.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/content/changelog.md b/docs/content/changelog.md index 1cb4255b63fba..e4745b24dde50 100644 --- a/docs/content/changelog.md +++ b/docs/content/changelog.md @@ -5,6 +5,43 @@ description: "Rclone Changelog" # Changelog +## v1.59.1 - 2022-08-08 + +[See commits](https://github.com/rclone/rclone/compare/v1.59.0...v1.59.1) + +* Bug Fixes + * accounting: Fix panic in core/stats-reset with unknown group (Nick Craig-Wood) + * build: Fix android build after GitHub actions change (Nick Craig-Wood) + * dlna: Fix SOAP action header parsing (Joram Schrijver) + * docs: Fix links to mount command from install docs (albertony) + * dropox: Fix ChangeNotify was unable to decrypt errors (Nick Craig-Wood) + * fs: Fix parsing of times and durations of the form "YYYY-MM-DD HH:MM:SS" (Nick Craig-Wood) + * serve sftp: Fix checksum detection (Nick Craig-Wood) + * sync: Add accidentally missed filter-sensitivity to --backup-dir option (Nick Naumann) +* Combine + * Fix docs showing `remote=` instead of `upstreams=` (Nick Craig-Wood) + * Throw error if duplicate directory name is specified (Nick Craig-Wood) + * Fix errors with backends shutting down while in use (Nick Craig-Wood) +* Dropbox + * Fix hang on quit with --dropbox-batch-mode off (Nick Craig-Wood) + * Fix infinite loop on uploading a corrupted file (Nick Craig-Wood) +* Internetarchive + * Ignore checksums for files using the different method (Lesmiscore) + * Handle hash symbol in the middle of filename (Lesmiscore) +* Jottacloud + * Fix working with whitelabel Elgiganten Cloud + * Do not store username in config when using standard auth (albertony) +* Mega + * Fix nil pointer exception when bad node received (Nick Craig-Wood) +* S3 + * Fix --s3-no-head panic: reflect: Elem of invalid type s3.PutObjectInput (Nick Craig-Wood) +* SFTP + * Fix issue with WS_FTP by working around failing RealPath (albertony) +* Union + * Fix duplicated files when using directories with leading / (Nick Craig-Wood) + * Fix multiple files being uploaded when roots don't exist (Nick Craig-Wood) + * Fix panic due to misalignment of struct field in 32 bit architectures (r-ricci) + ## v1.59.0 - 2022-07-09 [See commits](https://github.com/rclone/rclone/compare/v1.58.0...v1.59.0) From d347ac0154a824c25dcd8c4dd982be7703a2495d Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 8 Aug 2022 10:03:25 +0100 Subject: [PATCH 236/560] local: disable xattr support if the filesystems indicates it is not supported Before this change, if rclone was run with `-M` on a filesystem without xattr support, it would error out. This patch makes rclone detect the not supported errors and disable xattrs from then on. It prints one ERROR level message about this. See: https://forum.rclone.org/t/metadata-update-local-s3/32277/7 --- backend/local/local.go | 22 +++++++++++++--------- backend/local/xattr.go | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/backend/local/local.go b/backend/local/local.go index c625b426042b4..7f08be673173c 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -234,15 +234,16 @@ type Options struct { // Fs represents a local filesystem rooted at root type Fs struct { - name string // the name of the remote - root string // The root directory (OS path) - opt Options // parsed config options - features *fs.Features // optional features - dev uint64 // device number of root node - precisionOk sync.Once // Whether we need to read the precision - precision time.Duration // precision of local filesystem - warnedMu sync.Mutex // used for locking access to 'warned'. - warned map[string]struct{} // whether we have warned about this string + name string // the name of the remote + root string // The root directory (OS path) + opt Options // parsed config options + features *fs.Features // optional features + dev uint64 // device number of root node + precisionOk sync.Once // Whether we need to read the precision + precision time.Duration // precision of local filesystem + warnedMu sync.Mutex // used for locking access to 'warned'. + warned map[string]struct{} // whether we have warned about this string + xattrSupported int32 // whether xattrs are supported (atomic access) // do os.Lstat or os.Stat lstat func(name string) (os.FileInfo, error) @@ -286,6 +287,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e dev: devUnset, lstat: os.Lstat, } + if xattrSupported { + f.xattrSupported = 1 + } f.root = cleanRootPath(root, f.opt.NoUNC, f.opt.Enc) f.features = (&fs.Features{ CaseInsensitive: f.caseInsensitive(), diff --git a/backend/local/xattr.go b/backend/local/xattr.go index a2c0cc5e868e7..81e86d8a569f0 100644 --- a/backend/local/xattr.go +++ b/backend/local/xattr.go @@ -6,6 +6,8 @@ package local import ( "fmt" "strings" + "sync/atomic" + "syscall" "github.com/pkg/xattr" "github.com/rclone/rclone/fs" @@ -16,12 +18,30 @@ const ( xattrSupported = xattr.XATTR_SUPPORTED ) +// Check to see if the error supplied is a not supported error, and if +// so, disable xattrs +func (f *Fs) xattrIsNotSupported(err error) bool { + xattrErr, ok := err.(*xattr.Error) + if !ok { + return false + } + // Xattrs not supported can be ENOTSUP or ENOATTR or EINVAL (on Solaris) + if xattrErr.Err == syscall.EINVAL || xattrErr.Err == syscall.ENOTSUP || xattrErr.Err == xattr.ENOATTR { + // Show xattrs not supported + if atomic.CompareAndSwapInt32(&f.xattrSupported, 1, 0) { + fs.Errorf(f, "xattrs not supported - disabling: %v", err) + } + return true + } + return false +} + // getXattr returns the extended attributes for an object // // It doesn't return any attributes owned by this backend in // metadataKeys func (o *Object) getXattr() (metadata fs.Metadata, err error) { - if !xattrSupported { + if !xattrSupported || atomic.LoadInt32(&o.fs.xattrSupported) == 0 { return nil, nil } var list []string @@ -31,6 +51,9 @@ func (o *Object) getXattr() (metadata fs.Metadata, err error) { list, err = xattr.LList(o.path) } if err != nil { + if o.fs.xattrIsNotSupported(err) { + return nil, nil + } return nil, fmt.Errorf("failed to read xattr: %w", err) } if len(list) == 0 { @@ -45,6 +68,9 @@ func (o *Object) getXattr() (metadata fs.Metadata, err error) { v, err = xattr.LGet(o.path, k) } if err != nil { + if o.fs.xattrIsNotSupported(err) { + return nil, nil + } return nil, fmt.Errorf("failed to read xattr key %q: %w", k, err) } k = strings.ToLower(k) @@ -64,7 +90,7 @@ func (o *Object) getXattr() (metadata fs.Metadata, err error) { // // It doesn't set any attributes owned by this backend in metadataKeys func (o *Object) setXattr(metadata fs.Metadata) (err error) { - if !xattrSupported { + if !xattrSupported || atomic.LoadInt32(&o.fs.xattrSupported) == 0 { return nil } for k, value := range metadata { @@ -80,6 +106,9 @@ func (o *Object) setXattr(metadata fs.Metadata) (err error) { err = xattr.LSet(o.path, k, v) } if err != nil { + if o.fs.xattrIsNotSupported(err) { + return nil + } return fmt.Errorf("failed to set xattr key %q: %w", k, err) } } From 5439a2c5c67b6e8bbd21210ad232a70d0b0db74e Mon Sep 17 00:00:00 2001 From: Mark Trolley Date: Mon, 8 Aug 2022 19:08:07 -0400 Subject: [PATCH 237/560] docs: fix script installation command on downloads page Script installation instructions in `downloads.md` differ from those in `install.md` and fail on MacOS. --- docs/content/downloads.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/downloads.md b/docs/content/downloads.md index 682937b448d69..f365dc9e56751 100644 --- a/docs/content/downloads.md +++ b/docs/content/downloads.md @@ -28,11 +28,11 @@ You can also find a [mirror of the downloads on GitHub](https://github.com/rclon To install rclone on Linux/macOS/BSD systems, run: - curl https://rclone.org/install.sh | sudo bash + sudo -v ; curl https://rclone.org/install.sh | sudo bash For beta installation, run: - curl https://rclone.org/install.sh | sudo bash -s beta + sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta Note that this script checks the version of rclone installed first and won't re-download if not needed. From cb8842941b8ebca7f627320ad40c4b013df3c5aa Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 9 Aug 2022 12:57:29 +0100 Subject: [PATCH 238/560] Add Mark Trolley to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index fb8d03970e582..853b2f4dd199f 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -631,3 +631,4 @@ put them back in again.` >}} * Steve Kowalik * Jordi Gonzalez Muñoz * Joram Schrijver + * Mark Trolley From 0501773db13221171e102781809378a6555dd312 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 9 Aug 2022 10:44:54 +0100 Subject: [PATCH 239/560] azureblob,b2,s3: fix chunksize calculations producing too many parts Before this fix, the chunksize calculator was using the previous size of the object, not the new size of the object to calculate the chunk sizes. This meant that uploading a replacement object which needed a new chunk size would fail, using too many parts. This fix fixes the calculator to take the size explicitly. --- backend/azureblob/azureblob.go | 4 +- backend/b2/upload.go | 2 +- backend/s3/s3.go | 6 +- fs/chunksize/chunksize.go | 32 ++++++---- fs/chunksize/chunksize_test.go | 104 +++++++++++++++++++++++++++------ 5 files changed, 111 insertions(+), 37 deletions(-) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 0814b0168041d..edb8630708886 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -1678,14 +1678,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } } - uploadParts := int64(maxUploadParts) + uploadParts := maxUploadParts if uploadParts < 1 { uploadParts = 1 } else if uploadParts > maxUploadParts { uploadParts = maxUploadParts } // calculate size of parts/blocks - partSize := chunksize.Calculator(o, int(uploadParts), o.fs.opt.ChunkSize) + partSize := chunksize.Calculator(o, src.Size(), uploadParts, o.fs.opt.ChunkSize) putBlobOptions := azblob.UploadStreamToBlockBlobOptions{ BufferSize: int(partSize), diff --git a/backend/b2/upload.go b/backend/b2/upload.go index 8a1f676354887..47ba53473e078 100644 --- a/backend/b2/upload.go +++ b/backend/b2/upload.go @@ -97,7 +97,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs if size == -1 { fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize) } else { - chunkSize = chunksize.Calculator(src, maxParts, defaultChunkSize) + chunkSize = chunksize.Calculator(o, size, maxParts, defaultChunkSize) parts = size / int64(chunkSize) if size%int64(chunkSize) != 0 { parts++ diff --git a/backend/s3/s3.go b/backend/s3/s3.go index f0a02b391abcc..1f86b859297fd 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2116,7 +2116,7 @@ type Options struct { UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` CopyCutoff fs.SizeSuffix `config:"copy_cutoff"` ChunkSize fs.SizeSuffix `config:"chunk_size"` - MaxUploadParts int64 `config:"max_upload_parts"` + MaxUploadParts int `config:"max_upload_parts"` DisableChecksum bool `config:"disable_checksum"` SharedCredentialsFile string `config:"shared_credentials_file"` Profile string `config:"profile"` @@ -4718,10 +4718,10 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si if size == -1 { warnStreamUpload.Do(func() { fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v", - f.opt.ChunkSize, fs.SizeSuffix(int64(partSize)*uploadParts)) + f.opt.ChunkSize, fs.SizeSuffix(int64(partSize)*int64(uploadParts))) }) } else { - partSize = chunksize.Calculator(o, int(uploadParts), f.opt.ChunkSize) + partSize = chunksize.Calculator(o, size, uploadParts, f.opt.ChunkSize) } memPool := f.getMemoryPool(int64(partSize)) diff --git a/fs/chunksize/chunksize.go b/fs/chunksize/chunksize.go index 5ecf1d5f38ccc..1a60dda5f51cc 100644 --- a/fs/chunksize/chunksize.go +++ b/fs/chunksize/chunksize.go @@ -5,18 +5,26 @@ import ( "github.com/rclone/rclone/fs" ) -/* -Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi +// Calculator calculates the minimum chunk size needed to fit within +// the maximum number of parts, rounded up to the nearest fs.Mebi. +// +// For most backends, (chunk_size) * (concurrent_upload_routines) +// memory will be required so we want to use the smallest possible +// chunk size that's going to allow the upload to proceed. Rounding up +// to the nearest fs.Mebi on the assumption that some backends may +// only allow integer type parameters when specifying the chunk size. +// +// Returns the default chunk size if it is sufficiently large enough +// to support the given file size otherwise returns the smallest chunk +// size necessary to allow the upload to proceed. +func Calculator(o interface{}, size int64, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix { + // If streaming then use default chunk size + if size < 0 { + fs.Debugf(o, "Streaming upload with chunk_size %s allows uploads of up to %s and will fail only when that limit is reached.", defaultChunkSize, fs.SizeSuffix(maxParts)*defaultChunkSize) -For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest -possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption -that some backends may only allow integer type parameters when specifying the chunk size. - -Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the -smallest chunk size necessary to allow the upload to proceed. -*/ -func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix { - fileSize := fs.SizeSuffix(objInfo.Size()) + return defaultChunkSize + } + fileSize := fs.SizeSuffix(size) requiredChunks := fileSize / defaultChunkSize if requiredChunks < fs.SizeSuffix(maxParts) || (requiredChunks == fs.SizeSuffix(maxParts) && fileSize%defaultChunkSize == 0) { return defaultChunkSize @@ -31,6 +39,6 @@ func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuf minChunk += fs.Mebi } - fs.Debugf(objInfo, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk) + fs.Debugf(o, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk) return minChunk } diff --git a/fs/chunksize/chunksize_test.go b/fs/chunksize/chunksize_test.go index 9cdeb70d6b69b..a1a778f94d29f 100644 --- a/fs/chunksize/chunksize_test.go +++ b/fs/chunksize/chunksize_test.go @@ -2,34 +2,100 @@ package chunksize import ( "testing" - "time" "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/object" ) func TestComputeChunkSize(t *testing.T) { - tests := map[string]struct { - fileSize fs.SizeSuffix + for _, test := range []struct { + name string + size fs.SizeSuffix maxParts int defaultChunkSize fs.SizeSuffix - expected fs.SizeSuffix + want fs.SizeSuffix }{ - "default size returned when file size is small enough": {fileSize: 1000, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)}, - "default size returned when file size is just 1 byte small enough": {fileSize: toSizeSuffixMiB(100000) - 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)}, - "no rounding up when everything divides evenly": {fileSize: toSizeSuffixMiB(1000000), maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(100)}, - "rounding up to nearest MiB when not quite enough parts": {fileSize: toSizeSuffixMiB(1000000), maxParts: 9999, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)}, - "rounding up to nearest MiB when one extra byte": {fileSize: toSizeSuffixMiB(1000000) + 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)}, - "expected MiB value when rounding sets to absolute minimum": {fileSize: toSizeSuffixMiB(1) - 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(1)}, - "expected MiB value when rounding to absolute min with extra": {fileSize: toSizeSuffixMiB(1) + 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(2)}, - } + { + name: "streaming file", + size: -1, + maxParts: 10000, + defaultChunkSize: toSizeSuffixMiB(10), + want: toSizeSuffixMiB(10), + }, { + name: "default size returned when file size is small enough", + size: 1000, + maxParts: 10000, + defaultChunkSize: toSizeSuffixMiB(10), + want: toSizeSuffixMiB(10), + }, { + name: "default size returned when file size is just 1 byte small enough", + size: toSizeSuffixMiB(100000) - 1, + maxParts: 10000, + defaultChunkSize: toSizeSuffixMiB(10), + want: toSizeSuffixMiB(10), + }, { + name: "no rounding up when everything divides evenly", + size: toSizeSuffixMiB(1000000), + maxParts: 10000, + defaultChunkSize: toSizeSuffixMiB(100), + want: toSizeSuffixMiB(100), + }, { + name: "rounding up to nearest MiB when not quite enough parts", + size: toSizeSuffixMiB(1000000), + maxParts: 9999, + defaultChunkSize: toSizeSuffixMiB(100), + want: toSizeSuffixMiB(101), + }, { + name: "rounding up to nearest MiB when one extra byte", + size: toSizeSuffixMiB(1000000) + 1, + maxParts: 10000, + defaultChunkSize: toSizeSuffixMiB(100), + want: toSizeSuffixMiB(101), + }, { + name: "expected MiB value when rounding sets to absolute minimum", + size: toSizeSuffixMiB(1) - 1, + maxParts: 1, + defaultChunkSize: toSizeSuffixMiB(1), + want: toSizeSuffixMiB(1), + }, { + name: "expected MiB value when rounding to absolute min with extra", + size: toSizeSuffixMiB(1) + 1, + maxParts: 1, + defaultChunkSize: toSizeSuffixMiB(1), + want: toSizeSuffixMiB(2), + }, { + name: "issue from forum #1", + size: 120864818840, + maxParts: 10000, + defaultChunkSize: 5 * 1024 * 1024, + want: toSizeSuffixMiB(12), + }, + } { + t.Run(test.name, func(t *testing.T) { + got := Calculator(test.name, int64(test.size), test.maxParts, test.defaultChunkSize) + if got != test.want { + t.Fatalf("expected: %v, got: %v", test.want, got) + } + if test.size < 0 { + return + } + parts := func(result fs.SizeSuffix) int { + n := test.size / result + r := test.size % result + if r != 0 { + n++ + } + return int(n) + } + // Check this gives the parts in range + if parts(got) > test.maxParts { + t.Fatalf("too many parts %d", parts(got)) + } + // Check that setting chunk size smaller gave too many parts + if got > test.defaultChunkSize { + if parts(got-toSizeSuffixMiB(1)) <= test.maxParts { + t.Fatalf("chunk size %v too big as %v only gives %d parts", got, got-toSizeSuffixMiB(1), parts(got-toSizeSuffixMiB(1))) + } - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - src := object.NewStaticObjectInfo("mock", time.Now(), int64(tc.fileSize), true, nil, nil) - result := Calculator(src, tc.maxParts, tc.defaultChunkSize) - if result != tc.expected { - t.Fatalf("expected: %v, got: %v", tc.expected, result) } }) } From 1ad22b8881ceaa6219f0bc1ae67507c74f49bdb2 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 9 Aug 2022 11:15:04 +0100 Subject: [PATCH 240/560] gcs: add --gcs-endpoint flag and config parameter See: https://forum.rclone.org/t/how-to-modify-google-cloud-storage-endpoint-uri/32342 --- backend/googlecloudstorage/googlecloudstorage.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index 0e6a2b4ed7136..5ce96e672a46d 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -319,6 +319,10 @@ can't check the size and hash but the file contents will be decompressed. `, Advanced: true, Default: false, + }, { + Name: "endpoint", + Help: "Endpoint for the service.\n\nLeave blank normally.", + Advanced: true, }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, @@ -343,6 +347,7 @@ type Options struct { StorageClass string `config:"storage_class"` NoCheckBucket bool `config:"no_check_bucket"` Decompress bool `config:"decompress"` + Endpoint string `config:"endpoint"` Enc encoder.MultiEncoder `config:"encoding"` } @@ -523,7 +528,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e // Create a new authorized Drive client. f.client = oAuthClient - f.svc, err = storage.NewService(context.Background(), option.WithHTTPClient(f.client)) + gcsOpts := []option.ClientOption{option.WithHTTPClient(f.client)} + if opt.Endpoint != "" { + gcsOpts = append(gcsOpts, option.WithEndpoint(opt.Endpoint)) + } + f.svc, err = storage.NewService(context.Background(), gcsOpts...) if err != nil { return nil, fmt.Errorf("couldn't create Google Cloud Storage client: %w", err) } From 2c5923ab1a76e321ed752377c54e73ced2ae3c4c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 11 Aug 2022 12:20:17 +0100 Subject: [PATCH 241/560] filter: make sure we check --files-from when looking for a single file --- fs/filter/filter.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fs/filter/filter.go b/fs/filter/filter.go index 6f4ba9d5ad864..997070bd79397 100644 --- a/fs/filter/filter.go +++ b/fs/filter/filter.go @@ -375,6 +375,11 @@ func (f *Filter) InActive() bool { // IncludeRemote returns whether this remote passes the filter rules. func (f *Filter) IncludeRemote(remote string) bool { + // filesFrom takes precedence + if f.files != nil { + _, include := f.files[remote] + return include + } for _, rule := range f.fileRules.rules { if rule.Match(remote) { return rule.Include From 8d1fff9a8237c64f73cc02e273921efde1553487 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 11 Aug 2022 12:20:34 +0100 Subject: [PATCH 242/560] local: obey file filters in listing to fix errors on excluded files Fixes #6376 --- backend/local/local.go | 15 +++++++++++++ backend/local/local_internal_test.go | 32 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/backend/local/local.go b/backend/local/local.go index 7f08be673173c..18763ace3c860 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -22,6 +22,7 @@ import ( "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/encoder" @@ -443,6 +444,8 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { // This should return ErrDirNotFound if the directory isn't // found. func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + filter, useFilter := filter.GetConfig(ctx), filter.GetUseFilter(ctx) + fsDirPath := f.localPath(dir) _, err = os.Stat(fsDirPath) if err != nil { @@ -493,6 +496,13 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e continue } if fierr != nil { + // Don't report errors on any file names that are excluded + if useFilter { + newRemote := f.cleanRemote(dir, name) + if !filter.IncludeRemote(newRemote) { + continue + } + } err = fmt.Errorf("failed to read directory %q: %w", namepath, err) fs.Errorf(dir, "%v", fierr) _ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync @@ -510,6 +520,11 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e name := fi.Name() mode := fi.Mode() newRemote := f.cleanRemote(dir, name) + // Don't include non directory if not included + // we leave directory filtering to the layer above + if useFilter && !fi.IsDir() && !filter.IncludeRemote(newRemote) { + continue + } // Follow symlinks if required if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 { localPath := filepath.Join(fsDirPath, name) diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go index 55b98bb53728b..698d263a88415 100644 --- a/backend/local/local_internal_test.go +++ b/backend/local/local_internal_test.go @@ -9,11 +9,13 @@ import ( "path" "path/filepath" "runtime" + "sort" "testing" "time" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/object" "github.com/rclone/rclone/fstest" @@ -366,3 +368,33 @@ func TestMetadata(t *testing.T) { }) } + +func TestFilter(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + defer r.Finalise() + when := time.Now() + r.WriteFile("included", "included file", when) + r.WriteFile("excluded", "excluded file", when) + f := r.Flocal.(*Fs) + + // Add a filter + ctx, fi := filter.AddConfig(ctx) + require.NoError(t, fi.AddRule("+ included")) + require.NoError(t, fi.AddRule("- *")) + + // Check listing without use filter flag + entries, err := f.List(ctx, "") + require.NoError(t, err) + sort.Sort(entries) + require.Equal(t, "[excluded included]", fmt.Sprint(entries)) + + // Add user filter flag + ctx = filter.SetUseFilter(ctx, true) + + // Check listing with use filter flag + entries, err = f.List(ctx, "") + require.NoError(t, err) + sort.Sort(entries) + require.Equal(t, "[included]", fmt.Sprint(entries)) +} From 1107da7247db32199fde92f42c916bcb4c60dc3f Mon Sep 17 00:00:00 2001 From: Simon Bos Date: Mon, 8 Aug 2022 18:48:36 +0200 Subject: [PATCH 243/560] dlna: specify SSDP interface names from command line --- cmd/serve/dlna/dlna.go | 26 ++++++++++++++++++++++---- cmd/serve/dlna/dlna_test.go | 4 +++- cmd/serve/dlna/dlna_util.go | 6 +++++- cmd/serve/dlna/dlnaflags/dlnaflags.go | 15 +++++++++------ 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go index fc802226c3e9d..da6a08a19ce77 100644 --- a/cmd/serve/dlna/dlna.go +++ b/cmd/serve/dlna/dlna.go @@ -51,7 +51,10 @@ files that they are not able to play back correctly. f := cmd.NewFsSrc(args) cmd.Run(false, false, command, func() error { - s := newServer(f, &dlnaflags.Opt) + s, err := newServer(f, &dlnaflags.Opt) + if err != nil { + return err + } if err := s.Serve(); err != nil { return err } @@ -92,17 +95,32 @@ type server struct { vfs *vfs.VFS } -func newServer(f fs.Fs, opt *dlnaflags.Options) *server { +func newServer(f fs.Fs, opt *dlnaflags.Options) (*server, error) { friendlyName := opt.FriendlyName if friendlyName == "" { friendlyName = makeDefaultFriendlyName() } + interfaces := make([]net.Interface, 0, len(opt.InterfaceNames)) + for _, interfaceName := range opt.InterfaceNames { + var err error + intf, err := net.InterfaceByName(interfaceName) + if err != nil { + return nil, fmt.Errorf("failed to resolve interface name '%s': %w", interfaceName, err) + } + if !isAppropriatelyConfigured(*intf) { + return nil, fmt.Errorf("interface '%s' is not appropriately configured (it should be UP, MULTICAST and MTU > 0)", interfaceName) + } + interfaces = append(interfaces, *intf) + } + if len(interfaces) == 0 { + interfaces = listInterfaces() + } s := &server{ AnnounceInterval: 10 * time.Second, FriendlyName: friendlyName, RootDeviceUUID: makeDeviceUUID(friendlyName), - Interfaces: listInterfaces(), + Interfaces: interfaces, httpListenAddr: opt.ListenAddr, @@ -138,7 +156,7 @@ func newServer(f fs.Fs, opt *dlnaflags.Options) *server { http.FileServer(data.Assets)))) s.handler = logging(withHeader("Server", serverField, r)) - return s + return s, nil } // UPnPService is the interface for the SOAP service. diff --git a/cmd/serve/dlna/dlna_test.go b/cmd/serve/dlna/dlna_test.go index 8327ee0545a97..3a3d2fa4be417 100644 --- a/cmd/serve/dlna/dlna_test.go +++ b/cmd/serve/dlna/dlna_test.go @@ -35,7 +35,9 @@ const ( func startServer(t *testing.T, f fs.Fs) { opt := dlnaflags.DefaultOpt opt.ListenAddr = testBindAddress - dlnaServer = newServer(f, &opt) + var err error + dlnaServer, err = newServer(f, &opt) + assert.NoError(t, err) assert.NoError(t, dlnaServer.Serve()) baseURL = "http://" + dlnaServer.HTTPConn.Addr().String() } diff --git a/cmd/serve/dlna/dlna_util.go b/cmd/serve/dlna/dlna_util.go index ad961e067e137..d54dfea5b942f 100644 --- a/cmd/serve/dlna/dlna_util.go +++ b/cmd/serve/dlna/dlna_util.go @@ -47,13 +47,17 @@ func listInterfaces() []net.Interface { var active []net.Interface for _, intf := range ifs { - if intf.Flags&net.FlagUp != 0 && intf.Flags&net.FlagMulticast != 0 && intf.MTU > 0 { + if isAppropriatelyConfigured(intf) { active = append(active, intf) } } return active } +func isAppropriatelyConfigured(intf net.Interface) bool { + return intf.Flags&net.FlagUp != 0 && intf.Flags&net.FlagMulticast != 0 && intf.MTU > 0 +} + func didlLite(chardata string) string { return ` Date: Thu, 18 Aug 2022 15:14:26 -0400 Subject: [PATCH 244/560] docs: fix typo in filter pattern example --- docs/content/filtering.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/filtering.md b/docs/content/filtering.md index 0b7d3d99c49e6..cf526961cf991 100644 --- a/docs/content/filtering.md +++ b/docs/content/filtering.md @@ -177,7 +177,7 @@ them into regular expressions. | Rooted | `/*.jpg` | `/file.jpg` | `/file.png` | | | | `/file2.jpg` | `/dir/file.jpg` | | Alternates | `*.{jpg,png}` | `/file.jpg` | `/file.gif` | -| | | `/dir/file.gif` | `/dir/file.gif` | +| | | `/dir/file.png` | `/dir/file.gif` | | Path Wildcard | `dir/**` | `/dir/anyfile` | `file.png` | | | | `/subdir/dir/subsubdir/anyfile` | `/subdir/file.png` | | Any Char | `*.t?t` | `/file.txt` | `/file.qxt` | From 47539ec0e6bba833cf178dec8f3a8f16684ad598 Mon Sep 17 00:00:00 2001 From: anonion Date: Sun, 21 Aug 2022 14:09:33 -0600 Subject: [PATCH 245/560] docs: fix minor typo in onedrive docs --- docs/content/onedrive.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index 13242910ff8c9..b7a40ba787fd7 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -124,7 +124,7 @@ rclone uses a default Client ID when talking to OneDrive, unless a custom `clien The default Client ID and Key are shared by all rclone users when performing requests. You may choose to create and use your own Client ID, in case the default one does not work well for you. -For example, you might see throtting. +For example, you might see throttling. #### Creating Client ID for OneDrive Personal From 85eb9776bd5b700e0be0cdd23dab280f0d2d946d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Henrique=20Franco?= Date: Mon, 22 Aug 2022 05:43:54 -0300 Subject: [PATCH 246/560] crypt: fix typo in comment strign -> string --- backend/crypt/cipher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/crypt/cipher.go b/backend/crypt/cipher.go index 10485776b0e3e..ae1e623936dec 100644 --- a/backend/crypt/cipher.go +++ b/backend/crypt/cipher.go @@ -131,7 +131,7 @@ type fileNameEncoding interface { // - we strip the padding character `=` type caseInsensitiveBase32Encoding struct{} -// EncodeToString encodes a strign using the modified version of +// EncodeToString encodes a string using the modified version of // base32 encoding. func (caseInsensitiveBase32Encoding) EncodeToString(src []byte) string { encoded := base32.HexEncoding.EncodeToString(src) From 8bf2d6b6c819b5533330b9a7cc5ff4ab93c0d3dd Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 28 Aug 2022 10:16:37 +0200 Subject: [PATCH 247/560] =?UTF-8?q?Add=20Jo=C3=A3o=20Henrique=20Franco=20t?= =?UTF-8?q?o=20contributors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 853b2f4dd199f..c5520d86fbba1 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -632,3 +632,4 @@ put them back in again.` >}} * Jordi Gonzalez Muñoz * Joram Schrijver * Mark Trolley + * João Henrique Franco \ No newline at end of file From 5cc7797f9eee2e23c94096130a4457d4e92bb8b7 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 28 Aug 2022 10:18:14 +0200 Subject: [PATCH 248/560] Add anonion to contributors --- docs/content/authors.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/content/authors.md b/docs/content/authors.md index c5520d86fbba1..92d88fcfb94d7 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -632,4 +632,5 @@ put them back in again.` >}} * Jordi Gonzalez Muñoz * Joram Schrijver * Mark Trolley - * João Henrique Franco \ No newline at end of file + * João Henrique Franco + * anonion \ No newline at end of file From 21fd13f10df1273085a0e27ec3c6c18053223088 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 28 Aug 2022 10:18:33 +0200 Subject: [PATCH 249/560] Add Ryan Morey to contributors --- docs/content/authors.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/content/authors.md b/docs/content/authors.md index 92d88fcfb94d7..9b84a6b9752bd 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -633,4 +633,5 @@ put them back in again.` >}} * Joram Schrijver * Mark Trolley * João Henrique Franco - * anonion \ No newline at end of file + * anonion + * Ryan Morey <4590343+rmorey@users.noreply.github.com> \ No newline at end of file From 8a6857c295af68f929a94b616ce3299b19ae7765 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 28 Aug 2022 10:19:39 +0200 Subject: [PATCH 250/560] Add Simon Bos to contributors --- docs/content/authors.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/content/authors.md b/docs/content/authors.md index 9b84a6b9752bd..ea9e4ce9843a0 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -634,4 +634,5 @@ put them back in again.` >}} * Mark Trolley * João Henrique Franco * anonion - * Ryan Morey <4590343+rmorey@users.noreply.github.com> \ No newline at end of file + * Ryan Morey <4590343+rmorey@users.noreply.github.com> + * Simon Bos \ No newline at end of file From b342c6cf9c5668270e546cf0324c8d7fa71d7f67 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sat, 27 Aug 2022 18:30:55 +0200 Subject: [PATCH 251/560] docs/ftp: improve documentation of anonymous ftp - fixes #5650 --- docs/content/ftp.md | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/docs/content/ftp.md b/docs/content/ftp.md index 617ca38d76272..7e95a0a94d5d1 100644 --- a/docs/content/ftp.md +++ b/docs/content/ftp.md @@ -23,8 +23,7 @@ To create an FTP configuration named `remote`, run Rclone config guides you through an interactive setup process. A minimal rclone FTP remote definition only requires host, username and password. -For an anonymous FTP server, use `anonymous` as username and your email -address as password. +For an anonymous FTP server, see [below](#anonymous-ftp). ``` No remotes found, make a new one? @@ -101,11 +100,33 @@ excess files in the directory. rclone sync -i /home/local/directory remote:directory -### Example without a config file ### +### Anonymous FTP + +When connecting to a FTP server that allows anonymous login, you can use the +special "anonymous" username. Traditionally, this user account accepts any +string as a password, although it is common to use either the password +"anonymous" or "guest". Some servers require the use of a valid e-mail +address as password. + +Using [on-the-fly](#backend-path-to-dir) or +[connection string](/docs/#connection-strings) remotes makes it easy to access +such servers, without requiring any configuration in advance. The following +are examples of that: + + rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=$(rclone obscure dummy) + rclone lsf :ftp,host=speedtest.tele2.net,user=anonymous,pass=$(rclone obscure dummy): + +The above examples work in Linux shells and in PowerShell, but not Windows +Command Prompt. They execute the [rclone obscure](/commands/rclone_obscure/) +command to create a password string in the format required by the +[pass](#ftp-pass) option. The following examples are exactly the same, except use +an already obscured string representation of the same password "dummy", and +therefore works even in Windows Command Prompt: - rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=`rclone obscure dummy` + rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=IXs2wc8OJOz7SYLBk47Ji1rHTmxM + rclone lsf :ftp,host=speedtest.tele2.net,user=anonymous,pass=IXs2wc8OJOz7SYLBk47Ji1rHTmxM: -### Implicit TLS ### +### Implicit TLS Rlone FTP supports implicit FTP over TLS servers (FTPS). This has to be enabled in the FTP backend config for the remote, or with From 02b761310477959593a9eff437cbe422c87ecf24 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 24 Aug 2022 09:19:17 +0200 Subject: [PATCH 252/560] docs/jottacloud: improve description of the standard authentication --- docs/content/jottacloud.md | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/content/jottacloud.md b/docs/content/jottacloud.md index 7d815e8c4b0af..87b1e9e284916 100644 --- a/docs/content/jottacloud.md +++ b/docs/content/jottacloud.md @@ -34,10 +34,33 @@ and you have to choose the correct one when setting up the remote. ### Standard authentication -To configure Jottacloud you will need to generate a personal security token in the Jottacloud web interface. -You will the option to do in your [account security settings](https://www.jottacloud.com/web/secure) -(for whitelabel version you need to find this page in its web interface). -Note that the web interface may refer to this token as a JottaCli token. +The standard authentication method used by the official service (jottacloud.com), as well as +some of the whitelabel services, requires you to generate a single-use personal login token +from the account security settings in the service's web interface. Log in to your account, +go to "Settings" and then "Security", or use the direct link presented to you by rclone when +configuring the remote: . Scroll down to the section +"Personal login token", and click the "Generate" button. Note that if you are using a +whitelabel service you probably can't use the direct link, you need to find the same page in +their dedicated web interface, and also it may be in a different location than described above. + +To access your account from multiple instances of rclone, you need to configure each of them +with a separate personal login token. E.g. you create a Jottacloud remote with rclone in one +location, and copy the configuration file to a second location where you also want to run +rclone and access the same remote. Then you need to replace the token for one of them, using +the [config reconnect](https://rclone.org/commands/rclone_config_reconnect/) command, which +requires you to generate a new personal login token and supply as input. If you do not +do this, the token may easily end up being invalidated, resulting in both instances failing +with an error message something along the lines of: + + oauth2: cannot fetch token: 400 Bad Request + Response: {"error":"invalid_grant","error_description":"Stale token"} + +When this happens, you need to replace the token as described above to be able to use your +remote again. + +All personal login tokens you have taken into use will be listed in the web interface under +"My logged in devices", and from the right side of that list you can click the "X" button to +revoke individual tokens. ### Legacy authentication From 555def2da7f425225b9f8657593733b5d71f901e Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 28 Aug 2022 13:21:57 +0200 Subject: [PATCH 253/560] build: add package comments to silence revive linter --- backend/alias/alias.go | 1 + backend/all/all.go | 1 + backend/azureblob/azureblob.go | 3 +-- backend/b2/api/types.go | 1 + backend/b2/b2.go | 2 +- backend/cache/cache.go | 1 + backend/fichier/fichier.go | 1 + backend/googlephotos/api/types.go | 1 + backend/hdfs/hdfs.go | 1 + .../hidrive/hidrivehash/internal/internal.go | 1 + backend/jottacloud/api/types.go | 1 + backend/jottacloud/jottacloud.go | 1 + backend/koofr/koofr.go | 1 + backend/mailru/api/m1.go | 1 + backend/mailru/mailru.go | 1 + backend/onedrive/api/types.go | 3 +-- backend/opendrive/opendrive.go | 1 + backend/putio/putio.go | 1 + backend/qingstor/qingstor.go | 5 ++--- backend/seafile/api/types.go | 1 + backend/seafile/seafile.go | 1 + backend/sftp/sftp.go | 3 +-- backend/sia/api/types.go | 1 + backend/sia/sia.go | 1 + backend/union/policy/all.go | 1 + backend/union/union.go | 1 + backend/union/upstream/upstream.go | 1 + backend/uptobox/api/types.go | 1 + backend/uptobox/uptobox.go | 1 + backend/yandex/api/types.go | 1 + backend/yandex/yandex.go | 1 + backend/zoho/api/types.go | 1 + cmd/about/about.go | 1 + cmd/authorize/authorize.go | 1 + cmd/backend/backend.go | 1 + cmd/cachestats/cachestats.go | 1 + cmd/cat/cat.go | 1 + cmd/check/check.go | 1 + cmd/checksum/checksum.go | 1 + cmd/cleanup/cleanup.go | 1 + cmd/cmount/mount.go | 7 +++--- cmd/cmount/mount_brew.go | 7 +++--- cmd/cmount/mount_test.go | 2 ++ cmd/cmount/mount_unsupported.go | 7 +++--- cmd/config/config.go | 1 + cmd/copy/copy.go | 1 + cmd/copyto/copyto.go | 1 + cmd/copyurl/copyurl.go | 1 + cmd/cryptcheck/cryptcheck.go | 1 + cmd/cryptdecode/cryptdecode.go | 1 + cmd/dedupe/dedupe.go | 1 + cmd/delete/delete.go | 1 + cmd/deletefile/deletefile.go | 1 + cmd/genautocomplete/genautocomplete.go | 1 + cmd/gendocs/gendocs.go | 1 + cmd/hashsum/hashsum.go | 1 + cmd/link/link.go | 1 + cmd/listremotes/listremotes.go | 1 + cmd/ls/ls.go | 1 + cmd/ls/lshelp/lshelp.go | 1 + cmd/lsd/lsd.go | 1 + cmd/lsf/lsf.go | 1 + cmd/lsjson/lsjson.go | 1 + cmd/lsl/lsl.go | 1 + cmd/md5sum/md5sum.go | 1 + cmd/mkdir/mkdir.go | 1 + cmd/mount/mount.go | 3 +-- cmd/mount/mount_unsupported.go | 11 +++++----- cmd/mount2/mount.go | 3 +-- cmd/mount2/mount_unsupported.go | 7 +++--- cmd/mountlib/mount.go | 1 + cmd/move/move.go | 1 + cmd/moveto/moveto.go | 1 + cmd/ncdu/ncdu.go | 3 +-- cmd/obscure/obscure.go | 1 + cmd/purge/purge.go | 1 + cmd/rc/rc.go | 1 + cmd/rcat/rcat.go | 1 + cmd/rcd/rcd.go | 1 + cmd/reveal/reveal.go | 1 + cmd/rmdir/rmdir.go | 1 + cmd/rmdirs/rmdirs.go | 1 + cmd/selfupdate/selfupdate.go | 1 + cmd/serve/dlna/data/data.go | 5 +++-- cmd/serve/dlna/dlna.go | 1 + cmd/serve/dlna/dlnaflags/dlnaflags.go | 1 + cmd/serve/dlna/upnpav/upnpav.go | 1 + cmd/serve/ftp/ftp.go | 3 +-- cmd/serve/http/data/data.go | 5 +++-- cmd/serve/http/http.go | 1 + cmd/serve/httplib/httpflags/httpflags.go | 1 + cmd/serve/serve.go | 1 + cmd/serve/sftp/sftp.go | 3 +-- cmd/settier/settier.go | 1 + cmd/sha1sum/sha1sum.go | 1 + cmd/size/size.go | 1 + cmd/sync/sync.go | 1 + cmd/test/histogram/histogram.go | 1 + cmd/test/info/info.go | 1 + cmd/test/info/internal/build_csv/main.go | 1 + cmd/test/info/internal/internal.go | 1 + cmd/test/memory/memory.go | 1 + cmd/test/test.go | 1 + cmd/touch/touch.go | 1 + cmd/tree/tree.go | 1 + cmd/version/version.go | 1 + fs/chunkedreader/chunkedreader.go | 1 + fs/driveletter/driveletter.go | 3 +-- fs/driveletter/driveletter_windows.go | 1 + fs/hash/hash.go | 1 + fs/rc/jobs/job.go | 3 +-- fs/rc/webgui/plugins.go | 1 + fs/rc/webgui/webgui.go | 3 +-- fstest/mockfs/mockfs.go | 1 + lib/buildinfo/tags.go | 1 + lib/daemonize/daemon_other.go | 3 +-- lib/daemonize/daemon_unix.go | 3 +-- lib/encoder/encoder.go | 22 +++++++++---------- lib/encoder/filename/init.go | 1 + lib/encoder/internal/gen/main.go | 1 + lib/errors/errors.go | 1 + lib/file/preallocate_windows.go | 2 +- lib/http/auth/auth.go | 1 + lib/jwtutil/jwtutil.go | 1 + lib/kv/bolt.go | 1 + lib/mmap/mmap.go | 1 + lib/oauthutil/oauthutil.go | 1 + lib/readers/context.go | 1 + vfs/vfs.go | 5 ++--- vfs/vfscache/downloaders/downloaders.go | 1 + vfs/vfscommon/cachemode.go | 1 + vfs/vfstest/vfs.go | 1 + 132 files changed, 164 insertions(+), 68 deletions(-) diff --git a/backend/alias/alias.go b/backend/alias/alias.go index 33ba5d664ecce..92dda17bf65b6 100644 --- a/backend/alias/alias.go +++ b/backend/alias/alias.go @@ -1,3 +1,4 @@ +// Package alias implements a virtual provider to rename existing remotes. package alias import ( diff --git a/backend/all/all.go b/backend/all/all.go index dc17911e4c349..5dae4d37db0a2 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -1,3 +1,4 @@ +// Package all imports all the backends package all import ( diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index edb8630708886..0db3592220d16 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -1,8 +1,7 @@ -// Package azureblob provides an interface to the Microsoft Azure blob object storage system - //go:build !plan9 && !solaris && !js // +build !plan9,!solaris,!js +// Package azureblob provides an interface to the Microsoft Azure blob object storage system package azureblob import ( diff --git a/backend/b2/api/types.go b/backend/b2/api/types.go index e139dc8c1155a..5089db2425b39 100644 --- a/backend/b2/api/types.go +++ b/backend/b2/api/types.go @@ -1,3 +1,4 @@ +// Package api provides types used by the Backblaze B2 API. package api import ( diff --git a/backend/b2/b2.go b/backend/b2/b2.go index 6fd77b6a0c30c..52794d8025c61 100644 --- a/backend/b2/b2.go +++ b/backend/b2/b2.go @@ -1,4 +1,4 @@ -// Package b2 provides an interface to the Backblaze B2 object storage system +// Package b2 provides an interface to the Backblaze B2 object storage system. package b2 // FIXME should we remove sha1 checks from here as rclone now supports diff --git a/backend/cache/cache.go b/backend/cache/cache.go index 1c75aea4a8529..5c4e5dc87654e 100644 --- a/backend/cache/cache.go +++ b/backend/cache/cache.go @@ -1,6 +1,7 @@ //go:build !plan9 && !js // +build !plan9,!js +// Package cache implements a virtual provider to cache existing remotes. package cache import ( diff --git a/backend/fichier/fichier.go b/backend/fichier/fichier.go index 454e391ff61ca..e6c05305154ef 100644 --- a/backend/fichier/fichier.go +++ b/backend/fichier/fichier.go @@ -1,3 +1,4 @@ +// Package fichier provides an interface to the 1Fichier storage system. package fichier import ( diff --git a/backend/googlephotos/api/types.go b/backend/googlephotos/api/types.go index d26cfc7149f65..9b7aa79f6e173 100644 --- a/backend/googlephotos/api/types.go +++ b/backend/googlephotos/api/types.go @@ -1,3 +1,4 @@ +// Package api provides types used by the Google Photos API. package api import ( diff --git a/backend/hdfs/hdfs.go b/backend/hdfs/hdfs.go index 234344eceeec0..c07c0d7e2bc89 100644 --- a/backend/hdfs/hdfs.go +++ b/backend/hdfs/hdfs.go @@ -1,6 +1,7 @@ //go:build !plan9 // +build !plan9 +// Package hdfs provides an interface to the HDFS storage system. package hdfs import ( diff --git a/backend/hidrive/hidrivehash/internal/internal.go b/backend/hidrive/hidrivehash/internal/internal.go index f1596a9e68d42..261233c0dc46e 100644 --- a/backend/hidrive/hidrivehash/internal/internal.go +++ b/backend/hidrive/hidrivehash/internal/internal.go @@ -1,3 +1,4 @@ +// Package internal provides utilities for HiDrive. package internal import ( diff --git a/backend/jottacloud/api/types.go b/backend/jottacloud/api/types.go index 91e0c30957668..8daf0f7ad35b4 100644 --- a/backend/jottacloud/api/types.go +++ b/backend/jottacloud/api/types.go @@ -1,3 +1,4 @@ +// Package api provides types used by the Jottacloud API. package api import ( diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 0044c7f79d1e2..00178745192ed 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -1,3 +1,4 @@ +// Package jottacloud provides an interface to the Jottacloud storage system. package jottacloud import ( diff --git a/backend/koofr/koofr.go b/backend/koofr/koofr.go index f893b898b4896..642ab79fe8a2c 100644 --- a/backend/koofr/koofr.go +++ b/backend/koofr/koofr.go @@ -1,3 +1,4 @@ +// Package koofr provides an interface to the Koofr storage system. package koofr import ( diff --git a/backend/mailru/api/m1.go b/backend/mailru/api/m1.go index 039b1e10465f1..c9d3e9c9ff250 100644 --- a/backend/mailru/api/m1.go +++ b/backend/mailru/api/m1.go @@ -1,3 +1,4 @@ +// Package api provides types used by the Mail.ru API. package api import ( diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go index bd527e8e23b6c..95507996c4f28 100644 --- a/backend/mailru/mailru.go +++ b/backend/mailru/mailru.go @@ -1,3 +1,4 @@ +// Package mailru provides an interface to the Mail.ru Cloud storage system. package mailru import ( diff --git a/backend/onedrive/api/types.go b/backend/onedrive/api/types.go index 12dca863b0281..bfde1016111c7 100644 --- a/backend/onedrive/api/types.go +++ b/backend/onedrive/api/types.go @@ -1,5 +1,4 @@ -// Types passed and returned to and from the API - +// Package api provides types used by the OneDrive API. package api import ( diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.go index 35e1871c31c2e..037372e1357ed 100644 --- a/backend/opendrive/opendrive.go +++ b/backend/opendrive/opendrive.go @@ -1,3 +1,4 @@ +// Package opendrive provides an interface to the OpenDrive storage system. package opendrive import ( diff --git a/backend/putio/putio.go b/backend/putio/putio.go index 766a99b77f63e..6923641666d46 100644 --- a/backend/putio/putio.go +++ b/backend/putio/putio.go @@ -1,3 +1,4 @@ +// Package putio provides an interface to the put.io storage system. package putio import ( diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go index cd4880358c725..a3d0bd639341d 100644 --- a/backend/qingstor/qingstor.go +++ b/backend/qingstor/qingstor.go @@ -1,9 +1,8 @@ -// Package qingstor provides an interface to QingStor object storage -// Home: https://www.qingcloud.com/ - //go:build !plan9 && !js // +build !plan9,!js +// Package qingstor provides an interface to QingStor object storage +// Home: https://www.qingcloud.com/ package qingstor import ( diff --git a/backend/seafile/api/types.go b/backend/seafile/api/types.go index 9fb95c0cfa1ab..d6224709ff2bb 100644 --- a/backend/seafile/api/types.go +++ b/backend/seafile/api/types.go @@ -1,3 +1,4 @@ +// Package api provides types used by the Seafile API. package api // Some api objects are duplicated with only small differences, diff --git a/backend/seafile/seafile.go b/backend/seafile/seafile.go index 00457676c457f..1413544ee031a 100644 --- a/backend/seafile/seafile.go +++ b/backend/seafile/seafile.go @@ -1,3 +1,4 @@ +// Package seafile provides an interface to the Seafile storage system. package seafile import ( diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 6a4259542096a..1c160996cc7e7 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -1,8 +1,7 @@ -// Package sftp provides a filesystem interface using github.com/pkg/sftp - //go:build !plan9 // +build !plan9 +// Package sftp provides a filesystem interface using github.com/pkg/sftp package sftp import ( diff --git a/backend/sia/api/types.go b/backend/sia/api/types.go index ebec8961efe6f..eb046ab22e2c9 100644 --- a/backend/sia/api/types.go +++ b/backend/sia/api/types.go @@ -1,3 +1,4 @@ +// Package api provides types used by the Sia API. package api import ( diff --git a/backend/sia/sia.go b/backend/sia/sia.go index 2a8235638f282..b86faae61d59a 100644 --- a/backend/sia/sia.go +++ b/backend/sia/sia.go @@ -1,3 +1,4 @@ +// Package sia provides an interface to the Sia storage system. package sia import ( diff --git a/backend/union/policy/all.go b/backend/union/policy/all.go index 1de36da2a4b15..fe43654ff194c 100644 --- a/backend/union/policy/all.go +++ b/backend/union/policy/all.go @@ -1,3 +1,4 @@ +// Package policy provides utilities for the union implementation. package policy import ( diff --git a/backend/union/union.go b/backend/union/union.go index 8bcbf66c940c1..7e10f06b8082c 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -1,3 +1,4 @@ +// Package union implements a virtual provider to join existing remotes. package union import ( diff --git a/backend/union/upstream/upstream.go b/backend/union/upstream/upstream.go index 7b9f44fd2a39d..119eb7bb04b31 100644 --- a/backend/union/upstream/upstream.go +++ b/backend/union/upstream/upstream.go @@ -1,3 +1,4 @@ +// Package upstream provides utility functionality to union. package upstream import ( diff --git a/backend/uptobox/api/types.go b/backend/uptobox/api/types.go index 872a5cff04790..4e842c8ab64b8 100644 --- a/backend/uptobox/api/types.go +++ b/backend/uptobox/api/types.go @@ -1,3 +1,4 @@ +// Package api provides types used by the Uptobox API. package api import "fmt" diff --git a/backend/uptobox/uptobox.go b/backend/uptobox/uptobox.go index 6bc5bd8235d93..fb3bbd7db5ce0 100644 --- a/backend/uptobox/uptobox.go +++ b/backend/uptobox/uptobox.go @@ -1,3 +1,4 @@ +// Package uptobox provides an interface to the Uptobox storage system. package uptobox import ( diff --git a/backend/yandex/api/types.go b/backend/yandex/api/types.go index 78d4c6850bfca..614ced7d3d78a 100644 --- a/backend/yandex/api/types.go +++ b/backend/yandex/api/types.go @@ -1,3 +1,4 @@ +// Package api provides types used by the Yandex API. package api import ( diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.go index 3d5bf53cc0c67..a643a7771265d 100644 --- a/backend/yandex/yandex.go +++ b/backend/yandex/yandex.go @@ -1,3 +1,4 @@ +// Package yandex provides an interface to the Yandex storage system. package yandex import ( diff --git a/backend/zoho/api/types.go b/backend/zoho/api/types.go index 43d8644ceeea7..755c57b9be842 100644 --- a/backend/zoho/api/types.go +++ b/backend/zoho/api/types.go @@ -1,3 +1,4 @@ +// Package api provides types used by the Zoho API. package api import ( diff --git a/cmd/about/about.go b/cmd/about/about.go index 46c1d6f66f2e5..967744b598782 100644 --- a/cmd/about/about.go +++ b/cmd/about/about.go @@ -1,3 +1,4 @@ +// Package about provides the about command. package about import ( diff --git a/cmd/authorize/authorize.go b/cmd/authorize/authorize.go index d3114c750e239..87dda815bdc5b 100644 --- a/cmd/authorize/authorize.go +++ b/cmd/authorize/authorize.go @@ -1,3 +1,4 @@ +// Package authorize provides the authorize command. package authorize import ( diff --git a/cmd/backend/backend.go b/cmd/backend/backend.go index 85b7e44bd7cf2..3199f8a5deba0 100644 --- a/cmd/backend/backend.go +++ b/cmd/backend/backend.go @@ -1,3 +1,4 @@ +// Package backend provides the backend command. package backend import ( diff --git a/cmd/cachestats/cachestats.go b/cmd/cachestats/cachestats.go index 16a1398d3f756..7638db8f4cb3b 100644 --- a/cmd/cachestats/cachestats.go +++ b/cmd/cachestats/cachestats.go @@ -1,6 +1,7 @@ //go:build !plan9 && !js // +build !plan9,!js +// Package cachestats provides the cachestats command. package cachestats import ( diff --git a/cmd/cat/cat.go b/cmd/cat/cat.go index 4f90f04780808..227ddb1f396c8 100644 --- a/cmd/cat/cat.go +++ b/cmd/cat/cat.go @@ -1,3 +1,4 @@ +// Package cat provides the cat command. package cat import ( diff --git a/cmd/check/check.go b/cmd/check/check.go index 886c36a58fe5e..3c126e176275b 100644 --- a/cmd/check/check.go +++ b/cmd/check/check.go @@ -1,3 +1,4 @@ +// Package check provides the check command. package check import ( diff --git a/cmd/checksum/checksum.go b/cmd/checksum/checksum.go index 1083e380032be..c25aa143d49f7 100644 --- a/cmd/checksum/checksum.go +++ b/cmd/checksum/checksum.go @@ -1,3 +1,4 @@ +// Package checksum provides the checksum command. package checksum import ( diff --git a/cmd/cleanup/cleanup.go b/cmd/cleanup/cleanup.go index 549e3b420ea6e..e54577466897d 100644 --- a/cmd/cleanup/cleanup.go +++ b/cmd/cleanup/cleanup.go @@ -1,3 +1,4 @@ +// Package cleanup provides the cleanup command. package cleanup import ( diff --git a/cmd/cmount/mount.go b/cmd/cmount/mount.go index 873adcaf0e35a..c9893f2f7639f 100644 --- a/cmd/cmount/mount.go +++ b/cmd/cmount/mount.go @@ -1,11 +1,10 @@ -// Package cmount implements a FUSE mounting system for rclone remotes. -// -// This uses the cgo based cgofuse library - //go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows) // +build cmount // +build linux,cgo darwin,cgo freebsd,cgo windows +// Package cmount implements a FUSE mounting system for rclone remotes. +// +// This uses the cgo based cgofuse library package cmount import ( diff --git a/cmd/cmount/mount_brew.go b/cmd/cmount/mount_brew.go index ec1b97881e277..964c0b234967b 100644 --- a/cmd/cmount/mount_brew.go +++ b/cmd/cmount/mount_brew.go @@ -1,9 +1,10 @@ -// Build for macos with the brew tag to handle the absence -// of fuse and print an appropriate error message - //go:build brew && darwin // +build brew,darwin +// Package cmount implements a FUSE mounting system for rclone remotes. +// +// Build for macos with the brew tag to handle the absence +// of fuse and print an appropriate error message package cmount import ( diff --git a/cmd/cmount/mount_test.go b/cmd/cmount/mount_test.go index 542c35b1b9215..7c9c852f6538c 100644 --- a/cmd/cmount/mount_test.go +++ b/cmd/cmount/mount_test.go @@ -3,6 +3,8 @@ // +build linux,cgo darwin,cgo freebsd,cgo windows // +build !race !windows +// Package cmount implements a FUSE mounting system for rclone remotes. +// // FIXME this doesn't work with the race detector under Windows either // hanging or producing lots of differences. diff --git a/cmd/cmount/mount_unsupported.go b/cmd/cmount/mount_unsupported.go index 3b6761beb514a..375cec4565444 100644 --- a/cmd/cmount/mount_unsupported.go +++ b/cmd/cmount/mount_unsupported.go @@ -1,10 +1,11 @@ -// Build for cmount for unsupported platforms to stop go complaining -// about "no buildable Go source files " - //go:build !((linux && cgo && cmount) || (darwin && cgo && cmount) || (freebsd && cgo && cmount) || (windows && cmount)) // +build !linux !cgo !cmount // +build !darwin !cgo !cmount // +build !freebsd !cgo !cmount // +build !windows !cmount +// Package cmount implements a FUSE mounting system for rclone remotes. +// +// Build for cmount for unsupported platforms to stop go complaining +// about "no buildable Go source files". package cmount diff --git a/cmd/config/config.go b/cmd/config/config.go index ed57003278043..318685c7d65b5 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -1,3 +1,4 @@ +// Package config provides the config command. package config import ( diff --git a/cmd/copy/copy.go b/cmd/copy/copy.go index 33175e1f77416..c6e42a70ea9bd 100644 --- a/cmd/copy/copy.go +++ b/cmd/copy/copy.go @@ -1,3 +1,4 @@ +// Package copy provides the copy command. package copy import ( diff --git a/cmd/copyto/copyto.go b/cmd/copyto/copyto.go index 02d2af21e2ea4..85b54fa609a62 100644 --- a/cmd/copyto/copyto.go +++ b/cmd/copyto/copyto.go @@ -1,3 +1,4 @@ +// Package copyto provides the copyto command. package copyto import ( diff --git a/cmd/copyurl/copyurl.go b/cmd/copyurl/copyurl.go index fc3cf38cccc86..a3c079b11c8ef 100644 --- a/cmd/copyurl/copyurl.go +++ b/cmd/copyurl/copyurl.go @@ -1,3 +1,4 @@ +// Package copyurl provides the copyurl command. package copyurl import ( diff --git a/cmd/cryptcheck/cryptcheck.go b/cmd/cryptcheck/cryptcheck.go index ec0597da73018..aab50ef06f8e5 100644 --- a/cmd/cryptcheck/cryptcheck.go +++ b/cmd/cryptcheck/cryptcheck.go @@ -1,3 +1,4 @@ +// Package cryptcheck provides the cryptcheck command. package cryptcheck import ( diff --git a/cmd/cryptdecode/cryptdecode.go b/cmd/cryptdecode/cryptdecode.go index d36c70ca47f52..9c6178e9e4c1a 100644 --- a/cmd/cryptdecode/cryptdecode.go +++ b/cmd/cryptdecode/cryptdecode.go @@ -1,3 +1,4 @@ +// Package cryptdecode provides the cryptdecode command. package cryptdecode import ( diff --git a/cmd/dedupe/dedupe.go b/cmd/dedupe/dedupe.go index b43c4c2795663..94a136ca2f69b 100644 --- a/cmd/dedupe/dedupe.go +++ b/cmd/dedupe/dedupe.go @@ -1,3 +1,4 @@ +// Package dedupe provides the dedupe command. package dedupe import ( diff --git a/cmd/delete/delete.go b/cmd/delete/delete.go index a432dbe56fb71..047c320af9855 100644 --- a/cmd/delete/delete.go +++ b/cmd/delete/delete.go @@ -1,3 +1,4 @@ +// Package delete provides the delete command. package delete import ( diff --git a/cmd/deletefile/deletefile.go b/cmd/deletefile/deletefile.go index 88750c75eb216..9d222771d2f4a 100644 --- a/cmd/deletefile/deletefile.go +++ b/cmd/deletefile/deletefile.go @@ -1,3 +1,4 @@ +// Package deletefile provides the deletefile command. package deletefile import ( diff --git a/cmd/genautocomplete/genautocomplete.go b/cmd/genautocomplete/genautocomplete.go index 5971135770d54..9502bbbc9d439 100644 --- a/cmd/genautocomplete/genautocomplete.go +++ b/cmd/genautocomplete/genautocomplete.go @@ -1,3 +1,4 @@ +// Package genautocomplete provides the genautocomplete command. package genautocomplete import ( diff --git a/cmd/gendocs/gendocs.go b/cmd/gendocs/gendocs.go index 7a7ef3f46c016..e80b8a924ba5b 100644 --- a/cmd/gendocs/gendocs.go +++ b/cmd/gendocs/gendocs.go @@ -1,3 +1,4 @@ +// Package gendocs provides the gendocs command. package gendocs import ( diff --git a/cmd/hashsum/hashsum.go b/cmd/hashsum/hashsum.go index 64dfde4ce5685..5655fdcf24b24 100644 --- a/cmd/hashsum/hashsum.go +++ b/cmd/hashsum/hashsum.go @@ -1,3 +1,4 @@ +// Package hashsum provides the hashsum command. package hashsum import ( diff --git a/cmd/link/link.go b/cmd/link/link.go index 3367178c5750e..a0356cd3369fd 100644 --- a/cmd/link/link.go +++ b/cmd/link/link.go @@ -1,3 +1,4 @@ +// Package link provides the link command. package link import ( diff --git a/cmd/listremotes/listremotes.go b/cmd/listremotes/listremotes.go index 908979a4fa34f..bf7b697f235d0 100644 --- a/cmd/listremotes/listremotes.go +++ b/cmd/listremotes/listremotes.go @@ -1,3 +1,4 @@ +// Package ls provides the ls command. package ls import ( diff --git a/cmd/ls/ls.go b/cmd/ls/ls.go index e08a2c21d3eed..334b12f8e8ecc 100644 --- a/cmd/ls/ls.go +++ b/cmd/ls/ls.go @@ -1,3 +1,4 @@ +// Package ls provides the ls command. package ls import ( diff --git a/cmd/ls/lshelp/lshelp.go b/cmd/ls/lshelp/lshelp.go index f4a678be2d1ea..897c22f1c0488 100644 --- a/cmd/ls/lshelp/lshelp.go +++ b/cmd/ls/lshelp/lshelp.go @@ -1,3 +1,4 @@ +// Package lshelp provides common help for list commands. package lshelp import ( diff --git a/cmd/lsd/lsd.go b/cmd/lsd/lsd.go index bd723daadb5aa..8d79266fb6985 100644 --- a/cmd/lsd/lsd.go +++ b/cmd/lsd/lsd.go @@ -1,3 +1,4 @@ +// Package lsd provides the lsd command. package lsd import ( diff --git a/cmd/lsf/lsf.go b/cmd/lsf/lsf.go index e370f930c2b48..2651e90e7263e 100644 --- a/cmd/lsf/lsf.go +++ b/cmd/lsf/lsf.go @@ -1,3 +1,4 @@ +// Package lsf provides the lsf command. package lsf import ( diff --git a/cmd/lsjson/lsjson.go b/cmd/lsjson/lsjson.go index 8e4e343da0f68..eb30f85784caf 100644 --- a/cmd/lsjson/lsjson.go +++ b/cmd/lsjson/lsjson.go @@ -1,3 +1,4 @@ +// Package lsjson provides the lsjson command. package lsjson import ( diff --git a/cmd/lsl/lsl.go b/cmd/lsl/lsl.go index f2f40f7a70ec7..f8108c5f7b8df 100644 --- a/cmd/lsl/lsl.go +++ b/cmd/lsl/lsl.go @@ -1,3 +1,4 @@ +// Package lsl provides the lsl command. package lsl import ( diff --git a/cmd/md5sum/md5sum.go b/cmd/md5sum/md5sum.go index 665ae44673e93..09da8526f19be 100644 --- a/cmd/md5sum/md5sum.go +++ b/cmd/md5sum/md5sum.go @@ -1,3 +1,4 @@ +// Package md5sum provides the md5sum command. package md5sum import ( diff --git a/cmd/mkdir/mkdir.go b/cmd/mkdir/mkdir.go index 33fa33667cd1b..3c99180feae52 100644 --- a/cmd/mkdir/mkdir.go +++ b/cmd/mkdir/mkdir.go @@ -1,3 +1,4 @@ +// Package mkdir provides the mkdir command. package mkdir import ( diff --git a/cmd/mount/mount.go b/cmd/mount/mount.go index db0f23bd38245..765419df4f30b 100644 --- a/cmd/mount/mount.go +++ b/cmd/mount/mount.go @@ -1,8 +1,7 @@ -// Package mount implements a FUSE mounting system for rclone remotes. - //go:build linux || freebsd // +build linux freebsd +// Package mount implements a FUSE mounting system for rclone remotes. package mount import ( diff --git a/cmd/mount/mount_unsupported.go b/cmd/mount/mount_unsupported.go index cc5ccb628fd03..84f8656dc2e64 100644 --- a/cmd/mount/mount_unsupported.go +++ b/cmd/mount/mount_unsupported.go @@ -1,9 +1,10 @@ -// Build for mount for unsupported platforms to stop go complaining -// about "no buildable Go source files " - -// Invert the build constraint: linux freebsd - //go:build !linux && !freebsd // +build !linux,!freebsd +// Package mount implements a FUSE mounting system for rclone remotes. +// +// Build for mount for unsupported platforms to stop go complaining +// about "no buildable Go source files". +// +// Invert the build constraint: linux freebsd package mount diff --git a/cmd/mount2/mount.go b/cmd/mount2/mount.go index d33f44533f3af..cbb8905009f92 100644 --- a/cmd/mount2/mount.go +++ b/cmd/mount2/mount.go @@ -1,8 +1,7 @@ -// Package mount implements a FUSE mounting system for rclone remotes. - //go:build linux || (darwin && amd64) // +build linux darwin,amd64 +// Package mount2 implements a FUSE mounting system for rclone remotes. package mount2 import ( diff --git a/cmd/mount2/mount_unsupported.go b/cmd/mount2/mount_unsupported.go index 975a25b3e4c21..649eab0a21996 100644 --- a/cmd/mount2/mount_unsupported.go +++ b/cmd/mount2/mount_unsupported.go @@ -1,8 +1,9 @@ -// Build for mount for unsupported platforms to stop go complaining -// about "no buildable Go source files " - //go:build !linux && (!darwin || !amd64) // +build !linux // +build !darwin !amd64 +// Package mount2 implements a FUSE mounting system for rclone remotes. +// +// Build for mount for unsupported platforms to stop go complaining +// about "no buildable Go source files". package mount2 diff --git a/cmd/mountlib/mount.go b/cmd/mountlib/mount.go index b0119dc12eb37..ba3773e5c0a9b 100644 --- a/cmd/mountlib/mount.go +++ b/cmd/mountlib/mount.go @@ -1,3 +1,4 @@ +// Package mountlib provides the mount command. package mountlib import ( diff --git a/cmd/move/move.go b/cmd/move/move.go index 3d7f87167eee9..65c8adc233bfa 100644 --- a/cmd/move/move.go +++ b/cmd/move/move.go @@ -1,3 +1,4 @@ +// Package move provides the move command. package move import ( diff --git a/cmd/moveto/moveto.go b/cmd/moveto/moveto.go index 8d81bb3d39c3a..698d7ee186346 100644 --- a/cmd/moveto/moveto.go +++ b/cmd/moveto/moveto.go @@ -1,3 +1,4 @@ +// Package moveto provides the moveto command. package moveto import ( diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go index abb748b6d5276..02427fb494a40 100644 --- a/cmd/ncdu/ncdu.go +++ b/cmd/ncdu/ncdu.go @@ -1,8 +1,7 @@ -// Package ncdu implements a text based user interface for exploring a remote - //go:build !plan9 && !js // +build !plan9,!js +// Package ncdu implements a text based user interface for exploring a remote package ncdu import ( diff --git a/cmd/obscure/obscure.go b/cmd/obscure/obscure.go index d26f1d72b660b..08cf8a06a86a8 100644 --- a/cmd/obscure/obscure.go +++ b/cmd/obscure/obscure.go @@ -1,3 +1,4 @@ +// Package obscure provides the obscure command. package obscure import ( diff --git a/cmd/purge/purge.go b/cmd/purge/purge.go index 335615cf96c2f..d41e22cf67ab5 100644 --- a/cmd/purge/purge.go +++ b/cmd/purge/purge.go @@ -1,3 +1,4 @@ +// Package purge provides the purge command. package purge import ( diff --git a/cmd/rc/rc.go b/cmd/rc/rc.go index 53e8697d80095..93c2e319eb24b 100644 --- a/cmd/rc/rc.go +++ b/cmd/rc/rc.go @@ -1,3 +1,4 @@ +// Package rc provides the rc command. package rc import ( diff --git a/cmd/rcat/rcat.go b/cmd/rcat/rcat.go index f082fa95d34b5..c7766265d897f 100644 --- a/cmd/rcat/rcat.go +++ b/cmd/rcat/rcat.go @@ -1,3 +1,4 @@ +// Package rcat provides the rcat command. package rcat import ( diff --git a/cmd/rcd/rcd.go b/cmd/rcd/rcd.go index cc4be57f9d814..940c300a6fec4 100644 --- a/cmd/rcd/rcd.go +++ b/cmd/rcd/rcd.go @@ -1,3 +1,4 @@ +// Package rcd provides the rcd command. package rcd import ( diff --git a/cmd/reveal/reveal.go b/cmd/reveal/reveal.go index 817dd92ddaeca..e569a26b44b04 100644 --- a/cmd/reveal/reveal.go +++ b/cmd/reveal/reveal.go @@ -1,3 +1,4 @@ +// Package reveal provides the reveal command. package reveal import ( diff --git a/cmd/rmdir/rmdir.go b/cmd/rmdir/rmdir.go index 9b532ca4709b1..778315be83e0a 100644 --- a/cmd/rmdir/rmdir.go +++ b/cmd/rmdir/rmdir.go @@ -1,3 +1,4 @@ +// Package rmdir provides the rmdir command. package rmdir import ( diff --git a/cmd/rmdirs/rmdirs.go b/cmd/rmdirs/rmdirs.go index 41788c5271dbe..11a09ca203bc2 100644 --- a/cmd/rmdirs/rmdirs.go +++ b/cmd/rmdirs/rmdirs.go @@ -1,3 +1,4 @@ +// Package rmdir provides the rmdir command. package rmdir import ( diff --git a/cmd/selfupdate/selfupdate.go b/cmd/selfupdate/selfupdate.go index 5bfd1cb667067..ab24290b30ac5 100644 --- a/cmd/selfupdate/selfupdate.go +++ b/cmd/selfupdate/selfupdate.go @@ -1,6 +1,7 @@ //go:build !noselfupdate // +build !noselfupdate +// Package selfupdate provides the selfupdate command. package selfupdate import ( diff --git a/cmd/serve/dlna/data/data.go b/cmd/serve/dlna/data/data.go index 8e142b99f388d..a5a92880cb814 100644 --- a/cmd/serve/dlna/data/data.go +++ b/cmd/serve/dlna/data/data.go @@ -1,6 +1,7 @@ -//go:generate go run assets_generate.go +// Package data provides utilities for DLNA server. // The "go:generate" directive compiles static assets by running assets_generate.go - +// +//go:generate go run assets_generate.go package data import ( diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go index da6a08a19ce77..c36e492d7fa09 100644 --- a/cmd/serve/dlna/dlna.go +++ b/cmd/serve/dlna/dlna.go @@ -1,3 +1,4 @@ +// Package dlna provides DLNA server. package dlna import ( diff --git a/cmd/serve/dlna/dlnaflags/dlnaflags.go b/cmd/serve/dlna/dlnaflags/dlnaflags.go index 06ea27a1973d8..85a1347dcf3d3 100644 --- a/cmd/serve/dlna/dlnaflags/dlnaflags.go +++ b/cmd/serve/dlna/dlnaflags/dlnaflags.go @@ -1,3 +1,4 @@ +// Package dlnaflags provides utility functionality to DLNA. package dlnaflags import ( diff --git a/cmd/serve/dlna/upnpav/upnpav.go b/cmd/serve/dlna/upnpav/upnpav.go index 5ac5e17b7a9fc..c6dc9dc4fa619 100644 --- a/cmd/serve/dlna/upnpav/upnpav.go +++ b/cmd/serve/dlna/upnpav/upnpav.go @@ -1,3 +1,4 @@ +// Package upnpav provides utilities for DLNA server. package upnpav import ( diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go index 11753e42fbf1d..08cca77f31363 100644 --- a/cmd/serve/ftp/ftp.go +++ b/cmd/serve/ftp/ftp.go @@ -1,8 +1,7 @@ -// Package ftp implements an FTP server for rclone - //go:build !plan9 // +build !plan9 +// Package ftp implements an FTP server for rclone package ftp import ( diff --git a/cmd/serve/http/data/data.go b/cmd/serve/http/data/data.go index 73f6a1ff28b91..075acf8b79f19 100644 --- a/cmd/serve/http/data/data.go +++ b/cmd/serve/http/data/data.go @@ -1,6 +1,7 @@ -//go:generate go run assets_generate.go +// Package data provides common functionality for http servers // The "go:generate" directive compiles static assets by running assets_generate.go - +// +//go:generate go run assets_generate.go package data import ( diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go index 8712f8ac82108..cdf8afd2860b9 100644 --- a/cmd/serve/http/http.go +++ b/cmd/serve/http/http.go @@ -1,3 +1,4 @@ +// Package http provides common functionality for http servers package http import ( diff --git a/cmd/serve/httplib/httpflags/httpflags.go b/cmd/serve/httplib/httpflags/httpflags.go index 899482b7fd406..109604f0e3ede 100644 --- a/cmd/serve/httplib/httpflags/httpflags.go +++ b/cmd/serve/httplib/httpflags/httpflags.go @@ -1,3 +1,4 @@ +// Package httpflags provides utility functionality to HTTP. package httpflags import ( diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go index 848cb24bb34db..f79162c821e56 100644 --- a/cmd/serve/serve.go +++ b/cmd/serve/serve.go @@ -1,3 +1,4 @@ +// Package serve provides the serve command. package serve import ( diff --git a/cmd/serve/sftp/sftp.go b/cmd/serve/sftp/sftp.go index c77450a5e2a5d..1a7b7717c7aad 100644 --- a/cmd/serve/sftp/sftp.go +++ b/cmd/serve/sftp/sftp.go @@ -1,8 +1,7 @@ -// Package sftp implements an SFTP server to serve an rclone VFS - //go:build !plan9 // +build !plan9 +// Package sftp implements an SFTP server to serve an rclone VFS package sftp import ( diff --git a/cmd/settier/settier.go b/cmd/settier/settier.go index 6be1c1c6a2951..8a1d234f07e63 100644 --- a/cmd/settier/settier.go +++ b/cmd/settier/settier.go @@ -1,3 +1,4 @@ +// Package settier provides the settier command. package settier import ( diff --git a/cmd/sha1sum/sha1sum.go b/cmd/sha1sum/sha1sum.go index 1c3115a8b2712..25badd19c17ff 100644 --- a/cmd/sha1sum/sha1sum.go +++ b/cmd/sha1sum/sha1sum.go @@ -1,3 +1,4 @@ +// Package sha1sum provides the sha1sum command. package sha1sum import ( diff --git a/cmd/size/size.go b/cmd/size/size.go index 5ed51c7a6b099..c084de5278a84 100644 --- a/cmd/size/size.go +++ b/cmd/size/size.go @@ -1,3 +1,4 @@ +// Package size provides the size command. package size import ( diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go index be233cdd941ad..33a5b0027e5f2 100644 --- a/cmd/sync/sync.go +++ b/cmd/sync/sync.go @@ -1,3 +1,4 @@ +// Package sync provides the sync command. package sync import ( diff --git a/cmd/test/histogram/histogram.go b/cmd/test/histogram/histogram.go index d4f02d5d9987f..a3a2865b8117a 100644 --- a/cmd/test/histogram/histogram.go +++ b/cmd/test/histogram/histogram.go @@ -1,3 +1,4 @@ +// Package histogram provides the histogram test command. package histogram import ( diff --git a/cmd/test/info/info.go b/cmd/test/info/info.go index c4eef5f046d2f..89cc10a01283d 100644 --- a/cmd/test/info/info.go +++ b/cmd/test/info/info.go @@ -1,3 +1,4 @@ +// Package info provides the info test command. package info // FIXME once translations are implemented will need a no-escape diff --git a/cmd/test/info/internal/build_csv/main.go b/cmd/test/info/internal/build_csv/main.go index 96aa20a191821..fbabe5b85a60e 100644 --- a/cmd/test/info/internal/build_csv/main.go +++ b/cmd/test/info/internal/build_csv/main.go @@ -1,3 +1,4 @@ +// Package main provides utilities for the info test command. package main import ( diff --git a/cmd/test/info/internal/internal.go b/cmd/test/info/internal/internal.go index 6636629043827..7af06b87a8494 100644 --- a/cmd/test/info/internal/internal.go +++ b/cmd/test/info/internal/internal.go @@ -1,3 +1,4 @@ +// Package internal provides internal implementation for the info test command. package internal import ( diff --git a/cmd/test/memory/memory.go b/cmd/test/memory/memory.go index 551723258598b..38b7b16838213 100644 --- a/cmd/test/memory/memory.go +++ b/cmd/test/memory/memory.go @@ -1,3 +1,4 @@ +// Package memory provides the memory test command. package memory import ( diff --git a/cmd/test/test.go b/cmd/test/test.go index 39f5ccb295597..774bf6c3a507b 100644 --- a/cmd/test/test.go +++ b/cmd/test/test.go @@ -1,3 +1,4 @@ +// Package test provides the test command. package test import ( diff --git a/cmd/touch/touch.go b/cmd/touch/touch.go index 741c920b3f5a6..9f3d69b141798 100644 --- a/cmd/touch/touch.go +++ b/cmd/touch/touch.go @@ -1,3 +1,4 @@ +// Package touch provides the touch command. package touch import ( diff --git a/cmd/tree/tree.go b/cmd/tree/tree.go index 2528d9dc2a42d..5705dcec84f41 100644 --- a/cmd/tree/tree.go +++ b/cmd/tree/tree.go @@ -1,3 +1,4 @@ +// Package tree provides the tree command. package tree import ( diff --git a/cmd/version/version.go b/cmd/version/version.go index 3ec69e135187b..25a17aa41b980 100644 --- a/cmd/version/version.go +++ b/cmd/version/version.go @@ -1,3 +1,4 @@ +// Package version provides the version command. package version import ( diff --git a/fs/chunkedreader/chunkedreader.go b/fs/chunkedreader/chunkedreader.go index 1e29ea7365211..66cf666d28bdb 100644 --- a/fs/chunkedreader/chunkedreader.go +++ b/fs/chunkedreader/chunkedreader.go @@ -1,3 +1,4 @@ +// Package chunkedreader provides functionality for reading in chunks. package chunkedreader import ( diff --git a/fs/driveletter/driveletter.go b/fs/driveletter/driveletter.go index 0a683509e2dab..0c02f39767965 100644 --- a/fs/driveletter/driveletter.go +++ b/fs/driveletter/driveletter.go @@ -1,8 +1,7 @@ -// Package driveletter returns whether a name is a valid drive letter - //go:build !windows // +build !windows +// Package driveletter returns whether a name is a valid drive letter package driveletter // IsDriveLetter returns a bool indicating whether name is a valid diff --git a/fs/driveletter/driveletter_windows.go b/fs/driveletter/driveletter_windows.go index 40ca94d43146a..f8b506a800c86 100644 --- a/fs/driveletter/driveletter_windows.go +++ b/fs/driveletter/driveletter_windows.go @@ -1,6 +1,7 @@ //go:build windows // +build windows +// Package driveletter returns whether a name is a valid drive letter package driveletter // IsDriveLetter returns a bool indicating whether name is a valid diff --git a/fs/hash/hash.go b/fs/hash/hash.go index 93b1de3b22b03..4540b40cf49c4 100644 --- a/fs/hash/hash.go +++ b/fs/hash/hash.go @@ -1,3 +1,4 @@ +// Package hash provides hash utilities for Fs. package hash import ( diff --git a/fs/rc/jobs/job.go b/fs/rc/jobs/job.go index cb3aaa833b371..613093e259cc6 100644 --- a/fs/rc/jobs/job.go +++ b/fs/rc/jobs/job.go @@ -1,5 +1,4 @@ -// Manage background jobs that the rc is running - +// Package jobs manages background jobs that the rc is running. package jobs import ( diff --git a/fs/rc/webgui/plugins.go b/fs/rc/webgui/plugins.go index a6730aacbe216..23f9692c2fbc6 100644 --- a/fs/rc/webgui/plugins.go +++ b/fs/rc/webgui/plugins.go @@ -1,3 +1,4 @@ +// Package webgui provides plugin functionality to the Web GUI. package webgui import ( diff --git a/fs/rc/webgui/webgui.go b/fs/rc/webgui/webgui.go index 8153024d173de..ee05730f7b61f 100644 --- a/fs/rc/webgui/webgui.go +++ b/fs/rc/webgui/webgui.go @@ -1,5 +1,4 @@ -// Define the Web GUI helpers - +// Package webgui defines the Web GUI helpers. package webgui import ( diff --git a/fstest/mockfs/mockfs.go b/fstest/mockfs/mockfs.go index fe73db5ca1a68..62489c2f6c381 100644 --- a/fstest/mockfs/mockfs.go +++ b/fstest/mockfs/mockfs.go @@ -1,3 +1,4 @@ +// Package mockfs provides mock Fs for testing. package mockfs import ( diff --git a/lib/buildinfo/tags.go b/lib/buildinfo/tags.go index 6e61a457dec9b..945a1e4455a04 100644 --- a/lib/buildinfo/tags.go +++ b/lib/buildinfo/tags.go @@ -1,3 +1,4 @@ +// Package buildinfo provides build information. package buildinfo import ( diff --git a/lib/daemonize/daemon_other.go b/lib/daemonize/daemon_other.go index 788d034e01739..a0781d45335bb 100644 --- a/lib/daemonize/daemon_other.go +++ b/lib/daemonize/daemon_other.go @@ -1,8 +1,7 @@ -// Daemonization stub for non-Unix platforms (implementation) - //go:build windows || plan9 || js // +build windows plan9 js +// Package daemonize provides daemonization stub for non-Unix platforms. package daemonize import ( diff --git a/lib/daemonize/daemon_unix.go b/lib/daemonize/daemon_unix.go index beb89f7ac4605..1fcb2c55ce94a 100644 --- a/lib/daemonize/daemon_unix.go +++ b/lib/daemonize/daemon_unix.go @@ -1,8 +1,7 @@ -// Daemonization interface for Unix platforms (implementation) - //go:build !windows && !plan9 && !js // +build !windows,!plan9,!js +// Package daemonize provides daemonization interface for Unix platforms. package daemonize import ( diff --git a/lib/encoder/encoder.go b/lib/encoder/encoder.go index d50a2fcfe775f..a14d7c8485b42 100644 --- a/lib/encoder/encoder.go +++ b/lib/encoder/encoder.go @@ -1,15 +1,13 @@ -/* -Translate file names for usage on restrictive storage systems - -The restricted set of characters are mapped to a unicode equivalent version -(most to their FULLWIDTH variant) to increase compatibility with other -storage systems. -See: http://unicode-search.net/unicode-namesearch.pl?term=FULLWIDTH - -Encoders will also quote reserved characters to differentiate between -the raw and encoded forms. -*/ - +// Package encoder provides functionality to translate file names +// for usage on restrictive storage systems. +// +// The restricted set of characters are mapped to a unicode equivalent version +// (most to their FULLWIDTH variant) to increase compatibility with other +// storage systems. +// See: http://unicode-search.net/unicode-namesearch.pl?term=FULLWIDTH +// +// Encoders will also quote reserved characters to differentiate between +// the raw and encoded forms. package encoder import ( diff --git a/lib/encoder/filename/init.go b/lib/encoder/filename/init.go index 0cbc53c3cdc96..ea206cb9e0f8c 100644 --- a/lib/encoder/filename/init.go +++ b/lib/encoder/filename/init.go @@ -1,3 +1,4 @@ +// Package filename provides utilities for encoder. package filename import ( diff --git a/lib/encoder/internal/gen/main.go b/lib/encoder/internal/gen/main.go index a94c3793580df..97bcb32ba0f27 100644 --- a/lib/encoder/internal/gen/main.go +++ b/lib/encoder/internal/gen/main.go @@ -1,3 +1,4 @@ +// Package main provides utilities for encoder. package main import ( diff --git a/lib/errors/errors.go b/lib/errors/errors.go index 68238521eb429..ed848cbd69c73 100644 --- a/lib/errors/errors.go +++ b/lib/errors/errors.go @@ -1,3 +1,4 @@ +// Package errors provides error handling utilites. package errors import ( diff --git a/lib/file/preallocate_windows.go b/lib/file/preallocate_windows.go index 49352356af155..39955092e99b1 100644 --- a/lib/file/preallocate_windows.go +++ b/lib/file/preallocate_windows.go @@ -92,7 +92,7 @@ func PreAllocate(size int64, out *os.File) error { } const ( - FSCTL_SET_SPARSE = 0x000900c4 + FSCTL_SET_SPARSE = 0x000900c4 // Control code to set or clears the FILE_ATTRIBUTE_SPARSE_FILE attribute of a file. ) // SetSparseImplemented is a constant indicating whether the diff --git a/lib/http/auth/auth.go b/lib/http/auth/auth.go index e30ee9383f8cc..db50c8a303148 100644 --- a/lib/http/auth/auth.go +++ b/lib/http/auth/auth.go @@ -1,3 +1,4 @@ +// Package auth provides authentication for http. package auth import ( diff --git a/lib/jwtutil/jwtutil.go b/lib/jwtutil/jwtutil.go index af99ed3f80606..583036be92dbf 100644 --- a/lib/jwtutil/jwtutil.go +++ b/lib/jwtutil/jwtutil.go @@ -1,3 +1,4 @@ +// Package jwtutil provides JWT utilities. package jwtutil import ( diff --git a/lib/kv/bolt.go b/lib/kv/bolt.go index 5f9f3d7a6ab68..6fcfe49ddc994 100644 --- a/lib/kv/bolt.go +++ b/lib/kv/bolt.go @@ -1,6 +1,7 @@ //go:build !plan9 && !js // +build !plan9,!js +// Package kv provides key/value database. package kv import ( diff --git a/lib/mmap/mmap.go b/lib/mmap/mmap.go index 719bcf816d1ef..4f323c938c568 100644 --- a/lib/mmap/mmap.go +++ b/lib/mmap/mmap.go @@ -1,3 +1,4 @@ +// Package mmap provides memory mapped related utilities. package mmap import "os" diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go index 350fe1671aded..a7326e8866242 100644 --- a/lib/oauthutil/oauthutil.go +++ b/lib/oauthutil/oauthutil.go @@ -1,3 +1,4 @@ +// Package oauthutil provides OAuth utilities. package oauthutil import ( diff --git a/lib/readers/context.go b/lib/readers/context.go index 6af6efbaf6360..177b188094983 100644 --- a/lib/readers/context.go +++ b/lib/readers/context.go @@ -1,3 +1,4 @@ +// Package readers provides io.Reader related utilities. package readers import ( diff --git a/vfs/vfs.go b/vfs/vfs.go index b6dc141b3f854..77ccdf49998de 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -10,15 +10,14 @@ // may be referred to as "". However Stat strips slashes so you can // use paths with slashes in. // -// It also includes directory caching +// # It also includes directory caching // // The vfs package returns Error values to signal precisely which // error conditions have ocurred. It may also return general errors // it receives. It tries to use os Error values (e.g. os.ErrExist) // where possible. - +// //go:generate sh -c "go run make_open_tests.go | gofmt > open_test.go" - package vfs import ( diff --git a/vfs/vfscache/downloaders/downloaders.go b/vfs/vfscache/downloaders/downloaders.go index 46018c7e1988e..8d6e90fb2d7f2 100644 --- a/vfs/vfscache/downloaders/downloaders.go +++ b/vfs/vfscache/downloaders/downloaders.go @@ -1,3 +1,4 @@ +// Package downloaders provides utilities for the VFS layer package downloaders import ( diff --git a/vfs/vfscommon/cachemode.go b/vfs/vfscommon/cachemode.go index a2d91bd4ea38b..c82f3bc98808e 100644 --- a/vfs/vfscommon/cachemode.go +++ b/vfs/vfscommon/cachemode.go @@ -1,3 +1,4 @@ +// Package vfscommon provides utilities for VFS. package vfscommon import ( diff --git a/vfs/vfstest/vfs.go b/vfs/vfstest/vfs.go index 382fb8844617d..ad56fd42e572a 100644 --- a/vfs/vfstest/vfs.go +++ b/vfs/vfstest/vfs.go @@ -1,3 +1,4 @@ +// Package vfstest provides tests for VFS. package vfstest import ( From b5818454f7a5f147861230206bd16b5448669c85 Mon Sep 17 00:00:00 2001 From: YFdyh000 Date: Tue, 30 Aug 2022 16:23:29 +0800 Subject: [PATCH 254/560] onedrive: cleanup brand name --- backend/box/box.go | 2 +- backend/filefabric/filefabric.go | 2 +- backend/googlephotos/googlephotos.go | 2 +- backend/onedrive/api/types.go | 2 +- backend/onedrive/onedrive.go | 10 +++++----- backend/seafile/seafile.go | 2 +- backend/sugarsync/sugarsync.go | 2 +- backend/webdav/webdav.go | 2 +- backend/zoho/zoho.go | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/box/box.go b/backend/box/box.go index 93b2cd0ccb506..a6a33efdb1ea6 100644 --- a/backend/box/box.go +++ b/backend/box/box.go @@ -266,7 +266,7 @@ type Fs struct { root string // the path we are working on opt Options // parsed options features *fs.Features // optional features - srv *rest.Client // the connection to the one drive server + srv *rest.Client // the connection to the server dirCache *dircache.DirCache // Map of directory path to directory id pacer *fs.Pacer // pacer for API calls tokenRenewer *oauthutil.Renew // renew the token on expiry diff --git a/backend/filefabric/filefabric.go b/backend/filefabric/filefabric.go index fa3e7a178f226..ecfedbd0bacaf 100644 --- a/backend/filefabric/filefabric.go +++ b/backend/filefabric/filefabric.go @@ -150,7 +150,7 @@ type Fs struct { opt Options // parsed options features *fs.Features // optional features m configmap.Mapper // to save config - srv *rest.Client // the connection to the one drive server + srv *rest.Client // the connection to the server dirCache *dircache.DirCache // Map of directory path to directory id pacer *fs.Pacer // pacer for API calls tokenMu sync.Mutex // hold when reading the token diff --git a/backend/googlephotos/googlephotos.go b/backend/googlephotos/googlephotos.go index 238d6c783a3dc..c743ddec4a806 100644 --- a/backend/googlephotos/googlephotos.go +++ b/backend/googlephotos/googlephotos.go @@ -178,7 +178,7 @@ type Fs struct { opt Options // parsed options features *fs.Features // optional features unAuth *rest.Client // unauthenticated http client - srv *rest.Client // the connection to the one drive server + srv *rest.Client // the connection to the server ts *oauthutil.TokenSource // token source for oauth2 pacer *fs.Pacer // To pace the API calls startTime time.Time // time Fs was started - used for datestamps diff --git a/backend/onedrive/api/types.go b/backend/onedrive/api/types.go index bfde1016111c7..9d5cad6bae334 100644 --- a/backend/onedrive/api/types.go +++ b/backend/onedrive/api/types.go @@ -13,7 +13,7 @@ const ( PackageTypeOneNote = "oneNote" ) -// Error is returned from one drive when things go wrong +// Error is returned from OneDrive when things go wrong type Error struct { ErrorInfo struct { Code string `json:"code"` diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index c0674c9c596c5..29afd3976416f 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -600,14 +600,14 @@ type Options struct { Enc encoder.MultiEncoder `config:"encoding"` } -// Fs represents a remote one drive +// Fs represents a remote OneDrive type Fs struct { name string // name of this remote root string // the path we are working on opt Options // parsed options ci *fs.ConfigInfo // global config features *fs.Features // optional features - srv *rest.Client // the connection to the one drive server + srv *rest.Client // the connection to the OneDrive server dirCache *dircache.DirCache // Map of directory path to directory id pacer *fs.Pacer // pacer for API calls tokenRenewer *oauthutil.Renew // renew the token on expiry @@ -615,7 +615,7 @@ type Fs struct { driveType string // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive } -// Object describes a one drive object +// Object describes a OneDrive object // // Will definitely have info but maybe not meta type Object struct { @@ -645,7 +645,7 @@ func (f *Fs) Root() string { // String converts this Fs to a string func (f *Fs) String() string { - return fmt.Sprintf("One drive root '%s'", f.root) + return fmt.Sprintf("OneDrive root '%s'", f.root) } // Features returns the optional features of this Fs @@ -653,7 +653,7 @@ func (f *Fs) Features() *fs.Features { return f.features } -// parsePath parses a one drive 'url' +// parsePath parses a OneDrive 'url' func parsePath(path string) (root string) { root = strings.Trim(path, "/") return diff --git a/backend/seafile/seafile.go b/backend/seafile/seafile.go index 1413544ee031a..d1c2abdbcae15 100644 --- a/backend/seafile/seafile.go +++ b/backend/seafile/seafile.go @@ -137,7 +137,7 @@ type Fs struct { features *fs.Features // optional features endpoint *url.URL // URL of the host endpointURL string // endpoint as a string - srv *rest.Client // the connection to the one drive server + srv *rest.Client // the connection to the server pacer *fs.Pacer // pacer for API calls authMu sync.Mutex // Mutex to protect library decryption createDirMutex sync.Mutex // Protect creation of directories diff --git a/backend/sugarsync/sugarsync.go b/backend/sugarsync/sugarsync.go index 7078de975fdf0..1676c09492ea6 100644 --- a/backend/sugarsync/sugarsync.go +++ b/backend/sugarsync/sugarsync.go @@ -200,7 +200,7 @@ type Fs struct { root string // the path we are working on opt Options // parsed options features *fs.Features // optional features - srv *rest.Client // the connection to the one drive server + srv *rest.Client // the connection to the server dirCache *dircache.DirCache // Map of directory path to directory id pacer *fs.Pacer // pacer for API calls m configmap.Mapper // config file access diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go index ff5595f1b5159..8a815cef8187a 100644 --- a/backend/webdav/webdav.go +++ b/backend/webdav/webdav.go @@ -148,7 +148,7 @@ type Fs struct { features *fs.Features // optional features endpoint *url.URL // URL of the host endpointURL string // endpoint as a string - srv *rest.Client // the connection to the one drive server + srv *rest.Client // the connection to the server pacer *fs.Pacer // pacer for API calls precision time.Duration // mod time precision canStream bool // set if can stream diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.go index 167fa03ea7551..2af35daace23a 100644 --- a/backend/zoho/zoho.go +++ b/backend/zoho/zoho.go @@ -206,7 +206,7 @@ type Fs struct { root string // the path we are working on opt Options // parsed options features *fs.Features // optional features - srv *rest.Client // the connection to the one drive server + srv *rest.Client // the connection to the server dirCache *dircache.DirCache // Map of directory path to directory id pacer *fs.Pacer // pacer for API calls } From 2ea5b4f0b85b9a3231c3a0be184540458ca1d443 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Tue, 30 Aug 2022 10:26:03 +0200 Subject: [PATCH 255/560] Add YFdyh000 to contributors --- docs/content/authors.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/content/authors.md b/docs/content/authors.md index ea9e4ce9843a0..56d32925832c6 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -635,4 +635,5 @@ put them back in again.` >}} * João Henrique Franco * anonion * Ryan Morey <4590343+rmorey@users.noreply.github.com> - * Simon Bos \ No newline at end of file + * Simon Bos + * YFdyh000 \ No newline at end of file From 0008cb4934242757606b4c1305971fc51e7adef4 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Mon, 29 Aug 2022 14:32:00 +0200 Subject: [PATCH 256/560] docs: document that serve sftp uses chunk size 32 KiB See #6404 --- cmd/serve/sftp/sftp.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/cmd/serve/sftp/sftp.go b/cmd/serve/sftp/sftp.go index 1a7b7717c7aad..46c485d755c36 100644 --- a/cmd/serve/sftp/sftp.go +++ b/cmd/serve/sftp/sftp.go @@ -48,7 +48,7 @@ func AddFlags(flagSet *pflag.FlagSet, Opt *Options) { flags.StringVarP(flagSet, &Opt.User, "user", "", Opt.User, "User name for authentication") flags.StringVarP(flagSet, &Opt.Pass, "pass", "", Opt.Pass, "Password for authentication") flags.BoolVarP(flagSet, &Opt.NoAuth, "no-auth", "", Opt.NoAuth, "Allow connections with no authentication if set") - flags.BoolVarP(flagSet, &Opt.Stdio, "stdio", "", Opt.Stdio, "Run an sftp server on run stdin/stdout") + flags.BoolVarP(flagSet, &Opt.Stdio, "stdio", "", Opt.Stdio, "Run an sftp server on stdin/stdout") } func init() { @@ -61,11 +61,19 @@ func init() { var Command = &cobra.Command{ Use: "sftp remote:path", Short: `Serve the remote over SFTP.`, - Long: `Run a SFTP server to serve a remote over SFTP. This can be used -with an SFTP client or you can make a remote of type sftp to use with it. + Long: `Run an SFTP server to serve a remote over SFTP. This can be used +with an SFTP client or you can make a remote of type [sftp](/sftp) to use with it. -You can use the filter flags (e.g. ` + "`--include`, `--exclude`" + `) to control what -is served. +You can use the [filter](/filtering) flags (e.g. ` + "`--include`, `--exclude`" + `) +to control what is served. + +The server will respond to a small number of shell commands, mainly +md5sum, sha1sum and df, which enable it to provide support for checksums +and the about feature when accessed from an sftp remote. + +Note that this server uses fixed 32 KiB packet size, which means you must not +configure the client to use any other values, e.g. with the +[chunk_size](/sftp/#sftp-chunk-size) option on an sftp remote. The server will log errors. Use ` + "`-v`" + ` to see access logs. @@ -78,11 +86,6 @@ You must provide some means of authentication, either with ` + "`--auth-proxy`" + `, or set the ` + "`--no-auth`" + ` flag for no authentication when logging in. -Note that this also implements a small number of shell commands so -that it can provide md5sum/sha1sum/df information for the rclone sftp -backend. This means that is can support SHA1SUMs, MD5SUMs and the -about command when paired with the rclone sftp backend. - If you don't supply a host ` + "`--key`" + ` then rclone will generate rsa, ecdsa and ed25519 variants, and cache them for later use in rclone's cache directory (see ` + "`rclone help flags cache-dir`" + `) in the "serve-sftp" From ce3b65e6dc0866cac041b82aabbf790ebd9778b6 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sat, 13 Aug 2022 22:56:32 -0400 Subject: [PATCH 257/560] all: fix spelling across the project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * abcdefghijklmnopqrstuvwxyz * accounting * additional * allowed * almost * already * appropriately * arise * bandwidth * behave * bidirectional * brackets * cached * characters * cloud * committing * concatenating * configured * constructs * current * cutoff * deferred * different * directory * disposition * dropbox * either way * error * excess * experiments * explicitly * externally * files * github * gzipped * hierarchies * huffman * hyphen * implicitly * independent * insensitive * integrity * libraries * literally * metadata * mimics * missing * modification * multipart * multiple * nightmare * nonexistent * number * obscure * ourselves * overridden * potatoes * preexisting * priority * received * remote * replacement * represents * reproducibility * response * satisfies * sensitive * separately * separator * specifying * string * successful * synchronization * syncing * šenfeld * take * temporarily * testcontents * that * the * themselves * throttling * timeout * transaction * transferred * unnecessary * using * webbrowser * which * with * workspace Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- CONTRIBUTING.md | 4 +-- backend/b2/api/types.go | 2 +- backend/box/api/types.go | 2 +- backend/chunker/chunker.go | 4 +-- backend/combine/combine.go | 2 +- backend/compress/compress.go | 10 +++---- backend/crypt/crypt.go | 2 +- backend/drive/drive.go | 2 +- backend/dropbox/batcher.go | 2 +- backend/dropbox/dropbox.go | 4 +-- backend/fichier/structs.go | 2 +- backend/filefabric/api/types.go | 4 +-- backend/filefabric/filefabric.go | 2 +- .../googlecloudstorage/googlecloudstorage.go | 2 +- backend/hidrive/hidrive.go | 6 ++-- .../hidrive/hidrivehash/hidrivehash_test.go | 2 +- backend/internetarchive/internetarchive.go | 6 ++-- backend/jottacloud/jottacloud.go | 2 +- backend/koofr/koofr.go | 2 +- backend/local/local_internal_test.go | 2 +- backend/local/setbtime.go | 2 +- backend/local/setbtime_windows.go | 2 +- backend/mega/mega.go | 4 +-- backend/netstorage/netstorage.go | 6 ++-- backend/onedrive/api/types.go | 2 +- backend/pcloud/api/types.go | 2 +- backend/s3/s3.go | 4 +-- backend/s3/s3_internal_test.go | 4 +-- backend/swift/swift_internal_test.go | 4 +-- backend/uptobox/api/types.go | 2 +- backend/uptobox/uptobox.go | 8 +++--- backend/zoho/api/types.go | 2 +- backend/zoho/zoho.go | 6 ++-- cmd/bisync/deltas.go | 2 +- cmd/bisync/help.go | 2 +- cmd/cmount/mountpoint_windows.go | 2 +- cmd/config/config.go | 2 +- cmd/hashsum/hashsum.go | 2 +- cmd/help.go | 2 +- cmd/ls/lshelp/lshelp.go | 2 +- cmd/lsjson/lsjson.go | 2 +- cmd/md5sum/md5sum.go | 2 +- cmd/mountlib/help.go | 8 +++--- cmd/mountlib/mount.go | 2 +- cmd/mountlib/utils.go | 4 +-- cmd/ncdu/ncdu.go | 2 +- cmd/serve/docker/unix.go | 2 +- cmd/serve/ftp/ftp.go | 2 +- cmd/sha1sum/sha1sum.go | 2 +- cmd/touch/touch.go | 4 +-- cmdtest/environment_test.go | 2 +- docs/content/bisync.md | 8 +++--- docs/content/changelog.md | 28 +++++++++---------- docs/content/compress.md | 2 +- docs/content/crypt.md | 6 ++-- docs/content/docs.md | 6 ++-- docs/content/drive.md | 2 +- docs/content/dropbox.md | 2 +- docs/content/filefabric.md | 2 +- docs/content/flags.md | 2 +- docs/content/ftp.md | 2 +- docs/content/googlecloudstorage.md | 2 +- docs/content/hidrive.md | 2 +- docs/content/install.md | 4 +-- docs/content/jottacloud.md | 2 +- docs/content/mega.md | 2 +- docs/content/netstorage.md | 2 +- docs/content/onedrive.md | 2 +- docs/content/overview.md | 2 +- docs/content/rc.md | 4 +-- docs/content/s3.md | 2 +- docs/content/sftp.md | 8 +++--- fs/accounting/stats.go | 2 +- fs/cache/cache_test.go | 2 +- fs/config/configfile/configfile.go | 2 +- fs/config/configmap/configmap.go | 2 +- fs/dirtree/dirtree.go | 2 +- fs/newfs.go | 2 +- fs/operations/check_test.go | 4 +-- fs/operations/lsjson_test.go | 2 +- fs/operations/operations.go | 2 +- fs/rc/webgui/plugins.go | 4 +-- fs/rc/webgui/rc.go | 2 +- fs/registry.go | 4 +-- fstest/fstests/fstests.go | 10 +++---- fstest/run.go | 2 +- lib/cache/cache_test.go | 6 ++-- lib/dircache/dircache.go | 2 +- lib/jwtutil/jwtutil.go | 4 +-- lib/rest/rest.go | 2 +- librclone/librclone/librclone.go | 2 +- vfs/dir.go | 2 +- vfs/vfscache/cache_test.go | 4 +-- vfs/vfscache/item.go | 2 +- vfs/vfscache/writeback/writeback_test.go | 4 +-- 95 files changed, 160 insertions(+), 160 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f299a4691ba32..6f8ea2aface69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,7 +77,7 @@ Make sure you * Add [documentation](#writing-documentation) for a new feature. * [Commit your changes](#committing-your-changes) using the [message guideline](#commit-messages). -When you are done with that push your changes to Github: +When you are done with that push your changes to GitHub: git push -u origin my-new-feature @@ -88,7 +88,7 @@ Your changes will then get reviewed and you might get asked to fix some stuff. I You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master) or [squash your commits](#squashing-your-commits). -## Using Git and Github ## +## Using Git and GitHub ## ### Committing your changes ### diff --git a/backend/b2/api/types.go b/backend/b2/api/types.go index 5089db2425b39..4d981d950daec 100644 --- a/backend/b2/api/types.go +++ b/backend/b2/api/types.go @@ -239,7 +239,7 @@ type GetFileInfoRequest struct { // If the original source of the file being uploaded has a last // modified time concept, Backblaze recommends using // src_last_modified_millis as the name, and a string holding the base -// 10 number number of milliseconds since midnight, January 1, 1970 +// 10 number of milliseconds since midnight, January 1, 1970 // UTC. This fits in a 64 bit integer such as the type "long" in the // programming language Java. It is intended to be compatible with // Java's time long. For example, it can be passed directly into the diff --git a/backend/box/api/types.go b/backend/box/api/types.go index d3549f6aa7a63..6ff87761c25bc 100644 --- a/backend/box/api/types.go +++ b/backend/box/api/types.go @@ -14,7 +14,7 @@ const ( timeFormat = `"` + time.RFC3339 + `"` ) -// Time represents represents date and time information for the +// Time represents date and time information for the // box API, by using RFC3339 type Time time.Time diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.go index 4ed8ec9e87294..17da9393c20b7 100644 --- a/backend/chunker/chunker.go +++ b/backend/chunker/chunker.go @@ -64,7 +64,7 @@ import ( // length of 13 decimals it makes a 7-digit base-36 number. // // When transactions is set to the norename style, data chunks will -// keep their temporary chunk names (with the transacion identifier +// keep their temporary chunk names (with the transaction identifier // suffix). To distinguish them from temporary chunks, the txn field // of the metadata file is set to match the transaction identifier of // the data chunks. @@ -1079,7 +1079,7 @@ func (o *Object) readMetadata(ctx context.Context) error { // readXactID returns the transaction ID stored in the passed metadata object func (o *Object) readXactID(ctx context.Context) (xactID string, err error) { - // if xactID has already been read and cahced return it now + // if xactID has already been read and cached return it now if o.xIDCached { return o.xactID, nil } diff --git a/backend/combine/combine.go b/backend/combine/combine.go index dab80a82b5b30..34644a9681326 100644 --- a/backend/combine/combine.go +++ b/backend/combine/combine.go @@ -1,4 +1,4 @@ -// Package combine implents a backend to combine multipe remotes in a directory tree +// Package combine implents a backend to combine multiple remotes in a directory tree package combine /* diff --git a/backend/compress/compress.go b/backend/compress/compress.go index b53e725b8a8b8..2ae192eb9982e 100644 --- a/backend/compress/compress.go +++ b/backend/compress/compress.go @@ -90,7 +90,7 @@ Generally -1 (default, equivalent to 5) is recommended. Levels 1 to 9 increase compression at the cost of speed. Going past 6 generally offers very little return. -Level -2 uses Huffmann encoding only. Only use if you know what you +Level -2 uses Huffman encoding only. Only use if you know what you are doing. Level 0 turns off compression.`, Default: sgzip.DefaultCompression, @@ -130,7 +130,7 @@ type Fs struct { features *fs.Features // optional features } -// NewFs contstructs an Fs from the path, container:path +// NewFs constructs an Fs from the path, container:path func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, error) { // Parse config into Options struct opt := new(Options) @@ -451,7 +451,7 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...) } - // Need to include what we allready read + // Need to include what we already read in = &ReadCloserWrapper{ Reader: io.MultiReader(bytes.NewReader(buf), in), Closer: in, @@ -731,7 +731,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt } // If our new object is compressed we have to rename it with the correct size. - // Uncompressed objects don't store the size in the name so we they'll allready have the correct name. + // Uncompressed objects don't store the size in the name so we they'll already have the correct name. if compressible { wrapObj, err := operations.Move(ctx, f.Fs, nil, f.dataName(src.Remote(), newObj.size, compressible), newObj.Object) if err != nil { @@ -742,7 +742,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt return newObj, nil } -// Temporarely disabled. There might be a way to implement this correctly but with the current handling metadata duplicate objects +// Temporarily disabled. There might be a way to implement this correctly but with the current handling metadata duplicate objects // will break stuff. Right no I can't think of a way to make this work. // PutUnchecked uploads the object diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go index bae2c97999afb..85972298e9011 100644 --- a/backend/crypt/crypt.go +++ b/backend/crypt/crypt.go @@ -125,7 +125,7 @@ names, or for debugging purposes.`, This option could help with shortening the encrypted filename. The suitable option would depend on the way your remote count the filename -length and if it's case sensitve.`, +length and if it's case sensitive.`, Default: "base32", Examples: []fs.OptionExample{ { diff --git a/backend/drive/drive.go b/backend/drive/drive.go index b4f310eafb172..30c165b6149e4 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -3305,7 +3305,7 @@ drives found and a combined drive. upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:" Adding this to the rclone config file will cause those team drives to -be accessible with the aliases shown. Any illegal charactes will be +be accessible with the aliases shown. Any illegal characters will be substituted with "_" and duplicate names will have numbers suffixed. It will also add a remote called AllDrives which shows all the shared drives combined into one directory tree. diff --git a/backend/dropbox/batcher.go b/backend/dropbox/batcher.go index 874bb93d945b4..82a4b5524d76e 100644 --- a/backend/dropbox/batcher.go +++ b/backend/dropbox/batcher.go @@ -309,7 +309,7 @@ func (b *batcher) Shutdown() { } b.shutOnce.Do(func() { atexit.Unregister(b.atexit) - fs.Infof(b.f, "Commiting uploads - please wait...") + fs.Infof(b.f, "Committing uploads - please wait...") // show that batcher is shutting down close(b.closed) // quit the commitLoop by sending a quitRequest message diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 42686394d62ef..740bd76e34f1c 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -268,7 +268,7 @@ default based on the batch_mode in use. Advanced: true, }, { Name: "batch_commit_timeout", - Help: `Max time to wait for a batch to finish comitting`, + Help: `Max time to wait for a batch to finish committing`, Default: fs.Duration(10 * time.Minute), Advanced: true, }, { @@ -1669,7 +1669,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f correctOffset := uErr.EndpointError.IncorrectOffset.CorrectOffset delta := int64(correctOffset) - int64(cursor.Offset) skip += delta - what := fmt.Sprintf("incorrect offset error receved: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip) + what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip) if skip < 0 { return false, fmt.Errorf("can't seek backwards to correct offset: %s", what) } else if skip == chunkSize { diff --git a/backend/fichier/structs.go b/backend/fichier/structs.go index 89673f7f32af0..02e50a632715e 100644 --- a/backend/fichier/structs.go +++ b/backend/fichier/structs.go @@ -84,7 +84,7 @@ type CopyFileResponse struct { URLs []FileCopy `json:"urls"` } -// FileCopy is used in the the CopyFileResponse +// FileCopy is used in the CopyFileResponse type FileCopy struct { FromURL string `json:"from_url"` ToURL string `json:"to_url"` diff --git a/backend/filefabric/api/types.go b/backend/filefabric/api/types.go index 926ad82a6fd66..9f4fb59847747 100644 --- a/backend/filefabric/api/types.go +++ b/backend/filefabric/api/types.go @@ -19,7 +19,7 @@ const ( timeFormatJSON = `"` + timeFormatParameters + `"` ) -// Time represents represents date and time information for the +// Time represents date and time information for the // filefabric API type Time time.Time @@ -95,7 +95,7 @@ type Status struct { // Warning string `json:"warning"` // obsolete } -// Status statisfies the error interface +// Status satisfies the error interface func (e *Status) Error() string { return fmt.Sprintf("%s (%s)", e.Message, e.Code) } diff --git a/backend/filefabric/filefabric.go b/backend/filefabric/filefabric.go index ecfedbd0bacaf..7442814c51aab 100644 --- a/backend/filefabric/filefabric.go +++ b/backend/filefabric/filefabric.go @@ -843,7 +843,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { return f.purgeCheck(ctx, dir, false) } -// Wait for the the background task to complete if necessary +// Wait for the background task to complete if necessary func (f *Fs) waitForBackgroundTask(ctx context.Context, taskID api.String) (err error) { if taskID == "" || taskID == "0" { // No task to wait for diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go index 5ce96e672a46d..dc233bfcd147e 100644 --- a/backend/googlecloudstorage/googlecloudstorage.go +++ b/backend/googlecloudstorage/googlecloudstorage.go @@ -311,7 +311,7 @@ rclone does if you know the bucket exists already. Help: `If set this will decompress gzip encoded objects. It is possible to upload objects to GCS with "Content-Encoding: gzip" -set. Normally rclone will download these files files as compressed objects. +set. Normally rclone will download these files as compressed objects. If this flag is set then rclone will decompress these files with "Content-Encoding: gzip" as they are received. This means that rclone diff --git a/backend/hidrive/hidrive.go b/backend/hidrive/hidrive.go index b0f56844a8767..5004b81910a9e 100644 --- a/backend/hidrive/hidrive.go +++ b/backend/hidrive/hidrive.go @@ -330,7 +330,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, transaction) } - // Do not allow the root-prefix to be non-existent nor a directory, + // Do not allow the root-prefix to be nonexistent nor a directory, // but it can be empty. if f.opt.RootPrefix != "" { item, err := f.fetchMetadataForPath(ctx, f.opt.RootPrefix, api.HiDriveObjectNoMetadataFields) @@ -623,7 +623,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // should be retried after the parent-directories of the destination have been created. // If so, it will create the parent-directories. // -// If any errors arrise while finding the source or +// If any errors arise while finding the source or // creating the parent-directory those will be returned. // Otherwise returns the originalError. func (f *Fs) shouldRetryAndCreateParents(ctx context.Context, destinationPath string, sourcePath string, originalError error) (bool, error) { @@ -961,7 +961,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } else { _, _, err = o.fs.uploadFileChunked(ctx, resolvedPath, in, modTime, int(o.fs.opt.UploadChunkSize), o.fs.opt.UploadConcurrency) } - // Try to check if object was updated, eitherway. + // Try to check if object was updated, either way. // Metadata should be updated even if the upload fails. info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields) } else { diff --git a/backend/hidrive/hidrivehash/hidrivehash_test.go b/backend/hidrive/hidrivehash/hidrivehash_test.go index d27970c3462e1..07f2435b46b86 100644 --- a/backend/hidrive/hidrivehash/hidrivehash_test.go +++ b/backend/hidrive/hidrivehash/hidrivehash_test.go @@ -138,7 +138,7 @@ var testTable = []struct { // pattern describes how to use data to construct the hash-input. // For every entry n at even indices this repeats the data n times. // For every entry m at odd indices this repeats a null-byte m times. - // The input-data is constructed by concatinating the results in order. + // The input-data is constructed by concatenating the results in order. pattern []int64 out []byte name string diff --git a/backend/internetarchive/internetarchive.go b/backend/internetarchive/internetarchive.go index b3d4a55cb302d..4c1dc7f829a43 100644 --- a/backend/internetarchive/internetarchive.go +++ b/backend/internetarchive/internetarchive.go @@ -227,7 +227,7 @@ type Object struct { rawData json.RawMessage } -// IAFile reprensents a subset of object in MetadataResponse.Files +// IAFile represents a subset of object in MetadataResponse.Files type IAFile struct { Name string `json:"name"` // Source string `json:"source"` @@ -243,7 +243,7 @@ type IAFile struct { rawData json.RawMessage } -// MetadataResponse reprensents subset of the JSON object returned by (frontend)/metadata/ +// MetadataResponse represents subset of the JSON object returned by (frontend)/metadata/ type MetadataResponse struct { Files []IAFile `json:"files"` ItemSize int64 `json:"item_size"` @@ -1273,7 +1273,7 @@ func trimPathPrefix(s, prefix string, enc encoder.MultiEncoder) string { return enc.ToStandardPath(strings.TrimPrefix(s, prefix+"/")) } -// mimicks urllib.parse.quote() on Python; exclude / from url.PathEscape +// mimics urllib.parse.quote() on Python; exclude / from url.PathEscape func quotePath(s string) string { seg := strings.Split(s, "/") newValues := []string{} diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go index 00178745192ed..45e8489eab0ea 100644 --- a/backend/jottacloud/jottacloud.go +++ b/backend/jottacloud/jottacloud.go @@ -1418,7 +1418,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, } info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote) - // if destination was a trashed file then after a successfull copy the copied file is still in trash (bug in api?) + // if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?) if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" { fs.Debugf(src, "Server-side copied to trashed destination, restoring") info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5) diff --git a/backend/koofr/koofr.go b/backend/koofr/koofr.go index 642ab79fe8a2c..54513f0c1da48 100644 --- a/backend/koofr/koofr.go +++ b/backend/koofr/koofr.go @@ -668,7 +668,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, // // https://app.koofr.net/content/links/39a6cc01-3b23-477a-8059-c0fb3b0f15de/files/get?path=%2F // - // I am not sure about meaning of "path" parameter; in my expriments + // I am not sure about meaning of "path" parameter; in my experiments // it is always "%2F", and omitting it or putting any other value // results in 404. // diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go index 698d263a88415..811831d915b59 100644 --- a/backend/local/local_internal_test.go +++ b/backend/local/local_internal_test.go @@ -192,7 +192,7 @@ func TestHashOnUpdate(t *testing.T) { require.NoError(t, err) assert.Equal(t, "9a0364b9e99bb480dd25e1f0284c8555", md5) - // Reupload it with diferent contents but same size and timestamp + // Reupload it with different contents but same size and timestamp var b = bytes.NewBufferString("CONTENT") src := object.NewStaticObjectInfo(filePath, when, int64(b.Len()), true, nil, f) err = o.Update(ctx, b, src) diff --git a/backend/local/setbtime.go b/backend/local/setbtime.go index 20a914bf2a38f..5d2c462eefab8 100644 --- a/backend/local/setbtime.go +++ b/backend/local/setbtime.go @@ -9,7 +9,7 @@ import ( const haveSetBTime = false -// setBTime changes the the birth time of the file passed in +// setBTime changes the birth time of the file passed in func setBTime(name string, btime time.Time) error { // Does nothing return nil diff --git a/backend/local/setbtime_windows.go b/backend/local/setbtime_windows.go index 2b8fd98d5e1b1..adb9efa3a0c00 100644 --- a/backend/local/setbtime_windows.go +++ b/backend/local/setbtime_windows.go @@ -11,7 +11,7 @@ import ( const haveSetBTime = true -// setBTime sets the the birth time of the file passed in +// setBTime sets the birth time of the file passed in func setBTime(name string, btime time.Time) (err error) { h, err := syscall.Open(name, os.O_RDWR, 0755) if err != nil { diff --git a/backend/mega/mega.go b/backend/mega/mega.go index bb038506be834..92176c795f7ca 100644 --- a/backend/mega/mega.go +++ b/backend/mega/mega.go @@ -347,7 +347,7 @@ func (f *Fs) mkdir(ctx context.Context, rootNode *mega.Node, dir string) (node * } } if err != nil { - return nil, fmt.Errorf("internal error: mkdir called with non-existent root node: %w", err) + return nil, fmt.Errorf("internal error: mkdir called with nonexistent root node: %w", err) } // i is number of directories to create (may be 0) // node is directory to create them from @@ -387,7 +387,7 @@ func (f *Fs) findRoot(ctx context.Context, create bool) (*mega.Node, error) { return f._rootNode, nil } - // Check for pre-existing root + // Check for preexisting root absRoot := f.srv.FS.GetRoot() node, err := f.findDir(absRoot, f.root) //log.Printf("findRoot findDir %p %v", node, err) diff --git a/backend/netstorage/netstorage.go b/backend/netstorage/netstorage.go index 313679e272f8d..31adcbe7a4fab 100755 --- a/backend/netstorage/netstorage.go +++ b/backend/netstorage/netstorage.go @@ -118,7 +118,7 @@ type Fs struct { filetype string // dir, file or symlink dirscreated map[string]bool // if implicit dir has been created already dirscreatedMutex sync.Mutex // mutex to protect dirscreated - statcache map[string][]File // cache successfull stat requests + statcache map[string][]File // cache successful stat requests statcacheMutex sync.RWMutex // RWMutex to protect statcache } @@ -424,7 +424,7 @@ func (f *Fs) getFileName(file *File) string { func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { if f.filetype == "" { // This happens in two scenarios. - // 1. NewFs is done on a non-existent object, then later rclone attempts to List/ListR this NewFs. + // 1. NewFs is done on a nonexistent object, then later rclone attempts to List/ListR this NewFs. // 2. List/ListR is called from the context of test_all and not the regular rclone binary. err := f.initFs(ctx, dir) if err != nil { @@ -488,7 +488,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { if f.filetype == "" { // This happens in two scenarios. - // 1. NewFs is done on a non-existent object, then later rclone attempts to List/ListR this NewFs. + // 1. NewFs is done on a nonexistent object, then later rclone attempts to List/ListR this NewFs. // 2. List/ListR is called from the context of test_all and not the regular rclone binary. err := f.initFs(ctx, dir) if err != nil { diff --git a/backend/onedrive/api/types.go b/backend/onedrive/api/types.go index 9d5cad6bae334..ca56268589fd5 100644 --- a/backend/onedrive/api/types.go +++ b/backend/onedrive/api/types.go @@ -70,7 +70,7 @@ type Drive struct { Quota Quota `json:"quota"` } -// Timestamp represents represents date and time information for the +// Timestamp represents date and time information for the // OneDrive API, by using ISO 8601 and is always in UTC time. type Timestamp time.Time diff --git a/backend/pcloud/api/types.go b/backend/pcloud/api/types.go index 5ae2b6b6bb494..c1b5dc2176ff0 100644 --- a/backend/pcloud/api/types.go +++ b/backend/pcloud/api/types.go @@ -13,7 +13,7 @@ const ( timeFormat = `"` + time.RFC1123Z + `"` ) -// Time represents represents date and time information for the +// Time represents date and time information for the // pcloud API, by using RFC1123Z type Time time.Time diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 1f86b859297fd..7b3a81a294d40 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2009,7 +2009,7 @@ See [the time option docs](/docs/#time-option) for valid formats. Help: `If set this will decompress gzip encoded objects. It is possible to upload objects to S3 with "Content-Encoding: gzip" -set. Normally rclone will download these files files as compressed objects. +set. Normally rclone will download these files as compressed objects. If this flag is set then rclone will decompress these files with "Content-Encoding: gzip" as they are received. This means that rclone @@ -5199,7 +5199,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op var head s3.HeadObjectOutput //structs.SetFrom(&head, &req) setFrom_s3HeadObjectOutput_s3PutObjectInput(&head, &req) - head.ETag = &md5sumHex // doesn't matter quotes are misssing + head.ETag = &md5sumHex // doesn't matter quotes are missing head.ContentLength = &size // If we have done a single part PUT request then we can read these if gotEtag != "" { diff --git a/backend/s3/s3_internal_test.go b/backend/s3/s3_internal_test.go index be12cc42db80e..9d22ca69bef42 100644 --- a/backend/s3/s3_internal_test.go +++ b/backend/s3/s3_internal_test.go @@ -78,7 +78,7 @@ func (f *Fs) InternalTestMetadata(t *testing.T) { } t.Run("GzipEncoding", func(t *testing.T) { - // Test that the gziped file we uploaded can be + // Test that the gzipped file we uploaded can be // downloaded with and without decompression checkDownload := func(wantContents string, wantSize int64, wantHash string) { gotContents := fstests.ReadObject(ctx, t, o, -1) @@ -116,7 +116,7 @@ func (f *Fs) InternalTestNoHead(t *testing.T) { defer func() { assert.NoError(t, obj.Remove(ctx)) }() - // PutTestcontests checks the received object + // PutTestcontents checks the received object } diff --git a/backend/swift/swift_internal_test.go b/backend/swift/swift_internal_test.go index c8535508440db..49963fc61b9aa 100644 --- a/backend/swift/swift_internal_test.go +++ b/backend/swift/swift_internal_test.go @@ -16,8 +16,8 @@ func TestInternalUrlEncode(t *testing.T) { want string }{ {"", ""}, - {"abcdefghijklmopqrstuvwxyz", "abcdefghijklmopqrstuvwxyz"}, - {"ABCDEFGHIJKLMOPQRSTUVWXYZ", "ABCDEFGHIJKLMOPQRSTUVWXYZ"}, + {"abcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyz"}, + {"ABCDEFGHIJKLMNOPQRSTUVWXYZ", "ABCDEFGHIJKLMNOPQRSTUVWXYZ"}, {"0123456789", "0123456789"}, {"abc/ABC/123", "abc/ABC/123"}, {" ", "%20%20%20"}, diff --git a/backend/uptobox/api/types.go b/backend/uptobox/api/types.go index 4e842c8ab64b8..8cf1977546f44 100644 --- a/backend/uptobox/api/types.go +++ b/backend/uptobox/api/types.go @@ -80,7 +80,7 @@ type UploadInfo struct { } `json:"data"` } -// UploadResponse is the respnse to a successful upload +// UploadResponse is the response to a successful upload type UploadResponse struct { Files []struct { Name string `json:"name"` diff --git a/backend/uptobox/uptobox.go b/backend/uptobox/uptobox.go index fb3bbd7db5ce0..d403341bf8416 100644 --- a/backend/uptobox/uptobox.go +++ b/backend/uptobox/uptobox.go @@ -163,7 +163,7 @@ func (f *Fs) splitPathFull(pth string) (string, string) { return "//" + fullPath[:i], fullPath[i+1:] } -// splitPath is modified splitPath version that doesn't include the seperator +// splitPath is modified splitPath version that doesn't include the separator // in the base path func (f *Fs) splitPath(pth string) (string, string) { // chop of any leading or trailing '/' @@ -479,7 +479,7 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size } else if size == 0 { return nil, fs.ErrorCantUploadEmptyFiles } - // yes it does take take 4 requests if we're uploading to root and 6+ if we're uploading to any subdir :( + // yes it does take 4 requests if we're uploading to root and 6+ if we're uploading to any subdir :( // create upload request opts := rest.Opts{ @@ -757,7 +757,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string if err != nil { return fmt.Errorf("dirmove: source not found: %w", err) } - // check if the destination allready exists + // check if the destination already exists dstPath := f.dirPath(dstRemote) _, err = f.readMetaDataForPath(ctx, dstPath, &api.MetadataRequestOptions{Limit: 1}) if err == nil { @@ -782,7 +782,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string needMove := srcBase != dstBase // if we have to rename we'll have to use a temporary name since - // there could allready be a directory with the same name as the src directory + // there could already be a directory with the same name as the src directory if needRename { // rename to a temporary name tmpName := "rcloneTemp" + random.String(8) diff --git a/backend/zoho/api/types.go b/backend/zoho/api/types.go index 755c57b9be842..cb8b7bf44f231 100644 --- a/backend/zoho/api/types.go +++ b/backend/zoho/api/types.go @@ -6,7 +6,7 @@ import ( "time" ) -// Time represents represents date and time information for Zoho +// Time represents date and time information for Zoho // Zoho uses milliseconds since unix epoch (Java currentTimeMillis) type Time time.Time diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.go index 2af35daace23a..1e7556b03ac67 100644 --- a/backend/zoho/zoho.go +++ b/backend/zoho/zoho.go @@ -150,8 +150,8 @@ func init() { return workspace.ID, workspace.Attributes.Name }) case "workspace_end": - worksspaceID := config.Result - m.Set(configRootID, worksspaceID) + workspaceID := config.Result + m.Set(configRootID, workspaceID) return nil, nil } return nil, fmt.Errorf("unknown state %q", config.State) @@ -1264,7 +1264,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return err } - // upload was successfull, need to delete old object before rename + // upload was successful, need to delete old object before rename if err = o.Remove(ctx); err != nil { return fmt.Errorf("failed to remove old object: %w", err) } diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go index ee66980c49df9..d1dee1a1d1bf4 100644 --- a/cmd/bisync/deltas.go +++ b/cmd/bisync/deltas.go @@ -290,7 +290,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change return } -// exccessDeletes checks whether number of deletes is within allowed range +// excessDeletes checks whether number of deletes is within allowed range func (ds *deltaSet) excessDeletes() bool { maxDelete := ds.opt.MaxDelete maxRatio := float64(maxDelete) / 100.0 diff --git a/cmd/bisync/help.go b/cmd/bisync/help.go index 74f588489aa3c..14e6f78663d31 100644 --- a/cmd/bisync/help.go +++ b/cmd/bisync/help.go @@ -15,7 +15,7 @@ func makeHelp(help string) string { return replacer.Replace(help) } -var shortHelp = `Perform bidirectonal synchronization between two paths.` +var shortHelp = `Perform bidirectional synchronization between two paths.` var rcHelp = makeHelp(`This takes the following parameters diff --git a/cmd/cmount/mountpoint_windows.go b/cmd/cmount/mountpoint_windows.go index 5cd0aa1d06888..a1e511409b537 100644 --- a/cmd/cmount/mountpoint_windows.go +++ b/cmd/cmount/mountpoint_windows.go @@ -80,7 +80,7 @@ func handleDefaultMountpath() (string, error) { func handleNetworkShareMountpath(mountpath string, opt *mountlib.Options) (string, error) { // Assuming mount path is a valid network share path (UNC format, "\\Server\Share"). // Always mount as network drive, regardless of the NetworkMode option. - // Find an unused drive letter to use as mountpoint, the the supplied path can + // Find an unused drive letter to use as mountpoint, the supplied path can // be used as volume prefix (network share path) instead of mountpoint. if !opt.NetworkMode { fs.Debugf(nil, "Forcing --network-mode because mountpoint path is network share UNC format") diff --git a/cmd/config/config.go b/cmd/config/config.go index 318685c7d65b5..66b2a1c6db702 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -140,7 +140,7 @@ are 100% certain you are already passing obscured passwords then use |rclone config password| command. The flag |--non-interactive| is for use by applications that wish to -configure rclone themeselves, rather than using rclone's text based +configure rclone themselves, rather than using rclone's text based configuration questions. If this flag is set, and rclone needs to ask the user a question, a JSON blob will be returned with the question in it. diff --git a/cmd/hashsum/hashsum.go b/cmd/hashsum/hashsum.go index 5655fdcf24b24..4a1993716eb07 100644 --- a/cmd/hashsum/hashsum.go +++ b/cmd/hashsum/hashsum.go @@ -99,7 +99,7 @@ For the MD5 and SHA1 algorithms there are also dedicated commands, This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path -when there is data to read (if not, the hypen will be treated literaly, +when there is data to read (if not, the hyphen will be treated literally, as a relative path). Run without a hash to see the list of all supported hashes, e.g. diff --git a/cmd/help.go b/cmd/help.go index 0d1295d34050f..ecc34b63ea490 100644 --- a/cmd/help.go +++ b/cmd/help.go @@ -343,7 +343,7 @@ func showBackend(name string) { defaultValue := opt.GetValue() // Default value and Required are related: Required means option must // have a value, but if there is a default then a value does not have - // to be explicitely set and then Required makes no difference. + // to be explicitly set and then Required makes no difference. if defaultValue != "" { fmt.Printf("- Default: %s\n", quoteString(defaultValue)) } else { diff --git a/cmd/ls/lshelp/lshelp.go b/cmd/ls/lshelp/lshelp.go index 897c22f1c0488..3a98fff145f80 100644 --- a/cmd/ls/lshelp/lshelp.go +++ b/cmd/ls/lshelp/lshelp.go @@ -26,7 +26,7 @@ Note that |ls| and |lsl| recurse by default - use |--max-depth 1| to stop the re The other list commands |lsd|,|lsf|,|lsjson| do not recurse by default - use |-R| to make them recurse. -Listing a non-existent directory will produce an error except for +Listing a nonexistent directory will produce an error except for remotes which can't have empty directories (e.g. s3, swift, or gcs - the bucket-based remotes). `, "|", "`") diff --git a/cmd/lsjson/lsjson.go b/cmd/lsjson/lsjson.go index eb30f85784caf..a17ade32749a8 100644 --- a/cmd/lsjson/lsjson.go +++ b/cmd/lsjson/lsjson.go @@ -84,7 +84,7 @@ If ` + "`--files-only`" + ` is not specified directories in addition to the file will be returned. If ` + "`--metadata`" + ` is set then an additional Metadata key will be returned. -This will have metdata in rclone standard format as a JSON object. +This will have metadata in rclone standard format as a JSON object. if ` + "`--stat`" + ` is set then a single JSON blob will be returned about the item pointed to. This will return an error if the item isn't found. diff --git a/cmd/md5sum/md5sum.go b/cmd/md5sum/md5sum.go index 09da8526f19be..91bc66e4468f4 100644 --- a/cmd/md5sum/md5sum.go +++ b/cmd/md5sum/md5sum.go @@ -35,7 +35,7 @@ to running ` + "`rclone hashsum MD5 remote:path`" + `. This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path -when there is data to read (if not, the hypen will be treated literaly, +when there is data to read (if not, the hyphen will be treated literally, as a relative path). `, RunE: func(command *cobra.Command, args []string) error { diff --git a/cmd/mountlib/help.go b/cmd/mountlib/help.go index 10aea7b9fe1b3..7516157e81223 100644 --- a/cmd/mountlib/help.go +++ b/cmd/mountlib/help.go @@ -88,7 +88,7 @@ and experience unexpected program errors, freezes or other issues, consider moun as a network drive instead. When mounting as a fixed disk drive you can either mount to an unused drive letter, -or to a path representing a **non-existent** subdirectory of an **existing** parent +or to a path representing a **nonexistent** subdirectory of an **existing** parent directory or drive. Using the special value |*| will tell rclone to automatically assign the next available drive letter, starting with Z: and moving backward. Examples: @@ -119,7 +119,7 @@ the mapped drive, shown in Windows Explorer etc, while the complete |\\server\share| will be reported as the remote UNC path by |net use| etc, just like a normal network drive mapping. -If you specify a full network share UNC path with |--volname|, this will implicitely +If you specify a full network share UNC path with |--volname|, this will implicitly set the |--network-mode| option, so the following two examples have same result: rclone @ remote:path/to/files X: --network-mode @@ -128,7 +128,7 @@ set the |--network-mode| option, so the following two examples have same result: You may also specify the network share UNC path as the mountpoint itself. Then rclone will automatically assign a drive letter, same as with |*| and use that as mountpoint, and instead use the UNC path specified as the volume name, as if it were -specified with the |--volname| option. This will also implicitely set +specified with the |--volname| option. This will also implicitly set the |--network-mode| option. This means the following two examples have same result: rclone @ remote:path/to/files \\cloud\remote @@ -164,7 +164,7 @@ The permissions on each entry will be set according to [options](#options) The default permissions corresponds to |--file-perms 0666 --dir-perms 0777|, i.e. read and write permissions to everyone. This means you will not be able -to start any programs from the the mount. To be able to do that you must add +to start any programs from the mount. To be able to do that you must add execute permissions, e.g. |--file-perms 0777 --dir-perms 0777| to add it to everyone. If the program needs to write files, chances are you will have to enable [VFS File Caching](#vfs-file-caching) as well (see also [limitations](#limitations)). diff --git a/cmd/mountlib/mount.go b/cmd/mountlib/mount.go index ba3773e5c0a9b..907f16efdbe7d 100644 --- a/cmd/mountlib/mount.go +++ b/cmd/mountlib/mount.go @@ -238,7 +238,7 @@ func (m *MountPoint) Mount() (daemon *os.Process, err error) { return nil, err } - if err = m.CheckAllowings(); err != nil { + if err = m.CheckAllowed(); err != nil { return nil, err } m.SetVolumeName(m.MountOpt.VolumeName) diff --git a/cmd/mountlib/utils.go b/cmd/mountlib/utils.go index 1ec4d786a701f..099e3a7677fdc 100644 --- a/cmd/mountlib/utils.go +++ b/cmd/mountlib/utils.go @@ -62,9 +62,9 @@ func absPath(path string) string { return path } -// CheckAllowings informs about ignored flags on Windows. If not on Windows +// CheckAllowed informs about ignored flags on Windows. If not on Windows // and not --allow-non-empty flag is used, verify that mountpoint is empty. -func (m *MountPoint) CheckAllowings() error { +func (m *MountPoint) CheckAllowed() error { opt := &m.MountOpt if runtime.GOOS == "windows" { if opt.AllowNonEmpty { diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go index 02427fb494a40..d56f69bfc527e 100644 --- a/cmd/ncdu/ncdu.go +++ b/cmd/ncdu/ncdu.go @@ -48,7 +48,7 @@ press '?' to toggle the help on and off. The supported keys are: ` + strings.Join(helpText()[1:], "\n ") + ` Listed files/directories may be prefixed by a one-character flag, -some of them combined with a description in brackes at end of line. +some of them combined with a description in brackets at end of line. These flags have the following meaning: e means this is an empty directory, i.e. contains no files (but diff --git a/cmd/serve/docker/unix.go b/cmd/serve/docker/unix.go index 3c533d4543e37..8bf7b6fbc1155 100644 --- a/cmd/serve/docker/unix.go +++ b/cmd/serve/docker/unix.go @@ -25,7 +25,7 @@ func newUnixListener(path string, gid int) (net.Listener, string, error) { return nil, "", fmt.Errorf("expected only one socket from systemd, got %d", len(fds)) } - // create socket outselves + // create socket ourselves if filepath.Ext(path) == "" { path += ".sock" } diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go index 08cca77f31363..2023ba8514e9d 100644 --- a/cmd/serve/ftp/ftp.go +++ b/cmd/serve/ftp/ftp.go @@ -153,7 +153,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (*server, error) { } s.useTLS = s.opt.TLSKey != "" - // Check PassivePorts format since the the server library doesn't! + // Check PassivePorts format since the server library doesn't! if !passivePortsRe.MatchString(opt.PassivePorts) { return nil, fmt.Errorf("invalid format for passive ports %q", opt.PassivePorts) } diff --git a/cmd/sha1sum/sha1sum.go b/cmd/sha1sum/sha1sum.go index 25badd19c17ff..053ca0cbbf9e2 100644 --- a/cmd/sha1sum/sha1sum.go +++ b/cmd/sha1sum/sha1sum.go @@ -35,7 +35,7 @@ to running ` + "`rclone hashsum SHA1 remote:path`" + `. This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path -when there is data to read (if not, the hypen will be treated literaly, +when there is data to read (if not, the hyphen will be treated literally, as a relative path). This command can also hash data received on STDIN, if not passing diff --git a/cmd/touch/touch.go b/cmd/touch/touch.go index 9f3d69b141798..b0bea31b2c193 100644 --- a/cmd/touch/touch.go +++ b/cmd/touch/touch.go @@ -144,12 +144,12 @@ func Touch(ctx context.Context, f fs.Fs, remote string) error { return nil } if notCreateNewFile { - fs.Logf(f, "Not touching non-existent file due to --no-create") + fs.Logf(f, "Not touching nonexistent file due to --no-create") return nil } if recursive { // For consistency, --recursive never creates new files. - fs.Logf(f, "Not touching non-existent file due to --recursive") + fs.Logf(f, "Not touching nonexistent file due to --recursive") return nil } if operations.SkipDestructive(ctx, f, "touch (create)") { diff --git a/cmdtest/environment_test.go b/cmdtest/environment_test.go index 2124dc31c4c4f..b5b27eab56f42 100644 --- a/cmdtest/environment_test.go +++ b/cmdtest/environment_test.go @@ -81,7 +81,7 @@ func TestEnvironmentVariables(t *testing.T) { // Backend flags and remote name // - The listremotes command includes names from environment variables, // the part between "RCLONE_CONFIG_" and "_TYPE", converted to lowercase. - // - When using using a remote created from env, e.g. with lsd command, + // - When using a remote created from env, e.g. with lsd command, // the name is case insensitive in contrast to remotes in config file // (fs.ConfigToEnv converts to uppercase before checking environment). // - Previously using a remote created from env, e.g. with lsd command, diff --git a/docs/content/bisync.md b/docs/content/bisync.md index 68393d2a8a5e9..78bf06a0b267a 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -323,7 +323,7 @@ Most of these events come up due to a error status from an internal call. On such a critical error the `{...}.path1.lst` and `{...}.path2.lst` listing files are renamed to extension `.lst-err`, which blocks any future bisync runs (since the normal `.lst` files are not found). -Bisync keeps them under `bisync` subdirectory of the rclone cache direcory, +Bisync keeps them under `bisync` subdirectory of the rclone cache directory, typically at `${HOME}/.cache/rclone/bisync/` on Linux. Some errors are considered temporary and re-running the bisync is not blocked. @@ -421,7 +421,7 @@ don't have spelling case differences (`Smile.jpg` vs. `smile.jpg`). ## Windows support {#windows} Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on Windows -Github runners. +GitHub runners. Drive letters are allowed, including drive letters mapped to network drives (`rclone bisync J:\localsync GDrive:`). @@ -929,7 +929,7 @@ test command flags can be equally prefixed by a single `-` or double dash. synched tree even if there are check file mismatches in the test tree. - Some Dropbox tests can fail, notably printing the following message: `src and dst identical but can't set mod time without deleting and re-uploading` - This is expected and happens due a way Dropbox handles modificaion times. + This is expected and happens due a way Dropbox handles modification times. You should use the `-refresh-times` test flag to make up for this. - If Dropbox tests hit request limit for you and print error message `too_many_requests/...: Too many requests or write operations.` @@ -939,7 +939,7 @@ test command flags can be equally prefixed by a single `-` or double dash. ### Updating golden results Sometimes even a slight change in the bisync source can cause little changes -spread around many log files. Updating them manually would be a nighmare. +spread around many log files. Updating them manually would be a nightmare. The `-golden` flag will store the `test.log` and `*.lst` listings from each test case into respective golden directories. Golden results will diff --git a/docs/content/changelog.md b/docs/content/changelog.md index e4745b24dde50..0a6f3211918c5 100644 --- a/docs/content/changelog.md +++ b/docs/content/changelog.md @@ -14,7 +14,7 @@ description: "Rclone Changelog" * build: Fix android build after GitHub actions change (Nick Craig-Wood) * dlna: Fix SOAP action header parsing (Joram Schrijver) * docs: Fix links to mount command from install docs (albertony) - * dropox: Fix ChangeNotify was unable to decrypt errors (Nick Craig-Wood) + * dropbox: Fix ChangeNotify was unable to decrypt errors (Nick Craig-Wood) * fs: Fix parsing of times and durations of the form "YYYY-MM-DD HH:MM:SS" (Nick Craig-Wood) * serve sftp: Fix checksum detection (Nick Craig-Wood) * sync: Add accidentally missed filter-sensitivity to --backup-dir option (Nick Naumann) @@ -274,7 +274,7 @@ description: "Rclone Changelog" * build * Fix ARM architecture version in .deb packages after nfpm change (Nick Craig-Wood) * Hard fork `github.com/jlaffaye/ftp` to fix `go get github.com/rclone/rclone` (Nick Craig-Wood) - * oauthutil: Fix crash when webrowser requests `/robots.txt` (Nick Craig-Wood) + * oauthutil: Fix crash when webbrowser requests `/robots.txt` (Nick Craig-Wood) * operations: Fix goroutine leak in case of copy retry (Ankur Gupta) * rc: * Fix `operations/publiclink` default for `expires` parameter (Nick Craig-Wood) @@ -360,7 +360,7 @@ description: "Rclone Changelog" * Add rclone to list of supported `md5sum`/`sha1sum` commands to look for (albertony) * Refactor so we only have one way of running remote commands (Nick Craig-Wood) * Fix timeout on hashing large files by sending keepalives (Nick Craig-Wood) - * Fix unecessary seeking when uploading and downloading files (Nick Craig-Wood) + * Fix unnecessary seeking when uploading and downloading files (Nick Craig-Wood) * Update docs on how to create `known_hosts` file (Nick Craig-Wood) * Storj * Rename tardigrade backend to storj backend (Nick Craig-Wood) @@ -961,8 +961,8 @@ description: "Rclone Changelog" * Add sort by average size in directory (Adam Plánský) * Add toggle option for average s3ize in directory - key 'a' (Adam Plánský) * Add empty folder flag into ncdu browser (Adam Plánský) - * Add `!` (errror) and `.` (unreadable) file flags to go with `e` (empty) (Nick Craig-Wood) - * obscure: Make `rclone osbcure -` ignore newline at end of line (Nick Craig-Wood) + * Add `!` (error) and `.` (unreadable) file flags to go with `e` (empty) (Nick Craig-Wood) + * obscure: Make `rclone obscure -` ignore newline at end of line (Nick Craig-Wood) * operations * Add logs when need to upload files to set mod times (Nick Craig-Wood) * Move and copy log name of the destination object in verbose (Adam Plánský) @@ -987,7 +987,7 @@ description: "Rclone Changelog" * Make the error count match up in the log message (Nick Craig-Wood) * move: Fix data loss when source and destination are the same object (Nick Craig-Wood) * operations - * Fix `--cutof-mode` hard not cutting off immediately (Nick Craig-Wood) + * Fix `--cutoff-mode` hard not cutting off immediately (Nick Craig-Wood) * Fix `--immutable` error message (Nick Craig-Wood) * sync * Fix `--cutoff-mode` soft & cautious so it doesn't end the transfer early (Nick Craig-Wood) @@ -1035,7 +1035,7 @@ description: "Rclone Changelog" * Fixed crash on an empty file name (lluuaapp) * Box * Fix NewObject for files that differ in case (Nick Craig-Wood) - * Fix finding directories in a case insentive way (Nick Craig-Wood) + * Fix finding directories in a case insensitive way (Nick Craig-Wood) * Chunker * Skip long local hashing, hash in-transit (fixes) (Ivan Andreev) * Set Features ReadMimeType to false as Object.MimeType not supported (Nick Craig-Wood) @@ -1116,7 +1116,7 @@ description: "Rclone Changelog" * Implement `--sftp-use-fstat` for unusual SFTP servers (Nick Craig-Wood) * Sugarsync * Fix NewObject for files that differ in case (Nick Craig-Wood) - * Fix finding directories in a case insentive way (Nick Craig-Wood) + * Fix finding directories in a case insensitive way (Nick Craig-Wood) * Swift * Fix deletion of parts of Static Large Object (SLO) (Nguyễn Hữu Luân) * Ensure partially uploaded large files are uploaded unless `--swift-leave-parts-on-error` (Nguyễn Hữu Luân) @@ -1190,7 +1190,7 @@ description: "Rclone Changelog" [See commits](https://github.com/rclone/rclone/compare/v1.53.1...v1.53.2) * Bug Fixes - * acounting + * accounting * Fix incorrect speed and transferTime in core/stats (Nick Craig-Wood) * Stabilize display order of transfers on Windows (Nick Craig-Wood) * operations @@ -2160,7 +2160,7 @@ all the docs and Edward Barker for helping re-write the front page. * rcat: Fix slowdown on systems with multiple hashes (Nick Craig-Wood) * rcd: Fix permissions problems on cache directory with web gui download (Nick Craig-Wood) * Mount - * Default `--daemon-timout` to 15 minutes on macOS and FreeBSD (Nick Craig-Wood) + * Default `--daemon-timeout` to 15 minutes on macOS and FreeBSD (Nick Craig-Wood) * Update docs to show mounting from root OK for bucket-based (Nick Craig-Wood) * Remove nonseekable flag from write files (Nick Craig-Wood) * VFS @@ -2468,7 +2468,7 @@ all the docs and Edward Barker for helping re-write the front page. * Update google cloud storage endpoints (weetmuts) * HTTP * Add an example with username and password which is supported but wasn't documented (Nick Craig-Wood) - * Fix backend with `--files-from` and non-existent files (Nick Craig-Wood) + * Fix backend with `--files-from` and nonexistent files (Nick Craig-Wood) * Hubic * Make error message more informative if authentication fails (Nick Craig-Wood) * Jottacloud @@ -2952,7 +2952,7 @@ Point release to fix hubic and azureblob backends. * FTP * Work around strange response from box FTP server * More workarounds for FTP servers to fix mkParentDir error - * Fix no error on listing non-existent directory + * Fix no error on listing nonexistent directory * Google Cloud Storage * Add service_account_credentials (Matt Holt) * Detect bucket presence by listing it - minimises permissions needed @@ -3025,7 +3025,7 @@ Point release to fix hubic and azureblob backends. * Add .deb and .rpm packages as part of the build * Make a beta release for all branches on the main repo (but not pull requests) * Bug Fixes - * config: fixes errors on non existing config by loading config file only on first access + * config: fixes errors on nonexistent config by loading config file only on first access * config: retry saving the config after failure (Mateusz) * sync: when using `--backup-dir` don't delete files if we can't set their modtime * this fixes odd behaviour with Dropbox and `--backup-dir` @@ -3560,7 +3560,7 @@ Point release to fix hubic and azureblob backends. * Update B2 docs with Data usage, and Crypt section - thanks Tomasz Mazur * S3 * Command line and config file support for - * Setting/overriding ACL - thanks Radek Senfeld + * Setting/overriding ACL - thanks Radek Šenfeld * Setting storage class - thanks Asko Tamm * Drive * Make exponential backoff work exactly as per Google specification diff --git a/docs/content/compress.md b/docs/content/compress.md index 9a2d6b974ecbf..f9f3afbc043a1 100644 --- a/docs/content/compress.md +++ b/docs/content/compress.md @@ -129,7 +129,7 @@ Generally -1 (default, equivalent to 5) is recommended. Levels 1 to 9 increase compression at the cost of speed. Going past 6 generally offers very little return. -Level -2 uses Huffmann encoding only. Only use if you know what you +Level -2 uses Huffman encoding only. Only use if you know what you are doing. Level 0 turns off compression. diff --git a/docs/content/crypt.md b/docs/content/crypt.md index 92390bda39f1c..b0110f63533a5 100644 --- a/docs/content/crypt.md +++ b/docs/content/crypt.md @@ -241,7 +241,7 @@ the password configured for an existing crypt remote means you will no longer able to decrypt any of the previously encrypted content. The only possibility is to re-upload everything via a crypt remote configured with your new password. -Depending on the size of your data, your bandwith, storage quota etc, there are +Depending on the size of your data, your bandwidth, storage quota etc, there are different approaches you can take: - If you have everything in a different location, for example on your local system, you could remove all of the prior encrypted files, change the password for your @@ -254,7 +254,7 @@ effectively decrypting everything on the fly using the old password and re-encrypting using the new password. When done, delete the original crypt remote directory and finally the rclone crypt configuration with the old password. All data will be streamed from the storage system and back, so you will -get half the bandwith and be charged twice if you have upload and download quota +get half the bandwidth and be charged twice if you have upload and download quota on the storage system. **Note**: A security problem related to the random password generator @@ -567,7 +567,7 @@ How to encode the encrypted filename to text string. This option could help with shortening the encrypted filename. The suitable option would depend on the way your remote count the filename -length and if it's case sensitve. +length and if it's case sensitive. Properties: diff --git a/docs/content/docs.md b/docs/content/docs.md index c031a94bebdca..95609d7578910 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -498,7 +498,7 @@ backends can also store arbitrary user metadata. Where possible the key names are standardized, so, for example, it is possible to copy object metadata from s3 to azureblob for example and -metadata will be translated apropriately. +metadata will be translated appropriately. Some backends have limits on the size of the metadata and rclone will give errors on upload if they are exceeded. @@ -641,7 +641,7 @@ would mean limit the upload and download bandwidth to 10 MiB/s. single limit, specify the desired bandwidth in KiB/s, or use a suffix B|K|M|G|T|P. The default is `0` which means to not limit bandwidth. -The upload and download bandwidth can be specified seperately, as +The upload and download bandwidth can be specified separately, as `--bwlimit UP:DOWN`, so --bwlimit 10M:100k @@ -2011,7 +2011,7 @@ In all other cases the file will not be updated. Consider using the `--modify-window` flag to compensate for time skews between the source and the backend, for backends that do not support mod times, and instead use uploaded times. However, if the backend -does not support checksums, note that sync'ing or copying within the +does not support checksums, note that syncing or copying within the time skew window may still result in additional transfers for safety. ### --use-mmap ### diff --git a/docs/content/drive.md b/docs/content/drive.md index 7bb23ed595020..0277eba31be45 100644 --- a/docs/content/drive.md +++ b/docs/content/drive.md @@ -1335,7 +1335,7 @@ drives found and a combined drive. upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:" Adding this to the rclone config file will cause those team drives to -be accessible with the aliases shown. Any illegal charactes will be +be accessible with the aliases shown. Any illegal characters will be substituted with "_" and duplicate names will have numbers suffixed. It will also add a remote called AllDrives which shows all the shared drives combined into one directory tree. diff --git a/docs/content/dropbox.md b/docs/content/dropbox.md index 62b351c51789b..a8f4fe83f6ea2 100644 --- a/docs/content/dropbox.md +++ b/docs/content/dropbox.md @@ -409,7 +409,7 @@ Properties: #### --dropbox-batch-commit-timeout -Max time to wait for a batch to finish comitting +Max time to wait for a batch to finish committing Properties: diff --git a/docs/content/filefabric.md b/docs/content/filefabric.md index 225573ba576ea..d66b48162e164 100644 --- a/docs/content/filefabric.md +++ b/docs/content/filefabric.md @@ -13,7 +13,7 @@ through a global file system. ## Configuration The initial setup for the Enterprise File Fabric backend involves -getting a token from the the Enterprise File Fabric which you need to +getting a token from the Enterprise File Fabric which you need to do in your browser. `rclone config` walks you through it. Here is an example of how to make a remote called `remote`. First run: diff --git a/docs/content/flags.md b/docs/content/flags.md index cd9801851b9a5..2426a4a4ae5b7 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -313,7 +313,7 @@ and may be set in the config file. --drive-use-trash Send files to the trash instead of deleting permanently (default true) --drive-v2-download-min-size SizeSuffix If Object's are greater, use drive v2 API to download (default off) --dropbox-auth-url string Auth server URL - --dropbox-batch-commit-timeout Duration Max time to wait for a batch to finish comitting (default 10m0s) + --dropbox-batch-commit-timeout Duration Max time to wait for a batch to finish committing (default 10m0s) --dropbox-batch-mode string Upload file batching sync|async|off (default "sync") --dropbox-batch-size int Max number of files in upload batch --dropbox-batch-timeout Duration Max time to allow an idle upload batch before uploading (default 0s) diff --git a/docs/content/ftp.md b/docs/content/ftp.md index 7e95a0a94d5d1..fa29a682139e8 100644 --- a/docs/content/ftp.md +++ b/docs/content/ftp.md @@ -138,7 +138,7 @@ can be set with [`--ftp-port`](#ftp-port). In addition to the [default restricted characters set](/overview/#restricted-characters) the following characters are also replaced: -File names cannot end with the following characters. Repacement is +File names cannot end with the following characters. Replacement is limited to the last character in a file name: | Character | Value | Replacement | diff --git a/docs/content/googlecloudstorage.md b/docs/content/googlecloudstorage.md index 61b751d15d29e..e2309058539cb 100644 --- a/docs/content/googlecloudstorage.md +++ b/docs/content/googlecloudstorage.md @@ -607,7 +607,7 @@ Properties: If set this will decompress gzip encoded objects. It is possible to upload objects to GCS with "Content-Encoding: gzip" -set. Normally rclone will download these files files as compressed objects. +set. Normally rclone will download these files as compressed objects. If this flag is set then rclone will decompress these files with "Content-Encoding: gzip" as they are received. This means that rclone diff --git a/docs/content/hidrive.md b/docs/content/hidrive.md index 68375d1a22001..cd763cec3e851 100644 --- a/docs/content/hidrive.md +++ b/docs/content/hidrive.md @@ -124,7 +124,7 @@ the process is very similar to the process of initial setup exemplified before. HiDrive allows modification times to be set on objects accurate to 1 second. HiDrive supports [its own hash type](https://static.hidrive.com/dev/0001) -which is used to verify the integrety of file contents after successful transfers. +which is used to verify the integrity of file contents after successful transfers. ### Restricted filename characters diff --git a/docs/content/install.md b/docs/content/install.md index 714794f3d5344..f69b04fe70949 100644 --- a/docs/content/install.md +++ b/docs/content/install.md @@ -360,7 +360,7 @@ the system. Both scheduled task and Windows service can be used to achieve this. NOTE: Remember that when rclone runs as the `SYSTEM` user, the user profile that it sees will not be yours. This means that if you normally run rclone with configuration file in the default location, to be able to use the same configuration -when running as the system user you must explicitely tell rclone where to find +when running as the system user you must explicitly tell rclone where to find it with the [`--config`](https://rclone.org/docs/#config-config-file) option, or else it will look in the system users profile path (`C:\Windows\System32\config\systemprofile`). To test your command manually from a Command Prompt, you can run it with @@ -424,7 +424,7 @@ it should be possible through path rewriting as described [here](https://github. To Windows service running any rclone command, the excellent third-party utility [NSSM](http://nssm.cc), the "Non-Sucking Service Manager", can be used. -It includes some advanced features such as adjusting process periority, defining +It includes some advanced features such as adjusting process priority, defining process environment variables, redirect to file anything written to stdout, and customized response to different exit codes, with a GUI to configure everything from (although it can also be used from command line ). diff --git a/docs/content/jottacloud.md b/docs/content/jottacloud.md index 87b1e9e284916..4a61d2e78bd49 100644 --- a/docs/content/jottacloud.md +++ b/docs/content/jottacloud.md @@ -18,7 +18,7 @@ it also provides white-label solutions to different companies, such as: * Elgiganten Sweden (cloud.elgiganten.se) * Elgiganten Denmark (cloud.elgiganten.dk) * Giganti Cloud (cloud.gigantti.fi) - * ELKO Clouud (cloud.elko.is) + * ELKO Cloud (cloud.elko.is) Most of the white-label versions are supported by this backend, although may require different authentication setup - described below. diff --git a/docs/content/mega.md b/docs/content/mega.md index cd26c70108ece..7cc95adb62001 100644 --- a/docs/content/mega.md +++ b/docs/content/mega.md @@ -110,7 +110,7 @@ Use `rclone dedupe` to fix duplicated files. #### Object not found If you are connecting to your Mega remote for the first time, -to test access and syncronisation, you may receive an error such as +to test access and synchronization, you may receive an error such as ``` Failed to create file system for "my-mega-remote:": diff --git a/docs/content/netstorage.md b/docs/content/netstorage.md index 0c1bef5bdda4c..eab479c1ce462 100644 --- a/docs/content/netstorage.md +++ b/docs/content/netstorage.md @@ -152,7 +152,7 @@ Individual symlink files on the remote can be used with the commands like "cat" With NetStorage, directories can exist in one of two forms: 1. **Explicit Directory**. This is an actual, physical directory that you have created in a storage group. -2. **Implicit Directory**. This refers to a directory within a path that has not been physically created. For example, during upload of a file, non-existent subdirectories can be specified in the target path. NetStorage creates these as "implicit." While the directories aren't physically created, they exist implicitly and the noted path is connected with the uploaded file. +2. **Implicit Directory**. This refers to a directory within a path that has not been physically created. For example, during upload of a file, nonexistent subdirectories can be specified in the target path. NetStorage creates these as "implicit." While the directories aren't physically created, they exist implicitly and the noted path is connected with the uploaded file. Rclone will intercept all file uploads and mkdir commands for the NetStorage remote and will explicitly issue the mkdir command for each directory in the uploading path. This will help with the interoperability with the other Akamai services such as SFTP and the Content Management Shell (CMShell). Rclone will not guarantee correctness of operations with implicit directories which might have been created as a result of using an upload API directly. diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index b7a40ba787fd7..0f6897e34d02f 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -568,7 +568,7 @@ An official document about the limitations for different types of OneDrive can b ## Versions Every change in a file OneDrive causes the service to create a new -version of the the file. This counts against a users quota. For +version of the file. This counts against a users quota. For example changing the modification time of a file creates a second version, so the file apparently uses twice the space. diff --git a/docs/content/overview.md b/docs/content/overview.md index 2a9301acf45dc..e2054323876b0 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -111,7 +111,7 @@ systems they must support a common hash type. ### ModTime ### -Allmost all cloud storage systems store some sort of timestamp +Almost all cloud storage systems store some sort of timestamp on objects, but several of them not something that is appropriate to use for syncing. E.g. some backends will only write a timestamp that represent the time of the upload. To be relevant for syncing diff --git a/docs/content/rc.md b/docs/content/rc.md index fc6316e51974e..9d87e2e6530dc 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -397,7 +397,7 @@ The parameters can be a string as per the rest of rclone, eg `s3:bucket/path` or `:sftp:/my/dir`. They can also be specified as JSON blobs. -If specifyng a JSON blob it should be a object mapping strings to +If specifying a JSON blob it should be a object mapping strings to strings. These values will be used to configure the remote. There are 3 special values which may be set: @@ -1568,7 +1568,7 @@ check that parameter passing is working properly. **Authentication is required for this call.** -### sync/bisync: Perform bidirectonal synchronization between two paths. {#sync-bisync} +### sync/bisync: Perform bidirectional synchronization between two paths. {#sync-bisync} This takes the following parameters diff --git a/docs/content/s3.md b/docs/content/s3.md index dc5fd6d474ad5..d09dd35925c19 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -332,7 +332,7 @@ upload. Rclone's default directory traversal is to process each directory individually. This takes one API call per directory. Using the -`--fast-list` flag will read all info about the the objects into +`--fast-list` flag will read all info about the objects into memory first using a smaller number of API calls (one per 1000 objects). See the [rclone docs](/docs/#fast-list) for more details. diff --git a/docs/content/sftp.md b/docs/content/sftp.md index 8f3066a930504..48ebfb3a3af72 100644 --- a/docs/content/sftp.md +++ b/docs/content/sftp.md @@ -21,7 +21,7 @@ SSH installations. Paths are specified as `remote:path`. If the path does not begin with a `/` it is relative to the home directory of the user. An empty path `remote:` refers to the user's home directory. For example, `rclone lsd remote:` -would list the home directory of the user cofigured in the rclone remote config +would list the home directory of the user configured in the rclone remote config (`i.e /home/sftpuser`). However, `rclone lsd remote:/` would list the root directory for remote machine (i.e. `/`) @@ -264,7 +264,7 @@ can also run a SSH server, which is a port of OpenSSH (see official [installation guide](https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse)). On a Windows server the shell handling is different: Although it can also be set up to use a Unix type shell, e.g. Cygwin bash, the default is to use Windows Command Prompt (cmd.exe), and PowerShell is a recommended -alternative. All of these have bahave differently, which rclone must handle. +alternative. All of these have behave differently, which rclone must handle. Rclone tries to auto-detect what type of shell is used on the server, first time you access the SFTP remote. If a remote shell session is @@ -296,7 +296,7 @@ a new sftp remote is accessed. If you configure a sftp remote without a config file, e.g. an [on the fly](/docs/#backend-path-to-dir]) remote, rclone will have nowhere to store the result, and it will re-run the command on every access. To avoid this you should -explicitely set the `shell_type` option to the correct value, +explicitly set the `shell_type` option to the correct value, or to `none` if you want to prevent rclone from executing any remote shell commands. @@ -304,7 +304,7 @@ It is also important to note that, since the shell type decides how quoting and escaping of file paths used as command-line arguments are performed, configuring the wrong shell type may leave you exposed to command injection exploits. Make sure to confirm the auto-detected -shell type, or explicitely set the shell type you know is correct, +shell type, or explicitly set the shell type you know is correct, or disable shell access until you know. ### Checksum diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go index fc17f95ac23ca..3101ef993ba91 100644 --- a/fs/accounting/stats.go +++ b/fs/accounting/stats.go @@ -278,7 +278,7 @@ type transferStats struct { speed float64 } -// calculateTransferStats calculates some addtional transfer stats not +// calculateTransferStats calculates some additional transfer stats not // stored directly in StatsInfo func (s *StatsInfo) calculateTransferStats() (ts transferStats) { // checking and transferring have their own locking so read diff --git a/fs/cache/cache_test.go b/fs/cache/cache_test.go index c7620e25cd0e4..c03aa7dfaa098 100644 --- a/fs/cache/cache_test.go +++ b/fs/cache/cache_test.go @@ -154,7 +154,7 @@ func TestPin(t *testing.T) { cleanup, create := mockNewFs(t) defer cleanup() - // Test pinning and unpinning non-existent + // Test pinning and unpinning nonexistent f := mockfs.NewFs(context.Background(), "mock", "/alien") Pin(f) Unpin(f) diff --git a/fs/config/configfile/configfile.go b/fs/config/configfile/configfile.go index e9c498995f3a3..572ac15e1c723 100644 --- a/fs/config/configfile/configfile.go +++ b/fs/config/configfile/configfile.go @@ -40,7 +40,7 @@ func (s *Storage) check() { if err == nil { // check to see if config file has changed and if it has, reload it if s.fi == nil || !fi.ModTime().Equal(s.fi.ModTime()) || fi.Size() != s.fi.Size() { - fs.Debugf(nil, "Config file has changed externaly - reloading") + fs.Debugf(nil, "Config file has changed externally - reloading") err := s._load() if err != nil { fs.Errorf(nil, "Failed to read config file - using previous config: %v", err) diff --git a/fs/config/configmap/configmap.go b/fs/config/configmap/configmap.go index 516a919926fb3..ee8d4918fd45e 100644 --- a/fs/config/configmap/configmap.go +++ b/fs/config/configmap/configmap.go @@ -137,7 +137,7 @@ func (c Simple) Set(key, value string) { } // String the map value the same way the config parser does, but with -// sorted keys for reproducability. +// sorted keys for reproducibility. func (c Simple) String() string { var ks = make([]string, 0, len(c)) for k := range c { diff --git a/fs/dirtree/dirtree.go b/fs/dirtree/dirtree.go index 2785dcbed2ec8..6b4be8b6dce47 100644 --- a/fs/dirtree/dirtree.go +++ b/fs/dirtree/dirtree.go @@ -1,5 +1,5 @@ // Package dirtree contains the DirTree type which is used for -// building filesystem heirachies in memory. +// building filesystem hierarchies in memory. package dirtree import ( diff --git a/fs/newfs.go b/fs/newfs.go index f64cb2c8dfa75..adf8582fac22f 100644 --- a/fs/newfs.go +++ b/fs/newfs.go @@ -33,7 +33,7 @@ func NewFs(ctx context.Context, path string) (Fs, error) { overridden := fsInfo.Options.Overridden(config) if len(overridden) > 0 { extraConfig := overridden.String() - //Debugf(nil, "detected overriden config %q", extraConfig) + //Debugf(nil, "detected overridden config %q", extraConfig) md5sumBinary := md5.Sum([]byte(extraConfig)) suffix := base64.RawURLEncoding.EncodeToString(md5sumBinary[:]) // 5 characters length is 5*6 = 30 bits of base64 diff --git a/fs/operations/check_test.go b/fs/operations/check_test.go index 21692486d2ecc..be22051fe5fd4 100644 --- a/fs/operations/check_test.go +++ b/fs/operations/check_test.go @@ -186,11 +186,11 @@ func TestCheck(t *testing.T) { func TestCheckFsError(t *testing.T) { ctx := context.Background() - dstFs, err := fs.NewFs(ctx, "non-existent") + dstFs, err := fs.NewFs(ctx, "nonexistent") if err != nil { t.Fatal(err) } - srcFs, err := fs.NewFs(ctx, "non-existent") + srcFs, err := fs.NewFs(ctx, "nonexistent") if err != nil { t.Fatal(err) } diff --git a/fs/operations/lsjson_test.go b/fs/operations/lsjson_test.go index d5d7ca319c160..a8943cfbd0602 100644 --- a/fs/operations/lsjson_test.go +++ b/fs/operations/lsjson_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/require" ) -// Compare a and b in a file system idependent way +// Compare a and b in a file system independent way func compareListJSONItem(t *testing.T, a, b *operations.ListJSONItem, precision time.Duration) { assert.Equal(t, a.Path, b.Path, "Path") assert.Equal(t, a.Name, b.Name, "Name") diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 2bfddd07d3c7e..59de5aac0c6c9 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1787,7 +1787,7 @@ func copyURLFn(ctx context.Context, dstFileName string, url string, autoFilename _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) headerFilename := path.Base(strings.Replace(params["filename"], "\\", "/", -1)) if err != nil || headerFilename == "" { - return fmt.Errorf("CopyURL failed: filename not found in the Content-Dispoition header") + return fmt.Errorf("CopyURL failed: filename not found in the Content-Disposition header") } fs.Debugf(headerFilename, "filename found in Content-Disposition header.") return fn(ctx, headerFilename, resp.Body, resp.ContentLength, modTime) diff --git a/fs/rc/webgui/plugins.go b/fs/rc/webgui/plugins.go index 23f9692c2fbc6..859e6d9991df8 100644 --- a/fs/rc/webgui/plugins.go +++ b/fs/rc/webgui/plugins.go @@ -227,14 +227,14 @@ func (p *Plugins) GetPluginByName(name string) (out *PackageJSON, err error) { } -// getAuthorRepoBranchGithub gives author, repoName and branch from a github.com url +// getAuthorRepoBranchGitHub gives author, repoName and branch from a github.com url // // url examples: // https://github.com/rclone/rclone-webui-react/ // http://github.com/rclone/rclone-webui-react // https://github.com/rclone/rclone-webui-react/tree/caman-js // github.com/rclone/rclone-webui-react -func getAuthorRepoBranchGithub(url string) (author string, repoName string, branch string, err error) { +func getAuthorRepoBranchGitHub(url string) (author string, repoName string, branch string, err error) { repoURL := url repoURL = strings.Replace(repoURL, "https://", "", 1) repoURL = strings.Replace(repoURL, "http://", "", 1) diff --git a/fs/rc/webgui/rc.go b/fs/rc/webgui/rc.go index 8c54040e3a8ef..c3bada21f750d 100644 --- a/fs/rc/webgui/rc.go +++ b/fs/rc/webgui/rc.go @@ -102,7 +102,7 @@ func rcAddPlugin(_ context.Context, in rc.Params) (out rc.Params, err error) { return nil, err } - author, repoName, repoBranch, err := getAuthorRepoBranchGithub(pluginURL) + author, repoName, repoBranch, err := getAuthorRepoBranchGitHub(pluginURL) if err != nil { return nil, err } diff --git a/fs/registry.go b/fs/registry.go index 33dba915b6aa1..adf891ad2efe7 100644 --- a/fs/registry.go +++ b/fs/registry.go @@ -28,7 +28,7 @@ type RegInfo struct { // Prefix for command line flags for this fs - defaults to Name if not set Prefix string // Create a new file system. If root refers to an existing - // object, then it should return an Fs which which points to + // object, then it should return an Fs which points to // the parent of that object and ErrorIsFile. NewFs func(ctx context.Context, name string, root string, config configmap.Mapper) (Fs, error) `json:"-"` // Function to call to help with config - see docs for ConfigIn for more info @@ -179,7 +179,7 @@ func (o *Option) MarshalJSON() ([]byte, error) { }) } -// GetValue gets the current current value which is the default if not set +// GetValue gets the current value which is the default if not set func (o *Option) GetValue() interface{} { val := o.Value if val == nil { diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 28e02a44b2cc6..3b9767eaf0f91 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -531,14 +531,14 @@ func Run(t *testing.T, opt *Opt) { assert.True(t, len(fsInfo.CommandHelp) > 0, "Command is declared, must return some help in CommandHelp") }) - // TestFsRmdirNotFound tests deleting a non-existent directory + // TestFsRmdirNotFound tests deleting a nonexistent directory t.Run("FsRmdirNotFound", func(t *testing.T) { skipIfNotOk(t) if isBucketBasedButNotRoot(f) { t.Skip("Skipping test as non root bucket-based remote") } err := f.Rmdir(ctx, "") - assert.Error(t, err, "Expecting error on Rmdir non-existent") + assert.Error(t, err, "Expecting error on Rmdir nonexistent") }) // Make the directory @@ -729,7 +729,7 @@ func Run(t *testing.T, opt *Opt) { o, err := f.NewObject(ctx, "potato") assert.Nil(t, o) assert.Equal(t, fs.ErrorObjectNotFound, err) - // Now try an object in a non existing directory + // Now try an object in a nonexistent directory o, err = f.NewObject(ctx, "directory/not/found/potato") assert.Nil(t, o) assert.Equal(t, fs.ErrorObjectNotFound, err) @@ -1632,7 +1632,7 @@ func Run(t *testing.T, opt *Opt) { fstest.CheckListingWithRoot(t, rootRemote, configLeaf, []fstest.Item{file1Root, file2Root}, dirs, rootRemote.Precision()) }) - // Check that that listing the entries is OK + // Check that listing the entries is OK t.Run("ListEntries", func(t *testing.T) { entries, err := rootRemote.List(context.Background(), configLeaf) require.NoError(t, err) @@ -2068,7 +2068,7 @@ func Run(t *testing.T, opt *Opt) { // TestFsRootCollapse tests if the root of an fs "collapses" to the // absolute root. It creates a new fs of the same backend type with its - // root set to a *non-existent* folder, and attempts to read the info of + // root set to a *nonexistent* folder, and attempts to read the info of // an object in that folder, whose name is taken from a directory that // exists in the absolute root. // This test is added after diff --git a/fstest/run.go b/fstest/run.go index a9b8b0a46f182..8751b3d23ef23 100644 --- a/fstest/run.go +++ b/fstest/run.go @@ -17,7 +17,7 @@ func TestMkdir(t *testing.T) { // test stuff } -This will make r.Fremote and r.Flocal for a remote remote and a local +This will make r.Fremote and r.Flocal for a remote and a local remote. The remote is determined by the -remote flag passed in. */ diff --git a/lib/cache/cache_test.go b/lib/cache/cache_test.go index 415e167c3f1c1..71e1a4dbb3cea 100644 --- a/lib/cache/cache_test.go +++ b/lib/cache/cache_test.go @@ -158,7 +158,7 @@ func TestCachePin(t *testing.T) { _, err := c.Get("/", create) require.NoError(t, err) - // Pin a non-existent item to show nothing happens + // Pin a nonexistent item to show nothing happens c.Pin("notfound") c.mu.Lock() @@ -312,7 +312,7 @@ func TestCacheRename(t *testing.T) { assert.Equal(t, 2, c.Entries()) - // rename to non-existent + // rename to nonexistent value, found := c.Rename("existing1", "EXISTING1") assert.Equal(t, true, found) assert.Equal(t, existing1, value) @@ -326,7 +326,7 @@ func TestCacheRename(t *testing.T) { assert.Equal(t, 1, c.Entries()) - // rename non-existent + // rename nonexistent value, found = c.Rename("notfound", "NOTFOUND") assert.Equal(t, false, found) assert.Nil(t, value) diff --git a/lib/dircache/dircache.go b/lib/dircache/dircache.go index 920215e08b629..8076e7c8f72f0 100644 --- a/lib/dircache/dircache.go +++ b/lib/dircache/dircache.go @@ -140,7 +140,7 @@ func (dc *DirCache) SetRootIDAlias(rootID string) { dc.Put("", dc.rootID) } -// FlushDir flushes the map of all data starting with with the path +// FlushDir flushes the map of all data starting with the path // dir. // // If dir is empty string then this is equivalent to calling ResetRoot diff --git a/lib/jwtutil/jwtutil.go b/lib/jwtutil/jwtutil.go index 583036be92dbf..dfb4dc33be799 100644 --- a/lib/jwtutil/jwtutil.go +++ b/lib/jwtutil/jwtutil.go @@ -70,8 +70,8 @@ func Config(id, name string, claims *jws.ClaimSet, header *jws.Header, queryPara return fmt.Errorf("jwtutil: failed making auth request: %w", err) } defer func() { - deferedErr := resp.Body.Close() - if deferedErr != nil { + deferredErr := resp.Body.Close() + if deferredErr != nil { err = fmt.Errorf("jwtutil: failed to close resp.Body: %w", err) } }() diff --git a/lib/rest/rest.go b/lib/rest/rest.go index 69bb1deff7ede..a424b9d3eeb27 100644 --- a/lib/rest/rest.go +++ b/lib/rest/rest.go @@ -419,7 +419,7 @@ func MultipartUpload(ctx context.Context, in io.Reader, params url.Values, conte // opts.Body are set then CallJSON will do a multipart upload with a // file attached. opts.MultipartContentName is the name of the // parameter and opts.MultipartFileName is the name of the file. If -// MultpartContentName is set, and request != nil is supplied, then +// MultipartContentName is set, and request != nil is supplied, then // the request will be marshalled into JSON and added to the form with // parameter name MultipartMetadataName. // diff --git a/librclone/librclone/librclone.go b/librclone/librclone/librclone.go index 5b29e4c5975de..1e7dd6e1187a4 100644 --- a/librclone/librclone/librclone.go +++ b/librclone/librclone/librclone.go @@ -1,7 +1,7 @@ // Package librclone exports shims for library use // // This is the internal implementation which is used for C and -// Gomobile libaries which need slightly different export styles. +// Gomobile libraries which need slightly different export styles. // // The shims are a thin wrapper over the rclone RPC. package librclone diff --git a/vfs/dir.go b/vfs/dir.go index 7f68305821fa2..c45033d0eb9cd 100644 --- a/vfs/dir.go +++ b/vfs/dir.go @@ -186,7 +186,7 @@ func (d *Dir) ForgetAll() (hasVirtual bool) { } } } - // Purge any unecessary virtual entries + // Purge any unnecessary virtual entries d._purgeVirtual() d.read = time.Time{} diff --git a/vfs/vfscache/cache_test.go b/vfs/vfscache/cache_test.go index 6755dcee48072..bd08736a090e2 100644 --- a/vfs/vfscache/cache_test.go +++ b/vfs/vfscache/cache_test.go @@ -450,7 +450,7 @@ func TestCachePurgeClean(t *testing.T) { _, err = os.Stat(potato1.c.toOSPath(potato1.name)) require.NoError(t, err) - // Add some potatos + // Add some potatoes potato2 := c.Item("sub/dir/potato2") require.NoError(t, potato2.Open(nil)) require.NoError(t, potato2.Truncate(5)) @@ -603,7 +603,7 @@ func TestCacheRename(t *testing.T) { assertPathNotExist(t, osPathMeta) assert.False(t, c.Exists("sub/newPotato")) - // non-existent file - is ignored + // nonexistent file - is ignored assert.NoError(t, c.Rename("nonexist", "nonexist2", nil)) } diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go index 36355d98133d2..b7f40252d9a5b 100644 --- a/vfs/vfscache/item.go +++ b/vfs/vfscache/item.go @@ -1152,7 +1152,7 @@ func (item *Item) _ensure(offset, size int64) (err error) { // This is called by the downloader downloading file segments and the // vfs layer writing to the file. // -// This doesn't mark the item as Dirty - that the the responsibility +// This doesn't mark the item as Dirty - that the responsibility // of the caller as we don't know here whether we are adding reads or // writes to the cache file. // diff --git a/vfs/vfscache/writeback/writeback_test.go b/vfs/vfscache/writeback/writeback_test.go index 512d0eeb70d0a..cfc7f4fa96cfe 100644 --- a/vfs/vfscache/writeback/writeback_test.go +++ b/vfs/vfscache/writeback/writeback_test.go @@ -408,7 +408,7 @@ func TestWriteBackAddUpdateNotModified(t *testing.T) { pi2 := newPutItem(t) id2 := wb.Add(id, "one", false, pi2.put) assert.Equal(t, id, id2) - checkNotOnHeap(t, wb, wbItem) // object still being transfered + checkNotOnHeap(t, wb, wbItem) // object still being transferred checkInLookup(t, wb, wbItem) // Because modified was false above this should not cancel the @@ -525,7 +525,7 @@ func TestWriteBackMaxQueue(t *testing.T) { assert.Equal(t, toTransfer-maxTransfers, queued) assert.Equal(t, maxTransfers, inProgress) - // now finish the the first maxTransfers + // now finish the first maxTransfers for i := 0; i < maxTransfers; i++ { pis[i].finish(nil) } From 35349657cd2d5708b2b02a8f8810270c4c999abb Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Tue, 30 Aug 2022 19:51:00 +0200 Subject: [PATCH 258/560] docs/sftp: document use of chunk_size option in sftp remote paired with serve sftp Related to 0008cb4934242757606b4c1305971fc51e7adef4 --- backend/sftp/sftp.go | 31 ++++++++++++++++++------------- cmd/serve/sftp/sftp.go | 6 +++--- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 1c160996cc7e7..37fcbdcb2f51e 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -275,19 +275,24 @@ Set to 0 to keep connections indefinitely. Name: "chunk_size", Help: `Upload and download chunk size. -This controls the maximum packet size used in the SFTP protocol. The -RFC limits this to 32768 bytes (32k), however a lot of servers -support larger sizes and setting it larger will increase transfer -speed dramatically on high latency links. - -Only use a setting higher than 32k if you always connect to the same -server or after sufficiently broad testing. - -For example using the value of 252k with OpenSSH works well with its -maximum packet size of 256k. - -If you get the error "failed to send packet header: EOF" when copying -a large file, try lowering this number. +This controls the maximum size of payload in SFTP protocol packets. +The RFC limits this to 32768 bytes (32k), which is the default. However, +a lot of servers support larger sizes, typically limited to a maximum +total package size of 256k, and setting it larger will increase transfer +speed dramatically on high latency links. This includes OpenSSH, and, +for example, using the value of 255k works well, leaving plenty of room +for overhead while still being within a total packet size of 256k. + +Make sure to test thoroughly before using a value higher than 32k, +and only use it if you always connect to the same server or after +sufficiently broad testing. If you get errors such as +"failed to send packet payload: EOF", lots of "connection lost", +or "corrupted on transfer", when copying a larger file, try lowering +the value. The server run by [rclone serve sftp](/commands/rclone_serve_sftp) +sends packets with standard 32k maximum payload so you must not +set a different chunk_size when downloading files, but it accepts +packets up to the 256k total size, so for uploads the chunk_size +can be set as for the OpenSSH example above. `, Default: 32 * fs.Kibi, Advanced: true, diff --git a/cmd/serve/sftp/sftp.go b/cmd/serve/sftp/sftp.go index 46c485d755c36..8244435ee2865 100644 --- a/cmd/serve/sftp/sftp.go +++ b/cmd/serve/sftp/sftp.go @@ -71,9 +71,9 @@ The server will respond to a small number of shell commands, mainly md5sum, sha1sum and df, which enable it to provide support for checksums and the about feature when accessed from an sftp remote. -Note that this server uses fixed 32 KiB packet size, which means you must not -configure the client to use any other values, e.g. with the -[chunk_size](/sftp/#sftp-chunk-size) option on an sftp remote. +Note that this server uses standard 32 KiB packet payload size, which +means you must not configure the client to expect anything else, e.g. +with the [chunk_size](/sftp/#sftp-chunk-size) option on an sftp remote. The server will log errors. Use ` + "`-v`" + ` to see access logs. From f279e4ab01235d1d62bf5b19bd6cccea344916f7 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 5 Sep 2022 12:10:59 +0100 Subject: [PATCH 259/560] Add Josh Soref to contributors --- docs/content/authors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/authors.md b/docs/content/authors.md index 56d32925832c6..a0ffd369022cd 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -636,4 +636,4 @@ put them back in again.` >}} * anonion * Ryan Morey <4590343+rmorey@users.noreply.github.com> * Simon Bos - * YFdyh000 \ No newline at end of file + * YFdyh000 * Josh Soref <2119212+jsoref@users.noreply.github.com> From d08ed7d1e93e2b40d2de03ee73081a61b26031bd Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 11 Aug 2022 16:45:21 +0100 Subject: [PATCH 260/560] ftp: add notes on how to avoid deadlocks with concurrency - fixes #6370 --- backend/ftp/ftp.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index 24916d1613dac..c5c885e00fb72 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -81,8 +81,22 @@ security from the server in order to upgrade a plain text connection to an encrypted one. Cannot be used in combination with implicit FTP.`, Default: false, }, { - Name: "concurrency", - Help: "Maximum number of FTP simultaneous connections, 0 for unlimited.", + Name: "concurrency", + Help: strings.Replace(`Maximum number of FTP simultaneous connections, 0 for unlimited. + +Note that setting this is very likely to cause deadlocks so it should +be used with care. + +If you are doing a sync or copy then make sure concurrency is one more +than the sum of |--transfers| and |--checkers|. + +If you use |--check-first| then it just needs to be one more than the +maximum of |--checkers| and |--transfers|. + +So for |concurrency 3| you'd use |--checkers 2 --transfers 2 +--check-first| or |--checkers 1 --transfers 1|. + +`, "|", "`", -1), Default: 0, Advanced: true, }, { From 3cb7734eac78b562f9ef1a572f586e5e5198c45c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 13 Aug 2022 11:59:42 +0100 Subject: [PATCH 261/560] config: move locking to fix fatal error: concurrent map read and map write Before this change we assumed that github.com/Unknwon/goconfig was threadsafe as documented. However it turns out it is not threadsafe and looking at the code it appears that making it threadsafe might be quite hard. So this change increases the lock coverage in configfile to cover the goconfig uses also. Fixes #6378 --- fs/config/configfile/configfile.go | 49 ++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/fs/config/configfile/configfile.go b/fs/config/configfile/configfile.go index 572ac15e1c723..c0bef0389d73a 100644 --- a/fs/config/configfile/configfile.go +++ b/fs/config/configfile/configfile.go @@ -24,16 +24,15 @@ func Install() { // Storage implements config.Storage for saving and loading config // data in a simple INI based file. type Storage struct { - gc *goconfig.ConfigFile // config file loaded - thread safe mu sync.Mutex // to protect the following variables + gc *goconfig.ConfigFile // config file loaded - not thread safe fi os.FileInfo // stat of the file when last loaded } // Check to see if we need to reload the config -func (s *Storage) check() { - s.mu.Lock() - defer s.mu.Unlock() - +// +// mu must be held when calling this +func (s *Storage) _check() { if configPath := config.GetConfigPath(); configPath != "" { // Check to see if config file has changed since it was last loaded fi, err := os.Stat(configPath) @@ -174,7 +173,10 @@ func (s *Storage) Save() error { // Serialize the config into a string func (s *Storage) Serialize() (string, error) { - s.check() + s.mu.Lock() + defer s.mu.Unlock() + + s._check() var buf bytes.Buffer if err := goconfig.SaveConfigData(s.gc, &buf); err != nil { return "", fmt.Errorf("failed to save config file: %w", err) @@ -185,7 +187,10 @@ func (s *Storage) Serialize() (string, error) { // HasSection returns true if section exists in the config file func (s *Storage) HasSection(section string) bool { - s.check() + s.mu.Lock() + defer s.mu.Unlock() + + s._check() _, err := s.gc.GetSection(section) return err == nil } @@ -193,26 +198,38 @@ func (s *Storage) HasSection(section string) bool { // DeleteSection removes the named section and all config from the // config file func (s *Storage) DeleteSection(section string) { - s.check() + s.mu.Lock() + defer s.mu.Unlock() + + s._check() s.gc.DeleteSection(section) } // GetSectionList returns a slice of strings with names for all the // sections func (s *Storage) GetSectionList() []string { - s.check() + s.mu.Lock() + defer s.mu.Unlock() + + s._check() return s.gc.GetSectionList() } // GetKeyList returns the keys in this section func (s *Storage) GetKeyList(section string) []string { - s.check() + s.mu.Lock() + defer s.mu.Unlock() + + s._check() return s.gc.GetKeyList(section) } // GetValue returns the key in section with a found flag func (s *Storage) GetValue(section string, key string) (value string, found bool) { - s.check() + s.mu.Lock() + defer s.mu.Unlock() + + s._check() value, err := s.gc.GetValue(section, key) if err != nil { return "", false @@ -222,7 +239,10 @@ func (s *Storage) GetValue(section string, key string) (value string, found bool // SetValue sets the value under key in section func (s *Storage) SetValue(section string, key string, value string) { - s.check() + s.mu.Lock() + defer s.mu.Unlock() + + s._check() if strings.HasPrefix(section, ":") { fs.Logf(nil, "Can't save config %q for on the fly backend %q", key, section) return @@ -232,7 +252,10 @@ func (s *Storage) SetValue(section string, key string, value string) { // DeleteKey removes the key under section func (s *Storage) DeleteKey(section string, key string) bool { - s.check() + s.mu.Lock() + defer s.mu.Unlock() + + s._check() return s.gc.DeleteKey(section, key) } From bd787e8f457705de9072335fda879f575da4f3cf Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 5 Sep 2022 16:19:50 +0100 Subject: [PATCH 262/560] filter: Fix incorrect filtering with UseFilter context flag and wrapping backends In this commit 8d1fff9a8237c64f local: obey file filters in listing to fix errors on excluded files We started using filters in the local backend so the user could short circuit troublesome files/directories at a low level. However this caused a number of integration tests to fail. This turned out to be in backends wrapping the local backend. For example the combine backend test failed because it changes the paths passed to the local backend so they no longer match the paths in the current filter. To fix this, a new feature flag `FilterAware` was added and the UseFilter context flag is only passed to backends which support it. As the wrapping backends don't support the flag, this fixes the problems in the integration tests. In future the wrapping backends could modify the active filters to match the path modifications and then they could set the FilterAware flag. See #6376 --- backend/drive/drive.go | 1 + backend/drive/drive_internal_test.go | 3 +++ backend/local/local.go | 1 + backend/local/local_internal_test.go | 3 +++ fs/features.go | 2 ++ fs/march/march.go | 4 ++-- fs/walk/walk.go | 4 ++-- 7 files changed, 14 insertions(+), 4 deletions(-) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 30c165b6149e4..e0b4abb61fc3c 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -1210,6 +1210,7 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err WriteMimeType: true, CanHaveEmptyDirectories: true, ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs, + FilterAware: true, }).Fill(ctx, f) // Create a new authorized Drive client. diff --git a/backend/drive/drive_internal_test.go b/backend/drive/drive_internal_test.go index 22a84b0bb5f3f..c17da706293c9 100644 --- a/backend/drive/drive_internal_test.go +++ b/backend/drive/drive_internal_test.go @@ -518,6 +518,9 @@ func (f *Fs) InternalTestCopyID(t *testing.T) { // TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery func (f *Fs) InternalTestAgeQuery(t *testing.T) { + // Check set up for filtering + assert.True(t, f.Features().FilterAware) + opt := &filter.Opt{} err := opt.MaxAge.Set("1h") assert.NoError(t, err) diff --git a/backend/local/local.go b/backend/local/local.go index 18763ace3c860..7d56d8012f0e5 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -300,6 +300,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e ReadMetadata: true, WriteMetadata: true, UserMetadata: xattrSupported, // can only R/W general purpose metadata if xattrs are supported + FilterAware: true, }).Fill(ctx, f) if opt.FollowSymlinks { f.lstat = os.Stat diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go index 811831d915b59..7ecc9986bf5fb 100644 --- a/backend/local/local_internal_test.go +++ b/backend/local/local_internal_test.go @@ -378,6 +378,9 @@ func TestFilter(t *testing.T) { r.WriteFile("excluded", "excluded file", when) f := r.Flocal.(*Fs) + // Check set up for filtering + assert.True(t, f.Features().FilterAware) + // Add a filter ctx, fi := filter.AddConfig(ctx) require.NoError(t, fi.AddRule("+ included")) diff --git a/fs/features.go b/fs/features.go index 6fce02d608df8..b8bbffc2136fd 100644 --- a/fs/features.go +++ b/fs/features.go @@ -29,6 +29,7 @@ type Features struct { ReadMetadata bool // can read metadata from objects WriteMetadata bool // can write metadata to objects UserMetadata bool // can read/write general purpose metadata + FilterAware bool // can make use of filters if provided for listing // Purge all files in the directory specified // @@ -320,6 +321,7 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features { // ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal ft.SlowModTime = ft.SlowModTime && mask.SlowModTime ft.SlowHash = ft.SlowHash && mask.SlowHash + ft.FilterAware = ft.FilterAware && mask.FilterAware if mask.Purge == nil { ft.Purge = nil diff --git a/fs/march/march.go b/fs/march/march.go index db045156d5417..771f079f4a7e7 100644 --- a/fs/march/march.go +++ b/fs/march/march.go @@ -83,7 +83,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and !(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse) return func(dir string) (entries fs.DirEntries, err error) { - dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List + dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List return list.DirSorted(dirCtx, f, includeAll, dir) } } @@ -100,7 +100,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD mu.Lock() defer mu.Unlock() if !started { - dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List + dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List dirs, dirsErr = walk.NewDirTree(dirCtx, f, m.Dir, includeAll, ci.MaxDepth) started = true } diff --git a/fs/walk/walk.go b/fs/walk/walk.go index f63350a477393..01251ea1bec0d 100644 --- a/fs/walk/walk.go +++ b/fs/walk/walk.go @@ -64,7 +64,7 @@ type Func func(path string, entries fs.DirEntries, err error) error func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error { ci := fs.GetConfig(ctx) fi := filter.GetConfig(ctx) - ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List + ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List if ci.NoTraverse && fi.HaveFilesFrom() { return walkR(ctx, f, path, includeAll, maxLevel, fn, fi.MakeListR(ctx, f.NewObject)) } @@ -158,7 +158,7 @@ func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel fi.UsesDirectoryFilters() { // ...using any directory filters return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn) } - ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List + ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased) } From be53dcc9c97737b524bd3144364d9b2f74113fda Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 12 Sep 2022 11:54:35 +0100 Subject: [PATCH 263/560] docs: add more information about --track-renames See: https://forum.rclone.org/t/feature-question-how-does-rclone-track-renames-and-moves/32911/4 --- docs/content/docs.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index 95609d7578910..47757d431a37e 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -1868,13 +1868,22 @@ By default, rclone doesn't keep track of renamed files, so if you rename a file locally then sync it to a remote, rclone will delete the old file on the remote and upload a new copy. -If you use this flag, and the remote supports server-side copy or -server-side move, and the source and destination have a compatible -hash, then this will track renames during `sync` -operations and perform renaming server-side. - -Files will be matched by size and hash - if both match then a rename -will be considered. +An rclone sync with `--track-renames` runs like a normal sync, but keeps +track of objects which exist in the destination but not in the source +(which would normally be deleted), and which objects exist in the +source but not the destination (which would normally be transferred). +These objects are then candidates for renaming. + +After the sync, rclone matches up the source only and destination only +objects using the `--track-renames-strategy` specified and either +renames the destination object or transfers the source and deletes the +destination object. `--track-renames` is stateless like all of +rclone's syncs. + +To use this flag the destination must support server-side copy or +server-side move, and to use a hash based `--track-renames-strategy` +(the default) the source and the destination must have a compatible +hash. If the destination does not support server-side copy or move, rclone will fall back to the default behaviour and log an error level message @@ -1892,7 +1901,7 @@ Note also that `--track-renames` is incompatible with ### --track-renames-strategy (hash,modtime,leaf,size) ### -This option changes the matching criteria for `--track-renames`. +This option changes the file matching criteria for `--track-renames`. The matching is controlled by a comma separated selection of these tokens: @@ -1901,15 +1910,15 @@ The matching is controlled by a comma separated selection of these tokens: - `leaf` - the name of the file not including its directory name - `size` - the size of the file (this is always enabled) -So using `--track-renames-strategy modtime,leaf` would match files +The default option is `hash`. + +Using `--track-renames-strategy modtime,leaf` would match files based on modification time, the leaf of the file name and the size only. Using `--track-renames-strategy modtime` or `leaf` can enable `--track-renames` support for encrypted destinations. -If nothing is specified, the default option is matching by `hash`es. - Note that the `hash` strategy is not supported with encrypted destinations. ### --delete-(before,during,after) ### From 57803bee22df737fe58282c54b47583fa0991d05 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 12 Sep 2022 20:30:34 +0100 Subject: [PATCH 264/560] build: update tidy-beta to new layout --- RELEASE.md | 8 ++++++++ bin/tidy-beta | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 3747fb8e8e9ed..32f5d664bce0c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -53,6 +53,14 @@ doing that so it may be necessary to roll back dependencies to the version specified by `make updatedirect` in order to get rclone to build. +## Tidy beta + +At some point after the release run + + bin/tidy-beta v1.55 + +where the version number is that of a couple ago to remove old beta binaries. + ## Making a point release If rclone needs a point release due to some horrendous bug: diff --git a/bin/tidy-beta b/bin/tidy-beta index d884f49cef75c..9d5d29fadf460 100755 --- a/bin/tidy-beta +++ b/bin/tidy-beta @@ -15,6 +15,7 @@ else fi rclone ${dry_run} -vv -P --checkers 16 --transfers 16 delete \ + --fast-list \ --include "/${version}**" \ - --include "/branch/${version}**" \ + --include "/branch/*/${version}**" \ memstore:beta-rclone-org From d3d843a11d4a85bc9ed72f369d82d53f888cc047 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 12 Sep 2022 20:43:53 +0100 Subject: [PATCH 265/560] fs: warn the user when using a remote name without a colon A very common mistake for new users of rclone is to use a remote name without a colon. This can be on the command line or in the config when setting up a crypt backend. This change checks to see if the user uses a path which matches a remote name and gives an NOTICE like this if they do NOTICE: "remote" refers to a local folder, use "remote:" to refer to your remote or "./remote" to hide this warning See: https://forum.rclone.org/t/sync-to-onedrive-personal-lands-file-in-localfilesystem-but-not-in-onedrive/32956 --- fs/config.go | 6 ++++++ fs/config/config.go | 3 +++ fs/newfs.go | 3 +++ 3 files changed, 12 insertions(+) diff --git a/fs/config.go b/fs/config.go index b253a5c30feb4..13d5235589765 100644 --- a/fs/config.go +++ b/fs/config.go @@ -29,6 +29,12 @@ var ( return errors.New("no config file set handler") } + // Check if the config file has the named section + // + // This is a function pointer to decouple the config + // implementation from the fs + ConfigFileHasSection = func(section string) bool { return false } + // CountError counts an error. If any errors have been // counted then rclone will exit with a non zero error code. // diff --git a/fs/config/config.go b/fs/config/config.go index 0c84127e6bfe1..4a67252379844 100644 --- a/fs/config/config.go +++ b/fs/config/config.go @@ -117,6 +117,9 @@ func init() { // Set the function pointers up in fs fs.ConfigFileGet = FileGetFlag fs.ConfigFileSet = SetValueAndSave + fs.ConfigFileHasSection = func(section string) bool { + return LoadedData().HasSection(section) + } configPath = makeConfigPath() cacheDir = makeCacheDir() // Has fallback to tempDir, so set that first data = newDefaultStorage() diff --git a/fs/newfs.go b/fs/newfs.go index adf8582fac22f..927d278b49eb6 100644 --- a/fs/newfs.go +++ b/fs/newfs.go @@ -26,6 +26,9 @@ import ( // up with drive letters. func NewFs(ctx context.Context, path string) (Fs, error) { Debugf(nil, "Creating backend with remote %q", path) + if ConfigFileHasSection(path) { + Logf(nil, "%q refers to a local folder, use %q to refer to your remote or %q to hide this warning", path, path+":", "./"+path) + } fsInfo, configName, fsPath, config, err := ConfigFs(path) if err != nil { return nil, err From 655d63b4fde02698422cb039b0e0e76a0e5b8301 Mon Sep 17 00:00:00 2001 From: partev Date: Wed, 14 Sep 2022 06:14:32 -0400 Subject: [PATCH 266/560] docs: fix a typo: aftering -> after --- docs/content/onedrive.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index 0f6897e34d02f..fcff1b76e3f76 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -152,7 +152,7 @@ A common error is that the publisher of the App is not verified. You may try to [verify you account](https://docs.microsoft.com/en-us/azure/active-directory/develop/publisher-verification-overview), or try to limit the App to your organization only, as shown below. 1. Make sure to create the App with your business account. -2. Follow the steps above to create an App. However, we need a different account type here: `Accounts in this organizational directory only (*** - Single tenant)`. Note that you can also change the account type aftering creating the App. +2. Follow the steps above to create an App. However, we need a different account type here: `Accounts in this organizational directory only (*** - Single tenant)`. Note that you can also change the account type after creating the App. 3. Find the [tenant ID](https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/active-directory-how-to-find-tenant) of your organization. 4. In the rclone config, set `auth_url` to `https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/authorize`. 5. In the rclone config, set `token_url` to `https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/token`. From 91f8894285c451dd62b3b82f757a2ecd5cb08097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Heddeland=20Instefjord?= Date: Mon, 5 Sep 2022 21:54:41 +0200 Subject: [PATCH 267/560] ftp: Add force_list_hidden option Forces the use of `LIST -a` command when listing a directory which should list all hidden folders and files. --- backend/ftp/ftp.go | 9 +++++++++ docs/content/flags.md | 1 + docs/content/ftp.md | 11 +++++++++++ go.mod | 2 +- go.sum | 2 ++ 5 files changed, 24 insertions(+), 1 deletion(-) diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index c5c885e00fb72..b48a06eee4f4b 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -124,6 +124,11 @@ So for |concurrency 3| you'd use |--checkers 2 --transfers 2 Help: "Use MDTM to set modification time (VsFtpd quirk)", Default: false, Advanced: true, + }, { + Name: "force_list_hidden", + Help: "Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD.", + Default: false, + Advanced: true, }, { Name: "idle_timeout", Default: fs.Duration(60 * time.Second), @@ -205,6 +210,7 @@ type Options struct { DisableMLSD bool `config:"disable_mlsd"` DisableUTF8 bool `config:"disable_utf8"` WritingMDTM bool `config:"writing_mdtm"` + ForceListHidden bool `config:"force_list_hidden"` IdleTimeout fs.Duration `config:"idle_timeout"` CloseTimeout fs.Duration `config:"close_timeout"` ShutTimeout fs.Duration `config:"shut_timeout"` @@ -367,6 +373,9 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) { if f.opt.WritingMDTM { ftpConfig = append(ftpConfig, ftp.DialWithWritingMDTM(true)) } + if f.opt.ForceListHidden { + ftpConfig = append(ftpConfig, ftp.DialWithForceListHidden(true)) + } if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 { ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0})) } diff --git a/docs/content/flags.md b/docs/content/flags.md index 2426a4a4ae5b7..f9753336e47d0 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -357,6 +357,7 @@ and may be set in the config file. --ftp-tls-cache-size int Size of TLS session cache for all control and data connections (default 32) --ftp-user string FTP username (default "$USER") --ftp-writing-mdtm Use MDTM to set modification time (VsFtpd quirk) + --ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD. --gcs-anonymous Access public buckets and objects without credentials --gcs-auth-url string Auth server URL --gcs-bucket-acl string Access Control List for new buckets diff --git a/docs/content/ftp.md b/docs/content/ftp.md index fa29a682139e8..7d4da7d7b1e6b 100644 --- a/docs/content/ftp.md +++ b/docs/content/ftp.md @@ -310,6 +310,17 @@ Properties: - Type: bool - Default: false +#### --ftp-force-list-hidden + +Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD. + +Properties: + +- Config: force_list_hidden +- Env Var: RCLONE_FTP_FORCE_LIST_HIDDEN +- Type: bool +- Default: false + #### --ftp-idle-timeout Max time before closing idle connections. diff --git a/go.mod b/go.mod index 31072dbff8dc2..6358a144c3df1 100644 --- a/go.mod +++ b/go.mod @@ -135,7 +135,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff + github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a github.com/pkg/xattr v0.4.7 golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 diff --git a/go.sum b/go.sum index d09ac3566dff5..5445443073bc7 100644 --- a/go.sum +++ b/go.sum @@ -380,6 +380,8 @@ github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0 github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff h1:tN6UCYCBFNrPwvKf4RP9cIhGo6GcZ/IQTN8nqD7eCok= github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE= +github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a h1:s4ryRQyC5HKZh6qkjNAFcvmD7gImK5bZuj/YZkXy1vw= +github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= From 67f5f04a77b97e11ddfad3107f8c062bd851eba1 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Fri, 9 Sep 2022 00:18:18 +0200 Subject: [PATCH 268/560] build: fix lint option max-issues-per-linter --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 67b3edda9afa1..9828c53dd9df4 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -20,7 +20,7 @@ issues: exclude-use-default: false # Maximum issues count per one linter. Set to 0 to disable. Default is 50. - max-per-linter: 0 + max-issues-per-linter: 0 # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. max-same-issues: 0 From 6ce0168ba53555053f90f1e6f50c45e51ddf2255 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sun, 4 Sep 2022 10:14:11 +0200 Subject: [PATCH 269/560] docs: better alignment of icons Fixes issue with spacing between icon and text in backend docs headers. This reverts the changes from PR #5889 and #5701, which aligned menu/dropdown items when icons have different sizes, and implements an alternative fix which gives slightly better results, and also is more of a native Font Awesome solution: Font Awesome icons are designed on grid and share a consistent height. But they vary in width depending on how wide or narrow each symbol is. If you prefer to work with icons that have a consistent width, adding fa-fw will render each icon using the same width. --- docs/layouts/chrome/menu.html | 18 ++--- docs/layouts/chrome/navbar.html | 132 ++++++++++++++++---------------- docs/static/css/custom.css | 12 --- 3 files changed, 75 insertions(+), 87 deletions(-) diff --git a/docs/layouts/chrome/menu.html b/docs/layouts/chrome/menu.html index 3db7f54bd8c95..85ea1e5435df0 100644 --- a/docs/layouts/chrome/menu.html +++ b/docs/layouts/chrome/menu.html @@ -17,9 +17,9 @@

    - Twitter
    - Facebook
    - Reddit
    + Twitter
    + Facebook
    + Reddit

    @@ -31,12 +31,12 @@

    - Rclone forum
    - GitHub project
    - Rclone slack
    - Rclone Wiki
    - Donate
    - @njcw + Rclone forum
    + GitHub project
    + Rclone slack
    + Rclone Wiki
    + Donate
    + @njcw

    diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index f732756e32438..591219cf30670 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -14,18 +14,18 @@ Docs
    - Installation - Usage - Filtering - GUI - Remote Control - Changelog - Bugs - FAQ - Flags - Licence - Authors - Privacy Policy + Installation + Usage + Filtering + GUI + Remote Control + Changelog + Bugs + FAQ + Flags + Licence + Authors + Privacy Policy
  • @@ -33,12 +33,12 @@ Commands
    - Overview + Overview
    {{ with .Site.GetPage "/commands" }} {{ range .Data.Pages }} {{ if lt (countwords .Title) 3 }} - {{ .Title | markdownify }} + {{ .Title | markdownify }} {{ end }} {{ end }} {{ end }} @@ -49,65 +49,65 @@ Storage Systems
    - Overview + Overview
    - 1Fichier - Akamai NetStorage - Alias - Amazon Drive - Amazon S3 - Backblaze B2 - Box - Chunker (splits large files) - Compress (transparent gzip compression) - Combine (remotes into a directory tree) - Citrix ShareFile - Crypt (encrypts the others) - Digi Storage - Dropbox - Enterprise File Fabric - FTP - Google Cloud Storage - Google Drive - Google Photos - Hasher (better checksums for others) - HDFS (Hadoop Distributed Filesystem) - HiDrive - HTTP - Hubic - Internet Archive - Jottacloud - Koofr - Mail.ru Cloud - Mega - Memory - Microsoft Azure Blob Storage - Microsoft OneDrive - OpenDrive - QingStor - Openstack Swift - pCloud - premiumize.me - put.io - Seafile - SFTP - Sia - Storj - SugarSync - Uptobox - Union (merge backends) - WebDAV - Yandex Disk - Zoho WorkDrive + 1Fichier + Akamai NetStorage + Alias + Amazon Drive + Amazon S3 + Backblaze B2 + Box + Chunker (splits large files) + Compress (transparent gzip compression) + Combine (remotes into a directory tree) + Citrix ShareFile + Crypt (encrypts the others) + Digi Storage + Dropbox + Enterprise File Fabric + FTP + Google Cloud Storage + Google Drive + Google Photos + Hasher (better checksums for others) + HDFS (Hadoop Distributed Filesystem) + HiDrive + HTTP + Hubic + Internet Archive + Jottacloud + Koofr + Mail.ru Cloud + Mega + Memory + Microsoft Azure Blob Storage + Microsoft OneDrive + OpenDrive + QingStor + Openstack Swift + pCloud + premiumize.me + put.io + Seafile + SFTP + Sia + Storj + SugarSync + Uptobox + Union (merge backends) + WebDAV + Yandex Disk + Zoho WorkDrive
    - The local filesystem + The local filesystem
  • - Contact + Contact
  • - Donate + Donate
    • diff --git a/docs/static/css/custom.css b/docs/static/css/custom.css index d112bbbfc74ba..2ac4eb16ab7d7 100644 --- a/docs/static/css/custom.css +++ b/docs/static/css/custom.css @@ -142,18 +142,6 @@ h5 { font-size: 95%; } -/* Align dropdown items when icons have different sizes */ -.dropdown-item .fa, .fab, .fad, .fal, .far, .fas { - width: 20px; - text-align: center; -} - -/* Align menu items when icons have different sizes */ -.menu .fa, .fab, .fad, .fal, .far, .fas { - width: 18px; - text-align: center; -} - /* Make primary buttons rclone colours. Should learn sass and do this the proper way! */ .btn-primary { background-color: #3f79ad; From 67240bd5410a8c4ea700359073eb420fc2812072 Mon Sep 17 00:00:00 2001 From: Lesmiscore Date: Thu, 15 Sep 2022 00:45:35 +0900 Subject: [PATCH 270/560] sftp: fix directory creation races If mkdir fails then before this change it would have thrown an error. After this change, if the error indicated that the directory already exists then the error is not returned to the user. This fixes a race condition when two rclone threads are trying to create the same directory. --- backend/sftp/sftp.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 37fcbdcb2f51e..19596e59d3cde 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -1171,6 +1171,10 @@ func (f *Fs) mkdir(ctx context.Context, dirPath string) error { err = c.sftpClient.Mkdir(dirPath) f.putSftpConnection(&c, err) if err != nil { + if os.IsExist(err) { + fs.Debugf(f, "directory %q exists after Mkdir is attempted", dirPath) + return nil + } return fmt.Errorf("mkdir %q failed: %w", dirPath, err) } return nil From 8504da496bcd7a3e8468d99b3fd1a294c396bb91 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 15 Sep 2022 11:57:07 +0100 Subject: [PATCH 271/560] Changelog updates from Version v1.59.2 --- docs/content/changelog.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/content/changelog.md b/docs/content/changelog.md index 0a6f3211918c5..c11146b4f55c4 100644 --- a/docs/content/changelog.md +++ b/docs/content/changelog.md @@ -5,6 +5,21 @@ description: "Rclone Changelog" # Changelog +## v1.59.2 - 2022-09-15 + +[See commits](https://github.com/rclone/rclone/compare/v1.59.1...v1.59.2) + +* Bug Fixes + * config: Move locking to fix fatal error: concurrent map read and map write (Nick Craig-Wood) +* Local + * Disable xattr support if the filesystems indicates it is not supported (Nick Craig-Wood) +* Azure Blob + * Fix chunksize calculations producing too many parts (Nick Craig-Wood) +* B2 + * Fix chunksize calculations producing too many parts (Nick Craig-Wood) +* S3 + * Fix chunksize calculations producing too many parts (Nick Craig-Wood) + ## v1.59.1 - 2022-08-08 [See commits](https://github.com/rclone/rclone/compare/v1.59.0...v1.59.1) From c080b39e470913367f7e12a08a8c27e2706dd32e Mon Sep 17 00:00:00 2001 From: Dmitry Deniskin <110819396+ddeniskin@users.noreply.github.com> Date: Thu, 15 Sep 2022 17:04:34 +0200 Subject: [PATCH 272/560] s3: add support for IONOS Cloud Storage --- README.md | 1 + backend/s3/s3.go | 43 ++++++++++- docs/content/_index.md | 1 + docs/content/s3.md | 164 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 205 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7f2302f8998b5..6c0e35752af55 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/) * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/) * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3) + * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos) * Koofr [:page_facing_up:](https://rclone.org/koofr/) * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/) * Memset Memstore [:page_facing_up:](https://rclone.org/swift/) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 7b3a81a294d40..25257bb85aaec 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -64,7 +64,7 @@ import ( func init() { fs.Register(&fs.RegInfo{ Name: "s3", - Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi", + Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi", NewFs: NewFs, CommandHelp: commandHelp, Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { @@ -116,6 +116,9 @@ func init() { }, { Value: "IDrive", Help: "IDrive e2", + }, { + Value: "IONOS", + Help: "IONOS Cloud", }, { Value: "LyveCloud", Help: "Seagate Lyve Cloud", @@ -384,10 +387,24 @@ func init() { Value: "auto", Help: "R2 buckets are automatically distributed across Cloudflare's data centers for low latency.", }}, + }, { + Name: "region", + Help: "Region where your bucket will be created and your data stored.\n", + Provider: "IONOS", + Examples: []fs.OptionExample{{ + Value: "de", + Help: "Frankfurt, Germany", + }, { + Value: "eu-central-2", + Help: "Berlin, Germany", + }, { + Value: "eu-south-2", + Help: "Logrono, Spain", + }}, }, { Name: "region", Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.", - Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive", + Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive", Examples: []fs.OptionExample{{ Value: "", Help: "Use this if unsure.\nWill use v4 signatures and an empty region.", @@ -698,6 +715,20 @@ func init() { Value: "s3.private.sng01.cloud-object-storage.appdomain.cloud", Help: "Singapore Single Site Private Endpoint", }}, + }, { + Name: "endpoint", + Help: "Endpoint for IONOS S3 Object Storage.\n\nSpecify the endpoint from the same region.", + Provider: "IONOS", + Examples: []fs.OptionExample{{ + Value: "s3-eu-central-1.ionoscloud.com", + Help: "Frankfurt, Germany", + }, { + Value: "s3-eu-central-2.ionoscloud.com", + Help: "Berlin, Germany", + }, { + Value: "s3-eu-south-2.ionoscloud.com", + Help: "Logrono, Spain", + }}, }, { // oss endpoints: https://help.aliyun.com/document_detail/31837.html Name: "endpoint", @@ -1001,7 +1032,7 @@ func init() { }, { Name: "endpoint", Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.", - Provider: "!AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp", + Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp", Examples: []fs.OptionExample{{ Value: "objects-us-east-1.dream.io", Help: "Dream Objects endpoint", @@ -1411,7 +1442,7 @@ func init() { }, { Name: "location_constraint", Help: "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.", - Provider: "!AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS", + Provider: "!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS", }, { Name: "acl", Help: `Canned ACL used when creating buckets and storing or copying objects. @@ -2537,6 +2568,10 @@ func setQuirks(opt *Options) { useMultipartEtag = false // untested case "IDrive": virtualHostStyle = false + case "IONOS": + // listObjectsV2 supported - https://api.ionos.com/docs/s3/#Basic-Operations-get-Bucket-list-type-2 + virtualHostStyle = false + urlEncodeListings = false case "LyveCloud": useMultipartEtag = false // LyveCloud seems to calculate multipart Etags differently from AWS case "Minio": diff --git a/docs/content/_index.md b/docs/content/_index.md index 9d0f86bf439d9..98bcd578406e7 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -135,6 +135,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}} {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} {{< provider name="IDrive e2" home="https://www.idrive.com/e2/" config="/s3/#idrive-e2" >}} +{{< provider name="IONOS Cloud" home="https://cloud.ionos.com/storage/object-storage" config="/s3/#ionos" >}} {{< provider name="Koofr" home="https://koofr.eu/" config="/koofr/" >}} {{< provider name="Mail.ru Cloud" home="https://cloud.mail.ru/" config="/mailru/" >}} {{< provider name="Memset Memstore" home="https://www.memset.com/cloud/storage/" config="/swift/" >}} diff --git a/docs/content/s3.md b/docs/content/s3.md index d09dd35925c19..e8b10aae51594 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -19,6 +19,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}} {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} {{< provider name="IDrive e2" home="https://www.idrive.com/e2/" config="/s3/#idrive-e2" >}} +{{< provider name="IONOS Cloud" home="https://cloud.ionos.com/storage/object-storage" config="/s3/#ionos" >}} {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}} {{< provider name="RackCorp Object Storage" home="https://www.rackcorp.com/" config="/s3/#RackCorp" >}} {{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}} @@ -3533,6 +3534,169 @@ d) Delete this remote y/e/d> y ``` +### IONOS Cloud {#ionos} + +[IONOS S3 Object Storage](https://cloud.ionos.com/storage/object-storage) is a service offered by IONOS for storing and accessing unstructured data. +To connect to the service, you will need an access key and a secret key. These can be found in the [Data Center Designer](https://dcd.ionos.com/), by selecting **Manager resources** > **Object Storage Key Manager**. + + +Here is an example of a configuration. First, run `rclone config`. This will walk you through an interactive setup process. Type `n` to add the new remote, and then enter a name: + +``` +Enter name for new remote. +name> ionos-fra +``` + +Type `s3` to choose the connection type: +``` +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +[snip] +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + \ (s3) +[snip] +Storage> s3 +``` + +Type `IONOS`: +``` +Option provider. +Choose your S3 provider. +Choose a number from below, or type in your own value. +Press Enter to leave empty. +[snip] +XX / IONOS Cloud + \ (IONOS) +[snip] +provider> IONOS +``` + +Press Enter to choose the default option `Enter AWS credentials in the next step`: +``` +Option env_auth. +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own boolean value (true or false). +Press Enter for the default (false). + 1 / Enter AWS credentials in the next step. + \ (false) + 2 / Get AWS credentials from the environment (env vars or IAM). + \ (true) +env_auth> +``` + +Enter your Access Key and Secret key. These can be retrieved in the [Data Center Designer](https://dcd.ionos.com/), click on the menu “Manager resources” / "Object Storage Key Manager". +``` +Option access_key_id. +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +access_key_id> YOUR_ACCESS_KEY + +Option secret_access_key. +AWS Secret Access Key (password). +Leave blank for anonymous access or runtime credentials. +Enter a value. Press Enter to leave empty. +secret_access_key> YOUR_SECRET_KEY +``` + +Choose the region where your bucket is located: +``` +Option region. +Region where your bucket will be created and your data stored. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / Frankfurt, Germany + \ (de) + 2 / Berlin, Germany + \ (eu-central-2) + 3 / Logrono, Spain + \ (eu-south-2) +region> 2 +``` + +Choose the endpoint from the same region: +``` +Option endpoint. +Endpoint for IONOS S3 Object Storage. +Specify the endpoint from the same region. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / Frankfurt, Germany + \ (s3-eu-central-1.ionoscloud.com) + 2 / Berlin, Germany + \ (s3-eu-central-2.ionoscloud.com) + 3 / Logrono, Spain + \ (s3-eu-south-2.ionoscloud.com) +endpoint> 1 +``` + +Press Enter to choose the default option or choose the desired ACL setting: +``` +Option acl. +Canned ACL used when creating buckets and storing or copying objects. +This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Note that this ACL is applied when server-side copying objects as S3 +doesn't copy the ACL from the source but rather writes a fresh one. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \ (private) + / Owner gets FULL_CONTROL. +[snip] +acl> +``` + +Press Enter to skip the advanced config: +``` +Edit advanced config? +y) Yes +n) No (default) +y/n> +``` + +Press Enter to save the configuration, and then `q` to quit the configuration process: +``` +Configuration complete. +Options: +- type: s3 +- provider: IONOS +- access_key_id: YOUR_ACCESS_KEY +- secret_access_key: YOUR_SECRET_KEY +- endpoint: s3-eu-central-1.ionoscloud.com +Keep this "ionos-fra" remote? +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +Done! Now you can try some commands (for macOS, use `./rclone` instead of `rclone`). + +1) Create a bucket (the name must be unique within the whole IONOS S3) +``` +rclone mkdir ionos-fra:my-bucket +``` +2) List available buckets +``` +rclone lsd ionos-fra: +``` +4) Copy a file from local to remote +``` +rclone copy /Users/file.txt ionos-fra:my-bucket +``` +3) List contents of a bucket +``` +rclone ls ionos-fra:my-bucket +``` +5) Copy a file from remote to local +``` +rclone copy ionos-fra:my-bucket/file.txt +``` + ### Minio [Minio](https://minio.io/) is an object storage server built for cloud application developers and devops. From 67c675d7ad4962bc2f5aab5601d7bae721da7b44 Mon Sep 17 00:00:00 2001 From: Alexander Knorr <106825+opexxx@users.noreply.github.com> Date: Thu, 15 Sep 2022 17:12:39 +0200 Subject: [PATCH 273/560] docs: add cholateley package manager to install instructions --- docs/content/install.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/content/install.md b/docs/content/install.md index f69b04fe70949..40ff31ae72369 100644 --- a/docs/content/install.md +++ b/docs/content/install.md @@ -188,6 +188,15 @@ ls ~/data/mount kill %1 ``` +## Install on Windows via Chocolateley Packet Manager + +Make sure you have [Choco](https://chocolatey.org/) installed +``` +choco search rclone +choco install rclone +``` +This will install rclone on your windows machine + ## Install from source Make sure you have git and [Go](https://golang.org/) installed. From cbc18e2693da296770f22eec8fce92518f9b72dc Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 15 Sep 2022 17:06:31 +0100 Subject: [PATCH 274/560] docs: update install docs to make more consistent This also adds repology badges where appropriate to show versions in external repositories. --- docs/content/install.md | 108 +++++++++++++++++++++++++++++++--------- 1 file changed, 85 insertions(+), 23 deletions(-) diff --git a/docs/content/install.md b/docs/content/install.md index 40ff31ae72369..0480bf96c3ad2 100644 --- a/docs/content/install.md +++ b/docs/content/install.md @@ -14,7 +14,7 @@ Rclone is a Go program and comes as a single binary file. * Run `rclone config` to setup. See [rclone config docs](/docs/) for more details. * Optionally configure [automatic execution](#autostart). -See below for some expanded Linux / macOS instructions. +See below for some expanded Linux / macOS / Windows instructions. See the [usage](/docs/) docs for how to use rclone, or run `rclone -h`. @@ -35,7 +35,9 @@ For beta installation, run: Note that this script checks the version of rclone installed first and won't re-download if not needed. -## Linux installation from precompiled binary +## Linux installation {#linux} + +### Precompiled binary {#linux-precompiled} Fetch and unpack @@ -59,7 +61,9 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details. rclone config -## macOS installation with brew +## macOS installation {#macos} + +### Installation with brew {#macos-brew} brew install rclone @@ -68,7 +72,12 @@ NOTE: This version of rclone will not support `mount` any more (see on macOS, either install a precompiled binary or enable the relevant option when [installing from source](#install-from-source). -## macOS installation from precompiled binary, using curl +Note that this is a third party installer not controlled by the rclone +developers so it may be out of date. Its current version is as below. + +[![Homebrew package](https://repology.org/badge/version-for-repo/homebrew/rclone.svg)](https://repology.org/project/rclone/versions) + +### Precompiled binary, using curl {#macos-precompiled} To avoid problems with macOS gatekeeper enforcing the binary to be signed and notarized it is enough to download with `curl`. @@ -96,7 +105,7 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details. rclone config -## macOS installation from precompiled binary, using a web browser +### Precompiled binary, using a web browser {#macos-precompiled-web} When downloading a binary with a web browser, the browser will set the macOS gatekeeper quarantine attribute. Starting from Catalina, when attempting to run @@ -109,11 +118,73 @@ The simplest fix is to run xattr -d com.apple.quarantine rclone -## Install with docker +## Windows installation {#windows} + +### Precompiled binary {#windows-precompiled} + +Fetch the correct binary for your processor type by clicking on these +links. If not sure, use the first link. + +- [Intel/AMD - 64 Bit](https://downloads.rclone.org/rclone-current-linux-amd64.zip) +- [Intel/AMD - 32 Bit](https://downloads.rclone.org/rclone-current-linux-386.zip) +- [ARM - 64 Bit](https://downloads.rclone.org/rclone-current-linux-arm64.zip) + +Open this file in the Explorer and extract `rclone.exe`. Rclone is a +portable executable so you can place it wherever is convenient. + +Open a CMD window (or powershell) and run the binary. Note that rclone +does not launch a GUI by default, it runs in the CMD Window. + +- Run `rclone.exe config` to setup. See [rclone config docs](/docs/) for more details. +- Optionally configure [automatic execution](#autostart). + +If you are planning to use the [rclone mount](/commands/rclone_mount/) +feature then you will need to install the third party utility +[WinFsp](https://winfsp.dev/) also. + +### Chocolatey package manager {#windows-chocolatey} + +Make sure you have [Choco](https://chocolatey.org/) installed + +``` +choco search rclone +choco install rclone +``` + +This will install rclone on your Windows machine. If you are planning +to use [rclone mount](/commands/rclone_mount/) then -The rclone maintains a [docker image for rclone](https://hub.docker.com/r/rclone/rclone). -These images are autobuilt by docker hub from the rclone source based -on a minimal Alpine linux image. +``` +choco install winfsp +``` + +will install that too. + +Note that this is a third party installer not controlled by the rclone +developers so it may be out of date. Its current version is as below. + +[![Chocolatey package](https://repology.org/badge/version-for-repo/chocolatey/rclone.svg)](https://repology.org/project/rclone/versions) + +## Package manager installation {#package-manager} + +Many Linux, Windows, macOS and other OS distributions package and +distribute rclone. + +The distributed versions of rclone are often quite out of date and for +this reason we recommend one of the other installation methods if +possible. + +You can get an idea of how up to date or not your OS distribution's +package is here. + +[![Packaging status](https://repology.org/badge/vertical-allrepos/rclone.svg?columns=3)](https://repology.org/project/rclone/versions) + +## Docker installation {#docker} + +The rclone developers maintain a [docker image for rclone](https://hub.docker.com/r/rclone/rclone). + +These images are built as part of the release process based on a +minimal Alpine Linux. The `:latest` tag will always point to the latest stable release. You can use the `:beta` tag to get the latest build from master. You can @@ -188,16 +259,7 @@ ls ~/data/mount kill %1 ``` -## Install on Windows via Chocolateley Packet Manager - -Make sure you have [Choco](https://chocolatey.org/) installed -``` -choco search rclone -choco install rclone -``` -This will install rclone on your windows machine - -## Install from source +## Source installation {#source} Make sure you have git and [Go](https://golang.org/) installed. Go version 1.17 or newer is required, latest release is recommended. @@ -216,7 +278,7 @@ in the same folder. As an initial check you can now run `./rclone version` (`.\rclone version` on Windows). Note that on macOS and Windows the [mount](https://rclone.org/commands/rclone_mount/) -command will not be available unless you specify additional build tag `cmount`. +command will not be available unless you specify an additional build tag `cmount`. ``` go build -tags cmount @@ -235,7 +297,7 @@ distribution (make sure you install it in the classic mingw64 subsystem, the ucrt64 version is not compatible). Additionally, on Windows, you must install the third party utility -[WinFsp](http://www.secfs.net/winfsp/), with the "Developer" feature selected. +[WinFsp](https://winfsp.dev/), with the "Developer" feature selected. If building with cgo, you must also set environment variable CPATH pointing to the fuse include directory within the WinFsp installation (normally `C:\Program Files (x86)\WinFsp\inc\fuse`). @@ -283,7 +345,7 @@ with the current version): go get github.com/rclone/rclone ``` -## Installation with Ansible +## Ansible installation {#ansible} This can be done with [Stefan Weichinger's ansible role](https://github.com/stefangweichinger/ansible-rclone). @@ -299,7 +361,7 @@ Instructions - rclone ``` -## Portable installation +## Portable installation {#portable} As mentioned [above](https://rclone.org/install/#quickstart), rclone is single executable (`rclone`, or `rclone.exe` on Windows) that you can download as a From dbd9ce78e6ccb8769f027b3948b8c078996d5ffd Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 16 Sep 2022 16:57:23 +0100 Subject: [PATCH 275/560] =?UTF-8?q?Add=20=C3=98yvind=20Heddeland=20Instefj?= =?UTF-8?q?ord=20to=20contributors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index a0ffd369022cd..61b92dbc2f220 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -637,3 +637,4 @@ put them back in again.` >}} * Ryan Morey <4590343+rmorey@users.noreply.github.com> * Simon Bos * YFdyh000 * Josh Soref <2119212+jsoref@users.noreply.github.com> + * Øyvind Heddeland Instefjord From 90cda2d6c225293f5b73c035d6ba803ca3451445 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 16 Sep 2022 16:57:23 +0100 Subject: [PATCH 276/560] Add Dmitry Deniskin to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 61b92dbc2f220..0d35466105490 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -638,3 +638,4 @@ put them back in again.` >}} * Simon Bos * YFdyh000 * Josh Soref <2119212+jsoref@users.noreply.github.com> * Øyvind Heddeland Instefjord + * Dmitry Deniskin <110819396+ddeniskin@users.noreply.github.com> From 04b54bbb1e077a005f925cac7c7f7d158a37703c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 16 Sep 2022 16:57:23 +0100 Subject: [PATCH 277/560] Add Alexander Knorr to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 0d35466105490..a8f03173344a1 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -639,3 +639,4 @@ put them back in again.` >}} * YFdyh000 * Josh Soref <2119212+jsoref@users.noreply.github.com> * Øyvind Heddeland Instefjord * Dmitry Deniskin <110819396+ddeniskin@users.noreply.github.com> + * Alexander Knorr <106825+opexxx@users.noreply.github.com> From 1c99661d8cfbafa86ea186a3d0c8f07993a122a9 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 15 Sep 2022 17:18:35 +0100 Subject: [PATCH 278/560] onedrive: disable change notify in China region since it is not supported Fixes #6444 --- backend/onedrive/onedrive.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 29afd3976416f..e7674c019935b 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -891,6 +891,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e }).Fill(ctx, f) f.srv.SetErrorHandler(errorHandler) + // Disable change polling in China region + // See: https://github.com/rclone/rclone/issues/6444 + if f.opt.Region == regionCN { + f.features.ChangeNotify = nil + } + // Renew the token in the background f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { _, _, err := f.readMetaDataForPath(ctx, "") From 4f374bc26475316aaf9f4392c58017507de9e057 Mon Sep 17 00:00:00 2001 From: Richard Bateman Date: Sat, 17 Sep 2022 10:28:44 -0600 Subject: [PATCH 279/560] s3: add --s3-sse-customer-key-base64 to supply keys with binary data Fixes #6400 --- backend/s3/s3.go | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 25257bb85aaec..6abfd541c4aee 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1566,8 +1566,21 @@ isn't set then "acl" is used instead.`, Help: "arn:aws:kms:*", }}, }, { - Name: "sse_customer_key", - Help: "If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.", + Name: "sse_customer_key", + Help: `To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data. + +Alternatively you can provide --sse-customer-key-base64.`, + Provider: "AWS,Ceph,ChinaMobile,Minio", + Advanced: true, + Examples: []fs.OptionExample{{ + Value: "", + Help: "None", + }}, + }, { + Name: "sse_customer_key_base64", + Help: `If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data. + +Alternatively you can provide --sse-customer-key.`, Provider: "AWS,Ceph,ChinaMobile,Minio", Advanced: true, Examples: []fs.OptionExample{{ @@ -2142,6 +2155,7 @@ type Options struct { SSEKMSKeyID string `config:"sse_kms_key_id"` SSECustomerAlgorithm string `config:"sse_customer_algorithm"` SSECustomerKey string `config:"sse_customer_key"` + SSECustomerKeyBase64 string `config:"sse_customer_key_base64"` SSECustomerKeyMD5 string `config:"sse_customer_key_md5"` StorageClass string `config:"storage_class"` UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` @@ -2679,6 +2693,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e if opt.BucketACL == "" { opt.BucketACL = opt.ACL } + if opt.SSECustomerKeyBase64 != "" && opt.SSECustomerKey != "" { + return nil, errors.New("s3: can't use sse_customer_key and sse_customer_key_base64 at the same time") + } else if opt.SSECustomerKeyBase64 != "" { + // Decode the base64-encoded key and store it in the SSECustomerKey field + decoded, err := base64.StdEncoding.DecodeString(opt.SSECustomerKeyBase64) + if err != nil { + return nil, fmt.Errorf("s3: Could not decode sse_customer_key_base64: %w", err) + } + opt.SSECustomerKey = string(decoded) + } if opt.SSECustomerKey != "" && opt.SSECustomerKeyMD5 == "" { // calculate CustomerKeyMD5 if not supplied md5sumBinary := md5.Sum([]byte(opt.SSECustomerKey)) From bc09105d2eaef22b388b88d2196e1caa4f3f13ee Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 18 Sep 2022 11:30:59 +0100 Subject: [PATCH 280/560] Add Richard Bateman to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index a8f03173344a1..a4375fc9abf95 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -640,3 +640,4 @@ put them back in again.` >}} * Øyvind Heddeland Instefjord * Dmitry Deniskin <110819396+ddeniskin@users.noreply.github.com> * Alexander Knorr <106825+opexxx@users.noreply.github.com> + * Richard Bateman From fb6ddd680c6c2723dcc682a5a0ecb8070b25395c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 14 Sep 2022 09:02:58 +0100 Subject: [PATCH 281/560] compress: fix crash due to nil metadata #6434 Before this fix, if an error ocurred reading the metadata, it could be set as nil and then used, causing a crash. This fix changes the readMetadata function so it returns an error, and the error is always set if the metadata returned is nil. --- backend/compress/compress.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/backend/compress/compress.go b/backend/compress/compress.go index 2ae192eb9982e..b0ada83c6973d 100644 --- a/backend/compress/compress.go +++ b/backend/compress/compress.go @@ -367,9 +367,9 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { if err != nil { return nil, err } - meta := readMetadata(ctx, mo) - if meta == nil { - return nil, errors.New("error decoding metadata") + meta, err := readMetadata(ctx, mo) + if err != nil { + return nil, fmt.Errorf("error decoding metadata: %w", err) } // Create our Object o, err := f.Fs.NewObject(ctx, makeDataName(remote, meta.CompressionMetadata.Size, meta.Mode)) @@ -1040,24 +1040,19 @@ func newMetadata(size int64, mode int, cmeta sgzip.GzipMetadata, md5 string, mim } // This function will read the metadata from a metadata object. -func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata) { +func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err error) { // Open our meradata object rc, err := mo.Open(ctx) if err != nil { - return nil + return nil, err } - defer func() { - err := rc.Close() - if err != nil { - fs.Errorf(mo, "Error closing object: %v", err) - } - }() + defer fs.CheckClose(rc, &err) jr := json.NewDecoder(rc) meta = new(ObjectMetadata) if err = jr.Decode(meta); err != nil { - return nil + return nil, err } - return meta + return meta, nil } // Remove removes this object @@ -1167,7 +1162,7 @@ func (o *Object) loadMetadataIfNotLoaded(ctx context.Context) (err error) { return err } if o.meta == nil { - o.meta = readMetadata(ctx, o.mo) + o.meta, err = readMetadata(ctx, o.mo) } return err } From 07efdb55fa025939a488765ad216a906654186a7 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 15 Sep 2022 17:38:56 +0100 Subject: [PATCH 282/560] compress: fix error handling to not use or return nil objects #6434 --- backend/compress/compress.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/backend/compress/compress.go b/backend/compress/compress.go index b0ada83c6973d..6429e5a7e6652 100644 --- a/backend/compress/compress.go +++ b/backend/compress/compress.go @@ -373,7 +373,10 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { } // Create our Object o, err := f.Fs.NewObject(ctx, makeDataName(remote, meta.CompressionMetadata.Size, meta.Mode)) - return f.newObject(o, mo, meta), err + if err != nil { + return nil, err + } + return f.newObject(o, mo, meta), nil } // checkCompressAndType checks if an object is compressible and determines it's mime type @@ -677,7 +680,7 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob } return nil, err } - return f.newObject(dataObject, mo, meta), err + return f.newObject(dataObject, mo, meta), nil } // Put in to the remote path with the modTime given of the given size @@ -1097,6 +1100,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op origName := o.Remote() if o.meta.Mode != Uncompressed || compressible { newObject, err = o.f.putWithCustomFunctions(ctx, in, o.f.wrapInfo(src, origName, src.Size()), options, o.f.Fs.Put, updateMeta, compressible, mimeType) + if err != nil { + return err + } if newObject.Object.Remote() != o.Object.Remote() { if removeErr := o.Object.Remove(ctx); removeErr != nil { return removeErr @@ -1110,9 +1116,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } // If we are, just update the object and metadata newObject, err = o.f.putWithCustomFunctions(ctx, in, src, options, update, updateMeta, compressible, mimeType) - } - if err != nil { - return err + if err != nil { + return err + } } // Update object metadata and return o.Object = newObject.Object From 5f13d84135f8d164cb17562a602de2af157295d6 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 17 Sep 2022 16:40:45 +0100 Subject: [PATCH 283/560] compress: add extra debugging in case we have a repeat of #6434 --- backend/compress/compress.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/compress/compress.go b/backend/compress/compress.go index 6429e5a7e6652..6bfe1210c6595 100644 --- a/backend/compress/compress.go +++ b/backend/compress/compress.go @@ -29,6 +29,7 @@ import ( "github.com/rclone/rclone/fs/config/configstruct" "github.com/rclone/rclone/fs/fspath" "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/log" "github.com/rclone/rclone/fs/object" "github.com/rclone/rclone/fs/operations" ) @@ -1129,6 +1130,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op // This will initialize the variables of a new press Object. The metadata object, mo, and metadata struct, meta, must be specified. func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object { + if o == nil { + log.Trace(nil, "newObject(%#v, %#v, %#v) called with nil o", o, mo, meta) + } return &Object{ Object: o, f: f, @@ -1141,6 +1145,9 @@ func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object // This initializes the variables of a press Object with only the size. The metadata will be loaded later on demand. func (f *Fs) newObjectSizeAndNameOnly(o fs.Object, moName string, size int64) *Object { + if o == nil { + log.Trace(nil, "newObjectSizeAndNameOnly(%#v, %#v, %#v) called with nil o", o, moName, size) + } return &Object{ Object: o, f: f, From 88086643f72d718c10ecb9831e101e5c787eb5cb Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 15 Sep 2022 08:56:29 +0100 Subject: [PATCH 284/560] ftp: adapt to library changes to fix connection errors #6426 In https://github.com/jlaffaye/ftp/commit/212daf295f the upstream FTP library changed the way adding your own dialer works which meant that connections when using explicit FTP were failing. This patch reworks our connection code to bring it into the expectations of the library. --- backend/ftp/ftp.go | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index b48a06eee4f4b..480f63aba16ad 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -336,14 +336,44 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) { fs.Debugf(f, "Connecting to FTP server") // Make ftp library dial with fshttp dialer optionally using TLS + initialConnection := true dial := func(network, address string) (conn net.Conn, err error) { + fs.Debugf(f, "dial(%q,%q)", network, address) + defer func() { + fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err) + }() conn, err = fshttp.NewDialer(ctx).Dial(network, address) - if f.tlsConf != nil && err == nil { - conn = tls.Client(conn, f.tlsConf) + if err != nil { + return nil, err } - return + // Connect using cleartext only for non TLS + if f.tlsConf == nil { + return conn, nil + } + // Initial connection only needs to be cleartext for explicit TLS + if f.opt.ExplicitTLS && initialConnection { + initialConnection = false + return conn, nil + } + // Upgrade connection to TLS + tlsConn := tls.Client(conn, f.tlsConf) + // Do the initial handshake - tls.Client doesn't do it for us + // If we do this then connections to proftpd/pureftpd lock up + // See: https://github.com/rclone/rclone/issues/6426 + // See: https://github.com/jlaffaye/ftp/issues/282 + if false { + err = tlsConn.HandshakeContext(ctx) + if err != nil { + _ = conn.Close() + return nil, err + } + } + return tlsConn, nil + } + ftpConfig := []ftp.DialOption{ + ftp.DialWithContext(ctx), + ftp.DialWithDialFunc(dial), } - ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)} if f.opt.TLS { // Our dialer takes care of TLS but ftp library also needs tlsConf @@ -351,12 +381,6 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) { ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf)) } else if f.opt.ExplicitTLS { ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf)) - // Initial connection needs to be cleartext for explicit TLS - conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr) - if err != nil { - return nil, err - } - ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn)) } if f.opt.DisableEPSV { ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true)) From 67625b1dbd763e70be4995441c1e18ac631249e9 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 19 Sep 2022 19:45:52 +0100 Subject: [PATCH 285/560] ftp: increase timeouts on tests as they were failing locally --- backend/ftp/ftp_internal_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/ftp/ftp_internal_test.go b/backend/ftp/ftp_internal_test.go index bd1d8c6773469..3e06bb9b4a8e3 100644 --- a/backend/ftp/ftp_internal_test.go +++ b/backend/ftp/ftp_internal_test.go @@ -34,9 +34,9 @@ func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs { // test that big file uploads do not cause network i/o timeout func (f *Fs) testUploadTimeout(t *testing.T) { const ( - fileSize = 100000000 // 100 MiB - idleTimeout = 40 * time.Millisecond // small because test server is local - maxTime = 10 * time.Second // prevent test hangup + fileSize = 100000000 // 100 MiB + idleTimeout = 1 * time.Second // small because test server is local + maxTime = 10 * time.Second // prevent test hangup ) if testing.Short() { From 7e547822d667e644576d250655fa3013a7159076 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Mon, 19 Sep 2022 20:51:07 +0200 Subject: [PATCH 286/560] build: update GitHub actions to latest versions --- .github/workflows/build.yml | 14 +++++++------- .github/workflows/build_publish_docker_image.yml | 2 +- .../build_publish_release_docker_image.yml | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c7c76dee1ed6b..79ffc62a6de99 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,12 +97,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Install Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: stable: 'false' go-version: ${{ matrix.go }} @@ -162,7 +162,7 @@ jobs: env - name: Go module cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} @@ -226,7 +226,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Code quality test uses: golangci/golangci-lint-action@v3 @@ -242,18 +242,18 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 # Upgrade together with NDK version - name: Set up Go - uses: actions/setup-go@v1 + uses: actions/setup-go@v3 with: go-version: 1.19.x - name: Go module cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} diff --git a/.github/workflows/build_publish_docker_image.yml b/.github/workflows/build_publish_docker_image.yml index 053a6eb03167c..af52509732534 100644 --- a/.github/workflows/build_publish_docker_image.yml +++ b/.github/workflows/build_publish_docker_image.yml @@ -12,7 +12,7 @@ jobs: name: Build image job steps: - name: Checkout master - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Build and publish image diff --git a/.github/workflows/build_publish_release_docker_image.yml b/.github/workflows/build_publish_release_docker_image.yml index a8f3f91f72ce1..6fe9213c78761 100644 --- a/.github/workflows/build_publish_release_docker_image.yml +++ b/.github/workflows/build_publish_release_docker_image.yml @@ -11,7 +11,7 @@ jobs: name: Build image job steps: - name: Checkout master - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Get actual patch version @@ -40,7 +40,7 @@ jobs: name: Build docker plugin job steps: - name: Checkout master - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Build and publish docker plugin From 27096323db392ebc2dc4ad6b872a8e21ea7e7d1a Mon Sep 17 00:00:00 2001 From: Lorenzo Milesi Date: Mon, 26 Sep 2022 19:11:29 +0200 Subject: [PATCH 287/560] docs: remove "After" in automount example According to [systemd.automount](https://www.freedesktop.org/software/systemd/man/systemd.automount.html) manual > Note that automount units are separate from the mount itself, so you should > not set After= or Requires= for mount dependencies here. > For example, you should not set After=network-online.target or > similar on network filesystems. Doing so may result in an ordering cycle. --- docs/content/commands/rclone_mount.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/content/commands/rclone_mount.md b/docs/content/commands/rclone_mount.md index 2b88e0720570f..dfa8be1d6f7b7 100644 --- a/docs/content/commands/rclone_mount.md +++ b/docs/content/commands/rclone_mount.md @@ -354,7 +354,6 @@ optionally accompanied by systemd automount unit ``` # /etc/systemd/system/mnt-data.automount [Unit] -After=network-online.target Before=remote-fs.target [Automount] Where=/mnt/data From 7fbc928a19431f566b87c8c33d98fbb5099dad5f Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Mon, 26 Sep 2022 19:14:10 +0200 Subject: [PATCH 288/560] docs: remove "After" in systemd mount example See #6459 --- docs/content/commands/rclone_mount.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/content/commands/rclone_mount.md b/docs/content/commands/rclone_mount.md index dfa8be1d6f7b7..fa132d24c689e 100644 --- a/docs/content/commands/rclone_mount.md +++ b/docs/content/commands/rclone_mount.md @@ -341,8 +341,6 @@ mount sftp1:subdir /mnt/data -t rclone -o vfs_cache_mode=writes,sftp_key_file=/p or create systemd mount units: ``` # /etc/systemd/system/mnt-data.mount -[Unit] -After=network-online.target [Mount] Type=rclone What=sftp1:subdir From 4e078765f9429dc315a337e6d7826ebadd35c969 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Wed, 28 Sep 2022 08:44:32 +0200 Subject: [PATCH 289/560] docs: improve description of make command in install docs --- docs/content/install.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/content/install.md b/docs/content/install.md index 0480bf96c3ad2..041010092a418 100644 --- a/docs/content/install.md +++ b/docs/content/install.md @@ -312,9 +312,10 @@ go build -trimpath -ldflags -s -tags cmount ``` Instead of executing the `go build` command directly, you can run it via the -Makefile, which also sets version information and copies the resulting rclone -executable into your GOPATH bin folder (`$(go env GOPATH)/bin`, which -corresponds to `~/go/bin/rclone` by default). +Makefile. It changes the version number suffix from "-DEV" to "-beta" and +appends commit details. It also copies the resulting rclone executable into +your GOPATH bin folder (`$(go env GOPATH)/bin`, which corresponds to +`~/go/bin/rclone` by default). ``` make @@ -326,7 +327,15 @@ To include mount command on macOS and Windows with Makefile build: make GOTAGS=cmount ``` -As an alternative you can download the source, build and install rclone in one +There are other make targets that can be used for more advanced builds, +such as cross-compiling for all supported os/architectures, embedding +icon and version info resources into windows executable, and packaging +results into release artifacts. +See [Makefile](https://github.com/rclone/rclone/blob/master/Makefile) +and [cross-compile.go](https://github.com/rclone/rclone/blob/master/bin/cross-compile.go) +for details. + +Another alternative is to download the source, build and install rclone in one operation, as a regular Go package. The source will be stored it in the Go module cache, and the resulting executable will be in your GOPATH bin folder (`$(go env GOPATH)/bin`, which corresponds to `~/go/bin/rclone` by default). From 2aa264b33cb098f37eaeea43a87feec36b391ad0 Mon Sep 17 00:00:00 2001 From: Lesmiscore Date: Sat, 1 Oct 2022 00:10:57 +0900 Subject: [PATCH 290/560] smb: backend to support SMB - fixes #2042 --- README.md | 1 + backend/all/all.go | 1 + backend/smb/connpool.go | 229 +++++++++ backend/smb/smb.go | 789 +++++++++++++++++++++++++++++++ backend/smb/smb_test.go | 17 + bin/make_manual.py | 1 + docs/content/_index.md | 1 + docs/content/docs.md | 1 + docs/content/overview.md | 2 + docs/content/smb.md | 231 +++++++++ docs/layouts/chrome/navbar.html | 1 + fstest/test_all/config.yaml | 3 + fstest/testserver/init.d/TestSMB | 28 ++ go.mod | 2 + go.sum | 5 + 15 files changed, 1312 insertions(+) create mode 100644 backend/smb/connpool.go create mode 100644 backend/smb/smb.go create mode 100644 backend/smb/smb_test.go create mode 100644 docs/content/smb.md create mode 100755 fstest/testserver/init.d/TestSMB diff --git a/README.md b/README.md index 6c0e35752af55..44c8f8552cda3 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and * Seafile [:page_facing_up:](https://rclone.org/seafile/) * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs) * SFTP [:page_facing_up:](https://rclone.org/sftp/) + * SMB / CIFS [:page_facing_up:](https://rclone.org/smb/) * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath) * Storj [:page_facing_up:](https://rclone.org/storj/) * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/) diff --git a/backend/all/all.go b/backend/all/all.go index 5dae4d37db0a2..19b1e1df518de 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -44,6 +44,7 @@ import ( _ "github.com/rclone/rclone/backend/sftp" _ "github.com/rclone/rclone/backend/sharefile" _ "github.com/rclone/rclone/backend/sia" + _ "github.com/rclone/rclone/backend/smb" _ "github.com/rclone/rclone/backend/storj" _ "github.com/rclone/rclone/backend/sugarsync" _ "github.com/rclone/rclone/backend/swift" diff --git a/backend/smb/connpool.go b/backend/smb/connpool.go new file mode 100644 index 0000000000000..6f6fe82325f30 --- /dev/null +++ b/backend/smb/connpool.go @@ -0,0 +1,229 @@ +package smb + +import ( + "context" + "fmt" + "net" + "sync/atomic" + "time" + + smb2 "github.com/hirochachacha/go-smb2" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/fshttp" +) + +// dial starts a client connection to the given SMB server. It is a +// convenience function that connects to the given network address, +// initiates the SMB handshake, and then sets up a Client. +func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) { + dialer := fshttp.NewDialer(ctx) + tconn, err := dialer.Dial(network, addr) + if err != nil { + return nil, err + } + + pass := "" + if f.opt.Pass != "" { + pass, err = obscure.Reveal(f.opt.Pass) + if err != nil { + return nil, err + } + } + + d := &smb2.Dialer{ + Initiator: &smb2.NTLMInitiator{ + User: f.opt.User, + Password: pass, + Domain: f.opt.Domain, + }, + } + + session, err := d.DialContext(ctx, tconn) + if err != nil { + return nil, err + } + + return &conn{ + smbSession: session, + conn: &tconn, + }, nil +} + +// conn encapsulates a SMB client and corresponding SMB client +type conn struct { + conn *net.Conn + smbSession *smb2.Session + smbShare *smb2.Share + shareName string +} + +// Closes the connection +func (c *conn) close() (err error) { + if c.smbShare != nil { + err = c.smbShare.Umount() + } + sessionLogoffErr := c.smbSession.Logoff() + if err != nil { + return err + } + return sessionLogoffErr +} + +// True if it's closed +func (c *conn) closed() bool { + var nopErr error + if c.smbShare != nil { + // stat the current directory + _, nopErr = c.smbShare.Stat(".") + } else { + // list the shares + _, nopErr = c.smbSession.ListSharenames() + } + return nopErr == nil +} + +// Show that we are using a SMB session +// +// Call removeSession() when done +func (f *Fs) addSession() { + atomic.AddInt32(&f.sessions, 1) +} + +// Show the SMB session is no longer in use +func (f *Fs) removeSession() { + atomic.AddInt32(&f.sessions, -1) +} + +// getSessions shows whether there are any sessions in use +func (f *Fs) getSessions() int32 { + return atomic.LoadInt32(&f.sessions) +} + +// Open a new connection to the SMB server. +func (f *Fs) newConnection(ctx context.Context, share string) (c *conn, err error) { + c, err = f.dial(ctx, "tcp", f.opt.Host+":"+f.opt.Port) + if err != nil { + return nil, fmt.Errorf("couldn't connect SMB: %w", err) + } + if share != "" { + // mount the specified share as well if user requested + c.smbShare, err = c.smbSession.Mount(share) + if err != nil { + _ = c.smbSession.Logoff() + return nil, fmt.Errorf("couldn't initialize SMB: %w", err) + } + c.smbShare = c.smbShare.WithContext(ctx) + } + return c, nil +} + +// Ensure the specified share is mounted or the session is unmounted +func (c *conn) mountShare(share string) (err error) { + if c.shareName == share { + return nil + } + if c.smbShare != nil { + err = c.smbShare.Umount() + c.smbShare = nil + } + if err != nil { + return + } + if share != "" { + c.smbShare, err = c.smbSession.Mount(share) + if err != nil { + return + } + } + c.shareName = share + return nil +} + +// Get a SMB connection from the pool, or open a new one +func (f *Fs) getConnection(ctx context.Context, share string) (c *conn, err error) { + accounting.LimitTPS(ctx) + f.poolMu.Lock() + for len(f.pool) > 0 { + c = f.pool[0] + f.pool = f.pool[1:] + err = c.mountShare(share) + if err == nil { + break + } + fs.Debugf(f, "Discarding unusable SMB connection: %v", err) + c = nil + } + f.poolMu.Unlock() + if c != nil { + return c, nil + } + err = f.pacer.Call(func() (bool, error) { + c, err = f.newConnection(ctx, share) + if err != nil { + return true, err + } + return false, nil + }) + return c, err +} + +// Return a SMB connection to the pool +// +// It nils the pointed to connection out so it can't be reused +func (f *Fs) putConnection(pc **conn) { + c := *pc + *pc = nil + + var nopErr error + if c.smbShare != nil { + // stat the current directory + _, nopErr = c.smbShare.Stat(".") + } else { + // list the shares + _, nopErr = c.smbSession.ListSharenames() + } + if nopErr != nil { + fs.Debugf(f, "Connection failed, closing: %v", nopErr) + _ = c.close() + return + } + + f.poolMu.Lock() + f.pool = append(f.pool, c) + if f.opt.IdleTimeout > 0 { + f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer + } + f.poolMu.Unlock() +} + +// Drain the pool of any connections +func (f *Fs) drainPool(ctx context.Context) (err error) { + f.poolMu.Lock() + defer f.poolMu.Unlock() + if sessions := f.getSessions(); sessions != 0 { + fs.Debugf(f, "Not closing %d unused connections as %d sessions active", len(f.pool), sessions) + if f.opt.IdleTimeout > 0 { + f.drain.Reset(time.Duration(f.opt.IdleTimeout)) // nudge on the pool emptying timer + } + return nil + } + if f.opt.IdleTimeout > 0 { + f.drain.Stop() + } + if len(f.pool) != 0 { + fs.Debugf(f, "Closing %d unused connections", len(f.pool)) + } + for i, c := range f.pool { + if !c.closed() { + cErr := c.close() + if cErr != nil { + err = cErr + } + } + f.pool[i] = nil + } + f.pool = nil + return err +} diff --git a/backend/smb/smb.go b/backend/smb/smb.go new file mode 100644 index 0000000000000..3e750cc1fb9df --- /dev/null +++ b/backend/smb/smb.go @@ -0,0 +1,789 @@ +// Package smb provides an interface to SMB servers +package smb + +import ( + "context" + "fmt" + "io" + "os" + "path" + "strings" + "sync" + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/bucket" + "github.com/rclone/rclone/lib/encoder" + "github.com/rclone/rclone/lib/env" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/readers" +) + +const ( + minSleep = 100 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential +) + +var ( + currentUser = env.CurrentUser() +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "smb", + Description: "SMB / CIFS", + NewFs: NewFs, + + Options: []fs.Option{{ + Name: "host", + Help: "SMB server hostname to connect to.\n\nE.g. \"example.com\".", + Required: true, + }, { + Name: "user", + Help: "SMB username.", + Default: currentUser, + }, { + Name: "port", + Help: "SMB port number.", + Default: 445, + }, { + Name: "pass", + Help: "SMB password.", + IsPassword: true, + }, { + Name: "domain", + Help: "Domain name for NTLM authentication.", + Default: "WORKGROUP", + }, { + Name: "idle_timeout", + Default: fs.Duration(60 * time.Second), + Help: `Max time before closing idle connections. + +If no connections have been returned to the connection pool in the time +given, rclone will empty the connection pool. + +Set to 0 to keep connections indefinitely. +`, + Advanced: true, + }, { + Name: "hide_special_share", + Help: "Hide special shares (e.g. print$) which users aren't supposed to access.", + Default: true, + Advanced: true, + }, { + Name: "case_insensitive", + Help: "Whether the server is configured to be case-insensitive.\n\nAlways true on Windows shares.", + Default: true, + Advanced: true, + }, { + Name: config.ConfigEncoding, + Help: config.ConfigEncodingHelp, + Advanced: true, + Default: encoder.EncodeZero | + // path separator + encoder.EncodeSlash | + encoder.EncodeBackSlash | + // windows + encoder.EncodeWin | + encoder.EncodeCtl | + encoder.EncodeDot | + // the file turns into 8.3 names (and cannot be converted back) + encoder.EncodeRightSpace | + encoder.EncodeRightPeriod | + // + encoder.EncodeInvalidUtf8, + }, + }}) +} + +// Options defines the configuration for this backend +type Options struct { + Host string `config:"host"` + Port string `config:"port"` + User string `config:"user"` + Pass string `config:"pass"` + Domain string `config:"domain"` + HideSpecial bool `config:"hide_special_share"` + CaseInsensitive bool `config:"case_insensitive"` + IdleTimeout fs.Duration `config:"idle_timeout"` + + Enc encoder.MultiEncoder `config:"encoding"` +} + +// Fs represents a SMB remote +type Fs struct { + name string // name of this remote + root string // the path we are working on if any + opt Options // parsed config options + features *fs.Features // optional features + pacer *fs.Pacer // pacer for operations + + sessions int32 + poolMu sync.Mutex + pool []*conn + drain *time.Timer // used to drain the pool when we stop using the connections + + ctx context.Context +} + +// Object describes a file at the server +type Object struct { + fs *Fs // reference to Fs + remote string // the remote path + statResult os.FileInfo +} + +// NewFs constructs an Fs from the path +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + root = strings.Trim(root, "/") + + f := &Fs{ + name: name, + opt: *opt, + ctx: ctx, + root: root, + } + f.features = (&fs.Features{ + CaseInsensitive: opt.CaseInsensitive, + CanHaveEmptyDirectories: true, + BucketBased: true, + }).Fill(ctx, f) + + f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))) + // set the pool drainer timer going + if opt.IdleTimeout > 0 { + f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) }) + } + + // test if the root exists as a file + share, dir := f.split("") + if share == "" || dir == "" { + return f, nil + } + cn, err := f.getConnection(ctx, share) + if err != nil { + return nil, err + } + stat, err := cn.smbShare.Stat(f.toSambaPath(dir)) + f.putConnection(&cn) + if err != nil { + // ignore stat error here + return f, nil + } + if !stat.IsDir() { + f.root, err = path.Dir(root), fs.ErrorIsFile + } + fs.Debugf(f, "Using root directory %q", f.root) + return f, err +} + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + bucket, file := f.split("") + if bucket == "" { + return fmt.Sprintf("smb://%s@%s:%s/", f.opt.User, f.opt.Host, f.opt.Port) + } + return fmt.Sprintf("smb://%s@%s:%s/%s/%s", f.opt.User, f.opt.Host, f.opt.Port, bucket, file) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Hashes returns nothing as SMB itself doesn't have a way to tell checksums +func (f *Fs) Hashes() hash.Set { + return hash.NewHashSet() +} + +// Precision returns the precision of mtime +func (f *Fs) Precision() time.Duration { + return time.Millisecond +} + +// NewObject creates a new file object +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + share, path := f.split(remote) + return f.findObjectSeparate(ctx, share, path) +} + +func (f *Fs) findObjectSeparate(ctx context.Context, share, path string) (fs.Object, error) { + if share == "" || path == "" { + return nil, fs.ErrorIsDir + } + cn, err := f.getConnection(ctx, share) + if err != nil { + return nil, err + } + stat, err := cn.smbShare.Stat(f.toSambaPath(path)) + f.putConnection(&cn) + if err != nil { + return nil, translateError(err, false) + } + if stat.IsDir() { + return nil, fs.ErrorIsDir + } + + return f.makeEntry(share, path, stat), nil +} + +// Mkdir creates a directory on the server +func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) { + share, path := f.split(dir) + if share == "" || path == "" { + return nil + } + cn, err := f.getConnection(ctx, share) + if err != nil { + return err + } + err = cn.smbShare.MkdirAll(f.toSambaPath(path), 0o755) + f.putConnection(&cn) + return err +} + +// Rmdir removes an empty directory on the server +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + share, path := f.split(dir) + if share == "" || path == "" { + return nil + } + cn, err := f.getConnection(ctx, share) + if err != nil { + return err + } + err = cn.smbShare.Remove(f.toSambaPath(path)) + f.putConnection(&cn) + return err +} + +// Put uploads a file +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + o := &Object{ + fs: f, + remote: src.Remote(), + } + + err := o.Update(ctx, in, src, options...) + if err == nil { + return o, nil + } + + return nil, err +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +// +// May create the object even if it returns an error - if so +// will return the object and the error, otherwise will return +// nil and the error +func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + o := &Object{ + fs: f, + remote: src.Remote(), + } + + err := o.Update(ctx, in, src, options...) + if err == nil { + return o, nil + } + + return nil, err +} + +// Move src to this remote using server-side move operations. +// +// This is stored with the remote path given. +// +// It returns the destination Object and a possible error. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (_ fs.Object, err error) { + dstShare, dstPath := f.split(remote) + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + srcShare, srcPath := srcObj.split() + if dstShare != srcShare { + fs.Debugf(src, "Can't move - must be on the same share") + return nil, fs.ErrorCantMove + } + + err = f.ensureDirectory(ctx, dstShare, dstPath) + if err != nil { + return nil, fmt.Errorf("failed to make parent directories: %w", err) + } + + cn, err := f.getConnection(ctx, dstShare) + if err != nil { + return nil, err + } + err = cn.smbShare.Rename(f.toSambaPath(srcPath), f.toSambaPath(dstPath)) + f.putConnection(&cn) + if err != nil { + return nil, translateError(err, false) + } + return f.findObjectSeparate(ctx, dstShare, dstPath) +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server-side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) { + dstShare, dstPath := f.split(dstRemote) + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return fs.ErrorCantDirMove + } + srcShare, srcPath := srcFs.split(srcRemote) + if dstShare != srcShare { + fs.Debugf(src, "Can't move - must be on the same share") + return fs.ErrorCantDirMove + } + + err = f.ensureDirectory(ctx, dstShare, dstPath) + if err != nil { + return fmt.Errorf("failed to make parent directories: %w", err) + } + + cn, err := f.getConnection(ctx, dstShare) + if err != nil { + return err + } + defer f.putConnection(&cn) + + _, err = cn.smbShare.Stat(dstPath) + if os.IsNotExist(err) { + err = cn.smbShare.Rename(f.toSambaPath(srcPath), f.toSambaPath(dstPath)) + return translateError(err, true) + } + return fs.ErrorDirExists +} + +// List files and directories in a directory +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + share, _path := f.split(dir) + + cn, err := f.getConnection(ctx, share) + if err != nil { + return nil, err + } + defer f.putConnection(&cn) + + if share == "" { + shares, err := cn.smbSession.ListSharenames() + for _, shh := range shares { + shh = f.toNativePath(shh) + if strings.HasSuffix(shh, "$") && f.opt.HideSpecial { + continue + } + entries = append(entries, fs.NewDir(shh, time.Time{})) + } + return entries, err + } + + dirents, err := cn.smbShare.ReadDir(f.toSambaPath(_path)) + if err != nil { + return entries, translateError(err, true) + } + for _, file := range dirents { + nfn := f.toNativePath(file.Name()) + if file.IsDir() { + entries = append(entries, fs.NewDir(path.Join(dir, nfn), file.ModTime())) + } else { + entries = append(entries, f.makeEntryRelative(share, _path, nfn, file)) + } + } + + return entries, nil +} + +// About returns things about remaining and used spaces +func (f *Fs) About(ctx context.Context) (_ *fs.Usage, err error) { + share, dir := f.split("/") + if share == "" { + return nil, fs.ErrorListBucketRequired + } + dir = f.toSambaPath(dir) + + cn, err := f.getConnection(ctx, share) + if err != nil { + return nil, err + } + stat, err := cn.smbShare.Statfs(dir) + f.putConnection(&cn) + if err != nil { + return nil, err + } + + bs := int64(stat.BlockSize()) + usage := &fs.Usage{ + Total: fs.NewUsageValue(bs * int64(stat.TotalBlockCount())), + Used: fs.NewUsageValue(bs * int64(stat.TotalBlockCount()-stat.FreeBlockCount())), + Free: fs.NewUsageValue(bs * int64(stat.AvailableBlockCount())), + } + return usage, nil +} + +// Shutdown the backend, closing any background tasks and any +// cached connections. +func (f *Fs) Shutdown(ctx context.Context) error { + return f.drainPool(ctx) +} + +func (f *Fs) makeEntry(share, _path string, stat os.FileInfo) *Object { + remote := path.Join(share, _path) + return &Object{ + fs: f, + remote: trimPathPrefix(remote, f.root), + statResult: stat, + } +} + +func (f *Fs) makeEntryRelative(share, _path, relative string, stat os.FileInfo) *Object { + return f.makeEntry(share, path.Join(_path, relative), stat) +} + +func (f *Fs) ensureDirectory(ctx context.Context, share, _path string) error { + cn, err := f.getConnection(ctx, share) + if err != nil { + return err + } + err = cn.smbShare.MkdirAll(f.toSambaPath(path.Dir(_path)), 0o755) + f.putConnection(&cn) + return err +} + +/// Object + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// ModTime is the last modified time (read-only) +func (o *Object) ModTime(ctx context.Context) time.Time { + return o.statResult.ModTime() +} + +// Size is the file length +func (o *Object) Size() int64 { + return o.statResult.Size() +} + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Hash always returns empty value +func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// Storable returns if this object is storable +func (o *Object) Storable() bool { + return true +} + +// SetModTime sets modTime on a particular file +func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) { + share, reqDir := o.split() + if share == "" || reqDir == "" { + return fs.ErrorCantSetModTime + } + reqDir = o.fs.toSambaPath(reqDir) + + cn, err := o.fs.getConnection(ctx, share) + if err != nil { + return err + } + defer o.fs.putConnection(&cn) + + err = cn.smbShare.Chtimes(reqDir, t, t) + if err != nil { + return err + } + + fi, err := cn.smbShare.Stat(reqDir) + if err == nil { + o.statResult = fi + } + return err +} + +// Open an object for read +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + share, filename := o.split() + if share == "" || filename == "" { + return nil, fs.ErrorIsDir + } + filename = o.fs.toSambaPath(filename) + + var offset, limit int64 = 0, -1 + for _, option := range options { + switch x := option.(type) { + case *fs.SeekOption: + offset = x.Offset + case *fs.RangeOption: + offset, limit = x.Decode(o.Size()) + default: + if option.Mandatory() { + fs.Logf(o, "Unsupported mandatory option: %v", option) + } + } + } + + o.fs.addSession() // Show session in use + defer o.fs.removeSession() + + cn, err := o.fs.getConnection(ctx, share) + if err != nil { + return nil, err + } + fl, err := cn.smbShare.OpenFile(filename, os.O_RDONLY, 0) + if err != nil { + o.fs.putConnection(&cn) + return nil, fmt.Errorf("failed to open: %w", err) + } + pos, err := fl.Seek(offset, io.SeekStart) + if err != nil { + o.fs.putConnection(&cn) + return nil, fmt.Errorf("failed to seek: %w", err) + } + if pos != offset { + o.fs.putConnection(&cn) + return nil, fmt.Errorf("failed to seek: wrong position (expected=%d, reported=%d)", offset, pos) + } + + in = readers.NewLimitedReadCloser(fl, limit) + in = &boundReadCloser{ + rc: in, + close: func() error { + o.fs.putConnection(&cn) + return nil + }, + } + + return in, nil +} + +// Update the Object from in with modTime and size +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + share, filename := o.split() + if share == "" || filename == "" { + return fs.ErrorIsDir + } + + err = o.fs.ensureDirectory(ctx, share, filename) + if err != nil { + return fmt.Errorf("failed to make parent directories: %w", err) + } + + filename = o.fs.toSambaPath(filename) + + o.fs.addSession() // Show session in use + defer o.fs.removeSession() + + cn, err := o.fs.getConnection(ctx, share) + if err != nil { + return err + } + defer func() { + o.statResult, _ = cn.smbShare.Stat(filename) + o.fs.putConnection(&cn) + }() + + fl, err := cn.smbShare.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("failed to open: %w", err) + } + + // remove the file if upload failed + remove := func() { + // Windows doesn't allow removal of files without closing file + removeErr := fl.Close() + if removeErr != nil { + fs.Debugf(src, "failed to close the file for delete: %v", removeErr) + // try to remove the file anyway; the file may be already closed + } + + removeErr = cn.smbShare.Remove(filename) + if removeErr != nil { + fs.Debugf(src, "failed to remove: %v", removeErr) + } else { + fs.Debugf(src, "removed after failed upload: %v", err) + } + } + + _, err = fl.ReadFrom(in) + if err != nil { + remove() + return fmt.Errorf("Update ReadFrom failed: %w", err) + } + + err = fl.Close() + if err != nil { + remove() + return fmt.Errorf("Update Close failed: %w", err) + } + + // Set the modified time + err = o.SetModTime(ctx, src.ModTime(ctx)) + if err != nil { + return fmt.Errorf("Update SetModTime failed: %w", err) + } + + return nil +} + +// Remove an object +func (o *Object) Remove(ctx context.Context) (err error) { + share, filename := o.split() + if share == "" || filename == "" { + return fs.ErrorIsDir + } + filename = o.fs.toSambaPath(filename) + + cn, err := o.fs.getConnection(ctx, share) + if err != nil { + return err + } + + err = cn.smbShare.Remove(filename) + o.fs.putConnection(&cn) + + return err +} + +// String converts this Object to a string +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +/// Misc + +// split returns share name and path in the share from the rootRelativePath +// relative to f.root +func (f *Fs) split(rootRelativePath string) (shareName, filepath string) { + return bucket.Split(path.Join(f.root, rootRelativePath)) +} + +// split returns share name and path in the share from the object +func (o *Object) split() (shareName, filepath string) { + return o.fs.split(o.remote) +} + +func (f *Fs) toSambaPath(path string) string { + // 1. encode via Rclone's escaping system + // 2. convert to backslash-separated path + return strings.ReplaceAll(f.opt.Enc.FromStandardPath(path), "/", "\\") +} + +func (f *Fs) toNativePath(path string) string { + // 1. convert *back* to slash-separated path + // 2. encode via Rclone's escaping system + return f.opt.Enc.ToStandardPath(strings.ReplaceAll(path, "\\", "/")) +} + +func ensureSuffix(s, suffix string) string { + if strings.HasSuffix(s, suffix) { + return s + } + return s + suffix +} + +func trimPathPrefix(s, prefix string) string { + // we need to clean the paths to make tests pass! + s = betterPathClean(s) + prefix = betterPathClean(prefix) + if s == prefix || s == prefix+"/" { + return "" + } + prefix = ensureSuffix(prefix, "/") + return strings.TrimPrefix(s, prefix) +} + +func betterPathClean(p string) string { + d := path.Clean(p) + if d == "." { + return "" + } + return d +} + +type boundReadCloser struct { + rc io.ReadCloser + close func() error +} + +func (r *boundReadCloser) Read(p []byte) (n int, err error) { + return r.rc.Read(p) +} + +func (r *boundReadCloser) Close() error { + err1 := r.rc.Close() + err2 := r.close() + if err1 != nil { + return err1 + } + return err2 +} + +func translateError(e error, dir bool) error { + if os.IsNotExist(e) { + if dir { + return fs.ErrorDirNotFound + } + return fs.ErrorObjectNotFound + } + + return e +} + +var ( + _ fs.Fs = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.Mover = &Fs{} + _ fs.DirMover = &Fs{} + _ fs.Abouter = &Fs{} + _ fs.Shutdowner = &Fs{} + _ fs.Object = &Object{} + _ io.ReadCloser = &boundReadCloser{} +) diff --git a/backend/smb/smb_test.go b/backend/smb/smb_test.go new file mode 100644 index 0000000000000..aa5a0e4196778 --- /dev/null +++ b/backend/smb/smb_test.go @@ -0,0 +1,17 @@ +// Test smb filesystem interface +package smb_test + +import ( + "testing" + + "github.com/rclone/rclone/backend/smb" + "github.com/rclone/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestSMB:rclone", + NilObject: (*smb.Object)(nil), + }) +} diff --git a/bin/make_manual.py b/bin/make_manual.py index fa14b36a3748c..cc2e81a5eec7f 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -68,6 +68,7 @@ "putio.md", "seafile.md", "sftp.md", + "smb.md", "storj.md", "sugarsync.md", "tardigrade.md", # stub only to redirect to storj.md diff --git a/docs/content/_index.md b/docs/content/_index.md index 98bcd578406e7..5d1bf2cda4d25 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -162,6 +162,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}} {{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol" config="/sftp/" >}} {{< provider name="Sia" home="https://sia.tech/" config="/sia/" >}} +{{< provider name="SMB / CIFS" home="https://en.wikipedia.org/wiki/Server_Message_Block" config="/smb/" >}} {{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}} {{< provider name="Storj" home="https://storj.io/" config="/storj/" >}} {{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}} diff --git a/docs/content/docs.md b/docs/content/docs.md index 47757d431a37e..7f3b20378ea1d 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -69,6 +69,7 @@ See the following for detailed instructions for * [Seafile](/seafile/) * [SFTP](/sftp/) * [Sia](/sia/) + * [SMB](/smb/) * [Storj](/storj/) * [SugarSync](/sugarsync/) * [Union](/union/) diff --git a/docs/content/overview.md b/docs/content/overview.md index e2054323876b0..34f059eb34ea1 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -50,6 +50,7 @@ Here is an overview of the major features of each cloud storage system. | Seafile | - | - | No | No | - | - | | SFTP | MD5, SHA1 ² | R/W | Depends | No | - | - | | Sia | - | - | No | No | - | - | +| SMB | - | - | Yes | No | - | - | | SugarSync | - | - | No | No | - | - | | Storj | - | R | No | No | - | - | | Uptobox | - | - | No | Yes | - | - | @@ -501,6 +502,7 @@ upon backend-specific capabilities. | Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | | SFTP | No | No | Yes | Yes | No | No | Yes | No | Yes | Yes | | Sia | No | No | No | No | No | No | Yes | No | No | Yes | +| SMB | No | No | Yes | Yes | No | No | Yes | No | No | Yes | | SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes | | Storj | Yes † | No | Yes | No | No | Yes | Yes | No | No | No | | Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | diff --git a/docs/content/smb.md b/docs/content/smb.md new file mode 100644 index 0000000000000..62c1717bb9274 --- /dev/null +++ b/docs/content/smb.md @@ -0,0 +1,231 @@ +--- +title: "SMB / CIFS" +description: "Rclone docs for SMB backend" +--- + +# {{< icon "fa fa-server" >}} SMB + +SMB is [a communication protocol to share files over network](https://en.wikipedia.org/wiki/Server_Message_Block). + +This relies on [go-smb2 library](https://github.com/hirochachacha/go-smb2/) for communication with SMB protocol. + +Paths are specified as `remote:sharename` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`. + +## Notes + +The first path segment must be the name of the share, which you entered when you started to share on Windows. On smbd, it's the section title in `smb.conf` (usually in `/etc/samba/`) file. +You can find shares by quering the root if you're unsure (e.g. `rclone lsd remote:`). + +You can't access to the shared printers from rclone, obviously. + +You can't use Anonymous access for logging in. You have to use the `guest` user with an empty password instead. +The rclone client tries to avoid 8.3 names when uploading files by encoding trailing spaces and periods. +Alternatively, [the local backend](/local/#paths-on-windows) on Windows can access SMB servers using UNC paths, by `\\server\share`. This doesn't apply to non-Windows OSes, such as Linux and macOS. + +## Configuration + +Here is an example of making a SMB configuration. + +First run + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +name> remote +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +XX / SMB / CIFS + \ (smb) +Storage> smb + +Option host. +Samba hostname to connect to. +E.g. "example.com". +Enter a value. +host> localhost + +Option user. +Samba username. +Enter a string value. Press Enter for the default (lesmi). +user> guest + +Option port. +Samba port number. +Enter a signed integer. Press Enter for the default (445). +port> + +Option pass. +Samba password. +Choose an alternative below. Press Enter for the default (n). +y) Yes, type in my own password +g) Generate random password +n) No, leave this optional password blank (default) +y/g/n> g +Password strength in bits. +64 is just about memorable +128 is secure +1024 is the maximum +Bits> 64 +Your password is: XXXX +Use this password? Please note that an obscured version of this +password (and not the password itself) will be stored under your +configuration file, so keep this generated password in a safe place. +y) Yes (default) +n) No +y/n> y + +Option domain. +Domain name for NTLM authentication. +Enter a string value. Press Enter for the default (WORKGROUP). +domain> + +Edit advanced config? +y) Yes +n) No (default) +y/n> n + +Configuration complete. +Options: +- type: samba +- host: localhost +- user: guest +- pass: *** ENCRYPTED *** +Keep this "remote" remote? +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> d +``` + +{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/smb/smb.go then run make backenddocs" >}} +### Standard options + +Here are the Standard options specific to smb (SMB / CIFS). + +#### --smb-host + +SMB server hostname to connect to. + +E.g. "example.com". + +Properties: + +- Config: host +- Env Var: RCLONE_SMB_HOST +- Type: string +- Required: true + +#### --smb-user + +SMB username. + +Properties: + +- Config: user +- Env Var: RCLONE_SMB_USER +- Type: string +- Default: "$USER" + +#### --smb-port + +SMB port number. + +Properties: + +- Config: port +- Env Var: RCLONE_SMB_PORT +- Type: int +- Default: 445 + +#### --smb-pass + +SMB password. + +**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/). + +Properties: + +- Config: pass +- Env Var: RCLONE_SMB_PASS +- Type: string +- Required: false + +#### --smb-domain + +Domain name for NTLM authentication. + +Properties: + +- Config: domain +- Env Var: RCLONE_SMB_DOMAIN +- Type: string +- Default: "WORKGROUP" + +### Advanced options + +Here are the Advanced options specific to smb (SMB / CIFS). + +#### --smb-idle-timeout + +Max time before closing idle connections. + +If no connections have been returned to the connection pool in the time +given, rclone will empty the connection pool. + +Set to 0 to keep connections indefinitely. + + +Properties: + +- Config: idle_timeout +- Env Var: RCLONE_SMB_IDLE_TIMEOUT +- Type: Duration +- Default: 1m0s + +#### --smb-hide-special-share + +Hide special shares (e.g. print$) which users aren't supposed to access. + +Properties: + +- Config: hide_special_share +- Env Var: RCLONE_SMB_HIDE_SPECIAL_SHARE +- Type: bool +- Default: true + +#### --smb-case-insensitive + +Whether the server is configured to be case-insensitive. + +Always true on Windows shares. + +Properties: + +- Config: case_insensitive +- Env Var: RCLONE_SMB_CASE_INSENSITIVE +- Type: bool +- Default: true + +#### --smb-encoding + +The encoding for the backend. + +See the [encoding section in the overview](/overview/#encoding) for more info. + +Properties: + +- Config: encoding +- Env Var: RCLONE_SMB_ENCODING +- Type: MultiEncoder +- Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot + +{{< rem autogenerated options stop >}} diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 591219cf30670..1499511eacac2 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -92,6 +92,7 @@ Seafile SFTP Sia + SMB / CIFS Storj SugarSync Uptobox diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 23cdee6db673c..b946a3c914d20 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -369,6 +369,9 @@ backends: - backend: "sia" remote: "TestSia:" fastlist: false + - backend: "smb" + remote: "TestSMB:rclone" + fastlist: false - backend: "storj" remote: "TestStorj:" fastlist: true diff --git a/fstest/testserver/init.d/TestSMB b/fstest/testserver/init.d/TestSMB new file mode 100755 index 0000000000000..788b5a84a8fcd --- /dev/null +++ b/fstest/testserver/init.d/TestSMB @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e + +NAME=smb +USER=rclone +PASS=GNF3Cqeu +WORKGROUP=thepub + +. $(dirname "$0")/docker.bash + +start() { + docker run --rm -d --name $NAME dperson/samba \ + -p \ + -u "rclone;${PASS}" \ + -w "${WORKGROUP}" \ + -s "public;/share" \ + -s "rclone;/rclone;yes;no;no;rclone" + + echo type=smb + echo host=$(docker_ip) + echo user=$USER + echo pass=$(rclone obscure $PASS) + echo domain=$WORKGROUP + echo _connect=$(docker_ip):139 +} + +. $(dirname "$0")/run.bash diff --git a/go.mod b/go.mod index 6358a144c3df1..f40f1f11e09a1 100644 --- a/go.mod +++ b/go.mod @@ -84,6 +84,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gdamore/encoding v1.0.0 // indirect + github.com/geoffgarside/ber v1.1.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -92,6 +93,7 @@ require ( github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hirochachacha/go-smb2 v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 5445443073bc7..26c74ec7829e7 100644 --- a/go.sum +++ b/go.sum @@ -204,6 +204,8 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.5.2 h1:tKzG29kO9p2V++3oBY2W9zUjYu7IK1MENFeY/BzJSVY= github.com/gdamore/tcell/v2 v2.5.2/go.mod h1:wSkrPaXoiIWZqW/g7Px4xc79di6FTcpB8tvaKJ6uGBo= +github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= +github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -353,6 +355,8 @@ github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/C github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= +github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -676,6 +680,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= From a3c06b9bbe148ac4995c26dac3a5ce1b603a176e Mon Sep 17 00:00:00 2001 From: Marco Molteni Date: Sat, 1 Oct 2022 11:28:56 +0200 Subject: [PATCH 291/560] docs/content: remove duplicate Scaleway C14 Glacier Scaleway S3/C14 is now called S3/Glacier. Since Glacier is already mentioned in the Rclone Scaleway section, let's just remove this entry from here. --- docs/content/_index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/content/_index.md b/docs/content/_index.md index 5d1bf2cda4d25..848e0f17593a0 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -115,7 +115,6 @@ WebDAV or S3, that work out of the box.) {{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}} {{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}} {{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}} -{{< provider name="C14" home="https://www.online.net/en/storage/c14-cold-storage" config="/s3/#scaleway" >}} {{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}} {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}} {{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}} From c5109408c02db3ca14a9a5382373066299c39d25 Mon Sep 17 00:00:00 2001 From: Isaac Aymerich Date: Mon, 3 Oct 2022 12:24:58 +0200 Subject: [PATCH 292/560] rc: handle external unmount when mounting Before this change, if the a mount was created via the rc but unmounted externally with `fusermount -u` say, rclone would still believe the mount was active when it wasn't. --- cmd/mountlib/rc.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/mountlib/rc.go b/cmd/mountlib/rc.go index 7a89787feb339..b675abda99a65 100644 --- a/cmd/mountlib/rc.go +++ b/cmd/mountlib/rc.go @@ -122,7 +122,15 @@ func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) { log.Printf("mount FAILED: %v", err) return nil, err } - + go func() { + if err = mnt.Wait(); err != nil { + log.Printf("unmount FAILED: %v", err) + return + } + mountMu.Lock() + defer mountMu.Unlock() + delete(liveMounts, mountPoint) + }() // Add mount to list if mount point was successfully created liveMounts[mountPoint] = mnt From 6c3b7d5820fa7305d3fe94b252b98dcf45cb9ae5 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Tue, 4 Oct 2022 15:38:58 +0200 Subject: [PATCH 293/560] Create LICENSE --- LICENSE | 339 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000..d159169d10508 --- /dev/null +++ b/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. From 5efb880772e818d24f4c176ea8819765074fbf22 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Tue, 4 Oct 2022 15:40:37 +0200 Subject: [PATCH 294/560] Remove LICENSE --- LICENSE | 339 -------------------------------------------------------- 1 file changed, 339 deletions(-) delete mode 100644 LICENSE diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d159169d10508..0000000000000 --- a/LICENSE +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. From c10965ecfb2390a9511bb855cc30724958c0c848 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 5 Oct 2022 13:33:29 +0100 Subject: [PATCH 295/560] Add Dimitri Papadopoulos Orfanos to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index a4375fc9abf95..ed7e13930cb4b 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -641,3 +641,4 @@ put them back in again.` >}} * Dmitry Deniskin <110819396+ddeniskin@users.noreply.github.com> * Alexander Knorr <106825+opexxx@users.noreply.github.com> * Richard Bateman + * Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> From 910af597a12dca980ce5847eb2f1b392bcccc634 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 5 Oct 2022 13:33:29 +0100 Subject: [PATCH 296/560] Add Lorenzo Milesi to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index ed7e13930cb4b..f7cd0418331c3 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -642,3 +642,4 @@ put them back in again.` >}} * Alexander Knorr <106825+opexxx@users.noreply.github.com> * Richard Bateman * Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> + * Lorenzo Milesi From b437d9461a00d9bb864ef27d6544c72d3d498077 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 5 Oct 2022 13:33:29 +0100 Subject: [PATCH 297/560] Add Isaac Aymerich to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index f7cd0418331c3..ac192c72ff0a9 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -643,3 +643,4 @@ put them back in again.` >}} * Richard Bateman * Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> * Lorenzo Milesi + * Isaac Aymerich From 4e79de106a24d7b7416a0d30a7605137f2a53b27 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 4 Oct 2022 17:10:00 +0100 Subject: [PATCH 298/560] hubic: remove backend as service has now shutdown - fixes #6481 --- README.md | 1 - backend/all/all.go | 1 - backend/hubic/auth.go | 62 --------- backend/hubic/hubic.go | 200 --------------------------- backend/hubic/hubic_test.go | 19 --- backend/swift/swift.go | 2 +- bin/make_manual.py | 1 - cmd/mountlib/help.go | 4 +- cmd/serve/restic/restic-test.sh | 1 - cmd/test/info/all.sh | 2 - docs/content/_index.md | 1 - docs/content/crypt.md | 2 +- docs/content/docs.md | 3 +- docs/content/hubic.md | 238 -------------------------------- docs/content/overview.md | 4 +- docs/layouts/chrome/navbar.html | 1 - fstest/test_all/config.yaml | 3 - 17 files changed, 6 insertions(+), 539 deletions(-) delete mode 100644 backend/hubic/auth.go delete mode 100644 backend/hubic/hubic.go delete mode 100644 backend/hubic/hubic_test.go delete mode 100644 docs/content/hubic.md diff --git a/README.md b/README.md index 44c8f8552cda3..16e0be145144f 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and * HiDrive [:page_facing_up:](https://rclone.org/hidrive/) * HTTP [:page_facing_up:](https://rclone.org/http/) * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs) - * Hubic [:page_facing_up:](https://rclone.org/hubic/) * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/) * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/) * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3) diff --git a/backend/all/all.go b/backend/all/all.go index 19b1e1df518de..f611bfc61a32e 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -24,7 +24,6 @@ import ( _ "github.com/rclone/rclone/backend/hdfs" _ "github.com/rclone/rclone/backend/hidrive" _ "github.com/rclone/rclone/backend/http" - _ "github.com/rclone/rclone/backend/hubic" _ "github.com/rclone/rclone/backend/internetarchive" _ "github.com/rclone/rclone/backend/jottacloud" _ "github.com/rclone/rclone/backend/koofr" diff --git a/backend/hubic/auth.go b/backend/hubic/auth.go deleted file mode 100644 index b1bbac80e79ca..0000000000000 --- a/backend/hubic/auth.go +++ /dev/null @@ -1,62 +0,0 @@ -package hubic - -import ( - "context" - "net/http" - "time" - - "github.com/ncw/swift/v2" - "github.com/rclone/rclone/fs" -) - -// auth is an authenticator for swift -type auth struct { - f *Fs -} - -// newAuth creates a swift authenticator -func newAuth(f *Fs) *auth { - return &auth{ - f: f, - } -} - -// Request constructs an http.Request for authentication -// -// returns nil for not needed -func (a *auth) Request(ctx context.Context, c *swift.Connection) (r *http.Request, err error) { - const retries = 10 - for try := 1; try <= retries; try++ { - err = a.f.getCredentials(context.TODO()) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - fs.Debugf(a.f, "retrying auth request %d/%d: %v", try, retries, err) - } - return nil, err -} - -// Response parses the result of an http request -func (a *auth) Response(ctx context.Context, resp *http.Response) error { - return nil -} - -// The public storage URL - set Internal to true to read -// internal/service net URL -func (a *auth) StorageUrl(Internal bool) string { // nolint - return a.f.credentials.Endpoint -} - -// The access token -func (a *auth) Token() string { - return a.f.credentials.Token -} - -// The CDN url if available -func (a *auth) CdnUrl() string { // nolint - return "" -} - -// Check the interfaces are satisfied -var _ swift.Authenticator = (*auth)(nil) diff --git a/backend/hubic/hubic.go b/backend/hubic/hubic.go deleted file mode 100644 index 5c284194eff0d..0000000000000 --- a/backend/hubic/hubic.go +++ /dev/null @@ -1,200 +0,0 @@ -// Package hubic provides an interface to the Hubic object storage -// system. -package hubic - -// This uses the normal swift mechanism to update the credentials and -// ignores the expires field returned by the Hubic API. This may need -// to be revisited after some actual experience. - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "strings" - "time" - - swiftLib "github.com/ncw/swift/v2" - "github.com/rclone/rclone/backend/swift" - "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/config/configmap" - "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/config/obscure" - "github.com/rclone/rclone/fs/fshttp" - "github.com/rclone/rclone/lib/oauthutil" - "golang.org/x/oauth2" -) - -const ( - rcloneClientID = "api_hubic_svWP970PvSWbw5G3PzrAqZ6X2uHeZBPI" - rcloneEncryptedClientSecret = "leZKCcqy9movLhDWLVXX8cSLp_FzoiAPeEJOIOMRw1A5RuC4iLEPDYPWVF46adC_MVonnLdVEOTHVstfBOZ_lY4WNp8CK_YWlpRZ9diT5YI" -) - -// Globals -var ( - // Description of how to auth for this app - oauthConfig = &oauth2.Config{ - Scopes: []string{ - "credentials.r", // Read OpenStack credentials - }, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://api.hubic.com/oauth/auth/", - TokenURL: "https://api.hubic.com/oauth/token/", - }, - ClientID: rcloneClientID, - ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), - RedirectURL: oauthutil.RedirectLocalhostURL, - } -) - -// Register with Fs -func init() { - fs.Register(&fs.RegInfo{ - Name: "hubic", - Description: "Hubic", - NewFs: NewFs, - Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { - return oauthutil.ConfigOut("", &oauthutil.Options{ - OAuth2Config: oauthConfig, - }) - }, - Options: append(oauthutil.SharedOptions, swift.SharedOptions...), - }) -} - -// credentials is the JSON returned from the Hubic API to read the -// OpenStack credentials -type credentials struct { - Token string `json:"token"` // OpenStack token - Endpoint string `json:"endpoint"` // OpenStack endpoint - Expires string `json:"expires"` // Expires date - e.g. "2015-11-09T14:24:56+01:00" -} - -// Fs represents a remote hubic -type Fs struct { - fs.Fs // wrapped Fs - features *fs.Features // optional features - client *http.Client // client for oauth api - credentials credentials // returned from the Hubic API - expires time.Time // time credentials expire -} - -// Object describes a swift object -type Object struct { - *swift.Object -} - -// Return a string version -func (o *Object) String() string { - if o == nil { - return "" - } - return o.Object.String() -} - -// ------------------------------------------------------------ - -// String converts this Fs to a string -func (f *Fs) String() string { - if f.Fs == nil { - return "Hubic" - } - return fmt.Sprintf("Hubic %s", f.Fs.String()) -} - -// getCredentials reads the OpenStack Credentials using the Hubic API -// -// The credentials are read into the Fs -func (f *Fs) getCredentials(ctx context.Context) (err error) { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.hubic.com/1.0/account/credentials", nil) - if err != nil { - return err - } - resp, err := f.client.Do(req) - if err != nil { - return err - } - defer fs.CheckClose(resp.Body, &err) - if resp.StatusCode < 200 || resp.StatusCode > 299 { - body, _ := ioutil.ReadAll(resp.Body) - bodyStr := strings.TrimSpace(strings.ReplaceAll(string(body), "\n", " ")) - return fmt.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr) - } - decoder := json.NewDecoder(resp.Body) - var result credentials - err = decoder.Decode(&result) - if err != nil { - return err - } - // fs.Debugf(f, "Got credentials %+v", result) - if result.Token == "" || result.Endpoint == "" || result.Expires == "" { - return errors.New("couldn't read token, result and expired from credentials") - } - f.credentials = result - expires, err := time.Parse(time.RFC3339, result.Expires) - if err != nil { - return err - } - f.expires = expires - fs.Debugf(f, "Got swift credentials (expiry %v in %v)", f.expires, time.Until(f.expires)) - return nil -} - -// NewFs constructs an Fs from the path, container:path -func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { - client, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig) - if err != nil { - return nil, fmt.Errorf("failed to configure Hubic: %w", err) - } - - f := &Fs{ - client: client, - } - - // Make the swift Connection - ci := fs.GetConfig(ctx) - c := &swiftLib.Connection{ - Auth: newAuth(f), - ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport - Timeout: 10 * ci.Timeout, // Use the timeouts in the transport - Transport: fshttp.NewTransport(ctx), - } - err = c.Authenticate(ctx) - if err != nil { - return nil, fmt.Errorf("error authenticating swift connection: %w", err) - } - - // Parse config into swift.Options struct - opt := new(swift.Options) - err = configstruct.Set(m, opt) - if err != nil { - return nil, err - } - - // Make inner swift Fs from the connection - swiftFs, err := swift.NewFsWithConnection(ctx, opt, name, root, c, true) - if err != nil && err != fs.ErrorIsFile { - return nil, err - } - f.Fs = swiftFs - f.features = f.Fs.Features().Wrap(f) - return f, err -} - -// Features returns the optional features of this Fs -func (f *Fs) Features() *fs.Features { - return f.features -} - -// UnWrap returns the Fs that this Fs is wrapping -func (f *Fs) UnWrap() fs.Fs { - return f.Fs -} - -// Check the interfaces are satisfied -var ( - _ fs.Fs = (*Fs)(nil) - _ fs.UnWrapper = (*Fs)(nil) -) diff --git a/backend/hubic/hubic_test.go b/backend/hubic/hubic_test.go deleted file mode 100644 index 08d6f122dd382..0000000000000 --- a/backend/hubic/hubic_test.go +++ /dev/null @@ -1,19 +0,0 @@ -// Test Hubic filesystem interface -package hubic_test - -import ( - "testing" - - "github.com/rclone/rclone/backend/hubic" - "github.com/rclone/rclone/fstest/fstests" -) - -// TestIntegration runs integration tests against the remote -func TestIntegration(t *testing.T) { - fstests.Run(t, &fstests.Opt{ - RemoteName: "TestHubic:", - NilObject: (*hubic.Object)(nil), - SkipFsCheckWrap: true, - SkipObjectCheckWrap: true, - }) -} diff --git a/backend/swift/swift.go b/backend/swift/swift.go index bea898bda2d9c..ad4cfb712deed 100644 --- a/backend/swift/swift.go +++ b/backend/swift/swift.go @@ -40,7 +40,7 @@ const ( minSleep = 10 * time.Millisecond // In case of error, start at 10ms sleep. ) -// SharedOptions are shared between swift and hubic +// SharedOptions are shared between swift and backends which depend on swift var SharedOptions = []fs.Option{{ Name: "chunk_size", Help: `Above this size files will be chunked into a _segments container. diff --git a/bin/make_manual.py b/bin/make_manual.py index cc2e81a5eec7f..8ab7f314d5685 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -49,7 +49,6 @@ "hdfs.md", "hidrive.md", "http.md", - "hubic.md", "internetarchive.md", "jottacloud.md", "koofr.md", diff --git a/cmd/mountlib/help.go b/cmd/mountlib/help.go index 7516157e81223..9713448f303a0 100644 --- a/cmd/mountlib/help.go +++ b/cmd/mountlib/help.go @@ -235,8 +235,8 @@ applications won't work with their files on an rclone mount without |--vfs-cache-mode writes| or |--vfs-cache-mode full|. See the [VFS File Caching](#vfs-file-caching) section for more info. -The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2, -Hubic) do not support the concept of empty directories, so empty +The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2) +do not support the concept of empty directories, so empty directories will have a tendency to disappear once they fall out of the directory cache. diff --git a/cmd/serve/restic/restic-test.sh b/cmd/serve/restic/restic-test.sh index 0e40566eb1907..b26160d0b7fb6 100755 --- a/cmd/serve/restic/restic-test.sh +++ b/cmd/serve/restic/restic-test.sh @@ -15,7 +15,6 @@ TestDropbox: TestFichier: TestFTP: TestGoogleCloudStorage: -TestHubic: TestNetStorage: TestOneDrive: TestPcloud: diff --git a/cmd/test/info/all.sh b/cmd/test/info/all.sh index bb8dea10fd2db..fe1a65438779e 100755 --- a/cmd/test/info/all.sh +++ b/cmd/test/info/all.sh @@ -15,5 +15,3 @@ exec rclone --check-normalization=true --check-control=true --check-length=true TestSwift:testInfo \ TestYandex:testInfo \ TestFTP:testInfo - -# TestHubic:testInfo \ diff --git a/docs/content/_index.md b/docs/content/_index.md index 848e0f17593a0..cab7b03c7a50f 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -129,7 +129,6 @@ WebDAV or S3, that work out of the box.) {{< provider name="Hetzner Storage Box" home="https://www.hetzner.com/storage/storage-box" config="/sftp/#hetzner-storage-box" >}} {{< provider name="HiDrive" home="https://www.strato.de/cloud-speicher/" config="/hidrive/" >}} {{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}} -{{< provider name="Hubic" home="https://hubic.com/" config="/hubic/" >}} {{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}} {{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}} {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}} diff --git a/docs/content/crypt.md b/docs/content/crypt.md index b0110f63533a5..c3af9de33f2cb 100644 --- a/docs/content/crypt.md +++ b/docs/content/crypt.md @@ -225,7 +225,7 @@ If you intend to use the wrapped remote both directly for keeping unencrypted content, as well as through a crypt remote for encrypted content, it is recommended to point the crypt remote to a separate directory within the wrapped remote. If you use a bucket-based storage -system (e.g. Swift, S3, Google Compute Storage, B2, Hubic) it is generally +system (e.g. Swift, S3, Google Compute Storage, B2) it is generally advisable to wrap the crypt remote around a specific bucket (`s3:bucket`). If wrapping around the entire root of the storage (`s3:`), and use the optional file name encryption, rclone will encrypt the bucket name. diff --git a/docs/content/docs.md b/docs/content/docs.md index 7f3b20378ea1d..330d255812d98 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -51,7 +51,6 @@ See the following for detailed instructions for * [HDFS](/hdfs/) * [HiDrive](/hidrive/) * [HTTP](/http/) - * [Hubic](/hubic/) * [Internet Archive](/internetarchive/) * [Jottacloud](/jottacloud/) * [Koofr](/koofr/) @@ -1955,7 +1954,7 @@ quickly using the least amount of memory. However, some remotes have a way of listing all files beneath a directory in one (or a small number) of transactions. These tend to -be the bucket-based remotes (e.g. S3, B2, GCS, Swift, Hubic). +be the bucket-based remotes (e.g. S3, B2, GCS, Swift). If you use the `--fast-list` flag then rclone will use this method for listing directories. This will have the following consequences for diff --git a/docs/content/hubic.md b/docs/content/hubic.md deleted file mode 100644 index 3000bc6af4290..0000000000000 --- a/docs/content/hubic.md +++ /dev/null @@ -1,238 +0,0 @@ ---- -title: "Hubic" -description: "Rclone docs for Hubic" ---- - -# {{< icon "fa fa-space-shuttle" >}} Hubic - -Paths are specified as `remote:path` - -Paths are specified as `remote:container` (or `remote:` for the `lsd` -command.) You may put subdirectories in too, e.g. `remote:container/path/to/dir`. - -## Configuration - -The initial setup for Hubic involves getting a token from Hubic which -you need to do in your browser. `rclone config` walks you through it. - -Here is an example of how to make a remote called `remote`. First run: - - rclone config - -This will guide you through an interactive setup process: - -``` -n) New remote -s) Set configuration password -n/s> n -name> remote -Type of storage to configure. -Choose a number from below, or type in your own value -[snip] -XX / Hubic - \ "hubic" -[snip] -Storage> hubic -Hubic Client Id - leave blank normally. -client_id> -Hubic Client Secret - leave blank normally. -client_secret> -Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine -y) Yes -n) No -y/n> y -If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth -Log in and authorize rclone for access -Waiting for code... -Got code --------------------- -[remote] -client_id = -client_secret = -token = {"access_token":"XXXXXX"} --------------------- -y) Yes this is OK -e) Edit this remote -d) Delete this remote -y/e/d> y -``` - -See the [remote setup docs](/remote_setup/) for how to set it up on a -machine with no Internet browser available. - -Note that rclone runs a webserver on your local machine to collect the -token as returned from Hubic. This only runs from the moment it opens -your browser to the moment you get back the verification code. This -is on `http://127.0.0.1:53682/` and this it may require you to unblock -it temporarily if you are running a host firewall. - -Once configured you can then use `rclone` like this, - -List containers in the top level of your Hubic - - rclone lsd remote: - -List all the files in your Hubic - - rclone ls remote: - -To copy a local directory to an Hubic directory called backup - - rclone copy /home/source remote:backup - -If you want the directory to be visible in the official *Hubic -browser*, you need to copy your files to the `default` directory - - rclone copy /home/source remote:default/backup - -### --fast-list ### - -This remote supports `--fast-list` which allows you to use fewer -transactions in exchange for more memory. See the [rclone -docs](/docs/#fast-list) for more details. - -### Modified time ### - -The modified time is stored as metadata on the object as -`X-Object-Meta-Mtime` as floating point since the epoch accurate to 1 -ns. - -This is a de facto standard (used in the official python-swiftclient -amongst others) for storing the modification time for an object. - -Note that Hubic wraps the Swift backend, so most of the properties of -are the same. - -{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/hubic/hubic.go then run make backenddocs" >}} -### Standard options - -Here are the Standard options specific to hubic (Hubic). - -#### --hubic-client-id - -OAuth Client Id. - -Leave blank normally. - -Properties: - -- Config: client_id -- Env Var: RCLONE_HUBIC_CLIENT_ID -- Type: string -- Required: false - -#### --hubic-client-secret - -OAuth Client Secret. - -Leave blank normally. - -Properties: - -- Config: client_secret -- Env Var: RCLONE_HUBIC_CLIENT_SECRET -- Type: string -- Required: false - -### Advanced options - -Here are the Advanced options specific to hubic (Hubic). - -#### --hubic-token - -OAuth Access Token as a JSON blob. - -Properties: - -- Config: token -- Env Var: RCLONE_HUBIC_TOKEN -- Type: string -- Required: false - -#### --hubic-auth-url - -Auth server URL. - -Leave blank to use the provider defaults. - -Properties: - -- Config: auth_url -- Env Var: RCLONE_HUBIC_AUTH_URL -- Type: string -- Required: false - -#### --hubic-token-url - -Token server url. - -Leave blank to use the provider defaults. - -Properties: - -- Config: token_url -- Env Var: RCLONE_HUBIC_TOKEN_URL -- Type: string -- Required: false - -#### --hubic-chunk-size - -Above this size files will be chunked into a _segments container. - -Above this size files will be chunked into a _segments container. The -default for this is 5 GiB which is its maximum value. - -Properties: - -- Config: chunk_size -- Env Var: RCLONE_HUBIC_CHUNK_SIZE -- Type: SizeSuffix -- Default: 5Gi - -#### --hubic-no-chunk - -Don't chunk files during streaming upload. - -When doing streaming uploads (e.g. using rcat or mount) setting this -flag will cause the swift backend to not upload chunked files. - -This will limit the maximum upload size to 5 GiB. However non chunked -files are easier to deal with and have an MD5SUM. - -Rclone will still chunk files bigger than chunk_size when doing normal -copy operations. - -Properties: - -- Config: no_chunk -- Env Var: RCLONE_HUBIC_NO_CHUNK -- Type: bool -- Default: false - -#### --hubic-encoding - -The encoding for the backend. - -See the [encoding section in the overview](/overview/#encoding) for more info. - -Properties: - -- Config: encoding -- Env Var: RCLONE_HUBIC_ENCODING -- Type: MultiEncoder -- Default: Slash,InvalidUtf8 - -{{< rem autogenerated options stop >}} - -## Limitations - -This uses the normal OpenStack Swift mechanism to refresh the Swift -API credentials and ignores the expires field returned by the Hubic -API. - -The Swift API doesn't return a correct MD5SUM for segmented files -(Dynamic or Static Large Objects) so rclone won't check or use the -MD5SUM for these. diff --git a/docs/content/overview.md b/docs/content/overview.md index 34f059eb34ea1..79f2d6786602f 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -32,7 +32,6 @@ Here is an overview of the major features of each cloud storage system. | HDFS | - | R/W | No | No | - | - | | HiDrive | HiDrive ¹² | R/W | No | No | - | - | | HTTP | - | R | No | No | R | - | -| Hubic | MD5 | R/W | No | No | R/W | - | | Internet Archive | MD5, SHA1, CRC32 | R/W ¹¹ | No | No | - | RWU | | Jottacloud | MD5 | R/W | Yes | No | R | - | | Koofr | MD5 | - | Yes | No | - | - | @@ -484,7 +483,6 @@ upon backend-specific capabilities. | HDFS | Yes | No | Yes | Yes | No | No | Yes | No | Yes | Yes | | HiDrive | Yes | Yes | Yes | Yes | No | No | Yes | No | No | Yes | | HTTP | No | No | No | No | No | No | No | No | No | Yes | -| Hubic | Yes † | Yes | No | No | No | Yes | Yes | No | Yes | No | | Internet Archive | No | Yes | No | No | Yes | Yes | No | Yes | Yes | No | | Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | | Koofr | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | Yes | @@ -516,7 +514,7 @@ upon backend-specific capabilities. This deletes a directory quicker than just deleting all the files in the directory. -† Note Swift, Hubic, and Storj implement this in order to delete +† Note Swift and Storj implement this in order to delete directory markers but they don't actually have a quicker way of deleting files other than deleting them individually. diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 1499511eacac2..e32550e7f8c50 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -74,7 +74,6 @@ HDFS (Hadoop Distributed Filesystem) HiDrive HTTP - Hubic Internet Archive Jottacloud Koofr diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index b946a3c914d20..3fea2f48f9f65 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -136,9 +136,6 @@ backends: - backend: "hidrive" remote: "TestHiDrive:" fastlist: false - - backend: "hubic" - remote: "TestHubic:" - fastlist: false - backend: "internetarchive" remote: "TestIA:rclone-integration-test" fastlist: true From 09b6d939f5ea1b15823c5c8d9157037944770921 Mon Sep 17 00:00:00 2001 From: YanceyChiew <35898533+YanceyChiew@users.noreply.github.com> Date: Sat, 1 Oct 2022 14:28:50 +0800 Subject: [PATCH 299/560] dlna: add support for more external subtitle --- cmd/serve/dlna/cds.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/serve/dlna/cds.go b/cmd/serve/dlna/cds.go index f3daf0cd8cb34..f7cd77d1ff683 100644 --- a/cmd/serve/dlna/cds.go +++ b/cmd/serve/dlna/cds.go @@ -158,7 +158,9 @@ func mediaWithResources(nodes vfs.Nodes) (vfs.Nodes, map[vfs.Node]vfs.Nodes) { for _, node := range nodes { baseName, ext := splitExt(strings.ToLower(node.Name())) switch ext { - case ".srt": + case ".srt", ".ass", ".ssa", ".sub", ".idx", ".sup", ".jss", ".txt", ".usf", ".cue", ".vtt", ".css": + // .idx should be with .sub, .css should be with vtt otherwise they should be culled, + // and their mimeTypes are not consistent, but anyway these negatives don't throw errors. subtitlesByName[baseName] = node default: mediaByName[baseName] = append(mediaByName[baseName], node) From 4a35aff33c92dab5c3d35a663be64e6f92d51974 Mon Sep 17 00:00:00 2001 From: YanceyChiew <35898533+YanceyChiew@users.noreply.github.com> Date: Sat, 1 Oct 2022 15:26:02 +0800 Subject: [PATCH 300/560] dlna: add verification of addresses Verify the http service listening address and the SSDP server announcement address to prevent accidental listening of IPv6 addresses that do not support dlna yet and may be globally accessible. Unlistened addresses on the interface will also be filtered out of the SSDP announcement to avoid misleading other services in the multicast domain. --- cmd/serve/dlna/dlna.go | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go index c36e492d7fa09..4c2837217da77 100644 --- a/cmd/serve/dlna/dlna.go +++ b/cmd/serve/dlna/dlna.go @@ -279,7 +279,14 @@ func (s *server) resourceHandler(w http.ResponseWriter, r *http.Request) { // use s.Wait() to block on the listener indefinitely. func (s *server) Serve() (err error) { if s.HTTPConn == nil { - s.HTTPConn, err = net.Listen("tcp", s.httpListenAddr) + // Currently, the SSDP server only listens on an IPv4 multicast address. + // Differentiate between two INADDR_ANY addresses, + // so that 0.0.0.0 can only listen on IPv4 addresses. + network := "tcp4" + if strings.Count(s.httpListenAddr, ":") > 1 { + network = "tcp" + } + s.HTTPConn, err = net.Listen(network, s.httpListenAddr) if err != nil { return } @@ -336,6 +343,30 @@ func (s *server) startSSDP() { // Run SSDP server on an interface. func (s *server) ssdpInterface(intf net.Interface) { + // Figure out whether should an ip be announced + ipfilterFn := func(ip net.IP) bool { + listenaddr := s.HTTPConn.Addr().String() + listenip := listenaddr[:strings.LastIndex(listenaddr, ":")] + switch listenip { + case "0.0.0.0": + if strings.Contains(ip.String(), ":") { + // Any IPv6 address should not be announced + // because SSDP only listen on IPv4 multicast address + return false + } + return true + case "[::]": + // In the @Serve() section, the default settings have been made to not listen on IPv6 addresses. + // If actually still listening on [::], then allow to announce any address. + return true + default: + if listenip == ip.String() { + return true + } + return false + } + } + // Figure out which HTTP location to advertise based on the interface IP. advertiseLocationFn := func(ip net.IP) string { url := url.URL{ @@ -349,6 +380,12 @@ func (s *server) ssdpInterface(intf net.Interface) { return url.String() } + _, err := intf.Addrs() + if err != nil { + panic(err) + } + fs.Logf(s, "Started SSDP on %v", intf.Name) + // Note that the devices and services advertised here via SSDP should be // in agreement with the rootDesc XML descriptor that is defined above. ssdpServer := ssdp.Server{ @@ -359,6 +396,7 @@ func (s *server) ssdpInterface(intf net.Interface) { "urn:schemas-upnp-org:service:ContentDirectory:1", "urn:schemas-upnp-org:service:ConnectionManager:1", "urn:microsoft.com:service:X_MS_MediaReceiverRegistrar:1"}, + IPFilter: ipfilterFn, Location: advertiseLocationFn, Server: serverField, UUID: s.RootDeviceUUID, From 13b65104eb56756d728ce57d0b361da87eb89947 Mon Sep 17 00:00:00 2001 From: YanceyChiew <35898533+YanceyChiew@users.noreply.github.com> Date: Sat, 1 Oct 2022 17:18:50 +0800 Subject: [PATCH 301/560] dlna: add SSDP AnnounceInterval flag option The current default AnnounceInterval is too short, causing the multicast domain to be flooded with NOTIFY announcements, which may prevent other dlna devices from sleeping. This change allows users to set the announcement interval, and it's default value also increased to 12 minutes. Even within the interval, rclone can still passively respond to M-SEARCH requests from other devices. --- cmd/serve/dlna/dlna.go | 2 +- cmd/serve/dlna/dlnaflags/dlnaflags.go | 21 +++++++++++++-------- docs/content/commands/rclone_serve_dlna.md | 5 +++++ 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go index 4c2837217da77..579d695d8f5a4 100644 --- a/cmd/serve/dlna/dlna.go +++ b/cmd/serve/dlna/dlna.go @@ -118,7 +118,7 @@ func newServer(f fs.Fs, opt *dlnaflags.Options) (*server, error) { } s := &server{ - AnnounceInterval: 10 * time.Second, + AnnounceInterval: opt.AnnounceInterval, FriendlyName: friendlyName, RootDeviceUUID: makeDeviceUUID(friendlyName), Interfaces: interfaces, diff --git a/cmd/serve/dlna/dlnaflags/dlnaflags.go b/cmd/serve/dlna/dlnaflags/dlnaflags.go index 85a1347dcf3d3..10745474c4d8c 100644 --- a/cmd/serve/dlna/dlnaflags/dlnaflags.go +++ b/cmd/serve/dlna/dlnaflags/dlnaflags.go @@ -2,6 +2,8 @@ package dlnaflags import ( + "time" + "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/rc" "github.com/spf13/pflag" @@ -24,18 +26,20 @@ logging of all UPNP traffic. // Options is the type for DLNA serving options. type Options struct { - ListenAddr string - FriendlyName string - LogTrace bool - InterfaceNames []string + ListenAddr string + FriendlyName string + LogTrace bool + InterfaceNames []string + AnnounceInterval time.Duration } // DefaultOpt contains the defaults options for DLNA serving. var DefaultOpt = Options{ - ListenAddr: ":7879", - FriendlyName: "", - LogTrace: false, - InterfaceNames: []string{}, + ListenAddr: ":7879", + FriendlyName: "", + LogTrace: false, + InterfaceNames: []string{}, + AnnounceInterval: 12 * time.Minute, } // Opt contains the options for DLNA serving. @@ -49,6 +53,7 @@ func addFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *Options) { flags.StringVarP(flagSet, &Opt.FriendlyName, prefix+"name", "", Opt.FriendlyName, "Name of DLNA server") flags.BoolVarP(flagSet, &Opt.LogTrace, prefix+"log-trace", "", Opt.LogTrace, "Enable trace logging of SOAP traffic") flags.StringArrayVarP(flagSet, &Opt.InterfaceNames, prefix+"interface", "", Opt.InterfaceNames, "The interface to use for SSDP (repeat as necessary)") + flags.DurationVarP(flagSet, &Opt.AnnounceInterval, prefix+"announce-interval", "", Opt.AnnounceInterval, "The interval between SSDP announcements") } // AddFlags add the command line flags for DLNA serving. diff --git a/docs/content/commands/rclone_serve_dlna.md b/docs/content/commands/rclone_serve_dlna.md index 0343debb40b14..9ab1e83b942bd 100644 --- a/docs/content/commands/rclone_serve_dlna.md +++ b/docs/content/commands/rclone_serve_dlna.md @@ -32,6 +32,11 @@ IPs. Use `--name` to choose the friendly server name, which is by default "rclone (hostname)". +Use `--announce-interval` to specify the interval at which SSDP server +announce devices and services. Larger active announcement intervals help +keep the multicast domain clean, this value does not affect unicast +responses to `M-SEARCH` requests from other devices. + Use `--log-trace` in conjunction with `-vv` to enable additional debug logging of all UPNP traffic. From 966654e23a2ddd5e46fa3aaf2a031dd66fc4e0d8 Mon Sep 17 00:00:00 2001 From: YanceyChiew <35898533+YanceyChiew@users.noreply.github.com> Date: Sat, 1 Oct 2022 14:36:26 +0800 Subject: [PATCH 302/560] dlna: run assets_generate to make new icons --- cmd/serve/dlna/data/assets_vfsdata.go | 35 +++++++++++++-------------- go.mod | 2 ++ go.sum | 4 +++ 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/cmd/serve/dlna/data/assets_vfsdata.go b/cmd/serve/dlna/data/assets_vfsdata.go index a33659f0a53c9..642be7b01fa70 100644 --- a/cmd/serve/dlna/data/assets_vfsdata.go +++ b/cmd/serve/dlna/data/assets_vfsdata.go @@ -1,6 +1,5 @@ // Code generated by vfsgen; DO NOT EDIT. -//go:build !dev // +build !dev package data @@ -22,45 +21,45 @@ var Assets = func() http.FileSystem { fs := vfsgen۰FS{ "/": &vfsgen۰DirInfo{ name: "/", - modTime: time.Date(2019, 9, 15, 16, 40, 10, 576397038, time.UTC), + modTime: time.Date(2022, 10, 1, 4, 23, 36, 728914000, time.UTC), }, "/ConnectionManager.xml": &vfsgen۰CompressedFileInfo{ name: "ConnectionManager.xml", - modTime: time.Date(2019, 5, 27, 15, 4, 10, 117829131, time.UTC), - uncompressedSize: 5505, + modTime: time.Date(2022, 10, 1, 4, 23, 36, 726914600, time.UTC), + uncompressedSize: 5686, - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x57\x4d\x6f\xdb\x3c\x0c\x3e\x3b\xbf\x22\xf0\x3d\xaf\x5b\xe0\x3d\x0c\x85\xe2\xa2\x4b\x3f\x10\x6c\x45\x83\x36\x0d\xb0\x53\xa1\xc9\x6c\xaa\xd5\xa6\x0c\x89\xee\xc7\xbf\x1f\x62\xc7\xb5\xdd\x38\x8b\xe3\xc8\x59\x76\x93\x68\x91\xcf\x63\x92\x22\x29\x76\xfa\x16\x85\xfd\x17\xd0\x46\x2a\x1c\xba\xc7\xff\x1d\xb9\x7d\x40\xa1\x02\x89\xf3\xa1\x7b\x3f\xbd\x1c\x7c\x71\x4f\xfd\x1e\x33\x22\x0e\xfa\x6f\x51\x88\x66\xe8\x26\x1a\x4f\x8c\x78\x82\x88\x9b\x41\x12\x63\x3c\x50\x7a\x7e\x62\x40\xbf\x48\x01\x83\xe3\xc1\x91\xeb\xf7\x1c\x66\x62\x10\xb3\xcc\xac\xdf\x73\x1c\x16\xf1\x5f\x4a\xfb\xc7\xcc\xcb\x16\xa9\x48\xa2\xd2\xfe\x11\xf3\xb2\x45\xcf\x61\x5e\x55\x8b\x71\x41\x52\xe1\x77\x69\x28\x55\xc8\xb6\x8b\xa5\xc3\x90\x47\xe0\x5f\x01\x4d\xb4\x22\x25\x54\x38\xc6\x47\xc5\xbc\x54\x9a\x7e\xe7\x7a\x9e\x44\x80\x94\x2b\x97\x44\xd9\x76\x69\xe2\x4e\x25\x5a\x40\x49\xd3\x71\x58\x20\x35\x64\x50\x2a\x21\xe6\x15\xdb\xe5\x77\x0d\x21\x27\x08\xee\x88\x13\xcc\xb8\x96\xfc\x67\x98\x1b\xaa\xd2\xa9\x3d\x98\x91\xf1\xaa\x6c\xd6\x90\x93\xf8\x6c\x83\x9a\xc4\xe7\x96\xc4\x8a\xed\x47\x14\xbc\x22\x0c\xab\x11\x99\x68\x88\xb9\x86\x4b\xa5\x47\x0a\x31\xe3\xd6\x26\x2c\xb7\x10\x29\x82\x35\xc1\xad\xf8\x41\x62\x53\x37\x9c\x3d\x9c\xdd\x5e\x3d\x4c\x7f\x4c\x2e\x1e\xec\x86\x69\x02\x50\xfa\xdd\x6b\x8e\x7c\x0e\xda\x2a\xdf\x1a\xeb\x76\x49\x8f\xcf\x3b\xe2\xbb\x30\xbc\x2b\xd5\xf3\x1c\xde\x2a\xc7\x92\xd5\x5d\x09\x36\xf1\xe3\x16\xf7\xb5\x33\x47\x9e\xcd\xa6\x9a\xa3\x89\x95\x26\xdb\x44\x3f\x99\xde\x95\xe9\xad\x30\xb6\x19\x2e\x4d\x76\x56\xfa\x8a\x50\x8d\x54\x14\x87\x40\xd0\xa6\xf0\x1d\xda\x95\xdc\xd6\x0b\x57\x40\xa3\x44\x6b\x40\x2a\x03\x9a\x5d\x5d\x61\x2c\xe4\x42\x3d\xaf\xfd\x7a\xa2\xe5\x94\x72\x68\x59\x71\xb8\xb7\xf6\x9f\xaf\x7c\x4d\x66\x9e\x76\x44\xff\xe2\xd0\xb3\x6b\xf3\xdb\xfb\xd4\x73\x08\xdd\x7a\xe3\xd8\xd3\x8e\xa4\xc5\xb9\x67\xa1\x98\xd8\x28\xcd\xb5\x3e\xcc\xad\xdb\xa9\xd0\xf9\x72\xf9\x89\x2d\x1f\xac\xa9\xd5\x69\x6e\x92\x99\x32\x48\xdf\x00\x06\x17\x2f\x80\x64\x86\xee\x3b\x18\xb7\x54\xdd\xeb\x9e\x7b\x45\x5d\x0f\x38\xf1\xe9\x7b\x0c\xbe\x21\x2d\x71\xce\xbc\x0f\x41\x4a\xca\x7c\xfe\x95\x2d\x70\x57\xde\x72\xfb\x40\xdd\xd4\xd2\x2d\x22\xa3\x2a\x03\xff\x31\x31\x1a\xe2\x3b\x8c\x87\xa1\x7a\x85\x60\xc6\xc3\x04\xca\xad\xb6\x24\xf6\x6f\xbe\x31\xaf\x22\xa8\x39\x33\x52\x48\x80\x74\xa9\x74\xc4\xe9\x5a\x9a\x88\x93\x78\xda\xac\x36\x46\x93\x3c\x3e\x4a\x21\x01\xe9\x2b\xc7\xe0\x55\x06\xd4\x40\xed\x1e\x35\x84\xa9\x83\x46\x4f\x1c\x11\xc2\x26\x2a\xcf\xa8\x5e\xb1\xe6\x60\x55\x54\xdc\x0f\xcb\xa1\x59\xed\x03\x7b\xc9\x8d\xba\x52\x69\x23\x29\xc6\x18\x2f\x8a\xd8\x26\xb7\xdf\x24\x54\x7f\xae\x53\xaf\xef\xa1\x0c\x34\x88\x78\xa5\x87\x16\xd8\xf2\xff\x4e\x70\xd7\xcd\x71\x9d\x03\x7f\x1e\x6d\xb7\x01\x64\x5e\x4d\xb3\x61\x9e\x11\x71\xe0\xff\x0e\x00\x00\xff\xff\x2a\x62\x9d\xe1\x81\x15\x00\x00"), + compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xdc\x57\x5f\x6f\xda\x30\x10\x7f\x4e\xa5\x7e\x07\x94\x77\x96\x56\xda\xc3\x54\x99\x54\x1d\xfd\x23\xb4\x55\x45\x2d\x45\xda\x53\xe5\x39\x57\xea\x35\x39\x47\xf6\xa5\xb4\xdf\x7e\x82\x10\x48\x20\x8c\x90\x39\xc0\xf6\x66\x5f\xee\xee\xf7\xcb\x9d\x7d\x77\x66\xe7\xef\x51\xd8\x7a\x03\x6d\xa4\xc2\x8e\x7b\xfa\xe9\xc4\x6d\x01\x0a\x15\x48\x1c\x75\xdc\xc7\xc1\x75\xfb\x8b\x7b\xee\x1f\x1f\x31\x23\xe2\xa0\xf5\x1e\x85\x68\x3a\x6e\xa2\xf1\xcc\x88\x17\x88\xb8\x69\x27\x31\xc6\x6d\xa5\x47\x67\x06\xf4\x9b\x14\xd0\x3e\x6d\x9f\xb8\xfe\xf1\x91\xc3\x4c\x0c\x62\x98\x3a\x9e\xec\x1d\x16\xf1\x5f\x4a\xfb\xa7\xcc\x4b\x17\xa9\x4c\xa2\xd2\xfe\x09\xf3\xd2\xc5\xc4\xce\x5b\x32\x64\x5c\x90\x54\xf8\x5d\x1a\x4a\x6d\xd2\xfd\x74\xed\x30\xe4\x11\xf8\x37\x40\x7d\xad\x48\x09\x15\xf6\xf0\x59\x31\x6f\x2a\x4d\x15\xb8\x1e\x25\x11\x20\xcd\xed\x73\xb2\xd9\x7e\xe6\xe5\x41\x25\x5a\x40\xde\xd8\x71\x58\x20\x35\xa4\x78\x2a\x21\xe6\x2d\xb6\x99\x82\x86\x90\x13\x04\x0f\xc4\x09\x86\x5c\x4b\xfe\x33\xcc\x5c\x15\x39\x95\x2a\xce\x08\x79\x4b\x8c\xd6\x31\x94\xf8\x6a\x87\x9f\xc4\xd7\xda\xec\x16\xfb\x45\x4e\xbc\x5c\x52\x4a\x12\xd4\xd7\x10\x73\x0d\xd7\x4a\x77\x15\x62\xca\xb0\x66\x96\xee\x21\x52\x04\xeb\xd2\x5d\x88\x88\xc4\xca\x01\xb9\x78\xba\xb8\xbf\x79\x1a\xfc\xe8\x5f\x3d\x59\xcf\x5a\x1f\x20\xf7\xdb\xb7\x1c\xf9\x08\xb4\x65\xd2\x25\xfe\xad\x33\xef\x5d\x36\x46\x7a\xe2\xda\x02\xdf\xcb\x8c\x82\x65\xa2\x39\xbf\x16\x58\x56\x8a\xe8\x36\xb7\xb9\xc9\x90\x5e\x0c\x07\x9a\xa3\x89\x95\x26\xfb\x6c\x97\x9c\x5b\xa0\x7b\x2f\x8c\x7d\x9a\x33\xa7\xcd\x96\xc8\x45\xe6\xba\x2a\x8a\x43\x20\xa8\x59\x20\x0f\xf1\xc2\xd6\x88\xc7\x0d\x50\x37\xd1\x1a\x90\xf2\xa8\xc6\x42\x50\x8c\x95\xf3\x51\x4e\x6e\x0f\x31\xa9\x3f\xf0\xfc\x4b\xa5\xfd\x40\x2e\xf6\x7f\x52\x26\x2b\x8d\x4f\x35\xd9\xee\x79\x7e\xfa\xeb\xc6\xb9\x97\x01\xea\x50\xda\xfd\xe6\x09\xaa\x26\x53\xbb\x23\xd4\xc4\x36\x31\xcd\x45\x33\xf3\x6f\xaf\x9c\x67\xeb\xec\x23\x9b\xbd\x97\xa7\xae\x07\x73\xbf\xcc\xe4\xa1\x5a\x06\x30\xb8\x7a\x03\x24\xd3\x71\x3f\xc0\xb8\xf9\x66\x50\xf6\xc6\xcc\xb5\x81\x80\x13\x1f\x7c\xc4\xe0\x1b\xd2\x12\x47\xcc\x9b\x0b\x52\x6a\x66\xe5\x9f\xb6\xc1\x5e\x79\x3f\xee\x0a\x79\xe3\x4c\x60\x15\x1d\x55\x01\xfc\x8f\x67\xa5\x32\x07\x87\xf1\x30\x54\x63\x08\x86\x3c\x4c\xa0\xd0\xa6\x73\x72\xff\xee\x1b\xf3\x0a\x82\x32\xa5\xae\x42\x02\xa4\x6b\xa5\x23\x4e\xb7\xd2\x44\x9c\xc4\x4b\x05\xbb\x1e\x9a\xe4\xf9\x59\x0a\x09\x48\x5f\x39\x06\x63\x19\x50\x15\xbb\x47\xd4\x10\x4e\xe3\xd4\x7d\xe1\x88\x10\x56\xb2\x79\x45\x35\xc6\x32\xcd\xa2\x2c\x77\x71\xac\x67\xa9\xa4\x6d\xec\xea\xa8\x94\xd6\x54\x3b\x67\xa4\x87\xf1\xa4\xd4\x6d\x4c\xc1\x5d\x42\x6b\x14\x9b\xce\xc0\x6e\xaa\x44\x95\x13\x50\xec\xbe\x0b\x7c\xf9\xb9\x29\xec\xb5\x43\xe1\x2e\xc0\x57\x06\xe6\xed\x40\x99\x57\xd6\xa0\x98\x67\x44\x1c\xf8\xbf\x03\x00\x00\xff\xff\xb5\x30\x72\xd3\x36\x16\x00\x00"), }, "/ContentDirectory.xml": &vfsgen۰CompressedFileInfo{ name: "ContentDirectory.xml", - modTime: time.Date(2019, 5, 27, 15, 4, 10, 118094855, time.UTC), - uncompressedSize: 14527, + modTime: time.Date(2022, 10, 1, 4, 23, 36, 726914600, time.UTC), + uncompressedSize: 15030, - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5a\x5f\x6f\xa3\x38\x10\x7f\x4e\x3f\x45\x95\xf7\x1e\xad\x74\x4f\x2b\x37\xab\xdd\x24\xad\x22\xb5\x4d\x04\xd9\x6a\xef\x29\x72\x61\x9a\x78\x17\x6c\xce\x1e\xda\xe4\xdb\x9f\x80\x90\x40\x0a\xf9\x03\x86\x65\xf7\xf2\xc6\x3f\xff\xe6\x37\xc3\x78\xc6\x1e\x0f\xf9\xbc\xf4\xdc\xcb\x37\x90\x8a\x09\x7e\xdb\xbd\xf9\xeb\xba\xfb\xb9\x77\x41\x94\xed\x3b\x97\x4b\xcf\xe5\xea\xb6\x1b\x48\xfe\x49\xd9\x0b\xf0\xa8\xba\x0a\x7c\xee\x5f\x09\x39\xff\xa4\x40\xbe\x31\x1b\xae\x6e\xae\xae\xbb\xbd\x8b\x0e\x51\x3e\xd8\xcf\x31\x4a\xef\xa2\xd3\x21\x1e\xfd\x21\x64\xef\x86\x18\xf1\x45\xf4\x88\x71\x21\x7b\xd7\xc4\x88\x2f\x2e\x3a\xc4\xc8\x8e\x22\xd4\x46\x26\xf8\x03\x53\x18\x0d\x88\x6f\xc3\xcb\x0e\xe1\xd4\x83\xde\x3d\xa0\x05\x54\xda\x8b\x3e\xf5\xe9\x0b\x73\x19\x32\x50\xc4\x88\xde\x45\x5f\x51\x39\x0f\x3c\xe0\x98\x40\xa4\x1e\xc5\xb7\x6b\xa0\x0d\x4a\x7a\x74\xa7\x43\x1c\x26\x21\x16\x2a\x02\x24\xc6\xf6\x76\xfd\x5e\x82\x4b\x11\x1c\x0b\x29\xc2\x33\x95\x8c\xbe\xb8\x29\xb0\x14\xa5\xdc\x0f\x63\x42\x46\x86\xd1\xf6\x76\xa3\xb6\xb1\xd5\x3b\xdf\x04\x42\x62\x65\x03\xc4\x18\x5a\xd4\xff\x40\xa7\x6e\xe5\x87\x4b\x04\x1e\xfa\x8c\x0e\x2b\xa4\xc1\x74\x99\xa3\x80\x60\x9d\x76\xb9\x03\x8a\x81\x84\xf0\xfb\x32\x96\xc8\x1f\x5e\xd6\x06\x19\xb4\x5a\xbd\x61\xa5\x10\xbc\x6f\xbe\x43\x11\x46\x83\x32\x8a\x8f\x1c\x1d\xff\x7c\x87\x46\x6d\x2a\x7f\x95\xe2\x5d\x41\x19\x3d\xc7\x2f\x3f\xc0\xc6\x8c\x8d\x32\xda\x32\x7e\xac\xb2\x5f\x66\x5f\xcc\xfb\xd9\xf4\x9f\xc9\x70\xb6\x05\x3d\x5a\xe3\x02\x7a\xb1\x62\x77\x2e\x9d\x6b\x25\x98\x86\xad\x4a\xf1\x8e\xb9\x08\x52\x2b\xbd\x04\xb2\x2a\x35\x0b\xa9\x44\xc6\xe7\x23\xee\xc0\x52\x2b\xc3\x35\x62\x55\x82\x26\xfc\x1b\x80\x42\x70\xfa\x22\xe0\x85\x11\xa6\x14\xc3\x35\x62\x65\x13\x86\x59\x4c\x32\x04\xc9\xa8\x56\x7e\x59\xe0\xea\x86\x54\x81\xab\x23\x44\xa7\x18\x26\x98\x55\xb9\x3d\x05\xde\x0b\x48\x13\x30\x90\x1c\x74\x84\x55\xfd\x7f\x79\x2a\x90\xba\x8f\x14\xed\x05\xe8\xc8\xf5\xfa\x09\xe6\xa4\x32\x0d\xe4\x1a\xc8\x4c\xf1\x22\xb8\x4c\x66\xea\x0b\x8e\x94\x71\x90\xad\x4d\x4e\xeb\x05\x7e\x2d\xd1\x61\x07\xfa\x9c\xa4\xce\x49\xea\x9c\xa4\xce\x49\xea\x9c\xa4\xea\x48\x52\x7d\x09\x14\x21\x4e\x0c\x7f\x66\xaa\x1a\xba\x10\x3e\x2b\xf4\x9b\x52\xf4\x74\xcd\xbd\x43\x7b\xd0\x72\x7e\xa3\xcf\x78\xad\x88\x5b\xa7\xfa\xf4\x00\x14\x4a\xb1\x2a\xef\xd4\x6d\xa9\x0c\x9c\xaa\x78\x1c\x2f\x7e\x7f\xbd\x8b\x62\x4d\x20\x25\x70\x9c\xd2\xf9\x33\x75\x03\xd0\xca\x32\x01\x3d\xb1\x40\x57\x94\x52\xe1\xbd\x4d\x2c\x4f\xf5\xa3\x47\xf1\xf6\xe7\x7a\xd1\x13\xbc\x4f\x68\xe8\x47\x6d\x66\xd8\x9a\xbc\x70\xaa\xeb\x8c\x3c\x5f\x48\x34\x41\x89\x40\xda\xa5\xca\xb2\x56\x34\xf2\x9b\x39\xd2\xfa\x77\x22\xbc\xaa\x3f\x26\x4c\x2c\x8c\xd3\x50\x6e\x2b\xf9\x4d\x25\xe5\xea\x75\xdf\x5a\xac\x9c\xdf\xa4\x71\x6b\xf3\x9c\xe1\xf2\xec\x39\x67\xcf\x29\xe3\x39\x16\x0a\x3f\x11\x54\xc5\x7f\x0e\x1b\xa1\x5c\xd2\x6e\xc2\x06\x03\x70\x01\xa1\x8a\xf6\xc9\xd8\x5f\xeb\x9f\x25\x4e\x3c\x13\xfb\x4e\xa4\x98\x4b\x50\xa5\x8e\xbd\xdb\xf4\xeb\x0f\x50\x0c\x01\x02\xcd\x55\x90\x5d\x6c\x5d\x5c\x1f\x80\xcf\x71\x51\x0f\xd7\x04\x5b\x17\xd7\xa8\xc6\x54\x0f\xd5\x35\x74\xcd\x45\x1c\x13\x5e\x41\x02\x2f\x37\xfb\xdb\x5f\xc7\x69\xff\xb6\xe2\x37\x5c\xae\x7f\x9f\xb5\xad\x51\x26\x7d\xcc\x12\x03\xbb\xf5\x6e\x76\xbf\xcf\x2c\xc0\xaf\x42\xfc\xf4\xa8\xfc\x59\x6a\xea\x50\x84\xb9\x90\xab\xe9\xca\xd7\xbb\xdb\xcf\x02\x57\x2e\xe5\x69\x9e\x3a\xe6\xff\x60\x52\x4f\x84\xb2\xc0\x16\xbc\xf0\x54\xa4\x14\xbf\x18\x55\x97\x4b\x27\x97\xeb\x57\x64\xdd\x86\x1a\xa1\x4e\x13\x48\xa2\xd2\x42\x2e\x15\x70\x67\xf8\x06\x1c\xd5\x6d\x97\x8b\x6e\x7a\x2d\xbd\xb7\x99\xd4\xa1\x48\x43\x6f\xec\x29\x94\x8c\xcf\x89\xb1\x79\x10\x71\x52\xbb\x9a\x1c\x2f\x76\x4f\x03\x67\xad\x42\x0f\x36\x4e\x6a\x94\xbe\x02\x95\x11\x5f\xd8\xa9\xb7\x11\x11\xb0\xbf\x35\x0a\xdc\xe4\xf8\x44\x66\x43\x8a\x6e\x57\xc1\xcd\xfc\xd7\x82\x7c\x56\x9b\xbc\xdc\xd0\xd3\xac\xdc\x0f\xe7\x34\x8d\x48\x2d\xec\x02\x69\x44\x7a\x6e\x83\xe4\x7e\xc9\x1d\x42\x5d\x57\xbc\x83\xb3\xa9\xa2\x27\xd1\x3f\xf5\x78\xdd\x79\xf9\x08\x48\xc3\xb1\xc4\xc8\xbc\x2c\xfc\x7e\x10\xa5\x81\xfe\x82\xb9\x8e\x04\x9e\x33\x2a\xfb\x68\x1b\xc9\xb5\x18\xe3\x43\x97\x4b\x33\x0e\x90\xdf\x7d\xd1\x88\xec\xdd\xa6\x19\x6d\x51\xb3\x50\xe2\x6e\x13\x4c\xfd\x12\xeb\x4c\x0e\x85\x42\x73\x6b\x21\xcd\x89\xfd\x50\xdf\xd0\x31\xa5\xfb\xe3\xc7\xc9\xc3\x70\x3a\x1c\x1c\x9e\xcd\x43\xd3\x1c\x9b\x87\x3f\x1b\x3d\xcd\x26\xe6\xf8\xde\x1c\x5a\xd6\xe1\x8f\xad\xe9\x78\x32\xc9\x15\x5e\x6b\x50\x28\x2c\xc3\x34\x32\x41\x8b\x0a\x2b\xcd\x08\xcf\x9c\x94\x36\x2b\x3b\x5b\x41\x4d\xcd\x1c\xc9\xea\x89\x4b\xf9\x1b\xd0\x3d\x53\xb6\x43\x1c\x78\xa5\x81\x8b\x91\x8d\x2e\x0d\x9d\x2b\x91\x63\xc3\x46\x8d\x1c\x92\x0d\xd6\x2f\xa6\x91\xa9\x5f\x1c\x1d\xce\x8e\xa1\x43\x8c\x9c\x5d\x1e\x31\x94\xed\x3b\xbd\xff\x02\x00\x00\xff\xff\x1a\x57\x6a\x92\xbf\x38\x00\x00"), + compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xec\x5a\x51\x6f\xa3\x38\x10\x7e\xce\x4a\xfb\x1f\xaa\xbc\xf7\x68\xa5\x7b\x5a\xb9\x59\xed\xa6\x69\x15\xa9\x6d\x22\x92\xad\xf6\x9e\x22\x17\xa6\xc4\xbb\x60\x73\xf6\xd0\x26\xff\xfe\x44\x80\x04\x9a\xd0\xa6\x74\x60\x69\x2f\x6f\x81\xe0\x6f\x3e\x7f\x8c\x67\xf0\x78\xd8\xd7\x45\xe0\x1f\x3d\x80\x36\x42\xc9\xb3\xee\xe9\x5f\x27\xdd\xaf\xbd\xcf\x9f\x98\x71\x42\xf7\x68\x11\xf8\xd2\x9c\x75\x23\x2d\xbf\x18\x67\x0e\x01\x37\xc7\x51\x28\xc3\x63\xa5\xbd\x2f\x06\xf4\x83\x70\xe0\xf8\xf4\xf8\xa4\xdb\xfb\xfc\xa9\xc3\x4c\x08\xce\x6d\x82\x13\x5f\x77\x58\xc0\x7f\x29\xdd\x3b\x65\x56\xf2\x23\xb9\x27\xa4\xd2\xbd\x13\x66\x25\x3f\xe2\x71\xd6\x93\x81\x8c\x3b\x28\x94\xbc\x12\x06\x93\x31\xc9\xf5\xea\x77\x87\x49\x1e\x40\xef\x12\x70\x02\x5c\x3b\xf3\x3e\x0f\xf9\x9d\xf0\x05\x0a\x30\xcc\x5a\xfd\x97\x3c\xc6\xb5\x17\x05\x20\x71\x8d\x92\xbb\x97\x5e\xa7\x58\x6b\xa0\x02\x40\xa7\xc3\x5c\xa1\x21\xb1\xac\x22\x64\xd6\xe6\x32\x7b\x40\x83\xcf\x11\xdc\x09\x72\x84\x5b\xae\x05\xbf\xf3\x73\x70\x39\x5e\x3b\x1f\x4c\x49\x59\x45\x56\x9b\xeb\xcd\xfc\xad\x9c\x00\x25\x62\x28\x8d\x14\x52\x24\x30\x44\x42\x6c\x71\x6a\x44\x86\xc1\x02\x41\xc6\xae\x44\xa4\x47\x1e\x8f\x4e\x98\x12\x96\xb5\x2b\x74\x01\x1c\x23\x0d\xf1\x90\x8a\x9a\x94\x20\x54\x56\xa3\x80\x57\xbf\x87\x2c\x0d\x42\xf0\x23\x74\x39\xc2\xf0\xbc\xa2\x04\x43\x97\xc6\x0f\x9e\x70\xa9\x77\xf2\xdf\xb5\x7a\x34\x50\x71\xc6\xa3\xbb\x5f\xe0\x60\x51\xb0\xc2\xbc\x85\xdc\x7b\xda\xdf\x66\xdf\xec\xcb\xd9\xf4\x9f\xf1\x60\xb6\x81\x7d\xc5\xdc\xcb\x38\x26\x13\xbc\xf0\xb9\x47\xcc\x32\x0f\x4c\xc0\xf3\x42\xf8\x08\x9a\x98\x63\x06\x4a\xc0\x6f\x82\x5c\xa3\x90\xde\x50\xba\xb0\x20\xa6\x99\x62\x12\xb0\xb4\xe1\xdf\x08\x0c\x82\xdb\x57\x91\x44\x62\x9a\x29\x26\x85\x98\x71\x12\xd4\x02\x41\x0b\x4e\x4c\xb2\x08\x4d\x22\xa9\x89\x7c\x9a\xa0\x9e\xa3\x99\xa1\x12\x10\xbc\x89\x82\x3b\xd0\x36\x60\xa4\x25\xb8\xd4\x44\xe9\x5e\xfa\x54\x21\xf7\xaf\x39\x3a\x73\x30\xed\x65\xb9\x2b\x09\x52\x30\x6c\x2a\xa1\x25\x1f\xda\x15\x13\x5a\x5f\x49\xe4\x42\x82\x6e\x77\x4e\x4b\x37\x13\x35\x05\x90\x27\xe0\x87\xdc\x76\xc8\x6d\x87\xdc\x76\xc8\x6d\x87\xdc\xf6\xc7\x73\x5b\x5f\x03\x47\x48\x52\xc9\xc7\xce\x70\x03\x1f\xe2\x9b\x86\x98\x23\xe1\xca\x7c\x71\xef\x5b\xd1\x97\x48\x65\x6c\x4d\x7c\xab\xe0\xec\xe7\x60\x50\xab\xe5\x9b\xbc\xbd\x4d\x05\x8a\x0a\x12\x24\x71\xe5\xc3\x28\x50\x1a\x93\x22\xad\x41\xe2\x94\x7b\xb7\xdc\x8f\x80\x98\x6a\x06\xfb\xea\x52\x62\x69\x4e\x86\xc7\xb6\x51\xad\xe0\x5b\xd7\xea\xe1\xe3\x7b\xd6\x0d\x3c\x8e\x79\xec\x5b\xad\xa7\xd9\xaa\x74\x52\xc1\x9d\x86\x41\xa8\x34\xda\x60\x54\xa4\x1d\xa8\x7c\xb0\x12\x0f\xfe\x61\x0f\x89\x5f\xd6\x0a\x91\xe0\x3d\xc5\x29\x49\x48\x1e\xdb\x6e\x2f\xc9\xa9\xe6\xd2\xdc\x3f\xfb\x81\x57\xd1\x97\xf2\xc8\xf5\x7a\xd3\x60\x71\xf0\xa6\x83\x37\x91\x95\x04\x51\x85\x99\xb5\x37\xfa\xd4\x1e\x72\x54\x4c\xfd\x8d\xa9\x71\x0e\x3e\x20\xbc\x51\x87\x6c\xf8\x9f\x77\xdc\x6a\x07\xbe\x99\xdc\x63\xad\x3c\x0d\xc6\x7c\x10\x77\x78\x89\x67\x8c\x11\x99\xba\x56\x72\x86\x4e\x48\xf8\x0a\xa4\x87\xf3\xba\x08\x67\xe8\x84\x84\x57\x85\xb0\xba\xf8\xa6\xe0\x4d\x14\x99\x6c\xb8\x07\x0d\xb2\x72\x80\x78\x27\x75\xa6\x77\xb2\x89\x79\xbf\xfb\x82\x9f\xb3\x56\xb6\x17\xe5\xcf\x91\x12\x68\xbf\x81\x3d\xf7\xcf\xd9\x04\xf0\xbb\x52\xbf\x03\xae\x7f\x57\x5d\x58\x1c\xc1\x53\x7a\x39\x5d\x86\xd4\xe5\x87\x22\x34\x45\xf9\x91\x7c\x61\xd9\xff\xa3\x75\x3f\x56\x66\x02\x8e\x92\x2e\x31\xc9\x04\x97\xd2\xd7\xb3\xdf\xd9\x9f\x2c\x6d\x02\x5e\x41\x4f\xd7\xb8\xcc\xe4\x4d\x1d\x19\x90\xee\xe0\x01\x24\x9a\xb3\xae\x54\xdd\xed\x43\xfc\xd2\x56\x4d\x97\x23\x8f\x9d\xb4\x67\x50\x0b\xe9\x31\x6b\x7d\x23\x61\x66\xb6\xa6\xf4\x0a\xd3\xcf\xf5\xcc\xd6\x6d\xf8\xe5\x2e\x55\x52\x06\x4b\x30\x45\x0a\xe5\x6d\x90\x6b\x33\x91\xf8\x9b\xd6\xe8\xfa\x4b\x21\xb3\xdb\xe0\x84\x37\x9f\xd9\xcd\xbd\xe7\xb2\x64\x58\xa7\xcd\x9d\x21\xaa\x79\xdb\xdb\xc7\x51\x4d\x59\x2e\xef\x98\x69\x8a\xc1\xee\x3e\xd4\x17\xac\x77\x18\xf7\x7d\xf5\x08\xee\xfa\x58\x60\x9d\x32\x72\xf7\xd3\x1e\xd7\x6b\x40\x1e\x8f\x66\x56\xe1\xcf\xf2\x01\xe7\xab\xd4\xd1\x9f\x0b\xdf\xd5\x20\x77\x0d\x2b\xde\xcb\xc5\x7e\x22\x51\xb6\x1b\x83\x1a\x73\x88\x92\x2e\x95\xa6\xec\x6f\xb5\x1a\x11\x46\xd8\x72\xab\x5b\x9d\x43\x8d\x58\xad\x39\xa1\x94\x1b\xde\x5d\x9e\x69\xd4\xf4\x76\xc5\x85\x66\xc9\xf7\x47\xd7\xe3\xab\xc1\x74\x70\xbe\xc7\x6a\x1f\xd8\xf6\xc8\xde\xe3\xb9\xe1\xcd\x6c\x6c\x8f\x2e\xed\xc1\x64\xb2\xc7\xd3\x93\xe9\x68\x3c\xde\x6d\xbf\xee\xb0\x51\x5e\x1d\x6a\x6a\xf9\x96\x96\x7b\x1a\x23\x50\x38\x2a\x6e\xde\xfe\x93\xc2\x6f\x6e\x4d\x69\x51\x5b\xf4\x2a\xd9\xfa\x3e\xb7\xa0\x3b\xcc\x85\x7b\x1e\xf9\xb8\x52\xeb\xc8\x22\xfe\xa0\xd9\x3f\xb2\xd4\x4b\x24\xdb\xd3\xb5\x81\x4b\xa1\xa0\xf2\x8a\xb8\xb7\x1f\x27\x66\xed\xda\x5c\x32\xcb\x38\xa1\xdb\xfb\x2f\x00\x00\xff\xff\x3e\xc6\x59\x3c\xb6\x3a\x00\x00"), }, "/X_MS_MediaReceiverRegistrar.xml": &vfsgen۰CompressedFileInfo{ name: "X_MS_MediaReceiverRegistrar.xml", - modTime: time.Date(2019, 5, 27, 15, 4, 10, 118232995, time.UTC), - uncompressedSize: 2485, + modTime: time.Date(2022, 10, 1, 4, 23, 36, 727914200, time.UTC), + uncompressedSize: 2572, - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x56\xc1\x6a\xdb\x40\x10\x3d\xcb\x5f\x61\x74\x77\x36\x86\xd2\x43\x58\x6f\x30\x38\x04\x43\x0b\x45\x71\x0d\x3d\x99\xb5\x76\x50\xa6\x95\x66\xd5\x9d\x95\x49\xfa\xf5\x45\x5a\x39\x92\x13\x25\x14\x27\x4e\x2e\xbd\xcd\xcc\x7a\xde\x7b\xde\xa7\x19\x49\x5e\xde\x15\xf9\x78\x07\x8e\xd1\xd2\x2c\x9e\x9e\x9d\xc7\xe3\x4b\x35\x92\x9c\x96\x66\x7c\x57\xe4\xc4\xb3\xb8\x72\x74\xc1\xe9\x2d\x14\x9a\x27\x55\x49\xe5\xc4\xba\xec\x82\xc1\xed\x30\x85\xc9\x74\x72\x1e\xab\x51\x24\xb9\x84\x74\x1d\x60\xd4\x28\x8a\x64\xa1\x7f\x5a\xa7\xa6\x52\x84\xa0\x29\x21\x59\xa7\xce\xa5\x08\xc1\x28\x92\xe2\xb0\x4b\xea\xd4\xa3\xa5\x2f\xc8\xbe\x69\x08\x69\x1d\x46\x92\x74\x01\x6a\xc9\xf3\xca\xdf\x5a\x87\x7f\xc0\x48\xd1\x94\x9a\x43\xed\xb2\xaa\x00\xf2\xfb\xce\x5e\x29\xa4\x6d\xff\x02\x6a\xcd\xcb\x45\xaf\x37\x8a\xa4\x41\x07\x81\x09\x49\x8a\x2e\x6b\x8f\x1d\xe4\xda\x83\xb9\xf1\xda\xc3\x5a\x3b\xd4\xdb\x1c\xd4\x7c\x33\x4f\xae\x37\xab\x1f\xdf\xae\x36\x1d\xe8\xe0\x2f\x83\x1c\x71\xa8\x67\x58\x5e\x02\x5c\xe5\xfe\x39\x71\xb6\xf2\x47\xa8\xdb\x63\xfe\xb3\xb6\x2e\x7d\xf0\x41\x74\x46\x3c\xf5\x24\x81\x0c\xd9\x83\x0b\xd7\x70\x8c\x2b\x01\xc1\xe9\x1a\x38\x81\xdf\x5f\x39\x7b\x53\x7f\x86\xe0\x5f\xef\x54\x1f\x93\xcb\x17\x34\x1f\x6b\xdb\x00\xc1\xc9\x3c\x5c\xf2\x5a\xe7\x68\x6a\xf0\xff\x63\xf5\x01\x63\xb5\x0f\xdb\x23\xd9\x2e\xd7\x06\x75\xb5\x87\x94\xdc\x27\x19\x33\x90\xb9\xda\x01\x79\x9e\xc5\x64\xe3\x9e\x9b\x83\x97\xd8\xb9\x6a\xb4\xd7\xab\xfb\x12\x14\x7b\x87\x94\x49\xf1\x50\x68\x44\xf1\xe3\xbf\x72\x0c\xef\x93\x2b\xef\x58\x91\xfc\x89\x28\x5f\xd8\x22\x1d\xfd\x16\xe9\x6c\xab\x19\x3e\x7f\x7a\x07\x15\x8f\x17\xc3\x5b\xcb\xb8\x07\x3e\xd0\xd1\xbe\x1c\x1b\xf6\x6b\xa7\xc9\x83\xf9\x5e\xd6\x63\xfd\xcc\x13\x50\xe1\xc9\xe8\x17\x40\xf8\xae\xec\xed\x06\x43\x4b\x37\x55\x9a\x02\x98\x0f\x62\x4f\x60\x67\x7f\xbd\x9e\x5b\x8a\x81\x25\x20\x45\xfd\x5d\xa6\xfe\x06\x00\x00\xff\xff\xfa\x10\x27\xa0\xb5\x09\x00\x00"), + compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xec\x96\xd1\x6b\xdb\x30\x10\xc6\x9f\x5d\xe8\xff\x10\xfc\x9e\x2a\x81\xb1\x87\xa2\xb8\x04\x52\x4a\x60\x83\xe1\x66\x81\x3d\x05\xc5\x3a\xdc\xdb\xec\x93\xa7\x93\x43\xbb\xbf\x7e\x38\x72\x62\xaf\x71\x0a\x0b\x4e\xf3\xd2\x37\x9d\xd4\xfb\xbe\x5f\xf5\x45\x97\xc8\xbb\xe7\x3c\x1b\x6c\xc0\x32\x1a\x9a\x84\xe3\x9b\x51\x38\xb8\x8b\xae\xaf\x24\x27\x85\x1e\x3c\xe7\x19\xf1\x24\x2c\x2d\xdd\x72\xf2\x04\xb9\xe2\x61\x59\x50\x31\x34\x36\xbd\x65\xb0\x1b\x4c\x60\x38\x1e\x8e\xc2\xe8\xfa\x2a\x90\x5c\x40\xb2\xf4\x42\x55\x1d\xc8\x5c\xfd\x34\x36\x1a\x4b\xe1\x17\x7e\x0f\xc9\xd8\x68\x24\x85\x5f\x54\x7d\xe2\x55\xa3\x54\x89\x43\x43\x5f\x90\x9d\xef\xf1\xf5\x76\x1d\x48\x52\x39\x44\x73\x9e\x96\xee\xc9\x58\xfc\x03\x5a\x8a\xed\x96\x3f\x55\x36\x2d\x73\x20\xb7\x6f\x6e\xed\xd5\x75\x2d\x31\x83\x8a\x7e\x3e\x6b\xb7\x07\x81\xd4\x68\xc1\xdb\x21\x49\xd1\x54\xbb\x73\x0b\x99\x72\xa0\x1f\x9d\x72\xb0\x54\x16\xd5\x3a\x83\x68\xba\x9a\xc6\x0f\xab\xc5\x8f\x6f\xf7\xab\x46\xb6\xf3\x2f\x6b\x24\xf1\x8a\xe9\x08\x63\x0c\x5c\x66\xee\x28\xa1\x29\xdd\x29\x88\x3b\xd5\xff\x00\x6c\xea\x26\x15\xd1\x8a\xa5\x23\xa2\x18\x52\x64\x07\xd6\x5f\xc8\x89\x21\x79\x11\xab\x2a\xf1\x18\x7e\x7f\xe5\xb4\xe7\xb8\xba\x0c\x7a\x09\xae\x2d\xcb\xc5\x5b\xe0\x27\xa7\xd8\x61\x71\xde\x48\xe7\xbc\x54\x19\xea\xca\xe1\xe3\xd1\x5d\xfc\xd1\xed\xd6\xbb\x43\x59\x0f\xe4\xad\xf4\x62\xaf\x2b\xb9\x6d\x35\x60\x20\x7d\xbf\x01\x72\x3c\x09\xc9\x84\xed\x78\x3b\x2f\xb4\x15\xb3\x56\x4e\x2d\x5e\x0a\x88\xd8\x59\xa4\x54\x8a\xfd\x86\x47\xe3\x83\xff\xe9\x24\xef\xc3\xfb\x6f\x9c\x91\xdc\xf9\x6c\xdf\x1a\x35\x0d\xc2\x1a\xe9\x66\xad\x18\x3e\x7f\x7a\x1f\x92\x83\xe1\xd1\x3f\xca\x0b\xf0\xbf\x2c\xf5\x77\xeb\x96\xe0\xc1\x2a\x72\xa0\xbf\x17\xd5\xbb\x3f\xf6\x89\x28\xf1\x9c\x08\x33\x20\x7c\x77\x82\x7a\xd4\xa1\xa1\xc7\x32\x49\x00\xf4\x05\x09\x62\xd8\x98\x5f\xbd\xf8\x4b\xd1\x35\x25\xa4\xa8\x7e\xed\x45\x7f\x03\x00\x00\xff\xff\x63\x7d\x2d\x0d\x0c\x0a\x00\x00"), }, "/rclone-120x120.png": &vfsgen۰FileInfo{ name: "rclone-120x120.png", - modTime: time.Date(2019, 5, 27, 15, 4, 10, 118634349, time.UTC), - content: []byte("\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52\x00\x00\x00\x78\x00\x00\x00\x78\x08\x06\x00\x00\x00\x39\x64\x36\xd2\x00\x00\x00\x04\x67\x41\x4d\x41\x00\x00\xb1\x8f\x0b\xfc\x61\x05\x00\x00\x00\x20\x63\x48\x52\x4d\x00\x00\x7a\x26\x00\x00\x80\x84\x00\x00\xfa\x00\x00\x00\x80\xe8\x00\x00\x75\x30\x00\x00\xea\x60\x00\x00\x3a\x98\x00\x00\x17\x70\x9c\xba\x51\x3c\x00\x00\x00\x06\x62\x4b\x47\x44\x00\xff\x00\xff\x00\xff\xa0\xbd\xa7\x93\x00\x00\x00\x09\x70\x48\x59\x73\x00\x00\x0b\x13\x00\x00\x0b\x13\x01\x00\x9a\x9c\x18\x00\x00\x00\x07\x74\x49\x4d\x45\x07\xe3\x03\x0b\x05\x07\x12\x76\x96\x1b\x39\x00\x00\x50\x82\x49\x44\x41\x54\x78\xda\x9d\xbd\x77\x7c\x1c\xc5\xfd\xff\xff\x9c\xdd\xbb\x93\x74\xea\xd5\x92\x2c\x57\xd9\x96\xbb\x71\xc1\xc6\x60\x30\x18\x30\x98\x5e\x42\x0f\x04\x42\xe0\x13\x92\x40\xc2\x87\x34\x42\x12\x48\x03\xd2\x49\x08\x9f\x84\x10\x02\x84\xd0\x7b\x09\xc5\x98\xe2\x42\x31\xd8\xb8\x77\x5b\xee\xb6\x9a\xd5\xbb\x6e\x77\x7e\x7f\x6c\x9b\xd9\x5b\x99\x7c\x7f\xeb\xc7\xfa\x4e\x77\xb7\xbb\x33\xf3\x9a\xf7\x7b\xde\x6d\xde\x6f\xf1\xc0\x8e\x2e\x29\x01\x5b\x7a\xa7\xc4\x72\xff\xb6\x94\xcf\x6c\xf7\x33\x09\x48\xf7\xd5\x3b\x04\x20\x84\xfb\xea\xbe\xf7\x0e\xe9\xfe\x27\xd1\xaf\xd1\xae\x75\xaf\x31\x00\x53\x08\x4c\x01\xa6\x01\x31\x21\x88\x09\x9c\xbf\x05\x08\x21\x10\xca\x7d\xa5\x94\x4e\xfb\xb4\xf6\x06\x6d\xf5\xbe\xf3\xda\x80\xdb\x46\x43\x6d\xaf\xd2\xee\xa8\x43\x86\xde\x0b\xe5\x34\x84\xc0\x10\x28\x6d\x74\xdb\x6b\x38\xef\x0d\xc0\x10\xce\xe9\xf5\x55\x86\xda\x66\x49\x89\x25\x21\x65\x43\x4a\x4a\x52\xee\x7b\xef\x73\x1f\x03\xa4\x3f\xfe\xea\xd8\x19\x02\x0c\x04\xc2\x6d\x83\x21\xc0\x54\xdb\x96\x30\x84\x33\x50\xc2\xbb\x99\xc0\x90\x60\x21\x11\x80\xe5\xdd\x4a\x4a\x10\xce\xc3\xa2\x3a\xee\x7e\xed\x80\xab\x7c\x29\x43\xbf\x0f\x83\xec\x0f\x96\xf7\x87\x3b\x8c\xc2\xeb\x14\x01\xa8\x86\x72\xb5\x37\xc9\x54\x30\xad\x30\xb8\xee\x20\x82\x0c\xda\x87\x40\x0a\x30\xa4\x0e\x96\xff\x78\x11\x4c\x48\xbe\xa0\xcd\x12\x89\x54\xda\x27\x90\x58\x42\x04\x33\xca\x7f\x1f\x80\xab\x4e\x74\xe9\xfe\x21\x23\x1e\x16\xfe\xad\xfa\x6c\xbd\x1d\xc2\x27\x8e\x60\xf2\x0a\x17\x78\x88\x25\x0c\x90\x08\x9f\x02\x2c\x29\x31\x24\x18\xd2\x19\x64\x61\x7b\x40\x2b\x20\x03\x22\x82\x2a\xfd\x7e\x85\x3b\xe1\x37\x56\xf9\xc3\x9f\xd5\x22\xbd\xf3\x52\x22\x8c\x80\x56\x25\x02\xd3\x05\xce\xbb\xce\x07\x58\x99\xdd\x1a\xd7\x51\x66\xbc\xd7\x16\x21\x1d\x10\x0c\x9c\x3e\x0b\xe1\x7d\xa6\x0c\xdc\x20\x13\x32\x8d\x63\xb9\x83\x69\xba\x9d\xf1\xbf\xb7\x65\x40\xb2\x6e\xdb\x7d\xca\x57\x26\x8f\x37\x1e\x83\x4d\x7c\xf5\x59\xd2\x1f\xa7\xe0\x57\x0e\xe7\x11\x3e\x07\xf2\x38\x85\xa1\x80\x6b\x08\x88\x65\x18\xce\x65\x01\x4b\x10\xa4\x24\x18\x2e\xb0\xc2\x70\x81\xb5\x9d\x41\x91\x52\xa6\xb1\x33\x3b\xa2\x75\xd2\xed\x49\xe4\xac\x55\x26\xb8\x70\x39\x85\x0d\x98\x08\xe7\x7b\xa1\x0f\x94\x74\x29\xd9\x08\x01\x60\x4b\xe9\x83\x1c\x06\x56\x05\xd7\x7b\xa6\x37\xd3\x3d\x48\x0c\x77\xe0\xc2\x94\x81\xd6\x66\x99\xc6\x85\xfc\x25\x45\x0a\x6c\x01\x52\x48\xa4\x11\xdc\x41\xda\xd2\x7f\x98\x07\xa2\x21\x1c\x82\x51\xc7\x2c\x6d\x6c\x22\x26\x91\xf7\x87\x90\xce\x3d\x02\xb8\x83\xe5\xc5\x0c\x81\x6b\x2a\x80\xc7\x32\x4c\x8f\x9d\x39\xc0\x7a\x14\x9c\x22\x00\xd6\x7b\x82\xda\x20\xe1\x51\x91\x74\x3b\xa1\xb2\x65\x75\x90\xc2\x1d\x09\x4d\x06\x95\x1a\xa4\x94\x18\x42\x04\xbf\xb5\x9d\x81\x93\x12\x0c\x11\x4c\x2c\xef\x16\xbe\x5c\xe0\x02\x6b\x29\xc0\xaa\xf2\x42\x3a\x35\xe8\x40\x87\x01\x8e\x6a\x7f\x1a\xc0\x12\x84\x90\x98\xb8\x13\xdf\x96\x01\x29\x79\x13\xc3\x76\xe4\x09\x6f\x92\x7a\x2c\x35\x4c\x00\x47\x3b\xfc\xe5\xc0\x05\xd9\x13\x24\x34\x59\xc2\x93\x5f\x08\x28\xd7\x74\x01\x8f\x25\x0c\xe1\xcf\x78\x43\x4a\x52\x52\x20\x6c\x10\x86\x04\x5b\x01\xd6\x65\xcd\x86\x4b\xc5\x76\x44\x63\x54\x8a\x19\xfc\xbd\xc2\x66\x64\xc0\x62\x54\xb6\x85\x3b\x68\x1e\x35\xd8\x2e\xb5\x78\xc2\x5b\x70\x4f\xa9\x0b\x53\x8a\xc0\x15\x9e\x74\x01\xc0\x42\x01\x17\x0c\xc3\x70\x26\x95\xd0\x29\xd0\xb6\x6d\x6c\x29\xd3\xfa\xe5\xdf\xcb\x5d\xc7\xa5\x4b\xc1\x3e\xb0\x2e\xe7\x09\xae\x74\xde\x1b\x1e\x20\xa4\xdf\x73\xd0\xc9\x2f\x82\xc9\xa4\xce\xc2\x40\xc8\x73\x7e\xa3\x0a\x55\x1a\xab\xc6\x5d\x83\x3d\x16\x67\x48\xe1\xac\xb9\x06\x60\x0b\xa4\x21\xfd\xd9\x29\x0d\x81\x6d\x3b\x83\x26\x85\xc0\x70\x07\x57\x5d\x57\xc2\x6c\xd1\x26\x00\x55\x1d\xa0\xa0\xbd\xd2\x6f\xac\x2d\x83\xb5\x43\x02\xa6\x0c\x06\xca\x70\xa9\x45\x28\x33\x5e\x46\x81\xea\x4e\xbc\x30\xe5\x79\x83\x62\x78\x13\x48\x08\x4c\xc3\xc4\x4a\x0d\xd0\x5e\x7f\x88\xb6\x43\xfb\xe8\x3a\xd2\x40\xaa\xbf\x8f\x58\x46\x16\x85\xc3\x46\x51\x30\xbc\x9a\x78\x56\xb6\x0b\xb4\xad\xb5\xdd\x1b\x74\x3b\xc4\x6d\xc2\x20\xeb\xb4\xeb\x5e\x9b\x36\xa1\x3d\x61\x4b\x86\xc6\x46\x69\xb7\x08\xfa\xe3\x0b\x9c\x0a\xe5\x7a\x42\x95\xf7\xb7\x27\x70\x09\x01\xb1\xb8\x10\x2e\x85\xc8\x80\x05\xd8\x12\x69\x80\xb4\xbd\x19\xea\x0e\xa2\x10\xd8\xc2\x59\xbb\x0c\xb7\x0f\xe1\xf5\x57\x05\xd7\x5b\x23\x55\x30\xd4\x41\x52\x67\xa3\x10\xee\xe4\x51\x06\xc3\xa1\x5e\xb7\xf1\x32\xfd\x19\xe1\x75\xd8\x93\xaa\xa3\x58\x9f\x37\x48\xa6\x61\xd0\xdf\xd7\xc7\x81\xd5\xcb\xd9\xfc\xc6\x73\x1c\x5c\xf7\x09\x9d\x8d\x75\xa4\xfa\x7a\x91\xb6\x8d\x30\x4c\x32\xf3\x0b\xa9\x98\x38\x9d\xf1\x67\x5c\x4c\xf5\xfc\x45\x64\x16\x14\x63\x49\x2b\x8d\xdd\x1b\x0a\x1c\xc2\x9d\x75\xc2\x7f\x1f\xfc\xd6\x13\xc4\x7c\x6d\x41\x1d\xaf\x88\xb5\x38\xea\xf0\x08\xc9\x9b\x20\xfe\xd8\xb9\xaa\xa3\xaa\x6e\x0a\x11\x70\x46\xf1\x61\x63\x9f\xf4\xf4\x2d\x4b\xc2\x80\x2d\x19\xf0\x5e\x6d\xe8\xf7\x5e\xa5\xf3\xea\x7d\xef\xeb\x6d\x36\xa4\xa4\xab\x3f\x4b\x42\x3a\xa8\x2e\xf0\xa8\xe0\xf8\x8d\x46\x5f\x4f\x3c\xbd\xce\x54\xd8\x8d\x69\x18\x08\x21\xb0\x53\x29\x70\xd9\xa9\xa7\x03\x87\xd7\x5b\x55\x32\x55\x29\xc1\x10\x02\xd3\x30\xe8\xac\xdb\xcf\xca\xbf\xdf\xcb\xd6\xb7\x9e\xa7\xbf\xbb\x8b\xa3\x1d\x86\x69\x52\x35\x73\x1e\xc7\xdd\xf8\x43\x86\xce\x9c\xe7\xaf\x9b\xde\x8d\x1d\xbd\x5d\xd7\x7f\x3d\xdd\x3d\xe6\xbe\xf7\x74\x78\x53\xa1\xb6\xf0\x7a\xef\x4d\x52\x15\x07\x4f\x9f\x8f\x12\xf0\xbc\xb1\x33\x48\x17\xac\xbc\xb5\xd7\x1b\xbf\x98\x29\x84\xcf\x2a\xc1\xa5\x58\x85\x5a\x63\x42\x60\x1b\x92\x98\xed\xfc\x6d\x0b\xe1\x3c\x5c\x04\x54\x6c\xb8\x0d\x51\x75\x60\x6f\x0d\x0c\xb3\xa2\xb0\xda\xe2\x37\x5a\xa5\x52\xe1\xd2\xb8\x61\x62\x5b\x16\x47\xf6\x6c\x67\xf7\x8a\xb7\xa9\xdb\xb8\x9a\xec\x92\x72\x66\x5c\x7d\x33\xc9\xb2\x4a\x67\xc9\x88\x50\x87\xd4\x09\xe4\xcf\x6c\x43\xd0\xbc\xaf\x96\xf7\x7e\x71\x33\xfb\x3f\x5b\xca\x7f\x73\xd8\x96\xc5\xbe\x4f\x97\x72\xa4\x76\x1b\x27\xde\xfa\x2b\xc6\x2d\xba\x44\x53\x05\x83\x49\x24\xb1\x42\xba\xbb\x2d\x1d\x5b\x82\xfa\x4b\x6f\x45\x0e\xf4\x68\x15\xe4\xf4\x65\xe5\x68\xc7\xd1\x8c\x33\xea\x11\x33\x85\xd7\x44\x19\xcc\x08\x24\xa6\x70\x64\x2c\xdb\x9d\x1d\xb6\x3b\x3b\xbc\xcf\x3c\x96\x6d\xe0\x48\xdd\xb6\x10\x88\x08\x15\xca\xef\x9c\xb6\x3e\x2b\x22\x88\x4b\xc6\xfe\x7a\x8e\x40\x18\x26\xfd\xbd\x5d\x34\x6d\x59\xcb\xce\xb7\x5f\x60\xf7\xb2\x37\x69\x3f\xbc\xcf\xbf\x57\xa2\xb0\x84\x19\x5f\xfd\x2e\x96\xbb\x36\x46\x09\x41\xea\x20\x18\x42\x30\xd0\xd1\xce\x8a\x3f\xfd\x24\x12\xdc\xec\x92\x21\x94\x4f\x9c\x4e\x56\x61\x09\x5d\x47\xea\xa9\xdf\xbc\x96\xee\xe6\x46\xff\xfb\xae\xa6\x3a\x96\xfe\xf6\xfb\xc4\x72\xf2\x18\x75\xd2\x22\xa4\xb4\x40\x8a\x80\x18\xc0\x95\x49\x84\x3b\x2e\x12\x4b\x7a\xac\x33\x30\x85\x48\xa5\x4d\xaa\x46\xa0\x2f\x6b\xce\xf8\x44\xc9\x2c\xda\x78\x2a\xf7\xd3\xee\x83\x4e\x4c\xb1\x40\x37\x14\x18\x52\x3a\x14\x29\xd1\x24\x32\x13\x81\x25\x24\xa6\x50\xa8\x18\x87\x8a\xa5\x0b\xbe\x27\x1c\x79\xda\x82\xf0\x48\x32\xbc\x46\x87\x50\x90\x52\x3a\x42\x9b\xcb\x86\x7b\x5a\x8f\x70\x78\xd5\x32\x76\xbe\xf5\x2c\x87\x56\x2d\xa7\xaf\xa3\x35\xad\x73\xdd\xad\xcd\x0c\x58\x12\xdb\x0e\x26\x8c\xc6\x19\xdc\x57\xc3\xd3\x17\x4d\x83\x1d\xef\xbc\x44\xed\x07\xaf\xa7\xdd\x6b\xcc\x29\x67\x73\xc2\xd7\x6f\xa7\xa4\x7a\x02\x46\x2c\x4e\xaa\xbf\x9f\xc6\x5d\x5b\xf8\xf4\x91\x3f\xb2\xe3\x9d\x97\xfc\xf6\xf6\xb4\x1e\x61\xe5\xdf\x7e\x45\x49\xcd\x14\x72\xca\x86\x22\xa5\xed\x1b\x31\x24\x80\x61\x3a\x36\x03\x01\xb6\xb4\x5d\x6a\x0e\xe0\x90\x21\x0b\x5a\x18\x2c\x5f\x20\x0d\x99\x85\x3d\xe2\x50\x67\xad\xb7\xc6\x4b\xe1\xf5\x3d\x98\x6c\xde\x32\x65\xbb\x4b\x41\x2c\x6d\x81\x96\x2e\x15\xbb\x3a\xa9\x21\x1c\x0a\xd5\xa8\x57\xe0\xb0\x21\xe1\xdc\xdc\xf6\x54\x29\xe9\xfc\xde\xeb\xb4\x2d\xf5\x35\xc3\x61\xe3\x22\x0d\x64\x81\xa0\xe3\xe0\x1e\xf6\x2d\x7b\x83\xda\xc5\x2f\xd0\xb4\x75\x2d\xf6\x40\x7f\x1a\x18\xf1\xec\x5c\x2a\x66\x9e\xc8\x98\x73\xbe\x4c\x4a\x4a\x4d\x88\x0b\xaf\xbb\xc2\x55\xd4\x0d\x21\xe8\xeb\x68\x63\xfb\x1b\x4f\x23\x2d\x4b\xbb\x5f\xd5\xcc\x13\x38\xfd\x8e\xfb\xc8\x1d\x32\x14\x69\xa5\x1c\xe9\x3d\x91\xa0\x7c\xd2\x4c\x4e\xfb\xf1\x7d\x58\xfd\x7d\xec\xfa\xe0\x3f\xfe\xef\x1b\xb7\xac\x65\xe7\x92\x97\x39\xe6\xaa\x6f\xf9\x0f\xf4\x84\xab\x5d\x2b\x16\x53\xb7\xe1\x33\x86\x4e\x9c\xce\xe8\xd9\x27\x92\x9d\x97\x8f\x6d\x5b\xbe\xcd\x46\xb5\x6a\x79\xe3\xa1\x82\xe7\xad\xc5\xb6\xd2\x9f\xb4\x25\xcd\xfd\x4c\xb1\xa9\x04\x93\x48\xfb\x91\xf7\x81\x20\xc6\x20\x47\x00\xb6\x7b\xba\xe0\xf9\x54\x8c\xbe\x5e\x4b\xa4\xfb\xb7\x70\x2d\x4f\xce\x35\xd2\x95\xba\x55\xb0\x51\x40\x96\x52\x52\xfb\xf6\xb3\xac\xfd\xe7\x6f\x68\xdb\xb3\x3d\xba\x2d\xa6\x49\xcd\xb9\x57\x33\xe6\x9c\x2b\x29\x1a\x3b\x05\x33\x2b\x9b\x94\x65\x0f\x3a\x08\x3e\x0b\x14\x20\x0c\x83\xd6\x7d\xbb\x38\xb2\x63\x93\x76\xcf\x58\x46\x26\x33\xbf\xfc\x2d\x72\x86\x0c\xc5\xb6\x52\xda\xba\x28\xad\x14\xc9\x82\x12\xa6\x5f\x79\x13\xfb\x3f\x5d\x4a\x7f\x77\xa7\x7f\xdd\x9e\x65\x6f\x32\xf9\xc2\x6b\x89\x67\x25\xdd\xb5\xdd\xe0\xc8\x8e\x8d\xbc\xf5\xb3\x9b\xe9\xa8\x3b\x40\x2c\x23\x93\xd1\x73\x4f\xe1\x84\xaf\xdc\xcc\xe8\x59\x27\x60\xc4\xe3\xbe\x9a\x65\x28\x02\xa0\xa6\xf2\xa1\x03\x1a\x96\x29\x7c\xb9\xc6\xa5\x5e\x4b\x2a\xaa\x50\x1a\xba\xf8\xae\x03\xe9\x2e\x9f\x83\x8a\xe8\xaa\x42\xed\x49\x68\x9e\xd4\xe8\x78\x7c\x9c\xd7\x98\xfa\x99\xbb\x66\xfb\xe6\x32\x14\x43\xb8\x2a\xf4\x08\x81\x61\x9a\xb4\xef\xd9\xce\x67\xf7\xff\x64\x50\x70\x01\xa4\x65\x51\x38\x6e\x32\xe5\x33\x4f\xc4\xc8\x4c\x92\xb2\xed\xc0\x76\x8e\xea\xf5\x22\x4d\x92\x07\x41\xfb\x81\xdd\xf4\x77\xb6\x6b\xf7\x2c\x1c\x39\x8e\xca\x63\xe6\x22\x5d\x2a\xd3\xfa\x2c\x00\xdb\xa2\x74\xec\x24\xf2\x86\x8e\xd0\xae\x6b\xdd\xbb\x93\x9e\x23\x0d\x98\x86\xe1\x4b\xf8\xed\x87\xf6\xd1\xd9\x70\x18\x80\x54\x5f\x2f\xdb\x3f\x78\x93\xa7\xbf\x7d\x25\xaf\xdf\xfd\x3d\x1a\x76\xef\x08\x84\x2e\xe5\x4c\x85\x5e\x2d\x5b\xff\xdb\xf7\xe2\x49\x89\x85\x74\x35\x12\xc5\xe6\xae\xf4\x3b\xb8\xaf\xd4\xee\xe5\xd8\x36\x64\xb0\xb0\xa7\xb9\xa3\x44\xe0\x0c\xf0\xd4\x17\x0f\xe8\x98\xff\x2a\x5c\x95\xc0\x51\x0d\x54\xe0\xbd\xc9\x60\xb8\x2a\x84\x69\x9a\xc4\x62\x31\xe7\x7b\x02\xd5\x48\xda\xc1\x6a\x95\x91\x5b\x40\xe9\xc4\x19\x18\x31\x9d\xb9\xec\x79\xff\x35\x7a\xdb\x5a\xd2\x8c\x1b\xea\x40\x84\x75\x62\xef\xec\x6b\x6f\x45\xda\x3a\x7b\x2e\x19\x33\x91\xac\xfc\x42\x9f\x47\x8a\x88\x33\x91\x95\x24\x33\xaf\x40\xbb\x6e\xa0\xbb\x93\x81\x8e\x16\xc7\xb6\x8c\x33\x11\x2a\x27\xcf\x64\xec\xfc\x45\x18\xa6\xe9\xff\xae\xa7\xbd\x95\x95\x4f\xfd\x9d\xa7\x6e\xbd\x9a\xba\x1d\x9b\x91\x86\x91\x06\xac\x7f\xda\xde\x7b\x89\x65\x4b\x2c\x29\x9d\xf7\x72\x30\x10\xbd\xdf\xe9\x13\xc6\xbb\x87\xef\x76\x94\x8e\x4f\xc1\x19\x2c\x95\x55\x28\xe6\x34\xd5\xee\xa9\x53\xb2\xab\xef\xa9\x7a\xa0\x10\xe9\x54\xee\xbe\x17\x96\xc5\xa1\xcf\x3e\x60\xe7\x9b\x4f\x93\xea\x6a\xc7\x30\x04\x42\xda\x14\x8d\x1c\xc7\x8c\xeb\xbf\x47\xe9\x84\xe9\x8c\x5d\x74\x19\xa7\xff\xe6\xdf\x9c\xfe\xeb\x7f\x51\x34\x76\xb2\x36\xb0\x4d\x9b\x3f\xe7\xc8\xb6\x75\x60\x18\x21\xdd\x57\x06\x42\x8a\xf2\xb7\xb7\x0c\x49\x40\xc4\xd2\x57\xa2\xdc\xf2\xa1\x18\xb1\x58\x9a\xba\xa1\xca\x23\x58\x29\xac\xfe\xbe\x10\x3b\x09\x0c\xfc\x8e\x6e\x2b\xc9\x2f\xab\x60\xc1\xad\x3f\x27\xaf\xbc\x2a\xed\x39\x87\xb7\xae\x67\xe7\x27\x1f\x20\x31\xfc\x09\x99\x72\x41\x4a\xd9\xce\x39\xe0\x81\x62\x07\x40\x79\x94\x98\xb2\x3d\xff\x70\xd4\x29\x7d\x40\x7d\xa0\x95\xc9\x92\xb2\x25\x31\xcf\xe2\xa4\xb2\x39\x4f\x27\x53\x05\x09\xcf\xf6\x6a\xe2\xac\xab\xbe\x91\xdd\x13\xaa\x0c\xd7\xa4\x19\x5a\x8f\xbd\x9b\xac\x7f\xe9\x51\x3e\xbc\xff\x2e\x06\xba\x3b\x39\xe6\xea\x5b\x98\xf3\x8d\x9f\x62\x98\x26\xd2\x34\x98\x72\xc9\xd7\xa8\x39\xeb\x32\x62\x59\x49\xcc\x78\x06\x00\xe3\xcf\xbd\x8a\x15\x5b\xd7\xf9\x14\x36\xd0\xd5\xc1\xbe\xe5\x6f\x52\x72\xcc\xf1\x9a\x74\x99\xb6\xc4\x08\x43\x71\x35\x3a\x47\x66\x61\x09\x46\x2c\x8e\x9d\x1a\xf0\x3f\x8b\x67\xe5\xf8\x2a\x9a\xca\xad\xfc\xfe\x1a\x06\x7d\x6d\x2d\x74\x35\xd5\x6b\xf7\x4a\x24\xb3\x49\xe6\xe5\x3b\x56\x2c\x85\x43\xed\xfd\x74\x29\xed\x75\x07\x89\x3a\x84\x61\xba\xce\x10\x19\xdd\xf6\x41\xf4\xf8\xb4\x89\xa5\x18\x86\x20\x90\x91\x1c\xb3\xa9\x74\x4d\xba\xc2\xb7\x34\x9a\x02\x0c\x4b\x86\x7d\xa9\xba\xd7\xc8\x53\xb6\x02\x33\x58\x88\x92\xb5\x75\x38\xfd\x8c\xc5\x62\x58\x3d\x5d\x6c\x7d\xed\x49\xfa\x3b\xdb\x91\xb6\xcd\xc6\x67\x1f\x62\xf7\xfb\xaf\x12\x33\x4d\xff\x1e\x59\xb9\xf9\xc4\x63\x31\x84\xb4\x30\x90\x54\x2f\x38\x9f\x92\x10\x15\x1f\xfc\xe8\x1d\x7a\x9a\xea\x10\xc2\x08\x7a\xad\x81\x2b\xe8\x3a\xbc\x97\x8d\x0f\xdd\xcd\xda\x07\xee\xa4\xe3\xc0\x2e\x00\xb2\xcb\x2a\x89\x67\xe7\xe8\xe3\x65\xa7\xf0\xa6\x60\xda\x52\xe4\xae\xad\x47\x76\x6d\x4e\x03\x38\x7f\xe8\x08\x72\x4b\x86\x04\xd1\x27\x86\x41\xc7\xe1\xfd\x7c\xfe\xec\xc3\xd8\x56\x2a\x0d\x17\x33\x9e\x20\xbf\x72\xb8\x62\xdd\x33\xb0\x85\x89\x6d\x98\xa4\xa4\x20\x65\xe3\x50\xb0\x47\xcd\xb6\x54\xd8\x6c\xe8\x74\x7f\xa3\x51\x6c\x88\xcd\xab\xec\xdb\x67\xd1\x51\xa1\x39\xe1\xf5\x58\x65\x5f\xaa\xe7\xc2\x24\x42\xf8\x12\x82\x98\x69\x60\x22\x69\xd8\xbc\x86\x8f\xfe\xfa\x4b\x9a\x15\x21\x6a\xa0\xa7\x8b\xcf\x1e\xbc\x87\xd6\xdd\x5b\x5d\x90\xdd\x59\x87\xc0\xc4\x61\xdd\x79\x43\x2a\x19\x7f\xee\x55\xda\xf3\xdb\xf7\xed\xa0\x61\xed\x47\x18\xa6\xe1\xdb\x5c\xd5\x19\x0d\x82\x5d\x2f\x3e\xcc\xa6\x87\xef\x65\xcb\xe3\x7f\x64\xdb\xb3\x0f\x62\x5b\x36\xc9\xd2\x4a\xb2\x4b\x2b\xb5\x7b\xf5\x34\x37\x81\xbb\xf6\xab\xe0\xfa\xd2\xb4\x95\x62\xf7\x87\x4b\xb0\x42\xea\xda\xc8\x63\x4f\x24\x99\x9b\xe7\x1b\x83\x0c\x01\x1b\x5e\x7b\x8a\xfa\xed\x1b\x23\xa9\x37\x23\x3b\x87\xdc\x21\x43\xb1\x6c\x89\x2d\x0c\x1a\x6a\xb7\xf1\xfe\x5f\x7e\xc1\xda\x57\x9e\xa0\xa7\xab\x93\x14\x1e\xc8\xce\x99\x1a\xe4\xd4\xd6\x6f\x1b\x0d\x74\x6d\x2d\x56\xd7\x73\x29\x31\x82\x45\x3c\x1a\xdc\xb0\xdb\x4d\x35\xff\x05\xeb\xb2\x50\xec\xb2\x10\x33\x4d\x3a\x1b\x0e\xf1\xde\x9f\xee\xe2\x99\x6f\x5c\xc4\x67\x8f\x3f\x40\x5f\x48\x8a\x6d\xde\xbd\x8d\x95\x0f\xde\x83\xd5\xdd\x81\x69\x38\x82\x9a\x69\x04\x82\x9b\x09\x8c\x3b\xfd\x02\x8a\xc7\x4c\xf4\xaf\xb1\x53\x29\xf6\x7f\xf0\x3a\xb2\xbf\x57\x53\x13\x82\x53\x92\xea\x09\x54\x9a\x96\x5d\x9b\x48\xf5\x76\x93\x99\x5f\x44\x71\xcd\x34\xed\xf9\x47\x76\x6f\x63\xa0\xb7\x27\x98\x24\x0a\xfb\x33\x4d\x83\xd6\xfd\xbb\xd8\xb9\x7c\xb1\x76\x4d\xb2\xb0\x84\xf1\x0b\xce\xf1\x6d\xbd\xa6\x61\xd0\xb2\x77\x27\x6b\x5e\xfa\x17\x83\x1d\x59\x05\xc5\x64\x15\x96\x60\x49\x49\x57\x5b\x2b\x8b\x7f\xfd\x7d\x96\xff\xf5\x1e\xfe\xf3\x93\x9b\x58\xf2\x9b\x1f\xd0\xd1\x72\x84\x94\x30\x18\xb0\xf1\x29\x58\x3d\x1d\xaa\x4d\x07\x3b\xbc\x16\x7b\x12\xb4\x13\xe7\xa5\x78\x08\x7d\xa7\x40\x08\x5c\xcd\x1b\x14\x45\xc9\x9e\xc7\x42\x84\x04\x30\xd3\xe4\xc0\xda\x95\x3c\xfd\xbf\x57\xb3\xe2\xe1\x3f\xd0\x19\x62\x71\xea\xb1\x73\xc9\xcb\x6c\x79\xed\xc9\xc0\x28\x4f\x10\x9d\x20\xb0\xc9\xaf\x18\xc6\xc4\x73\xaf\xd4\xae\xa9\x5b\xf3\x21\x1d\xfb\x76\x6a\x54\xec\x4f\x38\xc3\x24\xb7\xaa\xda\xff\x6d\x7f\x5b\x33\x56\x6f\x37\x46\x3c\x41\xd5\x71\x0b\x10\x8a\x94\xdb\xb8\x6d\x3d\x4d\x3b\x36\x61\x98\x66\xc8\x1b\x23\xc0\xb6\x58\xf3\xfc\xa3\xb4\x1e\xdc\xa3\x3d\x7b\xfc\x82\xb3\xa9\x9c\x30\x15\x43\x5a\xee\xda\x67\xb3\xe6\xc5\xc7\x68\xde\x57\xeb\xff\x26\x91\xcc\x26\x91\xcc\xf6\xff\xce\x29\x2d\x27\x91\x93\x8f\x04\xba\xdb\x9a\x69\xdc\xb9\xd9\x99\xac\x56\x8a\x0d\x2f\x3d\xc6\x07\xbf\xbf\x9d\xce\x96\x23\x58\x42\x1c\x85\x6a\x3d\x4a\x95\x1a\x98\xaa\x9a\x68\x29\xec\x59\x95\xba\x0d\x35\x12\x42\x8d\x50\x54\x75\x2e\x48\xd7\x95\xa3\xa8\x38\x66\x9a\xec\xfd\xfc\x63\x9e\xfa\xc1\xd7\xd8\xfb\xf9\x27\xda\xe0\x24\x92\xd9\x64\xe6\xe6\x6b\x82\x8d\x6d\x59\xec\x7c\xf7\x55\xec\xbe\x6e\xcd\x03\xe2\xe9\xcf\x02\x49\xcd\xc2\x8b\x29\x1e\x3d\xde\xbf\xa6\xa7\xa9\x8e\x83\x1f\xbf\xe3\x73\x8d\x20\x54\x45\x20\x84\x20\x7f\x54\x0d\x46\xc2\x11\xd4\xfa\x3b\xdb\x5d\x8a\x96\x54\xce\x3a\x91\xc2\x51\xca\x7d\x5a\x9b\x59\xfd\xef\xbf\xd0\xdb\xde\x8a\x19\x8b\x21\x0c\x03\xc3\x34\x31\x0d\xc1\xe6\xb7\x5e\xe4\xf3\xe7\x1f\xd1\xda\x9f\x5f\x5e\xc5\x9c\x2b\xff\x87\x78\x22\xe1\x38\xd2\x4d\x93\x86\xed\x1b\x59\xf7\xda\xd3\xc1\x98\x18\x06\x93\x17\x5e\xe8\xa8\x5f\xde\x75\x15\xc3\x30\x33\x32\xb1\x2c\x9b\xec\xb2\xa1\x8c\x5f\x74\x29\xc2\x08\x9c\x86\x5b\x5f\x7b\x92\x15\xbf\xbf\x9d\xae\x96\x66\x2c\x61\xa4\xe9\xf4\x83\x9f\x9e\xba\x14\x02\x1a\x3d\x22\xd3\xd0\x6e\x86\x27\x51\x07\xe1\xa8\x9e\x4a\x12\x15\x02\x13\x50\xb3\xab\xf0\x37\x1c\xe2\x95\xdf\xdc\x41\xe3\x9e\x9d\x01\x8b\xca\x2b\xe0\x84\xaf\x7c\x8b\xab\xff\xf6\x22\x5f\x79\xf8\x75\xce\xfe\xe9\x7d\x0c\x9d\x32\x13\x21\x0c\x32\x73\xf3\x19\x7f\xda\x79\x24\x32\xb2\xf4\xa8\x04\x17\x38\x61\xdb\xe4\x55\x0e\x63\x42\x88\x8a\xf7\x2f\x7f\x93\x54\x47\xab\x2b\xe0\x09\xcc\x98\x89\x69\x9a\xc8\xbe\x1e\xc7\xfe\x9a\xc8\x04\x20\xd5\xdd\x49\xaa\xb3\x1d\x21\x21\xa7\xac\x92\xf1\xe7\x5d\x15\xf0\x62\x60\xeb\xe2\x97\x79\xfb\x17\xb7\x38\x12\xf0\xe1\xfd\x1c\xa9\xdd\xca\x8a\x87\x7f\xcf\x9b\xf7\x7c\x8f\xde\x8e\x36\xff\x77\x46\x2c\xc6\x89\x5f\xfd\x0e\x55\x13\xa6\x61\xd8\x36\xa6\x21\xc0\x4a\xb1\xea\xd9\x87\x69\xaf\x3f\xe4\xff\x6e\xf4\xec\x93\xa8\x99\x7f\x26\x3d\x6d\x2d\xfe\x67\x05\xc3\x46\x23\xcc\x18\xb6\x94\x88\x58\x9c\xd9\x37\xfe\x90\x29\x5f\xba\x5e\xeb\xcf\xb6\xd7\x9f\xe4\xe3\x3f\xfe\x88\xee\xd6\x66\x24\x3a\xc8\xaa\x0a\x98\x4e\x84\xf8\xd2\x79\xfa\xa4\x70\x3e\x8f\x0d\x16\x7b\x1b\x0e\x25\x51\x71\xd5\x30\x76\xc5\x74\x81\x64\xf9\x93\x0f\xb3\x6b\xf5\xc7\xfe\x57\x79\x65\x15\x5c\xf0\xe3\xdf\x31\xf1\xb4\xf3\xc1\x8c\x61\x49\xc9\xb0\xa9\xb3\x18\x7f\xca\xd9\xec\xf9\x6c\x39\xd9\xc5\x43\xa8\x9a\x71\x3c\xc2\x50\xed\xaf\x8e\x29\x14\x57\x05\xb0\x81\x9a\x85\x17\xb1\xe9\x95\xc7\x69\xd9\xb3\x03\x80\x23\x5b\xd7\xd1\xbc\x75\x1d\xe5\xc7\xce\xa7\xb7\xa9\x9e\x96\xdd\x5b\x69\x58\xbf\x92\xc6\x75\x9f\xd0\xb2\x73\x23\x03\x5d\xce\x7a\x9f\xea\xeb\xa1\xbf\xbd\xc5\xd7\xb9\x27\x9c\x73\x25\xfb\x3f\x5a\xc2\xfe\x4f\xde\x73\xfa\x61\x5b\x6c\x7e\xf3\x79\x76\x2e\x7b\x8b\x64\x61\x09\x56\x5f\x2f\x9d\x4d\xf5\x69\xb6\xf2\xd9\x97\x5c\xc7\xec\x4b\xae\xc5\x74\x4d\xb3\xa6\x69\xb2\x7b\xed\x4a\x36\xbc\xf5\x92\xff\x9b\x64\x7e\x21\xa7\xde\xf4\x43\x8e\x1c\xd8\xe3\xfb\x99\x85\x10\x14\x0c\x1d\xe1\xdb\xa0\x6d\x04\x56\x2a\xa5\x99\x3e\xbd\x63\xfb\xeb\x4f\x80\x69\x32\xe7\xd6\x7b\x88\x27\x73\x34\x93\xa6\xa7\x6c\xaa\xf6\x6b\xdd\x77\xe0\x2d\xa7\xc2\x0f\x3f\xf6\xbd\x49\x51\x31\xc5\x2a\xc0\x69\x80\x2a\xa0\xfb\x33\xdc\x30\xa8\xab\xdd\xc1\x47\x2f\x3d\xe1\x7f\x16\x4b\x64\x70\xd6\xb7\x7f\xcc\xf4\x45\x17\x91\xb2\x2c\x6c\x3b\xe5\xb8\x15\x81\x82\xb2\x0a\x8e\x39\xf7\x72\x67\x19\xb0\x2c\xe7\x99\x5e\xa3\x04\xa0\x80\x2c\xa4\x4d\xfe\xd0\x91\x4c\x38\xfb\x72\x3e\x7a\xe0\x17\x0e\x70\x3d\x5d\x6c\x78\xfc\x3e\x6a\x17\x3f\x4f\xc3\xc6\xcf\xe8\x38\xb8\x87\x54\x4f\xba\xf3\xde\xee\xef\xa7\xbf\xf5\x88\xcb\x0d\x24\xc9\xc2\x12\x4e\xfc\xce\x2f\x58\xfc\x93\x7a\x9a\x14\xdb\x74\x7f\x57\x27\xfd\x5d\xe9\x83\x2e\x0c\x83\xd9\x5f\xba\x96\x33\xbf\x73\x27\x99\x59\x49\x90\x36\xc2\x10\xa4\xfa\x7b\xf9\xe4\xa9\xbf\xd3\xdd\x7a\xc4\xff\xed\x9c\xcb\xae\xa7\x7a\xf6\x89\x6c\x79\xff\x8d\x60\x0c\x32\xb3\xc8\x2b\xaf\x72\x06\x5f\x08\xfa\x3a\x3b\xf8\xf0\xfe\xbb\xd8\xfa\x9f\xa7\x89\x3a\x76\xbc\xf6\x04\x25\x13\xa6\x33\xfe\xe2\xaf\x39\x96\x37\x55\x4f\x77\x31\xf1\x54\x56\x55\x85\xf5\x3d\x78\xc2\x19\x40\x89\xeb\x90\x70\xd6\xe0\x40\xc8\x1a\xd4\x2c\xe6\xb3\x6d\xd7\x1c\x28\x65\x10\x39\x21\x41\x08\x83\x4d\xcb\x97\xd0\xb4\x7f\x8f\xdf\xa0\x9a\xb9\x27\x73\xec\xb9\x97\x62\x48\xa9\x45\x36\xc4\x84\x13\xc2\x22\x6c\x0b\xc3\xb6\x83\x75\x14\x45\x42\x17\x8a\xab\xcf\x9d\x40\x35\x67\x5e\x42\x5e\xe5\x08\xff\xfe\x07\x3e\x5e\xc2\xb6\x97\x1f\xa5\x65\xe7\xa6\x48\x70\x01\xe2\xd9\x39\x64\xe4\xe4\xfa\xcf\xc0\xb6\x19\x32\x7e\x1a\x67\xfe\xe2\x41\x46\x1c\x77\x8a\xb6\x16\x86\x8f\xbc\xb2\x0a\x16\xdd\xfa\x33\xce\xfd\xe1\x3d\xe4\x14\x14\x61\xe0\xb0\x66\x03\xc9\xee\x4f\x97\xb1\xf9\xbd\x00\xc8\x11\xc7\xcc\xe6\xc4\x6b\x6e\xc2\x4e\x0d\xd0\xb8\x3b\x50\x07\x33\x72\xf2\xc9\x2e\x2d\x47\x02\xfd\x3d\x3d\x7c\xfc\xd7\x5f\xb1\xf1\x85\x7f\x6a\xeb\x9c\x2a\x93\x48\xdb\xa2\x71\xd3\xea\xc0\xac\x1a\x69\xf1\xd0\x09\x2d\x6c\xec\xb1\xbd\x40\x0b\x97\x98\x62\x69\x5b\x24\xa4\xf2\xa3\x41\x6e\x1c\x36\xed\xa5\xfa\xfb\xd8\xba\x72\x85\xf6\x9b\xa9\xa7\x9c\x49\x76\x6e\x1e\xa9\x81\x94\x6f\x48\xf0\x26\x92\xc0\xb3\xbe\x38\x77\x10\xee\xe7\x48\xc7\x22\x26\x84\x81\x65\xd9\xbe\xb3\xbc\xa3\x6e\x3f\x9b\x5e\x7a\x8c\xbe\xce\x36\xbe\xe8\x48\xe4\xe6\x53\x3c\x66\x12\x43\x67\x9f\xcc\xd0\x99\xf3\x18\x32\x79\x96\x33\x99\x5c\x3b\x3a\xd2\xa6\x7c\xc2\x31\x9c\xf7\xeb\x47\xd9\xf9\xfe\xeb\xec\x5a\xf6\x26\xcd\x7b\x76\xd2\xdb\xde\x8c\x10\x06\xb9\xa5\x15\x54\xcf\x39\x89\x19\xe7\x5e\xc6\xd0\x09\x53\x7d\xf6\x8e\xb4\xf9\xf4\xf9\xc7\xd9\xf6\xe1\x7b\x1c\xd9\x57\x4b\x5f\x57\x07\x00\x99\xb9\x79\x2c\xbc\xe9\x07\x14\x94\x57\xd1\x5a\x7f\x88\x96\x83\x7b\xfd\xb6\x64\x17\x97\x92\x55\x50\xc2\x40\x5f\x2f\x2b\x1f\xfe\x1d\x6b\x9f\xfa\x9b\x66\x77\x8f\x65\x65\x33\xe5\xcb\x37\xd3\xdb\x7a\x84\x5d\x6f\x3d\x4b\x56\x51\x19\xa3\x17\x7e\x09\x61\x18\x4e\x7c\x58\x28\xbc\x47\x8d\xc7\x0a\xe3\xe0\x23\x2e\x5c\x7f\xb0\x74\xa3\x2a\x55\xf3\xa4\x26\x5c\xc9\x68\xc9\xd9\x7b\xf5\x1f\x26\x04\x3d\x5d\x9d\x34\xec\xdb\xed\xff\xce\x8c\xc5\x28\xaa\xa8\xf2\x7d\xc8\xfe\x35\xa8\x51\x0e\x81\x24\x6e\x7b\x41\xbf\x42\xd0\xde\x58\x4f\xcb\xe1\xfd\x94\x54\x4f\xc4\xcc\x4c\x22\x53\x03\x7c\xf4\xd7\x5f\xb1\xfe\x28\xba\x66\x46\x5e\x21\x25\xe3\x26\x53\x35\xe7\x14\xaa\x8e\x9d\x4f\xe1\xe8\xf1\x64\xe4\xe6\x3b\x81\xe2\xae\xb7\x48\x35\x64\x20\x6d\x92\x05\x45\x4c\xff\xd2\x75\x4c\x3b\xef\x0a\xfa\xdb\x5b\xe9\xef\x68\xc1\x34\x0c\x72\x8b\x4a\xc8\x29\x28\xc2\x34\x0c\x90\x16\x42\x4a\x4c\xd3\x64\xdf\xfa\x35\xfc\xe7\x0f\x77\xd1\xa9\x44\x7a\x00\x1c\x7f\xe9\x57\x99\x70\xd2\x42\x90\x36\xed\x8d\x75\x74\x28\x6a\x61\x5e\x79\x15\x66\x46\x06\x9f\x3e\xf6\x27\x56\x3d\x7a\x9f\x66\xe9\xca\xc8\x2f\x64\xd6\x37\xee\x64\xec\xf9\x5f\xc1\x1e\xe8\x67\xcc\xb9\x57\x93\x91\x57\x40\x6e\xc5\x08\x67\xa4\xdc\x20\x08\xa4\x44\xb8\x8b\xae\x3a\xfe\x1e\xe0\xc1\xd8\x46\x1f\x31\x5b\x63\xc7\xe9\x9b\xb6\x3c\x81\xc3\x03\x33\x6c\x5c\x30\x04\xf4\xf7\xf7\xd1\xa7\x08\x0e\x56\x2a\x45\xc7\x91\x06\x5f\xf5\xf1\xa8\x35\xb8\x4e\xdf\x86\xe1\x1c\x06\x03\x3d\xdd\x2c\xfe\xcd\x0f\xd9\xf6\xfe\x1b\x4c\xbb\xf0\x6a\x4e\xf9\xce\x2f\x30\x12\x19\x91\xac\x34\xab\xa0\x88\xd2\x9a\x69\x0c\x3b\xee\x14\xaa\x66\x9d\x44\xe1\xa8\x1a\x12\x39\xb9\xce\x13\x6c\xcb\x59\x2f\xa5\x3e\x19\xb5\x89\x89\x44\x5a\x16\xb1\x58\x9c\x8c\x92\x32\xcc\xb2\x21\x04\xf1\x69\x12\x6c\xcb\x57\xc1\x4c\xc3\xe0\xe0\xa6\x35\x69\xe0\x0a\xc3\x00\x01\xfd\xbd\xdd\x64\xe4\x16\xd0\xbc\x7f\x0f\xbd\x1d\x81\x41\x27\x77\xc8\x50\xd6\xbd\xf0\x28\x1f\x3f\xf4\x1b\xcd\x22\x96\x5d\x56\xc9\x9c\x5b\xef\x61\xe4\xa9\x17\xb8\x41\x06\x99\x94\x4e\x98\xee\xb4\x49\x4a\x1a\xd6\x7f\xc2\xce\x37\x9e\xa6\x68\xf4\x78\x6a\xce\xbb\x9a\x78\x56\xd2\x8d\xd8\x8c\xb0\x47\x87\xc0\x46\xf9\x5b\x80\x23\x45\x87\x75\xae\xa8\x88\x3e\x87\xad\x4a\x65\xb0\x82\xad\x24\x86\x19\x23\x16\x4f\x68\x0f\xd8\xb4\xec\x1d\x4e\xb8\xe8\x2a\x8c\x78\x06\x02\xdb\xa1\x54\xbd\x09\x81\x38\xe8\x0e\x56\xe3\x8e\xcd\xec\x58\xbe\x98\xfe\x9e\x2e\xd6\xbd\xfc\x6f\x26\x9c\x71\x31\xc3\x8f\x3d\x89\x19\x97\xdd\xc8\xc1\x75\x2b\xe9\x6c\xac\xa3\x7c\xd2\x0c\x46\xce\x3d\x95\xaa\x59\x27\x52\x38\x72\x1c\xf1\xa4\x63\x63\x96\xb6\xc3\x46\xc3\x9d\x0d\x0f\x4a\x30\x30\x41\x20\x3d\x52\xfa\x8e\x12\xa1\xcc\x5e\xff\xf7\xd2\xa6\x7a\xc6\x5c\xaa\x67\x1d\xcf\xae\x55\x1f\xf9\xf7\x96\xb6\xcd\xfb\x0f\xff\x99\xbc\xb2\x0a\x4e\xbe\xfe\x56\x1a\x77\x6f\xd7\xa8\xf4\xe0\xfa\xcf\xd8\xfc\xe6\x73\xa4\x7a\x7b\xfc\xcf\xf2\x47\x8c\xe5\x84\xef\xff\x8e\xca\x39\x0b\xb0\x6d\xdb\x1f\x64\x69\x5b\x08\xc3\xa0\xf3\xe0\x1e\x56\xfc\xf2\x66\x5a\x77\x6f\x45\x98\x26\x76\x7f\x1f\xc7\x5c\x7d\x8b\xd6\x9e\x70\xff\x50\xfa\xa4\x82\x2b\x04\x8e\x37\xc9\x63\xcd\x96\xca\xa6\x65\x00\x88\x44\xb9\xb9\xf4\x3c\x4b\xae\xc7\xc8\x86\x44\x32\x87\x82\xf2\xa1\xec\xdf\x1a\xd8\x63\x37\x2e\x7b\x87\x35\xef\xbc\xc6\x9c\x73\x2f\xc5\xb2\x3c\x2e\x20\xfd\xeb\x7d\x2b\xb2\x22\xf3\xef\x59\xbd\xc2\xd7\x3f\x53\x7d\xbd\xf4\xb6\x1e\x41\x48\x9b\xf2\xf1\x53\xb9\xf8\x8f\x4f\xd1\xd7\xdd\x49\xd1\xc8\xb1\x0e\xa8\x12\x77\x80\xac\xb4\x9e\x86\xd9\x95\x08\xbd\x11\xee\x7a\xac\x3a\x50\xd2\xcd\x9e\x81\xb5\x0e\xdb\x66\xe8\xf8\xc9\xdc\xf0\x97\x27\xf9\xf0\x99\x7f\xb2\xfc\xc9\x7f\xd0\xea\xea\xbf\xd2\x9d\x54\x96\x95\xa2\xa1\x76\x9b\xf6\xdc\xfa\x2d\x6b\xb5\xbf\x4b\x27\x4c\x67\xde\x0f\xff\x40\xe9\xe4\x63\xb1\x42\xe1\x43\x1e\x4b\xee\x6f\x6b\xa6\xbb\xd1\xbd\xb7\x65\xb1\xe5\xc5\x7f\x32\xe6\xd4\xf3\x29\xa8\x1a\x89\x94\x76\x10\xef\x46\x3a\x35\x47\x4d\x68\xc3\xa3\x56\x95\x4d\xeb\xce\x68\xd5\x6f\x19\x65\x0b\x95\x98\x19\x99\x54\x4f\x9f\xa3\x35\xb8\xaf\xbb\x8b\x17\x7e\xf3\x53\xb6\x7e\xfc\x41\xe0\x50\x08\x6d\xaf\x08\x62\x78\x05\x7d\x1d\xad\x6c\x5f\x16\xd8\x7e\xe3\x99\x59\xe4\x94\x94\x01\x12\x03\x49\xc9\xc8\xb1\x54\x4e\x9a\x41\x46\x56\xd2\x91\xbe\xa5\xed\xfa\xa3\x85\x7f\xc6\x0d\xe7\x8c\x85\x4e\xd3\x3d\x0d\x84\x2f\x6c\xa9\x40\xaa\xa3\x13\x66\x7d\x3e\xb5\x4a\x9b\xfc\x92\x21\x9c\xf5\xcd\x1f\xf2\xf5\x07\x9f\x65\xe6\xd9\x5f\x22\x99\x5f\xc8\xf4\xb3\x2e\x66\xd6\x79\x57\xd0\xdb\xd9\x41\xd3\xde\x5a\x06\x3b\xaa\x8e\x9d\xcf\x82\x5f\xfc\x9d\xb2\x29\xc7\x62\x2b\xe0\x0a\xcf\x0a\x87\xa3\x49\x14\x8c\x18\x43\xd1\x98\x49\xfe\xf7\x6d\xfb\x6b\xd9\xb3\xf4\x75\x0c\xc3\x70\x9d\x39\xae\xf7\xce\x88\xf6\xde\x79\xf1\xd7\xfe\xfb\xb4\x70\x0f\xdf\x06\xaa\x83\x3a\xe0\x81\xed\xb9\xad\x42\x92\xf7\xd4\x53\xcf\x22\xbf\x74\x88\xd6\xa9\xc6\xfd\xbb\x79\xf8\x7b\x37\xb2\xf2\xb5\x67\xc1\xb6\x1c\x0f\x93\xeb\x48\x50\x41\x8e\x99\x26\x9b\xdf\x7d\x9d\xbd\x6b\x03\xf3\x66\xf1\xc8\xb1\x94\x8c\x18\x83\xf0\x66\xad\xb4\x31\x6c\x4b\xf3\x5e\xc5\x8c\xe0\x8c\x7b\x51\x25\xa1\x53\xed\xb4\xf7\xcc\xf0\x7a\xac\xaf\xcd\x21\x61\x52\x39\xa5\x74\x58\xea\xc8\xa9\xb3\xb8\xfa\xde\xbf\xf1\x9d\x27\xde\xe6\xb2\x9f\xdf\x4f\x4e\x49\x19\x1d\x47\x1a\x69\x1b\xc4\x1f\x5c\x7d\xea\xf9\x2c\xb8\xeb\xaf\x14\x8e\x9e\xa0\x83\xab\x9c\xa6\x69\xd2\x55\x7f\x80\xb5\xff\xfc\x1d\xcd\xbb\x36\x6b\xd7\xef\x78\xeb\x39\xfa\x9a\x1b\x9c\xc9\x1a\x02\x33\x16\xf1\xde\x8f\xb0\x11\x22\x64\xaa\x94\x6a\xd8\x87\xf2\xb7\xad\xb8\xa1\x54\xbf\xa3\xf7\x1b\xcb\xa2\xb2\x66\x32\xc7\x5f\x7c\x75\x5a\xe7\x9a\x0e\xec\xe5\x91\x1f\x7e\x83\x67\xee\xfe\x21\x0d\x7b\x76\x61\x1a\x06\xb1\x58\x9c\x98\x69\x12\x73\x4d\x8c\xdb\x3f\x7a\x8f\xb7\xff\xfc\x4b\x52\x7d\x41\xf4\xc4\x94\x33\x2f\x22\xb7\xa8\x58\x73\xcb\xa9\x71\x61\x31\x23\xe8\x54\x5c\x1c\x1d\x5c\x6d\x76\x8b\x10\xb8\x0a\xdb\xf6\xd9\x5e\x18\xec\x10\x2b\x94\x96\x45\x22\x2b\xc9\xd0\x09\x53\xc9\xca\x2b\x40\x4a\x68\x3d\x7c\x40\x33\x7c\x80\x23\x57\x4c\xba\xf0\x2b\x9c\x7c\xc7\x7d\x64\x57\x0c\xd3\xc0\xf5\x0e\xc3\x30\xc0\xb6\xd8\xbb\xf4\x75\x16\x7f\xef\x2a\xd6\x3d\xfe\x27\xfa\x15\x33\x29\x38\x8e\x91\xfd\x9f\xbc\x87\xe9\xf9\xcf\x85\xe2\x75\x53\x38\x98\xcf\xb5\x54\x4a\x4f\xa7\xde\xc0\x80\xed\x99\x2e\xfd\x06\xcb\xc0\x35\xe8\xa8\x50\xd2\xef\xb6\x61\x9a\x9c\xfa\xd5\x9b\xd9\xbf\x75\x03\x1b\x3f\x78\x5b\x6b\x60\x6f\x57\x07\x8b\xff\xf9\x17\xd6\xbd\xf7\x16\xb3\x16\x5d\xc0\xf8\xb9\x27\x93\x5f\x56\x41\x4f\x77\x17\x9b\x3f\x7c\x9f\xa5\x4f\xfc\x9d\x56\x65\xf6\x0f\x9f\x76\x2c\x33\xce\xbb\x3c\x78\x4e\xa4\xd0\x24\x34\xea\x0b\x1f\xaa\x89\xd5\x8b\xf2\xf4\x0d\x03\x22\x1d\x38\x5d\x00\x4b\xbf\x69\x3a\xcb\x96\xd8\xb6\xbb\x5f\xc9\x80\xc6\x3d\x3b\xe9\xef\xe9\xf6\xbf\x37\xe3\x09\xa6\x5f\xf5\x4d\x66\x7d\xed\xfb\x98\xc9\x1c\x52\x96\x9b\x6b\xc0\xb7\x2f\x08\x0c\xd3\xa0\xf3\xd0\x5e\x36\x3c\xf1\x17\xb6\xbd\xf6\x6f\x06\x5c\xdd\x3a\x7c\xd8\xa9\x14\x5b\xfe\xf3\x14\x35\x0b\xce\x26\x23\x99\xad\xc9\x31\x83\x49\xd6\xfe\x67\x7f\xdf\xd9\x25\xbd\xbd\x46\x03\x4a\x5c\x90\xaa\x13\xab\x9d\x0c\x28\x49\x28\xd4\x22\x30\x0d\x88\x9b\x06\x47\xf6\xed\xe6\xe9\x9f\xdd\xc6\xfa\xf7\xdf\x64\xb0\x23\x9e\x99\x49\x66\x32\x87\xd4\x40\x3f\x3d\x1d\xba\x9f\xb8\xa0\x7c\x28\x97\xff\xfa\x21\x46\x1f\x77\x0a\x96\x65\x69\xd2\xbc\x07\x8a\x2e\x08\x45\xeb\x80\xaa\x95\xc7\x0f\xd4\x8b\x70\x7f\x06\x7b\xa2\x02\x9f\xb6\xd7\x3f\xcd\xf1\x11\x02\x59\xcd\x0b\x62\x09\x83\xed\x9f\x2c\xe3\xc9\xdb\xbe\x42\x47\x63\x1d\x89\x64\x0e\xc7\xdd\xf8\x03\xa6\x5d\x79\x13\xc4\x32\x48\xd9\x36\x96\xea\xac\x31\x0c\xac\x81\x7e\xf6\x2f\x7b\x83\x35\xff\xfc\x9d\x13\x6b\x16\x71\x24\x8b\xcb\xe8\xef\xea\x24\xd5\xdb\x4d\x3c\x2b\x9b\x8b\xff\xf8\x24\x63\x4e\x3c\x1d\x2c\x2b\x8d\x13\xa5\x8f\x8f\x43\x00\xe6\x59\xb7\xdc\x71\x97\x15\x62\xcb\x9a\x75\x8b\x50\x10\x80\x61\x38\x21\x33\x52\x62\x18\xa6\xb3\x23\xc1\x33\x5c\x48\x49\x6e\x61\x31\x93\x4f\x3c\x95\x44\x46\x26\x75\xbb\x77\x6a\xfa\xb1\x3a\x23\xfb\x7b\xba\x49\x85\x02\xda\xf2\xcb\xca\xb9\xe4\xce\xfb\x98\x30\xff\x0c\x47\x8f\x55\x26\x95\x10\x3a\x8b\xf6\x12\xb5\xf8\xac\xd7\x08\x40\xf2\x25\x63\x15\x18\x8f\xea\x95\x01\xf1\xc1\x45\xc9\x73\xe1\xb1\x71\x05\x60\xf5\x3b\xdf\xe0\x4f\x30\x79\x2c\x69\x93\x57\x39\x82\xdc\x8a\x61\x34\x1f\xd8\xc3\xdc\x1b\xbe\xcf\xb4\xcb\x6e\x00\x33\xee\xc4\x61\x09\x37\x73\x01\x02\x0c\x93\xf6\x03\xbb\xf9\xfc\x6f\xbf\xe2\xf3\x87\xee\xa5\xab\x6e\x7f\xda\xf8\x64\x15\x96\x30\xf9\xc2\x6b\x98\x7f\xeb\xaf\xe8\x6e\x6e\xe0\x48\xed\x56\xec\xd4\x00\x02\xa8\x39\xf9\x4c\x62\xa6\xa1\x08\xa9\x81\xa0\xea\x79\xd7\x82\x80\x04\x10\x7f\xd9\xd1\x25\x53\xda\xae\x41\xa9\x49\xca\x5e\x94\xa2\x67\xe4\x68\xda\xb1\x91\x5d\x4b\xdf\xa0\xfd\xd0\x3e\xf2\xcb\x2a\x18\x77\xfc\x29\x54\x4f\x9f\x4d\x66\x46\x26\x26\xb6\xbb\x3e\x1a\x18\x48\xf6\x6d\x5c\xc3\xb2\x67\x1e\x61\xed\x7b\x6f\xd2\x7c\x38\x5a\x00\x71\x6e\x2d\x18\x39\x75\x26\xe7\xde\xfa\x53\x6a\xe6\x9d\xe6\xc7\x14\x85\x29\x4e\x07\x46\x68\xe0\xf9\xeb\xa3\x42\xb9\xea\x36\x90\x74\x0b\x9d\xd4\xd6\x5d\x2d\x68\x81\xd0\x86\x6a\x74\xb6\xa7\x86\xed\x7a\x9a\xc6\x80\x84\x01\xcb\xa6\xbd\xb5\x85\x58\x6e\x21\x29\x20\x25\x05\xfd\xfd\x7d\xf4\x74\xb4\x13\xcf\x2b\xc2\x4a\xa5\xd8\xfd\xfe\xab\xac\x7b\xe4\xf7\x34\xef\xdc\x14\x39\x16\xd5\x0b\xce\x63\xe6\xb5\xb7\x52\x31\x69\x06\x89\x8c\x0c\xb6\x2f\x7e\x81\xd7\x7f\x70\x1d\xd6\x40\x3f\xc9\xa2\x12\xbe\xfc\xd7\x97\xa8\x9a\x32\xd3\x8d\x5d\xd3\x73\x74\x80\x4e\xd5\x02\xdf\x92\x25\x89\x5c\x8b\x55\x73\xa5\x65\xb3\xf9\xf5\x27\xf9\xf0\xff\x7e\x45\x7b\xdd\x01\xbf\x41\x1f\x3f\xfe\x7f\xcc\x38\xef\x72\x16\x7d\xeb\x76\x8a\xca\xca\x9d\x10\x6d\x57\xb7\x1d\x3d\x6d\x16\xa3\x27\x4f\xa3\xfe\xab\xdf\x62\xcb\x27\xcb\xd9\xbe\xea\x23\x0e\xd7\xee\xa0\xb3\xad\x05\x69\x59\x64\x66\xe7\x30\x64\xd4\x18\x26\x9d\x78\x3a\xd3\x4e\x3d\x8b\xdc\xb2\x0a\x52\x96\xe5\x03\x66\xbb\x33\x3f\x6c\x2e\x1d\x4c\xfa\x55\x29\xcb\x73\xcf\x05\x9f\x3a\x2f\xbe\xe9\xd4\x5d\x88\x55\x2e\xe1\x07\xe9\x0b\x3d\x73\x4d\xda\x73\x04\x5a\xf2\x16\xcf\x6d\x27\x84\x20\x59\x50\x4c\xbf\xcb\xee\x3a\x0e\xef\xe3\xe3\x07\xef\xa1\x7e\xe3\xe7\x8c\x3a\xed\x02\xba\x1a\x0e\xb1\xf3\x8d\x67\x48\xf5\xf5\x30\xd8\x51\x34\x66\x02\xe5\xd3\x66\x83\x65\x61\xa5\x06\x18\x36\x73\x1e\x65\xe3\xa7\x72\x78\xc3\x2a\xba\x9b\x9b\xd8\xf8\xe6\x73\x54\x4d\x9e\xae\x71\x1d\x8f\xc3\x69\xfa\xbb\xf7\xfe\x8f\xdb\xba\xa4\xb7\x17\x78\xb0\x35\x58\x18\x06\xdb\x17\xbf\xc4\x3b\x3f\xff\x16\x7d\x21\x09\xcf\x3b\x66\x9c\x73\x09\x57\xfc\xe2\x7e\x72\x72\xf3\xd2\xa5\x57\xc3\xc4\x74\xd7\x9d\xde\xae\x4e\x7a\xbb\x3b\xb1\x6d\x9b\x78\x66\x16\x59\x39\x79\x98\x89\x4c\x52\xb6\xa5\x84\x9b\x28\xae\xcb\x30\xc0\x61\x1d\x36\x44\xbd\x28\x66\x56\x2d\x30\x1e\x7d\xa9\xd1\x8c\x37\x0a\x15\x84\xd7\x62\xdd\xcb\x25\x5c\x0a\x0e\xf6\xf3\xfa\x14\xac\xed\xa1\x76\xa8\xf9\x9d\x7b\x6e\x63\xfd\x73\xff\x18\x14\x4c\x23\x9e\xa0\x74\xe2\x0c\xda\xf7\xef\xa2\xc7\x35\x83\xe6\x0f\x1b\xcd\x79\x0f\xbc\x44\xe1\xb0\xd1\x8e\x9c\x63\x08\x3e\x7b\xe4\x0f\x7c\xf0\xc7\x9f\x00\x50\x58\x35\x8a\x6b\xff\xf1\x0a\xa5\x23\xc6\x60\x48\x5b\xdf\xd9\xef\x01\xab\x8c\x91\x79\xfa\xb7\x7e\x74\x57\x4a\x42\x8a\xc0\xc8\xe1\x85\xeb\x58\x00\xc2\xa0\x65\xdf\x2e\x96\xfc\xf2\x16\x3a\xeb\x07\x67\xb3\xf5\xbb\xb6\x51\x3a\xbc\x9a\xe1\x53\xa6\xbb\xba\xab\xc2\x42\x71\x0c\xe6\xb1\x58\x9c\x8c\x64\x92\x64\x4e\x1e\xd9\xb9\xf9\x64\x66\x25\x1d\x35\xc1\xb5\xff\x7a\x8d\x73\x06\x59\xa4\x27\xf8\x0a\xbd\x1a\x46\x10\xaa\x23\x94\xce\xa9\xd4\x2e\xa3\xf4\x9c\xc1\xc0\x25\x9d\x35\x1b\x22\x9d\x2a\x10\x81\x25\xdd\x9f\x48\xda\x84\x72\x34\x90\xda\x15\x8b\xa9\xdf\xf4\x79\xda\x58\x09\xc3\xa0\x6c\xf2\xb1\xcc\xbc\xe9\x27\xcc\xb8\xf1\x47\x0c\x74\xb6\x53\xbf\x7e\x25\x00\x7d\x1d\xad\x74\x35\x1c\x64\xdf\x47\xef\xd2\xd5\x78\x98\xb2\x9a\x29\xe4\x14\x95\xb2\xf3\x83\xff\xd0\xd7\xd9\x4e\x6f\x7b\x2b\xf9\xe5\x43\x19\x35\xf3\x78\xbc\x4c\x41\xde\xc4\x54\xfb\xe1\x8f\x51\x78\x7b\x84\x1f\xa3\xe5\x36\xd8\xb2\x6d\x36\xbc\xf8\x28\xcd\xb5\x5b\x39\xda\x61\x5b\x16\xeb\x16\xbf\x4a\x6f\x4f\x0f\xb6\xeb\x4a\x90\xae\x54\x22\x0c\x93\x54\x2a\xc5\xea\x77\x5e\xe7\xd3\x37\x5f\xa6\xa7\xbd\x05\xc3\x05\x55\xd5\xe9\x1c\x1d\xce\xd5\x6d\x0d\xef\x74\xad\x54\x8a\xbe\xe7\xe9\x7e\xfe\x1e\x28\x02\x70\x54\x2a\x0f\x4b\xdc\x3e\xb0\x61\x21\x4c\x05\x31\x2c\xa5\x87\xc1\x55\xee\x13\x9e\x2c\xea\xa7\xc2\x8c\x51\xb3\xe8\x32\x32\x42\x5b\x5f\xf2\x86\x55\x73\xdc\x77\x7f\xcb\xa9\xbf\x7b\x8a\xea\xb3\x2e\x27\x91\x5f\x44\xf5\x59\x97\x93\x55\x5c\xe6\xce\x18\xc9\xae\x77\x5f\x65\xf3\x2b\x8f\xb3\xe2\x4f\x3f\xe5\xe0\xda\x4f\x28\x1c\x39\x8e\x51\x27\x9c\xee\xdf\x63\xc3\x1b\xcf\xd3\xd1\x54\xe7\x4e\x6e\x65\x97\x7f\x94\xde\xef\x09\x33\x69\x71\x40\x12\xa4\x4b\xbd\x3b\xde\x79\x51\x6b\x64\xe1\x88\x31\x54\x4e\x9d\x4d\x66\x5e\xa1\xf6\x79\xe3\xbe\x5a\xba\x3b\x3b\xb0\xa5\xe3\x61\xea\xee\x68\xa7\xa5\xb1\x9e\xfa\xbd\xb5\x2c\x7d\xee\x5f\xfc\xf9\x5b\xd7\xf0\x87\xaf\x5f\xc1\xdf\xbe\xff\x0d\xda\x9a\xd2\x2d\x33\x0e\xb0\xc2\x07\x35\x66\x38\x46\x8c\xb8\xa1\x5a\x67\xd0\x0c\x1f\xbe\xd4\x1b\x02\x23\xea\xd0\x40\x8e\xf8\x3b\xfc\x5d\x94\x7e\x3d\xd8\xbd\xc3\x87\xb4\x2d\xca\x26\x4e\x67\xd8\xec\x53\xb4\xcf\x4b\x26\x4e\xa7\xe6\xa2\xaf\x92\x51\x58\x8a\x6d\xdb\xf4\x75\xb4\x52\xbb\xe4\x25\x52\x8a\x0e\xed\x1d\xfd\x5d\x1d\x34\xef\xdd\x85\x88\x27\x18\x7f\xc6\x45\x24\x5c\xc7\x4a\xdd\xb6\x0d\xec\x58\xfe\x0e\xc2\x08\x22\x42\xd5\xc0\x89\x20\x46\x5d\x38\x0e\x7f\x9f\xbd\x20\x35\xbd\x11\x43\xb0\xef\xe3\x77\x69\x57\x9c\xd8\x23\x4f\x38\x9d\x05\xb7\xff\x9e\xdc\xa2\x52\x1a\xb7\xac\x61\xc9\xbd\xdf\xa3\xc1\x0d\x7f\xe9\x6a\x3d\xc2\xbb\xff\xb8\x8f\xce\xa6\x7a\xba\x9a\x1b\xe9\x69\x6b\xa5\xa7\xa3\x8d\xbe\xae\x4e\x3a\xdb\x5a\x7c\x43\xc0\xaa\x77\x5e\x67\xd1\x57\xfe\x87\x92\xf2\x8a\xc0\xfb\x22\x42\x7a\xab\x70\xfc\xc4\xb6\xff\xb9\x74\xc3\x79\xa2\xb7\x5d\x7a\xd7\x7b\x9f\xab\x6f\x24\xa1\xcf\x55\xc0\x22\x59\xb0\x0e\xa8\x3a\x01\xa2\xee\x15\x48\xfa\xc1\xff\x52\x3a\x21\x3b\x35\x67\x5f\xce\x9e\xe5\x6f\x92\xea\xeb\x75\xc0\x59\xbd\x82\xe6\xda\xad\x14\x8e\x99\x8c\x10\x06\x0d\x1b\x57\xb1\xf1\xdf\x7f\x4e\xdb\x03\x65\x66\x64\x52\x39\xe3\x04\x2a\x67\xce\xc3\xb2\x6c\x2a\xa6\xcd\xa1\x62\xca\x2c\xf6\xae\xfc\x00\xdb\xb2\x58\xf3\xea\x53\x4c\x5d\x78\x3e\xd9\x39\x39\x7a\xae\x2c\x85\xa2\x35\x87\xbf\xd7\x28\x5b\x06\x83\x68\xa5\xfa\xa9\x5b\xff\xa9\xff\xd0\x8c\xdc\x7c\x66\x5e\x77\x1b\x05\xc3\xc7\x22\xec\x14\x23\x8e\x5b\xc0\xe4\xf3\xae\xe4\xbd\xdf\xdf\x01\x40\xe7\x91\x46\xde\x7b\xe8\x0f\x5f\x38\xbb\x87\x8d\x9d\x40\xd5\x98\x1a\xb0\x6d\x7f\x9d\x05\x7c\x00\x6d\x37\xe3\x8e\x2d\x50\xe2\xb5\x44\x00\xbe\x12\x2e\x34\x18\x70\x21\x8c\x8f\x1a\x9d\xe2\x53\xaf\xd0\x3f\x13\xea\x87\x83\xdc\x7b\xb0\xc9\x04\x8e\x3b\xb1\x72\xe6\x3c\xca\xa6\xcc\xe6\xd0\xaa\x65\x00\x74\x37\xd5\xb1\x6f\xe9\x7f\x28\x18\x33\x09\x29\x21\xa3\xa0\x84\x44\x5e\x21\x3d\x4d\x75\x98\x89\x0c\xf2\x86\x8f\xa1\x72\xd6\x49\x0c\x3f\x61\x21\x15\x53\x8e\x25\x23\x99\x64\xff\xea\x15\x1c\xf8\xe4\x5d\xda\x0f\x07\x3a\xf3\xde\x35\x9f\xb0\x7b\xd5\x0a\xa6\x2e\x38\xdb\x55\x99\x54\x9d\x98\x74\x80\xc3\x99\x70\x10\xd0\xdf\xd1\x4e\xab\x1b\xc9\x08\x50\x36\x71\x06\xa5\x13\xa7\x63\xa5\x52\x18\x42\x62\x4b\x9b\x44\x32\xf7\x0b\x01\x0d\x1f\xf3\xbf\x74\x15\xc5\x43\x2a\xc0\xb6\xfd\x99\xa7\x8e\xb8\x81\x70\x29\x58\x6a\x3b\x1f\xbd\x70\x1f\x67\xbf\xad\xd4\xa8\xd6\x1f\x70\x19\x4a\xfe\x72\x14\xf4\xd5\xa8\x94\xb0\xd0\x35\x98\xcf\x15\x02\x29\x3a\x3c\x79\xc2\xcf\x92\x52\x92\xc8\xcd\x67\xec\xa2\xcb\x38\xfc\xf9\x87\x7e\xac\xd5\xbe\x0f\x5e\x65\xec\x05\xd7\x92\x59\x54\x46\xfe\xe8\x09\x1c\xf7\xc3\xfb\x68\xd9\xb1\x91\xe2\x71\x53\x28\x99\x38\x83\x64\x71\x19\x31\xd3\xd9\xa7\xb5\xf6\xf1\x3f\xf1\xf9\xc3\xbf\x4b\x0b\x57\x1a\xe8\xed\x61\xc3\xdb\x2f\x33\x79\xfe\x42\x27\x0b\x11\x11\xc9\xd0\x10\xc4\xc2\xec\x25\xe8\xbc\x41\x57\xc3\x61\x3a\xea\x03\x9d\xb7\x7c\xea\x6c\xe2\x59\xd9\xbe\x0f\xd4\xb6\x6c\x9a\xf7\xed\x3a\x2a\x98\x86\x19\x23\x2b\x37\x8f\x82\xb2\x72\x4a\xab\x46\x30\xf6\x98\x59\xcc\xbf\xe8\x4a\xa4\x2b\xe2\xab\x92\xaa\x06\x14\x81\xb9\x4d\x8d\xd9\xf2\xb2\xfb\x78\x99\x04\x82\xc1\x0c\x03\xad\x0e\x7c\x3a\xca\x69\xc2\xd4\x51\x84\x2b\x42\x6d\x53\x1b\x19\x9d\x28\x45\x6d\x97\x64\xf8\x09\x0b\x29\x1a\x33\x91\x23\xdb\x37\x00\xd0\xba\x6b\x0b\x75\xab\x97\x31\x7a\xd1\xe5\x48\xc3\x60\xd8\xfc\xb3\x19\x3e\xff\x1c\x67\x57\xa4\x95\x02\x29\xb1\xad\x14\xb2\xbf\x97\xbd\x1f\x2e\x1e\x34\x16\xad\x79\xff\x6e\x52\x7d\x7d\x24\x92\x49\x27\x51\x1c\xca\x7a\xec\xba\x45\x7d\x80\xd3\x8c\x09\x86\x41\xfb\xfe\x5a\xfa\xda\x5b\x00\xc7\x78\x5e\x36\x69\xa6\x13\xa6\x62\x39\xb4\xd4\xd7\xdd\xc9\x61\x4d\x0d\x10\xe4\x94\x94\x51\x38\x74\x04\xa5\x23\x46\x53\x51\x3d\x9e\xca\x31\xe3\x28\x1f\x3e\x9a\x92\xf2\x0a\xf2\x0b\x0a\xc8\xca\xcc\xc4\x74\x43\x53\x50\xa5\xd6\xb0\x35\x0a\x05\xf4\x20\x51\x63\x90\xf1\x4d\xa0\xb1\x45\x9f\x7a\xd5\xbf\x49\xa7\xb2\x34\x36\xac\xf8\x62\xd3\xa4\x6a\x18\x54\xa8\x8a\xba\x77\xf4\x0f\x6d\xb2\x4b\xcb\xa9\x5e\x78\xb1\x0f\xb0\x9d\x1a\x60\xcf\xdb\xcf\x53\x30\x6a\x3c\x47\xb6\xae\xa5\x71\xfd\x4a\x62\x59\xd9\x4c\xfe\xf2\xcd\x64\x97\x55\xfa\xf7\x34\x33\x32\x29\xaa\x9e\xc8\xa1\x55\xcb\x83\x11\x36\x0c\x32\x73\xf2\xc8\x2f\xaf\x62\xda\x59\x5f\x22\x96\x91\xa9\xb5\x20\xbc\x03\x34\x36\xd8\x4e\x05\x90\xb4\xd4\x6e\x75\x92\x8f\xe1\x44\x2b\x26\x0b\x8a\xb1\xfb\xfb\x88\xc5\xe3\x98\xb1\x18\x6d\xfb\x0e\xd3\xb2\x37\x60\xe1\xe3\x4e\x5e\xc4\xe9\xb7\xfe\x8c\xe2\x8a\x2a\x92\x39\x39\x64\x26\xe2\x24\x4c\x83\xb8\x00\x53\xda\x18\xc2\xc6\xb6\x1d\xca\xd5\x04\x19\x81\x26\xe0\xf8\xa9\x8a\x45\x3a\xfb\x13\xea\xdf\x11\x02\x97\x3d\x08\x7b\x56\x27\x4f\x98\x52\x3d\x0e\x32\xa8\xde\x1b\x02\x52\x7e\xc1\x67\xae\x06\xae\x3d\x63\xd4\x82\xf3\xd9\xf2\xe2\x23\x74\x1c\x72\x04\xd6\x43\x2b\xdf\xa5\x71\xd3\x2a\xfa\x5a\x9a\x7c\x8e\x98\x53\x56\xc9\x94\x6b\xbe\x8d\xb4\x9c\x40\x41\xc3\x30\x98\x7c\xc9\xd7\x18\xe8\x6c\x63\xa0\xa3\x8d\x92\x51\xe3\xa8\x18\x3f\x85\xf2\xea\x1a\x4a\x86\x0e\x27\xaf\xa8\xd8\x31\xbe\xb8\x9d\x8a\xd2\x06\x62\x41\x93\xdc\x80\x33\x97\x05\x5a\xa9\x01\x8e\x28\x8e\xe7\xbe\xf6\x16\x96\xdc\xf5\x75\x0a\x86\x8f\xa1\x78\xf4\x78\xca\x6a\xa6\xd0\x51\x7f\x90\xee\x16\xc7\x07\x2a\x4c\x93\x29\xe7\x5e\x41\xc5\xa4\x19\x88\x54\x3f\x48\x9b\x54\xca\xa2\xab\xb1\x0e\x61\x0d\x50\x5c\x36\x84\x58\x22\xee\xbb\xeb\xa4\x46\xa2\x2a\xe5\x38\x9b\xca\x3d\xf0\x44\xa8\x7d\x69\x54\xe4\xc9\x0f\x32\x7d\x57\x46\xd4\x91\xc6\x82\x15\x6b\x95\x0f\xb4\x48\x77\x45\xa6\xad\xaf\x84\xd8\x75\xf8\x39\x22\x30\x67\x22\x6d\xf2\x87\x8d\x66\xe4\xc9\xe7\xb0\xe1\xc9\x07\x00\xb0\xfa\xfb\xb0\x9a\x1b\x08\xdf\xd5\x89\x15\x13\x7e\x1c\x77\xf1\xe8\x1a\x4e\xbf\xeb\xaf\xc4\xb0\xc9\xc8\xc8\x20\x61\x9a\xc4\x84\x24\xa6\x38\x6c\x8f\xc6\x49\x62\x5a\xa7\xbd\xf5\xcd\x10\xf4\xb5\xb7\x6b\xd4\x69\xa7\x52\xb4\xec\xd9\x41\xcb\x9e\x1d\xec\x5e\xf6\xa6\xbb\x59\x2b\xe6\xef\x9a\x17\x42\xb0\x73\xd9\xdb\xd8\xa9\x7e\xca\x46\x8d\xa5\xb4\x6a\x24\x1d\x87\xf7\xf1\xd2\xcf\xbe\x43\x77\xcb\x11\x26\xce\x5b\xc0\xc5\xb7\xfe\x98\x92\x21\x15\xda\xda\x19\x0d\x82\x03\xb2\x3f\x40\xa1\x41\xf5\xba\xa6\xdb\x9d\x23\xd6\xde\x10\xb0\xda\x6b\xd8\x5b\xa4\x80\xeb\x7d\x16\x06\xec\x0b\xe2\xd0\xd3\x27\x91\x27\x30\x22\x11\x86\xc9\x98\x85\x17\xb3\xfd\xb5\x27\xe8\xeb\x68\x45\x18\x06\x89\x9c\x7c\xb2\x87\x0c\xa5\x60\xe4\x38\x2a\x67\xce\x63\xf4\x69\x17\x61\x48\x5b\xb3\xae\x79\x61\xbb\xa6\x61\x82\xb4\xb1\x2d\x37\x8f\xa8\xbf\xa6\x39\x64\x90\x26\xf0\x79\xdf\xfc\x64\x43\x87\x54\xa5\x4e\x5b\x02\x86\x41\xc3\xf6\x8d\x3c\x7d\xc3\x39\x5a\xc6\xb7\xff\xaa\x73\xc2\x20\x23\x37\x8f\x82\xf2\xa1\x20\xa1\xce\xd5\x91\x85\x10\x7c\xeb\x6f\x4f\x33\xf7\xac\x0b\x31\x6c\xcb\x0d\xb1\x11\x5a\xa4\x85\x9a\x79\x41\xf5\xd8\x04\xf9\x28\x94\xdc\x15\x21\xc7\x88\x66\x2e\xe4\xbf\xa0\x62\x45\xc0\x0b\x9b\x29\xa3\xa8\xd7\x9b\x3c\xc1\x06\x3d\x75\xd3\xb5\xd3\xb6\x01\x5b\x6a\xb1\x6c\x5a\xa4\x2a\x92\x54\x7f\x3f\x9b\x5e\x7c\x84\xae\x86\xc3\x14\x8d\x9d\x44\xc1\xc8\x1a\x72\xca\xab\xc8\xc8\xcd\xc7\x8c\x27\x40\xda\x7e\xc2\x75\xdf\xec\xa8\xed\x0a\xd1\xf3\x60\x7a\x61\x4a\x71\xd7\x02\x18\x57\x2c\x81\x4e\x0c\x9a\x9b\x08\x0d\x45\x60\x31\x85\xb3\x90\xb7\xed\xaf\xa5\xd7\x15\xb0\xfe\x5f\x0e\x29\x6d\x7a\xdb\x5b\xa9\x6b\x6f\xd5\x3e\x8f\x67\x66\x91\xcc\x2b\xf0\xf7\xd1\x04\x54\x28\xb4\xc9\x15\x16\xb4\xb4\xb5\x55\xb1\xf3\x7a\xea\x93\xaa\x42\xfd\x37\xbb\x31\x54\x21\xca\xb7\x6b\x87\xc0\x0d\x17\xd1\xf0\xde\x28\xb2\xde\x20\x93\x3b\x60\xcb\x4e\x62\x73\x25\x2f\x16\x02\x91\xc8\xe0\x98\x2b\x6f\x72\x6e\xe8\xb1\x04\xe9\xa5\xeb\x0b\xa2\x43\xd3\x52\xf5\x93\xee\xb2\xf4\xc6\x48\x1d\x2b\x27\x87\xa8\x70\x4b\x2e\x48\x84\x6b\xfe\x0d\x2c\x41\xee\x0f\x0d\x01\xed\x87\xf6\xfa\x02\x96\x30\x0c\x86\xcd\x38\x9e\x78\x66\x92\xee\x96\x46\x7a\xdb\x5b\xe9\xeb\xec\x60\xa0\xb7\x1b\xab\xbf\x2f\x32\xd6\x28\x7c\x24\xf3\x0a\xc8\x1f\x52\x89\x2d\x6d\x3f\x31\x9a\x4d\xb0\x83\xd0\xcb\x3f\xaa\x35\xdc\xd7\xd1\x75\x0b\x9b\xef\x65\x0a\xaf\xbd\x04\x94\xa6\x81\x2a\x48\x13\x3e\xc2\x89\xb3\xc3\xbe\x5f\xdf\x3f\xe1\x59\xd7\x8e\x06\xae\x3b\x01\x1c\x60\xa4\x12\xd2\xa4\xfb\xb2\xbd\xac\xf6\x9e\x80\xe1\xb7\xc9\x50\x5b\xab\xb7\xd5\x9b\x74\xa6\x88\x8e\x06\x55\x97\x25\x29\x82\x71\x15\xee\xb7\x31\x8d\x15\xb9\x17\x19\x02\xb2\x92\x41\xd2\x92\x8c\x9c\x3c\x4e\xff\xee\xdd\x0c\xa9\x99\x4a\x5f\x77\x27\xfd\xdd\x5d\xf4\x76\xb6\xd1\xd3\xda\x42\x77\x4b\x13\xdd\xcd\x8d\x74\x35\xd5\xd3\x75\xa4\x9e\xae\x23\x8d\xf4\xb4\x36\xd1\x75\xa4\x91\x96\x03\x7b\x7c\x53\x64\x5e\xe9\x10\xb2\x0b\x8b\xdd\x00\x73\x2f\x1b\x8f\x03\x9e\x40\xf8\x03\xa8\x36\x5c\xf3\xce\x48\x2f\x01\x98\xea\xfa\x8b\x16\xac\xd2\x02\xdf\x15\xd0\xd2\xd8\x5f\x18\x5c\x97\x82\x7c\x2a\x11\x21\x30\x34\x28\xa2\x9f\x13\xe8\xa3\x4e\xa7\x0c\x77\x72\xfa\x2c\x21\xa2\x7d\x7e\xbb\x95\xef\x7d\x3b\x40\x68\x39\x51\xb9\xae\x9a\x10\xd5\xe1\x80\x4e\xe6\x5b\xef\x88\x19\x11\xad\x15\xd2\xa6\xe6\xa4\x85\xcc\xb8\xe8\x1a\x1a\x77\x6d\x65\xc2\x69\xe7\x51\x59\x33\x19\x23\x1e\x23\x96\x5f\x40\x32\xbf\x10\x18\xe6\x52\xbd\x02\x89\x6d\x23\x53\x03\x60\x0d\xd0\xb2\x67\x3b\x8f\xdf\x74\x09\x1d\x8d\x4e\x06\xb8\xe2\xa1\xc3\xc9\x48\xe6\xf8\xa9\x88\xd5\x99\x16\x16\x80\x7c\x80\x49\xdf\xf9\xa8\x6d\x82\x46\xa7\xd8\x34\x5d\x5e\x01\x56\x65\x71\xaa\x30\x15\xb6\xfc\x84\x03\x08\xbc\x7d\x53\x61\x81\x2f\xfc\xde\xd7\xaf\xa5\x73\x2f\x95\x35\x4b\x14\x6b\x9d\xd0\xaf\xd1\xfa\xae\x2c\x95\x88\xf4\x49\x15\xf6\xfb\xfa\x63\xe5\x86\x05\x05\x89\x60\x83\x68\x95\x98\xa1\xdc\x49\x99\xb7\x14\x56\x0c\xe5\xc2\xbb\xfe\x4c\xaa\xbf\x8f\x78\x56\x12\xaf\xe2\x8a\xed\xdd\xd0\x1b\x58\xa9\xb3\x21\x33\x16\xc3\x4c\xc4\x19\x32\x66\x22\xb3\x2f\xbd\x8e\x8d\x6f\xbd\x48\x2c\x1e\xe7\xd8\xf3\x2e\xc7\x8c\x27\xdc\x88\x0f\xfc\x2c\xf2\x5e\x83\x50\x6a\x32\x80\x27\x64\x85\x63\xc3\x94\x5d\x17\x61\x93\x64\x88\x40\xc2\x83\xa2\x3a\x15\x54\xcf\x8b\xca\xaa\xd5\xeb\x54\x6e\x66\xcb\x20\xa7\x76\x14\xf5\xaa\xd8\x19\x42\xb8\xc5\x45\xdc\xfb\x48\xb4\xa8\x50\xd4\xb6\x84\xde\xa7\x79\x34\x42\x37\x17\xe8\xeb\xb3\x47\xf1\x1e\x6b\xd6\x04\x6b\x2f\x28\xe9\x9e\xcd\x9d\x52\xbd\x37\x4a\xc3\xbc\xa7\xea\x59\x5d\x07\xc9\x02\xe0\x0d\x88\x3b\x28\x31\x43\x20\x6c\x9b\xfe\xf6\x66\xe2\xa6\x49\x7e\x61\xa1\xe6\x06\x0c\xe2\x7a\xd3\x63\x95\xd5\x7b\x0f\x16\x98\x1f\xb5\x13\x52\xed\x87\x16\xce\x02\x69\xba\xad\x0e\xb0\xd0\x26\x41\x30\xcd\x03\x69\x5e\xcd\x03\x19\xce\x53\xa5\x6e\xfd\x49\xcb\x94\xa0\x00\xa6\x6d\x29\x51\x84\xa9\xf0\xba\xaf\x12\x4d\x18\xef\xc1\x84\xc5\xa8\x48\x14\x0f\x87\x10\xf5\xea\x8d\xc1\x35\x7e\xa8\x5e\x1d\x14\x6f\x0f\x04\x82\x88\xc6\x76\x24\x18\xa6\x41\x6e\x71\xa9\x93\x94\x5a\x2d\xcb\xa3\x52\xb0\xd2\xf9\x70\x5d\x86\xa8\x35\xd8\x13\xac\xd4\x8e\x47\xb1\xcb\x70\xc7\xc3\x93\x28\x4a\x72\x0e\xab\x46\x83\x11\x53\xd4\xc0\xfb\xed\x10\x20\xbc\xb4\xca\xde\xb8\x78\xe3\xe4\x82\x1a\x7e\x66\x38\x2c\xd7\xa3\x78\x75\x82\xa8\x12\xb3\xd6\x5f\xff\x22\x97\x4d\x2b\x3f\x92\x12\x5d\xc8\x0a\x83\xac\x5a\x6a\xfc\xfa\x58\x02\xb7\x20\x94\x0c\x00\x11\xe9\x33\x2b\xe0\x04\x8a\x59\x43\x91\xd4\x7d\xdf\xaf\x32\x49\xbc\x6b\xfd\xef\x7c\x2a\x48\xb7\x31\x6b\x13\x3e\x24\x5d\x46\x82\x9b\xb6\x0e\x0b\x4d\x68\x09\x3b\x3c\x02\xc9\xf8\xbf\x03\x57\x03\xd9\x7d\x97\x26\xec\x29\xcf\xf2\xf6\x4a\xf9\x05\xba\xa2\xb8\x07\x28\x96\xbf\x68\xab\x95\xca\xd5\x3d\xce\x6b\x7b\x6d\x77\xb9\x64\xa4\x90\xa3\xf4\x51\x5b\xdc\x84\xff\x2a\x5c\x97\x61\xd4\x4c\x8e\xf6\xc2\xf8\x8d\xc5\x13\xa2\x5c\x09\x5a\x01\xd7\x77\x20\xa8\x20\x0f\xa2\xdf\x8a\x88\x57\x55\x10\x49\x8f\xad\xd2\x43\x72\xa2\xf4\x5e\x6d\xb0\x42\x1d\x08\xaf\xf9\x12\x7d\xe0\xa3\x58\xa9\xf7\x87\x1a\x5f\x6d\x80\xb3\x73\xd2\x96\xe4\x16\x16\x61\x9a\x26\x9e\x99\xd8\x7f\xae\x42\x00\xce\xb8\x89\x10\xdb\x77\x28\x56\x35\xaf\x79\x0e\x1c\x55\x4d\x8c\x45\x0d\xd2\xa0\x53\x53\x7a\x42\x87\x07\x86\xce\x8a\x06\xbb\x5e\x53\xc8\x65\x50\x70\x52\xb8\x20\x1b\x3e\x5f\x0c\x4d\x08\x02\xea\x55\xbf\x13\xa1\x36\x41\xba\x10\xa5\xe7\xdc\x12\x69\x2c\x5a\x9d\x0c\xde\xf5\x1a\x98\xe1\x89\xe9\x4f\x4a\xdd\xe8\xe2\x7f\x1f\x21\x8f\x78\x6d\xf4\x27\x14\x10\x33\x0d\x36\xbc\xfd\x32\x8b\x1f\xb8\x07\x2b\x35\xc0\xcc\xb3\x2e\xe2\xec\x9b\xbe\x4f\x3c\x23\xc3\xd9\xc9\xef\x3d\x57\xa8\xe3\xe5\xda\x0c\xf0\x34\x07\x87\x15\xf7\x77\x77\x61\x26\x32\x30\x63\x31\x45\xf0\x75\x4b\x05\xba\x4e\x88\x41\x33\xbe\x47\x60\xab\xfd\x2d\x06\xf9\x36\x6c\xc3\xd5\x06\x2a\x74\xfa\xeb\x85\x6f\x64\x57\xae\x73\x05\xba\xb0\xf3\x5e\x7a\x0f\x09\xef\x78\x1f\x24\xaa\x50\xdd\x82\x92\x66\xf0\x50\x24\xd3\xa0\xbd\xd2\x5f\xbf\xd4\xcf\x02\x20\x55\xa3\x8b\xb2\xe7\x29\xe8\x4a\xc8\x49\xa2\x0b\x76\x76\x6a\x80\xd5\xaf\x3d\xc3\xc1\xad\x8e\xeb\xf0\xc3\xa7\x1f\x61\xe6\xe9\xe7\x50\x59\x5d\x43\x3c\x1e\x27\x16\x8b\x23\xcc\xa0\x46\x9a\xed\x67\xaf\x73\x5e\x85\x84\x81\xfe\x7e\xde\x7f\xec\x01\xd6\x2f\x7e\x95\xb2\xea\x1a\xce\xbc\xe5\xc7\xe4\x97\x57\xb9\x75\x24\x3c\x19\xc8\x19\x9f\x58\xd0\x89\x48\x3d\x5c\xa7\x1a\xa1\x77\x40\x8a\xd0\x85\xa1\x23\x10\x54\x74\xaa\x55\xf5\x34\x7f\xad\x53\xd7\xf4\x41\x26\xc5\x60\x93\x8d\x08\xd5\x27\x5c\x3f\x48\x07\x58\x0f\xb1\xd5\xee\x29\x83\x40\x02\x6d\x32\x4a\x22\x42\x63\xd3\x77\x4d\xf8\x93\x44\x99\x36\x01\xc8\x82\xbe\xae\x4e\x5a\x0e\x05\x15\x64\xda\x8f\x34\xf2\xe0\x2d\xd7\x90\x5f\x52\x46\x6e\x61\x09\xf9\x25\x65\xe4\x97\x96\x91\x5f\x3a\x84\xbc\x92\x32\x72\x0a\x4b\x48\x16\x14\x91\x91\x93\x4f\x3c\x99\x4d\x3c\x2b\x9b\xc3\xdb\x36\xb0\xe4\xc1\xdf\xd2\xdd\xd6\xca\x9e\xb5\x2b\x19\x71\xcc\x6c\xe6\x5e\xf1\x35\xec\x94\x9d\xb6\x9c\xc5\xc2\x03\x77\x34\x90\xfd\x41\x55\x66\x78\x78\xc0\xdd\x65\x20\x42\xd7\x0d\xa8\xd6\x4b\x9c\xa6\x2a\xe0\x46\x84\xd4\xa0\x52\x6e\xfa\x64\x13\xfe\xfb\xf0\x7a\x1a\x36\xd2\x47\x19\x31\x06\x93\x37\xa4\xd2\xa1\x34\x81\x50\xa6\x5b\xd1\x74\x55\x4d\x49\xdc\x2a\xd2\x47\xd1\x10\x82\xee\xd6\x66\x3a\x1a\x83\x44\x2d\xb6\x95\xa2\x7e\xf7\x4e\xea\x77\xef\x24\x7c\x18\xa6\x49\x3c\x23\x93\x44\x56\x92\xac\xec\x5c\x92\xf9\x05\xe4\x14\x95\xd0\xd7\xd3\xad\x6d\xda\xeb\xef\xee\xd4\x64\x04\xbf\xf1\xe2\x28\x14\x3c\x98\x1e\x76\xb4\xc3\xfb\x9d\x2d\xdd\x2a\xa2\x42\x78\xbb\xc2\x5c\x6b\x0b\x48\x69\xfb\x09\x59\xd4\x8b\xa4\x07\x52\xc4\x3a\x1c\x39\xc9\x94\xd7\x30\xb8\x66\x08\x5c\x2f\x5d\x71\x58\x52\x55\xfb\xe9\x01\xa9\x0a\x5a\x48\x65\x0d\x54\x28\x56\xab\x90\x8e\xb7\x1e\x87\x9c\xa0\xd2\x01\x59\xf5\x62\x0b\xc3\xa0\xad\xfe\x20\xdd\x21\x47\xcc\x60\x87\x6d\x59\xf4\x75\x77\xd1\xd7\xdd\x45\xc7\x91\x68\xaf\x9e\x19\x8b\x53\x54\x35\x32\x78\x6c\x08\x64\x3d\xa2\xe3\x28\x64\xab\x52\x52\x38\xfe\x49\x05\x43\xb8\xa0\xa6\x06\xfa\xe9\x6f\x6d\xa3\xbd\xee\x00\x4d\xb5\xdb\xa8\xdf\xbe\x91\x82\x21\x95\x9c\x70\xf9\xf5\x64\x24\x12\xa1\xab\x85\x2e\xbc\xa9\xcf\x95\x3a\x15\x85\x41\xf6\x81\xf6\xd5\x20\xdd\x22\x15\x56\x95\x22\x29\x37\x82\x1b\xa5\x6d\x3d\x05\xcd\x8a\x16\x6c\x8f\x91\xfe\xf7\x91\x83\x16\x1a\xd7\xee\xb6\x56\x06\x94\xa4\x2c\xc3\x6a\x26\x81\x10\x34\x1f\x3e\x40\x77\x7b\xbb\x1f\xdd\xf1\xdf\x1e\x59\xf9\x05\x94\x8e\xa8\xd6\xe2\xce\x54\xc2\xd0\x84\x2c\x19\xd1\xa0\xf0\x77\x9a\x1e\xe6\xb2\xa5\x40\x55\x12\x74\xb5\x34\xb3\xe9\xed\x17\xd8\xfb\xd9\x0a\x9a\x76\x6f\xa3\xad\xee\x20\xbd\x1d\x6d\x48\xdb\x26\x2b\xaf\x80\x11\xd3\x8e\x65\xd4\xf4\x39\x7e\x84\xa1\xf4\x47\x58\x07\x26\x3c\x46\x83\xfd\x9d\x26\x15\x8b\xe0\x35\xec\xd0\x8f\xea\x9a\x67\x88\x50\x27\x6f\x34\xb8\x01\x90\x81\x97\x0b\xed\x33\xff\x7e\xca\x83\xd4\xc1\xb6\x6d\x9b\x51\xc7\xcc\xe6\x84\xcb\xae\x63\xdb\x47\xef\x93\x5b\x58\xcc\x57\x7f\xf5\x67\xca\xaa\x86\xd3\x7c\x70\x2f\x75\xb5\x3b\x58\xbf\xfc\x5d\x96\xbe\xf8\x14\x96\xeb\xc9\x8b\x25\x12\x58\xa9\x94\x96\x40\x4d\x3d\x0a\xca\x87\x92\x5f\x56\x01\xe1\x89\x21\x15\x80\xc3\xab\x45\x98\xaa\x05\xce\x6e\x74\x10\x58\xb6\x4d\x6a\xa0\x0f\x29\x0c\x84\x19\x73\xbc\x43\xde\x35\x86\xc1\xe7\x2f\x3d\xce\x12\x37\x4e\x3a\x7c\xf4\xb4\xb7\xb2\xfa\xf5\xe7\x18\x36\x65\x66\xa0\x1a\xb9\xcf\xf0\x8a\x43\x1b\x11\x02\xc0\x17\x51\x2f\x83\xbc\x0f\x57\xf7\x56\xaf\x89\x62\xfd\x69\x12\x7f\x48\x0d\xf2\xed\xe1\xe8\xef\x23\x55\x24\x4f\x16\x71\x47\x57\x02\xd2\xb6\xc9\x2b\x2b\xe7\xf2\xbb\xfe\x48\xf7\x91\x06\x32\x33\x12\x14\x95\x94\x11\x37\x05\x25\x65\x43\x98\x7a\xdc\xf1\x24\x12\x09\x3e\x78\xe1\x49\x07\x9c\x78\x9c\x2b\xbf\x77\x27\xc5\x15\x55\xd4\x6e\x5a\xcf\xfe\x1d\x5b\x38\x5c\xbb\x93\x86\x7d\xb5\xfe\x04\x18\x32\x6a\x2c\xc9\xdc\x7c\x6c\x19\xd5\x23\x45\xc8\x1a\x94\x78\x25\x74\xb6\x36\xd1\x72\x68\x3f\x0d\xbb\xb6\x71\x68\xeb\x06\x1a\x6a\xb7\x93\x99\x9b\xcf\x89\x37\x7e\x97\x92\xd1\xe3\xdd\x1a\xbb\x1e\x8b\x53\x14\x28\xc3\x20\x59\x58\x42\x5f\x47\x9b\xbf\xd9\x7b\xe3\x92\xd7\x98\x7b\xc5\x0d\x94\x8f\x1e\x8b\x15\x8e\x8a\x73\x25\x34\x43\x61\xd5\x51\xcb\xc1\x51\xdb\xfb\x05\x13\x21\xea\xd0\xc3\x7c\x06\xb1\xbb\x47\x00\x1e\x35\x21\xbc\x87\x0a\xf5\x1a\x4d\xfd\x93\x98\xb1\x04\xc5\x55\x23\x48\xb8\xb9\x2f\x6d\x2b\x45\x57\x67\x3b\x07\x0f\x1f\xe4\x83\xe7\x9f\xf0\xfd\xeb\x79\x45\x25\xcc\x3d\xf3\x7c\xaa\xc6\x4d\x64\xde\x85\x03\xf4\xf5\x0d\xb0\x63\xdd\x6a\x7e\x7f\xc3\x25\xb4\x37\x39\xf1\x5c\x95\xe3\x26\x11\x4b\xc4\x19\xf0\x52\x46\x86\x70\x4e\xd3\x83\x35\xa9\x5a\x4a\x3e\x79\xe6\x61\x3e\x79\xfa\x1f\xb4\x2a\xac\xd6\x3b\x12\xb9\xf9\x9c\x79\xfb\x6f\x03\x49\xd2\xb2\xa9\x39\xe3\x62\xda\xea\x0f\xd2\xdf\xdd\x45\xd5\x31\x73\xa8\x98\x34\x83\x65\xf7\xff\x8c\x1d\x1f\x38\xc9\x3b\x5b\x0e\xee\x65\xfd\xe2\x57\x28\xf9\x9f\xef\x3a\x45\x9b\xd4\x09\x21\x70\x03\xda\x85\x96\x46\x97\x88\xd7\xa3\x5a\x9d\x34\xb0\x34\x79\x3d\x72\x92\x84\x8d\x19\x61\x8b\x95\xba\xe6\xaa\x1b\xc8\x07\x93\x4b\xd4\x76\x05\x7f\x38\xeb\x84\x35\xd0\x4f\x7b\x47\x2b\x5d\x4d\x75\x34\xef\xdf\x4d\x43\xed\x76\x0e\xef\xda\x4a\xfd\x9e\x5d\x34\x1e\xd8\x47\x4b\x43\x20\x61\x0f\xad\x1e\x47\x49\x79\x25\x76\xaa\x1f\x6c\x49\x46\x66\x06\x5d\x2d\x47\xe8\x6e\x6b\x05\x1c\x01\xab\x72\xec\x84\x60\x87\xa5\x2b\x25\x0a\x6f\x52\x49\xd7\x92\xa5\x0a\x16\x81\x51\x49\x30\xd0\xd7\xcb\xba\x37\x9e\xe7\xf0\xb6\xe8\x82\x13\x0d\x3b\x37\xd3\xdf\xd7\x8b\x91\xc8\xf4\x4b\xc6\xe4\x54\x0e\xe7\x94\x1f\xfc\xd6\xb1\xa8\x08\x27\x0d\xe0\x84\x45\x97\xb2\x6b\xc5\x3b\x7e\x80\xde\xba\x37\x9e\x63\xc6\x05\x57\x51\x50\x56\xee\x56\x2c\x55\x32\xcb\x8a\xc0\xe4\x16\x05\x6c\xda\x44\x94\xaa\xd5\x07\xcd\x14\xaa\x1a\x54\x22\xb4\x96\x74\x2b\x95\x24\x92\x62\xbd\x1f\x87\x75\x72\xf5\x38\x6a\x50\x9e\x10\x74\xb5\x1e\x61\xe5\x93\x0f\xb2\x7f\xc3\x6a\x5a\x0e\xee\xa5\xbd\xb1\x9e\x9e\x8e\xf6\xc8\x4a\x2d\xde\x31\x6e\xda\x4c\x92\x39\x39\x58\x6e\xa8\xb1\x94\x92\xda\x75\xab\x48\xb9\x69\x11\x93\xf9\x05\x0c\x19\x59\x8d\x70\xdd\x93\x81\x21\xc8\x6d\xaf\x88\xa2\x60\xa5\x47\x66\x22\x83\xa2\x61\xa3\xa8\xfd\x6c\x45\xda\xc3\x33\x73\xf3\xa9\x9e\xb7\x10\x91\xc8\xc4\x36\x4c\xac\x94\x45\xe7\x91\x06\x0e\x6f\x5c\x4d\xe3\xf6\x0d\x94\x8c\x9d\xcc\xc8\x79\x67\x60\xc6\x62\x0c\x3d\xf6\x24\x4a\xc6\x4c\xa4\x61\xeb\x3a\x00\xea\xb6\x6f\x66\xcb\xd2\xb7\x99\x73\xe9\x75\x4e\x5e\x49\x02\x90\x71\x1b\x68\x10\x94\x46\x77\xda\x15\x31\x7a\xde\x3e\x5d\x6f\xf0\x45\x10\x21\x12\x36\xaa\x0c\x16\x72\xa3\x67\x55\x97\xe9\xe0\xca\xa3\x80\x8a\xae\x37\x0f\x06\xb2\x30\x0c\x3e\x7f\xe5\x29\xde\xbe\xff\x57\xfc\xbf\x1c\x95\xa3\xaa\x31\x4d\xc3\x8d\x25\x17\x74\x77\x77\xb2\xfd\xf3\x95\xfe\xf7\xc5\x95\xc3\x28\x1a\x52\xe9\xe4\x10\x03\xb7\xe2\xa8\x1e\x9c\xe0\x87\xcd\x86\xa5\x49\x00\x23\x16\x63\xd2\xc2\x0b\x58\xf7\xc6\xf3\xbe\x68\x5f\x34\xbc\x9a\x29\xe7\x5c\xc6\xa8\xb9\xa7\x32\x64\xd2\x0c\x7a\x3a\xda\x39\xb0\xfe\x53\x6a\x57\xbc\xc3\xbe\x4f\x97\xd2\xb2\x77\x27\x56\x7f\x1f\xd9\xa5\xe5\x9c\x7b\xff\x8b\x94\xd6\x4c\x25\x51\x54\xc6\x98\x05\xe7\xf9\x00\x4b\xdb\x62\xed\x6b\x4f\x33\xe9\x8c\x0b\xc9\xce\xcd\x43\x48\x89\xa5\x48\xc3\x01\xe5\x44\x93\x84\xcf\x96\xdd\x20\x33\xbf\xb4\x2d\xf8\x65\x5d\x45\xe8\x94\xa4\xaf\xc7\x3a\x8b\x0e\xf2\x2c\xfb\x46\x8b\x88\x16\x04\xf7\x54\x0a\x6b\x7b\x6c\x11\xdd\x3a\xe6\x81\xdb\xd1\x70\x98\xcf\x5f\x7d\xea\xff\x09\x5c\x80\x9e\xce\x0e\x0c\xe1\x94\xdd\x8d\x99\x31\x76\x6f\xf8\x9c\x5d\xeb\x56\xfb\xdf\x57\x8c\x1e\x47\x76\x5e\x1e\x16\xd2\xd7\x18\xd4\x62\x64\x12\xd7\x9b\xe4\x29\xf8\x61\x90\x6d\xdb\x66\xe4\xac\x79\x8c\x98\x7e\x1c\x3b\x3f\x7e\x1f\x70\x8a\x4c\xcc\xbd\xee\x56\xe2\xd9\x79\x34\xd4\x6e\x65\xf1\xdd\xb7\xb1\x7f\xf5\x0a\x7f\x7b\xa4\x77\xf4\xb6\xb5\xd0\xd9\xdc\x44\x11\x4e\xc0\xc0\xc8\x93\xcf\x66\xdd\x73\xff\xa0\xcb\x0d\xe1\xd9\xbf\xee\x53\x6a\x3f\x5d\xce\xe4\xd3\xce\xc5\x72\xd3\xf6\x7a\xe6\x4c\xd5\x40\x9f\x06\x86\xaa\x8e\xb8\xb3\xc1\x33\xac\x07\x23\xab\x53\x31\x52\xfa\x3a\xb2\x2a\x00\x79\x37\xf2\x84\x29\xd5\xae\x1c\x3e\x54\x55\x4c\xd3\xb3\x15\xee\xa1\x68\x7c\xc1\xf7\x86\x41\x63\xed\x36\x1a\x76\x6e\xf1\xef\x95\xc8\x4a\x92\x9d\x5f\x48\x66\x76\x0e\xd9\x79\xf9\xe4\xe4\x17\x50\x50\x52\x46\x51\xd9\x10\xde\x7b\xf6\x71\x5a\x5d\x01\x6a\xc9\x73\xff\x66\xec\xb4\x19\x8c\x99\x3a\x93\xe6\xa6\x06\x5e\xb8\xff\xb7\x74\x29\x46\x92\xb1\x33\xe7\x90\x88\x27\xe8\x4b\xa5\x1c\x2d\x44\x6d\xbb\xf0\x00\xf6\xfe\x90\xfa\x4c\xf7\x66\x75\x66\x6e\x3e\xc7\x9c\x7b\x05\xb5\x9f\x2e\xc3\xb6\x2c\x0e\x6e\x58\xc5\xbe\xb5\x2b\x19\x35\x6f\x21\x7b\x3f\x5b\xce\xee\x8f\x96\xa4\x0d\x46\x3c\x99\xc3\xa4\xcb\xbf\x41\xf1\xa4\x59\x0c\xa4\x2c\x0c\x21\xc8\x1f\x35\x9e\xca\x99\xf3\xd8\xf1\xd6\x73\x80\x93\x6c\x74\xed\x6b\x4f\x31\xf6\x84\xd3\xc8\xc8\x48\xf8\xf1\x57\x9e\xd3\x40\x75\x21\x0e\xe6\x7c\xf7\xda\xac\x8b\x51\xde\x3b\xa9\xbd\xf3\x95\x95\x90\x31\x45\xbd\x77\x54\xd0\x7c\x1a\xb0\x92\x60\x20\x85\x17\xf2\x1b\x98\x28\xa5\x42\xc1\x42\x00\xb6\xcd\x90\xea\xf1\x4c\x5c\x70\x36\x9b\x96\xbc\x46\x2c\x23\x83\x8b\x6f\xbf\x97\xc9\x27\x9e\x4a\x32\x2b\x8b\xec\x64\x16\x59\x99\x99\x64\x65\x64\xd0\xdd\xd2\xc4\x67\xef\xfc\xc7\x07\x78\xcf\x96\x8d\xfc\xf2\xfa\xcb\xa8\xaa\x1e\x47\x7b\x4b\x33\x07\x6b\x95\x9d\x9e\xc3\x46\x32\x6d\xfe\xe9\x08\x1c\xf6\x2c\x85\xd2\x2e\x17\x50\x89\x13\x43\xed\x47\xfc\xf9\x20\xab\xf6\x58\xdb\x62\xcc\x89\xa7\x33\xa4\x66\x0a\x87\x37\xaf\xa5\xbf\xbb\x93\xcd\x6f\xbd\x40\xd5\x9c\x93\xc9\x1f\x5e\x4d\x22\x3b\x97\xfe\x50\x86\xb6\x58\x56\x92\xe1\xa7\x9e\x8f\x99\x93\x87\x25\xa1\xbb\xb9\x81\xbd\xef\xbe\x4c\xc3\x66\x3d\x5f\x45\xed\xc7\xef\x73\x60\xd3\xe7\x8c\x9e\x75\x02\xb6\x6d\x29\x3b\x07\x83\x81\x8a\x34\xb0\x84\x29\x4b\x12\xf0\xc8\x34\xba\x17\x0a\xc0\x1e\xd5\x47\x70\x87\x90\xc5\x4c\x07\xd6\x2d\x98\x8d\x63\xf6\x54\x86\xd1\xbd\xb7\x48\xdb\x9c\x1e\x4c\x0e\x49\x5e\x69\x19\x97\xfc\xf2\x01\x8e\x3d\xef\x72\x62\x31\x93\xc9\x27\x9e\x46\x56\x66\x06\x31\xa4\x9f\xc5\x00\x20\x99\x9b\x47\xc5\xc8\x6a\xf6\xef\x08\xd2\x65\x74\xb4\xb6\xb0\x65\xf5\x4a\xc2\xc7\xdc\x73\xbf\x44\xc5\xc8\x6a\x52\x9e\x00\x86\xd0\xea\x13\x7b\xfd\x36\xcf\xbc\xf9\x8e\xbb\x7c\x56\x16\xe1\x5d\x91\x40\x22\x3b\x97\xde\x8e\x36\x76\x7d\xec\x54\x2b\xe9\x6e\x69\x62\xe4\x09\x0b\x29\x1e\x37\x85\x44\x5e\x21\x99\x45\xa5\xe4\x0e\x1d\x49\xab\x5b\xbe\x2e\xd5\xd3\x85\x99\x95\x43\xe1\xf8\xe9\xec\x7d\xf7\x25\x3e\xbd\xef\x47\x6c\x7d\xf9\x51\x7a\x5b\x9b\xb5\x46\xa6\xfa\x7a\x31\x13\x19\x8c\x9d\x77\x7a\x28\x66\x4a\x5f\xc8\xc2\x92\xac\x7a\x78\x98\xea\x03\x1b\x32\x70\x28\x24\x2b\x8f\xf2\x3a\x98\x0a\x95\x7e\x28\xe1\xac\x82\xb4\x90\x20\xdf\x8a\x66\xe0\x27\x27\xcb\xc8\xcc\x62\xe8\xd8\x09\x94\x57\x8f\xc3\x34\x94\x54\x0b\x04\xf9\x35\x32\x32\x32\x18\xe8\xed\xe1\xd3\x25\x6f\x44\x0b\x95\xee\x31\x6c\xec\x04\xae\xf9\xc9\xbd\xe4\x14\x16\xeb\x01\x90\x7a\x13\x9d\x04\x36\x8b\x6e\xb9\xe3\x2e\x94\x01\xf1\x97\x33\xa1\x77\x3a\x59\x5c\xc6\xf6\xa5\x6f\xb9\x41\xef\xed\xe4\x94\x57\x51\x39\x73\x1e\xc5\x13\xa6\x33\x7c\xfe\x39\x14\x8d\x9f\xce\xfe\x15\x6f\xfb\xdb\x4d\x3b\x0e\xd6\x72\xe8\x93\x77\xd9\xf6\xc2\xc3\x74\x1e\xde\x37\xa8\x0e\x51\x31\x69\x06\x63\x4f\x3a\xc3\x4f\x28\xe2\x51\xcc\xd1\x22\xf8\xc3\xa7\x0f\xa6\x66\x1c\xd3\x41\xf6\x17\x46\xf5\x7e\x21\xb5\x49\xbb\x87\x88\xb8\x5e\x19\x23\x67\x29\xf1\x52\x24\xeb\x21\xb8\xa6\x61\x10\x33\x0d\x64\x6a\x80\xde\xf6\x56\x7a\x5a\x8f\x90\xea\xea\x00\xdb\x22\x11\x33\x49\x24\xe2\x4e\x7e\x12\xa4\x53\xd6\x4f\xb8\x59\xfb\x80\xa1\x23\x47\x53\xbf\x6f\x37\x7b\xb6\x46\x27\x4a\xcb\xce\xcb\xe7\xab\x77\xfd\x86\x09\x73\x4e\xc0\xb6\x6c\x5f\xcf\x77\xfa\xa5\xab\x96\x00\xe6\x59\x37\xdf\x71\x97\xf0\x25\x9b\x68\x83\xad\x2d\x21\x23\xaf\x80\xb6\xba\x03\x1c\x58\xeb\xb0\x8b\xfe\xce\x0e\x46\x9d\x72\x0e\x46\x46\x16\x96\x2d\x49\x14\x14\xd3\xd3\xd2\x44\xfd\x9a\x0f\x01\x48\xf5\x74\xd3\x5d\x7f\x40\x2b\xcc\x1c\xcb\xcc\xa2\xe6\xdc\x2f\x33\x6a\xfe\x59\xf4\xb5\x1d\x61\xcc\x49\x67\x72\xc2\x8d\xdf\x27\x99\x57\xa0\x54\x30\x53\x86\x32\x64\xcf\x8d\x0c\x99\xd1\xc7\x3f\xe8\x86\xd0\xfd\xb1\x9a\xf4\x16\x9e\xe9\x04\xa7\xf6\x99\x7a\x0f\x0f\x54\x0f\x4c\x74\x7b\xb7\x07\xb4\x69\x98\x74\x36\xd6\xb1\xe6\xf5\x67\x79\xff\xa1\xdf\xb3\xfc\xd1\xbf\xf0\xc9\x33\xff\xe4\xd3\x97\x9e\x60\xed\xdb\x2f\xb1\x73\xd5\x87\x74\x34\x35\x90\xcc\xce\x26\xbf\xb0\x88\x44\x3c\xe6\x00\xed\x7a\xc0\x32\x33\xb3\x98\x3a\xe7\x78\xfa\x7a\x7a\x38\x50\xbb\x83\x01\x25\x03\x6f\x69\x65\x15\xd7\xde\x71\x37\x27\x9e\x7f\xe9\xa0\xfa\xba\x3f\x26\x1e\x94\x7f\xd9\xd1\x25\x75\x23\xba\x5e\x3f\xc9\x0f\x51\x15\x26\xfb\x37\xac\xe6\xc9\x9b\x2e\xa4\xbb\xb9\x09\x33\x9e\x60\xe1\x3d\x8f\x30\xfa\xf4\x8b\xe8\xed\xed\xa5\x7e\xf3\x1a\xd6\xff\xeb\x8f\xec\x7b\xff\xb5\xc8\x99\x57\x38\x66\x12\xd3\xae\xbd\x8d\xea\x05\xe7\x91\x48\xc4\x48\xb5\xb7\x90\x93\x9b\x47\x66\x46\x86\xb3\x61\x0a\x9b\xfe\x8e\x56\xf2\x0a\x0a\x9d\x80\x34\x77\x20\x51\x1a\x0e\xe8\x41\x7c\x3e\x6e\x7a\x34\x87\x5a\xc4\x3a\x48\x4d\xa8\x08\x4a\x9e\xfb\x10\x34\xa9\xd8\x03\x32\x6c\xfc\x70\x36\x9d\xc9\x34\x67\xbf\x16\x8f\x25\x41\x0a\xc1\xe6\xe5\x4b\x78\xed\x8f\x3f\x67\xdf\x86\xd5\x47\x65\xb3\x45\x15\x43\x99\x71\xea\x59\x9c\x7e\xc5\x75\x8c\x9b\x36\x9d\x84\x69\x60\x48\xe9\xe6\xcf\x36\x48\xf5\xf7\xb3\x79\xd5\x4a\x56\x2f\x7b\x97\x23\xf5\x75\x94\x55\x0d\x67\xce\xc2\xb3\x19\x31\x61\xaa\x56\x75\x34\xa8\x34\xea\x95\x76\x0f\xd2\x60\x49\x40\xfc\xdf\x8e\x2e\x19\x76\x89\x49\x29\x35\x9f\xa7\x77\x51\xff\x40\x8a\x57\x7f\x72\x13\x1b\x5f\x77\x0a\x3b\x55\x9f\x76\x21\xc7\x5c\xfb\xbf\x6c\x7d\xed\xdf\xec\x5a\xfc\x02\xbd\x2d\x4d\x69\x1d\x89\x65\x65\x33\x6a\xd1\xe5\x4c\xfa\xf2\x2d\x4e\xf6\x36\x6c\x77\x27\x9c\x41\x4c\x48\x12\x86\xc0\x94\x36\x1b\x5f\x79\x9c\xd5\xcf\x3e\xcc\xc9\x37\xdc\xc6\x31\x67\x5e\xe8\xe6\xd1\x72\xc0\x70\x84\xa0\xe8\xe0\x36\x0f\x64\x41\xc0\x01\xc2\x21\x3b\x5a\x54\x07\xde\x3a\x39\xf8\xce\x7e\x9f\x12\xa4\x02\x32\x41\xfc\xb3\x6f\xf5\x72\x7f\xe8\x08\x85\x82\xcf\x17\xbf\xca\x93\x3f\xfd\x36\x6d\x0d\x75\xfc\xb7\x47\x61\x59\x39\x67\x5c\xf5\x55\x2e\xb8\xfe\x9b\x14\x97\x96\x39\x46\x0b\x6f\xa2\x9a\xa6\xf3\x5c\xcb\x06\xc3\x70\x76\x58\xa6\xac\xa0\x60\xa5\xf4\xaa\x83\xcb\x50\x26\xfe\xa0\x9d\xe2\x6f\x3b\xbb\x64\x7a\x38\x8a\x0e\xb0\xbf\x15\xd2\x30\xd9\xbe\x7c\x31\x2f\xdd\x7a\x15\x03\x3d\x5d\xc4\x32\xb3\x88\x27\x73\xe9\x49\xdb\xc8\xec\x1c\x05\xe3\xa6\x32\xf1\xda\xef\x52\x75\xe2\x59\xc4\x13\x09\x4c\x69\x6b\x5b\x1f\xe3\x86\x81\x91\xea\x63\xcd\x93\x7f\xe5\xc3\x07\x7f\x4d\x7f\x77\x27\x79\xa5\xe5\x5c\xf9\x87\xc7\x18\x3b\xe7\x24\xd7\xca\xa5\xb3\x9e\x41\xb7\xa9\x88\x00\x64\x8f\x62\x03\xa7\xbf\x1e\x60\xef\x81\x6a\xaa\x94\x1f\xc1\xbd\x55\xa0\xb5\x60\xbb\x90\x13\x42\x18\x26\x9b\x56\xbc\xcf\x43\xb7\x5d\x4f\x4b\x28\xeb\x7b\x56\x6e\x1e\xb1\x78\xc2\xd9\xd3\xd5\xdb\xcb\x60\xc7\x9c\xd3\x16\x71\xf3\xdd\xf7\x51\x35\x6a\xb4\x9f\x9c\xc6\x3b\xf4\x3a\x85\x6e\xb0\xbd\x12\x80\x9f\xbe\x5d\x35\xd0\xeb\x63\x61\x8b\x8f\xbf\x1e\x7b\xc1\xe5\x8a\x29\xcf\xb6\x6c\x2a\xa7\x1d\xc7\xd0\xe9\x73\xd9\xf3\xd1\x12\x52\xbd\x3d\x5a\x45\x11\x9f\x6a\x93\xb9\x8c\x38\xfb\x2a\xc6\x5d\xf1\x4d\xb2\x2b\x46\xe0\x6c\x5c\xb6\x11\x86\xc2\xde\x84\x41\x5f\x57\x27\xab\xfe\xf1\x1b\x3e\x7f\xe2\x01\xbf\xec\x4c\x7b\x63\x1d\x0d\xbb\x77\x30\xe6\xb8\xf9\xba\x54\x3b\x88\xbe\xea\x1d\xfe\xee\x00\x4f\x07\x92\x7a\x28\x9d\xf7\x97\xb7\x6f\x48\xd5\xf9\xd5\x23\xcc\xaa\x3d\xb5\xd1\x1b\x1f\xc7\x4a\x26\xdc\xeb\x25\x08\x83\xe6\xfa\x83\xbc\xf0\xdb\x9f\x68\xe0\x16\x55\x0c\x65\xc1\x97\x6f\x64\xd2\x09\xa7\x90\x99\xcc\xa6\xad\xe1\x30\x9b\x3f\xfa\x80\xcf\xde\x7c\x99\xc3\x11\xe1\x39\x2b\x97\xbc\x09\x52\x72\xfb\xfd\xff\xa0\xb8\x74\x88\xe6\xf8\xf7\x27\xb0\x0c\xe2\xa8\xd5\xcd\x7a\xc1\xf7\xe1\xdf\x83\x79\xde\xb7\x7f\x7c\x57\xda\x8c\x45\x67\x43\xc1\x46\x6c\x89\x48\x64\x61\x4b\xa8\x5d\xf6\x46\xa4\x13\xba\x70\xc2\x0c\xa6\x7e\xfb\x5e\x46\x5f\x7c\x03\xf1\xbc\x22\xfc\x2a\xdb\x8a\x70\x62\x9a\x26\x7d\x2d\x4d\x7c\xfc\xa7\x1f\xb3\xee\xd9\xbf\x6b\x06\xf7\x78\x56\x92\xe3\xbf\x7c\x13\xc5\x55\x23\x5c\xe7\x87\xa9\x51\xaf\x16\xc5\xa8\xe8\xcc\x69\xab\x9d\x4b\xd2\xea\xba\x2b\xf4\xaf\xf0\x77\x19\x08\x95\x03\x84\x76\x39\x08\xfd\x96\xfa\xab\xc0\x10\x06\xef\xfe\xfb\x21\x96\x3d\xf7\xb8\xff\xbb\xd2\x61\x23\xb9\xf1\xb7\x0f\x72\xd2\x25\xd7\x50\x5a\x39\x8c\xc2\xd2\x32\x2a\x47\x8d\x65\xda\xbc\x05\xcc\x5c\x70\x06\xa6\x61\xb2\x7f\xfb\x16\x52\xfd\x7a\x75\xf1\x83\xbb\x77\x12\x8b\xc5\x98\x39\xef\x64\x0c\x75\x57\x60\x48\xa3\x50\xc7\x40\xe5\x24\xbe\x6f\x5a\x04\xba\x7f\x38\x1f\x4a\x40\x2e\xfe\x85\x52\x5b\x8b\x2c\xdb\xa2\xea\xb8\x05\x94\x8c\x9b\xaa\x5d\x12\xcf\xc9\x67\xdc\x95\xb7\x30\xe7\xee\x7f\x53\x7e\xd2\xd9\x48\xc3\xf4\xfd\xc4\xfe\x3e\x5e\x40\x1a\x26\x1d\x87\xf7\xb3\xf4\xee\xef\xb0\xe9\xa5\xc7\xd2\x26\x49\xb2\xa0\x98\xfc\x8a\x2a\x6c\x29\xe8\x6e\x6b\xe5\xd0\xd6\xf5\xf4\xf5\x74\x83\x61\x86\xa2\x29\xa4\xcb\x2a\xbd\x33\xa8\xa1\x1b\x1d\x14\x27\xb5\x32\x41\x69\xf3\x41\x3b\x55\xf5\x27\x3d\x80\x3e\x10\xe4\x04\x5d\x2d\x4d\xac\x7c\xfd\x85\x60\x1c\x32\x32\xb9\xe4\xb6\x9f\x32\xed\xa4\xd3\x9c\x74\x0c\xb6\x85\xb0\x6d\x84\x9d\xc2\xc0\xa6\x6a\xf4\x58\xae\xff\xe9\xbd\x7c\xfb\x77\x7f\x63\xc8\xb0\x91\x69\xed\x78\xf3\xa9\x7f\x51\xbb\x79\x83\x53\xb8\x7a\x90\x65\x43\xfd\x2c\xbc\x7b\x22\x7c\x81\x96\x83\x2c\x52\x15\x09\x09\x1a\xb6\x2d\xc9\x2c\x2a\x65\xcc\x99\x97\x68\x37\xca\x1f\x3d\x81\x9a\x6b\x6e\x23\xb3\xac\x0a\xcb\xb2\x34\x6a\xf3\xb8\x00\x86\x49\x6b\xed\x56\x96\xfd\xfc\x26\x76\xbf\xf7\x0a\x51\x47\x7e\xf9\x50\x92\x45\xa5\xb4\x37\xd5\xf3\xe2\x9d\xb7\xf0\xe0\x35\x67\xf2\xdc\x8f\x6e\x62\xdf\xfa\xd5\xae\xc5\x48\x04\x80\xca\x20\x00\xce\x07\x1a\x5d\x96\xd0\xc2\x6f\xbc\xfe\x44\x58\x9c\xbc\xc1\x52\x75\x70\x55\x20\x33\x5c\xd0\x63\xa6\x89\x69\x1a\x2e\x27\x32\xa8\xdb\xbd\x93\xc3\xb5\x41\x41\xca\x71\x33\xe7\x30\x7b\xe1\xb9\x08\xdb\xd6\x2a\xc4\xf8\xf2\x80\xb4\x89\xc7\x62\x2c\xb8\xf0\x52\xbe\xf7\xa7\x87\xa8\x18\x31\x4a\xeb\xff\x91\x86\x3a\x3e\x7c\xfb\x75\x0d\x93\xff\xbf\x87\xc4\x8d\x56\x8d\x3a\xc3\x91\x84\xaa\x41\xde\x96\x92\xe1\xf3\xcf\x26\x6f\xd8\x68\xff\x66\x6d\xb5\x9b\xe9\xd8\xbf\x53\x33\xfb\xa9\x96\x27\x61\x98\x34\x6d\xfc\x8c\x15\x3f\xbb\x91\xc3\x9f\x2d\x1d\xb4\x51\x85\xc3\x46\x13\x4f\xe6\xb0\xe6\xb5\xa7\x59\xff\xd6\x0b\x74\xb7\xb5\xb0\xfe\xcd\xe7\x79\xfc\x9b\x97\xf2\xde\xdf\xee\xa5\xbd\xa9\x1e\x29\x0c\x6d\x97\x7f\x5a\xe5\x72\x22\xd8\x96\x22\xa1\x69\xeb\x96\xca\x9e\x51\x52\x27\xa0\x08\x6d\x6e\x85\x98\x81\xde\x1e\x6a\xd7\x7f\xce\xe1\xda\x1d\x18\xae\x85\xaa\xfd\x48\x03\xfd\x3d\x81\x1c\x32\xfd\x94\x33\xc9\xc9\x2f\x70\xd6\x4a\x11\x06\xd8\x5b\xa6\x24\xd2\xb6\x98\x79\xe2\x29\xdc\x70\xc7\x2f\xc9\x54\x4a\xc2\x03\x6c\xf8\xec\x13\x7a\xbb\xbb\x35\x07\xcb\x60\xfa\x7f\x5a\xc4\x8b\xdb\x4f\xef\x73\x23\x3d\xf5\x9f\x54\xf6\x03\x49\x5d\x82\x74\x6f\x68\xdb\x36\x39\x95\x23\xa9\xb9\xf0\x3a\x84\x69\x02\x90\x59\x52\x41\x22\xbf\x38\x4d\x4f\x75\xd1\xa5\x6e\xe5\xbb\x7c\xf2\xb3\x1b\x69\xde\xb2\xc6\xff\x38\x23\xb7\x80\xd2\xf1\xd3\xb4\x9f\x96\x8c\xae\x71\x62\xbd\xa4\x44\x88\x80\xc1\xb4\x37\x1c\xe2\xdd\x3f\xff\x82\xa7\xbf\x73\x15\x75\xdb\x37\x22\x0d\x23\xd8\x18\x1e\x6e\x9f\x0c\x24\xde\x48\x2a\x70\x11\x56\xd9\xb2\xa1\xad\xc3\x82\x58\xcc\x01\xb5\xbf\xa7\x8b\x5d\xeb\x56\xf3\xe2\x5f\xff\xc0\x3d\x5f\xbb\x94\x3b\x2f\x3f\x8b\x5f\x7f\xed\x52\xf6\x6c\x5a\x87\x61\x18\xe4\x17\x15\x13\xcf\x74\x2a\x8e\x67\xe7\xe5\x33\xf9\xb8\x79\x18\x42\x2f\x05\xa4\x5a\xbb\xd4\xcf\xa5\x6d\x33\x6f\xd1\x79\xcc\x59\xb0\x50\x6b\x5e\xe3\xe1\x43\x74\x77\x75\xa1\xda\xd1\xd5\xd0\xdc\xc1\x0c\x3f\xc8\x74\xee\x1b\xd3\xa8\x35\x24\x54\xa9\x41\xdd\x2a\x25\x7b\x37\x1e\x77\xf1\xd7\x20\x9e\x49\x4b\xed\x16\x86\x9d\x79\x39\xd9\x95\x23\xb1\x00\xc3\x5d\x7f\x3d\xd7\xdd\x81\xf7\x5e\x62\xe3\x9f\x7f\x44\x4f\x63\x50\x0e\x3d\x96\x91\xc5\xb1\xff\x73\x3b\x6d\x7b\x77\xd0\xe8\xfa\x89\x0d\x33\x46\xf1\xc8\xb1\xd8\x52\x32\xe3\xe2\x6b\xe9\x68\xa8\xe3\xc3\xc7\xfe\xac\xad\xd3\xfb\x3e\xff\x98\xbd\x9f\x7f\x42\x69\xcd\x54\x6c\x19\x94\xaa\xf1\x6d\xd8\x86\x70\x6d\xcf\xd2\xff\xa7\x9b\xb9\xd2\x7d\xc5\x0e\xa8\x8e\x79\x51\x08\x1c\x1f\xf7\x8e\xad\xac\xfb\xf0\x03\x3e\x5f\xfa\x2e\xbb\x36\xae\xa3\xbd\x25\xa8\x89\xd4\xd1\xda\xcc\x5b\x8f\x3d\xc8\xd7\x7f\x3d\x85\xd1\x93\x8f\x61\xc1\xa5\xd7\xf0\xde\x33\x8f\x31\xfd\x94\x33\x18\x31\x7e\x92\xa3\xe6\xb8\x1c\x62\xf0\xd4\xfb\x8e\x98\x9e\x95\x4c\x32\x7b\xc1\x42\x96\xbe\xfe\x92\x7f\xff\x81\xfe\x7e\x06\x52\x29\x7f\x02\xfb\xaa\x19\xea\xe4\x0d\x19\x59\x42\x02\xa8\x87\xa3\x96\x8c\x54\x17\x4c\xf4\xf7\x69\x91\x83\x52\x12\xcb\x4c\x52\x73\xe9\xff\x38\x3a\x18\x90\xb2\x24\x3d\x87\xf6\xd0\xd7\xd1\x4a\xee\xc8\x1a\x44\x2c\xc1\xde\xd7\x1f\x63\xcb\x83\x3f\xa7\x5f\xc9\xd8\x63\xc4\x13\x4c\xff\xda\xf7\x19\x7f\xe1\xb5\xbc\xfb\xa3\xeb\x14\x8a\xce\x23\xbf\x6a\x24\xb6\x6d\x13\xcb\x4a\x3a\xf0\x84\x85\xb0\xa2\x12\xca\x27\x4e\xf7\xd3\xff\x0b\xe1\xb0\x7f\x29\x6d\xba\x5a\x9b\xe9\x73\xeb\x1d\xe6\x14\x14\x92\x5b\x50\x48\x22\x33\x0b\xe1\xa6\x28\xd7\x77\x16\x3a\x54\x15\x8f\x39\x15\x4c\x7a\x3b\xda\x39\xb0\x7d\x33\xeb\x56\x7c\xc0\xda\xe5\xef\x52\xbb\x69\x3d\x1d\xad\x2d\x0c\x7a\x48\x1b\x6c\x9b\xcc\x64\x92\xab\x6f\xff\x25\x27\x5f\x7c\x25\x25\x43\x87\x39\x95\xc2\xfd\x67\x85\x52\x35\xaa\xd2\xb9\xc2\x56\x0b\x4a\xca\x30\x63\x31\x3f\x52\x32\x33\x3b\x1b\x33\x9e\x81\x65\xeb\xcb\x8e\xb6\xc9\x5c\x3d\x91\xba\x25\x52\x59\xba\x62\xaa\xd4\x99\x9e\xfd\x5d\xdf\xbe\x99\xae\x83\x4a\x9f\x37\x0a\x21\x68\x5c\xf5\x3e\x6b\xff\xf0\x7d\xfa\x9a\x1b\xa8\x5a\x78\x29\x89\x82\x62\x76\x3d\x75\xbf\x56\xa1\x5b\x98\x31\xa6\x5c\xfd\x6d\xa6\x7c\xf9\x16\xac\x9e\x4e\x3a\x94\x02\x1f\xd9\xc5\x43\xc8\x2e\x29\x47\x0a\x83\x1d\xcb\x17\xb3\xea\xb9\x47\xfc\xef\x2a\x26\x4e\xa7\x70\xd8\x28\x46\xce\x5d\x40\xd9\x84\x69\x4e\x71\x47\x37\x5d\x42\xc3\xa6\x35\x6c\x7a\xe3\x19\xf6\x7d\xb6\x8c\x8e\xc6\x3a\x90\x36\xc9\xfc\x22\x4a\x47\x8e\x61\xf2\x82\xb3\x98\xb1\xf0\x3c\x0a\x8a\x4b\xfc\xd0\x16\x8f\x45\xa6\xfa\x7b\xd9\xb6\xf2\x53\xb6\xac\x5c\xce\xba\xe5\xef\xb3\x67\xcb\x06\xba\xda\xdb\x38\xda\x91\x93\x5f\xc0\xa2\x6b\x6e\xe4\xac\x6b\xbf\x4e\x2c\x16\xc7\x96\x0e\xc8\x35\x33\x67\x23\x6d\x35\xcf\x95\x9e\x0a\x29\x4a\x5b\xf1\xc6\xb3\xe1\xd0\x41\x1f\x5c\x80\x8a\x11\xa3\x48\x64\x67\x93\xb2\xed\x34\x0e\xaa\x17\x12\x8d\xce\x5b\xa2\x12\x6c\x2c\x9c\xf1\x3d\xbd\x02\xa9\x1e\x51\x88\xce\xf0\x7c\xf6\x23\xa5\xe4\xe0\x92\x17\xe8\xdc\xe7\x38\xa5\x77\xbf\xf8\x90\xee\xad\x07\x84\x61\x30\xe1\xb2\xaf\x33\xf5\xda\xdb\x88\x25\x32\x68\x3f\xbc\x57\xb3\x82\xe5\x0f\x1d\x41\x66\x5e\x21\x6d\x75\x07\x59\xf6\xe0\x6f\xe8\xeb\x74\xf6\xdf\x24\xb2\x73\x39\xf9\x96\x3b\x19\x75\xfc\xa9\x48\xbc\x54\xc2\x82\xfe\xae\x0e\xd6\x3c\xf3\x10\xab\x9e\xf8\x2b\x9d\x6e\xa4\x88\x77\x74\x36\x35\xd0\xb0\x6b\x2b\x9b\xdf\x7f\x83\x75\x6f\x3c\xcf\x97\x7e\xf0\x2b\xaa\xa7\x4e\x77\x41\x76\x42\x60\xd6\x7e\xbc\x98\x3f\x7c\xf3\x1a\xba\xbf\x00\x54\xf5\xc8\xce\x2b\xe0\x8c\x2b\xae\xa3\xac\x6a\x38\xa9\x81\x94\x9f\xc9\x4e\xba\xa1\xae\x86\xe7\x2a\x09\xdb\xb8\x55\x60\x25\xae\x95\xc2\xa0\xb3\xa3\x9d\x8f\xdf\x79\x43\x7b\xc6\xe4\xe3\x4e\x42\xc4\x33\x18\x48\xa5\xf4\x60\x7b\xf4\x54\x11\xe1\xba\xcf\xaa\x05\xcb\x72\x4f\x23\x6c\xc7\xd4\x13\x9d\x28\x6a\x45\x48\x52\x51\x85\x13\x43\x38\xc6\x8b\xe2\x89\x33\x11\x86\x19\xea\x89\x77\x81\x60\xec\x05\xd7\x31\xed\x86\xdb\x9d\x82\xc7\x38\xc9\xb1\xfb\x94\x4d\x54\x45\x23\xc7\x22\x62\x71\x3e\x7d\xe2\xaf\x1c\x58\x17\x24\x22\x1f\x77\xf2\x59\x0c\x3f\xf6\x24\x67\x76\xda\x8e\x8b\xac\xa7\xad\x85\xf7\x7e\xf7\x23\x96\xfe\xf9\xae\x34\x70\x35\x2a\xb1\x6d\xb6\x7e\xf8\x1e\x8f\x7c\xef\x06\xf6\x6f\x5a\x4b\xcc\x34\x5d\x97\x1e\xf4\x76\x76\xd0\xd3\x19\x5d\x4e\x2e\x33\x99\xcd\xe8\x09\x93\x39\xe7\xea\xeb\x99\xbd\xe0\x0c\xff\xf3\xfa\xfd\x7b\x58\xfa\xca\x33\xbe\x39\xd1\x67\xc5\xde\x3f\xa1\xb3\x60\xc3\x34\x31\x63\x31\xbc\x8d\xd9\xaa\xcc\x23\x85\x60\xc9\x8b\x4f\xb3\x66\xf9\xfb\xfe\xef\x87\x0c\x1f\xc9\xb4\xf9\xa7\x31\x60\xd9\x4a\x35\x3a\x19\xaa\x88\xe3\x99\x29\xd3\xcd\x93\x61\x1c\x63\xe1\xc2\xd0\x5a\x56\x1b\xd2\x03\xcf\x54\x35\x42\x0d\x03\x11\xd2\x66\xf4\x99\x97\xd2\xb2\x65\x0d\xbb\x5e\x4b\x2f\xc7\x3e\xfa\xcc\xcb\x99\xfe\xcd\xbb\xc8\xc8\xce\x05\x37\xa4\xb6\xab\xee\x00\x96\x52\x43\xa8\xa4\x7a\x02\x7b\x3f\x5b\xc6\xe7\xcf\x3d\xec\x7f\x96\x55\x50\xc4\xf4\xcb\x6e\x70\x42\x73\xad\x14\x52\x08\x06\x7a\xbb\x59\x76\xff\xcf\x58\xf7\xc2\x23\xda\x33\x84\x10\xe4\x94\x0c\x21\x96\x91\x49\x77\x4b\x13\x7d\x5d\x41\xd5\xb5\x83\xdb\x36\xf2\xec\xbd\x77\x70\xf3\xfd\xff\xa2\xa8\xb8\xd8\x29\xe5\x5a\x5c\x42\x3c\x91\x41\x7f\x6f\x0f\x42\x18\x14\x0f\x29\xa7\x7a\xd2\x14\xa6\xce\x39\x81\xc9\xb3\xe7\x32\x72\xdc\x04\x0a\xcb\xca\x59\xb5\xec\x3d\x36\xac\x5c\x41\x4f\x97\xb3\xd4\xbc\xff\xfc\x93\x2c\xb8\xf8\x4a\xca\x86\x8d\xc4\x0a\xd9\x8d\x55\x02\xb0\x2d\x8b\x8d\x1f\x2f\xa3\xa5\xa1\x8e\xa9\xc7\xcf\xa7\xb4\x62\x28\xee\x7a\x86\x95\x4a\xb1\xfc\xd5\x17\x78\xec\x37\x3f\x67\x40\x49\xe7\x7f\xf2\x97\xae\xa2\x74\x44\x35\x7d\x29\xdd\x96\x10\xde\x42\x13\xe8\xff\xa1\x4a\x75\xa1\xbf\x63\x29\x99\xce\xdb\xa3\xd6\x5e\x1f\x5c\xe9\x82\xab\x38\x4f\x1d\x6a\x96\x64\x66\xe7\x32\xe3\xe6\x9f\x23\xed\x14\xb5\xff\x79\x52\xeb\x70\x2c\x99\xed\xa7\x2a\xf0\xb6\x37\xb7\x1f\xda\xe3\xbb\xd4\xe2\xc9\x6c\xe2\xd9\xb9\x7c\xf8\xe0\xaf\xe9\x6d\x0b\x84\x9b\xf1\x0b\x2f\xa4\x62\xca\x2c\x2c\x2b\xe5\xb3\xaa\x75\x2f\x3c\xc6\xfa\x17\x1f\xd5\xee\x9f\x2c\x2a\x65\xce\x57\x6e\x66\xc2\x82\x73\xc8\x4a\x66\xd3\x71\x70\x0f\x9f\x3e\xf3\x0f\xd6\xbf\xf9\x22\xb6\xeb\xb4\xd8\xb8\xe2\x3d\x3e\x7c\xe1\x09\x2e\xf8\xfa\xb7\x31\x90\xe4\x17\x16\x91\x91\x99\x45\x7f\x6f\x0f\xd9\x79\x79\xfc\xe0\xbe\x07\x99\x79\xe2\x29\x64\x64\x66\x22\x84\x70\x0d\x36\x36\xe3\xa7\xcd\x60\xec\x94\xe9\xac\xff\xc4\x09\x1f\x3e\xb0\x6b\x3b\xcb\x5f\x79\x8e\x4b\x6e\xfe\x5e\xa4\x77\xcb\x3b\xde\x7c\xfc\x21\x1e\xff\xf5\x9d\x74\x77\x76\x30\x76\xda\x0c\x16\x5e\x7e\x2d\x63\xa7\xcd\xa4\xb7\xbb\x8b\x8f\xdf\x7a\x95\x25\xcf\xfc\x8b\x4e\x37\x88\x1d\x60\xfc\xb1\xc7\x73\xda\x55\x37\x30\x20\x05\x96\x3d\xc8\x0e\x0b\x55\xe7\x27\x7a\x79\x55\xa9\x3a\x96\x9e\x02\x48\xb1\x02\xa1\x70\x59\xdf\xb8\xed\xa4\x6d\x50\x59\xb4\x73\x9d\xc0\x92\x36\xd9\x05\x45\x1c\xfb\x9d\xbb\x11\xc0\x2e\x05\xe4\x1d\x2f\xfd\x93\x8c\x64\x36\xb3\xbe\xf1\x53\x8c\x58\x0c\x69\xa5\x68\x3f\xb0\x27\x60\x89\x79\x85\xec\x5a\xf6\x16\xfb\x94\xe4\xd7\xb9\x65\x95\x4c\xfb\xd2\xf5\x4e\x0d\x40\xcb\x02\xc3\xa4\x7e\xcb\x5a\x56\x3d\x76\x9f\x9f\x66\x11\x9c\x5a\x12\xa7\xfd\xe0\xb7\x4c\x3e\xeb\x12\x62\x86\x20\x2e\x24\x65\xc3\x46\x30\x62\xd2\x34\x0c\x24\x9f\xff\xe7\x79\xe7\x87\x52\xb2\xf4\xb9\x7f\x31\xff\x82\x4b\x28\x1f\x5a\x45\x6e\x7e\x3e\x59\x39\x39\x74\xb4\x36\xd3\xd7\xdb\x83\x21\x04\xc9\x64\x96\xb3\xd9\x0b\x4f\xb6\x80\xfc\x82\x02\xe6\x9d\x79\x8e\x0f\x30\x38\x11\x8f\x27\x5f\x78\x19\xa5\x55\xc3\x9d\x65\x43\xd1\xb9\x05\x82\xbe\xbe\x1e\x3e\x7e\xf3\x15\x5f\x68\xdb\xbe\x66\x15\x3b\xd6\xae\x26\x99\x9b\x8f\x95\x1a\xa0\xb7\x3b\x10\x3c\x01\xca\x47\x8d\xe1\x8a\xdb\x7f\x45\x4e\x59\x05\x7d\xaa\x25\x30\x42\xc0\xf5\x53\x3b\x2a\x98\xa5\xb3\x69\x07\x68\xad\x6e\x92\xea\x0f\xf6\x1a\xeb\x11\xab\x07\xa6\x56\xb2\x45\x04\xe5\x70\x62\x6e\x8d\x23\x13\x49\x76\x41\x11\x73\x6e\xbd\x9b\x31\x67\x5f\xe9\x77\x40\xda\x36\xb5\x6f\x3f\x47\x4f\xfd\x41\x47\x4f\xee\xeb\xa5\x53\x29\x32\xd1\xdb\xde\xc2\xf6\x25\x2f\x6b\x6a\xd1\xc4\x73\x2e\xa7\x64\xdc\x64\xbf\x0a\x69\x6a\xa0\x9f\xb5\xcf\xfc\x9d\x8e\x50\x81\xae\xa9\x17\x5f\x47\xcd\xc2\x0b\x9d\x14\x06\xb6\x85\xb4\x6d\xec\x54\x8a\x64\x5e\x01\x0b\xae\xbb\x99\xec\xc2\x62\xff\xb7\xfb\x77\x6c\x61\xf3\xc7\xcb\x30\x0d\x83\xec\x9c\x1c\x72\xf2\xf2\x01\x18\xe8\xeb\x63\xdf\x8e\x2d\x5a\x74\x86\x1a\x61\x72\xdc\xa9\x67\x38\x75\x26\xdc\x63\xdf\xf6\x2d\x2c\x7b\xf5\xf9\x41\xb6\xa7\x4a\x32\xb2\x92\xcc\x5d\x74\x3e\xa6\x19\xec\x2d\x90\x52\xd2\xd5\xde\x9a\x06\xee\xb8\x59\x73\xf9\xfa\x1f\x1f\x66\xf4\xcc\xb9\xf4\x0e\x58\xf4\xdb\x28\xa7\xf4\xcf\x01\xef\x94\x4e\x75\x35\x3d\xb3\xad\x5a\xd8\x2c\x58\x76\x0d\x75\x16\xa8\x01\xdf\x3e\xb0\xaa\xa1\x5d\x01\xd9\x01\xd5\x2d\x58\xe5\xbe\x7a\xc5\xac\x62\x48\x72\x0a\x8a\x99\xfb\xbf\x77\x33\xfe\xfc\x6b\x30\xdc\x4e\x16\x55\x4f\x20\xab\xa0\x10\x43\x48\x06\xba\xda\xe9\x6a\x0a\x84\xa3\x81\xee\x2e\x06\x94\x8e\x17\x0c\x1b\xcd\xa4\x0b\xae\x41\x0a\x03\x0b\x90\xc2\xa4\x7e\xeb\x7a\x76\x7d\xf0\x1f\x6d\x70\xf2\x87\x8e\x64\xf2\x85\x5f\x01\xd3\x4c\x13\x06\xa5\x6d\x31\xb4\x66\x12\xc3\x26\x04\x8e\x11\xdb\xb2\x58\xbb\xf4\x1d\xac\x81\x7e\xb2\xb2\xb2\xc9\x2b\x2c\xf2\xbf\x3b\x50\xbb\x0b\x69\x3b\xaa\x94\x5f\x70\x4b\x08\x84\xb4\x19\x39\x66\x1c\xd3\x8f\x3f\x51\x7b\xf6\x92\x67\x1f\xa7\xe9\xd0\x7e\x4c\xd7\x40\xa2\xe5\x99\x96\x92\x05\x97\x5e\xcd\xf1\x67\x5f\xc8\x60\x47\x7e\xe9\x10\xce\xf9\xfa\x6d\x7c\xf3\x2f\xff\x66\xd4\xf4\xe3\xe8\x4b\xd9\x1a\xa0\xce\x89\x5f\x2e\xaf\xdf\x03\xd6\x0e\x80\xf5\xaa\xc6\xaa\x15\xd9\x55\x2d\xc8\x38\x5a\x96\x74\x4f\x98\x52\x6b\xc2\xa7\x55\xda\x56\x8a\x57\xc5\x45\x00\xb4\x89\x4d\x4e\x41\x31\xf3\xbe\xfb\x6b\x4e\xb9\xf3\xff\x38\xf6\xc6\x1f\x32\xef\xbb\xbf\x76\xe3\xaf\x04\xbd\x2d\x8d\x69\x51\x96\xea\x31\xf9\x82\x6b\x28\x18\x3e\x06\xcb\xb2\xdd\x46\xdb\xec\xfc\xe0\x75\x7a\x42\x51\x23\x23\x4e\x58\x48\xfe\xb0\x6a\x37\x00\x4d\xe7\x3c\x52\x4a\x12\x59\xd9\x54\x54\xd7\x68\xd7\xec\xda\xb8\x8e\xce\x96\x23\x64\x25\x93\x8c\x9e\x38\xc5\xff\x3c\x99\x93\x13\x24\x6d\x51\x4f\x9c\x58\xa9\xf9\x67\x9f\x4f\x2c\x16\x50\xe4\x9e\xad\x9b\x59\xf1\xda\x8b\x5a\xa6\x3c\xef\x55\x4a\x49\x32\x27\x8f\xcb\x6f\xfb\xb1\x53\x42\x48\x39\xb2\x72\xf3\x38\xf3\xfa\x9b\xb9\xed\x91\x97\xb9\xe8\x7b\x3f\x27\x67\xc8\x50\x7a\x07\x52\x3e\xa0\xde\x39\xe0\x56\x85\xd5\xa9\xd7\x3d\x5d\x4a\x0e\x47\x72\xa8\x6a\x93\x2d\x43\x45\x39\xb4\x1c\x13\x1e\xc8\xbe\x67\x65\xf0\x8a\x62\xde\x9e\x20\x29\x1c\xef\x45\x50\xd0\x4a\x12\xcb\xce\x66\xd2\x79\x57\x39\x77\x77\xcd\x2b\x86\x69\xd2\x5d\x7f\x88\x81\x88\xda\xc2\x00\x25\x63\x27\x33\xe1\x9c\xcb\xb5\x84\xa3\x7d\x9d\xed\x1c\x5c\xa5\xef\x91\x32\xe3\x09\x86\x1f\x7f\x2a\x98\x31\xa4\x9d\x4a\xdb\x7c\xe6\xb4\x5f\x90\x5b\x5c\xaa\x5d\x77\xa4\xee\x20\x4d\x87\x0f\x51\x56\x5e\xc1\xc5\x5f\xfb\x26\xc9\xac\x2c\x32\x33\x33\x39\xf7\xca\xaf\x20\x5c\xaf\xb9\xb7\x2c\x05\x76\x5e\x9b\xe9\x73\x4f\x64\xf8\x98\x71\xd4\x6e\xf5\x4a\x1d\x48\x16\x3f\xf3\x2f\x4e\x39\xff\x12\x8a\xca\x2b\xb0\x5c\xf3\xac\x67\x86\xb4\x2c\x8b\xaa\xb1\x13\xb8\xf4\x7f\x7f\xc2\x03\xdf\xfd\x1f\xfa\x5c\x0e\x55\x31\x7a\x1c\x17\xdc\xfa\x53\x32\x72\xf2\x18\x48\x59\x58\x29\x2b\x00\x89\xf4\xac\x01\xaa\x9d\x19\xe5\x33\xef\x73\xd5\x1d\xaa\x1b\x3d\x64\xc0\xa2\xc3\xe0\xaa\x3a\x9e\xea\x13\x35\x5d\xaa\xd5\x4e\xa1\xd4\x18\xf4\x6a\x0e\xba\x54\x1d\x13\x60\x4a\xcb\x09\xd7\x41\xfa\xbf\xef\xac\xdb\xef\x47\x71\xa8\x87\x61\xc6\x98\x76\xc9\xf5\xe4\x96\x0f\x0b\xd6\x63\x21\xe8\x3e\xd2\x40\xbb\x5b\xd0\xc2\xa7\xb8\x92\x21\x14\x55\x4f\x74\x2c\x48\xea\x60\x68\x83\x20\x7c\x87\x88\x77\xf4\x74\x76\xd0\x74\xf8\x20\x52\x4a\xaa\x46\x8f\xe1\xa6\x3b\xef\xe6\x86\x1f\xde\x49\xe5\xf0\x91\x48\x5b\x2f\xa6\xe1\xaf\xaf\xd2\x66\x48\x45\x05\xc7\x9f\x76\xa6\x76\xaf\xda\xcd\x1b\x58\xfe\x9f\x97\x30\x0d\x03\xd3\x34\xb1\x52\x29\xba\xda\xdb\x7c\x5b\x71\xca\xb2\x98\xbd\xe8\x7c\xce\xb8\xe6\xeb\x08\xc3\x20\x2b\x37\x9f\xe3\x2f\xfe\x32\x66\x56\x36\x7d\xa9\x94\x4b\x91\xb8\xa7\xf4\x2b\x99\xfa\x55\xd9\x6d\xaf\x3a\xbb\x57\xc4\x5b\x06\x41\x77\x2e\x8b\x0e\xe7\xf1\xf4\x72\x6b\xda\x12\x62\x76\x04\xd5\xfa\x20\x23\x34\x33\x5b\x54\x59\x71\x6f\xba\x07\x52\x9d\x08\x2a\x96\x29\xf7\x4e\xd3\xa5\xed\xe8\x6d\x93\xb9\xe5\x43\x19\x36\xfb\x24\x9f\x8a\xa4\x4b\x85\x7d\x9d\x6d\x69\x14\x9f\x2c\x1e\x42\x66\x7e\x21\x5e\xbd\xe2\x80\x35\x2b\xd2\xa7\x94\xf4\x75\xe9\x42\x4d\x6a\x60\x80\xd6\x23\x8d\x8e\x55\xcc\xb6\x9c\x76\x1b\x42\xb7\x17\x13\x32\xc4\x09\x67\x9f\xd1\x49\x8b\xce\xe5\xa5\xc7\xfe\x41\x97\x6b\xa0\x91\x52\xf2\xf6\xd3\xff\x62\xee\xc2\xb3\x39\xb8\xb7\x96\x37\x9f\x7c\x94\xfd\x3b\xb7\xb3\xe0\xd2\xab\x59\xf8\xe5\x1b\xb0\x0d\x03\x61\xc6\x39\xef\x5b\xdf\x67\xe8\x84\x29\xe4\x94\x94\x53\x7d\xec\xf1\x6e\xac\x5b\x84\x6a\x43\xba\x33\x01\xa5\x5f\xea\xe4\xf5\x27\x71\x48\xd2\x56\xfd\x08\xb1\x41\x9d\xdf\x11\xe0\xea\x79\x1d\x43\x39\x8b\x01\x5b\x28\x86\x6e\x01\x52\x8a\x41\x4a\xe3\x48\x46\xcd\x39\x99\x21\xe3\xa7\x52\xbf\x75\xbd\x36\xf8\xed\x75\x07\x58\x7e\xdf\x4f\x39\xfd\x27\x7f\x26\xab\xb8\x0c\x61\x7b\x8a\x47\xba\x35\xd7\x4c\x64\x20\xcc\x18\xe9\xd2\x43\xd0\xa8\x54\x2a\x45\xf3\xe1\x03\x69\x5f\xa5\x52\xa9\x40\xbf\x14\xde\x80\x2a\x9e\x7e\x75\x32\xfa\x83\x6e\x33\x7e\xea\x31\x4c\x9c\x3e\x8b\xcf\x96\xbd\xe7\xff\x66\xd7\xe6\x0d\xfc\xec\x6b\x57\x70\x70\xf7\x4e\x5f\xaf\x6d\x3a\x7c\x80\x09\xc7\x9d\xc4\xd0\x9a\xc9\xa4\x2c\x8b\x44\x4e\x1e\xc7\x5d\x70\xa5\xb3\x76\x5a\x29\x25\x48\x2e\x44\x7d\x52\x46\xaa\x47\xa1\x6e\x69\xef\x83\xdf\xea\xd1\x37\x12\x30\xa2\x76\xd4\xa5\xed\xa0\x53\x84\x0d\x4f\x45\x0a\x8a\x3f\x3b\x67\x4c\x04\x42\x96\x23\x55\xab\x25\x62\xd1\xfe\x36\xb1\xa9\x18\x37\x91\x8b\x7f\xf7\x2f\x66\x5d\x7e\x03\x99\x4a\x09\x56\x69\x59\xd4\x2e\x7b\x8b\xfa\x4d\xab\x31\x0d\xd3\x9d\x4c\x92\xac\xbc\x02\x12\xd9\x39\x5a\x47\xfb\xda\x5b\xb1\x5c\xfd\x35\xca\x90\x8f\x10\x74\xb5\x35\x73\x68\xbb\xbe\x4b\xc0\x30\x4d\xb2\xdd\xfa\x11\xea\x60\xc8\xa8\xb1\x50\x5e\x05\x90\x5f\x58\xc0\x29\xe7\x5c\xa0\xdd\xcf\x4a\x0d\xb0\x6d\xed\x2a\xcd\x68\x21\xdc\xa0\x04\x4b\x06\x59\xea\xfa\x53\x29\x06\x52\x29\x8d\x72\x55\x09\x38\x3d\x65\xb2\xca\x72\xd5\xf4\xc5\x61\x73\x65\x10\xb6\x64\x87\xfa\x63\x44\xdb\x99\x43\x45\x21\x04\x8a\x7e\xa8\x48\xd5\x86\x2e\x4d\xeb\x52\xb5\xf0\xcf\x84\xf0\xde\x07\x60\xc7\x90\x0c\x19\x3d\x96\x45\xb7\xff\x96\x4b\xef\x7b\x82\x71\xf3\x17\xb9\x55\xbc\x20\xb7\xac\x82\x82\xca\xe1\x18\xee\xce\x39\x21\x25\x39\x25\x43\x28\x1c\x31\x46\xa7\xf6\x43\x7b\x69\xde\xb9\xc9\x89\x5f\x4a\xa3\x6f\x10\xa6\xc9\xae\x55\x1f\x51\xaf\x84\xd4\x00\x24\x73\xf3\x29\x1b\x31\xda\x4f\x11\x18\x06\x39\x8a\x5a\x0c\xc3\xa0\xa7\xbb\x8b\x55\xcb\x97\xb1\x7d\xe3\x3a\xcc\xd0\xba\xae\x1e\x43\xc7\xd4\x70\xf5\x8f\xef\xa1\x62\x4c\x0d\x29\xcb\x0a\xf4\x55\xc5\xee\xaf\xbe\x0f\x40\xd2\x6d\x12\xba\x64\x1c\x00\xa9\xfe\x7e\xb0\x0c\x7c\x1e\x65\xc7\x64\x78\x96\x86\xd6\x20\x15\xdc\x70\x5a\x40\xaf\xde\xbd\x3a\xdd\x9d\x4d\xd9\xee\xe0\x78\xec\xdf\x97\x6c\x85\xbf\x7f\xd6\x93\xfc\x8c\x58\x8c\xb1\x73\x4f\xa1\x6a\xf2\x4c\x76\x7f\xb6\x9c\x43\x9b\xd6\x50\x39\x75\x36\x43\xc6\x4c\xc4\xab\x6f\x88\x74\xcc\xa0\xd5\xf3\x17\xb1\xff\xd3\x20\xdc\x67\xa0\xbb\x93\x75\x4f\xfd\x95\xf2\x09\x53\xc9\x29\x2c\x71\x02\xdc\xdc\xe7\x19\xa6\x49\xcb\xe1\x83\x7c\xf0\xe8\x5f\x18\x08\xed\x5d\x1e\x31\x61\x32\x43\xdc\x9d\x79\xc2\xb3\xc4\xb9\x9a\x80\xe1\x46\x51\x78\x3b\x1d\x9d\x75\xd9\x60\xe7\xd6\xcd\xfc\xf9\x67\x77\xb0\x6a\xc5\x07\x74\x75\xa4\x3b\x28\xe2\x89\x04\xd5\x53\xa6\x33\xef\xbc\x4b\x98\xb5\xf0\x5c\x8a\xaa\x46\x3a\x0e\x03\xa9\xb0\x62\x5b\xa5\xbc\x74\x6f\x90\x1e\x60\xa1\xcb\x2f\xe1\x65\x4e\x9d\x7c\xe1\xf7\xea\xef\x94\xca\x67\x3a\xf5\xaa\xa5\x5c\xd2\xab\x5a\x46\x67\x49\xf7\xd8\x62\x18\x58\xe7\x3b\xa9\xab\x31\x92\xa0\x4a\x88\x6d\x93\xcc\xc9\x61\xe2\x82\x73\x18\x7f\xca\xd9\xd8\xd2\x91\x3e\x83\x34\xf5\x8e\xf9\xa5\xe6\xf4\x0b\xd9\xf2\xfa\xd3\xd4\x6f\x59\xeb\x77\xa4\x76\xe9\x1b\xbc\xfb\xcb\xef\x30\xe7\xba\x5b\x19\x32\x76\x02\x24\x32\xb0\x07\x06\x38\xb8\x6d\x03\xef\xff\xdf\xdd\x69\xe9\x27\x0c\xd3\x64\xde\x85\x57\x92\x99\x9b\xef\x44\x7d\x1a\xb8\xe5\x6c\x85\x96\x7a\xd1\xdb\x56\xeb\xd5\x60\x78\xe5\x89\x47\x59\xfa\x66\xfa\xb6\x9c\x9c\xfc\x02\xa6\x9d\x30\x9f\x93\x2f\xbc\x9c\xc9\x27\x9c\x4c\xb2\xb0\x98\x01\x4b\xd2\x97\xb2\x5d\x69\x17\xe5\x55\x0f\x56\xb7\xbf\x00\xdc\xb0\x89\x32\x0a\xc8\x30\xe8\xe1\x23\xa6\x4a\xc7\x2a\x0b\x56\xd3\x02\x06\xc9\x3d\xbd\x7d\x3e\xd1\xe9\x79\x03\x90\xd5\xc8\x0f\xef\xbd\x88\x00\x57\x29\xd0\x21\x71\x9d\x02\x4e\xe7\x4c\xa1\x06\xa7\x0b\x6c\x6c\x0a\xca\xab\x98\xf7\x8d\x3b\x78\xe3\xa7\x5f\xa7\xc7\x0d\xa1\x91\x52\xb2\x6d\xf1\x8b\xec\x5f\xb5\x9c\x8a\x89\xd3\xc9\x29\x29\xa3\xa7\xa5\x89\xc3\x9b\xd7\x3a\xce\xff\xd0\x31\x73\xe1\xb9\x1c\x7b\xf6\x45\x6e\x62\x13\xe9\x48\xfd\x3e\x05\x3b\x93\x49\x15\x20\xa5\x70\xb8\x54\xbe\x62\xee\x04\x28\x1f\x36\x9c\xb9\x0b\xcf\xe1\xe4\x0b\x2e\xa5\x7a\xea\x74\xcc\xcc\x6c\xfa\x53\x16\xfd\x03\x56\xa0\xd2\x28\xe0\x0e\x78\x6b\xae\xb2\xfe\xaa\xc5\x46\x82\x65\x22\x1a\xdc\x30\x05\x0f\x86\xa9\x4f\xa8\x9e\xbc\xf8\xdd\xb5\xed\x32\x6a\x7d\xfd\x22\x70\xc3\x96\x9b\x48\x21\x27\x62\xa6\xe9\xe2\x7c\xf4\x56\x19\x5b\x19\x00\x35\x6a\xd2\xbb\x6e\xd3\x1b\xcf\xf2\xfe\xef\xef\xa0\xb3\xe9\xbf\xdf\x03\x04\x30\x7e\xee\xc9\x7c\xf5\xde\x07\xa8\x1c\x55\x4d\x4c\x4a\x57\x26\x10\xbe\xcc\xe0\xd9\xd6\xbd\xaa\x61\x6a\xa1\xca\xfa\x43\x07\xf9\xf7\x03\x7f\xa4\xa9\xbe\x8e\x9a\xe9\xb3\x38\xee\xb4\x45\x54\x8c\x1a\x83\x34\x4c\xfa\x53\x76\x60\x65\x72\xad\x4f\x03\xae\x2f\x77\x40\xa1\x5c\x0d\xdc\xd0\xda\xe9\xa9\x74\x9a\xb0\xa7\x1a\x3b\x22\xd8\x75\x24\xb0\x2a\xb8\x80\xf8\xfe\xda\x76\x69\xa8\x40\xf2\x5f\x80\xab\xfc\xce\x1b\x80\x48\x21\x27\x02\x70\xaf\x23\x69\x09\x3e\x65\xd8\x1b\x12\xb0\xaa\x74\x69\x57\xb0\x6f\xd5\x72\x56\xfc\xed\x5e\xf6\xad\x5e\xf1\x85\x85\xb9\x32\x92\x39\xcc\x3a\xe7\x12\xce\xbd\xe5\x76\x86\x54\x0d\xc7\x94\xb6\x56\x0e\xce\x17\xfc\x0c\xa1\xed\x67\xd2\x0b\x57\x0b\x6c\x5b\x92\xb2\x2d\x44\x2c\x4e\xca\x92\x0c\xb8\x02\x94\x0a\x66\xe0\x0c\x08\xc0\x1d\xf0\x77\x02\xea\xae\xbd\x30\xd5\xaa\x00\x46\x51\xee\x60\x6b\xb1\x06\xa8\x42\x70\x86\x80\x98\x6a\x7b\x35\x45\xa0\x0a\x85\x3f\x8b\x02\x37\xbc\x43\x2f\xcc\x22\xa2\xa8\xda\x61\xcf\xc2\xd7\x95\x3d\x4a\x75\x1a\x27\xb0\x7c\x86\x0e\x16\x5e\xf6\x8d\x20\x36\xd2\xd3\xf7\x46\xcd\x99\x4f\xf9\xb8\xc9\x6c\x5f\xfa\x06\x9b\xdf\x7a\x81\xc3\x9b\xd6\xd0\xd3\xd6\x82\xb4\x2d\x84\x61\x10\xcb\xc8\x24\xaf\xb4\x82\x51\x33\xe6\x70\xec\xb9\x97\x31\xfe\xb8\x13\xc9\xca\xcc\xc4\xb6\x6c\x0c\x83\x40\xff\xc5\x59\x7f\x03\x16\x1d\x78\x84\xb4\x67\xfa\x54\x67\x60\x0d\x58\x3e\x58\x03\xea\x1a\x1b\x02\x77\x40\x65\xcf\x76\xb8\x44\x50\xba\x31\xe3\x68\x60\x0e\x06\xae\x4a\x4c\xbe\x60\xac\x12\xdf\x8f\xd7\x77\xc8\xb4\x04\xda\x28\xe0\x86\xd7\x65\xef\x62\x11\x66\x63\xba\xee\x1c\x84\xa7\xea\x74\xac\x52\x62\xb4\x93\x3a\xb4\xc7\x55\x7d\x0d\x85\x8a\x22\x04\x86\x61\xd0\xdf\xdd\x45\xcb\xfe\x5d\xb4\xec\xdf\x8d\xd5\xdb\x43\x3c\x23\x93\xbc\x92\x32\x4a\xaa\x86\x53\x58\x56\x41\x46\x22\x8e\x90\x16\x26\xf8\x65\x7d\xbc\x57\x8f\x72\xe3\x42\x75\x85\xea\xc9\x4b\xbd\x67\xaa\xed\xf5\xb6\x6e\xa6\x14\xca\xed\x57\xc0\xed\xb7\xa5\x4f\xdd\xaa\xd4\x6c\x47\xf5\x23\x82\x10\x06\x03\x3c\xca\x57\x10\x05\xae\xef\x09\xbb\x73\x43\x87\xd4\x36\x4b\x0f\xa2\xf7\xaa\x95\x4b\xc2\x12\x74\x3a\xc8\xea\x4e\x7d\xd5\x85\x16\xc4\x24\xc9\x88\x48\x84\xb4\x3d\xae\xfe\x4c\x0f\x3b\xbf\x43\xe6\x55\xe1\x64\xd4\x33\x0c\xc7\x6d\xe7\xb8\xfa\xa4\x7f\xfa\x5b\x48\x5c\x16\x1c\xe8\xee\x5e\xb5\x4e\xe1\xeb\xf1\xe6\x20\xda\x81\xba\x7d\xd3\x33\x33\xfa\x14\xaa\x80\xdb\x6f\xa7\x53\x70\xe0\xca\x0b\xfa\xa4\x02\xa8\x52\x61\x1a\xc8\x83\xa8\x4a\x51\x6c\x39\x5c\xe8\xcb\x04\x62\x4e\x91\x2a\xf1\x85\x82\x94\x4a\xb5\xaa\x8e\xac\xb2\xe8\xa8\x9a\x40\x6a\x43\xbc\xc2\xd0\x8e\x0a\xe2\xa8\x63\x96\x6f\x5b\x0b\x52\xe9\x7b\x57\x38\x15\x34\x03\xd5\xc5\x9f\x1c\x3e\x13\x55\x26\x91\xb4\xc1\x52\xe2\xa3\xbc\xf6\x0a\xd2\x6c\xcc\xe9\xc3\x19\x98\x57\x85\x3b\x7a\xe9\x35\x9c\xd4\x62\x58\x0a\x75\x1e\x05\x5c\x5f\xc0\xb2\x03\x89\x39\x2d\x0c\x4a\x6d\x49\x78\x4d\x0b\x81\xad\x52\xed\xd1\xc0\xf5\x26\xa9\x29\x20\xa6\xd6\xf0\x35\x07\xa1\xd8\xc1\xea\xdc\xeb\x54\x2c\xd2\x27\x44\x88\x12\x20\xd0\x7d\x83\x8e\x39\xa8\x4b\xe1\x64\x9c\xd1\x6b\x24\x39\xe9\x8b\x02\x43\x89\x0b\xb4\xbb\x7e\xaa\xe3\xa2\x59\xde\x08\xc9\x0c\x8a\xd0\xa4\x2d\x23\xfe\x00\x4a\xdf\x08\x63\xbb\x0f\xf0\x06\xd1\x03\xc4\x0a\x71\x9a\x81\xa8\x35\xd7\x63\xcf\x1e\xb8\x11\xec\x59\xa6\x4d\xe3\x80\x30\x50\x28\x53\xa5\xde\x28\xfc\x55\xee\x18\x06\x57\xdd\x0f\x15\xf3\x33\xbc\x08\xbd\x12\xa7\x07\x5a\xda\xfa\xea\x01\x3b\x08\xd5\x6a\xef\x15\x93\xa7\x3a\x1b\xb5\x3a\x80\xae\xd0\x65\xba\x1d\x37\x10\x81\xf0\x21\x1c\x4a\xf7\xe2\xae\xc3\xbe\xde\xf0\xac\xf6\x26\x63\x78\x0f\x90\x49\x7a\x21\x4a\xb5\x5d\x61\x4b\xb6\x5a\x91\xed\xa8\x6b\xaf\x0b\x6e\x4a\x95\x9e\xfd\x35\x59\x31\x47\xda\x81\x87\x27\x1d\x5c\x1c\x1f\xb4\x5b\xbb\x31\x5c\x04\x33\xea\x50\x39\xe3\x60\x94\xeb\x17\x94\x56\x55\x03\x55\x3d\x08\x83\x2a\x08\xff\x9d\x5e\x64\x2a\x52\xc2\x0e\xad\x65\x0e\x07\x74\x87\xd4\x0b\x12\x50\x06\xd5\xdf\xbd\x0e\x7e\x72\x70\xdb\xbd\x46\x05\x3e\xbd\xc3\x3a\x07\x09\x00\x0e\x95\x90\x55\xfa\xe8\xb7\x2d\x44\x35\x42\x04\x6d\x55\x77\x14\x58\x0a\xb8\x29\x25\x74\xc6\x8b\xac\x50\xfd\xb7\xaa\xa1\x43\x35\x66\x84\x01\x0e\xea\x16\x4a\xd7\x6a\x16\xce\xdb\xf7\xdf\x81\x6b\xb8\x93\xd8\x0c\xd5\x85\x8c\x79\x42\x47\x58\x62\x8e\x02\xd8\x9b\xed\x51\xdf\x85\xc1\x0d\xb3\x69\x95\x82\xfd\x0c\x30\xde\x29\x95\xdf\x85\x66\xb0\xf7\x1b\xe9\x82\x88\x0c\xd2\x6d\x0a\x97\x74\x07\x9b\x64\xe1\x2c\x3b\x51\x6d\x1b\x74\x5b\x89\x8c\x02\x59\xf7\xec\x04\xd2\xbf\x27\x20\xca\xb4\xcf\x75\xf6\x1c\xb1\xf6\x8a\x80\x5b\x38\x11\xaa\x6e\xba\x89\xd0\x12\x34\x18\xb8\xce\x24\x51\x88\x94\x80\x4d\xc7\x04\xfc\x7f\x12\x0b\x95\xdb\xce\x10\x29\x8f\x00\x00\x00\x19\x74\x45\x58\x74\x43\x6f\x6d\x6d\x65\x6e\x74\x00\x43\x72\x65\x61\x74\x65\x64\x20\x77\x69\x74\x68\x20\x47\x49\x4d\x50\x57\x81\x0e\x17\x00\x00\x00\x25\x74\x45\x58\x74\x64\x61\x74\x65\x3a\x63\x72\x65\x61\x74\x65\x00\x32\x30\x31\x39\x2d\x30\x33\x2d\x31\x31\x54\x31\x30\x3a\x30\x37\x3a\x31\x38\x2d\x30\x35\x3a\x30\x30\x3f\x9b\x66\xad\x00\x00\x00\x25\x74\x45\x58\x74\x64\x61\x74\x65\x3a\x6d\x6f\x64\x69\x66\x79\x00\x32\x30\x31\x39\x2d\x30\x33\x2d\x31\x31\x54\x31\x30\x3a\x30\x37\x3a\x31\x38\x2d\x30\x35\x3a\x30\x30\x4e\xc6\xde\x11\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82"), + modTime: time.Date(2022, 10, 1, 4, 23, 36, 727914200, time.UTC), + content: []byte("\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52\x00\x00\x00\x78\x00\x00\x00\x78\x08\x03\x00\x00\x00\x0e\xba\xc6\xe0\x00\x00\x00\x19\x74\x45\x58\x74\x53\x6f\x66\x74\x77\x61\x72\x65\x00\x77\x77\x77\x2e\x69\x6e\x6b\x73\x63\x61\x70\x65\x2e\x6f\x72\x67\x9b\xee\x3c\x1a\x00\x00\x00\x09\x70\x48\x59\x73\x00\x00\x06\xeb\x00\x00\x06\xeb\x01\x4c\x31\x9e\x4a\x00\x00\x00\x4e\x50\x4c\x54\x45\x47\x70\x4c\x70\xca\xf3\x3f\x79\xad\xb4\xe3\xf9\x40\x79\xae\x3f\x79\xad\x96\xd8\xf6\x77\xb6\xde\x75\xb6\xde\x8a\xd3\xf5\x70\xca\xf2\x3f\x79\xad\x70\xca\xf3\x70\xca\xf2\x70\xca\xf2\xb4\xe4\xfa\xb4\xe3\xf9\xb4\xe3\xf9\x3f\x79\xad\x3f\x79\xad\x9b\xda\xf7\xb4\xe3\xfa\x5d\xab\xd7\x3f\x79\xad\x70\xca\xf2\xb4\xe3\xf9\xb7\xab\x8d\x85\x00\x00\x00\x17\x74\x52\x4e\x53\x00\xef\xe8\xd0\x32\x7d\x51\x17\x09\x29\xb8\xc7\x96\x73\xd4\x6d\x8f\xea\xa6\x5d\x3c\xaf\x9c\x1b\x2f\x88\x80\x00\x00\x04\xc7\x49\x44\x41\x54\x68\xde\xed\x5b\xc9\x96\xa3\x30\x0c\x6c\xcc\xe2\x8d\xcd\x2c\x0f\xf3\xff\x3f\x3a\x40\xba\x5f\x02\xb1\x6c\xc9\xb8\x33\x73\x98\x3a\x93\x54\x24\x97\xe4\x92\x92\x7c\x7d\xfd\xc7\xbf\x0c\x5e\x75\xfd\x9c\xe7\xb9\xc9\xf3\xb9\xef\x26\xfe\x11\x52\x51\xf5\xf9\x7a\x81\xe9\x2b\xf1\xcb\xb4\xd5\x6c\x56\x37\xe6\xe9\x17\x83\xed\xf2\xd5\x03\xd3\xfd\x52\xd8\x9d\x59\x03\x30\xdd\x2f\xd0\x4e\xf9\x8a\x40\x9e\x3a\xe1\xa2\x5f\x91\xe8\xc5\xe7\xc3\xfd\x0e\x5a\x26\xd4\xf2\x4a\x42\x95\x4c\x55\x2b\x11\x89\x34\xd6\xaf\x64\xf4\x7f\x89\x37\x09\x73\xb7\x46\xa1\xfb\xb0\xae\x92\x29\x4c\x9a\x58\x62\x73\xab\xaa\x44\xbe\x46\x23\x17\x9f\x16\x56\x02\x81\xc9\xf5\x16\xe2\x93\x9d\xdf\x23\x9e\x3f\xae\xe8\x9b\xca\x0e\x29\xcb\x98\x3c\xa0\xf9\x3c\x7d\xc0\xa6\x7f\x58\x3c\x31\x79\x5d\x49\x95\xf8\x84\xcf\xfe\x6a\x9a\xd3\x9e\x32\x28\x69\xf3\x16\x47\x65\x52\x0a\x1b\xaa\xe1\xd9\x61\xa3\x79\x9e\xb0\x96\x81\x28\x66\x67\x43\x12\x73\x32\x79\x4d\xb4\x46\x08\x95\x80\x4c\x74\x1d\x1a\x70\x5c\xe1\x26\xd1\xf5\x38\x53\xdf\xa7\x4b\xa4\x6b\x43\x3e\x32\x67\xb2\x4d\x9a\x62\xaa\xe8\x0d\x87\xa7\x68\x5b\xc6\x7b\xc5\x0a\x67\x92\xa6\x14\xda\x9a\x23\x64\xd1\xa5\x68\x1f\x5d\xc4\x87\x4d\x42\x3c\x45\x94\x7e\x9f\xa2\x9a\x78\x84\x20\x3f\x41\x2c\x52\x14\xf2\x41\x6c\xe6\xbe\xab\x26\x29\xf9\x37\x42\x3b\x99\x14\xc4\xfd\x3a\x77\x32\x45\xed\x53\x53\x2d\x45\xa2\x7b\x25\xd6\xe4\xf2\x62\x1c\xcb\x0b\x46\x68\x37\x93\x68\x88\x12\x63\xa9\xd9\xe2\x02\x27\x34\xeb\x8e\xcc\xaa\x17\x10\x03\x5e\x5b\xc4\x0b\xb9\x18\x16\x2f\x0a\xbc\x57\x12\x09\x69\x97\x45\xa3\x9d\xc0\x7e\x91\x36\xb8\xa8\x79\x90\x76\xc3\x88\x0d\x78\x17\x75\x6b\x15\xe2\x72\x2c\x19\x82\x77\x61\x02\xe9\x86\xb7\x1b\x9c\x5b\x6b\xb3\x26\x14\xae\x5e\x70\x28\x71\x56\x69\xef\xb2\x95\xdd\xe1\x0f\xba\x60\x0b\x16\x05\x6a\xde\xd9\x8f\x58\x1d\xc4\x36\xf3\xdc\x6d\xe3\x82\xc7\x80\x72\xb7\x5b\xa6\x45\x66\xbf\x01\xa6\xbb\x5c\x28\x28\x10\xbb\xa1\xdd\x2b\x55\xd6\x06\x98\x69\xbc\xa7\xf6\x05\x98\xea\x43\xd3\xca\x06\x98\x47\x1a\x6f\x89\x99\xb3\xb6\x02\x96\xd6\xfa\x99\x89\xbc\xa7\x7a\x9a\x3c\x23\x5b\x7d\x22\xb6\xd5\x0d\x3d\xbf\x77\x90\x19\x0e\x98\x67\x67\xe2\xec\xd2\xc5\x84\xa6\xf1\x6a\xcc\xea\xc0\x11\xb0\xb5\xed\xb9\xf5\x0c\x34\xde\x93\xa4\x05\xa0\xac\x7d\xc0\xbb\x06\xbc\xa1\x3e\x25\x9a\xc8\x3b\x60\xd6\xac\x9d\x2b\xe0\x0d\x53\x7c\xa2\x19\x47\x0c\xa8\x7b\xd3\x92\x0e\xde\xd7\x64\x13\x2b\xf8\xda\xa8\x3b\x70\xc0\x53\x2e\xe2\x67\x4d\x09\x76\xa3\x94\xa0\x7e\xb9\xdb\xda\xca\xc9\x6b\x33\x81\x09\x58\x97\x63\x71\x01\xc7\x8c\x96\x5b\xd9\x88\xd6\x4d\xfc\x13\xb2\x27\x60\x56\xee\x24\x61\xf7\x32\x3b\x4b\xa9\x01\x78\x6d\x16\xea\x59\xa5\x88\xb5\xf2\xc7\x24\x0d\x12\x7f\xf7\x2f\xa8\x86\x59\x11\x3f\x5c\x3e\x4c\xad\x84\x72\xad\x8e\x6a\x80\x0e\x97\xb2\x44\xb8\x94\xd4\xcf\xae\x84\x2b\x80\x99\xc3\x99\xd6\xb4\x19\xa6\x73\x0f\xd2\x90\xbe\x2a\x30\xd3\x94\x3c\xbf\x8d\x11\x2f\x13\xa2\xa3\x67\xfe\xe4\x9a\x61\xfd\x6b\xe4\xf2\x19\xd2\x75\x41\x18\x52\xa2\xe0\x3e\x66\x09\x1c\x71\x91\x8e\x58\x02\x87\x5c\x62\x47\x94\x68\xb4\xee\xe6\x35\x50\x4f\x58\xd4\x2f\xc0\x4c\x45\x8d\x5b\x5d\x9a\x9a\xe9\xfa\xad\x13\xc4\xe4\xba\x75\x8a\x9a\xe1\xdf\x06\xf3\x75\x47\x86\x26\xd6\x78\x91\xb6\x22\xee\x90\xb3\x2f\xa2\xb6\x26\xf0\x56\x27\x16\x14\x95\xf8\xed\xd3\x67\x3c\x8a\x38\x23\xa6\xda\xa1\xd0\x3a\x32\xd5\x24\x71\x39\x3b\xaf\x8c\x11\x57\xe6\x2e\x27\x81\x29\xa5\xa7\x40\xfd\xfb\x23\xee\x7e\x91\x26\x34\x10\xe9\xf3\x13\xb4\x6b\x42\xb9\x3b\xd7\x40\xe9\xf7\x4f\xd7\x48\x78\x55\xed\xbe\x24\x98\xc0\x37\xbf\x50\x49\x71\xe0\x25\x05\x6e\xb7\xe2\xe9\xf7\xdf\x4e\x86\xd4\xaa\x37\x45\x0a\xc4\x90\x02\xb7\x8f\x70\xcb\x06\x2c\xc8\x96\x52\xbd\x10\x7a\x88\xb2\x54\x7d\x29\xf0\x93\x96\x0b\x21\xd9\x12\xb2\xe8\x92\x94\xe8\x43\x15\xc5\x42\x61\x6e\x68\xcc\xd0\xe3\xc7\xd3\x9a\xb2\x1c\x06\x8e\x0c\xd8\x9f\xd5\xc0\xc3\xad\x7f\x66\xd3\x05\xda\x35\x1e\xe9\xbb\xd6\xa0\x54\xd6\x5b\x7f\x3c\x30\xb3\x3d\x77\xe7\x87\xd9\x6e\x41\xe6\xf6\x44\xcd\x6b\xf0\xc1\x9f\x1b\xcd\xb7\x00\xd1\xe3\x23\x6c\x3e\x0e\xec\xa1\xf5\xc9\xc2\xc8\xea\xea\x38\x3d\x21\x1b\xe5\x79\xac\x46\x6e\x40\x18\x63\xaf\x4d\xbc\xb6\x7e\xb4\x6d\x16\x78\x42\xd2\x77\x3e\x7b\x2f\xe5\xa1\xf7\x0d\xe1\x79\x83\x73\x46\x5a\x7f\x34\xf7\x78\x5f\x3d\x4b\x49\xda\x70\x89\xf6\x16\x71\x13\xb7\xd8\x1b\x3c\xfd\x0b\x85\xb3\x2f\x25\x6c\xd8\x30\xfa\xf2\x25\x5a\xc6\x2e\x6f\x8f\x8b\x4b\x25\x49\x34\x51\xd9\x7b\xb2\x79\x7b\x5b\xd1\x11\xcc\xbb\xb2\x65\x5c\x4d\xb9\x2e\x6e\x82\xc0\xf6\x66\x36\x65\xb7\x85\x45\xfe\xce\xe9\xb1\x20\x89\x60\x56\xd0\x2f\x94\x06\x92\xc0\xc8\xcc\x0a\x74\xa3\x78\xe6\x63\x09\x26\x69\x0a\xab\x7d\x2e\x78\x64\x94\x98\x05\xa1\xaa\xb2\x80\xed\x2f\xb0\x07\xfd\x58\xc0\x55\xd8\x74\xab\xf0\xd2\x02\x17\xf4\xc0\x03\x4b\xc3\x73\xb8\x88\x29\x1a\xf5\x05\xf2\xab\x29\x92\x41\xea\xac\xc6\xae\x45\x79\x89\xa6\xdd\xb7\xd9\x93\x4a\x43\xeb\xff\x6d\x02\x2b\x1d\x16\x50\xd6\xe0\x82\x98\xfe\xbf\xa0\x87\xcb\xba\x7e\x4f\x00\x2e\xa2\xa6\x46\x5d\x85\x96\xa9\x26\xf6\xaf\x50\xc5\x58\x0e\x5a\x6b\xc6\xb4\x1e\xca\xb1\x08\x7c\xf8\xcd\xe2\x35\xb5\x6a\x37\x28\x55\x37\x55\xc2\x7f\x88\x20\x8e\xe8\xeb\x3f\x20\xfc\x01\x6b\xf7\x98\xc9\x61\xe7\x4c\xda\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82"), }, "/rclone-48x48.png": &vfsgen۰FileInfo{ name: "rclone-48x48.png", - modTime: time.Date(2019, 5, 27, 15, 4, 10, 118821186, time.UTC), - content: []byte("\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52\x00\x00\x00\x30\x00\x00\x00\x30\x08\x06\x00\x00\x00\x57\x02\xf9\x87\x00\x00\x00\x04\x67\x41\x4d\x41\x00\x00\xb1\x8f\x0b\xfc\x61\x05\x00\x00\x00\x20\x63\x48\x52\x4d\x00\x00\x7a\x26\x00\x00\x80\x84\x00\x00\xfa\x00\x00\x00\x80\xe8\x00\x00\x75\x30\x00\x00\xea\x60\x00\x00\x3a\x98\x00\x00\x17\x70\x9c\xba\x51\x3c\x00\x00\x00\x06\x62\x4b\x47\x44\x00\xff\x00\xff\x00\xff\xa0\xbd\xa7\x93\x00\x00\x00\x09\x70\x48\x59\x73\x00\x00\x0b\x13\x00\x00\x0b\x13\x01\x00\x9a\x9c\x18\x00\x00\x00\x07\x74\x49\x4d\x45\x07\xe3\x03\x0b\x05\x07\x12\x76\x96\x1b\x39\x00\x00\x15\x39\x49\x44\x41\x54\x68\xde\x6d\x9a\x79\x78\x54\xd5\xdd\xc7\x3f\xe7\xdc\x7b\x67\xcb\x9e\x10\x92\x10\x48\xc2\x8e\x06\x35\xb2\x28\x9b\x68\x5d\x00\x8b\xb8\x61\x71\xa3\xf6\xd5\x56\x5f\xb5\xae\xd5\x5a\x5b\xb5\x5a\xeb\x6e\xab\x52\x95\xd6\x22\xad\x88\x4a\x45\x2d\x95\x4a\x5b\x51\xc4\x6a\x6b\x40\x04\xc1\x40\x58\xc3\x92\x04\x08\x84\x84\xac\xb3\xde\x7b\xce\xfb\xc7\xbd\x77\x66\x68\xdf\x79\x9e\xfb\xcc\x3c\x33\xf3\x9c\x7b\x7e\xdb\xf7\xf7\xfd\x7d\xcf\x15\xaf\xec\xe9\xd7\x29\x0d\x09\x47\x93\x50\x90\x54\x1a\x5b\x81\x06\x04\x20\x04\x00\x28\xed\x7e\x07\x60\x08\xb0\x04\x84\x0c\x41\xd8\x10\x98\xd2\xfd\xde\x51\x90\xd4\xd9\x6b\x68\x24\x02\x43\x82\xf4\xd6\xd3\x64\xd6\x91\x02\x82\x52\xb8\xeb\x48\xb0\xa4\xc0\x10\xa0\x70\xd7\x88\x3b\x10\x75\x34\x71\x47\x93\x54\x1a\xa5\xdd\x35\x4c\x29\x08\x48\x08\x48\x81\x59\x60\x49\x92\x4a\x13\x13\x20\x1d\x77\x61\xad\x35\x29\x0d\x8e\x76\x3f\x6b\xed\x2e\xaa\x75\xc6\x00\x25\x45\x7a\x2b\xa6\x16\x68\xad\x71\x34\xa4\x14\xa4\xb4\x6b\x00\x80\x21\x35\x16\x06\x52\x4a\xd0\xa0\x94\x83\xa3\x14\xda\x5f\xc7\x5b\x53\x20\xd0\x68\x4c\x29\x40\xbb\xf7\xf2\x0d\x05\x90\x08\x84\x70\x8d\xb6\x04\x58\x42\x60\x09\x30\x0b\x2c\x41\x42\x81\x21\x04\xa0\x50\x1a\x6c\x0d\x29\x07\x1c\x6f\x53\xee\xe5\x79\x40\xb8\x8b\xd9\x5a\xa3\xb4\x40\x09\x81\x81\x44\x39\x0e\xb6\xd6\xa4\x54\xe6\xbf\x52\x0a\x64\xd2\xa1\x6d\x7f\x23\x9d\xfb\x77\x23\x0d\x83\xd2\x31\xa7\x91\x57\x51\x85\xd2\x1a\x01\xd8\x52\xa7\x1d\xa3\xb5\xc0\xd2\x1a\xe9\x19\xa6\xbd\x1f\xa4\x67\xac\x6f\xb4\x29\x05\x96\x17\x31\x33\xd7\x14\x98\x5e\xb8\x1d\x2d\x48\xf9\x5e\x14\x9a\x14\xee\xe6\x53\x5a\xe3\x28\xf7\xb3\x1b\x7a\x8d\x92\x06\x4a\x69\x3a\x9b\xf7\xd3\xdd\xba\x8f\xd2\x93\xc7\x61\x44\x72\xb1\xfd\x50\x4b\x81\xd3\xdd\xcd\xc6\xc5\xcf\xb2\x6d\xe5\x9b\xc4\x7b\xba\x40\x08\x0a\x06\x55\x31\xe1\x7f\xee\xe2\xa4\x39\xf3\x31\x4c\x03\x8d\x70\x0d\xf5\x16\x57\xb8\xce\xd4\xf8\x19\xe0\xe6\x9e\xc8\x32\xc4\xf0\x22\x21\x05\x98\x01\x43\xa0\x80\xa0\x84\xa4\x01\x21\x05\x29\xc3\x4f\x03\x81\x2d\xdc\xfc\x51\x7e\x1d\x08\x70\x6c\x9b\x63\x3b\xbf\x61\xff\x9a\x15\xec\xfb\x64\x25\xc9\x58\x94\x99\xbf\x7a\x8b\x92\xda\x09\x38\xb6\x0d\x42\x60\x00\x0d\xcb\x5f\x65\xc3\x92\x05\x94\x0c\x1b\xc3\xf4\xbb\x1e\x05\x0d\x1b\x96\xbe\xc8\xda\x67\xee\xc3\xc8\x2d\x64\xd4\x8c\xcb\x49\x26\xe3\x24\xbb\x7a\x30\x4a\x06\x60\x48\x89\x56\x1a\x43\xb8\xc6\x64\x47\xde\x4f\x27\xed\xa5\xb2\xf2\x2e\x33\xdb\x32\x4b\x40\x40\x42\x48\x82\x2d\x05\x8e\xe1\xa6\x90\x2d\x40\x09\x09\x52\x90\x4a\x24\xf8\xfa\xd5\x27\xd9\xf1\xee\x62\x12\x3d\xc7\x09\x16\x14\x33\xf5\xa7\x2f\x10\x19\x7a\x32\xd1\x94\x8d\xd6\xae\x67\x54\x6f\x17\x4d\x9f\xfc\x15\x21\x0d\xa6\xdc\xf2\x33\x6a\x67\x5f\x89\x06\x02\xf9\x05\xfc\xf5\x9e\xef\xb2\x73\xd5\x32\x46\x9d\x3b\x87\xc3\x0d\x5f\xb1\xf6\x99\x1f\x33\xfa\xac\x0b\x98\x72\xcd\x4d\x94\x54\x0c\x46\x78\xdb\x55\x5e\xfa\xfa\x06\x08\xe1\x45\x44\x6a\x50\x6e\xcd\x48\x3f\xc7\xdd\xd4\x10\x58\x52\x10\xf4\xd0\x25\x6c\xb8\x9f\x43\x96\x49\xfc\x68\x2b\x47\x36\x7e\x8e\xa1\x1d\x4c\xd3\x62\xcc\xc5\xf3\x19\x38\x76\x22\x76\x2c\x8a\x91\x93\x8f\x0e\x46\x48\x39\x5e\xf1\x0b\x49\x2a\x99\x20\xd1\xd7\x45\xa4\x78\x00\x65\x63\x4e\x43\xdb\x29\x84\x63\x53\x50\x56\x89\x19\x0a\x11\xeb\x38\x82\x48\xc5\x29\x2c\x1f\x44\xb2\xbf\x8f\xcf\x16\xfd\x9a\x0f\x5f\xf8\x05\xf1\xa4\x4d\x02\x83\x98\x87\x40\x31\x47\x13\x57\x9a\x84\xd2\x1e\x52\xba\xe8\x14\xf3\x7e\x93\x49\xa5\xdd\x1c\xf7\x8c\x30\xbd\x28\x04\x0d\x17\xde\x22\x96\x81\xd3\xd5\xce\xc6\x57\x9e\x60\xf5\x7d\xf3\x39\xb6\xa5\x9e\xc9\x37\xff\x8c\xe9\xf7\x3e\x45\xdd\xd5\x37\xa3\x52\x09\x0e\x7c\xf2\x3e\xca\x4e\xa1\x85\x20\xd1\x7d\x9c\x86\x57\x9f\x24\x7a\xe4\x20\xe1\xe2\x81\x08\x21\x90\x86\x81\x14\x02\x53\x4a\xba\xf6\xef\x22\x15\x8b\x52\x54\x59\x4d\x4e\x4e\x0e\x3d\xcd\x4d\x44\x3b\xdb\x01\x48\xc6\x63\x44\xe3\x31\xf6\x37\x7c\x4d\x77\x5f\x94\xa8\x23\xd2\x46\xc4\x3c\x38\x8d\x65\x6d\x3e\x63\x80\x72\x91\x47\x7b\xa1\x33\x84\x5b\xe5\x41\x43\xd2\xfa\xd5\xe7\x2c\xbf\x7d\x1e\xdb\x3e\xf8\x13\xf1\xae\x0e\xd6\x2f\x7c\x14\xa7\xab\x9d\x80\xd0\x0c\x3f\x6b\x06\x15\x75\x93\x38\xb8\x6e\x0d\xb1\x23\x2d\x98\xa6\x85\x4e\xc5\xd9\xf7\xc1\x9b\x74\xef\xdd\x4e\xd9\xa9\x93\x88\x75\x75\xd0\xb9\x77\x07\x96\x65\x91\xec\xe9\xa4\x71\xd5\xdb\x48\xc3\x60\xec\xcc\xcb\x90\xa9\x04\x1b\xde\x5c\x88\x9d\x88\x23\xa5\x41\x61\xd5\x70\xda\x5b\xf6\xb3\xfc\xae\x6b\xf8\xe4\xa5\x5f\xd2\xd5\xd7\xe7\x1a\x61\x9f\x68\x44\xdc\x8f\x8a\x03\x66\x52\x71\x02\xf4\x79\xf0\x8d\x69\x48\x3a\x9a\x76\xf2\xee\x43\xb7\x81\x61\x70\xf9\x2f\x5f\xa6\xbf\xab\x93\x63\x07\x9a\x08\x48\xb7\x5e\xf2\x8a\x4a\x38\x75\xee\xf5\x7c\xf4\xe8\xed\xb4\xad\xfb\x98\x41\x53\x2d\x3a\x36\xd7\xa3\x52\x49\x12\x9d\x47\x19\x35\x6b\x2e\x7b\x56\xbd\xc5\x3f\x5f\xf8\x39\x5d\xfb\x77\xd1\xb6\x6d\x13\x4d\x5f\xac\x61\xdc\xa5\xd7\x72\xea\xf9\x73\x68\xfc\x78\x25\x07\xbe\xfa\x17\xb5\x17\x5c\xcc\x9e\xfa\xb5\xe4\x57\x54\x93\x3b\x78\x18\x05\x55\xc3\xd9\xb4\xf4\x45\x52\x4a\x33\xee\xe6\x07\xb1\x42\x11\x84\xd6\x48\x0f\x01\x0d\x01\x8e\x16\x28\xa9\x31\xb3\x23\xe0\x36\x15\xbf\xde\x05\xeb\xff\xb2\x8c\xae\xa3\x87\xb9\x71\xe1\xdb\x0c\x9f\x76\x01\xc9\x94\x4d\xc2\xb6\x51\x48\x1c\x04\x0e\x82\xaa\x09\x67\x51\x50\x59\xc3\xe6\x45\x4f\xd2\xb0\xe4\x79\x62\xc7\x0e\x63\xe5\xe4\x11\xca\xcd\xa3\xfc\xa4\x3a\xce\xbe\xe7\x09\xd6\xfd\xee\x09\xd6\x2e\x78\x84\x40\x38\xc2\x99\xf3\x6e\x60\xc6\xed\x0f\xd0\xdb\xba\x8f\x2f\x5e\x7f\x99\x9a\x71\x93\x18\x3d\xf5\x3c\x76\x7e\xb6\x9a\xbc\x41\x43\xd8\xb1\x7a\x05\x6d\x0d\x1b\x08\xe4\x16\xd0\xf4\xe1\xbb\xd4\x9c\x7f\x39\x65\xa7\x4d\x42\xdb\x29\x94\xc8\xda\x1e\x1a\xad\x84\x1b\x81\xa4\x57\x7c\x7e\xab\x96\x02\x94\x9d\x60\x5f\xc3\x46\xf2\x4b\x4a\xa9\x1e\x53\x4b\x40\x3b\x20\x34\xc2\x90\xd8\x08\xf6\x6c\xf8\x37\xca\x08\xd0\xbe\x6f\x17\x3d\x87\x0e\x60\x45\x72\x19\x30\x6c\x0c\x55\xf3\x6f\xa3\x72\xfc\x34\x8a\x6b\x46\x62\xa1\xa9\xfd\xf6\x3c\x86\x4d\x98\x42\xff\xc1\xfd\xe4\x15\x14\x30\x64\x74\x2d\xdd\x87\x5a\x58\x72\xfb\x55\xb4\xef\x6f\xe2\xda\x67\x17\x73\xbc\xad\x15\x69\x5a\xb4\x7c\xf5\x6f\xbe\x5e\xfe\x2a\xb9\x83\xaa\x39\xe3\xce\xc7\x31\xc3\x39\x18\xc1\x10\x5d\x7b\xb6\x51\x3c\x74\xd4\x09\xd4\xc6\xa7\x24\x66\x52\x69\xe2\x1e\x07\xf2\xdb\xba\x29\xdd\x6e\x6b\x85\x22\x1c\x3f\x7c\x90\xa6\x0d\xff\x62\xe2\x45\xf3\x10\xb8\x5d\xd2\x89\x27\xf8\xfc\x95\xa7\xb1\xf2\x8a\x38\xeb\xd6\x07\xb8\xe0\xa7\xbf\xa6\x7c\xec\x78\x0a\xab\x86\x63\x85\x73\x3c\x1e\xa0\x40\x6b\xa4\x10\x14\x0d\xaa\xa2\x7c\x48\x8d\x97\x7a\x9a\x82\xe2\x12\x4e\x9f\x71\x09\xff\x5e\xfe\x1a\x1d\x2d\xfb\x38\xd6\xbc\x97\x58\x4f\x17\x1b\x96\xbe\x48\x69\xed\x78\xa6\xfe\xe4\x39\x0a\x46\xd4\x22\xa5\x41\xfd\x33\xf7\x70\x78\xe3\xe7\x5c\xb4\xe0\x1d\xf2\xcb\x2a\x41\x6b\x84\xc8\x38\x5a\x26\x1d\x97\xc8\xc5\x1c\x4d\xbf\xa3\xdd\xaa\xb7\x35\x8e\x11\x60\xec\x39\x33\x71\xec\x14\xcb\x9f\xb8\x9f\xf5\x2b\xde\x40\xc7\xa3\x58\x02\xf6\x7c\xbe\x9a\xe6\xaf\xd7\x33\x70\xd8\x28\x2a\x46\x8c\x61\xe2\xbc\x1b\x18\x72\xd2\xa9\xe4\x85\xc3\x84\x71\x08\x0b\x45\xc8\x43\x32\x53\x80\xd4\x0a\x94\x0d\xca\x01\xad\xc9\x29\x28\xe0\xe2\xbb\x7f\xce\x1d\x4b\x3e\xa0\x6e\xd6\x65\x1c\xdd\xbb\x0b\xb4\x62\xc8\xe4\xf3\x38\xf7\x17\xaf\x50\x3c\x72\x2c\x38\x0e\x87\xd7\x7f\x42\xdb\xa6\x7f\xd1\xb1\xab\x81\x96\xcf\xfe\x46\xc8\x94\x84\x0c\xb7\x4f\xb9\xef\x02\xb1\x70\x77\xbf\xf6\xa1\x2a\xa5\x40\x4a\x89\x25\x20\x6c\x19\xc8\x58\x2f\x7f\xfd\xf5\xc3\x7c\xfa\xd6\x22\xd0\x30\x62\xfc\x24\xf2\x4a\x06\xb2\xa3\xfe\x53\x8a\x87\xd4\x70\xf5\x73\x4b\xc9\xaf\xac\x46\x2b\x9d\x6e\xed\x5e\x84\xd3\x5d\xd4\xf1\x19\xa4\x87\x6c\x01\x8f\xc7\x48\x00\xc3\xe4\xf8\xf1\xe3\xfc\xf1\x8e\x6b\xb1\x8a\xcb\x99\xf2\xa3\x27\x30\x8b\xca\xe8\x3d\x72\x90\xed\xef\x2c\xa2\xf1\xdd\x45\xd8\xd1\x7e\xc2\x25\xa5\x14\x94\x0f\x66\xde\x8b\xcb\xc9\x2d\x2c\x42\xa2\xd3\x11\x10\xcf\xef\xec\xd7\x51\xdb\xf5\xbc\x8d\xe0\xe8\x8e\x6f\xd8\xf6\xfe\x52\x8a\x06\x94\x32\xfd\xca\xeb\x29\x2e\x2a\x62\xdb\xda\xbf\xb1\x7e\xe5\xdb\xb4\xec\xd8\x8a\x63\xdb\x0c\xab\x3b\x83\x19\xb7\xfe\x84\x01\x23\x6b\x49\x39\x8e\xb7\x98\x4b\x85\xf1\x5a\xbd\xed\x31\xda\x34\x2b\xf5\xfa\x8b\x25\x5d\x16\x29\x84\x6b\x64\xcc\xd1\x1c\x3e\x78\x90\xb8\x11\xa4\x75\x47\x03\xbd\x1d\xed\x6c\x7b\x67\x11\x47\x36\xd7\x23\xa4\x64\xda\xdd\x8f\x33\xe4\xf4\xc9\xac\xba\xef\x3a\x66\xde\xf3\x18\x75\xb3\xbf\x83\x54\x4e\x9a\x0f\x89\xa7\xb6\xf7\xe9\xa8\xad\x89\x29\xc1\xf1\xc3\xad\x7c\x70\xef\xb5\xb4\x6d\xdd\x08\xc0\xe4\x2b\xbe\xc7\x75\x4f\xbe\x4c\x7e\x4e\x84\x54\x6f\x37\x5d\x6d\x07\x09\x46\x22\x44\x8a\x4a\xc0\x0c\x92\x70\x14\x8e\x57\x38\xe2\x3f\xbc\xef\x92\x42\x7d\x82\x01\x96\xe7\x7d\x53\x80\x10\x02\x47\x7b\xa9\xab\x0c\x0e\x35\xef\x67\xd9\x8d\x73\xe8\x39\xb8\x9f\xa2\x11\x27\x53\x75\xd6\x85\xec\x5a\xb9\x94\xf2\x53\x26\x52\xf7\x9d\xef\xd3\xb8\xf2\x0d\x54\x7f\x0f\xf3\x17\xbc\x49\x24\x1c\xc2\xf4\x48\x9d\x99\x70\x34\x49\x05\x36\x82\x1d\xab\xff\xcc\xb1\xdd\xdb\x38\x79\xce\x35\x74\xec\xde\x4a\xe3\xe7\x1f\xb1\x7e\xe5\x72\x2c\x09\x2d\x0d\x9b\x38\xd6\xb2\x9f\x5b\x9e\x79\x99\xbc\x70\x08\xdb\x51\x58\x42\x60\x7b\x44\x0f\x8f\xaf\x38\x5e\x43\x14\x59\x88\xe6\x0f\x2f\x99\x4b\x78\x40\xed\xcd\x01\xca\x21\x32\xa0\x9c\x8a\xba\x49\xf4\xb6\xb5\x32\xe1\x8e\xc7\x08\x17\x97\xb2\x63\xc5\x6b\x34\xad\x79\x9f\xbc\xb2\x41\x8c\x3c\xef\x62\xd6\x3c\x7e\x37\xad\x5b\xbe\xa4\x76\xea\xb7\xb0\x70\xb0\x84\x40\xda\x1e\x17\x4a\x26\xe2\xb4\x7c\xf9\x29\x15\x75\x93\x39\xff\xe7\x2f\x71\xda\x15\x37\xd0\x7d\xe4\x10\x4b\xee\xfd\x01\x8b\x7e\xf4\x7d\x3e\x5e\xfa\x7b\x94\x63\x13\x0a\x06\x09\x08\x08\x1b\x82\x88\x01\x11\x53\x10\xf2\xf3\x3a\x2b\x85\x4e\x18\x46\x84\xdb\xdd\xa5\xf0\x72\x9f\xac\xff\x69\x8d\xd6\x1a\x23\x10\x64\xf4\xec\xab\x91\x86\xc1\xe1\x4d\xff\x86\x40\x98\x91\x17\x5f\xc7\xf9\xcf\xbd\x4d\xd5\x39\x73\xd8\xbc\xfc\x55\xfa\x3b\xdb\x69\xf8\xc7\x7b\x48\xed\x10\x10\x82\xa0\x04\xd3\xf1\xf8\x76\xb2\xe7\x38\xdd\x2d\x7b\xa9\x99\x36\x0b\xad\x35\x9d\xcd\x7b\x89\x14\x96\x70\xfa\x25\x57\x53\x3d\xe6\x14\xaa\x86\x0f\x67\xd8\x88\x91\xe4\x15\x14\x62\xa0\x91\x12\x94\x16\x48\xaf\xf9\x29\xe5\x4e\x65\x4a\x83\x93\x65\x84\xf0\x36\x2d\xd3\x3c\x5e\xa4\x47\x4b\xe5\x5d\x1a\x50\x8e\x43\xd9\x29\x13\xa9\x18\x3f\x8d\xa6\x0f\xde\x24\xd9\xd7\x43\xde\xa0\x2a\xaa\xa6\xcf\xa6\x73\xeb\x97\x24\xa3\xfd\x8c\xfa\xd6\x6c\xaa\xc7\x4d\x01\x8f\xf1\x9a\xd2\x45\x39\x0c\x29\x89\x1d\x39\x48\xac\xa3\x9d\x9d\xab\x96\xd1\xbe\x63\x33\x5d\xcd\x7b\xc8\x1d\x50\x46\xf5\xc4\xb3\xd0\xd1\x5e\x5a\x77\x6e\xa3\x76\xdc\x19\x99\x4d\x79\x39\xef\x93\x40\xad\xff\x9b\xbf\xfb\xde\x36\xbc\x1b\x99\x59\x85\xee\xe8\xec\x9e\x0f\x12\x4d\x30\x92\xc3\x49\x97\x7c\x97\x75\xbf\x79\x98\xee\xdd\xdf\x50\x3c\x64\x28\xd2\x4e\x52\x51\x3b\x8e\x2b\x7e\xfb\x17\xf2\x73\x22\x14\x84\x83\x20\x32\xf1\x35\x0d\x01\x86\x21\xe9\x6b\x6d\x22\x19\xed\xa3\xb0\xb2\x9a\x9e\xd6\x7d\x38\x89\x38\x9d\x2d\x7b\x59\xfe\xa3\xeb\xb0\x53\x49\xaa\x6b\xeb\x98\x7e\xe9\x95\xe4\x86\x02\xe9\x8e\x9d\xf6\xa2\x87\x3a\xb6\xd2\xae\xf7\xfd\xfc\xf6\xd2\xc6\xf2\x0a\xd8\xf4\xe0\x53\x65\x19\xee\x1b\x60\x08\x81\xa9\x1c\x46\x4c\x9f\x45\xe5\x29\x13\x09\xe6\x17\x62\x85\x22\x80\xcb\x7d\x02\x81\x7c\x4c\x09\x4a\x29\x94\x14\x69\x67\x99\xa6\x74\x09\x52\x69\x65\x15\x67\x5c\xf9\x7d\x26\xce\xbf\x15\x69\x05\x89\x76\x1f\x27\xd6\xdd\x49\xec\x58\x1b\xf5\x4b\x5e\xa4\x7c\xc4\x18\xcc\x70\x0e\xb6\xd2\xe9\x91\x4f\x79\x48\x93\x52\xfe\x2c\x9c\x19\xd2\x7d\x98\x33\xfc\x01\x5c\xba\x11\x40\x78\xfd\x81\x0c\x6a\xb9\x73\x88\x6b\xb6\x15\x0a\x92\x37\xa8\xd2\xed\xfa\xe9\x71\xdf\x4b\x99\xac\xa8\xa7\x94\x46\x08\x81\x78\xaa\xb1\x4f\xa7\xa5\x13\xa5\xd1\x52\xba\xc4\x0e\x81\xc6\x0d\x7d\x6f\xf3\x6e\x72\x83\x16\x83\xab\xaa\x09\x19\x6e\xf1\xf8\x83\x77\x4a\x41\xdc\x1b\x34\x52\x2a\x93\xf7\xd9\x29\x63\x4a\xd7\x08\x4f\x70\xc0\x56\x90\x50\x19\x7a\xec\x32\x62\x9d\x61\xc2\x52\x62\x4a\x81\xd0\xea\x84\x31\x52\x64\xad\xed\xf7\x13\xd3\x90\x3e\x9c\x81\x36\x5c\x6c\xd6\x1a\xd0\xca\x5d\x50\x40\x69\xcd\x08\xc2\xde\x6f\x8e\xd6\xa4\x94\x48\x1b\x60\x6b\x9d\x2e\x44\x1f\x85\xcc\xac\x94\x31\x7c\x23\xfe\x9f\x9a\xc9\x46\x2b\xc3\x9b\xa3\x0d\x29\xe8\x39\xd4\x8c\xa1\x6c\x2a\x87\x8f\x72\x1b\x9e\xf2\xd2\x53\x6b\x1c\xc7\x8d\x9d\x5f\x43\xa6\xbf\xf9\x74\x38\xbd\x8d\x68\xdc\x01\x54\x20\x50\x4a\xe3\x48\x17\x5d\x6c\x05\x42\x6a\xa4\x16\x9e\x92\x91\xf1\x8e\x21\x32\xc3\x50\x36\x65\x70\x0b\x3e\xf3\x7f\x85\x76\x6b\xe6\x3f\x66\x5e\x29\x40\x2a\x9b\xd5\x0b\x1e\xe5\x68\x53\x23\x33\xbf\x7f\x27\x45\x65\x15\xe4\x14\x16\x13\xc8\x2f\x24\x1e\x4f\xb0\xe6\xb5\x97\xa8\xbb\xe8\x4a\x86\x8e\x9f\x82\xa3\x1c\xcc\x6c\xbc\xf6\x2b\x4a\x22\x50\x9e\x26\xa3\x81\x44\x2c\x4a\xb4\xaf\x9b\xa2\x92\x12\x64\x28\x08\x8a\xb4\x72\x90\xb9\xb9\xf0\x54\x33\x77\xf3\x41\x8f\x36\xf8\x68\xe5\xa6\x8e\x40\xe1\x4a\x34\xa9\x74\xfd\x90\xbe\x97\x14\x92\x54\x2c\xca\x91\xa6\xed\xb4\x36\x7e\xc3\x6b\x3f\xbd\x05\x21\x04\x56\x30\x44\x20\x9c\x83\x19\x08\xd2\x71\xf0\x00\xd5\xe3\x26\x53\x3d\x7e\x2a\xb6\x02\x33\x1b\xaf\x85\x90\xd8\xc9\x24\x89\x58\x0c\x19\xce\xc1\x56\x8a\x0d\x7f\x5a\xc4\xf6\x8f\xfe\x42\x5f\x7b\x1b\x73\x7e\xfc\x18\x13\x2e\xbc\x0c\xad\x1c\x2c\x99\xc1\xf3\x13\x1b\xd6\x89\xe2\x53\x36\xbd\xf0\x0b\xdf\xf6\x38\x52\xca\x4f\x41\xed\x47\x5d\x63\x85\x42\x4c\x9d\x77\x3d\x85\xf9\xb9\x14\xe4\x17\xb0\xec\x99\x87\xe9\xeb\xea\xa4\xbc\x66\x18\x2d\xbb\xb6\x13\x8c\xe4\x32\xb0\x7a\x44\xba\xe7\x48\x7f\x30\x50\x5a\xb3\x71\xe5\x9f\x58\x7a\xe7\xb5\x2c\xbe\xf1\x62\x9a\xb7\x6e\x42\x99\x01\x12\xb6\xa2\x64\xf4\x69\x24\x13\x49\xea\xdf\x59\x42\x6f\x5f\x3f\x29\x2d\x48\x29\x7d\x42\xf8\xff\x23\x88\x27\xbc\xb2\xef\xe1\x68\x3f\x7d\x74\x66\x8c\xcd\xba\xfa\xbb\xbb\xa8\x1c\x5d\x8b\x63\x3b\x34\xae\xfb\x8c\xe3\x47\xdb\x38\xfb\xb2\xab\x78\x64\xd9\xdf\x98\x3e\xf7\x1a\xf2\x06\x0c\xa4\x64\xd0\x60\x84\x56\x6e\x41\x67\x6e\x22\xd8\xb7\xf1\x0b\x1a\x3f\x59\xc5\xd0\x49\xe7\x10\x8b\xc5\x69\x5c\xf3\x01\xfd\x5d\x1d\x9c\x7a\xe5\xff\x62\xe5\xe4\xb3\xe9\x8d\x17\xd9\xf3\xf5\x7a\x6a\xa7\x9e\x8b\xd0\x2e\xe2\x0b\x21\xd2\x12\xa0\x40\xa4\xb5\x1c\xc7\x13\xc6\xb2\x91\x27\x6d\xf0\x7f\xf4\x00\xe1\x91\xbb\x78\x6f\x37\xef\xfd\xf4\x66\xf6\xd4\xaf\xc5\xb6\x53\x6e\xdd\x68\x45\xe5\xd0\xe1\x58\x52\xd0\xba\x63\x2b\x65\xd5\xc3\x29\x28\x2a\x22\x25\x40\x09\x57\xbe\x74\x17\x91\x06\x63\x67\xcd\x25\xa7\xb8\x94\xd3\xe7\xdd\xc8\xce\x35\x2b\xf9\xf3\x1d\xf3\xf8\xe6\xcf\x4b\xe8\x89\xc6\x18\x78\xfa\x54\x1c\xdb\x61\xe3\xfb\x6f\x11\x8b\x27\xb2\x0a\x30\xa3\x9f\xfa\xda\x68\x52\xb9\xd0\x98\xf4\xf4\x9c\xa4\xf2\x3d\xce\x09\x9d\xdc\x14\x2e\x52\x99\xc2\x45\x1f\x95\x4c\x60\x98\x26\x67\xcf\xbf\x89\x9b\x9f\x5f\xcc\x9d\x0b\x16\x53\x52\x3e\x88\xbf\x2f\x5d\xc4\x4b\xf7\xde\xcc\xf6\x0d\xf5\x9c\x32\xed\x1c\x22\xe1\xb0\x27\xf0\x82\x69\x08\xbf\xb0\x14\x83\x4f\x9b\x48\xe5\xd8\x71\x6c\xff\xf8\x7d\x46\xcf\xb9\x96\x8e\xc3\xad\x1c\x69\xf8\x8a\x3d\x1f\xaf\xa0\xf5\xf3\xbf\xa3\x1d\x9b\x03\x9b\xd6\xd1\x79\xf4\x30\xa1\xaa\x1a\xa4\x76\xd2\xaa\xb5\xc6\xed\xf0\x4a\xf8\x7d\x58\xa3\xe4\x7f\x13\x38\x7f\xe3\x41\xaf\x3e\x6c\x0d\x5a\x48\x52\x89\x04\x06\x8a\x4b\xef\x7f\x92\x8a\xca\xc1\x0c\x2c\x29\xc2\x4c\xc5\x68\xac\xff\x27\xab\x97\xbd\x46\xd3\xd6\x2d\x8c\xac\x9b\xc0\xb4\x39\x57\x60\xa0\x31\xa5\x9b\x35\xc6\xec\x3b\x1e\x78\x04\xdc\x34\x90\x81\x20\x5a\x1a\x7c\xbd\xe2\x75\x46\x5c\x78\x15\x0e\x82\x03\x6b\x57\xd2\xb6\xf1\x5f\xe4\x0d\x1e\xc6\x99\xb7\x3d\xc2\xc9\x33\x2f\xa7\xb0\xbc\x92\x44\x77\x27\xe1\xdc\xfc\xb4\xf4\xae\xb4\xf6\x9a\x5f\x86\x26\x93\xf5\x9e\x99\xd8\x44\x9a\x52\x1b\x02\x02\x86\xc1\x91\x5d\xdb\x58\xf9\xd4\x4f\xf8\xe8\xe5\x27\xd9\xb4\x6a\x39\x7b\x37\x7e\x81\x29\x04\xd5\xc3\x47\x50\x77\xe6\x64\xf2\x0a\x0a\xa9\x19\x53\xcb\x55\x77\x3f\x40\xc5\xb0\x51\xd8\x8e\xca\xdc\x6b\xe1\xee\x7e\x6d\x6b\x3f\xdc\x82\xe3\xc7\x3b\x59\x76\xdb\x3c\xba\x0f\xb7\x12\xeb\x6c\xc7\x08\xe7\x30\x7c\xee\x0f\x38\xe9\x8a\x1b\x29\xa9\xa8\x44\x76\x1f\xe5\x8b\xe7\x7e\x46\xa2\xe3\x08\xd7\x2d\x58\x4a\xa4\xa0\x08\xc7\xb6\xd3\x5d\xd4\xa7\x0e\x01\xc3\x3d\xbc\x48\x37\x35\x91\x69\x74\x3e\x12\x39\x08\xda\x9a\xf7\xf3\xf2\x6d\xf3\x39\xd6\xbc\x8f\x93\xa7\x9c\x43\x4f\x47\x3b\xbb\xbe\xaa\x47\x08\xb8\xe8\xba\x9b\xb8\xf9\xa1\xc7\xc8\xc9\xcb\xc3\x76\x14\x49\x05\xfd\x29\x87\x98\x43\xfa\xd0\x43\x66\xe3\xb4\xa3\x15\x56\x41\x09\xa3\x66\x7d\x87\xbe\xb6\x16\xf2\x6a\x46\x31\xf1\xb1\x25\x8c\xbe\xfe\x7e\xac\x92\x72\x3a\x9b\x9b\xf8\xf0\xe1\x5b\x69\x58\xf9\x26\xf1\x68\x1f\x5d\xed\x47\x68\x69\xdc\x82\x2b\x4f\x8a\x13\x72\x3f\xe5\xd3\x83\xac\xbc\xf7\xe1\x35\x60\x1a\x84\x2c\x93\xa0\x69\x50\xff\xde\x52\x0e\xef\xd9\xc1\x8d\x4f\x2f\xe4\xce\x85\x6f\xf0\xc0\x92\x15\xdc\xf5\xfc\xef\x29\x1a\x58\xce\xfb\x7f\x58\xc8\xa7\x7f\x59\xee\x9e\xee\x78\x4a\x47\x3a\x92\x22\x43\xd5\xd3\x8c\xd2\x51\x90\x72\x14\x43\xa6\x7f\x9b\xe2\x51\xa7\x50\x52\x37\x95\xd2\x89\xdf\x42\x1b\x26\x9d\xbb\xb7\xf2\xf9\xc3\x37\xd1\xba\x6e\x0d\x08\xc9\x80\xa1\xa3\xd8\xf2\x8f\x15\x2c\xba\xe1\x22\x56\x2f\x78\x94\xbe\xee\x2e\x6c\x2d\xb0\x91\x38\xd2\xc0\x41\xa4\x29\x86\x00\x4c\xc3\xc0\x32\x2d\x54\x32\x41\xcb\xf6\x06\xd6\xbc\xb5\x98\xb6\x3d\xdb\x69\xdb\xbb\x8b\xd1\xe3\x27\x31\xf1\xbc\x59\x84\x0c\x41\x41\x5e\x2e\xb3\xae\xfa\x2e\xb7\x3c\xf2\x34\x86\x69\x51\xff\xd1\xdf\x49\x26\x93\xe9\xb5\x4e\xb8\x34\x98\xb6\x02\xdb\xf7\x98\x06\xdb\x51\x84\x06\x54\x30\xed\xd1\xc5\xc4\x92\x29\x3a\xb6\xd4\x63\xc7\xfa\x69\x7c\xe9\x41\xe2\x47\x5b\x38\xf5\xda\xdb\x68\x7c\xef\x0f\x94\x0c\x1d\x4d\x41\x79\x25\xd1\xae\xe3\x6c\xfb\x78\x25\xb5\x97\x7c\x97\xfc\x50\x0e\x9d\x7b\xb6\xe1\xf4\xf7\x30\xa8\x66\x18\x95\x83\x87\x20\xa5\xc0\x49\xc6\x39\xb0\x73\x37\x8d\xeb\x3e\x63\xe3\xda\xd5\xec\xfe\x66\x13\xb1\xfe\x3e\x6e\x79\xfa\x65\x2e\xbe\xf1\x0e\xec\x54\x92\x70\x28\x84\x44\x63\xa0\x11\x4a\x51\x37\x65\x3a\x03\x2b\x07\x13\x8d\x46\x49\xda\x0e\x98\x46\x9a\xf1\xfa\x6a\x87\xad\xc1\x4c\xa8\xec\xc3\xbd\x0c\x23\x2c\x1e\x59\xcb\xa1\xcd\xeb\xd9\xf0\xc0\x7c\x9c\x78\x0c\xd0\x4c\xba\xe7\x19\xf2\x06\x94\xb1\xf5\xed\x57\x08\xe4\x17\xb1\xe9\xcf\xaf\x53\x31\x76\x1c\xb3\x1f\xfd\x2d\x22\x18\xe6\x1f\x8f\xdf\xcd\x8e\x0f\x57\x90\x8a\x47\x29\xae\xac\xe2\xd2\xbb\x1e\xe4\xdc\xb9\x57\xf3\xc9\x9b\xaf\xf3\xa7\x67\x7f\x41\xac\xaf\x97\x92\xf2\x0a\xa4\x94\x94\x0d\xae\xe6\xb4\x29\x67\x53\x3a\xa4\x06\xad\x14\x68\x85\xd0\x1a\xc3\x34\x91\xa6\x41\x53\xe3\x56\x8e\xb7\x1f\x65\xfa\xa5\x57\x62\x9b\x41\x52\x8e\x22\x95\x4e\x51\xbc\xcf\x1a\x19\xf7\x84\xad\x54\xd6\x11\x92\x21\xc0\x54\x36\xa5\x23\x4e\x62\xd8\x8c\x2b\x48\xf5\x75\x53\x5a\x3b\x81\xa1\xe7\x5f\x46\x5f\x5b\x0b\x86\x69\xd1\xf4\xc5\x1a\x5a\x36\xaf\x63\xc2\xfc\x1f\x92\x57\x39\x94\x4f\x17\x3c\xcc\xe6\x77\xff\xc8\xb0\xa9\xe7\x73\xc1\x5d\x8f\x60\x58\x01\xde\x79\xfc\x27\x1c\x6c\xdc\x4c\x2a\xda\x47\x30\x14\xe2\xc1\x85\x7f\xe4\xb7\xab\xfe\xc9\xf5\xf7\xfd\x9c\xb6\xe6\xfd\x6c\xab\xff\x0c\x03\x10\x68\x7a\x3a\xda\xf9\xdd\x83\x77\xf3\xde\xef\x5e\x60\xd5\x1b\x7f\xe0\xa5\x07\xee\xa6\xa8\xac\x82\x49\x97\x5e\x4d\x9f\xad\xe9\xb3\x15\xfd\x59\x0a\xb5\x2f\x46\x98\x09\xe5\xb3\x42\x9d\xde\xbc\x94\x02\x53\x6b\xac\xdc\x1c\xce\xb8\xf9\x01\x0c\x01\xfb\x3f\xfd\x80\xd8\xc1\xbd\xf4\x1d\x3e\x40\xbc\xb7\x9b\x5d\x6b\x56\x52\x3d\xe9\x5b\xd4\x4c\xff\x36\x3b\xd6\xae\xa2\xf1\x6f\x6f\x53\x3d\xf9\x3c\x66\x3e\xb4\x80\x81\x03\x07\x52\x35\x72\x34\x8b\x7f\x78\x15\xeb\x56\xbe\x43\xf5\xa8\x31\x58\x81\x00\xe3\xa7\x9c\x45\xc5\x90\x2a\xa6\x9e\x37\x93\xb7\x7e\xf3\x2c\xff\x78\xe3\x55\x26\xcf\x98\x4d\x4e\x61\x31\x00\xbb\xb7\x6c\x64\xd5\x6b\xbf\xc3\xb0\x2c\x6a\xc6\x9e\xce\x35\xf7\x3f\x4e\x7e\xf5\x48\xba\x12\xb6\xc7\x62\x33\xc3\x8c\x0f\x16\x32\x7b\xf3\xa6\xcc\x9c\xdb\xe6\x98\x92\x5c\x03\x0a\xf3\x72\x99\xfa\xc3\x87\x98\xfd\xab\x37\x29\x19\x5c\x43\x6f\xcb\x5e\xb4\x72\x30\x83\x21\xea\xae\xbe\x05\x6d\x05\xd9\xb9\x7a\x05\xda\x71\x18\x7d\xe1\x3c\x82\x05\xc5\x38\xa9\x24\xe5\x23\x4e\x22\xb7\xa8\x84\x1d\x5f\xd5\x33\x7c\x4c\x2d\x17\x5d\xf3\x3d\x22\xe1\x10\x42\xd9\x0c\xa9\xaa\x62\xf2\x79\x33\x68\xdc\x50\xcf\xa6\xb5\x1f\x62\x1a\x82\xc2\xd2\x81\x5c\x73\xdf\x23\xe4\x15\x0f\x60\xca\xa5\xd7\x70\xdb\xa2\xf7\x18\x32\xe1\x2c\xba\x92\x8a\x9e\x94\xa6\xd7\xd6\xf4\xd9\x9a\xa8\xed\x9e\xd0\xa4\x23\x60\x67\x09\x53\x86\x10\x69\xc1\xc8\x3d\x74\x75\x75\x9f\x50\x6e\x84\xc2\xb1\x75\xd8\xa9\x24\xc1\x50\x08\x80\x48\x71\x29\x45\xd5\x23\x48\xc5\xfa\xe9\x3d\x7c\x00\x33\x14\xa1\x70\xe8\x28\x94\x76\x9b\x8c\xe3\x38\x68\xad\x39\xde\x7e\x94\xd2\xca\xc1\xdc\x70\xdf\x43\x04\xd0\x48\xad\xb1\x02\x16\xe7\x5f\x3c\x97\x55\xcb\x96\xb2\xe2\xd5\x97\xd8\xf5\xcd\x26\xac\x48\x2e\x17\xdd\x72\x2f\xb7\xff\x76\x19\x79\x95\xd5\x18\x85\x03\xe8\x4d\xda\x24\x55\x66\xe6\xc8\x90\xc2\x4c\x31\xcb\xf4\x34\x85\xdf\x74\xdc\x28\xb8\xba\x8f\x20\xc7\x14\xe4\x19\x82\x1c\xa9\xc8\x0f\x5a\xcc\xb8\xf3\x61\xa6\xfd\xe0\x1e\x82\x39\xb9\xc4\xda\x0f\x11\x0c\x85\xc9\x29\x29\xc3\x49\xc4\x89\x1e\x3d\x84\x34\x4c\xa4\x69\xd2\xbc\x6d\x0b\xbd\x1d\xc7\x88\xe4\x17\x20\xac\xa0\xdb\x3d\x3d\x65\x39\x11\x8b\xd2\xdf\xd3\x4d\x24\x27\x87\xed\x1b\xbf\xe4\xc3\x65\xaf\x93\x48\xda\xa4\x90\x0c\x3d\x63\x3a\x91\xb2\xc1\x44\x93\x8e\x97\xeb\x90\x70\x20\xae\xb2\x73\xdf\x9d\xc3\x6d\xed\xb1\x51\xe9\xcf\xad\xe9\x49\x4a\xa4\x47\xc0\xf4\xb1\xa6\x37\xe4\x84\x87\x54\x71\xe1\xdd\x8f\x30\xe5\xba\xdb\x30\x72\x0b\xc0\xb2\x38\x65\xce\xd5\x34\xaf\x5f\x4b\xfd\xc2\xc7\x50\x7d\xdd\x58\x52\x50\xff\xda\x02\xb4\x72\x98\x30\xeb\x12\x02\x79\x05\xa4\x94\x76\xc9\x9b\x30\xd8\xb9\xad\x81\x5f\xfd\xec\x47\xe4\x15\x15\x33\xf3\xda\x1b\x38\xf3\xa2\xb9\x94\x8e\xa8\x25\x86\x41\x34\x69\x13\xcb\x9a\xb1\x7d\x12\xa8\xb2\x22\x90\xcd\xdf\xc5\xfd\x5b\x7a\xb4\x95\xf5\xec\x81\x3f\x0a\x9a\xc2\x3f\xbd\xcf\xbc\x94\x37\x06\xa6\xbc\x67\x22\x12\x8e\xcb\x32\x93\x29\x87\x2d\xef\xbf\xc1\xba\x3f\x3c\x4f\xf7\xa1\x66\xb4\xd6\x44\x0a\x0a\x99\x32\x77\x3e\x97\xdf\x71\x3f\x03\x8b\x0a\xc9\x31\x5c\x35\x2f\x60\x08\xa2\xbd\xbd\x34\x6c\xfa\x8a\xd2\xaa\x61\xe4\x0f\xaa\xa2\x5f\x09\x7a\x12\x76\x3a\xc7\x13\x59\x93\x9a\x43\xd6\xbc\xe0\x6f\x3e\x4b\x2c\x13\x0f\x37\xf4\xea\x80\x67\x80\x2f\xbe\x5a\x5e\xcb\x4f\x9f\x88\x7b\xcf\x29\xf8\x06\xf8\x0f\x86\x24\x9c\xcc\x33\x15\x0a\xe8\x3a\xd8\xcc\x91\x9d\xdf\x20\x1d\x9b\x41\xc3\x47\x51\x3d\x6a\x0c\xb9\x41\x8b\xb0\x84\x1c\x4f\x82\x74\x1f\x0c\x11\x28\x21\xe9\xb7\x15\x7d\x49\x87\x5e\x5b\xd3\x93\x52\xf4\xdb\xee\x61\x8b\x4f\x41\x4e\x1c\x86\x32\xde\xcf\x28\x13\x60\x46\xcc\x13\xbd\xee\x4b\x21\x3e\x6f\xf1\x8b\xda\x55\x21\x04\x76\x5a\xd4\xd2\x68\x2d\x10\x59\x21\x2e\xab\xaa\x61\x50\xcd\x30\x4f\xf2\xd0\x04\x50\xe9\x67\x05\xd2\x0a\x86\xe7\xd9\x84\x72\x37\xdc\xe7\x6d\xbe\xd7\xd6\xf4\xdb\x2e\xb2\x38\x9e\x98\xe0\xf3\x1e\x7f\x00\x12\x64\x38\x95\xef\x74\x33\x6c\x78\xde\x17\xe2\x84\x67\x10\x7c\x44\xca\xd6\x74\x94\x06\xa1\x34\x8e\x14\x6e\x7e\x4a\xf7\xa0\x4d\xa0\xd1\x02\x04\x0a\xa9\x35\xa6\x06\x0b\x3f\x9a\x22\xc3\x42\x01\xe5\x51\x16\xff\xd8\xd4\x3f\x9b\xf0\x8f\x52\x93\xca\x07\x15\x97\xf3\x1b\x69\x42\x9e\x51\x3e\xdc\x5a\x75\x1f\xf7\xf9\x3f\x8a\x4e\x8b\x9e\xb4\xbd\x7d\x11\x00\x00\x00\x19\x74\x45\x58\x74\x43\x6f\x6d\x6d\x65\x6e\x74\x00\x43\x72\x65\x61\x74\x65\x64\x20\x77\x69\x74\x68\x20\x47\x49\x4d\x50\x57\x81\x0e\x17\x00\x00\x00\x25\x74\x45\x58\x74\x64\x61\x74\x65\x3a\x63\x72\x65\x61\x74\x65\x00\x32\x30\x31\x39\x2d\x30\x33\x2d\x31\x31\x54\x31\x30\x3a\x30\x37\x3a\x31\x38\x2d\x30\x35\x3a\x30\x30\x3f\x9b\x66\xad\x00\x00\x00\x25\x74\x45\x58\x74\x64\x61\x74\x65\x3a\x6d\x6f\x64\x69\x66\x79\x00\x32\x30\x31\x39\x2d\x30\x33\x2d\x31\x31\x54\x31\x30\x3a\x30\x37\x3a\x31\x38\x2d\x30\x35\x3a\x30\x30\x4e\xc6\xde\x11\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82"), + modTime: time.Date(2022, 10, 1, 4, 23, 36, 727914200, time.UTC), + content: []byte("\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52\x00\x00\x00\x30\x00\x00\x00\x30\x08\x03\x00\x00\x00\x60\xdc\x09\xb5\x00\x00\x00\x19\x74\x45\x58\x74\x53\x6f\x66\x74\x77\x61\x72\x65\x00\x77\x77\x77\x2e\x69\x6e\x6b\x73\x63\x61\x70\x65\x2e\x6f\x72\x67\x9b\xee\x3c\x1a\x00\x00\x00\x09\x70\x48\x59\x73\x00\x00\x02\xc4\x00\x00\x02\xc4\x01\x5b\x91\x9d\x0b\x00\x00\x00\x5d\x50\x4c\x54\x45\x47\x70\x4c\x3f\x7a\xad\xb4\xe3\xf9\x89\xd3\xf5\x3f\x79\xad\x9e\xdb\xf7\xb4\xe4\xfa\x84\xc5\xea\x70\xca\xf2\xb4\xe4\xf9\x70\xcb\xf3\x3f\x79\xad\x3f\x79\xad\x70\xcb\xf2\x70\xca\xf3\x3f\x79\xad\x70\xca\xf2\xb5\xe4\xf9\x3f\x79\xad\x70\xca\xf3\x3f\x79\xae\x3f\x79\xad\x3e\x79\xad\x70\xca\xf3\xb4\xe3\xfa\xb4\xe3\xfa\xb4\xe3\xf9\xb4\xe3\xfa\xb4\xe3\xf9\x3f\x79\xad\x70\xca\xf2\x48\xd5\xe6\x94\x00\x00\x00\x1c\x74\x52\x4e\x53\x00\x36\xed\x38\x94\x27\x68\x12\xda\xa4\x53\xee\xac\xa3\x80\x65\xef\x42\x7a\x6a\x4e\xda\xc7\xbc\x95\x8b\xd2\xb7\x4c\xbf\xbe\x53\x00\x00\x01\xd0\x49\x44\x41\x54\x48\xc7\xad\x56\xd9\x96\xab\x20\x10\x74\x41\x11\x15\xf7\xb8\x24\xca\xff\x7f\xe6\xa8\x08\xcd\xd2\xc6\x99\x73\x6f\x3d\x25\x48\xd9\x55\xd5\x8d\x1a\x04\xff\x01\xb4\x4d\xd9\x87\xb1\xa5\xa5\xbf\xda\x1e\xb3\x68\x55\x60\xf1\xf3\xdd\xd9\x6a\x81\x3d\x54\x69\xa3\xd5\x41\xd4\xfe\x6d\xff\x8e\x2f\x8c\x76\xc5\x10\xdd\x1a\xa1\xd1\x8a\x33\xee\x7c\xb0\xf5\x06\x33\xbe\x3f\x31\xb6\x7c\x96\xf9\x63\xfc\x4d\x1e\x0a\xcc\xd4\x49\x78\x41\x09\xda\x41\xaa\x56\x52\xed\xe2\x6b\x44\x0c\x29\x8a\x45\x9b\x22\x82\x13\xaf\xa8\x81\xf9\xba\xf6\x36\x17\xdf\x5f\x72\x62\xbe\x22\x58\xc4\x08\x0b\x4b\xe3\x84\xde\xdc\x05\xef\x44\x58\x13\x89\xce\x93\x84\xe4\x1a\x92\x62\xd3\x08\x1f\x4d\x87\xe5\x66\x62\xbc\x96\x75\xb7\x93\x9e\x5b\xdd\x1e\x8a\xcd\x46\x6d\xb7\x26\x0a\xb8\x68\x72\xd8\xdf\x6d\x2e\x5e\x96\x83\x75\xa1\x8d\x10\xa2\x52\xfb\x6b\x6f\xbf\xac\x90\xc2\xf0\xf5\xe2\x40\x7f\xe9\xdf\xf0\x02\x70\x3c\xe6\x20\x3b\x09\x42\x1e\xa5\x97\x4f\x18\x6c\x07\x54\x16\x10\x22\x73\x05\x8d\xe5\x09\x62\x67\x94\x9e\x0e\xb4\x28\x28\x40\x28\x7a\xa2\xde\x41\xae\xf6\x1f\x25\xb4\x83\x62\xf0\xa7\x45\x8d\x36\x30\x62\x88\xb4\x46\x9e\x0a\xe9\x81\xe3\x97\x32\xb1\x47\x3b\x5a\xc9\xdf\xe2\x8a\x49\x4c\x81\xea\x71\x67\x68\xcf\x0e\xd8\x86\x2a\x6d\xc2\x4a\x52\x62\x3a\x2f\x71\x8b\xa0\x5c\x34\x08\x21\x17\x66\x97\x3c\x82\x2f\x09\xe4\xa2\x92\x46\x67\xa2\x83\x58\x47\x98\xa3\xa6\x89\xa7\x89\xeb\x2e\x51\x4f\xd1\x1e\x2b\x34\x4e\x1d\x33\x18\x84\x49\x31\x92\x06\x1a\xa7\x4d\x6c\x85\x6c\x5d\xa8\xf5\xee\x35\xa4\xf1\xbe\x31\x46\xc3\x3c\x3d\x45\x49\xca\xd7\xd6\xd1\x4c\x00\x85\x57\xbc\x81\xbf\x15\x3a\xde\x14\x46\xc7\x81\x74\x35\xb8\x84\x12\x7c\x3b\xc8\x6f\x8e\x74\x6d\x8a\x12\x8e\xa0\x03\xc4\x21\x14\x21\xca\x30\x86\xa5\xf6\x19\xd3\xfd\xfd\xcf\x2c\x47\x9b\x50\x9b\x59\x9a\x09\x1b\xcf\x32\xa0\x14\x9d\x7c\x65\x55\xa0\x6b\xca\xb1\xb7\x6e\x4d\xca\xb1\x24\x5d\x68\x7c\x77\xf4\x7c\xe2\xbc\xca\xe9\x3f\x7c\xea\xfc\x00\xd9\x83\x7c\xed\x8a\x77\x2b\x47\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82"), }, "/rootDesc.xml.tmpl": &vfsgen۰CompressedFileInfo{ name: "rootDesc.xml.tmpl", - modTime: time.Date(2019, 9, 15, 16, 40, 10, 576918038, time.UTC), - uncompressedSize: 2521, + modTime: time.Date(2022, 10, 1, 4, 23, 36, 728914000, time.UTC), + uncompressedSize: 2586, - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x56\xd1\x6a\xe3\x3a\x10\x7d\xef\x57\x84\x3c\xdd\x0b\x8d\x65\x97\x5e\x08\x46\x55\x29\x31\x17\x02\x75\xb7\x24\x9b\xa5\x6f\x41\x95\x27\x8e\x76\x6d\xc9\x48\x72\x92\x52\xfa\xef\x8b\x64\x3b\xb6\x93\x36\x5b\x36\xb0\x6c\x5e\x2a\xcd\x9c\x73\x34\x3a\xd2\xa8\xc6\xb7\xbb\x3c\x1b\x6c\x40\x69\x2e\xc5\xcd\x30\xf0\xfc\xe1\x2d\xb9\xc0\x4a\x4a\x33\xd8\xe5\x99\xd0\x37\xc3\x52\x89\x50\xb3\x35\xe4\x54\x8f\xca\x42\x14\x23\xa9\xd2\x30\x81\x0d\x67\x30\x0a\x46\xfe\xf0\x62\xe0\x7e\x0e\x1d\x26\x99\xa0\x7d\x8a\x8d\x9c\xa4\x68\x60\x37\xc3\xb5\x31\x45\x88\xd0\x76\xbb\xf5\x34\x30\x8f\x49\xef\x87\x42\x96\x3a\x24\x17\x83\x01\xd6\x05\xb0\x6f\x55\x91\xc4\x91\x71\x4e\xbf\x4b\x45\x02\x8c\xaa\x41\x1d\xe4\x42\x2a\xe2\x63\x54\x0d\x2c\x13\x1d\x50\x71\x55\x46\x4d\xa8\x26\x5f\x5f\x0a\x20\x27\xb6\x19\xc6\x90\x70\x3a\x07\xb5\x01\x15\x06\x18\x75\x58\x95\xcc\x4a\x71\x10\x49\xf6\xf2\x40\x73\x20\xaf\xaf\xde\xff\x9d\xf9\xdb\x1b\x46\xbd\x7c\x53\xbf\x28\x57\x94\x99\x52\x81\x22\x8a\x65\x52\xc0\xe0\x9f\xea\xaf\x27\x55\xfa\xaf\xdd\x58\x07\x71\x4c\x5a\xcc\xee\x89\x75\x4d\x87\x08\xb5\x3c\xd4\xe7\x59\x50\x4d\x95\x09\x64\x11\x68\xa6\x78\x61\xac\x17\x15\x07\xa3\xa3\x44\x07\xef\xea\xed\x01\xbb\x3b\x70\xf3\x32\x7f\x06\x65\xf7\x1c\xb7\x53\xbb\xe5\x6e\xb6\x83\xff\xb8\xea\x26\x5b\x81\x35\x28\x4e\x1b\xbe\x5f\xff\x30\xea\x85\x2b\xe4\x22\x7a\xb0\xcb\xcf\xa4\x34\x91\x3b\x97\xc5\x62\x1a\xd9\x0a\x6c\xa2\x3e\xe5\x4c\xd0\xf0\x69\x19\xdd\x3f\xdc\x4d\xee\x1e\xd1\x71\x34\xfa\x32\x21\x51\x3c\x1f\x05\xde\x7f\x3e\x46\x07\x89\x77\xd1\xf1\xe8\x17\x78\x0d\x2c\x7c\x54\x32\x29\x99\x99\xd0\x82\xe8\x9c\x5f\x46\x93\x38\xf0\x2f\x53\x30\xee\x3a\x4d\xc5\x4a\xda\xbb\x6e\x03\x13\xea\xac\x6f\x42\x76\x9f\x3d\x76\x2b\xf9\xb4\x3c\x4f\xb4\xc7\xaf\x64\x39\x93\xe2\x9e\x6b\x43\xea\xae\x74\x81\x66\xe2\x9a\x2a\x07\x63\xef\x3a\xcf\x69\x0a\xa8\x10\xa9\xed\xaf\x3a\xd6\xc2\xb6\x3c\x31\x6b\x72\x3d\xc6\xa8\x1a\xb5\x99\x35\xf0\x74\x6d\x5c\xaa\x1e\xb6\xb9\x04\x0a\xb3\x26\x63\xdb\x53\x45\x8f\x54\xaa\x8c\x20\x6d\xa8\xe1\xac\xbe\x28\xa3\xeb\xf1\xee\x7a\xec\xb9\xf5\x6d\xb6\x29\x17\x75\xeb\x3d\xab\xf8\xe0\xca\xff\xb0\x7a\x97\x3b\xab\xfc\xe0\xca\xdf\x05\x57\xfe\xc9\x0d\x54\xe3\xf6\x34\x6c\x1f\xd8\x3b\xdd\x3b\x9f\x3a\xd6\x59\xad\x8e\x7c\xfc\x8e\xd5\x80\x70\x22\x85\x01\x61\x22\xae\x80\x19\xa9\x5e\xec\x6b\xd6\x25\x1f\x49\x4e\x13\x27\x78\x28\x34\x4d\x8e\xa4\xf6\x42\xd3\xa4\x23\x33\x9f\x3c\x46\xb6\xab\x1b\x2f\x0e\x59\xde\x2e\xcf\x30\x6a\x50\x2d\x8f\x49\x61\x94\x74\x0f\x02\x62\x26\xc3\xa8\x13\x68\x51\xb0\x01\x61\xe6\xe5\xb3\x8d\x62\xd4\x9d\xed\xbd\x3d\x30\xeb\x4c\xf3\x04\x30\xdb\x53\x31\x15\x34\xad\xfe\x17\xfc\xbe\x7b\x7d\xad\x4f\xdb\xd7\xa7\xfd\x85\xfe\xe5\x9c\x29\xa9\xe5\xca\x78\x4c\xe6\x7b\xf3\x9e\x96\xf1\x7c\xe9\xde\xa8\x19\x30\xe0\x1b\x50\x33\x48\xb9\x36\x8a\x7e\xda\xc6\x77\x85\xa7\xc9\x29\xe9\xcf\xb9\x7a\x42\xe0\xcf\xfa\xbb\x9f\x76\x5e\x80\x42\x81\x06\x61\x0b\x95\xc2\x2d\x87\xd1\x61\xc8\x7d\xe7\x34\xdf\x35\x18\xd9\x8f\x37\xf2\x33\x00\x00\xff\xff\xb4\xae\xdd\x58\xd9\x09\x00\x00"), + compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xcc\x56\xd1\x6a\xe3\x3a\x10\x7d\x2f\xf4\x1f\x42\x9e\xee\x85\xc6\xb2\x4b\x2f\x04\xa3\xaa\x94\x98\x0b\x81\xa6\x5b\x92\xcd\xd2\xb7\xa0\xca\x13\x47\xbb\xb6\x64\x24\x39\x49\x29\xfd\xf7\x45\xb2\x9d\xca\x69\x9a\x16\x02\xcb\xfa\x25\x9a\x33\x73\x8e\x67\x46\x1a\xc5\xf8\x66\x5b\xe4\xbd\x35\x28\xcd\xa5\xb8\xee\x47\x41\xd8\xbf\x21\xe7\x67\x58\x49\x69\x7a\xdb\x22\x17\xfa\xba\x5f\x29\x11\x6b\xb6\x82\x82\xea\x41\x55\x8a\x72\x20\x55\x16\xa7\xb0\xe6\x0c\x06\xd1\x20\xec\x9f\x9f\xf5\xdc\xe3\xc2\xe3\x34\x17\xb4\xcb\xb1\xc8\x71\x8e\x06\x76\xdd\x5f\x19\x53\xc6\x08\x6d\x36\x9b\x40\x03\x0b\x98\x0c\x7e\x29\x64\xb9\x7d\x62\xa3\xb1\x2e\x81\xfd\xa8\x13\x25\x35\x1d\x17\xf4\xa7\x54\x24\xc2\xa8\x5e\xb4\x28\x17\x52\x91\x10\xa3\x7a\xe1\xc8\x68\x9f\x8d\xeb\x64\x5a\x4e\x6d\x7d\x7f\x2e\x81\x1c\x29\x37\x9e\x40\xca\xe9\x0c\xd4\x1a\x54\x1c\x61\xe4\xb1\x1a\x9d\xa5\xe2\x20\xd2\xfc\xf9\x9e\x16\x40\x5e\x5e\x82\xff\x3d\xfb\xf5\x15\xa3\x8e\x7f\x57\x85\xa8\x96\x94\x99\x4a\x81\x22\x8a\xe5\x52\x40\xef\x9f\xfa\x37\x90\x2a\xfb\xd7\x96\xe7\x45\x1c\x60\xcd\xa7\x77\xc4\xb6\x4f\xc7\x08\xbd\x11\x51\x97\x68\x83\x5a\xae\x4c\x21\x4f\x40\x33\xc5\x4b\x63\x3b\x52\x93\x30\x7a\xe7\xf0\x09\x2e\xe7\x4e\x64\xa7\x0a\x07\x54\xc5\x13\x28\x5b\xf8\xe4\xcd\xb4\x75\xfb\x5e\x9f\xf0\x71\xe6\xad\xb7\x89\xd6\xa0\x38\x6d\x15\xc2\xe6\xc1\xa8\x03\x37\xa1\xf3\xe4\xde\x66\x30\x95\xd2\x24\x6e\x83\xe6\xf3\x71\x62\x93\xb0\x8e\x76\xbf\x73\x41\xe3\xc7\x45\x72\x77\x7f\x3b\xba\x7d\x40\x07\xe0\xe4\xdb\x88\x24\x93\xd9\x20\x0a\xfe\x0b\x31\xda\x73\x1c\x0e\x9f\x0c\x3e\x23\x68\x60\xf1\x83\x92\x69\xc5\xcc\x88\x96\x44\x17\xfc\x22\x19\x4d\xa2\xf0\x22\x03\xe3\xce\xd6\x58\x2c\xa5\x3d\xfe\x16\x18\x51\xb7\x07\x2d\x64\x8b\xed\xb0\x3d\xcd\xc7\xc5\x69\xaa\x1d\x7e\xa3\xcb\x99\x14\x77\x5c\x1b\xd2\xce\xaa\x43\x76\x96\x1b\xb4\x02\x8c\x3d\xfc\xbc\xa0\x19\xa0\x52\x64\x76\xe6\x1a\xcc\x8b\xdb\xf0\xd4\xac\xc8\xd5\x10\xa3\x7a\xe5\xb9\x56\xc0\xb3\x95\x71\xbe\x66\xe9\x39\x53\x28\xcd\x8a\x0c\xed\x9c\x95\x5d\x5a\xa5\x72\x82\xb4\xa1\x86\xb3\xe6\xd8\x0c\xae\x86\xdb\xab\x61\xe0\x72\xb0\xde\x5d\xd2\xa8\x93\xf5\x89\x35\x44\x97\xe1\xc7\x45\x38\xe7\x89\x55\x44\x97\xe1\x36\xba\x0c\x3f\xa9\xa3\x36\xbc\xcd\xb1\xe3\x61\x4f\x7a\x77\xbf\x1a\xd0\x7f\x65\x03\x7d\x7c\xd1\x35\x01\xf1\x48\x0a\x03\xc2\x24\x5c\x01\x33\x52\x3d\xdb\xeb\xce\x27\xbf\xd7\x1c\xa7\x4e\x71\x5f\x69\x9c\xbe\xd3\xda\x29\x8d\x53\x5f\x67\x36\x7a\x48\xec\xc4\xb7\x2d\xd9\xa7\x05\xdb\x22\xc7\xa8\x8d\xf2\x88\x4c\x0a\xa3\xa4\xbb\x2d\x10\x33\x39\x46\x1e\xe0\x85\xc1\x1a\x84\x99\x55\x4f\x16\xc6\xc8\xb7\xde\x9a\xbc\xdf\xb2\x53\x7b\x28\x80\xd9\x69\x9b\x50\x41\xb3\xfa\x3f\xe3\x84\x26\x76\xc5\xbe\xde\xc5\x2e\xef\x6f\x6d\x63\xc1\x99\x92\x5a\x2e\x4d\xc0\x64\xb1\xeb\xe1\xe3\x62\x32\x5b\xb8\x4b\x6c\x0a\x0c\xf8\x1a\xd4\x14\x32\xae\x8d\xa2\x5f\xef\xe6\x41\xe5\x71\x7a\x4c\xfb\x8b\xcd\x3d\xa2\xf0\xe7\xdb\xbc\xb3\xfd\x8b\xa1\x54\xa0\x41\xd8\x6c\xa5\x70\xaf\xc4\x68\x1f\xaa\x3f\x90\x76\xdf\x43\x18\xd9\x8f\x3f\xf2\x3b\x00\x00\xff\xff\xa9\x37\x2a\x77\x1a\x0a\x00\x00"), }, } fs["/"].(*vfsgen۰DirInfo).entries = []os.FileInfo{ diff --git a/go.mod b/go.mod index f40f1f11e09a1..3b01b8c189a03 100644 --- a/go.mod +++ b/go.mod @@ -115,6 +115,8 @@ require ( github.com/prometheus/procfs v0.7.3 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect + github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect github.com/spacemonkeygo/monkit/v3 v3.0.17 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect diff --git a/go.sum b/go.sum index 26c74ec7829e7..c79bbbf066cb3 100644 --- a/go.sum +++ b/go.sum @@ -564,6 +564,8 @@ github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9A github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= @@ -572,6 +574,8 @@ github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5J github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 h1:pXY9qYc/MP5zdvqWEUH6SjNiu7VhSjuVFTFiTcphaLU= +github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= From 0c1fb8b2b779faffdba90e95dc88d3f88f6256d2 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 7 Oct 2022 14:13:58 +0100 Subject: [PATCH 303/560] Add YanceyChiew to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index ac192c72ff0a9..d330e6024c0cb 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -644,3 +644,4 @@ put them back in again.` >}} * Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> * Lorenzo Milesi * Isaac Aymerich + * YanceyChiew <35898533+YanceyChiew@users.noreply.github.com> From 9bf78d0373ba33a15f61c1bb2a3fc7c62639aec0 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 5 Oct 2022 10:56:01 +0100 Subject: [PATCH 304/560] local: fix "Failed to read metadata: function not implemented" on old Linux kernels Before this change rclone used statx() to read the metadata for files from the local filesystem when `-M` was in use. Unfortunately statx() was only introduced in kernel 4.11 which was released in April 2017 so there are current systems (eg Centos 7) still on kernel versions which don't support statx(). This patch checks to see if statx() is available and if it isn't, it falls back to using fstatat() which was introduced in Linux 2.6.16 which is guaranteed for all Go versions. See: https://forum.rclone.org/t/metadata-from-linux-local-s3-failed-to-copy-failed-to-read-metadata-from-source-object-function-not-implemented/33233/ --- backend/local/metadata_linux.go | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/backend/local/metadata_linux.go b/backend/local/metadata_linux.go index 17d32c06bfde0..b274148e713c5 100644 --- a/backend/local/metadata_linux.go +++ b/backend/local/metadata_linux.go @@ -5,19 +5,41 @@ package local import ( "fmt" + "sync" "time" "github.com/rclone/rclone/fs" "golang.org/x/sys/unix" ) +var ( + statxCheckOnce sync.Once + readMetadataFromFileFn func(o *Object, m *fs.Metadata) (err error) +) + // Read the metadata from the file into metadata where possible func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) { + statxCheckOnce.Do(func() { + // Check statx() is available as it was only introduced in kernel 4.11 + // If not, fall back to fstatat() which was introduced in 2.6.16 which is guaranteed for all Go versions + var stat unix.Statx_t + if unix.Statx(unix.AT_FDCWD, ".", 0, unix.STATX_ALL, &stat) != unix.ENOSYS { + readMetadataFromFileFn = readMetadataFromFileStatx + } else { + readMetadataFromFileFn = readMetadataFromFileFstatat + } + }) + return readMetadataFromFileFn(o, m) +} + +// Read the metadata from the file into metadata where possible +func readMetadataFromFileStatx(o *Object, m *fs.Metadata) (err error) { flags := unix.AT_SYMLINK_NOFOLLOW if o.fs.opt.FollowSymlinks { flags = 0 } var stat unix.Statx_t + // statx() was added to Linux in kernel 4.11 err = unix.Statx(unix.AT_FDCWD, o.path, flags, (0 | unix.STATX_TYPE | // Want stx_mode & S_IFMT unix.STATX_MODE | // Want stx_mode & ~S_IFMT @@ -45,3 +67,36 @@ func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) { setTime("btime", stat.Btime) return nil } + +// Read the metadata from the file into metadata where possible +func readMetadataFromFileFstatat(o *Object, m *fs.Metadata) (err error) { + flags := unix.AT_SYMLINK_NOFOLLOW + if o.fs.opt.FollowSymlinks { + flags = 0 + } + var stat unix.Stat_t + // fstatat() was added to Linux in kernel 2.6.16 + // Go only supports 2.6.32 or later + err = unix.Fstatat(unix.AT_FDCWD, o.path, &stat, flags) + if err != nil { + return err + } + m.Set("mode", fmt.Sprintf("%0o", stat.Mode)) + m.Set("uid", fmt.Sprintf("%d", stat.Uid)) + m.Set("gid", fmt.Sprintf("%d", stat.Gid)) + if stat.Rdev != 0 { + m.Set("rdev", fmt.Sprintf("%x", stat.Rdev)) + } + setTime := func(key string, t unix.Timespec) { + // The types of t.Sec and t.Nsec vary from int32 to int64 on + // different Linux architectures so we need to cast them to + // int64 here and hence need to quiet the linter about + // unecessary casts. + // + // nolint: unconvert + m.Set(key, time.Unix(int64(t.Sec), int64(t.Nsec)).Format(metadataTimeFormat)) + } + setTime("atime", stat.Atim) + setTime("mtime", stat.Mtim) + return nil +} From 6654b66114521abc0031bbb0e76df43bcba6f137 Mon Sep 17 00:00:00 2001 From: Lesmiscore Date: Mon, 10 Oct 2022 12:18:54 +0900 Subject: [PATCH 305/560] union: propagate SlowHash feature --- backend/union/union.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/union/union.go b/backend/union/union.go index 7e10f06b8082c..fcfe4ac95884b 100644 --- a/backend/union/union.go +++ b/backend/union/union.go @@ -894,18 +894,22 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e WriteMetadata: true, UserMetadata: true, }).Fill(ctx, f) - canMove := true + canMove, slowHash := true, false for _, f := range upstreams { features = features.Mask(ctx, f) // Mask all upstream fs if !operations.CanServerSideMove(f) { canMove = false } + slowHash = slowHash || f.Features().SlowHash } // We can move if all remotes support Move or Copy if canMove { features.Move = f.Move } + // If any of upstreams are SlowHash, propagate it + features.SlowHash = slowHash + // Enable ListR when upstreams either support ListR or is local // But not when all upstreams are local if features.ListR == nil { From cf0bf159abbf8505fab84327c29fe2a24e361ae2 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 8 Oct 2022 11:27:17 +0100 Subject: [PATCH 306/560] s3: try to keep the maximum precision in ModTime with --user-server-modtime Before this change if --user-server-modtime was in use the ModTime could change for an object as we receive it accurate to the nearest ms in listings, but only accurate to the nearest second in HEAD and GET requests. Normally AWS returns the milliseconds as .000 in listings, but if versions are in use it may not. Storj S3 also seems to return milliseconds. This patch tries to keep the maximum precision in the last modified time, so it doesn't update a last modified time with a truncated version if the times were the same to the nearest second. See: https://forum.rclone.org/t/cache-fingerprint-miss-behavior-leading-to-false-positive-stalen-cache/33404/ --- backend/s3/s3.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 6abfd541c4aee..9aba03a3f2b16 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -4507,7 +4507,15 @@ func (o *Object) setMetaData(resp *s3.HeadObjectOutput) { o.lastModified = time.Now() fs.Logf(o, "Failed to read last modified") } else { - o.lastModified = *resp.LastModified + // Try to keep the maximum precision in lastModified. If we read + // it from listings then it may have millisecond precision, but + // if we read it from a HEAD/GET request then it will have + // second precision. + equalToWithinOneSecond := o.lastModified.Truncate(time.Second).Equal((*resp.LastModified).Truncate(time.Second)) + newHasNs := (*resp.LastModified).Nanosecond() != 0 + if !equalToWithinOneSecond || newHasNs { + o.lastModified = *resp.LastModified + } } o.mimeType = aws.StringValue(resp.ContentType) From 3f804224f424d859cefc9722fc2a52ac7483f3fb Mon Sep 17 00:00:00 2001 From: Isaac Aymerich Date: Wed, 12 Oct 2022 13:07:48 +0200 Subject: [PATCH 307/560] rc: validate Daemon option is not set when mounting a volume via RC - fixes #6469 --- cmd/mountlib/rc.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/mountlib/rc.go b/cmd/mountlib/rc.go index b675abda99a65..bde259c88c9cc 100644 --- a/cmd/mountlib/rc.go +++ b/cmd/mountlib/rc.go @@ -97,6 +97,10 @@ func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) { return nil, err } + if mountOpt.Daemon { + return nil, errors.New("Daemon Option not supported over the API") + } + mountType, err := in.GetString("mountType") mountMu.Lock() From 5ea9398b63af17d1002950b268873cfa4a281476 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 5 Oct 2022 17:39:24 +0100 Subject: [PATCH 308/560] swift: add --swift-no-large-objects to reduce HEAD requests Supplying the flag --swift-no-large-objects is a promise to the swift backend that there are no dynamic or static large objects stored. Using that knowledge rclone can speed its operations up reducing the number of HEAD requests. See: https://forum.rclone.org/t/handling-or-not-of-large-objects-in-swift/33389/ See: https://forum.rclone.org/t/swift-sync-checksum-calls-head-on-every-object-so-is-very-slow/30322 --- backend/swift/swift.go | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/backend/swift/swift.go b/backend/swift/swift.go index ad4cfb712deed..9ee5c76128d82 100644 --- a/backend/swift/swift.go +++ b/backend/swift/swift.go @@ -63,6 +63,32 @@ Rclone will still chunk files bigger than chunk_size when doing normal copy operations.`, Default: false, Advanced: true, +}, { + Name: "no_large_objects", + Help: strings.ReplaceAll(`Disable support for static and dynamic large objects + +Swift cannot transparently store files bigger than 5 GiB. There are +two schemes for doing that, static or dynamic large objects, and the +API does not allow rclone to determine whether a file is a static or +dynamic large object without doing a HEAD on the object. Since these +need to be treated differently, this means rclone has to issue HEAD +requests for objects for example when reading checksums. + +When |no_large_objects| is set, rclone will assume that there are no +static or dynamic large objects stored. This means it can stop doing +the extra HEAD calls which in turn increases performance greatly +especially when doing a swift to swift transfer with |--checksum| set. + +Setting this option implies |no_chunk| and also that no files will be +uploaded in chunks, so files bigger than 5 GiB will just fail on +upload. + +If you set this option and there *are* static or dynamic large objects, +then this will give incorrect hashes for them. Downloads will succeed, +but other operations such as Remove and Copy will fail. +`, "|", "`"), + Default: false, + Advanced: true, }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, @@ -222,6 +248,7 @@ type Options struct { EndpointType string `config:"endpoint_type"` ChunkSize fs.SizeSuffix `config:"chunk_size"` NoChunk bool `config:"no_chunk"` + NoLargeObjects bool `config:"no_large_objects"` Enc encoder.MultiEncoder `config:"encoding"` } @@ -1100,15 +1127,24 @@ func (o *Object) hasHeader(ctx context.Context, header string) (bool, error) { // isDynamicLargeObject checks for X-Object-Manifest header func (o *Object) isDynamicLargeObject(ctx context.Context) (bool, error) { + if o.fs.opt.NoLargeObjects { + return false, nil + } return o.hasHeader(ctx, "X-Object-Manifest") } // isStaticLargeObjectFile checks for the X-Static-Large-Object header func (o *Object) isStaticLargeObject(ctx context.Context) (bool, error) { + if o.fs.opt.NoLargeObjects { + return false, nil + } return o.hasHeader(ctx, "X-Static-Large-Object") } func (o *Object) isLargeObject(ctx context.Context) (result bool, err error) { + if o.fs.opt.NoLargeObjects { + return false, nil + } result, err = o.hasHeader(ctx, "X-Static-Large-Object") if result { return @@ -1464,7 +1500,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op headers := m.ObjectHeaders() fs.OpenOptionAddHeaders(options, headers) - if size > int64(o.fs.opt.ChunkSize) || (size == -1 && !o.fs.opt.NoChunk) { + if (size > int64(o.fs.opt.ChunkSize) || (size == -1 && !o.fs.opt.NoChunk)) && !o.fs.opt.NoLargeObjects { _, err = o.updateChunks(ctx, in, headers, size, contentType) if err != nil { return err From 90d23139f626bcaa3a7519698773faac86cb9afb Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 11 Oct 2022 17:54:22 +0100 Subject: [PATCH 309/560] s3: drop binary metadata with an ERROR message Before this change, rclone would attempt to upload metadata with binary contents which fail to be uploaded by net/http. This checks the keys and values for validity as http header values before uploading. See: https://forum.rclone.org/t/invalid-metadata-key-names-result-in-a-failure-to-transfer-xattr-results-in-failure-to-upload-net-http-invalid-header-field-value-for-x-amz-meta-samba-pai/33406/ --- backend/s3/s3.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 9aba03a3f2b16..d869efb8d235f 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -57,6 +57,7 @@ import ( "github.com/rclone/rclone/lib/readers" "github.com/rclone/rclone/lib/rest" "github.com/rclone/rclone/lib/version" + "golang.org/x/net/http/httpguts" "golang.org/x/sync/errgroup" ) @@ -5241,6 +5242,20 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op } } + // Check metadata keys and values are valid + for key, value := range req.Metadata { + if !httpguts.ValidHeaderFieldName(key) { + fs.Errorf(o, "Dropping invalid metadata key %q", key) + delete(req.Metadata, key) + } else if value == nil { + fs.Errorf(o, "Dropping nil metadata value for key %q", key) + delete(req.Metadata, key) + } else if !httpguts.ValidHeaderFieldValue(*value) { + fs.Errorf(o, "Dropping invalid metadata value %q for key %q", *value, key) + delete(req.Metadata, key) + } + } + var wantETag string // Multipart upload Etag to check var gotEtag string // Etag we got from the upload var lastModified time.Time // Time we got from the upload From b16e50851ad209371df2d5455900124880464529 Mon Sep 17 00:00:00 2001 From: Manoj Ghosh Date: Thu, 13 Oct 2022 05:04:56 -0700 Subject: [PATCH 310/560] Add a native backend for oracle object storage - fixes #6299 --- README.md | 1 + backend/all/all.go | 1 + backend/oracleobjectstorage/client.go | 158 ++++ backend/oracleobjectstorage/command.go | 228 ++++++ backend/oracleobjectstorage/copy.go | 155 ++++ backend/oracleobjectstorage/object.go | 621 ++++++++++++++++ backend/oracleobjectstorage/options.go | 229 ++++++ .../oracleobjectstorage.go | 691 ++++++++++++++++++ .../oracleobjectstorage_test.go | 33 + .../oracleobjectstorage_unsupported.go | 7 + backend/oracleobjectstorage/waiter.go | 362 +++++++++ docs/content/oracleobjectstorage.md | 515 +++++++++++++ fstest/test_all/config.yaml | 10 + go.mod | 7 +- go.sum | 13 +- 15 files changed, 3022 insertions(+), 9 deletions(-) create mode 100644 backend/oracleobjectstorage/client.go create mode 100644 backend/oracleobjectstorage/command.go create mode 100644 backend/oracleobjectstorage/copy.go create mode 100644 backend/oracleobjectstorage/object.go create mode 100644 backend/oracleobjectstorage/options.go create mode 100644 backend/oracleobjectstorage/oracleobjectstorage.go create mode 100644 backend/oracleobjectstorage/oracleobjectstorage_test.go create mode 100644 backend/oracleobjectstorage/oracleobjectstorage_unsupported.go create mode 100644 backend/oracleobjectstorage/waiter.go create mode 100644 docs/content/oracleobjectstorage.md diff --git a/README.md b/README.md index 16e0be145144f..8c5b1c6fbfbd8 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/) * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/) * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/) + * Oracle Object Storage [:page_facing_up:](https://rclone.org/oracleobjectstorage/) * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud) * pCloud [:page_facing_up:](https://rclone.org/pcloud/) * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/) diff --git a/backend/all/all.go b/backend/all/all.go index f611bfc61a32e..08786222a9732 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -34,6 +34,7 @@ import ( _ "github.com/rclone/rclone/backend/netstorage" _ "github.com/rclone/rclone/backend/onedrive" _ "github.com/rclone/rclone/backend/opendrive" + _ "github.com/rclone/rclone/backend/oracleobjectstorage" _ "github.com/rclone/rclone/backend/pcloud" _ "github.com/rclone/rclone/backend/premiumizeme" _ "github.com/rclone/rclone/backend/putio" diff --git a/backend/oracleobjectstorage/client.go b/backend/oracleobjectstorage/client.go new file mode 100644 index 0000000000000..b5dee80f41667 --- /dev/null +++ b/backend/oracleobjectstorage/client.go @@ -0,0 +1,158 @@ +//go:build !plan9 && !solaris && !js +// +build !plan9,!solaris,!js + +package oracleobjectstorage + +import ( + "context" + "crypto/rsa" + "errors" + "net/http" + "os" + + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/common/auth" + "github.com/oracle/oci-go-sdk/v65/objectstorage" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" +) + +func getConfigurationProvider(opt *Options) (common.ConfigurationProvider, error) { + switch opt.Provider { + case instancePrincipal: + return auth.InstancePrincipalConfigurationProvider() + case userPrincipal: + if opt.ConfigFile != "" && !fileExists(opt.ConfigFile) { + fs.Errorf(userPrincipal, "oci config file doesn't exist at %v", opt.ConfigFile) + } + return common.CustomProfileConfigProvider(opt.ConfigFile, opt.ConfigProfile), nil + case resourcePrincipal: + return auth.ResourcePrincipalConfigurationProvider() + case noAuth: + fs.Infof("client", "using no auth provider") + return getNoAuthConfiguration() + default: + } + return common.DefaultConfigProvider(), nil +} + +func newObjectStorageClient(ctx context.Context, opt *Options) (*objectstorage.ObjectStorageClient, error) { + p, err := getConfigurationProvider(opt) + if err != nil { + return nil, err + } + client, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(p) + if err != nil { + fs.Errorf(opt.Provider, "failed to create object storage client, %v", err) + return nil, err + } + if opt.Region != "" { + client.SetRegion(opt.Region) + } + modifyClient(ctx, opt, &client.BaseClient) + return &client, err +} + +func fileExists(filePath string) bool { + if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) { + return false + } + return true +} + +func modifyClient(ctx context.Context, opt *Options, client *common.BaseClient) { + client.HTTPClient = getHTTPClient(ctx) + if opt.Provider == noAuth { + client.Signer = getNoAuthSigner() + } +} + +// getClient makes http client according to the global options +// this has rclone specific options support like dump headers, body etc. +func getHTTPClient(ctx context.Context) *http.Client { + return fshttp.NewClient(ctx) +} + +var retryErrorCodes = []int{ + 408, // Request Timeout + 429, // Rate exceeded. + 500, // Get occasional 500 Internal Server Error + 503, // Service Unavailable + 504, // Gateway Time-out +} + +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } + // If this is an ocierr object, try and extract more useful information to determine if we should retry + if ociError, ok := err.(common.ServiceError); ok { + // Simple case, check the original embedded error in case it's generically retryable + if fserrors.ShouldRetry(err) { + return true, err + } + // If it is a timeout then we want to retry that + if ociError.GetCode() == "RequestTimeout" { + return true, err + } + } + // Ok, not an oci error, check for generic failure conditions + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +func getNoAuthConfiguration() (common.ConfigurationProvider, error) { + return &noAuthConfigurator{}, nil +} + +func getNoAuthSigner() common.HTTPRequestSigner { + return &noAuthSigner{} +} + +type noAuthConfigurator struct { +} + +type noAuthSigner struct { +} + +func (n *noAuthSigner) Sign(*http.Request) error { + return nil +} + +func (n *noAuthConfigurator) PrivateRSAKey() (*rsa.PrivateKey, error) { + return nil, nil +} + +func (n *noAuthConfigurator) KeyID() (string, error) { + return "", nil +} + +func (n *noAuthConfigurator) TenancyOCID() (string, error) { + return "", nil +} + +func (n *noAuthConfigurator) UserOCID() (string, error) { + return "", nil +} + +func (n *noAuthConfigurator) KeyFingerprint() (string, error) { + return "", nil +} + +func (n *noAuthConfigurator) Region() (string, error) { + return "", nil +} + +func (n *noAuthConfigurator) AuthType() (common.AuthConfig, error) { + return common.AuthConfig{ + AuthType: common.UnknownAuthenticationType, + IsFromConfigFile: false, + OboToken: nil, + }, nil +} + +// Check the interfaces are satisfied +var ( + _ common.ConfigurationProvider = &noAuthConfigurator{} + _ common.HTTPRequestSigner = &noAuthSigner{} +) diff --git a/backend/oracleobjectstorage/command.go b/backend/oracleobjectstorage/command.go new file mode 100644 index 0000000000000..1d5fb63bbab69 --- /dev/null +++ b/backend/oracleobjectstorage/command.go @@ -0,0 +1,228 @@ +//go:build !plan9 && !solaris && !js +// +build !plan9,!solaris,!js + +package oracleobjectstorage + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/objectstorage" + "github.com/rclone/rclone/fs" +) + +// ------------------------------------------------------------ +// Command Interface Implementation +// ------------------------------------------------------------ + +const ( + operationRename = "rename" + operationListMultiPart = "list-multipart-uploads" + operationCleanup = "cleanup" +) + +var commandHelp = []fs.CommandHelp{{ + Name: operationRename, + Short: "change the name of an object", + Long: `This command can be used to rename a object. + +Usage Examples: + + rclone backend rename oss:bucket relative-object-path-under-bucket object-new-name +`, + Opts: nil, +}, { + Name: operationListMultiPart, + Short: "List the unfinished multipart uploads", + Long: `This command lists the unfinished multipart uploads in JSON format. + + rclone backend list-multipart-uploads oss:bucket/path/to/object + +It returns a dictionary of buckets with values as lists of unfinished +multipart uploads. + +You can call it with no bucket in which case it lists all bucket, with +a bucket or with a bucket and path. + + { + "test-bucket": [ + { + "namespace": "test-namespace", + "bucket": "test-bucket", + "object": "600m.bin", + "uploadId": "51dd8114-52a4-b2f2-c42f-5291f05eb3c8", + "timeCreated": "2022-07-29T06:21:16.595Z", + "storageTier": "Standard" + } + ] +`, +}, { + Name: operationCleanup, + Short: "Remove unfinished multipart uploads.", + Long: `This command removes unfinished multipart uploads of age greater than +max-age which defaults to 24 hours. + +Note that you can use -i/--dry-run with this command to see what it +would do. + + rclone backend cleanup oss:bucket/path/to/object + rclone backend cleanup -o max-age=7w oss:bucket/path/to/object + +Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc. +`, + Opts: map[string]string{ + "max-age": "Max age of upload to delete", + }, +}, +} + +/* +Command the backend to run a named command + +The command run is name +args may be used to read arguments from +opts may be used to read optional arguments from + +The result should be capable of being JSON encoded +If it is a string or a []string it will be shown to the user +otherwise it will be JSON encoded and shown to the user like that +*/ +func (f *Fs) Command(ctx context.Context, commandName string, args []string, + opt map[string]string) (result interface{}, err error) { + // fs.Debugf(f, "command %v, args: %v, opts:%v", commandName, args, opt) + switch commandName { + case operationRename: + if len(args) < 2 { + return nil, fmt.Errorf("path to object or its new name to rename is empty") + } + remote := args[0] + newName := args[1] + return f.rename(ctx, remote, newName) + case operationListMultiPart: + return f.listMultipartUploadsAll(ctx) + case operationCleanup: + maxAge := 24 * time.Hour + if opt["max-age"] != "" { + maxAge, err = fs.ParseDuration(opt["max-age"]) + if err != nil { + return nil, fmt.Errorf("bad max-age: %w", err) + } + } + return nil, f.cleanUp(ctx, maxAge) + default: + return nil, fs.ErrorCommandNotFound + } +} + +func (f *Fs) rename(ctx context.Context, remote, newName string) (interface{}, error) { + if remote == "" { + return nil, fmt.Errorf("path to object file cannot be empty") + } + if newName == "" { + return nil, fmt.Errorf("the object's new name cannot be empty") + } + o := &Object{ + fs: f, + remote: remote, + } + bucketName, objectPath := o.split() + err := o.readMetaData(ctx) + if err != nil { + fs.Errorf(f, "failed to read object:%v %v ", objectPath, err) + if strings.HasPrefix(objectPath, bucketName) { + fs.Errorf(f, "warn: ensure object path: %v is relative to bucket:%v and doesn't include the bucket name", + objectPath, bucketName) + } + return nil, fs.ErrorNotAFile + } + details := objectstorage.RenameObjectDetails{ + SourceName: common.String(objectPath), + NewName: common.String(newName), + } + request := objectstorage.RenameObjectRequest{ + NamespaceName: common.String(f.opt.Namespace), + BucketName: common.String(bucketName), + RenameObjectDetails: details, + OpcClientRequestId: nil, + RequestMetadata: common.RequestMetadata{}, + } + var response objectstorage.RenameObjectResponse + err = f.pacer.Call(func() (bool, error) { + response, err = f.srv.RenameObject(ctx, request) + return shouldRetry(ctx, response.HTTPResponse(), err) + }) + if err != nil { + return nil, err + } + fs.Infof(f, "success: renamed object-path: %v to %v", objectPath, newName) + return "renamed successfully", nil +} + +func (f *Fs) listMultipartUploadsAll(ctx context.Context) (uploadsMap map[string][]*objectstorage.MultipartUpload, + err error) { + uploadsMap = make(map[string][]*objectstorage.MultipartUpload) + bucket, directory := f.split("") + if bucket != "" { + uploads, err := f.listMultipartUploads(ctx, bucket, directory) + if err != nil { + return uploadsMap, err + } + uploadsMap[bucket] = uploads + return uploadsMap, nil + } + entries, err := f.listBuckets(ctx) + if err != nil { + return uploadsMap, err + } + for _, entry := range entries { + bucket := entry.Remote() + uploads, listErr := f.listMultipartUploads(ctx, bucket, "") + if listErr != nil { + err = listErr + fs.Errorf(f, "%v", err) + } + uploadsMap[bucket] = uploads + } + return uploadsMap, err +} + +// listMultipartUploads lists all outstanding multipart uploads for (bucket, key) +// +// Note that rather lazily we treat key as a prefix, so it matches +// directories and objects. This could surprise the user if they ask +// for "dir" and it returns "dirKey" +func (f *Fs) listMultipartUploads(ctx context.Context, bucketName, directory string) ( + uploads []*objectstorage.MultipartUpload, err error) { + + uploads = []*objectstorage.MultipartUpload{} + req := objectstorage.ListMultipartUploadsRequest{ + NamespaceName: common.String(f.opt.Namespace), + BucketName: common.String(bucketName), + } + + var response objectstorage.ListMultipartUploadsResponse + for { + err = f.pacer.Call(func() (bool, error) { + response, err = f.srv.ListMultipartUploads(ctx, req) + return shouldRetry(ctx, response.HTTPResponse(), err) + }) + if err != nil { + // fs.Debugf(f, "failed to list multi part uploads %v", err) + return uploads, err + } + for index, item := range response.Items { + if directory != "" && item.Object != nil && !strings.HasPrefix(*item.Object, directory) { + continue + } + uploads = append(uploads, &response.Items[index]) + } + if response.OpcNextPage == nil { + break + } + req.Page = response.OpcNextPage + } + return uploads, nil +} diff --git a/backend/oracleobjectstorage/copy.go b/backend/oracleobjectstorage/copy.go new file mode 100644 index 0000000000000..9ed0b768e80f7 --- /dev/null +++ b/backend/oracleobjectstorage/copy.go @@ -0,0 +1,155 @@ +//go:build !plan9 && !solaris && !js +// +build !plan9,!solaris,!js + +package oracleobjectstorage + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/objectstorage" + "github.com/rclone/rclone/fs" +) + +// ------------------------------------------------------------ +// Implement Copier is an optional interfaces for Fs +//------------------------------------------------------------ + +// Copy src to this remote using server-side copy operations. +// This is stored with the remote path given +// It returns the destination Object and a possible error +// Will only be called if src.Fs().Name() == f.Name() +// If it isn't possible then return fs.ErrorCantCopy +func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + // fs.Debugf(f, "copying %v to %v", src.Remote(), remote) + srcObj, ok := src.(*Object) + if !ok { + // fs.Debugf(src, "Can't copy - not same remote type") + return nil, fs.ErrorCantCopy + } + // Temporary Object under construction + dstObj := &Object{ + fs: f, + remote: remote, + } + err := f.copy(ctx, dstObj, srcObj) + if err != nil { + return nil, err + } + return f.NewObject(ctx, remote) +} + +// copy does a server-side copy from dstObj <- srcObj +// +// If newInfo is nil then the metadata will be copied otherwise it +// will be replaced with newInfo +func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object) (err error) { + srcBucket, srcPath := srcObj.split() + dstBucket, dstPath := dstObj.split() + if dstBucket != srcBucket { + exists, err := f.bucketExists(ctx, dstBucket) + if err != nil { + return err + } + if !exists { + err = f.makeBucket(ctx, dstBucket) + if err != nil { + return err + } + } + } + copyObjectDetails := objectstorage.CopyObjectDetails{ + SourceObjectName: common.String(srcPath), + DestinationRegion: common.String(dstObj.fs.opt.Region), + DestinationNamespace: common.String(dstObj.fs.opt.Namespace), + DestinationBucket: common.String(dstBucket), + DestinationObjectName: common.String(dstPath), + DestinationObjectMetadata: metadataWithOpcPrefix(srcObj.meta), + } + req := objectstorage.CopyObjectRequest{ + NamespaceName: common.String(srcObj.fs.opt.Namespace), + BucketName: common.String(srcBucket), + CopyObjectDetails: copyObjectDetails, + } + var resp objectstorage.CopyObjectResponse + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CopyObject(ctx, req) + return shouldRetry(ctx, resp.HTTPResponse(), err) + }) + if err != nil { + return err + } + workRequestID := resp.OpcWorkRequestId + timeout := time.Duration(f.opt.CopyTimeout) + dstName := dstObj.String() + // https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/copyingobjects.htm + // To enable server side copy object, customers will have to + // grant policy to objectstorage service to manage object-family + // Allow service objectstorage- to manage object-family in tenancy + // Another option to avoid the policy is to download and reupload the file. + // This download upload will work for maximum file size limit of 5GB + err = copyObjectWaitForWorkRequest(ctx, workRequestID, dstName, timeout, f.srv) + if err != nil { + return err + } + return err +} + +func copyObjectWaitForWorkRequest(ctx context.Context, wID *string, entityType string, timeout time.Duration, + client *objectstorage.ObjectStorageClient) error { + + stateConf := &StateChangeConf{ + Pending: []string{ + string(objectstorage.WorkRequestStatusAccepted), + string(objectstorage.WorkRequestStatusInProgress), + string(objectstorage.WorkRequestStatusCanceling), + }, + Target: []string{ + string(objectstorage.WorkRequestSummaryStatusCompleted), + string(objectstorage.WorkRequestSummaryStatusCanceled), + string(objectstorage.WorkRequestStatusFailed), + }, + Refresh: func() (interface{}, string, error) { + getWorkRequestRequest := objectstorage.GetWorkRequestRequest{} + getWorkRequestRequest.WorkRequestId = wID + workRequestResponse, err := client.GetWorkRequest(context.Background(), getWorkRequestRequest) + wr := &workRequestResponse.WorkRequest + return workRequestResponse, string(wr.Status), err + }, + Timeout: timeout, + } + + wrr, e := stateConf.WaitForStateContext(ctx, entityType) + if e != nil { + return fmt.Errorf("work request did not succeed, workId: %s, entity: %s. Message: %s", *wID, entityType, e) + } + + wr := wrr.(objectstorage.GetWorkRequestResponse).WorkRequest + if wr.Status == objectstorage.WorkRequestStatusFailed { + errorMessage, _ := getObjectStorageErrorFromWorkRequest(ctx, wID, client) + return fmt.Errorf("work request did not succeed, workId: %s, entity: %s. Message: %s", *wID, entityType, errorMessage) + } + + return nil +} + +func getObjectStorageErrorFromWorkRequest(ctx context.Context, workRequestID *string, client *objectstorage.ObjectStorageClient) (string, error) { + req := objectstorage.ListWorkRequestErrorsRequest{} + req.WorkRequestId = workRequestID + res, err := client.ListWorkRequestErrors(ctx, req) + + if err != nil { + return "", err + } + + allErrs := make([]string, 0) + for _, errs := range res.Items { + allErrs = append(allErrs, *errs.Message) + } + + errorMessage := strings.Join(allErrs, "\n") + return errorMessage, nil +} diff --git a/backend/oracleobjectstorage/object.go b/backend/oracleobjectstorage/object.go new file mode 100644 index 0000000000000..d9f84792334c0 --- /dev/null +++ b/backend/oracleobjectstorage/object.go @@ -0,0 +1,621 @@ +//go:build !plan9 && !solaris && !js +// +build !plan9,!solaris,!js + +package oracleobjectstorage + +import ( + "context" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/ncw/swift/v2" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/objectstorage" + "github.com/oracle/oci-go-sdk/v65/objectstorage/transfer" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/atexit" +) + +// ------------------------------------------------------------ +// Object Interface Implementation +// ------------------------------------------------------------ + +const ( + metaMtime = "mtime" // the meta key to store mtime in - e.g. X-Amz-Meta-Mtime + metaMD5Hash = "md5chksum" // the meta key to store md5hash in + // StandardTier object storage tier + ociMetaPrefix = "opc-meta-" +) + +var archive = "archive" +var infrequentAccess = "infrequentaccess" +var standard = "standard" + +var storageTierMap = map[string]*string{ + archive: &archive, + infrequentAccess: &infrequentAccess, + standard: &standard, +} + +var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`) + +// Object describes a oci bucket object +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + md5 string // MD5 hash if known + bytes int64 // Size of the object + lastModified time.Time // The modified time of the object if known + meta map[string]string // The object metadata if known - may be nil + mimeType string // Content-Type of the object + + // Metadata as pointers to strings as they often won't be present + storageTier *string // e.g. Standard +} + +// split returns bucket and bucketPath from the object +func (o *Object) split() (bucket, bucketPath string) { + return o.fs.split(o.remote) +} + +// readMetaData gets the metadata if it hasn't already been fetched +func (o *Object) readMetaData(ctx context.Context) (err error) { + fs.Debugf(o, "trying to read metadata %v", o.remote) + if o.meta != nil { + return nil + } + info, err := o.headObject(ctx) + if err != nil { + return err + } + return o.decodeMetaDataHead(info) +} + +// headObject gets the metadata from the object unconditionally +func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObjectResponse, err error) { + bucketName, objectPath := o.split() + req := objectstorage.HeadObjectRequest{ + NamespaceName: common.String(o.fs.opt.Namespace), + BucketName: common.String(bucketName), + ObjectName: common.String(objectPath), + } + var response objectstorage.HeadObjectResponse + err = o.fs.pacer.Call(func() (bool, error) { + var err error + response, err = o.fs.srv.HeadObject(ctx, req) + return shouldRetry(ctx, response.HTTPResponse(), err) + }) + if err != nil { + if svcErr, ok := err.(common.ServiceError); ok { + if svcErr.GetHTTPStatusCode() == http.StatusNotFound { + return nil, fs.ErrorObjectNotFound + } + } + return nil, err + } + o.fs.cache.MarkOK(bucketName) + return &response, err +} + +func (o *Object) decodeMetaDataHead(info *objectstorage.HeadObjectResponse) (err error) { + return o.setMetaData( + info.ContentLength, + info.ContentMd5, + info.ContentType, + info.LastModified, + info.StorageTier, + info.OpcMeta) +} + +func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (err error) { + return o.setMetaData( + info.ContentLength, + info.ContentMd5, + info.ContentType, + info.LastModified, + info.StorageTier, + info.OpcMeta) +} + +func (o *Object) setMetaData( + contentLength *int64, + contentMd5 *string, + contentType *string, + lastModified *common.SDKTime, + storageTier interface{}, + meta map[string]string) error { + + if contentLength != nil { + o.bytes = *contentLength + } + if contentMd5 != nil { + md5, err := o.base64ToMd5(*contentMd5) + if err == nil { + o.md5 = md5 + } + } + o.meta = meta + if o.meta == nil { + o.meta = map[string]string{} + } + // Read MD5 from metadata if present + if md5sumBase64, ok := o.meta[metaMD5Hash]; ok { + md5, err := o.base64ToMd5(md5sumBase64) + if err != nil { + o.md5 = md5 + } + } + if lastModified == nil { + o.lastModified = time.Now() + fs.Logf(o, "Failed to read last modified") + } else { + o.lastModified = lastModified.Time + } + if contentType != nil { + o.mimeType = *contentType + } + if storageTier == nil || storageTier == "" { + o.storageTier = storageTierMap[standard] + } else { + tier := strings.ToLower(fmt.Sprintf("%v", storageTier)) + o.storageTier = storageTierMap[tier] + } + return nil +} + +func (o *Object) base64ToMd5(md5sumBase64 string) (md5 string, err error) { + md5sumBytes, err := base64.StdEncoding.DecodeString(md5sumBase64) + if err != nil { + fs.Debugf(o, "Failed to read md5sum from metadata %q: %v", md5sumBase64, err) + return "", err + } else if len(md5sumBytes) != 16 { + fs.Debugf(o, "failed to read md5sum from metadata %q: wrong length", md5sumBase64) + return "", fmt.Errorf("failed to read md5sum from metadata %q: wrong length", md5sumBase64) + } + return hex.EncodeToString(md5sumBytes), nil +} + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + return o.bytes +} + +// GetTier returns storage class as string +func (o *Object) GetTier() string { + if o.storageTier == nil || *o.storageTier == "" { + return standard + } + return *o.storageTier +} + +// SetTier performs changing storage class +func (o *Object) SetTier(tier string) (err error) { + ctx := context.TODO() + tier = strings.ToLower(tier) + bucketName, bucketPath := o.split() + tierEnum, ok := objectstorage.GetMappingStorageTierEnum(tier) + if !ok { + return fmt.Errorf("not a valid storage tier %v ", tier) + } + + req := objectstorage.UpdateObjectStorageTierRequest{ + NamespaceName: common.String(o.fs.opt.Namespace), + BucketName: common.String(bucketName), + UpdateObjectStorageTierDetails: objectstorage.UpdateObjectStorageTierDetails{ + ObjectName: common.String(bucketPath), + StorageTier: tierEnum, + }, + } + _, err = o.fs.srv.UpdateObjectStorageTier(ctx, req) + if err != nil { + return err + } + o.storageTier = storageTierMap[tier] + return err +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType(ctx context.Context) string { + err := o.readMetaData(ctx) + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return "" + } + return o.mimeType +} + +// Hash returns the MD5 of an object returning a lowercase hex string +func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { + if t != hash.MD5 { + return "", hash.ErrUnsupported + } + // Convert base64 encoded md5 into lower case hex + if o.md5 == "" { + err := o.readMetaData(ctx) + if err != nil { + return "", err + } + } + return o.md5, nil +} + +// ModTime returns the modification time of the object +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned to the http headers +func (o *Object) ModTime(ctx context.Context) (result time.Time) { + if o.fs.ci.UseServerModTime { + return o.lastModified + } + err := o.readMetaData(ctx) + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + // read mtime out of metadata if available + d, ok := o.meta[metaMtime] + if !ok || d == "" { + return o.lastModified + } + modTime, err := swift.FloatStringToTime(d) + if err != nil { + fs.Logf(o, "Failed to read mtime from object: %v", err) + return o.lastModified + } + return modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { + err := o.readMetaData(ctx) + if err != nil { + return err + } + o.meta[metaMtime] = swift.TimeToFloatString(modTime) + _, err = o.fs.Copy(ctx, o, o.remote) + return err +} + +// Storable returns if this object is storable +func (o *Object) Storable() bool { + return true +} + +// Remove an object +func (o *Object) Remove(ctx context.Context) error { + bucketName, bucketPath := o.split() + req := objectstorage.DeleteObjectRequest{ + NamespaceName: common.String(o.fs.opt.Namespace), + BucketName: common.String(bucketName), + ObjectName: common.String(bucketPath), + } + err := o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.DeleteObject(ctx, req) + return shouldRetry(ctx, resp.HTTPResponse(), err) + }) + return err +} + +// Open object file +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { + bucketName, bucketPath := o.split() + req := objectstorage.GetObjectRequest{ + NamespaceName: common.String(o.fs.opt.Namespace), + BucketName: common.String(bucketName), + ObjectName: common.String(bucketPath), + } + o.applyGetObjectOptions(&req, options...) + + var resp objectstorage.GetObjectResponse + err := o.fs.pacer.Call(func() (bool, error) { + var err error + resp, err = o.fs.srv.GetObject(ctx, req) + return shouldRetry(ctx, resp.HTTPResponse(), err) + }) + if err != nil { + return nil, err + } + // read size from ContentLength or ContentRange + bytes := resp.ContentLength + if resp.ContentRange != nil { + var contentRange = *resp.ContentRange + slash := strings.IndexRune(contentRange, '/') + if slash >= 0 { + i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64) + if err == nil { + bytes = &i + } else { + fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err) + } + } else { + fs.Debugf(o, "Failed to find length in %q", contentRange) + } + } + err = o.decodeMetaDataObject(&resp) + if err != nil { + return nil, err + } + o.bytes = *bytes + return resp.HTTPResponse().Body, nil +} + +// Update an object if it has changed +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + bucketName, bucketPath := o.split() + err = o.fs.makeBucket(ctx, bucketName) + if err != nil { + return err + } + + // determine if we like upload single or multipart. + size := src.Size() + multipart := size >= int64(o.fs.opt.UploadCutoff) + + // Set the mtime in the metadata + modTime := src.ModTime(ctx) + metadata := map[string]string{ + metaMtime: swift.TimeToFloatString(modTime), + } + + // read the md5sum if available + // - for non-multipart + // - so we can add a ContentMD5 + // - so we can add the md5sum in the metadata as metaMD5Hash if using SSE/SSE-C + // - for multipart provided checksums aren't disabled + // - so we can add the md5sum in the metadata as metaMD5Hash + var md5sumBase64 string + var md5sumHex string + if !multipart || !o.fs.opt.DisableChecksum { + md5sumHex, err = src.Hash(ctx, hash.MD5) + if err == nil && matchMd5.MatchString(md5sumHex) { + hashBytes, err := hex.DecodeString(md5sumHex) + if err == nil { + md5sumBase64 = base64.StdEncoding.EncodeToString(hashBytes) + if multipart && !o.fs.opt.DisableChecksum { + // Set the md5sum as metadata on the object if + // - a multipart upload + // - the ETag is not an MD5, e.g. when using SSE/SSE-C + // provided checksums aren't disabled + metadata[metaMD5Hash] = md5sumBase64 + } + } + } + } + // Guess the content type + mimeType := fs.MimeType(ctx, src) + + if multipart { + chunkSize := int64(o.fs.opt.ChunkSize) + uploadRequest := transfer.UploadRequest{ + NamespaceName: common.String(o.fs.opt.Namespace), + BucketName: common.String(bucketName), + ObjectName: common.String(bucketPath), + ContentType: common.String(mimeType), + PartSize: common.Int64(chunkSize), + AllowMultipartUploads: common.Bool(true), + AllowParrallelUploads: common.Bool(true), + ObjectStorageClient: o.fs.srv, + EnableMultipartChecksumVerification: common.Bool(!o.fs.opt.DisableChecksum), + NumberOfGoroutines: common.Int(o.fs.opt.UploadConcurrency), + Metadata: metadataWithOpcPrefix(metadata), + } + if o.fs.opt.StorageTier != "" { + storageTier, ok := objectstorage.GetMappingPutObjectStorageTierEnum(o.fs.opt.StorageTier) + if !ok { + return fmt.Errorf("not a valid storage tier: %v", o.fs.opt.StorageTier) + } + uploadRequest.StorageTier = storageTier + } + o.applyMultiPutOptions(&uploadRequest, options...) + uploadStreamRequest := transfer.UploadStreamRequest{ + UploadRequest: uploadRequest, + StreamReader: in, + } + uploadMgr := transfer.NewUploadManager() + var uploadID = "" + + defer atexit.OnError(&err, func() { + if uploadID == "" { + return + } + if o.fs.opt.LeavePartsOnError { + return + } + fs.Debugf(o, "Cancelling multipart upload") + errCancel := o.fs.abortMultiPartUpload( + context.Background(), + bucketName, + bucketPath, + uploadID) + if errCancel != nil { + fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel) + } + })() + + err = o.fs.pacer.Call(func() (bool, error) { + uploadResponse, err := uploadMgr.UploadStream(ctx, uploadStreamRequest) + var httpResponse *http.Response + if err == nil { + if uploadResponse.Type == transfer.MultipartUpload { + if uploadResponse.MultipartUploadResponse != nil { + httpResponse = uploadResponse.MultipartUploadResponse.HTTPResponse() + } + } else { + if uploadResponse.SinglepartUploadResponse != nil { + httpResponse = uploadResponse.SinglepartUploadResponse.HTTPResponse() + } + } + } + if err != nil { + uploadID := "" + if uploadResponse.MultipartUploadResponse != nil && uploadResponse.MultipartUploadResponse.UploadID != nil { + uploadID = *uploadResponse.MultipartUploadResponse.UploadID + fs.Debugf(o, "multipart streaming upload failed, aborting uploadID: %v, may retry", uploadID) + _ = o.fs.abortMultiPartUpload(ctx, bucketName, bucketPath, uploadID) + } + } + return shouldRetry(ctx, httpResponse, err) + }) + if err != nil { + fs.Errorf(o, "multipart streaming upload failed %v", err) + return err + } + } else { + req := objectstorage.PutObjectRequest{ + NamespaceName: common.String(o.fs.opt.Namespace), + BucketName: common.String(bucketName), + ObjectName: common.String(bucketPath), + ContentType: common.String(mimeType), + PutObjectBody: io.NopCloser(in), + OpcMeta: metadata, + } + if size >= 0 { + req.ContentLength = common.Int64(size) + } + if o.fs.opt.StorageTier != "" { + storageTier, ok := objectstorage.GetMappingPutObjectStorageTierEnum(o.fs.opt.StorageTier) + if !ok { + return fmt.Errorf("not a valid storage tier: %v", o.fs.opt.StorageTier) + } + req.StorageTier = storageTier + } + o.applyPutOptions(&req, options...) + err = o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.PutObject(ctx, req) + return shouldRetry(ctx, resp.HTTPResponse(), err) + }) + if err != nil { + fs.Errorf(o, "put object failed %v", err) + return err + } + } + // Read the metadata from the newly created object + o.meta = nil // wipe old metadata + return o.readMetaData(ctx) +} + +func (o *Object) applyPutOptions(req *objectstorage.PutObjectRequest, options ...fs.OpenOption) { + // Apply upload options + for _, option := range options { + key, value := option.Header() + lowerKey := strings.ToLower(key) + switch lowerKey { + case "": + // ignore + case "cache-control": + req.CacheControl = common.String(value) + case "content-disposition": + req.ContentDisposition = common.String(value) + case "content-encoding": + req.ContentEncoding = common.String(value) + case "content-language": + req.ContentLanguage = common.String(value) + case "content-type": + req.ContentType = common.String(value) + default: + if strings.HasPrefix(lowerKey, ociMetaPrefix) { + req.OpcMeta[lowerKey] = value + } else { + fs.Errorf(o, "Don't know how to set key %q on upload", key) + } + } + } +} + +func (o *Object) applyGetObjectOptions(req *objectstorage.GetObjectRequest, options ...fs.OpenOption) { + fs.FixRangeOption(options, o.bytes) + for _, option := range options { + switch option.(type) { + case *fs.RangeOption, *fs.SeekOption: + _, value := option.Header() + req.Range = &value + default: + if option.Mandatory() { + fs.Logf(o, "Unsupported mandatory option: %v", option) + } + } + } + // Apply upload options + for _, option := range options { + key, value := option.Header() + lowerKey := strings.ToLower(key) + switch lowerKey { + case "": + // ignore + case "cache-control": + req.HttpResponseCacheControl = common.String(value) + case "content-disposition": + req.HttpResponseContentDisposition = common.String(value) + case "content-encoding": + req.HttpResponseContentEncoding = common.String(value) + case "content-language": + req.HttpResponseContentLanguage = common.String(value) + case "content-type": + req.HttpResponseContentType = common.String(value) + case "range": + // do nothing + default: + fs.Errorf(o, "Don't know how to set key %q on upload", key) + } + } +} + +func (o *Object) applyMultiPutOptions(req *transfer.UploadRequest, options ...fs.OpenOption) { + // Apply upload options + for _, option := range options { + key, value := option.Header() + lowerKey := strings.ToLower(key) + switch lowerKey { + case "": + // ignore + case "content-encoding": + req.ContentEncoding = common.String(value) + case "content-language": + req.ContentLanguage = common.String(value) + case "content-type": + req.ContentType = common.String(value) + default: + if strings.HasPrefix(lowerKey, ociMetaPrefix) { + req.Metadata[lowerKey] = value + } else { + fs.Errorf(o, "Don't know how to set key %q on upload", key) + } + } + } +} + +func metadataWithOpcPrefix(src map[string]string) map[string]string { + dst := make(map[string]string) + for lowerKey, value := range src { + if !strings.HasPrefix(lowerKey, ociMetaPrefix) { + dst[ociMetaPrefix+lowerKey] = value + } + } + return dst +} diff --git a/backend/oracleobjectstorage/options.go b/backend/oracleobjectstorage/options.go new file mode 100644 index 0000000000000..a7d4e99efd8cd --- /dev/null +++ b/backend/oracleobjectstorage/options.go @@ -0,0 +1,229 @@ +//go:build !plan9 && !solaris && !js +// +build !plan9,!solaris,!js + +package oracleobjectstorage + +import ( + "time" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/lib/encoder" +) + +const ( + maxSizeForCopy = 4768 * 1024 * 1024 + minChunkSize = fs.SizeSuffix(1024 * 1024 * 5) + defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024) + defaultUploadConcurrency = 10 + maxUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024 * 1024) + minSleep = 100 * time.Millisecond + maxSleep = 5 * time.Minute + decayConstant = 1 // bigger for slower decay, exponential + defaultCopyTimeoutDuration = fs.Duration(time.Minute) +) + +const ( + userPrincipal = "user_principal_auth" + instancePrincipal = "instance_principal_auth" + resourcePrincipal = "resource_principal_auth" + environmentAuth = "env_auth" + noAuth = "no_auth" + + userPrincipalHelpText = `use an OCI user and an API key for authentication. +you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key. +https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm` + + instancePrincipalHelpText = `use instance principals to authorize an instance to make API calls. +each instance has its own identity, and authenticates using the certificates that are read from instance metadata. +https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm` + + resourcePrincipalHelpText = `use resource principals to make API calls` + + environmentAuthHelpText = `automatically pickup the credentials from runtime(env), first one to provide auth wins` + + noAuthHelpText = `no credentials needed, this is typically for reading public buckets` +) + +// Options defines the configuration for this backend +type Options struct { + Provider string `config:"provider"` + Compartment string `config:"compartment"` + Namespace string `config:"namespace"` + Region string `config:"region"` + Endpoint string `config:"endpoint"` + Enc encoder.MultiEncoder `config:"encoding"` + ConfigFile string `config:"config_file"` + ConfigProfile string `config:"config_profile"` + UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` + UploadConcurrency int `config:"upload_concurrency"` + DisableChecksum bool `config:"disable_checksum"` + CopyCutoff fs.SizeSuffix `config:"copy_cutoff"` + CopyTimeout fs.Duration `config:"copy_timeout"` + StorageTier string `config:"storage_tier"` + LeavePartsOnError bool `config:"leave_parts_on_error"` +} + +func newOptions() []fs.Option { + return []fs.Option{{ + Name: fs.ConfigProvider, + Help: "Choose your Auth Provider", + Required: true, + Default: environmentAuth, + Examples: []fs.OptionExample{{ + Value: environmentAuth, + Help: environmentAuthHelpText, + }, { + Value: userPrincipal, + Help: userPrincipalHelpText, + }, { + Value: instancePrincipal, + Help: instancePrincipalHelpText, + }, { + Value: resourcePrincipal, + Help: resourcePrincipalHelpText, + }, { + Value: noAuth, + Help: noAuthHelpText, + }}, + }, { + Name: "namespace", + Help: "Object storage namespace", + Required: true, + }, { + Name: "compartment", + Help: "Object storage compartment OCID", + Provider: "!no_auth", + Required: true, + }, { + Name: "region", + Help: "Object storage Region", + Required: true, + }, { + Name: "endpoint", + Help: "Endpoint for Object storage API.\n\nLeave blank to use the default endpoint for the region.", + Required: false, + }, { + Name: "config_file", + Help: "Path to OCI config file", + Provider: userPrincipal, + Default: "~/.oci/config", + Examples: []fs.OptionExample{{ + Value: "~/.oci/config", + Help: "oci configuration file location", + }}, + }, { + Name: "config_profile", + Help: "Profile name inside the oci config file", + Provider: userPrincipal, + Default: "Default", + Examples: []fs.OptionExample{{ + Value: "Default", + Help: "Use the default profile", + }}, + }, { + Name: "upload_cutoff", + Help: `Cutoff for switching to chunked upload. + +Any files larger than this will be uploaded in chunks of chunk_size. +The minimum is 0 and the maximum is 5 GiB.`, + Default: defaultUploadCutoff, + Advanced: true, + }, { + Name: "chunk_size", + Help: `Chunk size to use for uploading. + +When uploading files larger than upload_cutoff or files with unknown +size (e.g. from "rclone rcat" or uploaded with "rclone mount" or google +photos or google docs) they will be uploaded as multipart uploads +using this chunk size. + +Note that "upload_concurrency" chunks of this size are buffered +in memory per transfer. + +If you are transferring large files over high-speed links and you have +enough memory, then increasing this will speed up the transfers. + +Rclone will automatically increase the chunk size when uploading a +large file of known size to stay below the 10,000 chunks limit. + +Files of unknown size are uploaded with the configured +chunk_size. Since the default chunk size is 5 MiB and there can be at +most 10,000 chunks, this means that by default the maximum size of +a file you can stream upload is 48 GiB. If you wish to stream upload +larger files then you will need to increase chunk_size. + +Increasing the chunk size decreases the accuracy of the progress +statistics displayed with "-P" flag. +`, + Default: minChunkSize, + Advanced: true, + }, { + Name: "upload_concurrency", + Help: `Concurrency for multipart uploads. + +This is the number of chunks of the same file that are uploaded +concurrently. + +If you are uploading small numbers of large files over high-speed links +and these uploads do not fully utilize your bandwidth, then increasing +this may help to speed up the transfers.`, + Default: defaultUploadConcurrency, + Advanced: true, + }, { + Name: "copy_cutoff", + Help: `Cutoff for switching to multipart copy. + +Any files larger than this that need to be server-side copied will be +copied in chunks of this size. + +The minimum is 0 and the maximum is 5 GiB.`, + Default: fs.SizeSuffix(maxSizeForCopy), + Advanced: true, + }, { + Name: "copy_timeout", + Help: `Timeout for copy. + +Copy is an asynchronous operation, specify timeout to wait for copy to succeed +`, + Default: defaultCopyTimeoutDuration, + Advanced: true, + }, { + Name: "disable_checksum", + Help: `Don't store MD5 checksum with object metadata. + +Normally rclone will calculate the MD5 checksum of the input before +uploading it so it can add it to metadata on the object. This is great +for data integrity checking but can cause long delays for large files +to start uploading.`, + Default: false, + Advanced: true, + }, { + Name: config.ConfigEncoding, + Help: config.ConfigEncodingHelp, + Advanced: true, + // Any UTF-8 character is valid in a key, however it can't handle + // invalid UTF-8 and / have a special meaning. + // + // The SDK can't seem to handle uploading files called '. + // - initial / encoding + // - doubled / encoding + // - trailing / encoding + // so that OSS keys are always valid file names + Default: encoder.EncodeInvalidUtf8 | + encoder.EncodeSlash | + encoder.EncodeDot, + }, { + Name: "leave_parts_on_error", + Help: `If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery. + +It should be set to true for resuming uploads across different sessions. + +WARNING: Storing parts of an incomplete multipart upload counts towards space usage on object storage and will add +additional costs if not cleaned up. +`, + Default: false, + Advanced: true, + }} +} diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.go new file mode 100644 index 0000000000000..c229df0837fd1 --- /dev/null +++ b/backend/oracleobjectstorage/oracleobjectstorage.go @@ -0,0 +1,691 @@ +//go:build !plan9 && !solaris && !js +// +build !plan9,!solaris,!js + +// Package oracleobjectstorage provides an interface to the OCI object storage system. +package oracleobjectstorage + +import ( + "context" + "fmt" + "io" + "net/http" + "path" + "strings" + "time" + + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/objectstorage" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/lib/bucket" + "github.com/rclone/rclone/lib/pacer" +) + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "oracleobjectstorage", + Description: "Oracle Cloud Infrastructure Object Storage", + Prefix: "oos", + NewFs: NewFs, + CommandHelp: commandHelp, + Options: newOptions(), + }) +} + +// Fs represents a remote object storage server +type Fs struct { + name string // name of this remote + root string // the path we are working on if any + opt Options // parsed config options + ci *fs.ConfigInfo // global config + features *fs.Features // optional features + srv *objectstorage.ObjectStorageClient // the connection to the object storage + rootBucket string // bucket part of root (if any) + rootDirectory string // directory part of root (if any) + cache *bucket.Cache // cache for bucket creation status + pacer *fs.Pacer // To pace the API calls +} + +// NewFs Initialize backend +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + ci := fs.GetConfig(ctx) + objectStorageClient, err := newObjectStorageClient(ctx, opt) + if err != nil { + return nil, err + } + p := pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)) + f := &Fs{ + name: name, + opt: *opt, + ci: ci, + srv: objectStorageClient, + cache: bucket.NewCache(), + pacer: fs.NewPacer(ctx, p), + } + f.setRoot(root) + f.features = (&fs.Features{ + ReadMimeType: true, + WriteMimeType: true, + BucketBased: true, + BucketBasedRootOK: true, + SetTier: true, + GetTier: true, + SlowModTime: true, + }).Fill(ctx, f) + if f.rootBucket != "" && f.rootDirectory != "" && !strings.HasSuffix(root, "/") { + // Check to see if the (bucket,directory) is actually an existing file + oldRoot := f.root + newRoot, leaf := path.Split(oldRoot) + f.setRoot(newRoot) + _, err := f.NewObject(ctx, leaf) + if err != nil { + // File doesn't exist or is a directory so return old f + f.setRoot(oldRoot) + return f, nil + } + // return an error with fs which points to the parent + return f, fs.ErrorIsFile + } + return f, err +} + +func checkUploadChunkSize(cs fs.SizeSuffix) error { + if cs < minChunkSize { + return fmt.Errorf("%s is less than %s", cs, minChunkSize) + } + return nil +} + +func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) { + err = checkUploadChunkSize(cs) + if err == nil { + old, f.opt.ChunkSize = f.opt.ChunkSize, cs + } + return +} + +func checkUploadCutoff(cs fs.SizeSuffix) error { + if cs > maxUploadCutoff { + return fmt.Errorf("%s is greater than %s", cs, maxUploadCutoff) + } + return nil +} + +func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) { + err = checkUploadCutoff(cs) + if err == nil { + old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs + } + return +} + +// ------------------------------------------------------------ +// Implement backed that represents a remote object storage server +// Fs is the interface a cloud storage system must provide +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + if f.rootBucket == "" { + return "oss:root" + } + if f.rootDirectory == "" { + return fmt.Sprintf("oss:bucket %s", f.rootBucket) + } + return fmt.Sprintf("oss:bucket %s, path %s", f.rootBucket, f.rootDirectory) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// Precision of the remote +func (f *Fs) Precision() time.Duration { + return time.Millisecond +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +// setRoot changes the root of the Fs +func (f *Fs) setRoot(root string) { + f.root = parsePath(root) + f.rootBucket, f.rootDirectory = bucket.Split(f.root) +} + +// parsePath parses a remote 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// split returns bucket and bucketPath from the rootRelativePath +// relative to f.root +func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) { + bucketName, bucketPath = bucket.Split(path.Join(f.root, rootRelativePath)) + return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath) +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + bucketName, directory := f.split(dir) + fs.Debugf(f, "listing: bucket : %v, directory: %v", bucketName, dir) + if bucketName == "" { + if directory != "" { + return nil, fs.ErrorListBucketRequired + } + return f.listBuckets(ctx) + } + return f.listDir(ctx, bucketName, directory, f.rootDirectory, f.rootBucket == "") +} + +// listFn is called from list to handle an object. +type listFn func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error + +// list the objects into the function supplied from +// the bucket and root supplied +// (bucket, directory) is the starting directory +// If prefix is set then it is removed from all file names +// If addBucket is set then it adds the bucket to the start of the remotes generated +// If recurse is set the function will recursively list +// If limit is > 0 then it limits to that many files (must be less than 1000) +// If hidden is set then it will list the hidden (deleted) files too. +// if findFile is set it will look for files called (bucket, directory) +func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int, + fn listFn) (err error) { + if prefix != "" { + prefix += "/" + } + if directory != "" { + directory += "/" + } + + delimiter := "" + if !recurse { + delimiter = "/" + } + chunkSize := 1000 + if limit > 0 { + chunkSize = limit + } + var request = objectstorage.ListObjectsRequest{ + NamespaceName: common.String(f.opt.Namespace), + BucketName: common.String(bucket), + Prefix: common.String(directory), + Limit: common.Int(chunkSize), + Fields: common.String("name,size,etag,timeCreated,md5,timeModified,storageTier,archivalState"), + } + if delimiter != "" { + request.Delimiter = common.String(delimiter) + } + + for { + var resp objectstorage.ListObjectsResponse + err = f.pacer.Call(func() (bool, error) { + var err error + resp, err = f.srv.ListObjects(ctx, request) + return shouldRetry(ctx, resp.HTTPResponse(), err) + }) + if err != nil { + if ociError, ok := err.(common.ServiceError); ok { + // If it is a timeout then we want to retry that + if ociError.GetHTTPStatusCode() == http.StatusNotFound { + err = fs.ErrorDirNotFound + } + } + if f.rootBucket == "" { + // if listing from the root ignore wrong region requests returning + // empty directory + if reqErr, ok := err.(common.ServiceError); ok { + // 301 if wrong region for bucket + if reqErr.GetHTTPStatusCode() == http.StatusMovedPermanently { + fs.Errorf(f, "Can't change region for bucket %q with no bucket specified", bucket) + return nil + } + } + } + return err + } + if !recurse { + for _, commonPrefix := range resp.ListObjects.Prefixes { + if commonPrefix == "" { + fs.Logf(f, "Nil common prefix received") + continue + } + remote := commonPrefix + remote = f.opt.Enc.ToStandardPath(remote) + if !strings.HasPrefix(remote, prefix) { + fs.Logf(f, "Odd name received %q", remote) + continue + } + remote = remote[len(prefix):] + if addBucket { + remote = path.Join(bucket, remote) + } + remote = strings.TrimSuffix(remote, "/") + err = fn(remote, &objectstorage.ObjectSummary{Name: &remote}, true) + if err != nil { + return err + } + } + } + for i := range resp.Objects { + object := &resp.Objects[i] + // Finish if file name no longer has prefix + //if prefix != "" && !strings.HasPrefix(file.Name, prefix) { + // return nil + //} + remote := *object.Name + remote = f.opt.Enc.ToStandardPath(remote) + if !strings.HasPrefix(remote, prefix) { + // fs.Debugf(f, "Odd name received %v", object.Name) + continue + } + remote = remote[len(prefix):] + // Check for directory + isDirectory := remote == "" || strings.HasSuffix(remote, "/") + if addBucket { + remote = path.Join(bucket, remote) + } + // is this a directory marker? + if isDirectory && object.Size != nil && *object.Size == 0 { + continue // skip directory marker + } + if isDirectory && len(remote) > 1 { + remote = remote[:len(remote)-1] + } + err = fn(remote, object, isDirectory) + if err != nil { + return err + } + } + // end if no NextFileName + if resp.NextStartWith == nil { + break + } + request.Start = resp.NextStartWith + } + return nil +} + +// Convert a list item into a DirEntry +func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *objectstorage.ObjectSummary, isDirectory bool) (fs.DirEntry, error) { + if isDirectory { + size := int64(0) + if object.Size != nil { + size = *object.Size + } + d := fs.NewDir(remote, time.Time{}).SetSize(size) + return d, nil + } + o, err := f.newObjectWithInfo(ctx, remote, object) + if err != nil { + return nil, err + } + return o, nil +} + +// listDir lists a single directory +func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool) (entries fs.DirEntries, err error) { + fn := func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error { + entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory) + if err != nil { + return err + } + if entry != nil { + entries = append(entries, entry) + } + return nil + } + err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, fn) + if err != nil { + return nil, err + } + // bucket must be present if listing succeeded + f.cache.MarkOK(bucket) + return entries, nil +} + +// listBuckets returns all the buckets to out +func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) { + if f.opt.Provider == noAuth { + return nil, fmt.Errorf("can't list buckets with %v provider, use a valid auth provider in config file", noAuth) + } + var request = objectstorage.ListBucketsRequest{ + NamespaceName: common.String(f.opt.Namespace), + CompartmentId: common.String(f.opt.Compartment), + } + var resp objectstorage.ListBucketsResponse + for { + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.ListBuckets(ctx, request) + return shouldRetry(ctx, resp.HTTPResponse(), err) + }) + if err != nil { + return nil, err + } + for _, item := range resp.Items { + bucketName := f.opt.Enc.ToStandardName(*item.Name) + f.cache.MarkOK(bucketName) + d := fs.NewDir(bucketName, item.TimeCreated.Time) + entries = append(entries, d) + } + if resp.OpcNextPage == nil { + break + } + request.Page = resp.OpcNextPage + } + return entries, nil +} + +// Return an Object from a path +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *objectstorage.ObjectSummary) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + if info != nil { + // Set info but not meta + if info.TimeModified == nil { + fs.Logf(o, "Failed to read last modified") + o.lastModified = time.Now() + } else { + o.lastModified = info.TimeModified.Time + } + if info.Md5 != nil { + md5, err := o.base64ToMd5(*info.Md5) + if err != nil { + o.md5 = md5 + } + } + o.bytes = *info.Size + o.storageTier = storageTierMap[strings.ToLower(string(info.StorageTier))] + } else { + err := o.readMetaData(ctx) // reads info and headers, returning an error + if err != nil { + return nil, err + } + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + return f.newObjectWithInfo(ctx, remote, nil) +} + +// Put the object into the bucket +// Copy the reader in to the new object which is returned +// The new object may have been created if an error is returned +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + // Temporary Object under construction + o := &Object{ + fs: f, + remote: src.Remote(), + } + return o, o.Update(ctx, in, src, options...) +} + +// PutStream uploads to the remote path with the modTime given of indeterminate size +func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return f.Put(ctx, in, src, options...) +} + +// Mkdir creates the bucket if it doesn't exist +func (f *Fs) Mkdir(ctx context.Context, dir string) error { + bucketName, _ := f.split(dir) + return f.makeBucket(ctx, bucketName) +} + +// makeBucket creates the bucket if it doesn't exist +func (f *Fs) makeBucket(ctx context.Context, bucketName string) error { + return f.cache.Create(bucketName, func() error { + details := objectstorage.CreateBucketDetails{ + Name: common.String(bucketName), + CompartmentId: common.String(f.opt.Compartment), + PublicAccessType: objectstorage.CreateBucketDetailsPublicAccessTypeNopublicaccess, + } + req := objectstorage.CreateBucketRequest{ + NamespaceName: common.String(f.opt.Namespace), + CreateBucketDetails: details, + } + err := f.pacer.Call(func() (bool, error) { + resp, err := f.srv.CreateBucket(ctx, req) + return shouldRetry(ctx, resp.HTTPResponse(), err) + }) + if err == nil { + fs.Infof(f, "Bucket %q created with accessType %q", bucketName, + objectstorage.CreateBucketDetailsPublicAccessTypeNopublicaccess) + } + if svcErr, ok := err.(common.ServiceError); ok { + if code := svcErr.GetCode(); code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + err = nil + } + } + return err + }, func() (bool, error) { + return f.bucketExists(ctx, bucketName) + }) +} + +// Check if the bucket exists +// +// NB this can return incorrect results if called immediately after bucket deletion +func (f *Fs) bucketExists(ctx context.Context, bucketName string) (bool, error) { + req := objectstorage.HeadBucketRequest{ + NamespaceName: common.String(f.opt.Namespace), + BucketName: common.String(bucketName), + } + err := f.pacer.Call(func() (bool, error) { + resp, err := f.srv.HeadBucket(ctx, req) + return shouldRetry(ctx, resp.HTTPResponse(), err) + }) + if err == nil { + return true, nil + } + if err, ok := err.(common.ServiceError); ok { + if err.GetHTTPStatusCode() == http.StatusNotFound { + return false, nil + } + } + return false, err +} + +// Rmdir delete an empty bucket. if bucket is not empty this is will fail with appropriate error +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + bucketName, directory := f.split(dir) + if bucketName == "" || directory != "" { + return nil + } + return f.cache.Remove(bucketName, func() error { + req := objectstorage.DeleteBucketRequest{ + NamespaceName: common.String(f.opt.Namespace), + BucketName: common.String(bucketName), + } + err := f.pacer.Call(func() (bool, error) { + resp, err := f.srv.DeleteBucket(ctx, req) + return shouldRetry(ctx, resp.HTTPResponse(), err) + }) + if err == nil { + fs.Infof(f, "Bucket %q deleted", bucketName) + } + return err + }) +} + +func (f *Fs) abortMultiPartUpload(ctx context.Context, bucketName, bucketPath, uploadID string) (err error) { + if uploadID == "" { + return nil + } + request := objectstorage.AbortMultipartUploadRequest{ + NamespaceName: common.String(f.opt.Namespace), + BucketName: common.String(bucketName), + ObjectName: common.String(bucketPath), + UploadId: common.String(uploadID), + } + err = f.pacer.Call(func() (bool, error) { + resp, err := f.srv.AbortMultipartUpload(ctx, request) + return shouldRetry(ctx, resp.HTTPResponse(), err) + }) + return err +} + +// cleanUpBucket removes all pending multipart uploads for a given bucket over the age of maxAge +func (f *Fs) cleanUpBucket(ctx context.Context, bucket string, maxAge time.Duration, + uploads []*objectstorage.MultipartUpload) (err error) { + fs.Infof(f, "cleaning bucket %q of pending multipart uploads older than %v", bucket, maxAge) + for _, upload := range uploads { + if upload.TimeCreated != nil && upload.Object != nil && upload.UploadId != nil { + age := time.Since(upload.TimeCreated.Time) + what := fmt.Sprintf("pending multipart upload for bucket %q key %q dated %v (%v ago)", bucket, *upload.Object, + upload.TimeCreated, age) + if age > maxAge { + fs.Infof(f, "removing %s", what) + if operations.SkipDestructive(ctx, what, "remove pending upload") { + continue + } + ignoreErr := f.abortMultiPartUpload(ctx, *upload.Bucket, *upload.Object, *upload.UploadId) + if ignoreErr != nil { + // fs.Debugf(f, "ignoring error %s", ignoreErr) + } + } else { + // fs.Debugf(f, "ignoring %s", what) + } + } else { + fs.Infof(f, "MultipartUpload doesn't have sufficient details to abort.") + } + } + return err +} + +// CleanUp removes all pending multipart uploads +func (f *Fs) cleanUp(ctx context.Context, maxAge time.Duration) (err error) { + uploadsMap, err := f.listMultipartUploadsAll(ctx) + if err != nil { + return err + } + for bucketName, uploads := range uploadsMap { + cleanErr := f.cleanUpBucket(ctx, bucketName, maxAge, uploads) + if err != nil { + fs.Errorf(f, "Failed to cleanup bucket %q: %v", bucketName, cleanErr) + err = cleanErr + } + } + return err +} + +// CleanUp removes all pending multipart uploads older than 24 hours +func (f *Fs) CleanUp(ctx context.Context) (err error) { + return f.cleanUp(ctx, 24*time.Hour) +} + +// ------------------------------------------------------------ +// Implement ListRer is an optional interfaces for Fs +//------------------------------------------------------------ + +/* +ListR lists the objects and directories of the Fs starting +from dir recursively into out. + +dir should be "" to start from the root, and should not +have trailing slashes. + +This should return ErrDirNotFound if the directory isn't +found. + +It should call callback for each tranche of entries read. +These need not be returned in any particular order. If +callback returns an error then the listing will stop +immediately. + +Don't implement this unless you have a more efficient way +of listing recursively that doing a directory traversal. +*/ +func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { + bucketName, directory := f.split(dir) + list := walk.NewListRHelper(callback) + listR := func(bucket, directory, prefix string, addBucket bool) error { + return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error { + entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory) + if err != nil { + return err + } + return list.Add(entry) + }) + } + if bucketName == "" { + entries, err := f.listBuckets(ctx) + if err != nil { + return err + } + for _, entry := range entries { + err = list.Add(entry) + if err != nil { + return err + } + bucketName := entry.Remote() + err = listR(bucketName, "", f.rootDirectory, true) + if err != nil { + return err + } + // bucket must be present if listing succeeded + f.cache.MarkOK(bucketName) + } + } else { + err = listR(bucketName, directory, f.rootDirectory, f.rootBucket == "") + if err != nil { + return err + } + // bucket must be present if listing succeeded + f.cache.MarkOK(bucketName) + } + return list.Flush() +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = &Fs{} + _ fs.Copier = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.ListRer = &Fs{} + _ fs.Commander = &Fs{} + + _ fs.Object = &Object{} + _ fs.MimeTyper = &Object{} + _ fs.GetTierer = &Object{} + _ fs.SetTierer = &Object{} +) diff --git a/backend/oracleobjectstorage/oracleobjectstorage_test.go b/backend/oracleobjectstorage/oracleobjectstorage_test.go new file mode 100644 index 0000000000000..479da7b5ba9f7 --- /dev/null +++ b/backend/oracleobjectstorage/oracleobjectstorage_test.go @@ -0,0 +1,33 @@ +//go:build !plan9 && !solaris && !js +// +build !plan9,!solaris,!js + +package oracleobjectstorage + +import ( + "testing" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fstest/fstests" +) + +// TestIntegration runs integration tests against the remote +func TestIntegration(t *testing.T) { + fstests.Run(t, &fstests.Opt{ + RemoteName: "TestOracleObjectStorage:", + TiersToTest: []string{"standard", "archive"}, + NilObject: (*Object)(nil), + ChunkedUpload: fstests.ChunkedUploadConfig{ + MinChunkSize: minChunkSize, + }, + }) +} + +func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) { + return f.setUploadChunkSize(cs) +} + +func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) { + return f.setUploadCutoff(cs) +} + +var _ fstests.SetUploadChunkSizer = (*Fs)(nil) diff --git a/backend/oracleobjectstorage/oracleobjectstorage_unsupported.go b/backend/oracleobjectstorage/oracleobjectstorage_unsupported.go new file mode 100644 index 0000000000000..8df06ee6aa628 --- /dev/null +++ b/backend/oracleobjectstorage/oracleobjectstorage_unsupported.go @@ -0,0 +1,7 @@ +// Build for oracleobjectstorage for unsupported platforms to stop go complaining +// about "no buildable Go source files " + +//go:build plan9 || solaris || js +// +build plan9 solaris js + +package oracleobjectstorage diff --git a/backend/oracleobjectstorage/waiter.go b/backend/oracleobjectstorage/waiter.go new file mode 100644 index 0000000000000..d241ff8a783f7 --- /dev/null +++ b/backend/oracleobjectstorage/waiter.go @@ -0,0 +1,362 @@ +//go:build !plan9 && !solaris && !js +// +build !plan9,!solaris,!js + +package oracleobjectstorage + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/rclone/rclone/fs" +) + +var refreshGracePeriod = 30 * time.Second + +// StateRefreshFunc is a function type used for StateChangeConf that is +// responsible for refreshing the item being watched for a state change. +// +// It returns three results. `result` is any object that will be returned +// as the final object after waiting for state change. This allows you to +// return the final updated object, for example an EC2 instance after refreshing +// it. A nil result represents not found. +// +// `state` is the latest state of that object. And `err` is any error that +// may have happened while refreshing the state. +type StateRefreshFunc func() (result interface{}, state string, err error) + +// StateChangeConf is the configuration struct used for `WaitForState`. +type StateChangeConf struct { + Delay time.Duration // Wait this time before starting checks + Pending []string // States that are "allowed" and will continue trying + Refresh StateRefreshFunc // Refreshes the current state + Target []string // Target state + Timeout time.Duration // The amount of time to wait before timeout + MinTimeout time.Duration // Smallest time to wait before refreshes + PollInterval time.Duration // Override MinTimeout/backoff and only poll this often + NotFoundChecks int // Number of times to allow not found (nil result from Refresh) + + // This is to work around inconsistent APIs + ContinuousTargetOccurrence int // Number of times the Target state has to occur continuously +} + +// WaitForStateContext watches an object and waits for it to achieve the state +// specified in the configuration using the specified Refresh() func, +// waiting the number of seconds specified in the timeout configuration. +// +// If the Refresh function returns an error, exit immediately with that error. +// +// If the Refresh function returns a state other than the Target state or one +// listed in Pending, return immediately with an error. +// +// If the Timeout is exceeded before reaching the Target state, return an +// error. +// +// Otherwise, the result is the result of the first call to the Refresh function to +// reach the target state. +// +// Cancellation from the passed in context will cancel the refresh loop +func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType string) (interface{}, error) { + // fs.Debugf(entityType, "Waiting for state to become: %s", conf.Target) + + notfoundTick := 0 + targetOccurrence := 0 + + // Set a default for times to check for not found + if conf.NotFoundChecks == 0 { + conf.NotFoundChecks = 20 + } + + if conf.ContinuousTargetOccurrence == 0 { + conf.ContinuousTargetOccurrence = 1 + } + + type Result struct { + Result interface{} + State string + Error error + Done bool + } + + // Read every result from the refresh loop, waiting for a positive result.Done. + resCh := make(chan Result, 1) + // cancellation channel for the refresh loop + cancelCh := make(chan struct{}) + + result := Result{} + + go func() { + defer close(resCh) + + select { + case <-time.After(conf.Delay): + case <-cancelCh: + return + } + + // start with 0 delay for the first loop + var wait time.Duration + + for { + // store the last result + resCh <- result + + // wait and watch for cancellation + select { + case <-cancelCh: + return + case <-time.After(wait): + // first round had no wait + if wait == 0 { + wait = 100 * time.Millisecond + } + } + + res, currentState, err := conf.Refresh() + result = Result{ + Result: res, + State: currentState, + Error: err, + } + + if err != nil { + resCh <- result + return + } + + // If we're waiting for the absence of a thing, then return + if res == nil && len(conf.Target) == 0 { + targetOccurrence++ + if conf.ContinuousTargetOccurrence == targetOccurrence { + result.Done = true + resCh <- result + return + } + continue + } + + if res == nil { + // If we didn't find the resource, check if we have been + // not finding it for a while, and if so, report an error. + notfoundTick++ + if notfoundTick > conf.NotFoundChecks { + result.Error = &NotFoundError{ + LastError: err, + Retries: notfoundTick, + } + resCh <- result + return + } + } else { + // Reset the counter for when a resource isn't found + notfoundTick = 0 + found := false + + for _, allowed := range conf.Target { + if currentState == allowed { + found = true + targetOccurrence++ + if conf.ContinuousTargetOccurrence == targetOccurrence { + result.Done = true + resCh <- result + return + } + continue + } + } + + for _, allowed := range conf.Pending { + if currentState == allowed { + found = true + targetOccurrence = 0 + break + } + } + + if !found && len(conf.Pending) > 0 { + result.Error = &UnexpectedStateError{ + LastError: err, + State: result.State, + ExpectedState: conf.Target, + } + resCh <- result + return + } + } + + // Wait between refreshes using exponential backoff, except when + // waiting for the target state to reoccur. + if targetOccurrence == 0 { + wait *= 2 + } + + // If a poll interval has been specified, choose that interval. + // Otherwise, bound the default value. + if conf.PollInterval > 0 && conf.PollInterval < 180*time.Second { + wait = conf.PollInterval + } else { + if wait < conf.MinTimeout { + wait = conf.MinTimeout + } else if wait > 10*time.Second { + wait = 10 * time.Second + } + } + + // fs.Debugf(entityType, "[TRACE] Waiting %s before next try", wait) + } + }() + + // store the last value result from the refresh loop + lastResult := Result{} + + timeout := time.After(conf.Timeout) + for { + select { + case r, ok := <-resCh: + // channel closed, so return the last result + if !ok { + return lastResult.Result, lastResult.Error + } + + // we reached the intended state + if r.Done { + return r.Result, r.Error + } + + // still waiting, store the last result + lastResult = r + case <-ctx.Done(): + close(cancelCh) + return nil, ctx.Err() + case <-timeout: + // fs.Debugf(entityType, "[WARN] WaitForState timeout after %s", conf.Timeout) + // fs.Debugf(entityType, "[WARN] WaitForState starting %s refresh grace period", refreshGracePeriod) + + // cancel the goroutine and start our grace period timer + close(cancelCh) + timeout := time.After(refreshGracePeriod) + + // we need a for loop and a label to break on, because we may have + // an extra response value to read, but still want to wait for the + // channel to close. + forSelect: + for { + select { + case r, ok := <-resCh: + if r.Done { + // the last refresh loop reached the desired state + return r.Result, r.Error + } + + if !ok { + // the goroutine returned + break forSelect + } + + // target state not reached, save the result for the + // TimeoutError and wait for the channel to close + lastResult = r + case <-ctx.Done(): + fs.Errorf(entityType, "Context cancellation detected, abandoning grace period") + break forSelect + case <-timeout: + fs.Errorf(entityType, "WaitForState exceeded refresh grace period") + break forSelect + } + } + + return nil, &TimeoutError{ + LastError: lastResult.Error, + LastState: lastResult.State, + Timeout: conf.Timeout, + ExpectedState: conf.Target, + } + } + } +} + +// NotFoundError resource not found error +type NotFoundError struct { + LastError error + LastRequest interface{} + LastResponse interface{} + Message string + Retries int +} + +func (e *NotFoundError) Error() string { + if e.Message != "" { + return e.Message + } + + if e.Retries > 0 { + return fmt.Sprintf("couldn't find resource (%d retries)", e.Retries) + } + + return "couldn't find resource" +} + +func (e *NotFoundError) Unwrap() error { + return e.LastError +} + +// UnexpectedStateError is returned when Refresh returns a state that's neither in Target nor Pending +type UnexpectedStateError struct { + LastError error + State string + ExpectedState []string +} + +func (e *UnexpectedStateError) Error() string { + return fmt.Sprintf( + "unexpected state '%s', wanted target '%s'. last error: %s", + e.State, + strings.Join(e.ExpectedState, ", "), + e.LastError, + ) +} + +func (e *UnexpectedStateError) Unwrap() error { + return e.LastError +} + +// TimeoutError is returned when WaitForState times out +type TimeoutError struct { + LastError error + LastState string + Timeout time.Duration + ExpectedState []string +} + +func (e *TimeoutError) Error() string { + expectedState := "resource to be gone" + if len(e.ExpectedState) > 0 { + expectedState = fmt.Sprintf("state to become '%s'", strings.Join(e.ExpectedState, ", ")) + } + + extraInfo := make([]string, 0) + if e.LastState != "" { + extraInfo = append(extraInfo, fmt.Sprintf("last state: '%s'", e.LastState)) + } + if e.Timeout > 0 { + extraInfo = append(extraInfo, fmt.Sprintf("timeout: %s", e.Timeout.String())) + } + + suffix := "" + if len(extraInfo) > 0 { + suffix = fmt.Sprintf(" (%s)", strings.Join(extraInfo, ", ")) + } + + if e.LastError != nil { + return fmt.Sprintf("timeout while waiting for %s%s: %s", + expectedState, suffix, e.LastError) + } + + return fmt.Sprintf("timeout while waiting for %s%s", + expectedState, suffix) +} + +func (e *TimeoutError) Unwrap() error { + return e.LastError +} diff --git a/docs/content/oracleobjectstorage.md b/docs/content/oracleobjectstorage.md new file mode 100644 index 0000000000000..2510f9aeee32e --- /dev/null +++ b/docs/content/oracleobjectstorage.md @@ -0,0 +1,515 @@ +--- +title: "Oracle Object Storage" +description: "Rclone docs for Oracle Object Storage" +--- + +# {{< icon "fa-light fa-cloud" >}} Oracle Object Storage + +[Oracle Object Storage Overview](https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/objectstorageoverview.htm) + +[Oracle Object Storage FAQ](https://www.oracle.com/cloud/storage/object-storage/faq/) + +Paths are specified as `remote:bucket` (or `remote:` for the `lsd` +command.) You may put subdirectories in too, e.g. `remote:bucket/path/to/dir`. + +## Configuration + +Here is an example of making an oracle object storage configuration. `rclone config` walks you +through it. + +Here is an example of how to make a remote called `remote`. First run: + + rclone config + +This will guide you through an interactive setup process: + + +``` +n) New remote +d) Delete remote +r) Rename remote +c) Copy remote +s) Set configuration password +q) Quit config +e/n/d/r/c/s/q> n + +Enter name for new remote. +name> remote + +Option Storage. +Type of storage to configure. +Choose a number from below, or type in your own value. +[snip] +XX / Oracle Cloud Infrastructure Object Storage + \ (oracleobjectstorage) +Storage> oracleobjectstorage + +Option provider. +Choose your Auth Provider +Choose a number from below, or type in your own string value. +Press Enter for the default (env_auth). + 1 / automatically pickup the credentials from runtime(env), first one to provide auth wins + \ (env_auth) + / use an OCI user and an API key for authentication. + 2 | you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key. + | https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm + \ (user_principal_auth) + / use instance principals to authorize an instance to make API calls. + 3 | each instance has its own identity, and authenticates using the certificates that are read from instance metadata. + | https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm + \ (instance_principal_auth) + 4 / use resource principals to make API calls + \ (resource_principal_auth) + 5 / no credentials needed, this is typically for reading public buckets + \ (no_auth) +provider> 2 + +Option namespace. +Object storage namespace +Enter a value. +namespace> idbamagbg734 + +Option compartment. +Object storage compartment OCID +Enter a value. +compartment> ocid1.compartment.oc1..aaaaaaaapufkxc7ame3sthry5i7ujrwfc7ejnthhu6bhanm5oqfjpyasjkba + +Option region. +Object storage Region +Enter a value. +region> us-ashburn-1 + +Option endpoint. +Endpoint for Object storage API. +Leave blank to use the default endpoint for the region. +Enter a value. Press Enter to leave empty. +endpoint> + +Option config_file. +Path to OCI config file +Choose a number from below, or type in your own string value. +Press Enter for the default (~/.oci/config). + 1 / oci configuration file location + \ (~/.oci/config) +config_file> /etc/oci/dev.conf + +Option config_profile. +Profile name inside OCI config file +Choose a number from below, or type in your own string value. +Press Enter for the default (Default). + 1 / Use the default profile + \ (Default) +config_profile> Test + +Edit advanced config? +y) Yes +n) No (default) +y/n> n + +Configuration complete. +Options: +- type: oracleobjectstorage +- namespace: idbamagbg734 +- compartment: ocid1.compartment.oc1..aaaaaaaapufkxc7ame3sthry5i7ujrwfc7ejnthhu6bhanm5oqfjpyasjkba +- region: us-ashburn-1 +- provider: user_principal_auth +- oci_config_file: /etc/oci/dev.conf +- oci_config_profile: Test +Keep this "remote" remote? +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +See all buckets + + rclone lsd remote: + +Create a new bucket + + rclone mkdir remote:bucket + +List the contents of a bucket + + rclone ls remote:bucket + rclone ls remote:bucket --max-depth 1 + +### Modified time + +The modified time is stored as metadata on the object as +`opc-meta-mtime` as floating point since the epoch, accurate to 1 ns. + +If the modification time needs to be updated rclone will attempt to perform a server +side copy to update the modification if the object can be copied in a single part. +In the case the object is larger than 5Gb, the object will be uploaded rather than copied. + +Note that reading this from the object takes an additional `HEAD` request as the metadata +isn't returned in object listings. + +### Multipart uploads + +rclone supports multipart uploads with OOS which means that it can +upload files bigger than 5 GiB. + +Note that files uploaded *both* with multipart upload *and* through +crypt remotes do not have MD5 sums. + +rclone switches from single part uploads to multipart uploads at the +point specified by `--oos-upload-cutoff`. This can be a maximum of 5 GiB +and a minimum of 0 (ie always upload multipart files). + +The chunk sizes used in the multipart upload are specified by +`--oos-chunk-size` and the number of chunks uploaded concurrently is +specified by `--oos-upload-concurrency`. + +Multipart uploads will use `--transfers` * `--oos-upload-concurrency` * +`--oos-chunk-size` extra memory. Single part uploads to not use extra +memory. + +Single part transfers can be faster than multipart transfers or slower +depending on your latency from oos - the more latency, the more likely +single part transfers will be faster. + +Increasing `--oos-upload-concurrency` will increase throughput (8 would +be a sensible value) and increasing `--oos-chunk-size` also increases +throughput (16M would be sensible). Increasing either of these will +use more memory. The default values are high enough to gain most of +the possible performance without using too much memory. + +{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/oracleobjectstorage/oracleobjectstorage.go then run make backenddocs" >}} +### Standard options + +Here are the Standard options specific to oracleobjectstorage (Oracle Cloud Infrastructure Object Storage). + +#### --oos-provider + +Choose your Auth Provider + +Properties: + +- Config: provider +- Env Var: RCLONE_OOS_PROVIDER +- Type: string +- Default: "env_auth" +- Examples: + - "env_auth" + - automatically pickup the credentials from runtime(env), first one to provide auth wins + - "user_principal_auth" + - use an OCI user and an API key for authentication. + - you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key. + - https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm + - "instance_principal_auth" + - use instance principals to authorize an instance to make API calls. + - each instance has its own identity, and authenticates using the certificates that are read from instance metadata. + - https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm + - "resource_principal_auth" + - use resource principals to make API calls + - "no_auth" + - no credentials needed, this is typically for reading public buckets + +#### --oos-namespace + +Object storage namespace + +Properties: + +- Config: namespace +- Env Var: RCLONE_OOS_NAMESPACE +- Type: string +- Required: true + +#### --oos-compartment + +Object storage compartment OCID + +Properties: + +- Config: compartment +- Env Var: RCLONE_OOS_COMPARTMENT +- Provider: !no_auth +- Type: string +- Required: true + +#### --oos-region + +Object storage Region + +Properties: + +- Config: region +- Env Var: RCLONE_OOS_REGION +- Type: string +- Required: true + +#### --oos-endpoint + +Endpoint for Object storage API. + +Leave blank to use the default endpoint for the region. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_OOS_ENDPOINT +- Type: string +- Required: false + +#### --oos-config-file + +Path to OCI config file + +Properties: + +- Config: config_file +- Env Var: RCLONE_OOS_CONFIG_FILE +- Provider: user_principal_auth +- Type: string +- Default: "~/.oci/config" +- Examples: + - "~/.oci/config" + - oci configuration file location + +#### --oos-config-profile + +Path to OCI config file + +Properties: + +- Config: config_profile +- Env Var: RCLONE_OOS_CONFIG_PROFILE +- Provider: user_principal_auth +- Type: string +- Default: "Default" +- Examples: + - "Default" + - Use the default profile + +### Advanced options + +Here are the Advanced options specific to oracleobjectstorage (Oracle Cloud Infrastructure Object Storage). + +#### --oos-upload-cutoff + +Cutoff for switching to chunked upload. + +Any files larger than this will be uploaded in chunks of chunk_size. +The minimum is 0 and the maximum is 5 GiB. + +Properties: + +- Config: upload_cutoff +- Env Var: RCLONE_OOS_UPLOAD_CUTOFF +- Type: SizeSuffix +- Default: 200Mi + +#### --oos-chunk-size + +Chunk size to use for uploading. + +When uploading files larger than upload_cutoff or files with unknown +size (e.g. from "rclone rcat" or uploaded with "rclone mount" or google +photos or google docs) they will be uploaded as multipart uploads +using this chunk size. + +Note that "upload_concurrency" chunks of this size are buffered +in memory per transfer. + +If you are transferring large files over high-speed links and you have +enough memory, then increasing this will speed up the transfers. + +Rclone will automatically increase the chunk size when uploading a +large file of known size to stay below the 10,000 chunks limit. + +Files of unknown size are uploaded with the configured +chunk_size. Since the default chunk size is 5 MiB and there can be at +most 10,000 chunks, this means that by default the maximum size of +a file you can stream upload is 48 GiB. If you wish to stream upload +larger files then you will need to increase chunk_size. + +Increasing the chunk size decreases the accuracy of the progress +statistics displayed with "-P" flag. + + +Properties: + +- Config: chunk_size +- Env Var: RCLONE_OOS_CHUNK_SIZE +- Type: SizeSuffix +- Default: 5Mi + +#### --oos-upload-concurrency + +Concurrency for multipart uploads. + +This is the number of chunks of the same file that are uploaded +concurrently. + +If you are uploading small numbers of large files over high-speed links +and these uploads do not fully utilize your bandwidth, then increasing +this may help to speed up the transfers. + +Properties: + +- Config: upload_concurrency +- Env Var: RCLONE_OOS_UPLOAD_CONCURRENCY +- Type: int +- Default: 10 + +#### --oos-copy-cutoff + +Cutoff for switching to multipart copy. + +Any files larger than this that need to be server-side copied will be +copied in chunks of this size. + +The minimum is 0 and the maximum is 5 GiB. + +Properties: + +- Config: copy_cutoff +- Env Var: RCLONE_OOS_COPY_CUTOFF +- Type: SizeSuffix +- Default: 4.656Gi + +#### --oos-copy-timeout + +Timeout for copy. + +Copy is an asynchronous operation, specify timeout to wait for copy to succeed + + +Properties: + +- Config: copy_timeout +- Env Var: RCLONE_OOS_COPY_TIMEOUT +- Type: Duration +- Default: 1m0s + +#### --oos-disable-checksum + +Don't store MD5 checksum with object metadata. + +Normally rclone will calculate the MD5 checksum of the input before +uploading it so it can add it to metadata on the object. This is great +for data integrity checking but can cause long delays for large files +to start uploading. + +Properties: + +- Config: disable_checksum +- Env Var: RCLONE_OOS_DISABLE_CHECKSUM +- Type: bool +- Default: false + +#### --oos-encoding + +The encoding for the backend. + +See the [encoding section in the overview](/overview/#encoding) for more info. + +Properties: + +- Config: encoding +- Env Var: RCLONE_OOS_ENCODING +- Type: MultiEncoder +- Default: Slash,InvalidUtf8,Dot + +#### --oos-leave-parts-on-error + +If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery. + +It should be set to true for resuming uploads across different sessions. + +WARNING: Storing parts of an incomplete multipart upload counts towards space usage on object storage and will add +additional costs if not cleaned up. + + +Properties: + +- Config: leave_parts_on_error +- Env Var: RCLONE_OOS_LEAVE_PARTS_ON_ERROR +- Type: bool +- Default: false + +## Backend commands + +Here are the commands specific to the oracleobjectstorage backend. + +Run them with + + rclone backend COMMAND remote: + +The help below will explain what arguments each command takes. + +See the [backend](/commands/rclone_backend/) command for more +info on how to pass options and arguments. + +These can be run on a running backend using the rc command +[backend/command](/rc/#backend-command). + +### rename + +change the name of an object + + rclone backend rename remote: [options] [+] + +This command can be used to rename a object. + +Usage Examples: + + rclone backend rename oos:bucket relative-object-path-under-bucket object-new-name + + +### list-multipart-uploads + +List the unfinished multipart uploads + + rclone backend list-multipart-uploads remote: [options] [+] + +This command lists the unfinished multipart uploads in JSON format. + + rclone backend list-multipart-uploads oos:bucket/path/to/object + +It returns a dictionary of buckets with values as lists of unfinished +multipart uploads. + +You can call it with no bucket in which case it lists all bucket, with +a bucket or with a bucket and path. + + { + "test-bucket": [ + { + "namespace": "test-namespace", + "bucket": "test-bucket", + "object": "600m.bin", + "uploadId": "51dd8114-52a4-b2f2-c42f-5291f05eb3c8", + "timeCreated": "2022-07-29T06:21:16.595Z", + "storageTier": "Standard" + } + ] + + +### cleanup + +Remove unfinished multipart uploads. + + rclone backend cleanup remote: [options] [+] + +This command removes unfinished multipart uploads of age greater than +max-age which defaults to 24 hours. + +Note that you can use -i/--dry-run with this command to see what it +would do. + + rclone backend cleanup oos:bucket/path/to/object + rclone backend cleanup -o max-age=7w oos:bucket/path/to/object + +Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc. + + +Options: + +- "max-age": Max age of upload to delete + +{{< rem autogenerated options stop >}} diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 3fea2f48f9f65..242ab84f35d38 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -385,3 +385,13 @@ backends: fastlist: false ignore: - TestRWFileHandleWriteNoWrite + - backend: "oracleobjectstorage" + remote: "TestOracleObjectStorage:" + fastlist: true + ignore: + - TestIntegration/FsMkdir/FsEncoding/control_chars + - TestIntegration/FsMkdir/FsEncoding/leading_CR + - TestIntegration/FsMkdir/FsEncoding/leading_LF + - TestIntegration/FsMkdir/FsEncoding/trailing_CR + - TestIntegration/FsMkdir/FsEncoding/trailing_LF + - TestIntegration/FsMkdir/FsEncoding/leading_HT diff --git a/go.mod b/go.mod index 3b01b8c189a03..d343e0675157a 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/go-chi/chi/v5 v5.0.7 github.com/google/uuid v1.3.0 github.com/hanwen/go-fuse/v2 v2.1.0 + github.com/hirochachacha/go-smb2 v1.1.0 github.com/iguanesolutions/go-systemd/v5 v5.1.0 github.com/jcmturner/gokrb5/v8 v8.4.3 github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 @@ -39,6 +40,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 github.com/ncw/swift/v2 v2.0.1 + github.com/oracle/oci-go-sdk/v65 v65.1.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/sftp v1.13.5 github.com/pmezard/go-difflib v1.0.0 @@ -86,6 +88,7 @@ require ( github.com/gdamore/encoding v1.0.0 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect @@ -93,7 +96,6 @@ require ( github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/hirochachacha/go-smb2 v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect @@ -115,8 +117,7 @@ require ( github.com/prometheus/procfs v0.7.3 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect - github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect + github.com/sony/gobreaker v0.5.0 // indirect github.com/spacemonkeygo/monkit/v3 v3.0.17 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect diff --git a/go.sum b/go.sum index c79bbbf066cb3..9026764171183 100644 --- a/go.sum +++ b/go.sum @@ -227,6 +227,8 @@ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -382,8 +384,6 @@ github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZ github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= -github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff h1:tN6UCYCBFNrPwvKf4RP9cIhGo6GcZ/IQTN8nqD7eCok= -github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE= github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a h1:s4ryRQyC5HKZh6qkjNAFcvmD7gImK5bZuj/YZkXy1vw= github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -487,6 +487,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/oracle/oci-go-sdk/v65 v65.1.0 h1:CtEPYXdFvv6H+zfYPfTT7DT/V/a5UsULkrj/AnzOtBc= +github.com/oracle/oci-go-sdk/v65 v65.1.0/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/getopt v1.1.0/go.mod h1:FxXoW1Re00sQG/+KIkuSqRL/LwQgSkv7uyac+STFsbk= @@ -564,8 +566,6 @@ github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9A github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= @@ -574,8 +574,6 @@ github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5J github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= -github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 h1:pXY9qYc/MP5zdvqWEUH6SjNiu7VhSjuVFTFiTcphaLU= -github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -590,6 +588,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= +github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/monkit/v3 v3.0.4/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes= @@ -604,6 +604,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= From 66ed0ca7262e7e70909a9b4bf45b46594db23ccb Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Thu, 13 Oct 2022 22:49:22 +0800 Subject: [PATCH 311/560] s3: add Qiniu KODO to s3 provider list - fixes #6195 --- README.md | 1 + backend/s3/s3.go | 113 +++++++++++++++++++- docs/content/_index.md | 1 + docs/content/s3.md | 202 ++++++++++++++++++++++++++++++++++++ fstest/test_all/config.yaml | 11 ++ 5 files changed, 324 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8c5b1c6fbfbd8..fc8064ac7260a 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/) * put.io [:page_facing_up:](https://rclone.org/putio/) * QingStor [:page_facing_up:](https://rclone.org/qingstor/) + * Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu) * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/) * RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp) * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index d869efb8d235f..84da84e915d33 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -65,7 +65,7 @@ import ( func init() { fs.Register(&fs.RegInfo{ Name: "s3", - Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi", + Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi", NewFs: NewFs, CommandHelp: commandHelp, Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { @@ -150,6 +150,9 @@ func init() { }, { Value: "Wasabi", Help: "Wasabi Object Storage", + }, { + Value: "Qiniu", + Help: "Qiniu Object Storage (Kodo)", }, { Value: "Other", Help: "Any other S3 compatible provider", @@ -388,6 +391,34 @@ func init() { Value: "auto", Help: "R2 buckets are automatically distributed across Cloudflare's data centers for low latency.", }}, + }, { + // References: + // https://developer.qiniu.com/kodo/4088/s3-access-domainname + Name: "region", + Help: "Region to connect to.", + Provider: "Qiniu", + Examples: []fs.OptionExample{{ + Value: "cn-east-1", + Help: "The default endpoint - a good choice if you are unsure.\nEast China Region 1.\nNeeds location constraint cn-east-1.", + }, { + Value: "cn-east-2", + Help: "East China Region 2.\nNeeds location constraint cn-east-2.", + }, { + Value: "cn-north-1", + Help: "North China Region 1.\nNeeds location constraint cn-north-1.", + }, { + Value: "cn-south-1", + Help: "South China Region 1.\nNeeds location constraint cn-south-1.", + }, { + Value: "us-north-1", + Help: "North America Region.\nNeeds location constraint us-north-1.", + }, { + Value: "ap-southeast-1", + Help: "Southeast Asia Region 1.\nNeeds location constraint ap-southeast-1.", + }, { + Value: "ap-northeast-1", + Help: "Northeast Asia Region 1.\nNeeds location constraint ap-northeast-1.", + }}, }, { Name: "region", Help: "Region where your bucket will be created and your data stored.\n", @@ -405,7 +436,7 @@ func init() { }, { Name: "region", Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.", - Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive", + Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive", Examples: []fs.OptionExample{{ Value: "", Help: "Use this if unsure.\nWill use v4 signatures and an empty region.", @@ -1030,10 +1061,37 @@ func init() { Value: "nz.s3.rackcorp.com", Help: "Auckland (New Zealand) Endpoint", }}, + }, { + // Qiniu endpoints: https://developer.qiniu.com/kodo/4088/s3-access-domainname + Name: "endpoint", + Help: "Endpoint for Qiniu Object Storage.", + Provider: "Qiniu", + Examples: []fs.OptionExample{{ + Value: "s3-cn-east-1.qiniucs.com", + Help: "East China Endpoint 1", + }, { + Value: "s3-cn-east-2.qiniucs.com", + Help: "East China Endpoint 2", + }, { + Value: "s3-cn-north-1.qiniucs.com", + Help: "North China Endpoint 1", + }, { + Value: "s3-cn-south-1.qiniucs.com", + Help: "South China Endpoint 1", + }, { + Value: "s3-us-north-1.qiniucs.com", + Help: "North America Endpoint 1", + }, { + Value: "s3-ap-southeast-1.qiniucs.com", + Help: "Southeast Asia Endpoint 1", + }, { + Value: "s3-ap-northeast-1.qiniucs.com", + Help: "Northeast Asia Endpoint 1", + }}, }, { Name: "endpoint", Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.", - Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp", + Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu", Examples: []fs.OptionExample{{ Value: "objects-us-east-1.dream.io", Help: "Dream Objects endpoint", @@ -1440,10 +1498,36 @@ func init() { Value: "nz", Help: "Auckland (New Zealand) Region", }}, + }, { + Name: "location_constraint", + Help: "Location constraint - must be set to match the Region.\n\nUsed when creating buckets only.", + Provider: "Qiniu", + Examples: []fs.OptionExample{{ + Value: "cn-east-1", + Help: "East China Region 1", + }, { + Value: "cn-east-2", + Help: "East China Region 2", + }, { + Value: "cn-north-1", + Help: "North China Region 1", + }, { + Value: "cn-south-1", + Help: "South China Region 1", + }, { + Value: "us-north-1", + Help: "North America Region 1", + }, { + Value: "ap-southeast-1", + Help: "Southeast Asia Region 1", + }, { + Value: "ap-northeast-1", + Help: "Northeast Asia Region 1", + }}, }, { Name: "location_constraint", Help: "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.", - Provider: "!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS", + Provider: "!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS", }, { Name: "acl", Help: `Canned ACL used when creating buckets and storing or copying objects. @@ -1710,6 +1794,24 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke Value: "GLACIER", Help: "Archived storage.\nPrices are lower, but it needs to be restored first to be accessed.", }}, + }, { + // Mapping from here: https://developer.qiniu.com/kodo/5906/storage-type + Name: "storage_class", + Help: "The storage class to use when storing new objects in Qiniu.", + Provider: "Qiniu", + Examples: []fs.OptionExample{{ + Value: "STANDARD", + Help: "Standard storage class", + }, { + Value: "LINE", + Help: "Infrequent access storage mode", + }, { + Value: "GLACIER", + Help: "Archive storage mode", + }, { + Value: "DEEP_ARCHIVE", + Help: "Deep archive storage mode", + }}, }, { Name: "upload_cutoff", Help: `Cutoff for switching to chunked upload. @@ -2623,6 +2725,9 @@ func setQuirks(opt *Options) { useMultipartEtag = false // untested case "Wasabi": // No quirks + case "Qiniu": + useMultipartEtag = false + urlEncodeListings = false case "Other": listObjectsV2 = false virtualHostStyle = false diff --git a/docs/content/_index.md b/docs/content/_index.md index cab7b03c7a50f..e42e43ed4e883 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -152,6 +152,7 @@ WebDAV or S3, that work out of the box.) {{< provider name="premiumize.me" home="https://premiumize.me/" config="/premiumizeme/" >}} {{< provider name="put.io" home="https://put.io/" config="/putio/" >}} {{< provider name="QingStor" home="https://www.qingcloud.com/products/storage" config="/qingstor/" >}} +{{< provider name="Qiniu Cloud Object Storage (Kodo)" home="https://www.qiniu.com/en/products/kodo" config="/s3/#qiniu" >}} {{< provider name="Rackspace Cloud Files" home="https://www.rackspace.com/cloud/files" config="/swift/" >}} {{< provider name="rsync.net" home="https://rsync.net/products/rclone.html" config="/sftp/#rsync-net" >}} {{< provider name="Scaleway" home="https://www.scaleway.com/object-storage/" config="/s3/#scaleway" >}} diff --git a/docs/content/s3.md b/docs/content/s3.md index e8b10aae51594..a07e2dc27ded0 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -21,6 +21,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="IDrive e2" home="https://www.idrive.com/e2/" config="/s3/#idrive-e2" >}} {{< provider name="IONOS Cloud" home="https://cloud.ionos.com/storage/object-storage" config="/s3/#ionos" >}} {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}} +{{< provider name="Qiniu Cloud Object Storage (Kodo)" home="https://www.qiniu.com/en/products/kodo" config="/s3/#qiniu" >}} {{< provider name="RackCorp Object Storage" home="https://www.rackcorp.com/" config="/s3/#RackCorp" >}} {{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}} {{< provider name="Seagate Lyve Cloud" home="https://www.seagate.com/gb/en/services/cloud/storage/" config="/s3/#lyve" >}} @@ -3764,6 +3765,207 @@ So once set up, for example, to copy files into a bucket rclone copy /path/to/files minio:bucket ``` +### Qiniu Cloud Object Storage (Kodo) {#qiniu} + +[Qiniu Cloud Object Storage (Kodo)](https://www.qiniu.com/en/products/kodo), a completely independent-researched core technology which is proven by repeated customer experience has occupied absolute leading market leader position. Kodo can be widely applied to mass data management. + +To configure access to Qiniu Kodo, follow the steps below: + +1. Run `rclone config` and select `n` for a new remote. + +``` +rclone config +No remotes found, make a new one? +n) New remote +s) Set configuration password +q) Quit config +n/s/q> n +``` + +2. Give the name of the configuration. For example, name it 'qiniu'. + +``` +name> qiniu +``` + +3. Select `s3` storage. + +``` +Choose a number from below, or type in your own value + 1 / 1Fichier + \ (fichier) + 2 / Akamai NetStorage + \ (netstorage) + 3 / Alias for an existing remote + \ (alias) + 4 / Amazon Drive + \ (amazon cloud drive) + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi + \ (s3) +[snip] +Storage> s3 +``` + +4. Select `Qiniu` provider. +``` +Choose a number from below, or type in your own value +1 / Amazon Web Services (AWS) S3 + \ "AWS" +[snip] +22 / Qiniu Object Storage (Kodo) + \ (Qiniu) +[snip] +provider> Qiniu +``` + +5. Enter your SecretId and SecretKey of Qiniu Kodo. + +``` +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). +Only applies if access_key_id and secret_access_key is blank. +Enter a boolean value (true or false). Press Enter for the default ("false"). +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" +env_auth> 1 +AWS Access Key ID. +Leave blank for anonymous access or runtime credentials. +Enter a string value. Press Enter for the default (""). +access_key_id> AKIDxxxxxxxxxx +AWS Secret Access Key (password) +Leave blank for anonymous access or runtime credentials. +Enter a string value. Press Enter for the default (""). +secret_access_key> xxxxxxxxxxx +``` + +6. Select endpoint for Qiniu Kodo. This is the standard endpoint for different region. + +``` + / The default endpoint - a good choice if you are unsure. + 1 | East China Region 1. + | Needs location constraint cn-east-1. + \ (cn-east-1) + / East China Region 2. + 2 | Needs location constraint cn-east-2. + \ (cn-east-2) + / North China Region 1. + 3 | Needs location constraint cn-north-1. + \ (cn-north-1) + / South China Region 1. + 4 | Needs location constraint cn-south-1. + \ (cn-south-1) + / North America Region. + 5 | Needs location constraint us-north-1. + \ (us-north-1) + / Southeast Asia Region 1. + 6 | Needs location constraint ap-southeast-1. + \ (ap-southeast-1) + / Northeast Asia Region 1. + 7 | Needs location constraint ap-northeast-1. + \ (ap-northeast-1) +[snip] +endpoint> 1 + +Option endpoint. +Endpoint for Qiniu Object Storage. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / East China Endpoint 1 + \ (s3-cn-east-1.qiniucs.com) + 2 / East China Endpoint 2 + \ (s3-cn-east-2.qiniucs.com) + 3 / North China Endpoint 1 + \ (s3-cn-north-1.qiniucs.com) + 4 / South China Endpoint 1 + \ (s3-cn-south-1.qiniucs.com) + 5 / North America Endpoint 1 + \ (s3-us-north-1.qiniucs.com) + 6 / Southeast Asia Endpoint 1 + \ (s3-ap-southeast-1.qiniucs.com) + 7 / Northeast Asia Endpoint 1 + \ (s3-ap-northeast-1.qiniucs.com) +endpoint> 1 + +Option location_constraint. +Location constraint - must be set to match the Region. +Used when creating buckets only. +Choose a number from below, or type in your own value. +Press Enter to leave empty. + 1 / East China Region 1 + \ (cn-east-1) + 2 / East China Region 2 + \ (cn-east-2) + 3 / North China Region 1 + \ (cn-north-1) + 4 / South China Region 1 + \ (cn-south-1) + 5 / North America Region 1 + \ (us-north-1) + 6 / Southeast Asia Region 1 + \ (ap-southeast-1) + 7 / Northeast Asia Region 1 + \ (ap-northeast-1) +location_constraint> 1 +``` + +7. Choose acl and storage class. + +``` +Note that this ACL is applied when server-side copying objects as S3 +doesn't copy the ACL from the source but rather writes a fresh one. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + / Owner gets FULL_CONTROL. + 1 | No one else has access rights (default). + \ (private) + / Owner gets FULL_CONTROL. + 2 | The AllUsers group gets READ access. + \ (public-read) +[snip] +acl> 2 +The storage class to use when storing new objects in Tencent COS. +Enter a string value. Press Enter for the default (""). +Choose a number from below, or type in your own value + 1 / Standard storage class + \ (STANDARD) + 2 / Infrequent access storage mode + \ (LINE) + 3 / Archive storage mode + \ (GLACIER) + 4 / Deep archive storage mode + \ (DEEP_ARCHIVE) +[snip] +storage_class> 1 +Edit advanced config? (y/n) +y) Yes +n) No (default) +y/n> n +Remote config +-------------------- +[qiniu] +- type: s3 +- provider: Qiniu +- access_key_id: xxx +- secret_access_key: xxx +- region: cn-east-1 +- endpoint: s3-cn-east-1.qiniucs.com +- location_constraint: cn-east-1 +- acl: public-read +- storage_class: STANDARD +-------------------- +y) Yes this is OK (default) +e) Edit this remote +d) Delete this remote +y/e/d> y +Current remotes: + +Name Type +==== ==== +qiniu s3 +``` + ### RackCorp {#RackCorp} [RackCorp Object Storage](https://www.rackcorp.com/storage/s3storage) is an S3 compatible object storage platform from your friendly cloud provider RackCorp. diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 242ab84f35d38..282a9936757fb 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -210,6 +210,17 @@ backends: - backend: "s3" remote: "TestS3Alibaba:" fastlist: true + # - backend: "s3" + # remote: "TestS3Qiniu:" + # fastlist: true + # ignore: + # - TestIntegration/FsMkdir/FsEncoding/control_chars + # - TestIntegration/FsMkdir/FsEncoding/leading_VT + # - TestIntegration/FsMkdir/FsEncoding/trailing_VT + # - TestIntegration/FsMkdir/FsPutFiles/FromRoot/ListR + # - TestIntegration/FsMkdir/FsPutFiles/SetTier + # - TestIntegration/FsMkdir/FsPutFiles/FsPutStream/0 + # - TestIntegration/FsMkdir/FsPutFiles/Internal/Metadata - backend: "s3" remote: "TestS3R2:" fastlist: true From cb5b5635c7f528a976d3100740df8d282bf0233e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 14 Oct 2022 11:11:56 +0100 Subject: [PATCH 312/560] Add Manoj Ghosh to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index d330e6024c0cb..eda07cad7f05e 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -645,3 +645,4 @@ put them back in again.` >}} * Lorenzo Milesi * Isaac Aymerich * YanceyChiew <35898533+YanceyChiew@users.noreply.github.com> + * Manoj Ghosh From bb3272e83782c2c8a6ee243a463f6ac2b86bbdba Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 14 Oct 2022 11:11:56 +0100 Subject: [PATCH 313/560] Add Bachue Zhou to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index eda07cad7f05e..3d6f06d1b34a1 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -646,3 +646,4 @@ put them back in again.` >}} * Isaac Aymerich * YanceyChiew <35898533+YanceyChiew@users.noreply.github.com> * Manoj Ghosh + * Bachue Zhou From fce22c00656b8c62081d76c9854a95ee25ef827b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 12 Oct 2022 08:55:58 +0100 Subject: [PATCH 314/560] s3: add --s3-no-system-metadata to suppress read and write of system metadata See: https://forum.rclone.org/t/problems-with-content-disposition-and-backblaze-b2-using-s3/33292/ --- backend/s3/s3.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 84da84e915d33..c0fc888fc007d 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2164,6 +2164,11 @@ can't check the size and hash but the file contents will be decompressed. `, Advanced: true, Default: false, + }, { + Name: "no_system_metadata", + Help: `Suppress setting and reading of system metadata`, + Advanced: true, + Default: false, }, }}) } @@ -2290,6 +2295,7 @@ type Options struct { Versions bool `config:"versions"` VersionAt fs.Time `config:"version_at"` Decompress bool `config:"decompress"` + NoSystemMetadata bool `config:"no_system_metadata"` } // Fs represents a remote s3 server @@ -5227,6 +5233,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op for k, v := range meta { pv := aws.String(v) k = strings.ToLower(k) + if o.fs.opt.NoSystemMetadata { + req.Metadata[k] = pv + continue + } switch k { case "cache-control": req.CacheControl = pv @@ -5505,6 +5515,9 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) // Set system metadata setMetadata := func(k string, v *string) { + if o.fs.opt.NoSystemMetadata { + return + } if v == nil || *v == "" { return } From 5e59e7f442cb287146cea0c920e409b214093043 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 14 Oct 2022 12:10:03 +0100 Subject: [PATCH 315/560] ftp: Fix hang when using ExplicitTLS to certain servers. It was discovered that doing the tls Handshake immediately on connection causes some FTP servers (proftpd and pureftpd) to hang. This imports a fix for it by temporarily hard forking jlaffaye/ftp to include the fix submitted as a pull request. See: https://forum.rclone.org/t/rclone-ftps-explicit-rclone-touch-empty-files-proftpd-unable-to-build-data-connection-operation-not-permitted/22522 See: https://github.com/rclone/rclone/issues/6426#issuecomment-1243993039 See: https://github.com/jlaffaye/ftp/pull/283 See: https://github.com/jlaffaye/ftp/issues/282 --- backend/ftp/ftp.go | 2 +- go.mod | 2 +- go.sum | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index 480f63aba16ad..ede6d088ac981 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -15,7 +15,7 @@ import ( "sync" "time" - "github.com/jlaffaye/ftp" + "github.com/rclone/ftp" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/config" diff --git a/go.mod b/go.mod index d343e0675157a..b7aecb2e7137d 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.12.2 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 + github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6 github.com/rfjakob/eme v1.1.2 github.com/shirou/gopsutil/v3 v3.22.7 github.com/sirupsen/logrus v1.9.0 @@ -140,7 +141,6 @@ require ( github.com/golang-jwt/jwt/v4 v4.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a github.com/pkg/xattr v0.4.7 golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 diff --git a/go.sum b/go.sum index 9026764171183..be4dc9ecf2086 100644 --- a/go.sum +++ b/go.sum @@ -383,9 +383,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/ github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf h1:2IYBd5TD/maMqTU2YUzp2tJL4cNaOYQ9EBullN9t9pk= github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= -github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a h1:s4ryRQyC5HKZh6qkjNAFcvmD7gImK5bZuj/YZkXy1vw= -github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -535,6 +534,8 @@ github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU= +github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6 h1:J832KfU2Z44Ck3XR5bvw2UxShP0QnjueruNQ6dTYH+g= +github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6/go.mod h1:qRpxqlna6CaIq9fSRud1bDC5S7EEUEou0j8nMZ0lxO8= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= From daf3162bcf6f60056a5a7064fc16b74fc4f3a28a Mon Sep 17 00:00:00 2001 From: Manoj Ghosh Date: Sun, 16 Oct 2022 23:19:23 -0700 Subject: [PATCH 316/560] oracle-object-storage: minor docs update --- backend/oracleobjectstorage/command.go | 8 ++++---- backend/oracleobjectstorage/options.go | 13 ++++++++++++ .../oracleobjectstorage.go | 9 ++++++--- bin/make_manual.py | 1 + docs/content/flags.md | 16 +++++++++++++++ docs/content/oracleobjectstorage.md | 20 ++++++++++++++++++- 6 files changed, 59 insertions(+), 8 deletions(-) diff --git a/backend/oracleobjectstorage/command.go b/backend/oracleobjectstorage/command.go index 1d5fb63bbab69..6e0c7e11501bd 100644 --- a/backend/oracleobjectstorage/command.go +++ b/backend/oracleobjectstorage/command.go @@ -31,7 +31,7 @@ var commandHelp = []fs.CommandHelp{{ Usage Examples: - rclone backend rename oss:bucket relative-object-path-under-bucket object-new-name + rclone backend rename oos:bucket relative-object-path-under-bucket object-new-name `, Opts: nil, }, { @@ -39,7 +39,7 @@ Usage Examples: Short: "List the unfinished multipart uploads", Long: `This command lists the unfinished multipart uploads in JSON format. - rclone backend list-multipart-uploads oss:bucket/path/to/object + rclone backend list-multipart-uploads oos:bucket/path/to/object It returns a dictionary of buckets with values as lists of unfinished multipart uploads. @@ -68,8 +68,8 @@ max-age which defaults to 24 hours. Note that you can use -i/--dry-run with this command to see what it would do. - rclone backend cleanup oss:bucket/path/to/object - rclone backend cleanup -o max-age=7w oss:bucket/path/to/object + rclone backend cleanup oos:bucket/path/to/object + rclone backend cleanup -o max-age=7w oos:bucket/path/to/object Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc. `, diff --git a/backend/oracleobjectstorage/options.go b/backend/oracleobjectstorage/options.go index a7d4e99efd8cd..9e7e65fd5f380 100644 --- a/backend/oracleobjectstorage/options.go +++ b/backend/oracleobjectstorage/options.go @@ -63,6 +63,7 @@ type Options struct { CopyTimeout fs.Duration `config:"copy_timeout"` StorageTier string `config:"storage_tier"` LeavePartsOnError bool `config:"leave_parts_on_error"` + NoCheckBucket bool `config:"no_check_bucket"` } func newOptions() []fs.Option { @@ -222,6 +223,18 @@ It should be set to true for resuming uploads across different sessions. WARNING: Storing parts of an incomplete multipart upload counts towards space usage on object storage and will add additional costs if not cleaned up. +`, + Default: false, + Advanced: true, + }, { + Name: "no_check_bucket", + Help: `If set, don't attempt to check the bucket exists or create it. + +This can be useful when trying to minimise the number of transactions +rclone does if you know the bucket exists already. + +It can also be needed if the user you are using does not have bucket +creation permissions. `, Default: false, Advanced: true, diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.go index c229df0837fd1..d18a3467ac4b7 100644 --- a/backend/oracleobjectstorage/oracleobjectstorage.go +++ b/backend/oracleobjectstorage/oracleobjectstorage.go @@ -148,12 +148,12 @@ func (f *Fs) Root() string { // String converts this Fs to a string func (f *Fs) String() string { if f.rootBucket == "" { - return "oss:root" + return "oos:root" } if f.rootDirectory == "" { - return fmt.Sprintf("oss:bucket %s", f.rootBucket) + return fmt.Sprintf("oos:bucket %s", f.rootBucket) } - return fmt.Sprintf("oss:bucket %s, path %s", f.rootBucket, f.rootDirectory) + return fmt.Sprintf("oos:bucket %s, path %s", f.rootBucket, f.rootDirectory) } // Features returns the optional features of this Fs @@ -473,6 +473,9 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { // makeBucket creates the bucket if it doesn't exist func (f *Fs) makeBucket(ctx context.Context, bucketName string) error { + if f.opt.NoCheckBucket { + return nil + } return f.cache.Create(bucketName, func() error { details := objectstorage.CreateBucketDetails{ Name: common.String(bucketName), diff --git a/bin/make_manual.py b/bin/make_manual.py index 8ab7f314d5685..4fa6fe59e5e84 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -59,6 +59,7 @@ "azureblob.md", "onedrive.md", "opendrive.md", + "oracleobjectstorage.md", "qingstor.md", "sia.md", "swift.md", diff --git a/docs/content/flags.md b/docs/content/flags.md index f9753336e47d0..90fbb2a5ff596 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -487,6 +487,22 @@ and may be set in the config file. --onedrive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different onedrive configs --onedrive-token string OAuth Access Token as a JSON blob --onedrive-token-url string Token server url + --oos-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) + --oos-compartment string Object storage compartment OCID + --oos-config-file string Path to OCI config file (default "~/.oci/config") + --oos-config-profile string Profile name inside the oci config file (default "Default") + --oos-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) + --oos-copy-timeout Duration Timeout for copy (default 1m0s) + --oos-disable-checksum Don't store MD5 checksum with object metadata + --oos-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --oos-endpoint string Endpoint for Object storage API + --oos-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery + --oos-namespace string Object storage namespace + --oos-no-check-bucket If set, don't attempt to check the bucket exists or create it + --oos-provider string Choose your Auth Provider (default "env_auth") + --oos-region string Object storage Region + --oos-upload-concurrency int Concurrency for multipart uploads (default 10) + --oos-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) --opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size (default 10Mi) --opendrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot) --opendrive-password string Password (obscured) diff --git a/docs/content/oracleobjectstorage.md b/docs/content/oracleobjectstorage.md index 2510f9aeee32e..e67369e15571e 100644 --- a/docs/content/oracleobjectstorage.md +++ b/docs/content/oracleobjectstorage.md @@ -272,7 +272,7 @@ Properties: #### --oos-config-profile -Path to OCI config file +Profile name inside the oci config file Properties: @@ -432,6 +432,24 @@ Properties: - Type: bool - Default: false +#### --oos-no-check-bucket + +If set, don't attempt to check the bucket exists or create it. + +This can be useful when trying to minimise the number of transactions +rclone does if you know the bucket exists already. + +It can also be needed if the user you are using does not have bucket +creation permissions. + + +Properties: + +- Config: no_check_bucket +- Env Var: RCLONE_OOS_NO_CHECK_BUCKET +- Type: bool +- Default: false + ## Backend commands Here are the commands specific to the oracleobjectstorage backend. From 188b9f8cf1dad3820351a61d2072f67011ff80c9 Mon Sep 17 00:00:00 2001 From: Tom Mombourquette Date: Sun, 16 Oct 2022 01:42:12 -0300 Subject: [PATCH 317/560] rc: corrected mount/unmountall help msg and title --- cmd/mountlib/rc.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/mountlib/rc.go b/cmd/mountlib/rc.go index bde259c88c9cc..3717db28ad94a 100644 --- a/cmd/mountlib/rc.go +++ b/cmd/mountlib/rc.go @@ -274,8 +274,11 @@ func init() { Path: "mount/unmountall", AuthRequired: true, Fn: unmountAll, - Title: "Show current mount points", - Help: `This shows currently mounted points, which can be used for performing an unmount. + Title: "Unmount all active mounts", + Help: ` +rclone allows Linux, FreeBSD, macOS and Windows to +mount any of Rclone's cloud storage systems as a file system with +FUSE. This takes no parameters and returns error if unmount does not succeed. From d2fef05fe45074c458ae11d6b3361a2bd26c9c10 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Wed, 19 Oct 2022 17:13:12 +0100 Subject: [PATCH 318/560] httplib: Add --xxx-min-tls-version option to select minimum tls values for HTTP servers This allows administrators to disable TLS 1.0 and 1.1, for example. Example: rclone rcd --rc-min-tls-version=tls1.2 --rc-cert --rc-key --- cmd/serve/httplib/httpflags/httpflags.go | 1 + cmd/serve/httplib/httplib.go | 22 +++++++++++++++++++++- docs/content/flags.md | 1 + docs/content/rc.md | 5 +++++ lib/http/http.go | 24 +++++++++++++++++++++++- lib/http/http_test.go | 10 ++++++++++ 6 files changed, 61 insertions(+), 2 deletions(-) diff --git a/cmd/serve/httplib/httpflags/httpflags.go b/cmd/serve/httplib/httpflags/httpflags.go index 109604f0e3ede..d06ab8076d071 100644 --- a/cmd/serve/httplib/httpflags/httpflags.go +++ b/cmd/serve/httplib/httpflags/httpflags.go @@ -29,6 +29,7 @@ func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *httplib.Options) flags.StringVarP(flagSet, &Opt.BasicPass, prefix+"pass", "", Opt.BasicPass, "Password for authentication") flags.StringVarP(flagSet, &Opt.BaseURL, prefix+"baseurl", "", Opt.BaseURL, "Prefix for URLs - leave blank for root") flags.StringVarP(flagSet, &Opt.Template, prefix+"template", "", Opt.Template, "User-specified template") + flags.StringVarP(flagSet, &Opt.MinTLSVersion, prefix+"min-tls-version", "", Opt.MinTLSVersion, "Minimum TLS version that is acceptable") } diff --git a/cmd/serve/httplib/httplib.go b/cmd/serve/httplib/httplib.go index 82a29817f86de..10d8eb1ecd597 100644 --- a/cmd/serve/httplib/httplib.go +++ b/cmd/serve/httplib/httplib.go @@ -108,6 +108,10 @@ supply ` + "`--client-ca`" + ` also. of that with the CA certificate. ` + "`--key`" + ` should be the PEM encoded private key and ` + "`--client-ca`" + ` should be the PEM encoded client certificate authority certificate. + +--min-tls-version is minimum TLS version that is acceptable. Valid + values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default + "tls1.0"). ` // Options contains options for the http Server @@ -126,6 +130,7 @@ type Options struct { BasicPass string // password for BasicUser Auth AuthFn `json:"-"` // custom Auth (not set by command line flags) Template string // User specified template + MinTLSVersion string // MinTLSVersion contains the minimum TLS version that is acceptable } // AuthFn if used will be used to authenticate user, pass. If an error @@ -141,6 +146,7 @@ var DefaultOpt = Options{ ServerReadTimeout: 1 * time.Hour, ServerWriteTimeout: 1 * time.Hour, MaxHeaderBytes: 4096, + MinTLSVersion: "tls1.0", } // Server contains info about the running http server @@ -276,6 +282,20 @@ func NewServer(handler http.Handler, opt *Options) *Server { s.Opt.BaseURL = "/" + s.Opt.BaseURL } + var minTLSVersion uint16 + switch opt.MinTLSVersion { + case "tls1.0": + minTLSVersion = tls.VersionTLS10 + case "tls1.1": + minTLSVersion = tls.VersionTLS11 + case "tls1.2": + minTLSVersion = tls.VersionTLS12 + case "tls1.3": + minTLSVersion = tls.VersionTLS13 + default: + log.Fatalf("Invalid value for --min-tls-version") + } + // FIXME make a transport? s.httpServer = &http.Server{ Addr: s.Opt.ListenAddr, @@ -286,7 +306,7 @@ func NewServer(handler http.Handler, opt *Options) *Server { ReadHeaderTimeout: 10 * time.Second, // time to send the headers IdleTimeout: 60 * time.Second, // time to keep idle connections open TLSConfig: &tls.Config{ - MinVersion: tls.VersionTLS10, // disable SSL v3.0 and earlier + MinVersion: minTLSVersion, }, } diff --git a/docs/content/flags.md b/docs/content/flags.md index 90fbb2a5ff596..acf93f6c7735e 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -119,6 +119,7 @@ These flags are available for every command. --rc-job-expire-interval duration Interval to check for expired async jobs (default 10s) --rc-key string SSL PEM Private key --rc-max-header-bytes int Maximum size of request header (default 4096) + --rc-min-tls-version string Minimum TLS version that is acceptable --rc-no-auth Don't require auth for certain methods --rc-pass string Password for authentication --rc-realm string Realm for authentication (default "rclone") diff --git a/docs/content/rc.md b/docs/content/rc.md index 9d87e2e6530dc..51d291720bcfb 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -41,6 +41,11 @@ SSL PEM Private key Maximum size of request header (default 4096) +### --rc-min-tls-version=VALUE + +The minimum TLS version that is acceptable. Valid values are "tls1.0", +"tls1.1", "tls1.2" and "tls1.3" (default "tls1.0"). + ### --rc-user=VALUE User name for authentication. diff --git a/lib/http/http.go b/lib/http/http.go index 1ff7c11489271..527555504e034 100644 --- a/lib/http/http.go +++ b/lib/http/http.go @@ -59,6 +59,10 @@ supply ` + "`--client-ca`" + ` also. of that with the CA certificate. ` + "`--key`" + ` should be the PEM encoded private key and ` + "`--client-ca`" + ` should be the PEM encoded client certificate authority certificate. + +--min-tls-version is minimum TLS version that is acceptable. Valid + values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default + "tls1.0"). ` // Middleware function signature required by chi.Router.Use() @@ -76,6 +80,7 @@ type Options struct { SslCertBody []byte // SSL PEM key (concatenation of certificate and CA certificate) body, ignores SslCert SslKeyBody []byte // SSL PEM Private key body, ignores SslKey ClientCA string // Client certificate authority to verify clients with + MinTLSVersion string // MinTLSVersion contains the minimum TLS version that is acceptable. } // DefaultOpt is the default values used for Options @@ -84,6 +89,7 @@ var DefaultOpt = Options{ ServerReadTimeout: 1 * time.Hour, ServerWriteTimeout: 1 * time.Hour, MaxHeaderBytes: 4096, + MinTLSVersion: "tls1.0", } // Server interface of http server @@ -151,8 +157,23 @@ func NewServer(listeners, tlsListeners []net.Listener, opt Options) (Server, err if err != nil { log.Fatal(err) } + var minTLSVersion uint16 + switch opt.MinTLSVersion { + case "tls1.0": + minTLSVersion = tls.VersionTLS10 + case "tls1.1": + minTLSVersion = tls.VersionTLS11 + case "tls1.2": + minTLSVersion = tls.VersionTLS12 + case "tls1.3": + minTLSVersion = tls.VersionTLS13 + default: + err = errors.New("Invalid value for --min-tls-version") + log.Fatalf(err.Error()) + return nil, err + } tlsConfig = &tls.Config{ - MinVersion: tls.VersionTLS10, // disable SSL v3.0 and earlier + MinVersion: minTLSVersion, Certificates: []tls.Certificate{cert}, } } else if len(listeners) == 0 && len(tlsListeners) != 0 { @@ -410,6 +431,7 @@ func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *Options) { flags.StringVarP(flagSet, &Opt.SslKey, prefix+"key", "", Opt.SslKey, "SSL PEM Private key") flags.StringVarP(flagSet, &Opt.ClientCA, prefix+"client-ca", "", Opt.ClientCA, "Client certificate authority to verify clients with") flags.StringVarP(flagSet, &Opt.BaseURL, prefix+"baseurl", "", Opt.BaseURL, "Prefix for URLs - leave blank for root") + flags.StringVarP(flagSet, &Opt.MinTLSVersion, prefix+"min-tls-version", "", Opt.MinTLSVersion, "Minimum TLS version that is acceptable") } diff --git a/lib/http/http_test.go b/lib/http/http_test.go index 1c2618be2eeb4..fd9850059d769 100644 --- a/lib/http/http_test.go +++ b/lib/http/http_test.go @@ -494,6 +494,16 @@ func Test_useSSL(t *testing.T) { }}, want: true, }, + { + name: "basic", + args: args{opt: Options{ + SslCert: "", + SslKey: "test", + ClientCA: "", + MinTLSVersion: "tls1.2", + }}, + want: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From dff223f195bbd0ed4fbaec30b761b963550f2c57 Mon Sep 17 00:00:00 2001 From: Ole Frost <82263101+olefrost@users.noreply.github.com> Date: Fri, 21 Oct 2022 09:54:16 +0200 Subject: [PATCH 319/560] install.sh: fix arm-v7 download --- docs/content/install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/content/install.sh b/docs/content/install.sh index 8a8242fee9059..336dadce1d5a6 100755 --- a/docs/content/install.sh +++ b/docs/content/install.sh @@ -106,6 +106,9 @@ case "$OS_type" in aarch64|arm64) OS_type='arm64' ;; + armv7*) + OS_type='arm-v7' + ;; arm*) OS_type='arm' ;; From b75c20720870882b3492e42b297a15c0f1501939 Mon Sep 17 00:00:00 2001 From: Manoj Ghosh Date: Thu, 20 Oct 2022 12:24:19 -0700 Subject: [PATCH 320/560] oracle-object-storage: overview, docs update --- docs/content/_index.md | 3 ++- docs/content/docs.md | 1 + docs/content/oracleobjectstorage.md | 6 +++--- docs/content/overview.md | 2 ++ docs/content/swift.md | 2 +- docs/layouts/chrome/navbar.html | 1 + 6 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/content/_index.md b/docs/content/_index.md index e42e43ed4e883..744a42b319589 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -146,7 +146,8 @@ WebDAV or S3, that work out of the box.) {{< provider name="OVH" home="https://www.ovh.co.uk/public-cloud/storage/object-storage/" config="/swift/" >}} {{< provider name="OpenDrive" home="https://www.opendrive.com/" config="/opendrive/" >}} {{< provider name="OpenStack Swift" home="https://docs.openstack.org/swift/latest/" config="/swift/" >}} -{{< provider name="Oracle Cloud Storage" home="https://cloud.oracle.com/object-storage/buckets" config="/swift/" >}} +{{< provider name="Oracle Cloud Storage Swift" home="https://docs.oracle.com/en-us/iaas/integration/doc/configure-object-storage.html" config="/swift/" >}} +{{< provider name="Oracle Object Storage" home="https://www.oracle.com/cloud/storage/object-storage" config="/oracleobjectstorage/" >}} {{< provider name="ownCloud" home="https://owncloud.org/" config="/webdav/#owncloud" >}} {{< provider name="pCloud" home="https://www.pcloud.com/" config="/pcloud/" >}} {{< provider name="premiumize.me" home="https://premiumize.me/" config="/premiumizeme/" >}} diff --git a/docs/content/docs.md b/docs/content/docs.md index 330d255812d98..4b58920a22294 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -61,6 +61,7 @@ See the following for detailed instructions for * [Microsoft OneDrive](/onedrive/) * [OpenStack Swift / Rackspace Cloudfiles / Memset Memstore](/swift/) * [OpenDrive](/opendrive/) + * [Oracle Object Storage](/oracleobjectstorage/) * [Pcloud](/pcloud/) * [premiumize.me](/premiumizeme/) * [put.io](/putio/) diff --git a/docs/content/oracleobjectstorage.md b/docs/content/oracleobjectstorage.md index e67369e15571e..d25dfc303d5ad 100644 --- a/docs/content/oracleobjectstorage.md +++ b/docs/content/oracleobjectstorage.md @@ -3,7 +3,7 @@ title: "Oracle Object Storage" description: "Rclone docs for Oracle Object Storage" --- -# {{< icon "fa-light fa-cloud" >}} Oracle Object Storage +# {{< icon "fa fa-cloud" >}} Oracle Object Storage [Oracle Object Storage Overview](https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/objectstorageoverview.htm) @@ -113,8 +113,8 @@ Options: - compartment: ocid1.compartment.oc1..aaaaaaaapufkxc7ame3sthry5i7ujrwfc7ejnthhu6bhanm5oqfjpyasjkba - region: us-ashburn-1 - provider: user_principal_auth -- oci_config_file: /etc/oci/dev.conf -- oci_config_profile: Test +- config_file: /etc/oci/dev.conf +- config_profile: Test Keep this "remote" remote? y) Yes this is OK (default) e) Edit this remote diff --git a/docs/content/overview.md b/docs/content/overview.md index 79f2d6786602f..dbb15445cfb79 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -42,6 +42,7 @@ Here is an overview of the major features of each cloud storage system. | Microsoft OneDrive | SHA1 ⁵ | R/W | Yes | No | R | - | | OpenDrive | MD5 | R/W | Yes | Partial ⁸ | - | - | | OpenStack Swift | MD5 | R/W | No | No | R/W | - | +| Oracle Object Storage | MD5 | R/W | No | No | R/W | - | | pCloud | MD5, SHA1 ⁷ | R | No | No | W | - | | premiumize.me | - | - | Yes | No | R | - | | put.io | CRC-32 | R/W | No | Yes | R | - | @@ -493,6 +494,7 @@ upon backend-specific capabilities. | Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | | OpenStack Swift | Yes † | Yes | No | No | No | Yes | Yes | No | Yes | No | +| Oracle Object Storage | Yes | Yes | No | No | Yes | Yes | No | No | No | No | | pCloud | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | premiumize.me | Yes | No | Yes | Yes | No | No | No | Yes | Yes | Yes | | put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | Yes | Yes | diff --git a/docs/content/swift.md b/docs/content/swift.md index 70cde652f2d61..f75e4f6ddb868 100644 --- a/docs/content/swift.md +++ b/docs/content/swift.md @@ -11,7 +11,7 @@ Commercial implementations of that being: * [Rackspace Cloud Files](https://www.rackspace.com/cloud/files/) * [Memset Memstore](https://www.memset.com/cloud/storage/) * [OVH Object Storage](https://www.ovh.co.uk/public-cloud/storage/object-storage/) - * [Oracle Cloud Storage](https://cloud.oracle.com/object-storage/buckets) + * [Oracle Cloud Storage](https://docs.oracle.com/en-us/iaas/integration/doc/configure-object-storage.html) * [IBM Bluemix Cloud ObjectStorage Swift](https://console.bluemix.net/docs/infrastructure/objectstorage-swift/index.html) Paths are specified as `remote:container` (or `remote:` for the `lsd` diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index e32550e7f8c50..dfd1e83d9e742 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -85,6 +85,7 @@ OpenDrive QingStor Openstack Swift + Oracle Object Storage pCloud premiumize.me put.io From 0d2a62a927558e2a6e38db011093936d249c7d12 Mon Sep 17 00:00:00 2001 From: Ole Frost <82263101+olefrost@users.noreply.github.com> Date: Fri, 21 Oct 2022 11:45:24 +0200 Subject: [PATCH 321/560] docs: Describe connection strings in alias backend --- docs/content/alias.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/content/alias.md b/docs/content/alias.md index aa041f6f4eed1..eeb7464759f60 100644 --- a/docs/content/alias.md +++ b/docs/content/alias.md @@ -23,6 +23,11 @@ Invoking `rclone mkdir backup:../desktop` is exactly the same as invoking The empty path is not allowed as a remote. To alias the current directory use `.` instead. +The target remote can also be a [connection string](/docs/#connection-strings). +This can be used to modify the config of a remote for different uses, e.g. +the alias `myDriveTrash` with the target remote `myDrive,trashed_only:` +can be used to only show the trashed files in `myDrive`. + ## Configuration Here is an example of how to make an alias called `remote` for local folder. From 38f1f5b177e95ba92854bbfdd31de945255e0963 Mon Sep 17 00:00:00 2001 From: Tom Mombourquette Date: Wed, 19 Oct 2022 22:40:20 -0300 Subject: [PATCH 322/560] rc: Fix mount/listmounts not returning the full Fs entered in mount/mount --- cmd/mountlib/rc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/mountlib/rc.go b/cmd/mountlib/rc.go index 3717db28ad94a..d4c091235a618 100644 --- a/cmd/mountlib/rc.go +++ b/cmd/mountlib/rc.go @@ -258,7 +258,7 @@ func listMountsRc(_ context.Context, in rc.Params) (out rc.Params, err error) { for _, k := range keys { m := liveMounts[k] info := MountInfo{ - Fs: m.Fs.Name(), + Fs: fs.ConfigString(m.Fs), MountPoint: m.MountPoint, MountedOn: m.MountedOn, } From 0fb1b75a02100d8fb27d23f13d97d35ccd5778fa Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 21 Oct 2022 12:39:45 +0100 Subject: [PATCH 323/560] Add Manoj Ghosh to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 3d6f06d1b34a1..534e2e2d4b678 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -647,3 +647,4 @@ put them back in again.` >}} * YanceyChiew <35898533+YanceyChiew@users.noreply.github.com> * Manoj Ghosh * Bachue Zhou + * Manoj Ghosh From d9c4d95ab3c98840da8534e51b5bf5a639d40bd0 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 21 Oct 2022 12:39:45 +0100 Subject: [PATCH 324/560] Add Tom Mombourquette to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 534e2e2d4b678..85033a1077e3c 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -648,3 +648,4 @@ put them back in again.` >}} * Manoj Ghosh * Bachue Zhou * Manoj Ghosh + * Tom Mombourquette From 546dc82793086a56324e38b584ad3f8bfac99bfe Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 21 Oct 2022 12:39:45 +0100 Subject: [PATCH 325/560] Add Robert Newson to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 85033a1077e3c..012cd07b63927 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -649,3 +649,4 @@ put them back in again.` >}} * Bachue Zhou * Manoj Ghosh * Tom Mombourquette + * Robert Newson From afa61e702c3768c5cbbb72a26a9415b8ee61fc90 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 21 Oct 2022 12:39:29 +0100 Subject: [PATCH 326/560] docs: remove hosted by tag as server has moved --- docs/layouts/_default/baseof.html | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/layouts/_default/baseof.html b/docs/layouts/_default/baseof.html index e04260dbb4c3d..d3e327ebeab4d 100644 --- a/docs/layouts/_default/baseof.html +++ b/docs/layouts/_default/baseof.html @@ -42,9 +42,8 @@ © Nick Craig-Wood 2014-{{ now.Format "2006" }}
    {{ if .File}}{{ with $path := strings.TrimPrefix "/" .File.Path }}Source file {{ $path }}{{ end }} last updated {{ .Lastmod.Format "2006-01-02" }}
    {{end}} - Website hosted on a MEMSET CLOUD VPS, - uploaded with rclone - and built with Hugo. + Uploaded with rclone. + Built with Hugo. Logo by @andy23.

    From 01dbbff62e2470603d0c3abf4adf5c96e60cdbb5 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 21 Oct 2022 15:06:08 +0100 Subject: [PATCH 327/560] Version v1.60.0 --- MANUAL.html | 2343 ++++++++--- MANUAL.md | 2561 ++++++++++-- MANUAL.txt | 2700 ++++++++++--- docs/content/changelog.md | 76 + docs/content/commands/rclone.md | 2 +- docs/content/commands/rclone_bisync.md | 6 +- .../commands/rclone_completion_bash.md | 2 +- .../content/commands/rclone_completion_zsh.md | 6 +- docs/content/commands/rclone_config_create.md | 2 +- docs/content/commands/rclone_config_update.md | 2 +- docs/content/commands/rclone_hashsum.md | 2 +- docs/content/commands/rclone_ls.md | 2 +- docs/content/commands/rclone_lsd.md | 2 +- docs/content/commands/rclone_lsf.md | 2 +- docs/content/commands/rclone_lsjson.md | 4 +- docs/content/commands/rclone_lsl.md | 2 +- docs/content/commands/rclone_md5sum.md | 2 +- docs/content/commands/rclone_mount.md | 15 +- docs/content/commands/rclone_ncdu.md | 2 +- docs/content/commands/rclone_serve_dlna.md | 7 +- docs/content/commands/rclone_serve_http.md | 5 + docs/content/commands/rclone_serve_restic.md | 5 + docs/content/commands/rclone_serve_sftp.md | 23 +- docs/content/commands/rclone_serve_webdav.md | 5 + docs/content/commands/rclone_sha1sum.md | 2 +- docs/content/commands/rclone_sync.md | 5 + docs/content/flags.md | 33 +- docs/content/ftp.md | 14 + docs/content/googlecloudstorage.md | 13 + docs/content/rc.md | 12 +- docs/content/s3.md | 218 +- docs/content/sftp.md | 31 +- docs/content/swift.md | 32 + rclone.1 | 3555 ++++++++++++++--- 34 files changed, 9680 insertions(+), 2013 deletions(-) diff --git a/MANUAL.html b/MANUAL.html index 4c0d99e6195f2..4b87ec4b17fcf 100644 --- a/MANUAL.html +++ b/MANUAL.html @@ -19,7 +19,7 @@

    rclone(1) User Manual

    Nick Craig-Wood

    -

    Jul 09, 2022

    +

    Oct 21, 2022

    Rclone syncs your files to cloud storage

    rclone logo

    @@ -85,7 +85,6 @@

    Supported providers

  • China Mobile Ecloud Elastic Object Storage (EOS)
  • Arvan Cloud Object Storage (AOS)
  • Citrix ShareFile
    • -
  • C14
  • Cloudflare R2
  • DigitalOcean Spaces
  • Digi Storage
    • @@ -100,11 +99,11 @@

    Supported providers

  • Hetzner Storage Box
  • HiDrive
  • HTTP
    • -
  • Hubic
  • Internet Archive
  • Jottacloud
  • IBM COS S3
  • IDrive e2
    • +
  • IONOS Cloud
  • Koofr
  • Mail.ru Cloud
  • Memset Memstore
    • @@ -117,12 +116,14 @@

    Supported providers

  • OVH
  • OpenDrive
  • OpenStack Swift
    • -
  • Oracle Cloud Storage
    • +
  • Oracle Cloud Storage Swift
    • +
  • Oracle Object Storage
  • ownCloud
  • pCloud
  • premiumize.me
  • put.io
  • QingStor
    • +
  • Qiniu Cloud Object Storage (Kodo)
  • Rackspace Cloud Files
  • rsync.net
  • Scaleway
    • @@ -131,6 +132,7 @@

    Supported providers

  • SeaweedFS
  • SFTP
  • Sia
    • +
  • SMB / CIFS
  • StackPath
  • Storj
  • SugarSync
    • @@ -170,7 +172,7 @@

    Quickstart

  • Run rclone config to setup. See rclone config docs for more details.
  • Optionally configure automatic execution.
    • -

    See below for some expanded Linux / macOS instructions.

    +

    See below for some expanded Linux / macOS / Windows instructions.

    See the usage docs for how to use rclone, or run rclone -h.

    Already installed rclone can be easily updated to the latest version using the rclone selfupdate command.

    Script installation

    @@ -179,7 +181,8 @@

    Script installation

    For beta installation, run:

    sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta

    Note that this script checks the version of rclone installed first and won't re-download if not needed.

    -

    Linux installation from precompiled binary

    +

    Linux installation

    +

    Precompiled binary

    Fetch and unpack

    curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip
 unzip rclone-current-linux-amd64.zip
@@ -194,10 +197,13 @@ 
    Linux installation from prec
 sudo mandb

    Run rclone config to setup. See rclone config docs for more details.

    rclone config
    -

    macOS installation with brew

    +

    macOS installation

    +

    Installation with brew

    brew install rclone

    NOTE: This version of rclone will not support mount any more (see #5373). If mounting is wanted on macOS, either install a precompiled binary or enable the relevant option when installing from source.

    -

    macOS installation from precompiled binary, using curl

    +

    Note that this is a third party installer not controlled by the rclone developers so it may be out of date. Its current version is as below.

    +

    Homebrew package

    +

    Precompiled binary, using curl

    To avoid problems with macOS gatekeeper enforcing the binary to be signed and notarized it is enough to download with curl.

    Download the latest version of rclone.

    cd && curl -O https://downloads.rclone.org/rclone-current-osx-amd64.zip
    @@ -211,14 +217,44 @@

    macOS installatio 
    cd .. && rm -rf rclone-*-osx-amd64 rclone-current-osx-amd64.zip

    Run rclone config to setup. See rclone config docs for more details.

    rclone config
    -

    macOS installation from precompiled binary, using a web browser

    +

    Precompiled binary, using a web browser

    When downloading a binary with a web browser, the browser will set the macOS gatekeeper quarantine attribute. Starting from Catalina, when attempting to run rclone, a pop-up will appear saying:

    "rclone" cannot be opened because the developer cannot be verified.
 macOS cannot verify that this app is free from malware.

    The simplest fix is to run

    xattr -d com.apple.quarantine rclone
    -

    Install with docker

    -

    The rclone maintains a docker image for rclone. These images are autobuilt by docker hub from the rclone source based on a minimal Alpine linux image.

    +

    Windows installation

    +

    Precompiled binary

    +

    Fetch the correct binary for your processor type by clicking on these links. If not sure, use the first link.

    + +

    Open this file in the Explorer and extract rclone.exe. Rclone is a portable executable so you can place it wherever is convenient.

    +

    Open a CMD window (or powershell) and run the binary. Note that rclone does not launch a GUI by default, it runs in the CMD Window.

    + +

    If you are planning to use the rclone mount feature then you will need to install the third party utility WinFsp also.

    +

    Chocolatey package manager

    +

    Make sure you have Choco installed

    +
    choco search rclone
+choco install rclone
    +

    This will install rclone on your Windows machine. If you are planning to use rclone mount then

    +
    choco install winfsp
    +

    will install that too.

    +

    Note that this is a third party installer not controlled by the rclone developers so it may be out of date. Its current version is as below.

    +

    Chocolatey package

    +

    Package manager installation

    +

    Many Linux, Windows, macOS and other OS distributions package and distribute rclone.

    +

    The distributed versions of rclone are often quite out of date and for this reason we recommend one of the other installation methods if possible.

    +

    You can get an idea of how up to date or not your OS distribution's package is here.

    +

    Packaging status

    +

    Docker installation

    +

    The rclone developers maintain a docker image for rclone.

    +

    These images are built as part of the release process based on a minimal Alpine Linux.

    The :latest tag will always point to the latest stable release. You can use the :beta tag to get the latest build from master. You can also use version tags, e.g. :1.49.1, :1.49 or :1.

    $ docker pull rclone/rclone:latest
 latest: Pulling from rclone/rclone
@@ -264,28 +300,29 @@ 
    Install with docker
    
     mount dropbox:Photos /data/mount &
 ls ~/data/mount
 kill %1
    -

    Install from source

    -

    Make sure you have git and Go installed. Go version 1.16 or newer is required, latest release is recommended. You can get it from your package manager, or download it from golang.org/dl. Then you can run the following:

    +

    Source installation

    +

    Make sure you have git and Go installed. Go version 1.17 or newer is required, latest release is recommended. You can get it from your package manager, or download it from golang.org/dl. Then you can run the following:

    git clone https://github.com/rclone/rclone.git
 cd rclone
 go build

    This will check out the rclone source in subfolder rclone, which you can later modify and send pull requests with. Then it will build the rclone executable in the same folder. As an initial check you can now run ./rclone version (.\rclone version on Windows).

    -

    Note that on macOS and Windows the mount command will not be available unless you specify additional build tag cmount.

    +

    Note that on macOS and Windows the mount command will not be available unless you specify an additional build tag cmount.

    go build -tags cmount

    This assumes you have a GCC compatible C compiler (GCC or Clang) in your PATH, as it uses cgo. But on Windows, the cgofuse library that the cmount implementation is based on, also supports building without cgo, i.e. by setting environment variable CGO_ENABLED to value 0 (static linking). This is how the official Windows release of rclone is being built, starting with version 1.59. It is still possible to build with cgo on Windows as well, by using the MinGW port of GCC, e.g. by installing it in a MSYS2 distribution (make sure you install it in the classic mingw64 subsystem, the ucrt64 version is not compatible).

    -

    Additionally, on Windows, you must install the third party utility WinFsp, with the "Developer" feature selected. If building with cgo, you must also set environment variable CPATH pointing to the fuse include directory within the WinFsp installation (normally C:\Program Files (x86)\WinFsp\inc\fuse).

    +

    Additionally, on Windows, you must install the third party utility WinFsp, with the "Developer" feature selected. If building with cgo, you must also set environment variable CPATH pointing to the fuse include directory within the WinFsp installation (normally C:\Program Files (x86)\WinFsp\inc\fuse).

    You may also add arguments -ldflags -s (with or without -tags cmount), to omit symbol table and debug information, making the executable file smaller, and -trimpath to remove references to local file system paths. This is how the official rclone releases are built.

    go build -trimpath -ldflags -s -tags cmount
    -

    Instead of executing the go build command directly, you can run it via the Makefile, which also sets version information and copies the resulting rclone executable into your GOPATH bin folder ($(go env GOPATH)/bin, which corresponds to ~/go/bin/rclone by default).

    +

    Instead of executing the go build command directly, you can run it via the Makefile. It changes the version number suffix from "-DEV" to "-beta" and appends commit details. It also copies the resulting rclone executable into your GOPATH bin folder ($(go env GOPATH)/bin, which corresponds to ~/go/bin/rclone by default).

    make

    To include mount command on macOS and Windows with Makefile build:

    make GOTAGS=cmount
    -

    As an alternative you can download the source, build and install rclone in one operation, as a regular Go package. The source will be stored it in the Go module cache, and the resulting executable will be in your GOPATH bin folder ($(go env GOPATH)/bin, which corresponds to ~/go/bin/rclone by default).

    +

    There are other make targets that can be used for more advanced builds, such as cross-compiling for all supported os/architectures, embedding icon and version info resources into windows executable, and packaging results into release artifacts. See Makefile and cross-compile.go for details.

    +

    Another alternative is to download the source, build and install rclone in one operation, as a regular Go package. The source will be stored it in the Go module cache, and the resulting executable will be in your GOPATH bin folder ($(go env GOPATH)/bin, which corresponds to ~/go/bin/rclone by default).

    With Go version 1.17 or newer:

    go install github.com/rclone/rclone@latest

    With Go versions older than 1.17 (do not use the -u flag, it causes Go to try to update the dependencies that rclone uses and sometimes these don't work with the current version):

    go get github.com/rclone/rclone
    -

    Installation with Ansible

    +

    Ansible installation

    This can be done with Stefan Weichinger's ansible role.

    Instructions

      @@ -295,12 +332,12 @@

      Installation with Ansible

          - hosts: rclone-hosts
       roles:
           - rclone
      -

      Portable installation

      +

      Portable installation

      As mentioned above, rclone is single executable (rclone, or rclone.exe on Windows) that you can download as a zip archive and extract into a location of your choosing. When executing different commands, it may create files in different locations, such as a configuration file and various temporary files. By default the locations for these are according to your operating system, e.g. configuration file in your user profile directory and temporary files in the standard temporary directory, but you can customize all of them, e.g. to make a completely self-contained, portable installation.

      Run the config paths command to see the locations that rclone will use.

      To override them set the corresponding options (as command-line arguments, or as environment variables): - --config - --cache-dir - --temp-dir

      Autostart

      -

      After installing and configuring rclone, as described above, you are ready to use rclone as an interactive command line utility. If your goal is to perform periodic operations, such as a regular sync, you will probably want to configure your rclone command in your operating system's scheduler. If you need to expose service-like features, such as remote control, GUI, serve or mount, you will often want an rclone command always running in the background, and configuring it to run in a service infrastructure may be a better option. Below are some alternatives on how to achieve this on different operating systems.

      +

      After installing and configuring rclone, as described above, you are ready to use rclone as an interactive command line utility. If your goal is to perform periodic operations, such as a regular sync, you will probably want to configure your rclone command in your operating system's scheduler. If you need to expose service-like features, such as remote control, GUI, serve or mount, you will often want an rclone command always running in the background, and configuring it to run in a service infrastructure may be a better option. Below are some alternatives on how to achieve this on different operating systems.

      NOTE: Before setting up autorun it is highly recommended that you have tested your command manually from a Command Prompt first.

      Autostart on Windows

      The most relevant alternatives for autostart on Windows are: - Run at user log on using the Startup folder - Run at user log on, at system startup or at schedule using Task Scheduler - Run at system startup using Windows service

      @@ -309,8 +346,8 @@

      Running in background

      Example command to run a sync in background:

      c:\rclone\rclone.exe sync c:\files remote:/files --no-console --log-file c:\rclone\logs\sync_files.txt

      User account

      -

      As mentioned in the mount documentation, mounted drives created as Administrator are not visible to other accounts, not even the account that was elevated as Administrator. By running the mount command as the built-in SYSTEM user account, it will create drives accessible for everyone on the system. Both scheduled task and Windows service can be used to achieve this.

      -

      NOTE: Remember that when rclone runs as the SYSTEM user, the user profile that it sees will not be yours. This means that if you normally run rclone with configuration file in the default location, to be able to use the same configuration when running as the system user you must explicitely tell rclone where to find it with the --config option, or else it will look in the system users profile path (C:\Windows\System32\config\systemprofile). To test your command manually from a Command Prompt, you can run it with the PsExec utility from Microsoft's Sysinternals suite, which takes option -s to execute commands as the SYSTEM user.

      +

      As mentioned in the mount documentation, mounted drives created as Administrator are not visible to other accounts, not even the account that was elevated as Administrator. By running the mount command as the built-in SYSTEM user account, it will create drives accessible for everyone on the system. Both scheduled task and Windows service can be used to achieve this.

      +

      NOTE: Remember that when rclone runs as the SYSTEM user, the user profile that it sees will not be yours. This means that if you normally run rclone with configuration file in the default location, to be able to use the same configuration when running as the system user you must explicitly tell rclone where to find it with the --config option, or else it will look in the system users profile path (C:\Windows\System32\config\systemprofile). To test your command manually from a Command Prompt, you can run it with the PsExec utility from Microsoft's Sysinternals suite, which takes option -s to execute commands as the SYSTEM user.

      Start from Startup folder

      To quickly execute an rclone command you can simply create a standard Windows Explorer shortcut for the complete rclone command you want to run. If you store this shortcut in the special "Startup" start-menu folder, Windows will automatically run it at login. To open this folder in Windows Explorer, enter path %APPDATA%\Microsoft\Windows\Start Menu\Programs\Startup, or C:\ProgramData\Microsoft\Windows\Start Menu\Programs\StartUp if you want the command to start for every user that logs in.

      This is the easiest approach to autostarting of rclone, but it offers no functionality to set it to run as different user, or to set conditions or actions on certain events. Setting up a scheduled task as described below will often give you better results.

      @@ -324,7 +361,7 @@
      Mount command built-in servi 
      New-Service -Name Rclone -BinaryPathName 'c:\rclone\rclone.exe mount remote:/files X: --config c:\rclone\config\rclone.conf --log-file c:\rclone\logs\mount.txt'

      The WinFsp service infrastructure supports incorporating services for file system implementations, such as rclone, into its own launcher service, as kind of "child services". This has the additional advantage that it also implements a network provider that integrates into Windows standard methods for managing network drives. This is currently not officially supported by Rclone, but with WinFsp version 2019.3 B2 / v1.5B2 or later it should be possible through path rewriting as described here.

      Third-party service integration
      -

      To Windows service running any rclone command, the excellent third-party utility NSSM, the "Non-Sucking Service Manager", can be used. It includes some advanced features such as adjusting process periority, defining process environment variables, redirect to file anything written to stdout, and customized response to different exit codes, with a GUI to configure everything from (although it can also be used from command line ).

      +

      To Windows service running any rclone command, the excellent third-party utility NSSM, the "Non-Sucking Service Manager", can be used. It includes some advanced features such as adjusting process priority, defining process environment variables, redirect to file anything written to stdout, and customized response to different exit codes, with a GUI to configure everything from (although it can also be used from command line ).

      There are also several other alternatives. To mention one more, WinSW, "Windows Service Wrapper", is worth checking out. It requires .NET Framework, but it is preinstalled on newer versions of Windows, and it also provides alternative standalone distributions which includes necessary runtime (.NET 5). WinSW is a command-line only utility, where you have to manually create an XML file with service configuration. This may be a drawback for some, but it can also be an advantage as it is easy to back up and re-use the configuration settings, without having go through manual steps in a GUI. One thing to note is that by default it does not restart the service on error, one have to explicit enable this in the configuration file (via the "onfailure" parameter).

      Autostart on Linux

      Start as a service

      @@ -363,7 +400,6 @@

      Configure

    1. HDFS
    2. HiDrive
    3. HTTP
      4. -
    4. Hubic
    5. Internet Archive
    6. Jottacloud
    7. Koofr
      8. @@ -374,6 +410,7 @@

      Configure

    8. Microsoft OneDrive
    9. OpenStack Swift / Rackspace Cloudfiles / Memset Memstore
    10. OpenDrive
      11. +
    11. Oracle Object Storage
    12. Pcloud
    13. premiumize.me
    14. put.io
      15. @@ -381,6 +418,7 @@

      Configure

    15. Seafile
    16. SFTP
    17. Sia
      18. +
    18. SMB
    19. Storj
    20. SugarSync
    21. Union
      22. @@ -469,6 +507,7 @@

      Synopsis

      Note that files in the destination won't be deleted if there were any errors at any point. Duplicate objects (files with the same name, on those providers that support it) are also not yet handled.

      It is always the contents of the directory that is synced, not the directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents. See extended explanation in the copy command if unsure.

      If dest:path doesn't exist, it is created and the source:path contents go there.

      +

      It is not possible to sync overlapping remotes. However, you may exclude the destination from the sync with a filter rule or by putting an exclude-if-present file inside the destination directory and sync to a destination that is inside the source directory.

      Note: Use the -P/--progress flag to view real-time transfer statistics

      Note: Use the rclone dedupe command to deal with "Duplicate object/directory found in source/destination - ignoring" errors. See this forum post for more info.

      rclone sync source:path dest:path [flags]
      @@ -616,7 +655,7 @@

      Synopsis

      ls,lsl,lsd are designed to be human-readable. lsf is designed to be human and machine-readable. lsjson is designed to be machine-readable.

      Note that ls and lsl recurse by default - use --max-depth 1 to stop the recursion.

      The other list commands lsd,lsf,lsjson do not recurse by default - use -R to make them recurse.

      -

      Listing a non-existent directory will produce an error except for remotes which can't have empty directories (e.g. s3, swift, or gcs - the bucket-based remotes).

      +

      Listing a nonexistent directory will produce an error except for remotes which can't have empty directories (e.g. s3, swift, or gcs - the bucket-based remotes).

      rclone ls remote:path [flags]

      Options

        -h, --help   help for ls
      @@ -651,7 +690,7 @@

      Synopsis

      ls,lsl,lsd are designed to be human-readable. lsf is designed to be human and machine-readable. lsjson is designed to be machine-readable.

      Note that ls and lsl recurse by default - use --max-depth 1 to stop the recursion.

      The other list commands lsd,lsf,lsjson do not recurse by default - use -R to make them recurse.

      -

      Listing a non-existent directory will produce an error except for remotes which can't have empty directories (e.g. s3, swift, or gcs - the bucket-based remotes).

      +

      Listing a nonexistent directory will produce an error except for remotes which can't have empty directories (e.g. s3, swift, or gcs - the bucket-based remotes).

      rclone lsd remote:path [flags]

      Options

        -h, --help        help for lsd
@@ -683,7 +722,7 @@ 
      Synopsis
      
 
      ls,lsl,lsd are designed to be human-readable. lsf is designed to be human and machine-readable. lsjson is designed to be machine-readable.
      
 
      Note that ls and lsl recurse by default - use --max-depth 1 to stop the recursion.
      
 
      The other list commands lsd,lsf,lsjson do not recurse by default - use -R to make them recurse.
      
-
      Listing a non-existent directory will produce an error except for remotes which can't have empty directories (e.g. s3, swift, or gcs - the bucket-based remotes).
      
+
      Listing a nonexistent directory will produce an error except for remotes which can't have empty directories (e.g. s3, swift, or gcs - the bucket-based remotes).
      
 
      rclone lsl remote:path [flags]
      
 
      Options
      
 
        -h, --help   help for lsl
      
@@ -698,7 +737,7 @@ 
      Synopsis
      
 
      Produces an md5sum file for all the objects in the path. This is in the same format as the standard md5sum tool produces.
      
 
      By default, the hash is requested from the remote. If MD5 is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling MD5 for any remote.
      
 
      For other algorithms, see the hashsum command. Running rclone md5sum remote:path is equivalent to running rclone hashsum MD5 remote:path.
      
-
      This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a relative path).
      
+
      This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hyphen will be treated literally, as a relative path).
      
 
      rclone md5sum remote:path [flags]
      
 
      Options
      
 
            --base64               Output base64 encoded hashsum
@@ -717,7 +756,7 @@ 
      Synopsis
      
 
      Produces an sha1sum file for all the objects in the path. This is in the same format as the standard sha1sum tool produces.
      
 
      By default, the hash is requested from the remote. If SHA-1 is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling SHA-1 for any remote.
      
 
      For other algorithms, see the hashsum command. Running rclone sha1sum remote:path is equivalent to running rclone hashsum SHA1 remote:path.
      
-
      This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a relative path).
      
+
      This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hyphen will be treated literally, as a relative path).
      
 
      This command can also hash data received on STDIN, if not passing a remote:path.
      
 
      rclone sha1sum remote:path [flags]
      
 
      Options
      
@@ -959,9 +998,9 @@ 
      SEE ALSO
      
 
    22. rclone - Show help for rclone commands, flags and backends.
      23. 
 
 
      rclone bisync
      
-
      Perform bidirectonal synchronization between two paths.
      
+
      Perform bidirectional synchronization between two paths.
      
 
      Synopsis
      
-
      Perform bidirectonal synchronization between two paths.
      
+
      Perform bidirectional synchronization between two paths.
      
 
      Bisync provides a bidirectional cloud sync solution in rclone. It retains the Path1 and Path2 filesystem listings from the prior run. On each successive run it will: - list files on Path1 and Path2, and check for changes on each side. Changes include New, Newer, Older, and Deleted files. - Propagate changes on Path1 to Path2, and vice-versa.
      
 
      See full bisync description for details.
      
 
      rclone bisync remote1:path1 remote2:path2 [flags]
      
@@ -1061,10 +1100,10 @@ 
      Synopsis
      
 
      To load completions in your current shell session:
      
 
      source <(rclone completion bash)
      
 
      To load completions for every new session, execute once:
      
-
      Linux:
      
+
      Linux:
      
 
      rclone completion bash > /etc/bash_completion.d/rclone
      
-
      macOS:
      
-
      rclone completion bash > /usr/local/etc/bash_completion.d/rclone
      
+
      macOS:
      
+
      rclone completion bash > $(brew --prefix)/etc/bash_completion.d/rclone
      
 
      You will need to start a new shell for this setup to take effect.
      
 
      rclone completion bash
      
 
      Options
      
@@ -1115,11 +1154,13 @@ 
      Synopsis
      
 
      Generate the autocompletion script for the zsh shell.
      
 
      If shell completion is not already enabled in your environment you will need to enable it. You can execute the following once:
      
 
      echo "autoload -U compinit; compinit" >> ~/.zshrc
      
+
      To load completions in your current shell session:
      
+
      source <(rclone completion zsh); compdef _rclone rclone
      
 
      To load completions for every new session, execute once:
      
-
      Linux:
      
+
      Linux:
      
 
      rclone completion zsh > "${fpath[1]}/_rclone"
      
-
      macOS:
      
-
      rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone
      
+
      macOS:
      
+
      rclone completion zsh > $(brew --prefix)/share/zsh/site-functions/_rclone
      
 
      You will need to start a new shell for this setup to take effect.
      
 
      rclone completion zsh [flags]
      
 
      Options
      
@@ -1142,7 +1183,7 @@ 
      Synopsis
      
 
      Note that if the config process would normally ask a question the default is taken (unless --non-interactive is used). Each time that happens rclone will print or DEBUG a message saying how to affect the value taken.
      
 
      If any of the parameters passed is a password field, then rclone will automatically obscure them if they aren't already obscured before putting them in the config file.
      
 
      NB If the password parameter is 22 characters or longer and consists only of base64 characters then rclone can get confused about whether the password is already obscured or not and put unobscured passwords into the config file. If you want to be 100% certain that the passwords get obscured then use the --obscure flag, or if you are 100% certain you are already passing obscured passwords then use --no-obscure. You can also set obscured passwords using the rclone config password command.
      
-
      The flag --non-interactive is for use by applications that wish to configure rclone themeselves, rather than using rclone's text based configuration questions. If this flag is set, and rclone needs to ask the user a question, a JSON blob will be returned with the question in it.
      
+
      The flag --non-interactive is for use by applications that wish to configure rclone themselves, rather than using rclone's text based configuration questions. If this flag is set, and rclone needs to ask the user a question, a JSON blob will be returned with the question in it.
      
 
      This will look something like (some irrelevant detail removed):
      
 
      {
     "State": "*oauth-islocal,teamdrive,,",
@@ -1339,7 +1380,7 @@ 
      Synopsis
      
 
      Note that if the config process would normally ask a question the default is taken (unless --non-interactive is used). Each time that happens rclone will print or DEBUG a message saying how to affect the value taken.
      
 
      If any of the parameters passed is a password field, then rclone will automatically obscure them if they aren't already obscured before putting them in the config file.
      
 
      NB If the password parameter is 22 characters or longer and consists only of base64 characters then rclone can get confused about whether the password is already obscured or not and put unobscured passwords into the config file. If you want to be 100% certain that the passwords get obscured then use the --obscure flag, or if you are 100% certain you are already passing obscured passwords then use --no-obscure. You can also set obscured passwords using the rclone config password command.
      
-
      The flag --non-interactive is for use by applications that wish to configure rclone themeselves, rather than using rclone's text based configuration questions. If this flag is set, and rclone needs to ask the user a question, a JSON blob will be returned with the question in it.
      
+
      The flag --non-interactive is for use by applications that wish to configure rclone themselves, rather than using rclone's text based configuration questions. If this flag is set, and rclone needs to ask the user a question, a JSON blob will be returned with the question in it.
      
 
      This will look something like (some irrelevant detail removed):
      
 
      {
     "State": "*oauth-islocal,teamdrive,,",
@@ -1608,7 +1649,7 @@ 
      Synopsis
      
 
      Produces a hash file for all the objects in the path using the hash named. The output is in the same format as the standard md5sum/sha1sum tool.
      
 
      By default, the hash is requested from the remote. If the hash is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling any hash for any remote.
      
 
      For the MD5 and SHA1 algorithms there are also dedicated commands, md5sum and sha1sum.
      
-
      This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hypen will be treated literaly, as a relative path).
      
+
      This command can also hash data received on standard input (stdin), by not passing a remote:path, or by passing a hyphen as remote:path when there is data to read (if not, the hyphen will be treated literally, as a relative path).
      
 
      Run without a hash to see the list of all supported hashes, e.g.
      
 
      $ rclone hashsum
 Supported hashes are:
@@ -1742,7 +1783,7 @@ 
      Synopsis
      
 
      ls,lsl,lsd are designed to be human-readable. lsf is designed to be human and machine-readable. lsjson is designed to be machine-readable.
      
 
      Note that ls and lsl recurse by default - use --max-depth 1 to stop the recursion.
      
 
      The other list commands lsd,lsf,lsjson do not recurse by default - use -R to make them recurse.
      
-
      Listing a non-existent directory will produce an error except for remotes which can't have empty directories (e.g. s3, swift, or gcs - the bucket-based remotes).
      
+
      Listing a nonexistent directory will produce an error except for remotes which can't have empty directories (e.g. s3, swift, or gcs - the bucket-based remotes).
      
 
      rclone lsf remote:path [flags]
      
 
      Options
      
 
            --absolute           Put a leading / in front of path names
@@ -1790,7 +1831,7 @@ 
      Synopsis
      
 
      If --encrypted is not specified the Encrypted won't be emitted.
      
 
      If --dirs-only is not specified files in addition to directories are returned
      
 
      If --files-only is not specified directories in addition to the files will be returned.
      
-
      If --metadata is set then an additional Metadata key will be returned. This will have metdata in rclone standard format as a JSON object.
      
+
      If --metadata is set then an additional Metadata key will be returned. This will have metadata in rclone standard format as a JSON object.
      
 
      if --stat is set then a single JSON blob will be returned about the item pointed to. This will return an error if the item isn't found. However on bucket based backends (like s3, gcs, b2, azureblob etc) if the item isn't found it will return an empty directory as it isn't possible to tell empty directories from missing directories there.
      
 
      The Path field will only show folders below the remote path being listed. If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt" will be "subfolder/file.txt", not "remote:path/subfolder/file.txt". When used without --recursive the Path will always be the same as Name.
      
 
      If the directory is a bucket in a bucket-based backend, then "IsBucket" will be set to true. This key won't be present unless it is "true".
      
@@ -1808,7 +1849,7 @@ 
      Synopsis
      
 
      ls,lsl,lsd are designed to be human-readable. lsf is designed to be human and machine-readable. lsjson is designed to be machine-readable.
      
 
      Note that ls and lsl recurse by default - use --max-depth 1 to stop the recursion.
      
 
      The other list commands lsd,lsf,lsjson do not recurse by default - use -R to make them recurse.
      
-
      Listing a non-existent directory will produce an error except for remotes which can't have empty directories (e.g. s3, swift, or gcs - the bucket-based remotes).
      
+
      Listing a nonexistent directory will produce an error except for remotes which can't have empty directories (e.g. s3, swift, or gcs - the bucket-based remotes).
      
 
      rclone lsjson remote:path [flags]
      
 
      Options
      
 
            --dirs-only               Show only directories in the listing
@@ -1856,7 +1897,7 @@ 
      Installing on Windows
      
 
      Mounting modes on windows
      
 
      Unlike other operating systems, Microsoft Windows provides a different filesystem type for network and fixed drives. It optimises access on the assumption fixed disk drives are fast and reliable, while network drives have relatively high latency and less reliability. Some settings can also be differentiated between the two types, for example that Windows Explorer should just display icons and not create preview thumbnails for image and video files on network drives.
      
 
      In most cases, rclone will mount the remote as a normal, fixed disk drive by default. However, you can also choose to mount it as a remote network drive, often described as a network share. If you mount an rclone remote using the default, fixed drive mode and experience unexpected program errors, freezes or other issues, consider mounting as a network drive instead.
      
-
      When mounting as a fixed disk drive you can either mount to an unused drive letter, or to a path representing a non-existent subdirectory of an existing parent directory or drive. Using the special value * will tell rclone to automatically assign the next available drive letter, starting with Z: and moving backward. Examples:
      
+
      When mounting as a fixed disk drive you can either mount to an unused drive letter, or to a path representing a nonexistent subdirectory of an existing parent directory or drive. Using the special value * will tell rclone to automatically assign the next available drive letter, starting with Z: and moving backward. Examples:
      
 
      rclone mount remote:path/to/files *
 rclone mount remote:path/to/files X:
 rclone mount remote:path/to/files C:\path\parent\mount
@@ -1865,10 +1906,10 @@ 
      Mounting modes on windows
      
 
      To mount as network drive, you can add option --network-mode to your mount command. Mounting to a directory path is not supported in this mode, it is a limitation Windows imposes on junctions, so the remote must always be mounted to a drive letter.
      
 
      rclone mount remote:path/to/files X: --network-mode
      
 
      A volume name specified with --volname will be used to create the network share path. A complete UNC path, such as \\cloud\remote, optionally with path \\cloud\remote\madeup\path, will be used as is. Any other string will be used as the share part, after a default prefix \\server\. If no volume name is specified then \\server\share will be used. You must make sure the volume name is unique when you are mounting more than one drive, or else the mount command will fail. The share name will treated as the volume label for the mapped drive, shown in Windows Explorer etc, while the complete \\server\share will be reported as the remote UNC path by net use etc, just like a normal network drive mapping.
      
-
      If you specify a full network share UNC path with --volname, this will implicitely set the --network-mode option, so the following two examples have same result:
      
+
      If you specify a full network share UNC path with --volname, this will implicitly set the --network-mode option, so the following two examples have same result:
      
 
      rclone mount remote:path/to/files X: --network-mode
 rclone mount remote:path/to/files X: --volname \\server\share
      
-
      You may also specify the network share UNC path as the mountpoint itself. Then rclone will automatically assign a drive letter, same as with * and use that as mountpoint, and instead use the UNC path specified as the volume name, as if it were specified with the --volname option. This will also implicitely set the --network-mode option. This means the following two examples have same result:
      
+
      You may also specify the network share UNC path as the mountpoint itself. Then rclone will automatically assign a drive letter, same as with * and use that as mountpoint, and instead use the UNC path specified as the volume name, as if it were specified with the --volname option. This will also implicitly set the --network-mode option. This means the following two examples have same result:
      
 
      rclone mount remote:path/to/files \\cloud\remote
 rclone mount remote:path/to/files * --volname \\cloud\remote
      
 
      There is yet another way to enable network mode, and to set the share path, and that is to pass the "native" libfuse/WinFsp option directly: --fuse-flag --VolumePrefix=\server\share. Note that the path must be with just a single backslash prefix in this case.
      
@@ -1878,7 +1919,7 @@ 
      Mounting modes on windows
      
 
      Windows filesystem permissions
      
 
      The FUSE emulation layer on Windows must convert between the POSIX-based permission model used in FUSE, and the permission model used in Windows, based on access-control lists (ACL).
      
 
      The mounted filesystem will normally get three entries in its access-control list (ACL), representing permissions for the POSIX permission scopes: Owner, group and others. By default, the owner and group will be taken from the current user, and the built-in group "Everyone" will be used to represent others. The user/group can be customized with FUSE options "UserName" and "GroupName", e.g. -o UserName=user123 -o GroupName="Authenticated Users". The permissions on each entry will be set according to options --dir-perms and --file-perms, which takes a value in traditional numeric notation.
      
-
      The default permissions corresponds to --file-perms 0666 --dir-perms 0777, i.e. read and write permissions to everyone. This means you will not be able to start any programs from the the mount. To be able to do that you must add execute permissions, e.g. --file-perms 0777 --dir-perms 0777 to add it to everyone. If the program needs to write files, chances are you will have to enable VFS File Caching as well (see also limitations).
      
+
      The default permissions corresponds to --file-perms 0666 --dir-perms 0777, i.e. read and write permissions to everyone. This means you will not be able to start any programs from the mount. To be able to do that you must add execute permissions, e.g. --file-perms 0777 --dir-perms 0777 to add it to everyone. If the program needs to write files, chances are you will have to enable VFS File Caching as well (see also limitations).
      
 
      Note that the mapping of permissions is not always trivial, and the result you see in Windows Explorer may not be exactly like you expected. For example, when setting a value that includes write access, this will be mapped to individual permissions "write attributes", "write data" and "append data", but not "write extended attributes". Windows will then show this as basic permission "Special" instead of "Write", because "Write" includes the "write extended attributes" permission.
      
 
      If you set POSIX permissions for only allowing access to the owner, using --file-perms 0600 --dir-perms 0700, the user group and the built-in "Everyone" group will still be given some special permissions, such as "read attributes" and "read permissions", in Windows. This is done for compatibility reasons, e.g. to allow users without additional permissions to be able to read basic metadata about files like in UNIX. One case that may arise is that other programs (incorrectly) interprets this as the file being accessible by everyone. For example an SSH client may warn about "unprotected private key file".
      
 
      WinFsp 2021 (version 1.9) introduces a new FUSE option "FileSecurity", that allows the complete specification of file security descriptors using SDDL. With this you can work around issues such as the mentioned "unprotected private key file" by specifying -o FileSecurity="D:P(A;;FA;;;OW)", for file all access (FA) to the owner (OW).
      
@@ -1890,7 +1931,7 @@ 
      Windows caveats
      
 
      Note that mapping to a directory path, instead of a drive letter, does not suffer from the same limitations.
      
 
      Limitations
      
 
      Without the use of --vfs-cache-mode this can only write files sequentially, it can only seek when reading. This means that many applications won't work with their files on an rclone mount without --vfs-cache-mode writes or --vfs-cache-mode full. See the VFS File Caching section for more info.
      
-
      The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2, Hubic) do not support the concept of empty directories, so empty directories will have a tendency to disappear once they fall out of the directory cache.
      
+
      The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2) do not support the concept of empty directories, so empty directories will have a tendency to disappear once they fall out of the directory cache.
      
 
      When rclone mount is invoked on Unix with --daemon flag, the main rclone program will wait for the background mount to become ready or until the timeout specified by the --daemon-wait flag. On Linux it can check mount status using ProcFS so the flag in fact sets maximum time to wait, while the real wait can be less. On macOS / BSD the time to wait is constant and the check is performed only at the end. We advise you to set wait time on macOS reasonably.
      
 
      Only supported on Linux, FreeBSD, OS X and Windows at the moment.
      
 
      rclone mount vs rclone sync/copy
      
@@ -2159,7 +2200,7 @@ 
      Synopsis
      
  ^L refresh screen (fix screen corruption)
  ? to toggle help on and off
  q/ESC/^c to quit
      
-
      Listed files/directories may be prefixed by a one-character flag, some of them combined with a description in brackes at end of line. These flags have the following meaning:
      
+
      Listed files/directories may be prefixed by a one-character flag, some of them combined with a description in brackets at end of line. These flags have the following meaning:
      
 
      e means this is an empty directory, i.e. contains no files (but
   may contain empty subdirectories)
 ~ means this is a directory where some of the files (possibly in
@@ -2458,11 +2499,13 @@ 
      Alternate report of used bytes
      
 
      rclone serve dlna remote:path [flags]
      
 
      Options
      
 
            --addr string                            The ip:port or :port to bind the DLNA http server to (default ":7879")
+      --announce-interval duration             The interval between SSDP announcements (default 12m0s)
       --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
       --dir-perms FileMode                     Directory permissions (default 0777)
       --file-perms FileMode                    File permissions (default 0666)
       --gid uint32                             Override the gid field set by the filesystem (not supported on Windows) (default 1000)
   -h, --help                                   help for dlna
+      --interface stringArray                  The interface to use for SSDP (repeat as necessary)
       --log-trace                              Enable trace logging of SOAP traffic
       --name string                            Name of DLNA server
       --no-checksum                            Don't compare checksums on up/download
@@ -2885,6 +2928,7 @@ 
      Server options
      
 
      SSL/TLS
      
 
      By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.
      
 
      --cert should be a either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.
      
+
      --min-tls-version is minimum TLS version that is acceptable. Valid values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").
      
 
      Template
      
 
      --template allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages:
      
 
@@ -3105,6 +3149,7 @@ 
      Options
      
       --htpasswd string                        A htpasswd file - if not provided no authentication is done
       --key string                             SSL PEM Private key
       --max-header-bytes int                   Maximum size of request header (default 4096)
+      --min-tls-version string                 Minimum TLS version that is acceptable (default "tls1.0")
       --no-checksum                            Don't compare checksums on up/download
       --no-modtime                             Don't read/write the modification time (can speed things up)
       --no-seek                                Don't allow seeking in files
@@ -3279,6 +3324,7 @@ 
      Authentication
      SSL/TLS
      By default this will serve over HTTP. If you want you can serve over HTTPS. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.
      --cert should be either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.
      
+
      --min-tls-version is minimum TLS version that is acceptable. Valid values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").
      rclone serve restic remote:path [flags]
      Options
            --addr string                     IPaddress:Port or :Port to bind server to (default "localhost:8080")
@@ -3291,6 +3337,7 @@ 
      Options
      
       --htpasswd string                 htpasswd file - if not provided no authentication is done
       --key string                      SSL PEM Private key
       --max-header-bytes int            Maximum size of request header (default 4096)
+      --min-tls-version string          Minimum TLS version that is acceptable (default "tls1.0")
       --pass string                     Password for authentication
       --private-repos                   Users can only access their private repo
       --realm string                    Realm for authentication (default "rclone")
@@ -3307,12 +3354,13 @@ 
      SEE ALSO
      
 
      rclone serve sftp
      
 
      Serve the remote over SFTP.
      
 
      Synopsis
      
-
      Run a SFTP server to serve a remote over SFTP. This can be used with an SFTP client or you can make a remote of type sftp to use with it.
      
-
      You can use the filter flags (e.g. --include, --exclude) to control what is served.
      
+
      Run an SFTP server to serve a remote over SFTP. This can be used with an SFTP client or you can make a remote of type sftp to use with it.
      
+
      You can use the filter flags (e.g. --include, --exclude) to control what is served.
      
+
      The server will respond to a small number of shell commands, mainly md5sum, sha1sum and df, which enable it to provide support for checksums and the about feature when accessed from an sftp remote.
      
+
      Note that this server uses standard 32 KiB packet payload size, which means you must not configure the client to expect anything else, e.g. with the chunk_size option on an sftp remote.
      
 
      The server will log errors. Use -v to see access logs.
      
 
      --bwlimit will be respected for file transfers. Use --stats to control the stats printing.
      
 
      You must provide some means of authentication, either with --user/--pass, an authorized keys file (specify location with --authorized-keys - the default is the same as ssh), an --auth-proxy, or set the --no-auth flag for no authentication when logging in.
      
-
      Note that this also implements a small number of shell commands so that it can provide md5sum/sha1sum/df information for the rclone sftp backend. This means that is can support SHA1SUMs, MD5SUMs and the about command when paired with the rclone sftp backend.
      
 
      If you don't supply a host --key then rclone will generate rsa, ecdsa and ed25519 variants, and cache them for later use in rclone's cache directory (see rclone help flags cache-dir) in the "serve-sftp" directory.
      
 
      By default the server binds to localhost:2022 - if you want it to be reachable externally then supply --addr :2022 for example.
      
 
      Note that the default of --vfs-cache-mode off is fine for the rclone sftp backend, but it may not be with other SFTP clients.
      
@@ -3483,7 +3531,7 @@ 
      Options
      
       --pass string                            Password for authentication
       --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --read-only                              Only allow read-only access
-      --stdio                                  Run an sftp server on run stdin/stdout
+      --stdio                                  Run an sftp server on stdin/stdout
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
       --user string                            User name for authentication
@@ -3612,6 +3660,7 @@ 
      Authentication
      
 
      SSL/TLS
      
 
      By default this will serve over HTTP. If you want you can serve over HTTPS. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.
      
 
      --cert should be either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.
      
+
      --min-tls-version is minimum TLS version that is acceptable. Valid values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").
      
 
      VFS - Virtual File System
      
 
      This command uses the VFS layer. This adapts the cloud storage objects that rclone uses into something which looks much more like a disk filing system.
      
 
      Cloud storage objects have lots of properties which aren't like disk files - you can't extend them or write to the middle of them, so the VFS layer has to deal with that. Because there is no one right way of doing this there are various options explained below.
      
@@ -3774,6 +3823,7 @@ 
      Options
      
       --htpasswd string                        htpasswd file - if not provided no authentication is done
       --key string                             SSL PEM Private key
       --max-header-bytes int                   Maximum size of request header (default 4096)
+      --min-tls-version string                 Minimum TLS version that is acceptable (default "tls1.0")
       --no-checksum                            Don't compare checksums on up/download
       --no-modtime                             Don't read/write the modification time (can speed things up)
       --no-seek                                Don't allow seeking in files
@@ -4093,7 +4143,7 @@ 
      Linux / OSX
      
 
      If you want to send a ' you will need to use ", e.g.
      
 
      rclone copy "O'Reilly Reviews" remote:backup
      
 
      The rules for quoting metacharacters are complicated and if you want the full details you'll have to consult the manual page for your shell.
      
-
      Windows
      
+
      Windows
      
 
      If your names have spaces in you need to put them in ", e.g.
      
 
      rclone copy "E:\folder name\folder name\folder name" remote:backup
      
 
      If you are using the root directory on its own then don't quote it (see #464 for why), e.g.
      
@@ -4138,7 +4188,7 @@ 
      Metadata framework
      
 
    23. length is backend dependent
      24. 
 
 
      Each backend can provide system metadata that it understands. Some backends can also store arbitrary user metadata.
      
-
      Where possible the key names are standardized, so, for example, it is possible to copy object metadata from s3 to azureblob for example and metadata will be translated apropriately.
      
+
      Where possible the key names are standardized, so, for example, it is possible to copy object metadata from s3 to azureblob for example and metadata will be translated appropriately.
      
 
      Some backends have limits on the size of the metadata and rclone will give errors on upload if they are exceeded.
      
 
      Metadata preservation
      
 
      The goal of the implementation is to
      
@@ -4231,12 +4281,32 @@ 
      Standard system metadata
      
 
      Options
      
 
      Rclone has a number of options to control its behaviour.
      
 
      Options that take parameters can have the values passed in two ways, --option=value or --option value. However boolean (true/false) options behave slightly differently to the other options in that --boolean sets the option to true and the absence of the flag sets it to false. It is also possible to specify --boolean=false or --boolean=true. Note that --boolean false is not valid - this is parsed as --boolean and the false is parsed as an extra command line argument for rclone.
      
-
      Options which use TIME use the go time parser. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
      
+
      Time or duration options
      
+
      TIME or DURATION options can be specified as a duration string or a time string.
      
+
      A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Default units are seconds or the following abbreviations are valid:
      
+
        
+
      • ms - Milliseconds
        • 
+
      • s - Seconds
        • 
+
      • m - Minutes
        • 
+
      • h - Hours
        • 
+
      • d - Days
        • 
+
      • w - Weeks
        • 
+
      • M - Months
        • 
+
      • y - Years
        • 
+
      
+
      These can also be specified as an absolute time in the following formats:
      
+
        
+
      • RFC3339 - e.g. 2006-01-02T15:04:05Z or 2006-01-02T15:04:05+07:00
        • 
+
      • ISO8601 Date and time, local timezone - 2006-01-02T15:04:05
        • 
+
      • ISO8601 Date and time, local timezone - 2006-01-02 15:04:05
        • 
+
      • ISO8601 Date - 2006-01-02 (YYYY-MM-DD)
        • 
+
      
+
      Size options
      
 
      Options which use SIZE use KiB (multiples of 1024 bytes) by default. However, a suffix of B for Byte, K for KiB, M for MiB, G for GiB, T for TiB and P for PiB may be used. These are the binary units, e.g. 1, 2**10, 2**20, 2**30 respectively.
      
 
      --backup-dir=DIR
      
 
      When using sync, copy or move any files which would have been overwritten or deleted are moved in their original hierarchy into this directory.
      
 
      If --suffix is set, then the moved files will have the suffix added to them. If there is a file with the same path (after the suffix has been added) in DIR, then it will be overwritten.
      
-
      The remote in use must support server-side move or copy and you must use the same remote as the destination of the sync. The backup directory must not overlap the destination directory.
      
+
      The remote in use must support server-side move or copy and you must use the same remote as the destination of the sync. The backup directory must not overlap the destination directory without it being excluded by a filter rule.
      
 
      For example
      
 
      rclone sync -i /path/to/local remote:current --backup-dir remote:old
      
 
      will sync /path/to/local to remote:current, but for any files which would have been updated or deleted will be stored in remote:old.
      
@@ -4248,7 +4318,7 @@ 
      --bwlimit=BANDWIDTH_SPEC
      
 
      This option controls the bandwidth limit. For example
      
 
      --bwlimit 10M
      
 
      would mean limit the upload and download bandwidth to 10 MiB/s. NB this is bytes per second not bits per second. To use a single limit, specify the desired bandwidth in KiB/s, or use a suffix B|K|M|G|T|P. The default is 0 which means to not limit bandwidth.
      
-
      The upload and download bandwidth can be specified seperately, as --bwlimit UP:DOWN, so
      
+
      The upload and download bandwidth can be specified separately, as --bwlimit UP:DOWN, so
      
 
      --bwlimit 10M:100k
      
 
      would mean limit the upload bandwidth to 10 MiB/s and the download bandwidth to 100 KiB/s. Either limit can be "off" meaning no limit, so to just limit the upload bandwidth you would use
      
 
      --bwlimit 10M:off
      
@@ -4622,6 +4692,10 @@ 
      --retries int
      
 
      --retries-sleep=TIME
      
 
      This sets the interval between each retry specified by --retries
      
 
      The default is 0. Use 0 to disable.
      
+
      --server-side-across-configs
      
+
      Allow server-side operations (e.g. copy or move) to work across different configurations.
      
+
      This can be useful if you wish to do a server-side copy or move between two remotes which use the same backend but are configured differently.
      
+
      Note that this isn't enabled by default because it isn't easy for rclone to tell if it will work between any two configurations.
      
 
      --size-only
      
 
      Normally rclone will look at modification time and size of files to see if they are equal. If you set this flag then rclone will check only the size.
      
 
      This can be useful transferring files from Dropbox which have been modified by the desktop sync client which doesn't set checksums of modification times in the same way as rclone.
      
@@ -4685,14 +4759,15 @@ 
      --tpslimit-burst int
      
 
      This may be used to increase performance of --tpslimit without changing the long term average number of transactions per second.
      
 
      --track-renames
      
 
      By default, rclone doesn't keep track of renamed files, so if you rename a file locally then sync it to a remote, rclone will delete the old file on the remote and upload a new copy.
      
-
      If you use this flag, and the remote supports server-side copy or server-side move, and the source and destination have a compatible hash, then this will track renames during sync operations and perform renaming server-side.
      
-
      Files will be matched by size and hash - if both match then a rename will be considered.
      
+
      An rclone sync with --track-renames runs like a normal sync, but keeps track of objects which exist in the destination but not in the source (which would normally be deleted), and which objects exist in the source but not the destination (which would normally be transferred). These objects are then candidates for renaming.
      
+
      After the sync, rclone matches up the source only and destination only objects using the --track-renames-strategy specified and either renames the destination object or transfers the source and deletes the destination object. --track-renames is stateless like all of rclone's syncs.
      
+
      To use this flag the destination must support server-side copy or server-side move, and to use a hash based --track-renames-strategy (the default) the source and the destination must have a compatible hash.
      
 
      If the destination does not support server-side copy or move, rclone will fall back to the default behaviour and log an error level message to the console.
      
 
      Encrypted destinations are not currently supported by --track-renames if --track-renames-strategy includes hash.
      
 
      Note that --track-renames is incompatible with --no-traverse and that it uses extra memory to keep track of all the rename candidates.
      
 
      Note also that --track-renames is incompatible with --delete-before and will select --delete-after instead of --delete-during.
      
 
      --track-renames-strategy (hash,modtime,leaf,size)
      
-
      This option changes the matching criteria for --track-renames.
      
+
      This option changes the file matching criteria for --track-renames.
      
 
      The matching is controlled by a comma separated selection of these tokens:
      
 
        
 
      • modtime - the modification time of the file - not supported on all backends
        • 
@@ -4700,9 +4775,9 @@ 
        --track-renames-strategy (ha
 
      • leaf - the name of the file not including its directory name
        • 
 
      • size - the size of the file (this is always enabled)
        • 
 
      
-
      So using --track-renames-strategy modtime,leaf would match files based on modification time, the leaf of the file name and the size only.
      
+
      The default option is hash.
      
+
      Using --track-renames-strategy modtime,leaf would match files based on modification time, the leaf of the file name and the size only.
      
 
      Using --track-renames-strategy modtime or leaf can enable --track-renames support for encrypted destinations.
      
-
      If nothing is specified, the default option is matching by hashes.
      
 
      Note that the hash strategy is not supported with encrypted destinations.
      
 
      --delete-(before,during,after)
      
 
      This option allows you to specify when files on your destination are deleted when you sync folders.
      
@@ -4711,7 +4786,7 @@ 
      --delete-(before,during,after)
      
 
      Specifying --delete-after (the default value) will delay deletion of files until all new/updated files have been successfully transferred. The files to be deleted are collected in the copy pass then deleted after the copy pass has completed successfully. The files to be deleted are held in memory so this mode may use more memory. This is the safest mode as it will only delete files if there have been no errors subsequent to that. If there have been errors before the deletions start then you will get the message not deleting files as there were IO errors.
      
 
      --fast-list
      
 
      When doing anything which involves a directory listing (e.g. sync, copy, ls - in fact nearly every command), rclone normally lists a directory and processes it before using more directory lists to process any subdirectories. This can be parallelised and works very quickly using the least amount of memory.
      
-
      However, some remotes have a way of listing all files beneath a directory in one (or a small number) of transactions. These tend to be the bucket-based remotes (e.g. S3, B2, GCS, Swift, Hubic).
      
+
      However, some remotes have a way of listing all files beneath a directory in one (or a small number) of transactions. These tend to be the bucket-based remotes (e.g. S3, B2, GCS, Swift).
      
 
      If you use the --fast-list flag then rclone will use this method for listing directories. This will have the following consequences for the listing:
      
 
        
 
      • It will use fewer transactions (important if you pay for them)
        • 
@@ -4735,7 +4810,7 @@ 
        -u, --update
        
 
        If an existing destination file has a modification time older than the source file's, it will be updated if the sizes are different. If the sizes are the same, it will be updated if the checksum is different or not available.
        
 
        If an existing destination file has a modification time equal (within the computed modify window) to the source file's, it will be updated if the sizes are different. The checksum will not be checked in this case unless the --checksum flag is provided.
        
 
        In all other cases the file will not be updated.
        
-
        Consider using the --modify-window flag to compensate for time skews between the source and the backend, for backends that do not support mod times, and instead use uploaded times. However, if the backend does not support checksums, note that sync'ing or copying within the time skew window may still result in additional transfers for safety.
        
+
        Consider using the --modify-window flag to compensate for time skews between the source and the backend, for backends that do not support mod times, and instead use uploaded times. However, if the backend does not support checksums, note that syncing or copying within the time skew window may still result in additional transfers for safety.
        
 
        --use-mmap
        
 
        If this flag is set then rclone will use anonymous memory allocated by mmap on Unix based platforms and VirtualAlloc on Windows for its transfer buffers (size controlled by --buffer-size). Memory allocated like this does not go on the Go heap and can be returned to the OS immediately when it is finished with.
        
 
        If this flag is not set then rclone will allocate and free the buffers using the Go memory allocator which may use more memory as memory pages are returned less aggressively to the OS.
        
@@ -5149,7 +5224,7 @@ 
        Filter pattern examples
        
 
      
-
+
@@ -5430,34 +5505,21 @@ 
      Other filters
      
 
      --min-size - Don't transfer any file smaller than this
      
 
      Controls the minimum size file within the scope of an rclone command. Default units are KiB but abbreviations K, M, G, T or P are valid.
      
 
      E.g. rclone ls remote: --min-size 50k lists files on remote: of 50 KiB size or larger.
      
+
      See the size option docs for more info.
      
 
      --max-size - Don't transfer any file larger than this
      
 
      Controls the maximum size file within the scope of an rclone command. Default units are KiB but abbreviations K, M, G, T or P are valid.
      
 
      E.g. rclone ls remote: --max-size 1G lists files on remote: of 1 GiB size or smaller.
      
+
      See the size option docs for more info.
      
 
      --max-age - Don't transfer any file older than this
      
-
      Controls the maximum age of files within the scope of an rclone command. Default units are seconds or the following abbreviations are valid:
      
-
        
-
      • ms - Milliseconds
        • 
-
      • s - Seconds
        • 
-
      • m - Minutes
        • 
-
      • h - Hours
        • 
-
      • d - Days
        • 
-
      • w - Weeks
        • 
-
      • M - Months
        • 
-
      • y - Years
        • 
-
      
-
      --max-age can also be specified as an absolute time in the following formats:
      
-
        
-
      • RFC3339 - e.g. 2006-01-02T15:04:05Z or 2006-01-02T15:04:05+07:00
        • 
-
      • ISO8601 Date and time, local timezone - 2006-01-02T15:04:05
        • 
-
      • ISO8601 Date and time, local timezone - 2006-01-02 15:04:05
        • 
-
      • ISO8601 Date - 2006-01-02 (YYYY-MM-DD)
        • 
-
      
+
      Controls the maximum age of files within the scope of an rclone command.
      
 
      --max-age applies only to files and not to directories.
      
 
      E.g. rclone ls remote: --max-age 2d lists files on remote: of 2 days old or less.
      
+
      See the time option docs for valid formats.
      
 
      --min-age - Don't transfer any file younger than this
      
 
      Controls the minimum age of files within the scope of an rclone command. (see --max-age for valid formats)
      
 
      --min-age applies only to files and not to directories.
      
 
      E.g. rclone ls remote: --min-age 2d lists files on remote: of 2 days old or more.
      
+
      See the time option docs for valid formats.
      
 
      Other flags
      
 
      --delete-excluded - Delete files on dest excluded from sync
      
 
      Important this flag is dangerous to your data - use with --dry-run and -v first.
      
@@ -5573,6 +5635,8 @@ 
      --rc-key=PATH
      
 
      SSL PEM Private key
      
 
      --rc-max-header-bytes=VALUE
      
 
      Maximum size of request header (default 4096)
      
+
      --rc-min-tls-version=VALUE
      
+
      The minimum TLS version that is acceptable. Valid values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").
      
 
      --rc-user=VALUE
      
 
      User name for authentication.
      
 
      --rc-pass=VALUE
      
@@ -5743,7 +5807,7 @@ 
      Data types
      
 
      Specifying remotes to work on
      
 
      Remotes are specified with the fs=, srcFs=, dstFs= parameters depending on the command being used.
      
 
      The parameters can be a string as per the rest of rclone, eg s3:bucket/path or :sftp:/my/dir. They can also be specified as JSON blobs.
      
-
      If specifyng a JSON blob it should be a object mapping strings to strings. These values will be used to configure the remote. There are 3 special values which may be set:
      
+
      If specifying a JSON blob it should be a object mapping strings to strings. These values will be used to configure the remote. There are 3 special values which may be set:
      
 
        
 
      • type - set to type to specify a remote called :type:
        • 
 
      • _name - set to name to specify a remote called name:
        • 
@@ -6140,6 +6204,11 @@ 
        job/stop: Stop the running job
        
 
          
 
        • jobid - id of the job (integer).
          • 
 
        
+
        job/stopgroup: Stop all running jobs in a group
        
+
        Parameters:
        
+
          
+
        • group - name of the group (string).
          • 
+
        
 
        mount/listmounts: Show current mount points
        
 
        This shows currently mounted points, which can be used for performing an unmount.
        
 
        This takes no parameters and returns
        
@@ -6186,8 +6255,8 @@ 
        mount/unmount: Unmount selected active mount
        
 
        Example:
        
 
        rclone rc mount/unmount mountPoint=/home/<user>/mountPoint
        
 
        Authentication is required for this call.
        
-
        mount/unmountall: Show current mount points
        
-
        This shows currently mounted points, which can be used for performing an unmount.
        
+
        mount/unmountall: Unmount all active mounts
        
+
        rclone allows Linux, FreeBSD, macOS and Windows to mount any of Rclone's cloud storage systems as a file system with FUSE.
        
 
        This takes no parameters and returns error if unmount does not succeed.
        
 
        Eg
        
 
        rclone rc mount/unmountall
        
@@ -6569,7 +6638,7 @@ 
        rc/noop: Echo the input to the output parameters
        
 
        rc/noopauth: Echo the input to the output parameters requiring auth
        
 
        This echoes the input parameters to the output parameters for testing purposes. It can be used to check that rclone is still alive and to check that parameter passing is working properly.
        
 
        Authentication is required for this call.
        
-
        sync/bisync: Perform bidirectonal synchronization between two paths.
        
+
        sync/bisync: Perform bidirectional synchronization between two paths.
        
 
        This takes the following parameters
        
 
          
 
        • path1 - a remote directory string e.g. drive:path1
          • 
@@ -6954,15 +7023,6 @@ 
          Features
          
 
      
-
-
-
-
-
-
-
-
-
@@ -6971,7 +7031,7 @@ 
      Features
      
       
-
+
@@ -6980,7 +7040,7 @@ 
      Features
      
       
-
+
@@ -6989,7 +7049,7 @@ 
      Features
      
       
-
+
@@ -6998,7 +7058,7 @@ 
      Features
      
       
-
+
@@ -7007,7 +7067,7 @@ 
      Features
      
       
-
+
@@ -7016,7 +7076,7 @@ 
      Features
      
       
-
+
@@ -7025,7 +7085,7 @@ 
      Features
      
       
-
+
@@ -7034,7 +7094,7 @@ 
      Features
      
       
-
+
@@ -7043,7 +7103,7 @@ 
      Features
      
       
-
+
@@ -7052,6 +7112,15 @@ 
      Features
      
       
+
+
+
+
+
+
+
+
+
@@ -7116,6 +7185,15 @@ 
      Features
      
       
+
+
+
+
+
+
+
+
+
@@ -7124,7 +7202,7 @@ 
      Features
      
       
-
+
@@ -7133,7 +7211,7 @@ 
      Features
      
       
-
+
@@ -7142,7 +7220,7 @@ 
      Features
      
       
-
+
@@ -7151,7 +7229,7 @@ 
      Features
      
       
-
+
@@ -7160,7 +7238,7 @@ 
      Features
      
       
-
+
@@ -7169,7 +7247,7 @@ 
      Features
      
       
-
+
@@ -7197,7 +7275,7 @@ 
      Hash
      
 
      The cloud storage system supports various hash types of the objects. The hashes are used when transferring data as an integrity check and can be specifically used with the --checksum flag in syncs and in the check command.
      
 
      To use the verify checksums when transferring between cloud storage systems they must support a common hash type.
      
 
      ModTime
      
-
      Allmost all cloud storage systems store some sort of timestamp on objects, but several of them not something that is appropriate to use for syncing. E.g. some backends will only write a timestamp that represent the time of the upload. To be relevant for syncing it should be able to store the modification time of the source object. If this is not the case, rclone will only check the file size by default, though can be configured to check the file hash (with the --checksum flag). Ideally it should also be possible to change the timestamp of an existing file without having to re-upload it.
      
+
      Almost all cloud storage systems store some sort of timestamp on objects, but several of them not something that is appropriate to use for syncing. E.g. some backends will only write a timestamp that represent the time of the upload. To be relevant for syncing it should be able to store the modification time of the source object. If this is not the case, rclone will only check the file size by default, though can be configured to check the file hash (with the --checksum flag). Ideally it should also be possible to change the timestamp of an existing file without having to re-upload it.
      
 
      Storage systems with a - in the ModTime column, means the modification read on objects is not the modification time of the file when uploaded. It is most likely the time the file was uploaded, or possibly something else (like the time the picture was taken in Google Photos).
      
 
      Storage systems with a R (for read-only) in the ModTime column, means the it keeps modification times on objects, and updates them when uploading objects, but it does not support changing only the modification time (SetModTime operation) without re-uploading, possibly not even without deleting existing first. Some operations in rclone, such as copy and sync commands, will automatically check for SetModTime support and re-upload if necessary to keep the modification times in sync. Other commands will not work without SetModTime support, e.g. touch command on an existing file will fail, and changes to modification time only on a files in a mount will be silently ignored.
      
 
      Storage systems with R/W (for read/write) in the ModTime column, means they do also support modtime-only operations.
      
@@ -7872,19 +7950,6 @@ 
      Optional Features
      
       
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -7897,7 +7962,7 @@ 
      Optional Features
      
       
-
+
@@ -7910,7 +7975,7 @@ 
      Optional Features
      
       
-
+
@@ -7923,7 +7988,7 @@ 
      Optional Features
      
       
-
+
@@ -7936,7 +8001,7 @@ 
      Optional Features
      
       
-
+
@@ -7949,7 +8014,7 @@ 
      Optional Features
      
       
-
+
@@ -7962,7 +8027,7 @@ 
      Optional Features
      
       
-
+
@@ -7975,7 +8040,7 @@ 
      Optional Features
      
       
-
+
@@ -7988,7 +8053,7 @@ 
      Optional Features
      
       
-
+
@@ -8001,7 +8066,7 @@ 
      Optional Features
      
       
-
+
@@ -8014,6 +8079,19 @@ 
      Optional Features
      
       
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -8106,6 +8184,19 @@ 
      Optional Features
      
       
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -8118,7 +8209,7 @@ 
      Optional Features
      
       
-
+
@@ -8131,7 +8222,7 @@ 
      Optional Features
      
       
-
+
@@ -8144,7 +8235,7 @@ 
      Optional Features
      
       
-
+
@@ -8157,7 +8248,7 @@ 
      Optional Features
      
       
-
+
@@ -8170,7 +8261,7 @@ 
      Optional Features
      
       
-
+
@@ -8183,7 +8274,7 @@ 
      Optional Features
      
       
-
+
@@ -8200,7 +8291,7 @@ 
      Optional Features
      
 
      
 
 
 
 
 
 
      
 
 /dir/file.gif/dir/file.png
 /dir/file.gif
 
      
 
      -
 
      
 
      HubicMD5R/WNoNoR/W-
      
 Internet Archive
 MD5, SHA1, CRC32
 R/W ¹¹-
 RWU
 
      
 Jottacloud
 MD5
 R/WR
 -
 
      
 Koofr
 MD5
 --
 -
 
      
 Mail.ru Cloud
 Mailru ⁶
 R/W-
 -
 
      
 Mega
 -
 --
 -
 
      
 Memory
 MD5
 R/W-
 -
 
      
 Microsoft Azure Blob Storage
 MD5
 R/WR/W
 -
 
      
 Microsoft OneDrive
 SHA1 ⁵
 R/WR
 -
 
      
 OpenDrive
 MD5
 R/W-
 -
 
      
 OpenStack Swift
 MD5
 R/WR/W
 -
 
      Oracle Object StorageMD5R/WNoNoR/W-
      
 
      
 pCloud
 MD5, SHA1 ⁷-
 
      
 
      SMB--YesNo--
      
 SugarSync
 -
 --
 -
 
      
 Storj
 -
 R-
 -
 
      
 Uptobox
 -
 --
 -
 
      
 WebDAV
 MD5, SHA1 ³
 R ⁴-
 -
 
      
 Yandex Disk
 MD5
 R/WR
 -
 
      
 Zoho WorkDrive
 -
 --
 -
 
      
 The local filesystem
 All
 R/WYes
 
      
 
      HubicYes †YesNoNoNoYesYesNoYesNo
      
 Internet Archive
 No
 YesYes
 No
 
      
 Jottacloud
 Yes
 YesYes
 Yes
 
      
 Koofr
 Yes
 YesYes
 Yes
 
      
 Mail.ru Cloud
 Yes
 YesYes
 Yes
 
      
 Mega
 Yes
 NoYes
 Yes
 
      
 Memory
 No
 YesNo
 No
 
      
 Microsoft Azure Blob Storage
 Yes
 YesNo
 No
 
      
 Microsoft OneDrive
 Yes
 YesYes
 Yes
 
      
 OpenDrive
 Yes
 YesNo
 Yes
 
      
 OpenStack Swift
 Yes †
 YesYes
 No
 
      Oracle Object StorageYesYesNoNoYesYesNoNoNoNo
      
 
      
 pCloud
 YesYes
 
      
 
      SMBNoNoYesYesNoNoYesNoNoYes
      
 SugarSync
 Yes
 YesNo
 Yes
 
      
 Storj
 Yes †
 NoNo
 No
 
      
 Uptobox
 No
 YesNo
 No
 
      
 WebDAV
 Yes
 YesYes
 Yes
 
      
 Yandex Disk
 Yes
 YesYes
 Yes
 
      
 Zoho WorkDrive
 Yes
 YesYes
 Yes
 
      
 The local filesystem
 Yes
 No
      
 
      Purge
      
 
      This deletes a directory quicker than just deleting all the files in the directory.
      
-
      † Note Swift, Hubic, and Storj implement this in order to delete directory markers but they don't actually have a quicker way of deleting files other than deleting them individually.
      
+
      † Note Swift and Storj implement this in order to delete directory markers but they don't actually have a quicker way of deleting files other than deleting them individually.
      
 
      ‡ StreamUpload is not supported with Nextcloud
      
 
      Copy
      
 
      Used when copying an object to and from the same remote. This known as a server-side copy so you can copy a file without downloading it and uploading it again. It is used if you use rclone copy or rclone move if the remote doesn't support Move directly.
      
@@ -8337,6 +8428,7 @@ 
      Non Backend Flags
      
       --rc-job-expire-interval duration      Interval to check for expired async jobs (default 10s)
       --rc-key string                        SSL PEM Private key
       --rc-max-header-bytes int              Maximum size of request header (default 4096)
+      --rc-min-tls-version string            Minimum TLS version that is acceptable (default "tls1.0")
       --rc-no-auth                           Don't require auth for certain methods
       --rc-pass string                       Password for authentication
       --rc-realm string                      Realm for authentication (default "rclone")
@@ -8353,6 +8445,7 @@ 
      Non Backend Flags
      
       --refresh-times                        Refresh the modtime of remote files
       --retries int                          Retry operations this many times if they fail (default 3)
       --retries-sleep duration               Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)
+      --server-side-across-configs           Allow server-side operations (e.g. copy) to work across different configs
       --size-only                            Skip based on size only, not mod-time or checksum
       --stats duration                       Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s)
       --stats-file-name-length int           Max file name length in stats (0 for no limit) (default 45)
@@ -8378,7 +8471,7 @@ 
      Non Backend Flags
      
       --use-json-log                         Use json log format
       --use-mmap                             Use mmap allocator (see docs)
       --use-server-modtime                   Use server modified time instead of object metadata
-      --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.59.0")
+      --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.60.0")
   -v, --verbose count                        Print lots more stuff (repeat for more)
      
 
      Backend Flags
      
 
      These flags are available for every command. They control the backends and may be set in the config file.
      
@@ -8525,7 +8618,7 @@ 
      Backend Flags
      
       --drive-use-trash                              Send files to the trash instead of deleting permanently (default true)
       --drive-v2-download-min-size SizeSuffix        If Object's are greater, use drive v2 API to download (default off)
       --dropbox-auth-url string                      Auth server URL
-      --dropbox-batch-commit-timeout Duration        Max time to wait for a batch to finish comitting (default 10m0s)
+      --dropbox-batch-commit-timeout Duration        Max time to wait for a batch to finish committing (default 10m0s)
       --dropbox-batch-mode string                    Upload file batching sync|async|off (default "sync")
       --dropbox-batch-size int                       Max number of files in upload batch
       --dropbox-batch-timeout Duration               Max time to allow an idle upload batch before uploading (default 0s)
@@ -8559,6 +8652,7 @@ 
      Backend Flags
      
       --ftp-disable-utf8                             Disable using UTF-8 even if server advertises support
       --ftp-encoding MultiEncoder                    The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
       --ftp-explicit-tls                             Use Explicit FTPS (FTP over TLS)
+      --ftp-force-list-hidden                        Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
       --ftp-host string                              FTP host to connect to
       --ftp-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
       --ftp-no-check-certificate                     Do not verify the TLS certificate of the server
@@ -8577,6 +8671,7 @@ 
      Backend Flags
      
       --gcs-client-secret string                     OAuth Client Secret
       --gcs-decompress                               If set this will decompress gzip encoded objects
       --gcs-encoding MultiEncoder                    The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot)
+      --gcs-endpoint string                          Endpoint for the service
       --gcs-location string                          Location for the newly created buckets
       --gcs-no-check-bucket                          If set, don't attempt to check the bucket exists or create it
       --gcs-object-acl string                        Access Control List for new objects
@@ -8622,14 +8717,6 @@ 
      Backend Flags
      
       --http-no-head                                 Don't use HEAD requests
       --http-no-slash                                Set this if the site doesn't end directories with /
       --http-url string                              URL of HTTP host to connect to
-      --hubic-auth-url string                        Auth server URL
-      --hubic-chunk-size SizeSuffix                  Above this size files will be chunked into a _segments container (default 5Gi)
-      --hubic-client-id string                       OAuth Client Id
-      --hubic-client-secret string                   OAuth Client Secret
-      --hubic-encoding MultiEncoder                  The encoding for the backend (default Slash,InvalidUtf8)
-      --hubic-no-chunk                               Don't chunk files during streaming upload
-      --hubic-token string                           OAuth Access Token as a JSON blob
-      --hubic-token-url string                       Token server url
       --internetarchive-access-key-id string         IAS3 Access Key
       --internetarchive-disable-checksum             Don't ask the server to test against MD5 checksum calculated by rclone (default true)
       --internetarchive-encoding MultiEncoder        The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot)
@@ -8698,6 +8785,22 @@ 
      Backend Flags
      
       --onedrive-server-side-across-configs          Allow server-side operations (e.g. copy) to work across different onedrive configs
       --onedrive-token string                        OAuth Access Token as a JSON blob
       --onedrive-token-url string                    Token server url
+      --oos-chunk-size SizeSuffix                    Chunk size to use for uploading (default 5Mi)
+      --oos-compartment string                       Object storage compartment OCID
+      --oos-config-file string                       Path to OCI config file (default "~/.oci/config")
+      --oos-config-profile string                    Profile name inside the oci config file (default "Default")
+      --oos-copy-cutoff SizeSuffix                   Cutoff for switching to multipart copy (default 4.656Gi)
+      --oos-copy-timeout Duration                    Timeout for copy (default 1m0s)
+      --oos-disable-checksum                         Don't store MD5 checksum with object metadata
+      --oos-encoding MultiEncoder                    The encoding for the backend (default Slash,InvalidUtf8,Dot)
+      --oos-endpoint string                          Endpoint for Object storage API
+      --oos-leave-parts-on-error                     If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery
+      --oos-namespace string                         Object storage namespace
+      --oos-no-check-bucket                          If set, don't attempt to check the bucket exists or create it
+      --oos-provider string                          Choose your Auth Provider (default "env_auth")
+      --oos-region string                            Object storage Region
+      --oos-upload-concurrency int                   Concurrency for multipart uploads (default 10)
+      --oos-upload-cutoff SizeSuffix                 Cutoff for switching to chunked upload (default 200Mi)
       --opendrive-chunk-size SizeSuffix              Files will be uploaded in chunks this size (default 10Mi)
       --opendrive-encoding MultiEncoder              The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot)
       --opendrive-password string                    Password (obscured)
@@ -8729,6 +8832,7 @@ 
      Backend Flags
      
       --s3-bucket-acl string                         Canned ACL used when creating buckets
       --s3-chunk-size SizeSuffix                     Chunk size to use for uploading (default 5Mi)
       --s3-copy-cutoff SizeSuffix                    Cutoff for switching to multipart copy (default 4.656Gi)
+      --s3-decompress                                If set this will decompress gzip encoded objects
       --s3-disable-checksum                          Don't store MD5 checksum with object metadata
       --s3-disable-http2                             Disable usage of http2 for S3 backends
       --s3-download-url string                       Custom endpoint for downloads
@@ -8747,6 +8851,7 @@ 
      Backend Flags
      
       --s3-no-check-bucket                           If set, don't attempt to check the bucket exists or create it
       --s3-no-head                                   If set, don't HEAD uploaded objects to check integrity
       --s3-no-head-object                            If set, do not do HEAD before GET when getting objects
+      --s3-no-system-metadata                        Suppress setting and reading of system metadata
       --s3-profile string                            Profile to use in the shared credentials file
       --s3-provider string                           Choose your S3 provider
       --s3-region string                             Region to connect to
@@ -8756,7 +8861,8 @@ 
      Backend Flags
      
       --s3-session-token string                      An AWS session token
       --s3-shared-credentials-file string            Path to the shared credentials file
       --s3-sse-customer-algorithm string             If using SSE-C, the server-side encryption algorithm used when storing this object in S3
-      --s3-sse-customer-key string                   If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data
+      --s3-sse-customer-key string                   To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data
+      --s3-sse-customer-key-base64 string            If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data
       --s3-sse-customer-key-md5 string               If using SSE-C you may provide the secret encryption key MD5 checksum (optional)
       --s3-sse-kms-key-id string                     If using KMS ID you must provide the ARN of Key
       --s3-storage-class string                      The storage class to use when storing new objects in S3
@@ -8766,6 +8872,8 @@ 
      Backend Flags
      
       --s3-use-multipart-etag Tristate               Whether to use ETag in multipart uploads for verification (default unset)
       --s3-use-presigned-request                     Whether to use a presigned request or PutObject for single part uploads
       --s3-v2-auth                                   If true use v2 authentication
+      --s3-version-at Time                           Show file versions as they were at the specified time (default off)
+      --s3-versions                                  Include old versions in directory listings
       --seafile-2fa                                  Two-factor authentication ('true' if the account has 2FA enabled)
       --seafile-create-library                       Should rclone create a library if it doesn't exist
       --seafile-encoding MultiEncoder                The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
@@ -8812,6 +8920,15 @@ 
      Backend Flags
      
       --sia-encoding MultiEncoder                    The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot)
       --sia-user-agent string                        Siad User Agent (default "Sia-Agent")
       --skip-links                                   Don't warn about skipped symlinks
+      --smb-case-insensitive                         Whether the server is configured to be case-insensitive (default true)
+      --smb-domain string                            Domain name for NTLM authentication (default "WORKGROUP")
+      --smb-encoding MultiEncoder                    The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot)
+      --smb-hide-special-share                       Hide special shares (e.g. print$) which users aren't supposed to access (default true)
+      --smb-host string                              SMB server hostname to connect to
+      --smb-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
+      --smb-pass string                              SMB password (obscured)
+      --smb-port int                                 SMB port number (default 445)
+      --smb-user string                              SMB username (default "$USER")
       --storj-access-grant string                    Access grant
       --storj-api-key string                         API key
       --storj-passphrase string                      Encryption passphrase
@@ -8842,6 +8959,7 @@ 
      Backend Flags
      
       --swift-key string                             API key or password (OS_PASSWORD)
       --swift-leave-parts-on-error                   If true avoid calling abort upload on a failure
       --swift-no-chunk                               Don't chunk files during streaming upload
+      --swift-no-large-objects                       Disable support for static and dynamic large objects
       --swift-region string                          Region name - optional (OS_REGION_NAME)
       --swift-storage-policy string                  The storage policy to use when creating a new container
       --swift-storage-url string                     Storage URL - optional (OS_STORAGE_URL)
@@ -9314,7 +9432,7 @@ 
      Modification time
      
 
      Error handling
      
 
      Certain bisync critical errors, such as file copy/move failing, will result in a bisync lockout of following runs. The lockout is asserted because the sync status and history of the Path1 and Path2 filesystems cannot be trusted, so it is safer to block any further changes until someone checks things out. The recovery is to do a --resync again.
      
 
      It is recommended to use --resync --dry-run --verbose initially and carefully review what changes will be made before running the --resync without --dry-run.
      
-
      Most of these events come up due to a error status from an internal call. On such a critical error the {...}.path1.lst and {...}.path2.lst listing files are renamed to extension .lst-err, which blocks any future bisync runs (since the normal .lst files are not found). Bisync keeps them under bisync subdirectory of the rclone cache direcory, typically at ${HOME}/.cache/rclone/bisync/ on Linux.
      
+
      Most of these events come up due to a error status from an internal call. On such a critical error the {...}.path1.lst and {...}.path2.lst listing files are renamed to extension .lst-err, which blocks any future bisync runs (since the normal .lst files are not found). Bisync keeps them under bisync subdirectory of the rclone cache directory, typically at ${HOME}/.cache/rclone/bisync/ on Linux.
      
 
      Some errors are considered temporary and re-running the bisync is not blocked. The critical return blocks further bisync runs.
      
 
      Lock file
      
 
      When bisync is running, a lock file is created in the bisync working directory, typically at ~/.cache/rclone/bisync/PATH1..PATH2.lck on Linux. If bisync should crash or hang, the lock file will remain in place and block any further runs of bisync for the same paths. Delete the lock file as part of debugging the situation. The lock file effectively blocks follow-on (e.g., scheduled by cron) runs when the prior invocation is taking a long time. The lock file contains PID of the blocking process, which may help in debug.
      
@@ -9339,7 +9457,7 @@ 
      Renamed directories
      
 
      Case sensitivity
      
 
      Synching with case-insensitive filesystems, such as Windows or Box, can result in file name conflicts. This will be fixed in a future release. The near term workaround is to make sure that files on both sides don't have spelling case differences (Smile.jpg vs. smile.jpg).
      
 
      Windows support
      
-
      Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on Windows Github runners.
      
+
      Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on Windows GitHub runners.
      
 
      Drive letters are allowed, including drive letters mapped to network drives (rclone bisync J:\localsync GDrive:). If a drive letter is omitted, the shell current drive is the default. Drive letters are a single character follows by :, so cloud names must be more than one character long.
      
 
      Absolute paths (with or without a drive letter), and relative paths (with or without a drive letter) are supported.
      
 
      Working directory is created at C:\Users\MyLogin\AppData\Local\rclone\bisync.
      
@@ -9596,11 +9714,11 @@ 
      Notes about testing
      
 
    24. path1 and/or path2 subdirectories are created in a temporary directory under the respective local or cloud test remote.
      25. 
 
    25. By default, the Path1 and Path2 test dirs and workdir will be deleted after each test run. The -no-cleanup flag disables purging these directories when validating and debugging a given test. These directories will be flushed before running another test, independent of the -no-cleanup usage.
      26. 
 
    26. You will likely want to add `- /testdir/to your normal   bisync--filters-fileso that normal syncs do not attempt to sync   the test temporary directories, which may haveRCLONE_TESTmiscompares   in some testcases which would otherwise trip the--check-accesssystem.   The--check-accessmechanism is hard-coded to ignoreRCLONE_TESTfiles beneathbisync/testdata`, so the test cases may reside on the synched tree even if there are check file mismatches in the test tree.
      27. 
-
    27. Some Dropbox tests can fail, notably printing the following message: src and dst identical but can't set mod time without deleting and re-uploading This is expected and happens due a way Dropbox handles modificaion times. You should use the -refresh-times test flag to make up for this.
      28. 
+
    28. Some Dropbox tests can fail, notably printing the following message: src and dst identical but can't set mod time without deleting and re-uploading This is expected and happens due a way Dropbox handles modification times. You should use the -refresh-times test flag to make up for this.
      29. 
 
    29. If Dropbox tests hit request limit for you and print error message too_many_requests/...: Too many requests or write operations. then follow the Dropbox App ID instructions.
      30. 
 
 
      Updating golden results
      
-
      Sometimes even a slight change in the bisync source can cause little changes spread around many log files. Updating them manually would be a nighmare.
      
+
      Sometimes even a slight change in the bisync source can cause little changes spread around many log files. Updating them manually would be a nightmare.
      
 
      The -golden flag will store the test.log and *.lst listings from each test case into respective golden directories. Golden results will automatically contain generic strings instead of local or cloud paths which means that they should match when run with a different cloud service.
      
 
      Your normal workflow might be as follows: 1. Git-clone the rclone sources locally 2. Modify bisync source and check that it builds 3. Run the whole test suite go test ./cmd/bisync -remote local 4. If some tests show log difference, recheck them individually, e.g.: go test ./cmd/bisync -remote local -case basic 5. If you are convinced with the difference, goldenize all tests at once: go test ./cmd/bisync -remote local -golden 6. Use word diff: git diff --word-diff ./cmd/bisync/testdata/. Please note that normal line-level diff is generally useless here. 7. Check the difference carefully! 8. Commit the change (git commit) only if you are sure. If unsure, save your code changes then wipe the log diffs from git: git reset [--hard].
      
 
      Structure of test scenarios
      
@@ -9891,6 +10009,7 @@ 
      Alias
      
 
      During the initial setup with rclone config you will specify the target remote. The target remote can either be a local path or another remote.
      
 
      Subfolders can be used in target remote. Assume an alias remote named backup with the target mydrive:private/backup. Invoking rclone mkdir backup:desktop is exactly the same as invoking rclone mkdir mydrive:private/backup/desktop.
      
 
      There will be no special handling of paths containing .. segments. Invoking rclone mkdir backup:../desktop is exactly the same as invoking rclone mkdir mydrive:private/backup/../desktop. The empty path is not allowed as a remote. To alias the current directory use . instead.
      
+
      The target remote can also be a connection string. This can be used to modify the config of a remote for different uses, e.g. the alias myDriveTrash with the target remote myDrive,trashed_only: can be used to only show the trashed files in myDrive.
      
 
      Configuration
      
 
      Here is an example of how to make an alias called remote for local folder. First run:
      
 
       rclone config
      
@@ -10172,7 +10291,9 @@ 
      Amazon S3 Storage Providers
      
 
    30. Huawei OBS
      31. 
 
    31. IBM COS S3
      32. 
 
    32. IDrive e2
      33. 
+
    33. IONOS Cloud
      34. 
 
    34. Minio
      35. 
+
    35. Qiniu Cloud Object Storage (Kodo)
      36. 
 
    36. RackCorp Object Storage
      37. 
 
    37. Scaleway
      38. 
 
    38. Seagate Lyve Cloud
      39. 
@@ -10430,7 +10551,7 @@ 
      Avoiding HEAD requ
 
      These flags can and should be used in combination with --fast-list - see below.
      
 
      If using rclone mount or any command using the VFS (eg rclone serve) commands then you might want to consider using the VFS flag --no-modtime which will stop rclone reading the modification time for every object. You could also use --use-server-modtime if you are happy with the modification times of the objects being the time of upload.
      
 
      Avoiding GET requests to read directory listings
      
-
      Rclone's default directory traversal is to process each directory individually. This takes one API call per directory. Using the --fast-list flag will read all info about the the objects into memory first using a smaller number of API calls (one per 1000 objects). See the rclone docs for more details.
      
+
      Rclone's default directory traversal is to process each directory individually. This takes one API call per directory. Using the --fast-list flag will read all info about the objects into memory first using a smaller number of API calls (one per 1000 objects). See the rclone docs for more details.
      
 
      rclone sync --fast-list --checksum /path/to/source s3:bucket
      
 
      --fast-list trades off API transactions for memory use. As a rough guide rclone uses 1k of memory per object stored, so using --fast-list on a sync of a million objects will use roughly 1 GiB of RAM.
      
 
      If you are only copying a small number of files into a big repository then using --no-traverse is a good idea. This finds objects directly instead of through directory listings. You can do a "top-up" sync very cheaply by using --max-age and --no-traverse to copy only recent files, eg
      
@@ -10446,6 +10567,36 @@ 
      Hashes
      
 
      However for objects which were uploaded as multipart uploads or with server side encryption (SSE-AWS or SSE-C) the ETag header is no longer the MD5 sum of the data, so rclone adds an additional piece of metadata X-Amz-Meta-Md5chksum which is a base64 encoded MD5 hash (in the same format as is required for Content-MD5).
      
 
      For large objects, calculating this hash can take some time so the addition of this hash can be disabled with --s3-disable-checksum. This will mean that these objects do not have an MD5 checksum.
      
 
      Note that reading this from the object takes an additional HEAD request as the metadata isn't returned in object listings.
      
+
      Versions
      
+
      When bucket versioning is enabled (this can be done with rclone with the rclone backend versioning command) when rclone uploads a new version of a file it creates a new version of it Likewise when you delete a file, the old version will be marked hidden and still be available.
      
+
      Old versions of files, where available, are visible using the --s3-versions flag.
      
+
      It is also possible to view a bucket as it was at a certain point in time, using the --s3-version-at flag. This will show the file versions as they were at that time, showing files that have been deleted afterwards, and hiding files that were created since.
      
+
      If you wish to remove all the old versions then you can use the rclone backend cleanup-hidden remote:bucket command which will delete all the old hidden versions of files, leaving the current ones intact. You can also supply a path and only old versions under that path will be deleted, e.g. rclone backend cleanup-hidden remote:bucket/path/to/stuff.
      
+
      When you purge a bucket, the current and the old versions will be deleted then the bucket will be deleted.
      
+
      However delete will cause the current versions of the files to become hidden old versions.
      
+
      Here is a session showing the listing and retrieval of an old version followed by a cleanup of the old versions.
      
+
      Show current version and all the versions with --s3-versions flag.
      
+
      $ rclone -q ls s3:cleanup-test
+        9 one.txt
+
+$ rclone -q --s3-versions ls s3:cleanup-test
+        9 one.txt
+        8 one-v2016-07-04-141032-000.txt
+       16 one-v2016-07-04-141003-000.txt
+       15 one-v2016-07-02-155621-000.txt
      
+
      Retrieve an old version
      
+
      $ rclone -q --s3-versions copy s3:cleanup-test/one-v2016-07-04-141003-000.txt /tmp
+
+$ ls -l /tmp/one-v2016-07-04-141003-000.txt
+-rw-rw-r-- 1 ncw ncw 16 Jul  2 17:46 /tmp/one-v2016-07-04-141003-000.txt
      
+
      Clean up all the old versions and show that they've gone.
      
+
      $ rclone -q backend cleanup-hidden s3:cleanup-test
+
+$ rclone -q ls s3:cleanup-test
+        9 one.txt
+
+$ rclone -q --s3-versions ls s3:cleanup-test
+        9 one.txt
      
 
      Cleanup
      
 
      If you run rclone cleanup s3:bucket then it will remove all pending multipart uploads older than 24 hours. You can use the -i flag to see exactly what it will do. If you want more control over the expiry date then run rclone backend cleanup s3:bucket -o max-age=1h to expire all uploads older than one hour. You can use rclone backend list-multipart-uploads s3:bucket to see the pending multipart uploads.
      
 
      Restricted filename characters
      
@@ -10592,7 +10743,7 @@ 
      Object-lock enabled S3 bucket
      
 
 
      As mentioned in the Hashes section, small files that are not uploaded as multipart, use a different tag, causing the upload to fail. A simple solution is to set the --s3-upload-cutoff 0 and force all the files to be uploaded as multipart.
      
 
      Standard options
      
-
      Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi).
      
+
      Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
      
 
      --s3-provider
      
 
      Choose your S3 provider.
      
 
      Properties:
      
@@ -10647,6 +10798,10 @@ 
      --s3-provider
      
 
        
 
      • IDrive e2
        • 
 
      
+
    39. "IONOS"
+
        
+
      • IONOS Cloud
        • 
+
      40. 
 
    40. "LyveCloud"
 
        
 
      • Seagate Lyve Cloud
        • 
@@ -10687,6 +10842,10 @@ 
        --s3-provider
        
 
          
 
        • Wasabi Object Storage
          • 
 
        
+
      • "Qiniu"
+
          
+
        • Qiniu Object Storage (Kodo)
          • 
+
        • 
 
      • "Other"
 
          
 
        • Any other S3 compatible provider
          • 
@@ -11079,12 +11238,86 @@ 
          --s3-region
          
 
        
 
        --s3-region
        
 
        Region to connect to.
        
+
        Properties:
        
+
          
+
        • Config: region
          • 
+
        • Env Var: RCLONE_S3_REGION
          • 
+
        • Provider: Qiniu
          • 
+
        • Type: string
          • 
+
        • Required: false
          • 
+
        • Examples:
+
            
+
          • "cn-east-1"
+
              
+
            • The default endpoint - a good choice if you are unsure.
              • 
+
            • East China Region 1.
              • 
+
            • Needs location constraint cn-east-1.
              • 
+
            • 
+
          • "cn-east-2"
+
              
+
            • East China Region 2.
              • 
+
            • Needs location constraint cn-east-2.
              • 
+
            • 
+
          • "cn-north-1"
+
              
+
            • North China Region 1.
              • 
+
            • Needs location constraint cn-north-1.
              • 
+
            • 
+
          • "cn-south-1"
+
              
+
            • South China Region 1.
              • 
+
            • Needs location constraint cn-south-1.
              • 
+
            • 
+
          • "us-north-1"
+
              
+
            • North America Region.
              • 
+
            • Needs location constraint us-north-1.
              • 
+
            • 
+
          • "ap-southeast-1"
+
              
+
            • Southeast Asia Region 1.
              • 
+
            • Needs location constraint ap-southeast-1.
              • 
+
            • 
+
          • "ap-northeast-1"
+
              
+
            • Northeast Asia Region 1.
              • 
+
            • Needs location constraint ap-northeast-1.
              • 
+
            • 
+
          • 
+
        
+
        --s3-region
        
+
        Region where your bucket will be created and your data stored.
        
+
        Properties:
        
+
          
+
        • Config: region
          • 
+
        • Env Var: RCLONE_S3_REGION
          • 
+
        • Provider: IONOS
          • 
+
        • Type: string
          • 
+
        • Required: false
          • 
+
        • Examples:
+
            
+
          • "de"
+
              
+
            • Frankfurt, Germany
              • 
+
            • 
+
          • "eu-central-2"
+
              
+
            • Berlin, Germany
              • 
+
            • 
+
          • "eu-south-2"
+
              
+
            • Logrono, Spain
              • 
+
            • 
+
          • 
+
        
+
        --s3-region
        
+
        Region to connect to.
        
 
        Leave blank if you are using an S3 clone and you don't have a region.
        
 
        Properties:
        
 
          
 
        • Config: region
          • 
 
        • Env Var: RCLONE_S3_REGION
          • 
-
        • Provider: !AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
          • 
+
        • Provider: !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
          • 
 
        • Type: string
          • 
 
        • Required: false
          • 
 
        • Examples:
@@ -11531,6 +11764,32 @@ 
          --s3-endpoint
          
 
        • 
 
      
 
      --s3-endpoint
      
+
      Endpoint for IONOS S3 Object Storage.
      
+
      Specify the endpoint from the same region.
      
+
      Properties:
      
+
        
+
      • Config: endpoint
        • 
+
      • Env Var: RCLONE_S3_ENDPOINT
        • 
+
      • Provider: IONOS
        • 
+
      • Type: string
        • 
+
      • Required: false
        • 
+
      • Examples:
+
          
+
        • "s3-eu-central-1.ionoscloud.com"
+
            
+
          • Frankfurt, Germany
            • 
+
          • 
+
        • "s3-eu-central-2.ionoscloud.com"
+
            
+
          • Berlin, Germany
            • 
+
          • 
+
        • "s3-eu-south-2.ionoscloud.com"
+
            
+
          • Logrono, Spain
            • 
+
          • 
+
        • 
+
      
+
      --s3-endpoint
      
 
      Endpoint for OSS API.
      
 
      Properties:
      
 
        
@@ -11643,7 +11902,7 @@ 
        --s3-endpoint
        
 
      41. 
 
 
-
      --s3-endpoint
      
+
      --s3-endpoint
      
 
      Endpoint for OBS API.
      
 
      Properties:
      
 
        
@@ -11716,7 +11975,7 @@ 
        --s3-endpoint
        
 
      
 
 
-
      --s3-endpoint
      
+
      --s3-endpoint
      
 
      Endpoint for Scaleway Object Storage.
      
 
      Properties:
      
 
        
@@ -11741,7 +12000,7 @@ 
        --s3-endpoint
        
 
      
 
 
-
      --s3-endpoint
      
+
      --s3-endpoint
      
 
      Endpoint for StackPath Object Storage.
      
 
      Properties:
      
 
        
@@ -11766,7 +12025,7 @@ 
        --s3-endpoint
        
 
      
 
 
-
      --s3-endpoint
      
+
      --s3-endpoint
      
 
      Endpoint of the Shared Gateway.
      
 
      Properties:
      
 
        
@@ -11791,7 +12050,7 @@ 
        --s3-endpoint
        
 
      
 
 
-
      --s3-endpoint
      
+
      --s3-endpoint
      
 
      Endpoint for Tencent COS API.
      
 
      Properties:
      
 
        
@@ -11880,7 +12139,7 @@ 
        --s3-endpoint
        
 
      
 
 
-
      --s3-endpoint
      
+
      --s3-endpoint
      
 
      Endpoint for RackCorp Object Storage.
      
 
      Properties:
      
 
        
@@ -11969,14 +12228,55 @@ 
        --s3-endpoint
        
 
      
 
 
-
      --s3-endpoint
      
+
      --s3-endpoint
      
+
      Endpoint for Qiniu Object Storage.
      
+
      Properties:
      
+
        
+
      • Config: endpoint
        • 
+
      • Env Var: RCLONE_S3_ENDPOINT
        • 
+
      • Provider: Qiniu
        • 
+
      • Type: string
        • 
+
      • Required: false
        • 
+
      • Examples:
+
          
+
        • "s3-cn-east-1.qiniucs.com"
+
            
+
          • East China Endpoint 1
            • 
+
          • 
+
        • "s3-cn-east-2.qiniucs.com"
+
            
+
          • East China Endpoint 2
            • 
+
          • 
+
        • "s3-cn-north-1.qiniucs.com"
+
            
+
          • North China Endpoint 1
            • 
+
          • 
+
        • "s3-cn-south-1.qiniucs.com"
+
            
+
          • South China Endpoint 1
            • 
+
          • 
+
        • "s3-us-north-1.qiniucs.com"
+
            
+
          • North America Endpoint 1
            • 
+
          • 
+
        • "s3-ap-southeast-1.qiniucs.com"
+
            
+
          • Southeast Asia Endpoint 1
            • 
+
          • 
+
        • "s3-ap-northeast-1.qiniucs.com"
+
            
+
          • Northeast Asia Endpoint 1
            • 
+
          • 
+
        • 
+
      
+
      --s3-endpoint
      
 
      Endpoint for S3 API.
      
 
      Required when using an S3 clone.
      
 
      Properties:
      
 
        
 
      • Config: endpoint
        • 
 
      • Env Var: RCLONE_S3_ENDPOINT
        • 
-
      • Provider: !AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp
        • 
+
      • Provider: !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu
        • 
 
      • Type: string
        • 
 
      • Required: false
        • 
 
      • Examples:
@@ -12542,12 +12842,54 @@ 
        --s3-location-constraint
        
 
      
 
      --s3-location-constraint
      
 
      Location constraint - must be set to match the Region.
      
+
      Used when creating buckets only.
      
+
      Properties:
      
+
        
+
      • Config: location_constraint
        • 
+
      • Env Var: RCLONE_S3_LOCATION_CONSTRAINT
        • 
+
      • Provider: Qiniu
        • 
+
      • Type: string
        • 
+
      • Required: false
        • 
+
      • Examples:
+
          
+
        • "cn-east-1"
+
            
+
          • East China Region 1
            • 
+
          • 
+
        • "cn-east-2"
+
            
+
          • East China Region 2
            • 
+
          • 
+
        • "cn-north-1"
+
            
+
          • North China Region 1
            • 
+
          • 
+
        • "cn-south-1"
+
            
+
          • South China Region 1
            • 
+
          • 
+
        • "us-north-1"
+
            
+
          • North America Region 1
            • 
+
          • 
+
        • "ap-southeast-1"
+
            
+
          • Southeast Asia Region 1
            • 
+
          • 
+
        • "ap-northeast-1"
+
            
+
          • Northeast Asia Region 1
            • 
+
          • 
+
        • 
+
      
+
      --s3-location-constraint
      
+
      Location constraint - must be set to match the Region.
      
 
      Leave blank if not sure. Used when creating buckets only.
      
 
      Properties:
      
 
        
 
      • Config: location_constraint
        • 
 
      • Env Var: RCLONE_S3_LOCATION_CONSTRAINT
        • 
-
      • Provider: !AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS
        • 
+
      • Provider: !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS
        • 
 
      • Type: string
        • 
 
      • Required: false
        • 
 
      
@@ -12855,8 +13197,37 @@ 
      --s3-storage-class
      
 
 
 
+
      --s3-storage-class
      
+
      The storage class to use when storing new objects in Qiniu.
      
+
      Properties:
      
+
        
+
      • Config: storage_class
        • 
+
      • Env Var: RCLONE_S3_STORAGE_CLASS
        • 
+
      • Provider: Qiniu
        • 
+
      • Type: string
        • 
+
      • Required: false
        • 
+
      • Examples:
+
          
+
        • "STANDARD"
+
            
+
          • Standard storage class
            • 
+
          • 
+
        • "LINE"
+
            
+
          • Infrequent access storage mode
            • 
+
          • 
+
        • "GLACIER"
+
            
+
          • Archive storage mode
            • 
+
          • 
+
        • "DEEP_ARCHIVE"
+
            
+
          • Deep archive storage mode
            • 
+
          • 
+
        • 
+
      
 
      Advanced options
      
-
      Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi).
      
+
      Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
      
 
      --s3-bucket-acl
      
 
      Canned ACL used when creating buckets.
      
 
      For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
      
@@ -12924,7 +13295,8 @@ 
      --s3-sse-customer-algorithm
      
 
 
 
      --s3-sse-customer-key
      
-
      If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.
      
+
      To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data.
      
+
      Alternatively you can provide --sse-customer-key-base64.
      
 
      Properties:
      
 
        
 
      • Config: sse_customer_key
        • 
@@ -12940,6 +13312,24 @@ 
        --s3-sse-customer-key
        
 
      
 
 
+
      --s3-sse-customer-key-base64
      
+
      If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data.
      
+
      Alternatively you can provide --sse-customer-key.
      
+
      Properties:
      
+
        
+
      • Config: sse_customer_key_base64
        • 
+
      • Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_BASE64
        • 
+
      • Provider: AWS,Ceph,ChinaMobile,Minio
        • 
+
      • Type: string
        • 
+
      • Required: false
        • 
+
      • Examples:
+
          
+
        • ""
+
            
+
          • None
            • 
+
          • 
+
        • 
+
      
 
      --s3-sse-customer-key-md5
      
 
      If using SSE-C you may provide the secret encryption key MD5 checksum (optional).
      
 
      If you leave it blank, this is calculated automatically from the sse_customer_key provided.
      
@@ -13251,6 +13641,47 @@ 
      --s3-use-presigned-request
      
 
    41. Type: bool
      42. 
 
    42. Default: false
      43. 
 
+
      --s3-versions
      
+
      Include old versions in directory listings.
      
+
      Properties:
      
+
        
+
      • Config: versions
        • 
+
      • Env Var: RCLONE_S3_VERSIONS
        • 
+
      • Type: bool
        • 
+
      • Default: false
        • 
+
      
+
      --s3-version-at
      
+
      Show file versions as they were at the specified time.
      
+
      The parameter should be a date, "2006-01-02", datetime "2006-01-02 15:04:05" or a duration for that long ago, eg "100d" or "1h".
      
+
      Note that when using this no file write operations are permitted, so you can't upload files or delete them.
      
+
      See the time option docs for valid formats.
      
+
      Properties:
      
+
        
+
      • Config: version_at
        • 
+
      • Env Var: RCLONE_S3_VERSION_AT
        • 
+
      • Type: Time
        • 
+
      • Default: off
        • 
+
      
+
      --s3-decompress
      
+
      If set this will decompress gzip encoded objects.
      
+
      It is possible to upload objects to S3 with "Content-Encoding: gzip" set. Normally rclone will download these files as compressed objects.
      
+
      If this flag is set then rclone will decompress these files with "Content-Encoding: gzip" as they are received. This means that rclone can't check the size and hash but the file contents will be decompressed.
      
+
      Properties:
      
+
        
+
      • Config: decompress
        • 
+
      • Env Var: RCLONE_S3_DECOMPRESS
        • 
+
      • Type: bool
        • 
+
      • Default: false
        • 
+
      
+
      --s3-no-system-metadata
      
+
      Suppress setting and reading of system metadata
      
+
      Properties:
      
+
        
+
      • Config: no_system_metadata
        • 
+
      • Env Var: RCLONE_S3_NO_SYSTEM_METADATA
        • 
+
      • Type: bool
        • 
+
      • Default: false
        • 
+
      
 
      Metadata
      
 
      User metadata is stored as x-amz-meta- keys. S3 metadata keys are case insensitive and are always returned in lower case.
      
 
      Here are the possible system metadata items for the s3 backend.
      
@@ -13406,6 +13837,20 @@ 
      cleanup
      
 
        
 
      • "max-age": Max age of upload to delete
        • 
 
      
+
      cleanup-hidden
      
+
      Remove old versions of files.
      
+
      rclone backend cleanup-hidden remote: [options] [<arguments>+]
      
+
      This command removes any old hidden versions of files on a versions enabled bucket.
      
+
      Note that you can use -i/--dry-run with this command to see what it would do.
      
+
      rclone backend cleanup-hidden s3:bucket/path/to/dir
      
+
      versioning
      
+
      Set/get versioning support for a bucket.
      
+
      rclone backend versioning remote: [options] [<arguments>+]
      
+
      This command sets versioning support if a parameter is passed and then returns the current versioning status for the bucket supplied.
      
+
      rclone backend versioning s3:bucket # read status only
+rclone backend versioning s3:bucket Enabled
+rclone backend versioning s3:bucket Suspended
      
+
      It may return "Enabled", "Suspended" or "Unversioned". Note that once versioning has been enabled the status can't be set back to "Unversioned".
      
 
      Anonymous access to public buckets
      
 
      If you want to use rclone to access a public bucket, configure with a blank access_key_id and secret_access_key. Your config should end up looking like this:
      
 
      [anons3]
@@ -13973,6 +14418,132 @@ 
      IDrive e2
      
 e) Edit this remote
 d) Delete this remote
 y/e/d> y
      
+
      IONOS Cloud
      
+
      IONOS S3 Object Storage is a service offered by IONOS for storing and accessing unstructured data. To connect to the service, you will need an access key and a secret key. These can be found in the Data Center Designer, by selecting Manager resources > Object Storage Key Manager.
      
+
      Here is an example of a configuration. First, run rclone config. This will walk you through an interactive setup process. Type n to add the new remote, and then enter a name:
      
+
      Enter name for new remote.
+name> ionos-fra
      
+
      Type s3 to choose the connection type:
      
+
      Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+   \ (s3)
+[snip]
+Storage> s3
      
+
      Type IONOS:
      
+
      Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+[snip]
+XX / IONOS Cloud
+   \ (IONOS)
+[snip]
+provider> IONOS
      
+
      Press Enter to choose the default option Enter AWS credentials in the next step:
      
+
      Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth>
      
+
      Enter your Access Key and Secret key. These can be retrieved in the Data Center Designer, click on the menu “Manager resources” / "Object Storage Key Manager".
      
+
      Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> YOUR_ACCESS_KEY
+
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> YOUR_SECRET_KEY
      
+
      Choose the region where your bucket is located:
      
+
      Option region.
+Region where your bucket will be created and your data stored.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / Frankfurt, Germany
+   \ (de)
+ 2 / Berlin, Germany
+   \ (eu-central-2)
+ 3 / Logrono, Spain
+   \ (eu-south-2)
+region> 2
      
+
      Choose the endpoint from the same region:
      
+
      Option endpoint.
+Endpoint for IONOS S3 Object Storage.
+Specify the endpoint from the same region.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / Frankfurt, Germany
+   \ (s3-eu-central-1.ionoscloud.com)
+ 2 / Berlin, Germany
+   \ (s3-eu-central-2.ionoscloud.com)
+ 3 / Logrono, Spain
+   \ (s3-eu-south-2.ionoscloud.com)
+endpoint> 1
      
+
      Press Enter to choose the default option or choose the desired ACL setting:
      
+
      Option acl.
+Canned ACL used when creating buckets and storing or copying objects.
+This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+   / Owner gets FULL_CONTROL.
+[snip]
+acl>
      
+
      Press Enter to skip the advanced config:
      
+
      Edit advanced config?
+y) Yes
+n) No (default)
+y/n>
      
+
      Press Enter to save the configuration, and then q to quit the configuration process:
      
+
      Configuration complete.
+Options:
+- type: s3
+- provider: IONOS
+- access_key_id: YOUR_ACCESS_KEY
+- secret_access_key: YOUR_SECRET_KEY
+- endpoint: s3-eu-central-1.ionoscloud.com
+Keep this "ionos-fra" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
      
+
      Done! Now you can try some commands (for macOS, use ./rclone instead of rclone).
      
+
        
+
      1. Create a bucket (the name must be unique within the whole IONOS S3)
        2. 
+
      
+
      rclone mkdir ionos-fra:my-bucket
      
+
        
+
      1. List available buckets
        2. 
+
      
+
      rclone lsd ionos-fra:
      
+
        
+
      1. Copy a file from local to remote
        2. 
+
      
+
      rclone copy /Users/file.txt ionos-fra:my-bucket
      
+
        
+
      1. List contents of a bucket
        2. 
+
      
+
      rclone ls ionos-fra:my-bucket
      
+
        
+
      1. Copy a file from remote to local
        2. 
+
      
+
      rclone copy ionos-fra:my-bucket/file.txt
      
 
      Minio
      
 
      Minio is an object storage server built for cloud application developers and devops.
      
 
      It is very easy to install and provides an S3 compatible server which can be used by rclone.
      
@@ -14019,6 +14590,191 @@ 
      Minio
      
 server_side_encryption =
      
 
      So once set up, for example, to copy files into a bucket
      
 
      rclone copy /path/to/files minio:bucket
      
+
      Qiniu Cloud Object Storage (Kodo)
      
+
      Qiniu Cloud Object Storage (Kodo), a completely independent-researched core technology which is proven by repeated customer experience has occupied absolute leading market leader position. Kodo can be widely applied to mass data management.
      
+
      To configure access to Qiniu Kodo, follow the steps below:
      
+
        
+
      1. Run rclone config and select n for a new remote.
        2. 
+
      
+
      rclone config
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
      
+
        
+
      1. Give the name of the configuration. For example, name it 'qiniu'.
        2. 
+
      
+
      name> qiniu
      
+
        
+
      1. Select s3 storage.
        2. 
+
      
+
      Choose a number from below, or type in your own value
+ 1 / 1Fichier
+   \ (fichier)
+ 2 / Akamai NetStorage
+   \ (netstorage)
+ 3 / Alias for an existing remote
+   \ (alias)
+ 4 / Amazon Drive
+   \ (amazon cloud drive)
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi
+   \ (s3)
+[snip]
+Storage> s3
      
+
        
+
      1. Select Qiniu provider.
        2. 
+
      
+
      Choose a number from below, or type in your own value
+1 / Amazon Web Services (AWS) S3
+   \ "AWS"
+[snip]
+22 / Qiniu Object Storage (Kodo)
+   \ (Qiniu)
+[snip]
+provider> Qiniu
      
+
        
+
      1. Enter your SecretId and SecretKey of Qiniu Kodo.
        2. 
+
      
+
      Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Enter a boolean value (true or false). Press Enter for the default ("false").
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+access_key_id> AKIDxxxxxxxxxx
+AWS Secret Access Key (password)
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+secret_access_key> xxxxxxxxxxx
      
+
        
+
      1. Select endpoint for Qiniu Kodo. This is the standard endpoint for different region.
        2. 
+
      
+
         / The default endpoint - a good choice if you are unsure.
+ 1 | East China Region 1.
+   | Needs location constraint cn-east-1.
+   \ (cn-east-1)
+   / East China Region 2.
+ 2 | Needs location constraint cn-east-2.
+   \ (cn-east-2)
+   / North China Region 1.
+ 3 | Needs location constraint cn-north-1.
+   \ (cn-north-1)
+   / South China Region 1.
+ 4 | Needs location constraint cn-south-1.
+   \ (cn-south-1)
+   / North America Region.
+ 5 | Needs location constraint us-north-1.
+   \ (us-north-1)
+   / Southeast Asia Region 1.
+ 6 | Needs location constraint ap-southeast-1.
+   \ (ap-southeast-1)
+   / Northeast Asia Region 1.
+ 7 | Needs location constraint ap-northeast-1.
+   \ (ap-northeast-1)
+[snip]
+endpoint> 1
+
+Option endpoint.
+Endpoint for Qiniu Object Storage.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / East China Endpoint 1
+   \ (s3-cn-east-1.qiniucs.com)
+ 2 / East China Endpoint 2
+   \ (s3-cn-east-2.qiniucs.com)
+ 3 / North China Endpoint 1
+   \ (s3-cn-north-1.qiniucs.com)
+ 4 / South China Endpoint 1
+   \ (s3-cn-south-1.qiniucs.com)
+ 5 / North America Endpoint 1
+   \ (s3-us-north-1.qiniucs.com)
+ 6 / Southeast Asia Endpoint 1
+   \ (s3-ap-southeast-1.qiniucs.com)
+ 7 / Northeast Asia Endpoint 1
+   \ (s3-ap-northeast-1.qiniucs.com)
+endpoint> 1
+
+Option location_constraint.
+Location constraint - must be set to match the Region.
+Used when creating buckets only.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / East China Region 1
+   \ (cn-east-1)
+ 2 / East China Region 2
+   \ (cn-east-2)
+ 3 / North China Region 1
+   \ (cn-north-1)
+ 4 / South China Region 1
+   \ (cn-south-1)
+ 5 / North America Region 1
+   \ (us-north-1)
+ 6 / Southeast Asia Region 1
+   \ (ap-southeast-1)
+ 7 / Northeast Asia Region 1
+   \ (ap-northeast-1)
+location_constraint> 1
      
+
        
+
      1. Choose acl and storage class.
        2. 
+
      
+
      Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+   / Owner gets FULL_CONTROL.
+ 2 | The AllUsers group gets READ access.
+   \ (public-read)
+[snip]
+acl> 2
+The storage class to use when storing new objects in Tencent COS.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 1 / Standard storage class
+   \ (STANDARD)
+ 2 / Infrequent access storage mode
+   \ (LINE)
+ 3 / Archive storage mode
+   \ (GLACIER)
+ 4 / Deep archive storage mode
+   \ (DEEP_ARCHIVE)
+[snip]
+storage_class> 1
+Edit advanced config? (y/n)
+y) Yes
+n) No (default)
+y/n> n
+Remote config
+--------------------
+[qiniu]
+- type: s3
+- provider: Qiniu
+- access_key_id: xxx
+- secret_access_key: xxx
+- region: cn-east-1
+- endpoint: s3-cn-east-1.qiniucs.com
+- location_constraint: cn-east-1
+- acl: public-read
+- storage_class: STANDARD
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+Current remotes:
+
+Name                 Type
+====                 ====
+qiniu                s3
      
 
      RackCorp
      
 
      RackCorp Object Storage is an S3 compatible object storage platform from your friendly cloud provider RackCorp. The service is fast, reliable, well priced and located in many strategic locations unserviced by others, to ensure you can maintain data sovereignty.
      
 
      Before you can use RackCorp Object Storage, you'll need to "sign up" for an account on our "portal". Next you can create an access key, a secret key and buckets, in your location of choice with ease. These details are required for the next steps of configuration, when rclone config asks for your access_key_id and secret_access_key.
      
@@ -14986,7 +15742,7 @@ 
      SHA1 checksums
      
 
      Transfers
      
 
      Backblaze recommends that you do lots of transfers simultaneously for maximum speed. In tests from my SSD equipped laptop the optimum setting is about --transfers 32 though higher numbers may be used for a slight speed improvement. The optimum number for you may vary depending on your hardware, how big the files are, how much you want to load your computer, etc. The default of --transfers 4 is definitely too low for Backblaze B2 though.
      
 
      Note that uploading big files (bigger than 200 MiB by default) will use a 96 MiB RAM buffer by default. There can be at most --transfers of these in use at any moment, so this sets the upper limit on the memory used.
      
-
      Versions
      
+
      Versions
      
 
      When rclone uploads a new version of a file it creates a new version of it. Likewise when you delete a file, the old version will be marked hidden and still be available. Conversely, you may opt in to a "hard delete" of files with the --b2-hard-delete flag which would permanently remove the file instead of hiding it.
      
 
      Old versions of files, where available, are visible using the --b2-versions flag.
      
 
      It is also possible to view a bucket as it was at a certain point in time, using the --b2-version-at flag. This will show the file versions as they were at that time, showing files that have been deleted afterwards, and hiding files that were created since.
      
@@ -15033,7 +15789,7 @@ 
      Data usage
      
 /b2api/v1/b2_get_upload_part_url
 /b2api/v1/b2_upload_part/
 /b2api/v1/b2_finish_large_file
      
-
      Versions
      
+
      Versions
      
 
      Versions can be viewed with the --b2-versions flag. When it is set rclone will show and act on older versions of files. For example
      
 
      Listing without --b2-versions
      
 
      $ rclone -q ls b2:cleanup-test
@@ -16643,10 +17399,10 @@ 
      Specifying the remote
      
 
      Note: A string which do not contain a : will by rclone be treated as a relative path in the local filesystem. For example, if you enter the name remote without the trailing :, it will be treated as a subdirectory of the current directory with name "remote".
      
 
      If a path remote:path/to/dir is specified, rclone stores encrypted files in path/to/dir on the remote. With file name encryption, files saved to secret:subdir/subfile are stored in the unencrypted path path/to/dir but the subdir/subpath element is encrypted.
      
 
      The path you specify does not have to exist, rclone will create it when needed.
      
-
      If you intend to use the wrapped remote both directly for keeping unencrypted content, as well as through a crypt remote for encrypted content, it is recommended to point the crypt remote to a separate directory within the wrapped remote. If you use a bucket-based storage system (e.g. Swift, S3, Google Compute Storage, B2, Hubic) it is generally advisable to wrap the crypt remote around a specific bucket (s3:bucket). If wrapping around the entire root of the storage (s3:), and use the optional file name encryption, rclone will encrypt the bucket name.
      
+
      If you intend to use the wrapped remote both directly for keeping unencrypted content, as well as through a crypt remote for encrypted content, it is recommended to point the crypt remote to a separate directory within the wrapped remote. If you use a bucket-based storage system (e.g. Swift, S3, Google Compute Storage, B2) it is generally advisable to wrap the crypt remote around a specific bucket (s3:bucket). If wrapping around the entire root of the storage (s3:), and use the optional file name encryption, rclone will encrypt the bucket name.
      
 
      Changing password
      
 
      Should the password, or the configuration file containing a lightly obscured form of the password, be compromised, you need to re-encrypt your data with a new password. Since rclone uses secret-key encryption, where the encryption key is generated directly from the password kept on the client, it is not possible to change the password/key of already encrypted content. Just changing the password configured for an existing crypt remote means you will no longer able to decrypt any of the previously encrypted content. The only possibility is to re-upload everything via a crypt remote configured with your new password.
      
-
      Depending on the size of your data, your bandwith, storage quota etc, there are different approaches you can take: - If you have everything in a different location, for example on your local system, you could remove all of the prior encrypted files, change the password for your configured crypt remote (or delete and re-create the crypt configuration), and then re-upload everything from the alternative location. - If you have enough space on the storage system you can create a new crypt remote pointing to a separate directory on the same backend, and then use rclone to copy everything from the original crypt remote to the new, effectively decrypting everything on the fly using the old password and re-encrypting using the new password. When done, delete the original crypt remote directory and finally the rclone crypt configuration with the old password. All data will be streamed from the storage system and back, so you will get half the bandwith and be charged twice if you have upload and download quota on the storage system.
      
+
      Depending on the size of your data, your bandwidth, storage quota etc, there are different approaches you can take: - If you have everything in a different location, for example on your local system, you could remove all of the prior encrypted files, change the password for your configured crypt remote (or delete and re-create the crypt configuration), and then re-upload everything from the alternative location. - If you have enough space on the storage system you can create a new crypt remote pointing to a separate directory on the same backend, and then use rclone to copy everything from the original crypt remote to the new, effectively decrypting everything on the fly using the old password and re-encrypting using the new password. When done, delete the original crypt remote directory and finally the rclone crypt configuration with the old password. All data will be streamed from the storage system and back, so you will get half the bandwidth and be charged twice if you have upload and download quota on the storage system.
      
 
      Note: A security problem related to the random password generator was fixed in rclone version 1.53.3 (released 2020-11-19). Passwords generated by rclone config in version 1.49.0 (released 2019-08-26) to 1.53.2 (released 2020-10-26) are not considered secure and should be changed. If you made up your own password, or used rclone version older than 1.49.0 or newer than 1.53.2 to generate it, you are not affected by this issue. See issue #4783 for more details, and a tool you can use to check if you are affected.
      
 
      Example
      
 
      Create the following file structure using "standard" file name encryption.
      
@@ -16852,7 +17608,7 @@ 
      --crypt-no-data-encryption
      
 
 
      --crypt-filename-encoding
      
 
      How to encode the encrypted filename to text string.
      
-
      This option could help with shortening the encrypted filename. The suitable option would depend on the way your remote count the filename length and if it's case sensitve.
      
+
      This option could help with shortening the encrypted filename. The suitable option would depend on the way your remote count the filename length and if it's case sensitive.
      
 
      Properties:
      
 
        
 
      • Config: filename_encoding
        • 
@@ -17059,7 +17815,7 @@ 
        Advanced options
        
 
        --compress-level
        
 
        GZIP compression level (-2 to 9).
        
 
        Generally -1 (default, equivalent to 5) is recommended. Levels 1 to 9 increase compression at the cost of speed. Going past 6 generally offers very little return.
        
-
        Level -2 uses Huffmann encoding only. Only use if you know what you are doing. Level 0 turns off compression.
        
+
        Level -2 uses Huffman encoding only. Only use if you know what you are doing. Level 0 turns off compression.
        
 
        Properties:
        
 
          
 
        • Config: level
          • 
@@ -17156,7 +17912,7 @@ 
          Configuring for Google Drive
 
 [AllDrives]
 type = combine
-remote = "My Drive=My Drive:" "Test Drive=Test Drive:"
      
+upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
      
 
      If you then add that config to your config file (find it with rclone config file) then you can access all the shared drives in one place with the AllDrives: remote.
      
 
      See the Google Drive docs for full info.
      
 
      Standard options
      
@@ -17452,7 +18208,7 @@ 
      --dropbox-batch-timeout
      
 
    43. Default: 0s
      44. 
 
 
      --dropbox-batch-commit-timeout
      
-
      Max time to wait for a batch to finish comitting
      
+
      Max time to wait for a batch to finish committing
      
 
      Properties:
      
 
        
 
      • Config: batch_commit_timeout
        • 
@@ -17492,7 +18248,7 @@ 
        Get your own Dropbox App ID
        
 
        Enterprise File Fabric
        
 
        This backend supports Storage Made Easy's Enterprise File Fabric™ which provides a software solution to integrate and unify File and Object Storage accessible through a global file system.
        
 
        Configuration
        
-
        The initial setup for the Enterprise File Fabric backend involves getting a token from the the Enterprise File Fabric which you need to do in your browser. rclone config walks you through it.
        
+
        The initial setup for the Enterprise File Fabric backend involves getting a token from the Enterprise File Fabric which you need to do in your browser. rclone config walks you through it.
        
 
        Here is an example of how to make a remote called remote. First run:
        
 
         rclone config
        
 
        This will guide you through an interactive setup process:
        
@@ -17684,7 +18440,7 @@ 
        FTP
        
 
        Configuration
        
 
        To create an FTP configuration named remote, run
        
 
        rclone config
        
-
        Rclone config guides you through an interactive setup process. A minimal rclone FTP remote definition only requires host, username and password. For an anonymous FTP server, use anonymous as username and your email address as password.
        
+
        Rclone config guides you through an interactive setup process. A minimal rclone FTP remote definition only requires host, username and password. For an anonymous FTP server, see below.
        
 
        No remotes found, make a new one?
 n) New remote
 r) Rename remote
@@ -17748,13 +18504,19 @@ 
        Configuration
        
 
        rclone ls remote:path/to/directory
        
 
        Sync /home/local/directory to the remote directory, deleting any excess files in the directory.
        
 
        rclone sync -i /home/local/directory remote:directory
        
-
        Example without a config file
        
-
        rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=`rclone obscure dummy`
        
+
        Anonymous FTP
        
+
        When connecting to a FTP server that allows anonymous login, you can use the special "anonymous" username. Traditionally, this user account accepts any string as a password, although it is common to use either the password "anonymous" or "guest". Some servers require the use of a valid e-mail address as password.
        
+
        Using on-the-fly or connection string remotes makes it easy to access such servers, without requiring any configuration in advance. The following are examples of that:
        
+
        rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=$(rclone obscure dummy)
+rclone lsf :ftp,host=speedtest.tele2.net,user=anonymous,pass=$(rclone obscure dummy):
        
+
        The above examples work in Linux shells and in PowerShell, but not Windows Command Prompt. They execute the rclone obscure command to create a password string in the format required by the pass option. The following examples are exactly the same, except use an already obscured string representation of the same password "dummy", and therefore works even in Windows Command Prompt:
        
+
        rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=IXs2wc8OJOz7SYLBk47Ji1rHTmxM
+rclone lsf :ftp,host=speedtest.tele2.net,user=anonymous,pass=IXs2wc8OJOz7SYLBk47Ji1rHTmxM:
        
 
        Implicit TLS
        
 
        Rlone FTP supports implicit FTP over TLS servers (FTPS). This has to be enabled in the FTP backend config for the remote, or with --ftp-tls. The default FTPS port is 990, not 21 and can be set with --ftp-port.
        
 
        Restricted filename characters
        
 
        In addition to the default restricted characters set the following characters are also replaced:
        
-
        File names cannot end with the following characters. Repacement is limited to the last character in a file name:
        
+
        File names cannot end with the following characters. Replacement is limited to the last character in a file name:
        
 
@@ -17855,6 +18617,10 @@ 
        Advanced options
        Here are the Advanced options specific to ftp (FTP).
        --ftp-concurrency
        Maximum number of FTP simultaneous connections, 0 for unlimited.
        
+
        Note that setting this is very likely to cause deadlocks so it should be used with care.
        
+
        If you are doing a sync or copy then make sure concurrency is one more than the sum of --transfers and --checkers.
        
+
        If you use --check-first then it just needs to be one more than the maximum of --checkers and --transfers.
        
+
        So for concurrency 3 you'd use --checkers 2 --transfers 2 --check-first or --checkers 1 --transfers 1.
        Properties:
          
 
        • Config: concurrency
          • 
@@ -17907,6 +18673,15 @@ 
          --ftp-writing-mdtm
          
 
        • Type: bool
          • 
 
        • Default: false
          • 
 
        
+
        --ftp-force-list-hidden
        
+
        Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD.
        
+
        Properties:
        
+
          
+
        • Config: force_list_hidden
          • 
+
        • Env Var: RCLONE_FTP_FORCE_LIST_HIDDEN
          • 
+
        • Type: bool
          • 
+
        • Default: false
          • 
+
        --ftp-idle-timeout
        Max time before closing idle connections.
        If no connections have been returned to the connection pool in the time given, rclone will empty the connection pool.
        
@@ -18598,7 +19373,7 @@ 
        --gcs-no-check-bucket
        --gcs-decompress
        If set this will decompress gzip encoded objects.
        
-
        It is possible to upload objects to GCS with "Content-Encoding: gzip" set. Normally rclone will download these files files as compressed objects.
        
+
        It is possible to upload objects to GCS with "Content-Encoding: gzip" set. Normally rclone will download these files as compressed objects.
        If this flag is set then rclone will decompress these files with "Content-Encoding: gzip" as they are received. This means that rclone can't check the size and hash but the file contents will be decompressed.
        Properties:
          
@@ -18607,6 +19382,16 @@ 
          --gcs-decompress
          
 
        • Type: bool
          • 
 
        • Default: false
          • 
 
        
+
        --gcs-endpoint
        
+
        Endpoint for the service.
        
+
        Leave blank normally.
        
+
        Properties:
        
+
          
+
        • Config: endpoint
          • 
+
        • Env Var: RCLONE_GCS_ENDPOINT
          • 
+
        • Type: string
          • 
+
        • Required: false
          • 
+
        --gcs-encoding
        The encoding for the backend.
        See the encoding section in the overview for more info.
        
@@ -19657,8 +20442,8 @@ 
        drives
        
 
 [AllDrives]
 type = combine
-remote = "My Drive=My Drive:" "Test Drive=Test Drive:"
-
        Adding this to the rclone config file will cause those team drives to be accessible with the aliases shown. Any illegal charactes will be substituted with "_" and duplicate names will have numbers suffixed. It will also add a remote called AllDrives which shows all the shared drives combined into one directory tree.
        
+upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
+
        Adding this to the rclone config file will cause those team drives to be accessible with the aliases shown. Any illegal characters will be substituted with "_" and duplicate names will have numbers suffixed. It will also add a remote called AllDrives which shows all the shared drives combined into one directory tree.
        untrash
        Untrash files and directories
        rclone backend untrash remote: [options] [<arguments>+]
        
@@ -20447,7 +21232,7 @@ 
        Invalid refresh token
        the process is very similar to the process of initial setup exemplified before.
        Modified time and hashes
        HiDrive allows modification times to be set on objects accurate to 1 second.
        
-
        HiDrive supports its own hash type which is used to verify the integrety of file contents after successful transfers.
        
+
        HiDrive supports its own hash type which is used to verify the integrity of file contents after successful transfers.
        Restricted filename characters
        HiDrive cannot store files or folders that include / (0x2F) or null-bytes (0x00) in their name. Any other characters can be used in the names of files or folders. Additionally, files or folders cannot be named either of the following: . or ..
        Therefore rclone will automatically replace these characters, if files or folders are stored or accessed with such names.
        
@@ -20799,161 +21584,12 @@ 
        --http-no-head
        Limitations
        rclone about is not supported by the HTTP backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.
        See List of backends that do not support rclone about and rclone about
        
-
        Hubic
        
-
        Paths are specified as remote:path
        
-
        Paths are specified as remote:container (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:container/path/to/dir.
        
-
        Configuration
        
-
        The initial setup for Hubic involves getting a token from Hubic which you need to do in your browser. rclone config walks you through it.
        
-
        Here is an example of how to make a remote called remote. First run:
        
-
         rclone config
        
-
        This will guide you through an interactive setup process:
        
-
        n) New remote
-s) Set configuration password
-n/s> n
-name> remote
-Type of storage to configure.
-Choose a number from below, or type in your own value
-[snip]
-XX / Hubic
-   \ "hubic"
-[snip]
-Storage> hubic
-Hubic Client Id - leave blank normally.
-client_id>
-Hubic Client Secret - leave blank normally.
-client_secret>
-Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
-y) Yes
-n) No
-y/n> y
-If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
-Log in and authorize rclone for access
-Waiting for code...
-Got code
---------------------
-[remote]
-client_id =
-client_secret =
-token = {"access_token":"XXXXXX"}
---------------------
-y) Yes this is OK
-e) Edit this remote
-d) Delete this remote
-y/e/d> y
        
-
        See the remote setup docs for how to set it up on a machine with no Internet browser available.
        
-
        Note that rclone runs a webserver on your local machine to collect the token as returned from Hubic. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall.
        
-
        Once configured you can then use rclone like this,
        
-
        List containers in the top level of your Hubic
        
-
        rclone lsd remote:
        
-
        List all the files in your Hubic
        
-
        rclone ls remote:
        
-
        To copy a local directory to an Hubic directory called backup
        
-
        rclone copy /home/source remote:backup
        
-
        If you want the directory to be visible in the official Hubic browser, you need to copy your files to the default directory
        
-
        rclone copy /home/source remote:default/backup
        
-
        --fast-list
        
-
        This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.
        
-
        Modified time
        
-
        The modified time is stored as metadata on the object as X-Object-Meta-Mtime as floating point since the epoch accurate to 1 ns.
        
-
        This is a de facto standard (used in the official python-swiftclient amongst others) for storing the modification time for an object.
        
-
        Note that Hubic wraps the Swift backend, so most of the properties of are the same.
        
-
        Standard options
        
-
        Here are the Standard options specific to hubic (Hubic).
        
-
        --hubic-client-id
        
-
        OAuth Client Id.
        
-
        Leave blank normally.
        
-
        Properties:
        
-
          
-
        • Config: client_id
          • 
-
        • Env Var: RCLONE_HUBIC_CLIENT_ID
          • 
-
        • Type: string
          • 
-
        • Required: false
          • 
-
        
-
        --hubic-client-secret
        
-
        OAuth Client Secret.
        
-
        Leave blank normally.
        
-
        Properties:
        
-
          
-
        • Config: client_secret
          • 
-
        • Env Var: RCLONE_HUBIC_CLIENT_SECRET
          • 
-
        • Type: string
          • 
-
        • Required: false
          • 
-
        
-
        Advanced options
        
-
        Here are the Advanced options specific to hubic (Hubic).
        
-
        --hubic-token
        
-
        OAuth Access Token as a JSON blob.
        
-
        Properties:
        
-
          
-
        • Config: token
          • 
-
        • Env Var: RCLONE_HUBIC_TOKEN
          • 
-
        • Type: string
          • 
-
        • Required: false
          • 
-
        
-
        --hubic-auth-url
        
-
        Auth server URL.
        
-
        Leave blank to use the provider defaults.
        
-
        Properties:
        
-
          
-
        • Config: auth_url
          • 
-
        • Env Var: RCLONE_HUBIC_AUTH_URL
          • 
-
        • Type: string
          • 
-
        • Required: false
          • 
-
        
-
        --hubic-token-url
        
-
        Token server url.
        
-
        Leave blank to use the provider defaults.
        
-
        Properties:
        
-
          
-
        • Config: token_url
          • 
-
        • Env Var: RCLONE_HUBIC_TOKEN_URL
          • 
-
        • Type: string
          • 
-
        • Required: false
          • 
-
        
-
        --hubic-chunk-size
        
-
        Above this size files will be chunked into a _segments container.
        
-
        Above this size files will be chunked into a _segments container. The default for this is 5 GiB which is its maximum value.
        
-
        Properties:
        
-
          
-
        • Config: chunk_size
          • 
-
        • Env Var: RCLONE_HUBIC_CHUNK_SIZE
          • 
-
        • Type: SizeSuffix
          • 
-
        • Default: 5Gi
          • 
-
        
-
        --hubic-no-chunk
        
-
        Don't chunk files during streaming upload.
        
-
        When doing streaming uploads (e.g. using rcat or mount) setting this flag will cause the swift backend to not upload chunked files.
        
-
        This will limit the maximum upload size to 5 GiB. However non chunked files are easier to deal with and have an MD5SUM.
        
-
        Rclone will still chunk files bigger than chunk_size when doing normal copy operations.
        
-
        Properties:
        
-
          
-
        • Config: no_chunk
          • 
-
        • Env Var: RCLONE_HUBIC_NO_CHUNK
          • 
-
        • Type: bool
          • 
-
        • Default: false
          • 
-
        
-
        --hubic-encoding
        
-
        The encoding for the backend.
        
-
        See the encoding section in the overview for more info.
        
-
        Properties:
        
-
          
-
        • Config: encoding
          • 
-
        • Env Var: RCLONE_HUBIC_ENCODING
          • 
-
        • Type: MultiEncoder
          • 
-
        • Default: Slash,InvalidUtf8
          • 
-
        
-
        Limitations
        
-
        This uses the normal OpenStack Swift mechanism to refresh the Swift API credentials and ignores the expires field returned by the Hubic API.
        
-
        The Swift API doesn't return a correct MD5SUM for segmented files (Dynamic or Static Large Objects) so rclone won't check or use the MD5SUM for these.
        Internet Archive
        The Internet Archive backend utilizes Items on archive.org
        Refer to IAS3 API documentation for the API this backend uses.
        Paths are specified as remote:bucket (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:item/path/to/dir.
        
-
        Once you have made a remote (see the provider specific section above) you can use it like this:
        Unlike S3, listing up all items uploaded by you isn't supported.
        
+
        Once you have made a remote, you can use it like this:
        Make a new item
        rclone mkdir remote:item
        List the contents of a item
        
@@ -20965,12 +21601,12 @@ 
        Notes
        You can optionally wait for the server's processing to finish, by setting non-zero value to wait_archive key. By making it wait, rclone can do normal file comparison. Make sure to set a large enough value (e.g. 30m0s for smaller files) as it can take a long time depending on server's queue.
        About metadata
        This backend supports setting, updating and reading metadata of each file. The metadata will appear as file metadata on Internet Archive. However, some fields are reserved by both Internet Archive and rclone.
        
-
        The following are reserved by Internet Archive: - name - source - size - md5 - crc32 - sha1 - format - old_version - viruscheck
        
+
        The following are reserved by Internet Archive: - name - source - size - md5 - crc32 - sha1 - format - old_version - viruscheck - summation
        Trying to set values to these keys is ignored with a warning. Only setting mtime is an exception. Doing so make it the identical behavior as setting ModTime.
        rclone reserves all the keys starting with rclone-. Setting value for these keys will give you warnings, but values are set according to request.
        If there are multiple values for a key, only the first one is returned. This is a limitation of rclone, that supports one value per one key. It can be triggered when you did a server-side copy.
        Reading metadata will also provide custom (non-standard nor reserved) ones.
        
-
        Configuration
        
+
        Configuration
        Here is an example of making an internetarchive configuration. Most applies to the other providers as well, any differences are described below.
        First run
        rclone config
        
@@ -21039,7 +21675,7 @@ 
        Configuration
        
 e) Edit this remote
 d) Delete this remote
 y/e/d> y
-
        Standard options
        
+
        Standard options
        Here are the Standard options specific to internetarchive (Internet Archive).
        --internetarchive-access-key-id
        IAS3 Access Key.
        
@@ -21061,7 +21697,7 @@ 
        --internetarchive-secret-access-keyType: string
 
      • Required: false
        • 
 
-
        Advanced options
        
+
        Advanced options
        Here are the Advanced options specific to internetarchive (Internet Archive).
        --internetarchive-endpoint
        IAS3 Endpoint.
        
@@ -21138,42 +21774,42 @@ 
        Metadata
        
-
+
-
+
-
+
-
+
-
+
-
+
@@ -21201,48 +21837,60 @@ 
        Metadata
        
-
+
-
+
-
+
+
+
+
+
+
+
+
-
+
        
 
 
        
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 CRC32 calculated by Internet Archive
 string
 01234567NY
 
        
 
        
 format
 Name of format identified by Internet Archive
 string
 Comma-Separated ValuesNY
 
        
 
        
 md5
 MD5 hash calculated by Internet Archive
 string
 01234567012345670123456701234567NY
 
        
 
        
 mtime
 Time of last modification, managed by Rclone
 RFC 3339
 2006-01-02T15:04:05.999999999ZNY
 
        
 
        
 name
 Full file path, without the bucket part
 filename
 backend/internetarchive/internetarchive.goNY
 
        
 
        
 old_version
 Whether the file was replaced and moved by keep-old-version flag
 boolean
 trueNY
 
        
 
        
 rclone-ia-mtime
 SHA1 hash calculated by Internet Archive
 string
 0123456701234567012345670123456701234567NY
 
        
 
        
 size
 File size in bytes
 decimal number
 123456NY
 
        
 
        
 source
 The source of the file
 string
 originalNY
 
        
 
        summationCheck https://forum.rclone.org/t/31922 for how it is usedstringmd5Y
        
 viruscheck
 The last time viruscheck process was run for the file (?)
 unixtime
 1654191352NY
 
        
 
 
        
 
        See the metadata docs for more info.
        
 
        Jottacloud
        
-
        Jottacloud is a cloud storage service provider from a Norwegian company, using its own datacenters in Norway. In addition to the official service at jottacloud.com, it also provides white-label solutions to different companies, such as: * Telia * Telia Cloud (cloud.telia.se) * Telia Sky (sky.telia.no) * Tele2 * Tele2 Cloud (mittcloud.tele2.se) * Elkjøp (with subsidiaries): * Elkjøp Cloud (cloud.elkjop.no) * Elgiganten Sweden (cloud.elgiganten.se) * Elgiganten Denmark (cloud.elgiganten.dk) * Giganti Cloud (cloud.gigantti.fi) * ELKO Clouud (cloud.elko.is)
        
+
        Jottacloud is a cloud storage service provider from a Norwegian company, using its own datacenters in Norway. In addition to the official service at jottacloud.com, it also provides white-label solutions to different companies, such as: * Telia * Telia Cloud (cloud.telia.se) * Telia Sky (sky.telia.no) * Tele2 * Tele2 Cloud (mittcloud.tele2.se) * Elkjøp (with subsidiaries): * Elkjøp Cloud (cloud.elkjop.no) * Elgiganten Sweden (cloud.elgiganten.se) * Elgiganten Denmark (cloud.elgiganten.dk) * Giganti Cloud (cloud.gigantti.fi) * ELKO Cloud (cloud.elko.is)
        
 
        Most of the white-label versions are supported by this backend, although may require different authentication setup - described below.
        
 
        Paths are specified as remote:path
        
 
        Paths may be as deep as required, e.g. remote:directory/subdirectory.
        
 
        Authentication types
        
 
        Some of the whitelabel versions uses a different authentication method than the official service, and you have to choose the correct one when setting up the remote.
        
 
        Standard authentication
        
-
        To configure Jottacloud you will need to generate a personal security token in the Jottacloud web interface. You will the option to do in your account security settings (for whitelabel version you need to find this page in its web interface). Note that the web interface may refer to this token as a JottaCli token.
        
+
        The standard authentication method used by the official service (jottacloud.com), as well as some of the whitelabel services, requires you to generate a single-use personal login token from the account security settings in the service's web interface. Log in to your account, go to "Settings" and then "Security", or use the direct link presented to you by rclone when configuring the remote: https://www.jottacloud.com/web/secure. Scroll down to the section "Personal login token", and click the "Generate" button. Note that if you are using a whitelabel service you probably can't use the direct link, you need to find the same page in their dedicated web interface, and also it may be in a different location than described above.
        
+
        To access your account from multiple instances of rclone, you need to configure each of them with a separate personal login token. E.g. you create a Jottacloud remote with rclone in one location, and copy the configuration file to a second location where you also want to run rclone and access the same remote. Then you need to replace the token for one of them, using the config reconnect command, which requires you to generate a new personal login token and supply as input. If you do not do this, the token may easily end up being invalidated, resulting in both instances failing with an error message something along the lines of:
        
+
        oauth2: cannot fetch token: 400 Bad Request
+Response: {"error":"invalid_grant","error_description":"Stale token"}
        
+
        When this happens, you need to replace the token as described above to be able to use your remote again.
        
+
        All personal login tokens you have taken into use will be listed in the web interface under "My logged in devices", and from the right side of that list you can click the "X" button to revoke individual tokens.
        
 
        Legacy authentication
        
 
        If you are using one of the whitelabel versions (e.g. from Elkjøp) you may not have the option to generate a CLI token. In this case you'll have to use the legacy authentication. To do this select yes when the setup asks for legacy authentication and enter your username and password. The rest of the setup is identical to the default setup.
        
 
        Telia Cloud authentication
        
 
        Similar to other whitelabel versions Telia Cloud doesn't offer the option of creating a CLI token, and additionally uses a separate authentication flow where the username is generated internally. To setup rclone to use Telia Cloud, choose Telia Cloud authentication in the setup. The rest of the setup is identical to the default setup.
        
 
        Tele2 Cloud authentication
        
 
        As Tele2-Com Hem merger was completed this authentication can be used for former Com Hem Cloud and Tele2 Cloud customers as no support for creating a CLI token exists, and additionally uses a separate authentication flow where the username is generated internally. To setup rclone to use Tele2 Cloud, choose Tele2 Cloud authentication in the setup. The rest of the setup is identical to the default setup.
        
-
        Configuration
        
+
        Configuration
        
 
        Here is an example of how to make a remote called remote with the default setup. First run:
        
 
        rclone config
        
 
        This will guide you through an interactive setup process:
        
@@ -21339,7 +21987,7 @@ 
        Devices and Mountpoints
        
 
        With rclone you'll want to use the standard Jotta/Archive device/mountpoint in most cases. However, you may for example want to access files from the sync or backup functionality provided by the official clients, and rclone therefore provides the option to select other devices and mountpoints during config.
        
 
        You are allowed to create new devices and mountpoints. All devices except the built-in Jotta device are treated as backup devices by official Jottacloud clients, and the mountpoints on them are individual backup sets.
        
 
        With the built-in Jotta device, only existing, built-in, mountpoints can be selected. In addition to the mentioned Archive and Sync, it may contain several other mountpoints such as: Latest, Links, Shared and Trash. All of these are special mountpoints with a different internal representation than the "regular" mountpoints. Rclone will only to a very limited degree support them. Generally you should avoid these, unless you know what you are doing.
        
-
        --fast-list
        
+
        --fast-list
        
 
        This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.
        
 
        Note that the implementation in Jottacloud always uses only a single API request to get the entire list, so for large folders this could lead to long wait time before the first results are shown.
        
 
        Note also that with rclone version 1.58 and newer information about MIME types are not available when using --fast-list.
        
@@ -21398,12 +22046,12 @@ 
        Restricted filename characters
        
 
        Invalid UTF-8 bytes will also be replaced, as they can't be used in XML strings.
        
 
        Deleting files
        
 
        By default, rclone will send all files to the trash when deleting files. They will be permanently deleted automatically after 30 days. You may bypass the trash and permanently delete files immediately by using the --jottacloud-hard-delete flag, or set the equivalent environment variable. Emptying the trash is supported by the cleanup command.
        
-
        Versions
        
+
        Versions
        
 
        Jottacloud supports file versioning. When rclone uploads a new version of a file it creates a new version of it. Currently rclone only supports retrieving the current version but older versions can be accessed via the Jottacloud Website.
        
 
        Versioning can be disabled by --jottacloud-no-versions option. This is achieved by deleting the remote file prior to uploading a new version. If the upload the fails no version of the file will be available in the remote.
        
 
        Quota information
        
 
        To view your current quota you can use the rclone about remote: command which will display your usage limit (unless it is unlimited) and the current usage.
        
-
        Advanced options
        
+
        Advanced options
        
 
        Here are the Advanced options specific to jottacloud (Jottacloud).
        
 
        --jottacloud-md5-memory-limit
        
 
        Files bigger than this will be cached on disk to calculate the MD5 if required.
        
@@ -21461,7 +22109,7 @@ 
        --jottacloud-encoding
        
 
      • Type: MultiEncoder
        • 
 
      • Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot
        • 
 
      
-
      Limitations
      
+
      Limitations
      
 
      Note that Jottacloud is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".
      
 
      There are quite a few characters that can't be in Jottacloud file names. Rclone will map these names to and from an identical looking unicode equivalent. For example if a file has a ? in it will be mapped to ？ instead.
      
 
      Jottacloud only supports filenames up to 255 characters in length.
      
@@ -21470,7 +22118,7 @@ 
      Troubleshooting
      
 
      Koofr
      
 
      Paths are specified as remote:path
      
 
      Paths may be as deep as required, e.g. remote:directory/subdirectory.
      
-
      Configuration
      
+
      Configuration
      
 
      The initial setup for Koofr involves creating an application password for rclone. You can do that by opening the Koofr web application, giving the password a nice name like rclone and clicking on generate.
      
 
      Here is an example of how to make a remote called koofr. First run:
      
 
       rclone config
      
@@ -21557,7 +22205,7 @@ 
      Restricted filename characters
      
 
 
 
      Invalid UTF-8 bytes will also be replaced, as they can't be used in XML strings.
      
-
      Standard options
      
+
      Standard options
      
 
      Here are the Standard options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers).
      
 
      --koofr-provider
      
 
      Choose your storage provider.
      
@@ -21635,7 +22283,7 @@ 
      --koofr-password
      
 
    44. Type: string
      45. 
 
    45. Required: true
      46. 
 
-
      Advanced options
      
+
      Advanced options
      
 
      Here are the Advanced options specific to koofr (Koofr, Digi Storage and other Koofr-compatible storage providers).
      
 
      --koofr-mountid
      
 
      Mount ID of the mount to use.
      
@@ -21667,7 +22315,7 @@ 
      --koofr-encoding
      
 
    46. Type: MultiEncoder
      47. 
 
    47. Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot
      48. 
 
-
      Limitations
      
+
      Limitations
      
 
      Note that Koofr is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".
      
 
      Providers
      
 
      Koofr
      
@@ -21809,7 +22457,7 @@ 
      Features highlights
      
 
    48. Storage keeps hash for all files and performs transparent deduplication, the hash algorithm is a modified SHA1
      49. 
 
    49. If a particular file is already present in storage, one can quickly submit file hash instead of long file upload (this optimization is supported by rclone)
      50. 
 
-
      Configuration
      
+
      Configuration
      
 
      Here is an example of making a mailru configuration. First create a Mail.ru Cloud account and choose a tariff, then run
      
 
      rclone config
      
 
      This will guide you through an interactive setup process:
      
@@ -21875,7 +22523,7 @@ 
      Configuration
      
 
      rclone ls remote:directory
      
 
      Sync /home/local/directory to the remote path, deleting any excess files in the path.
      
 
      rclone sync -i /home/local/directory remote:directory
      
-
      Modified time
      
+
      Modified time
      
 
      Files support a modification time attribute with up to 1 second precision. Directories do not have a modification time, which is shown as "Jan 1 1970".
      
 
      Hash checksums
      
 
      Hash sums use a custom Mail.ru algorithm based on SHA1. If file size is less than or equal to the SHA1 block size (20 bytes), its hash is simply its data right-padded with zero bytes. Hash sum of a larger file is computed as a SHA1 sum of the file data bytes concatenated with a decimal representation of the data length.
      
@@ -21937,7 +22585,7 @@ 
      Restricted filename characters
      
 
 
 
      Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.
      
-
      Standard options
      
+
      Standard options
      
 
      Here are the Standard options specific to mailru (Mail.ru Cloud).
      
 
      --mailru-user
      
 
      User name (usually email).
      
@@ -21979,7 +22627,7 @@ 
      --mailru-speedup-enable
      
 
 
 
-
      Advanced options
      
+
      Advanced options
      
 
      Here are the Advanced options specific to mailru (Mail.ru Cloud).
      
 
      --mailru-speedup-file-patterns
      
 
      Comma separated list of file name patterns eligible for speedup (put by hash).
      
@@ -22109,7 +22757,7 @@ 
      --mailru-encoding
      
 
    50. Type: MultiEncoder
      51. 
 
    51. Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot
      52. 
 
-
      Limitations
      
+
      Limitations
      
 
      File size limits depend on your account. A single file size is limited by 2G for a free account and unlimited for paid tariffs. Please refer to the Mail.ru site for the total uploaded size limits.
      
 
      Note that Mailru is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".
      
 
      Mega
      
@@ -22117,7 +22765,7 @@ 
      Mega
      
 
      This is an rclone backend for Mega which supports the file transfer features of Mega using the same client side encryption.
      
 
      Paths are specified as remote:path
      
 
      Paths may be as deep as required, e.g. remote:directory/subdirectory.
      
-
      Configuration
      
+
      Configuration
      
 
      Here is an example of how to make a remote called remote. First run:
      
 
       rclone config
      
 
      This will guide you through an interactive setup process:
      
@@ -22195,7 +22843,7 @@ 
      Duplicated files
      
 
      Use rclone dedupe to fix duplicated files.
      
 
      Failure to log-in
      
 
      Object not found
      
-
      If you are connecting to your Mega remote for the first time, to test access and syncronisation, you may receive an error such as
      
+
      If you are connecting to your Mega remote for the first time, to test access and synchronization, you may receive an error such as
      
 
      Failed to create file system for "my-mega-remote:": 
 couldn't login: Object (typically, node or user) not found
      
 
      The diagnostic steps often recommended in the rclone forum start with the MEGAcmd utility. Note that this refers to the official C++ command from https://github.com/meganz/MEGAcmd and not the go language built command from t3rm1n4l/megacmd that is no longer maintained.
      
@@ -22218,7 +22866,7 @@ 
      Repeated commands blocks access
      
 
      Note that once blocked, the use of other tools (such as megacmd) is not a sure workaround: following megacmd login times have been observed in succession for blocked remote: 7 minutes, 20 min, 30min, 30 min, 30min. Web access looks unaffected though.
      
 
      Investigation is continuing in relation to workarounds based on timeouts, pacers, retrials and tpslimits - if you discover something relevant, please post on the forum.
      
 
      So, if rclone was working nicely and suddenly you are unable to log-in and you are sure the user and the password are correct, likely you have got the remote blocked for a while.
      
-
      Standard options
      
+
      Standard options
      
 
      Here are the Standard options specific to mega (Mega).
      
 
      --mega-user
      
 
      User name.
      
@@ -22239,7 +22887,7 @@ 
      --mega-pass
      
 
    52. Type: string
      53. 
 
    53. Required: true
      54. 
 
-
      Advanced options
      
+
      Advanced options
      
 
      Here are the Advanced options specific to mega (Mega).
      
 
      --mega-debug
      
 
      Output more debug from Mega.
      
@@ -22271,13 +22919,13 @@ 
      --mega-encoding
      
 
    54. Type: MultiEncoder
      55. 
 
    55. Default: Slash,InvalidUtf8,Dot
      56. 
 
-
      Limitations
      
+
      Limitations
      
 
      This backend uses the go-mega go library which is an opensource go library implementing the Mega API. There doesn't appear to be any documentation for the mega protocol beyond the mega C++ SDK source code so there are likely quite a few errors still remaining in this library.
      
 
      Mega allows duplicate files which may confuse rclone.
      
 
      Memory
      
 
      The memory backend is an in RAM backend. It does not persist its data - use the local backend for that.
      
 
      The memory backend behaves like a bucket-based remote (e.g. like s3). Because it has no parameters you can just use it with the :memory: remote name.
      
-
      Configuration
      
+
      Configuration
      
 
      You can configure it as a remote like this with rclone config too if you want to:
      
 
      No remotes found, make a new one?
 n) New remote
@@ -22317,7 +22965,7 @@ 
      Akamai NetStorage
      
 
      Paths are specified as remote: You may put subdirectories in too, e.g. remote:/path/to/dir. If you have a CP code you can use that as the folder after the domain such as <domain>/<cpcode>/<internal directories within cpcode>.
      
 
      For example, this is commonly configured with or without a CP code: * With a CP code. [your-domain-prefix]-nsu.akamaihd.net/123456/subdirectory/ * Without a CP code. [your-domain-prefix]-nsu.akamaihd.net
      
 
      See all buckets rclone lsd remote: The initial setup for Netstorage involves getting an account and secret. Use rclone config to walk you through the setup process.
      
-
      Configuration
      
+
      Configuration
      
 
      Here's an example of how to make a remote called ns1.
      
 
        
 
      1. To begin the interactive configuration process, enter this command:
        2. 
@@ -22411,7 +23059,7 @@ 
        Implicit vs. Explicit Directories
        With NetStorage, directories can exist in one of two forms:
        
 
          
 
        1. Explicit Directory. This is an actual, physical directory that you have created in a storage group.
          2. 
-
        2. Implicit Directory. This refers to a directory within a path that has not been physically created. For example, during upload of a file, non-existent subdirectories can be specified in the target path. NetStorage creates these as "implicit." While the directories aren't physically created, they exist implicitly and the noted path is connected with the uploaded file.
          3. 
+
        3. Implicit Directory. This refers to a directory within a path that has not been physically created. For example, during upload of a file, nonexistent subdirectories can be specified in the target path. NetStorage creates these as "implicit." While the directories aren't physically created, they exist implicitly and the noted path is connected with the uploaded file.
          4. 
 
        
 
        Rclone will intercept all file uploads and mkdir commands for the NetStorage remote and will explicitly issue the mkdir command for each directory in the uploading path. This will help with the interoperability with the other Akamai services such as SFTP and the Content Management Shell (CMShell). Rclone will not guarantee correctness of operations with implicit directories which might have been created as a result of using an upload API directly.
        
 
        --fast-list / ListR support
        
@@ -22425,7 +23073,7 @@ 
        --fast-list / ListR support
        
 
        Purge
        
 
        NetStorage remote supports the purge feature by using the "quick-delete" NetStorage API action. The quick-delete action is disabled by default for security reasons and can be enabled for the account through the Akamai portal. Rclone will first try to use quick-delete action for the purge command and if this functionality is disabled then will fall back to a standard delete method.
        
 
        Note: Read the NetStorage Usage API for considerations when using "quick-delete". In general, using quick-delete method will not delete the tree immediately and objects targeted for quick-delete may still be accessible.
        
-
        Standard options
        
+
        Standard options
        
 
        Here are the Standard options specific to netstorage (Akamai NetStorage).
        
 
        --netstorage-host
        
 
        Domain+path of NetStorage host to connect to.
        
@@ -22457,7 +23105,7 @@ 
        --netstorage-secret
        
 
      2. Type: string
        3. 
 
      3. Required: true
        4. 
 
-
        Advanced options
        
+
        Advanced options
        
 
        Here are the Advanced options specific to netstorage (Akamai NetStorage).
        
 
        --netstorage-protocol
        
 
        Select between HTTP or HTTPS protocol.
        
@@ -22497,7 +23145,7 @@ 
 
        The desired path location (including applicable sub-directories) ending in the object that will be the target of the symlink (for example, /links/mylink). Include the file extension for the object, if applicable. rclone backend symlink <src> <path>
        
 
        Microsoft Azure Blob Storage
        
 
        Paths are specified as remote:container (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:container/path/to/dir.
        
-
        Configuration
        
+
        Configuration
        
 
        Here is an example of making a Microsoft Azure Blob Storage configuration. For a remote called remote. First run:
        
 
         rclone config
        
 
        This will guide you through an interactive setup process:
        
@@ -22539,9 +23187,9 @@ 
        Configuration
        
 
        rclone ls remote:container
        
 
        Sync /home/local/directory to the remote container, deleting any excess files in the container.
        
 
        rclone sync -i /home/local/directory remote:container
        
-
        --fast-list
        
+
        --fast-list
        
 
        This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.
        
-
        Modified time
        
+
        Modified time
        
 
        The modified time is stored as metadata on the object with the mtime key. It is stored using RFC3339 Format time with nanosecond precision. The metadata is supplied during directory listings so there is no overhead to using it.
        
 
        Performance
        
 
        When uploading large files, increasing the value of --azureblob-upload-concurrency will increase performance at the cost of using more memory. The default of 16 is set quite conservatively to use less memory. It maybe be necessary raise it to 64 or higher to fully utilize a 1 GBit/s link with a single file transfer.
        
@@ -22604,7 +23252,7 @@ 
        SAS URL
        
 
        Note that you can't see or access any other containers - this will fail
        
 
        rclone ls azureblob:othercontainer
        
 
        Container level SAS URLs are useful for temporarily allowing third parties access to a single container or putting credentials into an untrusted environment such as a CI build server.
        
-
        Standard options
        
+
        Standard options
        
 
        Here are the Standard options specific to azureblob (Microsoft Azure Blob Storage).
        
 
        --azureblob-account
        
 
        Storage Account Name.
        
@@ -22672,7 +23320,7 @@ 
        --azureblob-use-emulator
        
 
      4. Type: bool
        5. 
 
      5. Default: false
        6. 
 
-
        Advanced options
        
+
        Advanced options
        
 
        Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage).
        
 
        --azureblob-msi-object-id
        
 
        Object ID of the user-assigned MSI to use, if any.
        
@@ -22852,7 +23500,7 @@ 
        --azureblob-no-head-object
        
 
      6. Type: bool
        7. 
 
      7. Default: false
        8. 
 
-
        Limitations
        
+
        Limitations
        
 
        MD5 sums are only uploaded with chunked files if the source has an MD5 sum. This will always be the case for a local to azure copy.
        
 
        rclone about is not supported by the Microsoft Azure Blob storage backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.
        
 
        See List of backends that do not support rclone about and rclone about
        
@@ -22863,7 +23511,7 @@ 
        Azure Storage Emulator Support
        
 
        Microsoft OneDrive
        
 
        Paths are specified as remote:path
        
 
        Paths may be as deep as required, e.g. remote:directory/subdirectory.
        
-
        Configuration
        
+
        Configuration
        
 
        The initial setup for OneDrive involves getting a token from Microsoft which you need to do in your browser. rclone config walks you through it.
        
 
        Here is an example of how to make a remote called remote. First run:
        
 
         rclone config
        
@@ -22950,7 +23598,7 @@ 
        Configuration
        
 
        rclone copy /home/source remote:backup
        
 
        Getting your own Client ID and Key
        
 
        rclone uses a default Client ID when talking to OneDrive, unless a custom client_id is specified in the config. The default Client ID and Key are shared by all rclone users when performing requests.
        
-
        You may choose to create and use your own Client ID, in case the default one does not work well for you. For example, you might see throtting.
        
+
        You may choose to create and use your own Client ID, in case the default one does not work well for you. For example, you might see throttling.
        
 
        Creating Client ID for OneDrive Personal
        
 
        To create your own Client ID, please follow these steps:
        
 
          
@@ -22968,7 +23616,7 @@ 
          Creating Client ID for OneDriv
 
          You may try to verify you account, or try to limit the App to your organization only, as shown below.
          
 
            
 
          1. Make sure to create the App with your business account.
            2. 
-
          2. Follow the steps above to create an App. However, we need a different account type here: Accounts in this organizational directory only (*** - Single tenant). Note that you can also change the account type aftering creating the App.
            3. 
+
          3. Follow the steps above to create an App. However, we need a different account type here: Accounts in this organizational directory only (*** - Single tenant). Note that you can also change the account type after creating the App.
            4. 
 
          4. Find the tenant ID of your organization.
            5. 
 
          5. In the rclone config, set auth_url to https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/authorize.
            6. 
 
          6. In the rclone config, set token_url to https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/token.
            7. 
@@ -23078,7 +23726,7 @@ 
            Restricted filename characters
            
 
            Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.
            
 
            Deleting files
            
 
            Any files you delete with rclone will end up in the trash. Microsoft doesn't provide an API to permanently delete files, nor to empty the trash, so you will have to do that with one of Microsoft's apps or via the OneDrive website.
            
-
            Standard options
            
+
            Standard options
            
 
            Here are the Standard options specific to onedrive (Microsoft OneDrive).
            
 
            --onedrive-client-id
            
 
            OAuth Client Id.
            
@@ -23124,11 +23772,11 @@ 
            --onedrive-region
            
 
 
          7. "cn"
 
              
-
            • Azure and Office 365 operated by 21Vianet in China
              • 
+
            • Azure and Office 365 operated by Vnet Group in China
              • 
 
            8. 
 
 
-
            Advanced options
            
+
            Advanced options
            
 
            Here are the Advanced options specific to onedrive (Microsoft OneDrive).
            
 
            --onedrive-token
            
 
            OAuth Access Token as a JSON blob.
            
@@ -23342,7 +23990,7 @@ 
            --onedrive-encoding
            
 
          8. Type: MultiEncoder
            9. 
 
          9. Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot
            10. 
 
-
            Limitations
            
+
            Limitations
            
 
            If you don't use rclone for 90 days the refresh token will expire. This will result in authorization problems. This is easy to fix by running the rclone config reconnect remote: command to get a new token and refresh token.
            
 
            Naming
            
 
            Note that OneDrive is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".
            
@@ -23354,8 +24002,8 @@ 
            Path length
            
 
            Number of files
            
 
            OneDrive seems to be OK with at least 50,000 files in a folder, but at 100,000 rclone will get errors listing the directory like couldn’t list files: UnknownError:. See #2707 for more info.
            
 
            An official document about the limitations for different types of OneDrive can be found here.
            
-
            Versions
            
-
            Every change in a file OneDrive causes the service to create a new version of the the file. This counts against a users quota. For example changing the modification time of a file creates a second version, so the file apparently uses twice the space.
            
+
            Versions
            
+
            Every change in a file OneDrive causes the service to create a new version of the file. This counts against a users quota. For example changing the modification time of a file creates a second version, so the file apparently uses twice the space.
            
 
            For example the copy command is affected by this as rclone copies the file and then afterwards sets the modification time to match the source file which uses another version.
            
 
            You can use the rclone cleanup command (see below) to remove all old versions.
            
 
            Or you can set the no_versions parameter to true and rclone will remove versions after operations which create new versions. This takes extra transactions so only enable it if you need it.
            
@@ -23414,7 +24062,7 @@ 
            OpenDrive
            
 
            Paths are specified as remote:path
            
 
            Paths may be as deep as required, e.g. remote:directory/subdirectory.
            
-
            Configuration
            
+
            Configuration
            
 
            Here is an example of how to make a remote called remote. First run:
            
 
             rclone config
            
 
            This will guide you through an interactive setup process:
            
@@ -23557,7 +24205,7 @@ 
            Restricted filename characters
            
 
 
 
            Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.
            
-
            Standard options
            
+
            Standard options
            
 
            Here are the Standard options specific to opendrive (OpenDrive).
            
 
            --opendrive-username
            
 
            Username.
            
@@ -23578,7 +24226,7 @@ 
            --opendrive-password
            
 
          10. Type: string
            11. 
 
          11. Required: true
            12. 
 
-
            Advanced options
            
+
            Advanced options
            
 
            Here are the Advanced options specific to opendrive (OpenDrive).
            
 
            --opendrive-encoding
            
 
            The encoding for the backend.
            
@@ -23600,11 +24248,388 @@ 
            --opendrive-chunk-size
            
 
          12. Type: SizeSuffix
            13. 
 
          13. Default: 10Mi
            14. 
 
-
            Limitations
            
+
            Limitations
            
 
            Note that OpenDrive is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".
            
 
            There are quite a few characters that can't be in OpenDrive file names. These can't occur on Windows platforms, but on non-Windows platforms they are common. Rclone will map these names to and from an identical looking unicode equivalent. For example if a file has a ? in it will be mapped to ？ instead.
            
 
            rclone about is not supported by the OpenDrive backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.
            
 
            See List of backends that do not support rclone about and rclone about
            
+
            Oracle Object Storage
            
+
            Oracle Object Storage Overview
            
+
            Oracle Object Storage FAQ
            
+
            Paths are specified as remote:bucket (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:bucket/path/to/dir.
            
+
            Configuration
            
+
            Here is an example of making an oracle object storage configuration. rclone config walks you through it.
            
+
            Here is an example of how to make a remote called remote. First run:
            
+
             rclone config
            
+
            This will guide you through an interactive setup process:
            
+
            n) New remote
+d) Delete remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+e/n/d/r/c/s/q> n
+
+Enter name for new remote.
+name> remote
+
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+XX / Oracle Cloud Infrastructure Object Storage
+   \ (oracleobjectstorage)
+Storage> oracleobjectstorage
+
+Option provider.
+Choose your Auth Provider
+Choose a number from below, or type in your own string value.
+Press Enter for the default (env_auth).
+ 1 / automatically pickup the credentials from runtime(env), first one to provide auth wins
+   \ (env_auth)
+   / use an OCI user and an API key for authentication.
+ 2 | you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key.
+   | https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm
+   \ (user_principal_auth)
+   / use instance principals to authorize an instance to make API calls. 
+ 3 | each instance has its own identity, and authenticates using the certificates that are read from instance metadata. 
+   | https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm
+   \ (instance_principal_auth)
+ 4 / use resource principals to make API calls
+   \ (resource_principal_auth)
+ 5 / no credentials needed, this is typically for reading public buckets
+   \ (no_auth)
+provider> 2
+
+Option namespace.
+Object storage namespace
+Enter a value.
+namespace> idbamagbg734
+
+Option compartment.
+Object storage compartment OCID
+Enter a value.
+compartment> ocid1.compartment.oc1..aaaaaaaapufkxc7ame3sthry5i7ujrwfc7ejnthhu6bhanm5oqfjpyasjkba
+
+Option region.
+Object storage Region
+Enter a value.
+region> us-ashburn-1
+
+Option endpoint.
+Endpoint for Object storage API.
+Leave blank to use the default endpoint for the region.
+Enter a value. Press Enter to leave empty.
+endpoint> 
+
+Option config_file.
+Path to OCI config file
+Choose a number from below, or type in your own string value.
+Press Enter for the default (~/.oci/config).
+ 1 / oci configuration file location
+   \ (~/.oci/config)
+config_file> /etc/oci/dev.conf
+
+Option config_profile.
+Profile name inside OCI config file
+Choose a number from below, or type in your own string value.
+Press Enter for the default (Default).
+ 1 / Use the default profile
+   \ (Default)
+config_profile> Test
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: oracleobjectstorage
+- namespace: idbamagbg734
+- compartment: ocid1.compartment.oc1..aaaaaaaapufkxc7ame3sthry5i7ujrwfc7ejnthhu6bhanm5oqfjpyasjkba
+- region: us-ashburn-1
+- provider: user_principal_auth
+- config_file: /etc/oci/dev.conf
+- config_profile: Test
+Keep this "remote" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
            
+
            See all buckets
            
+
            rclone lsd remote:
            
+
            Create a new bucket
            
+
            rclone mkdir remote:bucket
            
+
            List the contents of a bucket
            
+
            rclone ls remote:bucket
+rclone ls remote:bucket --max-depth 1
            
+
            Modified time
            
+
            The modified time is stored as metadata on the object as opc-meta-mtime as floating point since the epoch, accurate to 1 ns.
            
+
            If the modification time needs to be updated rclone will attempt to perform a server side copy to update the modification if the object can be copied in a single part. In the case the object is larger than 5Gb, the object will be uploaded rather than copied.
            
+
            Note that reading this from the object takes an additional HEAD request as the metadata isn't returned in object listings.
            
+
            Multipart uploads
            
+
            rclone supports multipart uploads with OOS which means that it can upload files bigger than 5 GiB.
            
+
            Note that files uploaded both with multipart upload and through crypt remotes do not have MD5 sums.
            
+
            rclone switches from single part uploads to multipart uploads at the point specified by --oos-upload-cutoff. This can be a maximum of 5 GiB and a minimum of 0 (ie always upload multipart files).
            
+
            The chunk sizes used in the multipart upload are specified by --oos-chunk-size and the number of chunks uploaded concurrently is specified by --oos-upload-concurrency.
            
+
            Multipart uploads will use --transfers * --oos-upload-concurrency * --oos-chunk-size extra memory. Single part uploads to not use extra memory.
            
+
            Single part transfers can be faster than multipart transfers or slower depending on your latency from oos - the more latency, the more likely single part transfers will be faster.
            
+
            Increasing --oos-upload-concurrency will increase throughput (8 would be a sensible value) and increasing --oos-chunk-size also increases throughput (16M would be sensible). Increasing either of these will use more memory. The default values are high enough to gain most of the possible performance without using too much memory.
            
+
            Standard options
            
+
            Here are the Standard options specific to oracleobjectstorage (Oracle Cloud Infrastructure Object Storage).
            
+
            --oos-provider
            
+
            Choose your Auth Provider
            
+
            Properties:
            
+
              
+
            • Config: provider
              • 
+
            • Env Var: RCLONE_OOS_PROVIDER
              • 
+
            • Type: string
              • 
+
            • Default: "env_auth"
              • 
+
            • Examples:
+
                
+
              • "env_auth"
+
                  
+
                • automatically pickup the credentials from runtime(env), first one to provide auth wins
                  • 
+
                • 
+
              • "user_principal_auth"
+
                  
+
                • use an OCI user and an API key for authentication.
                  • 
+
                • you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key.
                  • 
+
                • https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm
                  • 
+
                • 
+
              • "instance_principal_auth"
+
                  
+
                • use instance principals to authorize an instance to make API calls.
                  • 
+
                • each instance has its own identity, and authenticates using the certificates that are read from instance metadata.
                  • 
+
                • https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm
                  • 
+
                • 
+
              • "resource_principal_auth"
+
                  
+
                • use resource principals to make API calls
                  • 
+
                • 
+
              • "no_auth"
+
                  
+
                • no credentials needed, this is typically for reading public buckets
                  • 
+
                • 
+
              • 
+
            
+
            --oos-namespace
            
+
            Object storage namespace
            
+
            Properties:
            
+
              
+
            • Config: namespace
              • 
+
            • Env Var: RCLONE_OOS_NAMESPACE
              • 
+
            • Type: string
              • 
+
            • Required: true
              • 
+
            
+
            --oos-compartment
            
+
            Object storage compartment OCID
            
+
            Properties:
            
+
              
+
            • Config: compartment
              • 
+
            • Env Var: RCLONE_OOS_COMPARTMENT
              • 
+
            • Provider: !no_auth
              • 
+
            • Type: string
              • 
+
            • Required: true
              • 
+
            
+
            --oos-region
            
+
            Object storage Region
            
+
            Properties:
            
+
              
+
            • Config: region
              • 
+
            • Env Var: RCLONE_OOS_REGION
              • 
+
            • Type: string
              • 
+
            • Required: true
              • 
+
            
+
            --oos-endpoint
            
+
            Endpoint for Object storage API.
            
+
            Leave blank to use the default endpoint for the region.
            
+
            Properties:
            
+
              
+
            • Config: endpoint
              • 
+
            • Env Var: RCLONE_OOS_ENDPOINT
              • 
+
            • Type: string
              • 
+
            • Required: false
              • 
+
            
+
            --oos-config-file
            
+
            Path to OCI config file
            
+
            Properties:
            
+
              
+
            • Config: config_file
              • 
+
            • Env Var: RCLONE_OOS_CONFIG_FILE
              • 
+
            • Provider: user_principal_auth
              • 
+
            • Type: string
              • 
+
            • Default: "~/.oci/config"
              • 
+
            • Examples:
+
                
+
              • "~/.oci/config"
+
                  
+
                • oci configuration file location
                  • 
+
                • 
+
              • 
+
            
+
            --oos-config-profile
            
+
            Profile name inside the oci config file
            
+
            Properties:
            
+
              
+
            • Config: config_profile
              • 
+
            • Env Var: RCLONE_OOS_CONFIG_PROFILE
              • 
+
            • Provider: user_principal_auth
              • 
+
            • Type: string
              • 
+
            • Default: "Default"
              • 
+
            • Examples:
+
                
+
              • "Default"
+
                  
+
                • Use the default profile
                  • 
+
                • 
+
              • 
+
            
+
            Advanced options
            
+
            Here are the Advanced options specific to oracleobjectstorage (Oracle Cloud Infrastructure Object Storage).
            
+
            --oos-upload-cutoff
            
+
            Cutoff for switching to chunked upload.
            
+
            Any files larger than this will be uploaded in chunks of chunk_size. The minimum is 0 and the maximum is 5 GiB.
            
+
            Properties:
            
+
              
+
            • Config: upload_cutoff
              • 
+
            • Env Var: RCLONE_OOS_UPLOAD_CUTOFF
              • 
+
            • Type: SizeSuffix
              • 
+
            • Default: 200Mi
              • 
+
            
+
            --oos-chunk-size
            
+
            Chunk size to use for uploading.
            
+
            When uploading files larger than upload_cutoff or files with unknown size (e.g. from "rclone rcat" or uploaded with "rclone mount" or google photos or google docs) they will be uploaded as multipart uploads using this chunk size.
            
+
            Note that "upload_concurrency" chunks of this size are buffered in memory per transfer.
            
+
            If you are transferring large files over high-speed links and you have enough memory, then increasing this will speed up the transfers.
            
+
            Rclone will automatically increase the chunk size when uploading a large file of known size to stay below the 10,000 chunks limit.
            
+
            Files of unknown size are uploaded with the configured chunk_size. Since the default chunk size is 5 MiB and there can be at most 10,000 chunks, this means that by default the maximum size of a file you can stream upload is 48 GiB. If you wish to stream upload larger files then you will need to increase chunk_size.
            
+
            Increasing the chunk size decreases the accuracy of the progress statistics displayed with "-P" flag.
            
+
            Properties:
            
+
              
+
            • Config: chunk_size
              • 
+
            • Env Var: RCLONE_OOS_CHUNK_SIZE
              • 
+
            • Type: SizeSuffix
              • 
+
            • Default: 5Mi
              • 
+
            
+
            --oos-upload-concurrency
            
+
            Concurrency for multipart uploads.
            
+
            This is the number of chunks of the same file that are uploaded concurrently.
            
+
            If you are uploading small numbers of large files over high-speed links and these uploads do not fully utilize your bandwidth, then increasing this may help to speed up the transfers.
            
+
            Properties:
            
+
              
+
            • Config: upload_concurrency
              • 
+
            • Env Var: RCLONE_OOS_UPLOAD_CONCURRENCY
              • 
+
            • Type: int
              • 
+
            • Default: 10
              • 
+
            
+
            --oos-copy-cutoff
            
+
            Cutoff for switching to multipart copy.
            
+
            Any files larger than this that need to be server-side copied will be copied in chunks of this size.
            
+
            The minimum is 0 and the maximum is 5 GiB.
            
+
            Properties:
            
+
              
+
            • Config: copy_cutoff
              • 
+
            • Env Var: RCLONE_OOS_COPY_CUTOFF
              • 
+
            • Type: SizeSuffix
              • 
+
            • Default: 4.656Gi
              • 
+
            
+
            --oos-copy-timeout
            
+
            Timeout for copy.
            
+
            Copy is an asynchronous operation, specify timeout to wait for copy to succeed
            
+
            Properties:
            
+
              
+
            • Config: copy_timeout
              • 
+
            • Env Var: RCLONE_OOS_COPY_TIMEOUT
              • 
+
            • Type: Duration
              • 
+
            • Default: 1m0s
              • 
+
            
+
            --oos-disable-checksum
            
+
            Don't store MD5 checksum with object metadata.
            
+
            Normally rclone will calculate the MD5 checksum of the input before uploading it so it can add it to metadata on the object. This is great for data integrity checking but can cause long delays for large files to start uploading.
            
+
            Properties:
            
+
              
+
            • Config: disable_checksum
              • 
+
            • Env Var: RCLONE_OOS_DISABLE_CHECKSUM
              • 
+
            • Type: bool
              • 
+
            • Default: false
              • 
+
            
+
            --oos-encoding
            
+
            The encoding for the backend.
            
+
            See the encoding section in the overview for more info.
            
+
            Properties:
            
+
              
+
            • Config: encoding
              • 
+
            • Env Var: RCLONE_OOS_ENCODING
              • 
+
            • Type: MultiEncoder
              • 
+
            • Default: Slash,InvalidUtf8,Dot
              • 
+
            
+
            --oos-leave-parts-on-error
            
+
            If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery.
            
+
            It should be set to true for resuming uploads across different sessions.
            
+
            WARNING: Storing parts of an incomplete multipart upload counts towards space usage on object storage and will add additional costs if not cleaned up.
            
+
            Properties:
            
+
              
+
            • Config: leave_parts_on_error
              • 
+
            • Env Var: RCLONE_OOS_LEAVE_PARTS_ON_ERROR
              • 
+
            • Type: bool
              • 
+
            • Default: false
              • 
+
            
+
            --oos-no-check-bucket
            
+
            If set, don't attempt to check the bucket exists or create it.
            
+
            This can be useful when trying to minimise the number of transactions rclone does if you know the bucket exists already.
            
+
            It can also be needed if the user you are using does not have bucket creation permissions.
            
+
            Properties:
            
+
              
+
            • Config: no_check_bucket
              • 
+
            • Env Var: RCLONE_OOS_NO_CHECK_BUCKET
              • 
+
            • Type: bool
              • 
+
            • Default: false
              • 
+
            
+
            Backend commands
            
+
            Here are the commands specific to the oracleobjectstorage backend.
            
+
            Run them with
            
+
            rclone backend COMMAND remote:
            
+
            The help below will explain what arguments each command takes.
            
+
            See the backend command for more info on how to pass options and arguments.
            
+
            These can be run on a running backend using the rc command backend/command.
            
+
            rename
            
+
            change the name of an object
            
+
            rclone backend rename remote: [options] [<arguments>+]
            
+
            This command can be used to rename a object.
            
+
            Usage Examples:
            
+
            rclone backend rename oos:bucket relative-object-path-under-bucket object-new-name
            
+
            list-multipart-uploads
            
+
            List the unfinished multipart uploads
            
+
            rclone backend list-multipart-uploads remote: [options] [<arguments>+]
            
+
            This command lists the unfinished multipart uploads in JSON format.
            
+
            rclone backend list-multipart-uploads oos:bucket/path/to/object
            
+
            It returns a dictionary of buckets with values as lists of unfinished multipart uploads.
            
+
            You can call it with no bucket in which case it lists all bucket, with a bucket or with a bucket and path.
            
+
            {
+  "test-bucket": [
+            {
+                    "namespace": "test-namespace",
+                    "bucket": "test-bucket",
+                    "object": "600m.bin",
+                    "uploadId": "51dd8114-52a4-b2f2-c42f-5291f05eb3c8",
+                    "timeCreated": "2022-07-29T06:21:16.595Z",
+                    "storageTier": "Standard"
+            }
+    ]
            
+
            cleanup
            
+
            Remove unfinished multipart uploads.
            
+
            rclone backend cleanup remote: [options] [<arguments>+]
            
+
            This command removes unfinished multipart uploads of age greater than max-age which defaults to 24 hours.
            
+
            Note that you can use -i/--dry-run with this command to see what it would do.
            
+
            rclone backend cleanup oos:bucket/path/to/object
+rclone backend cleanup -o max-age=7w oos:bucket/path/to/object
            
+
            Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
            
+
            Options:
            
+
              
+
            • "max-age": Max age of upload to delete
              • 
+
            
 
            QingStor
            
 
            Paths are specified as remote:bucket (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:bucket/path/to/dir.
            
 
            Configuration
            
@@ -23675,9 +24700,9 @@ 
            Configuration
            
 
            rclone ls remote:bucket
            
 
            Sync /home/local/directory to the remote bucket, deleting any excess files in the bucket.
            
 
            rclone sync -i /home/local/directory remote:bucket
            
-
            --fast-list
            
+
            --fast-list
            
 
            This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.
            
-
            Multipart uploads
            
+
            Multipart uploads
            
 
            rclone supports multipart uploads with QingStor which means that it can upload files bigger than 5 GiB. Note that files uploaded with multipart upload don't have an MD5SUM.
            
 
            Note that incomplete multipart uploads older than 24 hours can be removed with rclone cleanup remote:bucket just for one bucket rclone cleanup remote: for all buckets. QingStor does not ever remove incomplete multipart uploads so it may be necessary to run this from time to time.
            
 
            Buckets and Zone
            
@@ -23838,7 +24863,7 @@ 
            --qingstor-encoding
            
 
          14. Type: MultiEncoder
            15. 
 
          15. Default: Slash,Ctl,InvalidUtf8
            16. 
 
-
            Limitations
            
+
            Limitations
            
 
            rclone about is not supported by the qingstor backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.
            
 
            See List of backends that do not support rclone about and rclone about
            
 
            Sia
            
@@ -23953,7 +24978,7 @@ 
            --sia-encoding
            
 
          16. Type: MultiEncoder
            17. 
 
          17. Default: Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot
            18. 
 
-
            Limitations
            
+
            Limitations
            
 
 
            Paths are specified as remote:container (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:container/path/to/dir.
            
@@ -24103,7 +25128,7 @@ 
            Using rclone without a config file
      
-
      --fast-list
      
+
      --fast-list
      
 
      This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.
      
 
      --update and --use-server-modtime
      
 
      As noted below, the modified time is stored on metadata on the object. It is used by default for all operations that require checking the time a file was last updated. It allows rclone to treat the remote more like a true filesystem, but it is inefficient because it requires an extra API call to retrieve the metadata.
      
@@ -24402,6 +25427,19 @@ 
      --swift-no-chunk
      
 
    56. Type: bool
      57. 
 
    57. Default: false
      58. 
 
+
      --swift-no-large-objects
      
+
      Disable support for static and dynamic large objects
      
+
      Swift cannot transparently store files bigger than 5 GiB. There are two schemes for doing that, static or dynamic large objects, and the API does not allow rclone to determine whether a file is a static or dynamic large object without doing a HEAD on the object. Since these need to be treated differently, this means rclone has to issue HEAD requests for objects for example when reading checksums.
      
+
      When no_large_objects is set, rclone will assume that there are no static or dynamic large objects stored. This means it can stop doing the extra HEAD calls which in turn increases performance greatly especially when doing a swift to swift transfer with --checksum set.
      
+
      Setting this option implies no_chunk and also that no files will be uploaded in chunks, so files bigger than 5 GiB will just fail on upload.
      
+
      If you set this option and there are static or dynamic large objects, then this will give incorrect hashes for them. Downloads will succeed, but other operations such as Remove and Copy will fail.
      
+
      Properties:
      
+
        
+
      • Config: no_large_objects
        • 
+
      • Env Var: RCLONE_SWIFT_NO_LARGE_OBJECTS
        • 
+
      • Type: bool
        • 
+
      • Default: false
        • 
+
      
 
      --swift-encoding
      
 
      The encoding for the backend.
      
 
      See the encoding section in the overview for more info.
      
@@ -24412,7 +25450,7 @@ 
      --swift-encoding
      
 
    58. Type: MultiEncoder
      59. 
 
    59. Default: Slash,InvalidUtf8
      60. 
 
-
      Limitations
      
+
      Limitations
      
 
      The Swift API doesn't return a correct MD5SUM for segmented files (Dynamic or Static Large Objects) so rclone won't check or use the MD5SUM for these.
      
 
      Troubleshooting
      
 
      Rclone gives Failed to create file system for "remote:": Bad Request
      
@@ -24732,7 +25770,7 @@ 
      --premiumizeme-encoding
      
 
    60. Type: MultiEncoder
      61. 
 
    61. Default: Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot
      62. 
 
-
      Limitations
      
+
      Limitations
      
 
      Note that premiumize.me is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".
      
 
      premiumize.me file names can't have the \ or " characters in. rclone maps these to and from an identical looking unicode equivalents ＼ and ＂
      
 
      premiumize.me only supports filenames up to 255 characters in length.
      
@@ -24833,7 +25871,7 @@ 
      --putio-encoding
      
 
    62. Type: MultiEncoder
      63. 
 
    63. Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot
      64. 
 
-
      Limitations
      
+
      Limitations
      
 
      put.io has rate limiting. When you hit a limit, rclone automatically retries after waiting the amount of time requested by the server.
      
 
      If you want to avoid ever hitting these limits, you may use the --tpslimit flag with a low number. Note that the imposed limits may be different for different operations, and may change over time.
      
 
      Seafile
      
@@ -24995,7 +26033,7 @@ 
      Configuration in library mode
      
 
      rclone ls seafile:directory
      
 
      Sync /home/local/directory to the remote library, deleting any excess files in the library.
      
 
      rclone sync -i /home/local/directory seafile:
      
-
      --fast-list
      
+
      --fast-list
      
 
      Seafile version 7+ supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details. Please note this is not supported on seafile server version 6.x
      
 
      Restricted filename characters
      
 
      In addition to the default restricted characters set the following characters are also replaced:
      
@@ -25143,7 +26181,7 @@ 
      SFTP
      
 
    64. rsync.net
      65. 
 
 
      SFTP runs over SSH v2 and is installed as standard with most modern SSH installations.
      
-
      Paths are specified as remote:path. If the path does not begin with a / it is relative to the home directory of the user. An empty path remote: refers to the user's home directory. For example, rclone lsd remote: would list the home directory of the user cofigured in the rclone remote config (i.e /home/sftpuser). However, rclone lsd remote:/ would list the root directory for remote machine (i.e. /)
      
+
      Paths are specified as remote:path. If the path does not begin with a / it is relative to the home directory of the user. An empty path remote: refers to the user's home directory. For example, rclone lsd remote: would list the home directory of the user configured in the rclone remote config (i.e /home/sftpuser). However, rclone lsd remote:/ would list the root directory for remote machine (i.e. /)
      
 
      Note that some SFTP servers will need the leading / - Synology is a good example of this. rsync.net and Hetzner, on the other hand, requires users to OMIT the leading /.
      
 
      Note that by default rclone will try to execute shell commands on the server, see shell access considerations.
      
 
      Configuration
      
@@ -25269,12 +26307,12 @@ 
      ssh-agent on macOS
      
 
      These commands can be used in scripts of course.
      
 
      Shell access
      
 
      Some functionality of the SFTP backend relies on remote shell access, and the possibility to execute commands. This includes checksum, and in some cases also about. The shell commands that must be executed may be different on different type of shells, and also quoting/escaping of file path arguments containing special characters may be different. Rclone therefore needs to know what type of shell it is, and if shell access is available at all.
      
-
      Most servers run on some version of Unix, and then a basic Unix shell can be assumed, without further distinction. Windows 10, Server 2019, and later can also run a SSH server, which is a port of OpenSSH (see official installation guide). On a Windows server the shell handling is different: Although it can also be set up to use a Unix type shell, e.g. Cygwin bash, the default is to use Windows Command Prompt (cmd.exe), and PowerShell is a recommended alternative. All of these have bahave differently, which rclone must handle.
      
+
      Most servers run on some version of Unix, and then a basic Unix shell can be assumed, without further distinction. Windows 10, Server 2019, and later can also run a SSH server, which is a port of OpenSSH (see official installation guide). On a Windows server the shell handling is different: Although it can also be set up to use a Unix type shell, e.g. Cygwin bash, the default is to use Windows Command Prompt (cmd.exe), and PowerShell is a recommended alternative. All of these have behave differently, which rclone must handle.
      
 
      Rclone tries to auto-detect what type of shell is used on the server, first time you access the SFTP remote. If a remote shell session is successfully created, it will look for indications that it is CMD or PowerShell, with fall-back to Unix if not something else is detected. If unable to even create a remote shell session, then shell command execution will be disabled entirely. The result is stored in the SFTP remote configuration, in option shell_type, so that the auto-detection only have to be performed once. If you manually set a value for this option before first run, the auto-detection will be skipped, and if you set a different value later this will override any existing. Value none can be set to avoid any attempts at executing shell commands, e.g. if this is not allowed on the server.
      
 
      When the server is rclone serve sftp, the rclone SFTP remote will detect this as a Unix type shell - even if it is running on Windows. This server does not actually have a shell, but it accepts input commands matching the specific ones that the SFTP backend relies on for Unix shells, e.g. md5sum and df. Also it handles the string escape rules used for Unix shell. Treating it as a Unix type shell from a SFTP remote will therefore always be correct, and support all features.
      
 
      Shell access considerations
      
-
      The shell type auto-detection logic, described above, means that by default rclone will try to run a shell command the first time a new sftp remote is accessed. If you configure a sftp remote without a config file, e.g. an on the fly remote, rclone will have nowhere to store the result, and it will re-run the command on every access. To avoid this you should explicitely set the shell_type option to the correct value, or to none if you want to prevent rclone from executing any remote shell commands.
      
-
      It is also important to note that, since the shell type decides how quoting and escaping of file paths used as command-line arguments are performed, configuring the wrong shell type may leave you exposed to command injection exploits. Make sure to confirm the auto-detected shell type, or explicitely set the shell type you know is correct, or disable shell access until you know.
      
+
      The shell type auto-detection logic, described above, means that by default rclone will try to run a shell command the first time a new sftp remote is accessed. If you configure a sftp remote without a config file, e.g. an on the fly remote, rclone will have nowhere to store the result, and it will re-run the command on every access. To avoid this you should explicitly set the shell_type option to the correct value, or to none if you want to prevent rclone from executing any remote shell commands.
      
+
      It is also important to note that, since the shell type decides how quoting and escaping of file paths used as command-line arguments are performed, configuring the wrong shell type may leave you exposed to command injection exploits. Make sure to confirm the auto-detected shell type, or explicitly set the shell type you know is correct, or disable shell access until you know.
      
 
      Checksum
      
 
      SFTP does not natively support checksums (file hash), but rclone is able to use checksumming if the same login has shell access, and can execute remote commands. If there is a command that can calculate compatible checksums on the remote system, Rclone can then be configured to execute this whenever a checksum is needed, and read back the results. Currently MD5 and SHA-1 are supported.
      
 
      Normally this requires an external utility being available on the server. By default rclone will try commands md5sum, md5 and rclone md5sum for MD5 checksums, and the first one found usable will be picked. Same with sha1sum, sha1 and rclone sha1sum commands for SHA-1 checksums. These utilities normally need to be in the remote's PATH to be found.
      
@@ -25601,10 +26639,8 @@ 
      --sftp-idle-timeout
      
 
 
      --sftp-chunk-size
      
 
      Upload and download chunk size.
      
-
      This controls the maximum packet size used in the SFTP protocol. The RFC limits this to 32768 bytes (32k), however a lot of servers support larger sizes and setting it larger will increase transfer speed dramatically on high latency links.
      
-
      Only use a setting higher than 32k if you always connect to the same server or after sufficiently broad testing.
      
-
      For example using the value of 252k with OpenSSH works well with its maximum packet size of 256k.
      
-
      If you get the error "failed to send packet header: EOF" when copying a large file, try lowering this number.
      
+
      This controls the maximum size of payload in SFTP protocol packets. The RFC limits this to 32768 bytes (32k), which is the default. However, a lot of servers support larger sizes, typically limited to a maximum total package size of 256k, and setting it larger will increase transfer speed dramatically on high latency links. This includes OpenSSH, and, for example, using the value of 255k works well, leaving plenty of room for overhead while still being within a total packet size of 256k.
      
+
      Make sure to test thoroughly before using a value higher than 32k, and only use it if you always connect to the same server or after sufficiently broad testing. If you get errors such as "failed to send packet payload: EOF", lots of "connection lost", or "corrupted on transfer", when copying a larger file, try lowering the value. The server run by rclone serve sftp sends packets with standard 32k maximum payload so you must not set a different chunk_size when downloading files, but it accepts packets up to the 256k total size, so for uploads the chunk_size can be set as for the OpenSSH example above.
      
 
      Properties:
      
 
        
 
      • Config: chunk_size
        • 
@@ -25638,7 +26674,7 @@ 
        --sftp-set-env
        
 
      • Type: SpaceSepList
        • 
 
      • Default:
        • 
 
      
-
      Limitations
      
+
      Limitations
      
 
      On some SFTP servers (e.g. Synology) the paths are different for SSH and SFTP so the hashes can't be calculated properly. For them using disable_hashcheck is a good idea.
      
 
      The only ssh agent supported under Windows is Putty's pageant.
      
 
      The Go SSH library disables the use of the aes128-cbc cipher by default, due to security concerns. This can be re-enabled on a per-connection basis by setting the use_insecure_cipher setting in the configuration file to true. Further details on the insecurity of this cipher can be found in this paper.
      
@@ -25651,6 +26687,180 @@ 
      rsync.net
      
 
      Hetzner Storage Box
      
 
      Hetzner Storage Boxes are supported through the SFTP backend on port 23.
      
 
      See Hetzner's documentation for details
      
+
      SMB
      
+
      SMB is a communication protocol to share files over network.
      
+
      This relies on go-smb2 library for communication with SMB protocol.
      
+
      Paths are specified as remote:sharename (or remote: for the lsd command.) You may put subdirectories in too, e.g. remote:item/path/to/dir.
      
+
      Notes
      
+
      The first path segment must be the name of the share, which you entered when you started to share on Windows. On smbd, it's the section title in smb.conf (usually in /etc/samba/) file. You can find shares by quering the root if you're unsure (e.g. rclone lsd remote:).
      
+
      You can't access to the shared printers from rclone, obviously.
      
+
      You can't use Anonymous access for logging in. You have to use the guest user with an empty password instead. The rclone client tries to avoid 8.3 names when uploading files by encoding trailing spaces and periods. Alternatively, the local backend on Windows can access SMB servers using UNC paths, by \\server\share. This doesn't apply to non-Windows OSes, such as Linux and macOS.
      
+
      Configuration
      
+
      Here is an example of making a SMB configuration.
      
+
      First run
      
+
      rclone config
      
+
      This will guide you through an interactive setup process.
      
+
      No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+XX / SMB / CIFS
+   \ (smb)
+Storage> smb
+
+Option host.
+Samba hostname to connect to.
+E.g. "example.com".
+Enter a value.
+host> localhost
+
+Option user.
+Samba username.
+Enter a string value. Press Enter for the default (lesmi).
+user> guest
+
+Option port.
+Samba port number.
+Enter a signed integer. Press Enter for the default (445).
+port> 
+
+Option pass.
+Samba password.
+Choose an alternative below. Press Enter for the default (n).
+y) Yes, type in my own password
+g) Generate random password
+n) No, leave this optional password blank (default)
+y/g/n> g
+Password strength in bits.
+64 is just about memorable
+128 is secure
+1024 is the maximum
+Bits> 64
+Your password is: XXXX
+Use this password? Please note that an obscured version of this 
+password (and not the password itself) will be stored under your 
+configuration file, so keep this generated password in a safe place.
+y) Yes (default)
+n) No
+y/n> y
+
+Option domain.
+Domain name for NTLM authentication.
+Enter a string value. Press Enter for the default (WORKGROUP).
+domain> 
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: samba
+- host: localhost
+- user: guest
+- pass: *** ENCRYPTED ***
+Keep this "remote" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> d
      
+
      Standard options
      
+
      Here are the Standard options specific to smb (SMB / CIFS).
      
+
      --smb-host
      
+
      SMB server hostname to connect to.
      
+
      E.g. "example.com".
      
+
      Properties:
      
+
        
+
      • Config: host
        • 
+
      • Env Var: RCLONE_SMB_HOST
        • 
+
      • Type: string
        • 
+
      • Required: true
        • 
+
      
+
      --smb-user
      
+
      SMB username.
      
+
      Properties:
      
+
        
+
      • Config: user
        • 
+
      • Env Var: RCLONE_SMB_USER
        • 
+
      • Type: string
        • 
+
      • Default: "$USER"
        • 
+
      
+
      --smb-port
      
+
      SMB port number.
      
+
      Properties:
      
+
        
+
      • Config: port
        • 
+
      • Env Var: RCLONE_SMB_PORT
        • 
+
      • Type: int
        • 
+
      • Default: 445
        • 
+
      
+
      --smb-pass
      
+
      SMB password.
      
+
      NB Input to this must be obscured - see rclone obscure.
      
+
      Properties:
      
+
        
+
      • Config: pass
        • 
+
      • Env Var: RCLONE_SMB_PASS
        • 
+
      • Type: string
        • 
+
      • Required: false
        • 
+
      
+
      --smb-domain
      
+
      Domain name for NTLM authentication.
      
+
      Properties:
      
+
        
+
      • Config: domain
        • 
+
      • Env Var: RCLONE_SMB_DOMAIN
        • 
+
      • Type: string
        • 
+
      • Default: "WORKGROUP"
        • 
+
      
+
      Advanced options
      
+
      Here are the Advanced options specific to smb (SMB / CIFS).
      
+
      --smb-idle-timeout
      
+
      Max time before closing idle connections.
      
+
      If no connections have been returned to the connection pool in the time given, rclone will empty the connection pool.
      
+
      Set to 0 to keep connections indefinitely.
      
+
      Properties:
      
+
        
+
      • Config: idle_timeout
        • 
+
      • Env Var: RCLONE_SMB_IDLE_TIMEOUT
        • 
+
      • Type: Duration
        • 
+
      • Default: 1m0s
        • 
+
      
+
      --smb-hide-special-share
      
+
      Hide special shares (e.g. print$) which users aren't supposed to access.
      
+
      Properties:
      
+
        
+
      • Config: hide_special_share
        • 
+
      • Env Var: RCLONE_SMB_HIDE_SPECIAL_SHARE
        • 
+
      • Type: bool
        • 
+
      • Default: true
        • 
+
      
+
      --smb-case-insensitive
      
+
      Whether the server is configured to be case-insensitive.
      
+
      Always true on Windows shares.
      
+
      Properties:
      
+
        
+
      • Config: case_insensitive
        • 
+
      • Env Var: RCLONE_SMB_CASE_INSENSITIVE
        • 
+
      • Type: bool
        • 
+
      • Default: true
        • 
+
      
+
      --smb-encoding
      
+
      The encoding for the backend.
      
+
      See the encoding section in the overview for more info.
      
+
      Properties:
      
+
        
+
      • Config: encoding
        • 
+
      • Env Var: RCLONE_SMB_ENCODING
        • 
+
      • Type: MultiEncoder
        • 
+
      • Default: Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot
        • 
+
      
 
      Storj
      
 
      Storj is an encrypted, secure, and cost-effective object storage service that enables you to store, back up, and archive large amounts of data in a decentralized manner.
      
 
      Backend options
      
@@ -25710,7 +26920,7 @@ 
      Backend options
      
 
    65. S3 backend: secret encryption key is shared with the gateway
      66. 
 
 
-
      Configuration
      
+
      Configuration
      
 
      To make a new Storj configuration you need one of the following: * Access Grant that someone else shared with you. * API Key of a Storj project you are a member of.
      
 
      Here is an example of how to make a remote called remote. First run:
      
 
       rclone config
      
@@ -25807,7 +27017,7 @@ 
      Setup with API key and passphrase
      
-
      Standard options
      
+
      Standard options
      
 
      Here are the Standard options specific to storj (Storj Decentralized Cloud Storage).
      
 
      --storj-provider
      
 
      Choose an authentication method.
      
@@ -25940,7 +27150,7 @@ 
      Sync two Locations
      
 
      rclone sync -i --progress remote-us:bucket/path/to/dir/ remote-europe:bucket/path/to/dir/
      
 
      Or even between another cloud storage and Storj.
      
 
      rclone sync -i --progress s3:bucket/path/to/dir/ storj:bucket/path/to/dir/
      
-
      Limitations
      
+
      Limitations
      
 
      rclone about is not supported by the rclone Storj backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.
      
 
      See List of backends that do not support rclone about and rclone about
      
 
      Known issues
      
@@ -25948,7 +27158,7 @@ 
      Known issues
      
 
      To fix these, please raise your system limits. You can do this issuing a ulimit -n 65536 just before you run rclone. To change the limits more permanently you can add this to your shell startup script, e.g. $HOME/.bashrc, or change the system-wide configuration, usually /etc/sysctl.conf and/or /etc/security/limits.conf, but please refer to your operating system manual.
      
 
      SugarSync
      
 
      SugarSync is a cloud service that enables active synchronization of files across computers and other devices for file backup, access, syncing, and sharing.
      
-
      Configuration
      
+
      Configuration
      
 
      The initial setup for SugarSync involves getting a token from SugarSync which you can do with rclone. rclone config walks you through it.
      
 
      Here is an example of how to make a remote called remote. First run:
      
 
       rclone config
      
@@ -26021,7 +27231,7 @@ 
      Restricted filename characters
      
 
      Deleting files
      
 
      Deleted files will be moved to the "Deleted items" folder by default.
      
 
      However you can supply the flag --sugarsync-hard-delete or set the config parameter hard_delete = true if you would like files to be deleted straight away.
      
-
      Standard options
      
+
      Standard options
      
 
      Here are the Standard options specific to sugarsync (Sugarsync).
      
 
      --sugarsync-app-id
      
 
      Sugarsync App ID.
      
@@ -26062,7 +27272,7 @@ 
      --sugarsync-hard-delete
      
 
    66. Type: bool
      67. 
 
    67. Default: false
      68. 
 
-
      Advanced options
      
+
      Advanced options
      
 
      Here are the Advanced options specific to sugarsync (Sugarsync).
      
 
      --sugarsync-refresh-token
      
 
      Sugarsync refresh token.
      
@@ -26134,7 +27344,7 @@ 
      --sugarsync-encoding
      
 
    68. Type: MultiEncoder
      69. 
 
    69. Default: Slash,Ctl,InvalidUtf8,Dot
      70. 
 
-
      Limitations
      
+
      Limitations
      
 
      rclone about is not supported by the SugarSync backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.
      
 
      See List of backends that do not support rclone about and rclone about
      
 
      Tardigrade
      
@@ -26143,7 +27353,7 @@ 
      Uptobox
      
 
      This is a Backend for Uptobox file storage service. Uptobox is closer to a one-click hoster than a traditional cloud storage provider and therefore not suitable for long term storage.
      
 
      Paths are specified as remote:path
      
 
      Paths may be as deep as required, e.g. remote:directory/subdirectory.
      
-
      Configuration
      
+
      Configuration
      
 
      To configure an Uptobox backend you'll need your personal api token. You'll find it in your account settings
      
 
      Here is an example of how to make a remote called remote with the default setup. First run:
      
 
      rclone config
      
@@ -26223,7 +27433,7 @@ 
      Restricted filename characters
      
 
 
 
      Invalid UTF-8 bytes will also be replaced, as they can't be used in XML strings.
      
-
      Standard options
      
+
      Standard options
      
 
      Here are the Standard options specific to uptobox (Uptobox).
      
 
      --uptobox-access-token
      
 
      Your access token.
      
@@ -26235,7 +27445,7 @@ 
      --uptobox-access-token
      
 
    70. Type: string
      71. 
 
    71. Required: false
      72. 
 
-
      Advanced options
      
+
      Advanced options
      
 
      Here are the Advanced options specific to uptobox (Uptobox).
      
 
      --uptobox-encoding
      
 
      The encoding for the backend.
      
@@ -26247,7 +27457,7 @@ 
      --uptobox-encoding
      
 
    72. Type: MultiEncoder
      73. 
 
    73. Default: Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot
      74. 
 
-
      Limitations
      
+
      Limitations
      
 
      Uptobox will delete inactive files that have not been accessed in 60 days.
      
 
      rclone about is not supported by this backend an overview of used space can however been seen in the uptobox web interface.
      
 
      Union
      
@@ -26257,7 +27467,7 @@ 
      Union
      
 
      Attribute :ro and :nc can be attach to the end of path to tag the remote as read only or no create, e.g. remote:directory/subdirectory:ro or remote:directory/subdirectory:nc.
      
 
      Subfolders can be used in upstream remotes. Assume a union remote named backup with the remotes mydrive:private/backup. Invoking rclone mkdir backup:desktop is exactly the same as invoking rclone mkdir mydrive:private/backup/desktop.
      
 
      There will be no special handling of paths containing .. segments. Invoking rclone mkdir backup:../desktop is exactly the same as invoking rclone mkdir mydrive:private/backup/../desktop.
      
-
      Configuration
      
+
      Configuration
      
 
      Here is an example of how to make a union called remote for local folders. First run:
      
 
       rclone config
      
 
      This will guide you through an interactive setup process:
      
@@ -26478,7 +27688,7 @@ 
      Policy descriptions
      
 
 
 
-
      Standard options
      
+
      Standard options
      
 
      Here are the Standard options specific to union (Union merges the contents of several upstream fs).
      
 
      --union-upstreams
      
 
      List of space separated upstreams.
      
@@ -26527,7 +27737,7 @@ 
      --union-cache-time
      
 
    74. Type: int
      75. 
 
    75. Default: 120
      76. 
 
-
      Advanced options
      
+
      Advanced options
      
 
      Here are the Advanced options specific to union (Union merges the contents of several upstream fs).
      
 
      --union-min-free-space
      
 
      Minimum viable free space for lfs/eplfs policies.
      
@@ -26545,7 +27755,7 @@ 
      Metadata
      
 
      WebDAV
      
 
      Paths are specified as remote:path
      
 
      Paths may be as deep as required, e.g. remote:directory/subdirectory.
      
-
      Configuration
      
+
      Configuration
      
 
      To configure the WebDAV remote you will need to have a URL for it, and a username and password. If you know what kind of system you are connecting to then rclone can enable extra features.
      
 
      Here is an example of how to make a remote called remote. First run:
      
 
       rclone config
      
@@ -26618,7 +27828,7 @@ 
      Configuration
      
 
      Modified time and hashes
      
 
      Plain WebDAV does not support modified times. However when used with Owncloud or Nextcloud rclone will support modified times.
      
 
      Likewise plain WebDAV does not support hashes, however when used with Owncloud or Nextcloud rclone will support SHA1 and MD5 hashes. Depending on the exact version of Owncloud or Nextcloud hashes may appear on all objects, or only on objects which had a hash uploaded with them.
      
-
      Standard options
      
+
      Standard options
      
 
      Here are the Standard options specific to webdav (WebDAV).
      
 
      --webdav-url
      
 
      URL of http host to connect to.
      
@@ -26691,7 +27901,7 @@ 
      --webdav-bearer-token
      
 
    76. Type: string
      77. 
 
    77. Required: false
      78. 
 
-
      Advanced options
      
+
      Advanced options
      
 
      Here are the Advanced options specific to webdav (WebDAV).
      
 
      --webdav-bearer-token-command
      
 
      Command to run to get a bearer token.
      
@@ -26798,7 +28008,7 @@ 
      OpenID-Connect
      
 bearer_token_command = oidc-token XDC
      
 
      Yandex Disk
      
 
      Yandex Disk is a cloud storage solution created by Yandex.
      
-
      Configuration
      
+
      Configuration
      
 
      Here is an example of making a yandex configuration. First run
      
 
      rclone config
      
 
      This will guide you through an interactive setup process:
      
@@ -26862,7 +28072,7 @@ 
      Quota information
      
 
      Restricted filename characters
      
 
      The default restricted characters set are replaced.
      
 
      Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.
      
-
      Standard options
      
+
      Standard options
      
 
      Here are the Standard options specific to yandex (Yandex Disk).
      
 
      --yandex-client-id
      
 
      OAuth Client Id.
      
@@ -26884,7 +28094,7 @@ 
      --yandex-client-secret
      
 
    78. Type: string
      79. 
 
    79. Required: false
      80. 
 
-
      Advanced options
      
+
      Advanced options
      
 
      Here are the Advanced options specific to yandex (Yandex Disk).
      
 
      --yandex-token
      
 
      OAuth Access Token as a JSON blob.
      
@@ -26934,13 +28144,13 @@ 
      --yandex-encoding
      
 
    80. Type: MultiEncoder
      81. 
 
    81. Default: Slash,Del,Ctl,InvalidUtf8,Dot
      82. 
 
-
      Limitations
      
+
      Limitations
      
 
      When uploading very large files (bigger than about 5 GiB) you will need to increase the --timeout parameter. This is because Yandex pauses (perhaps to calculate the MD5SUM for the entire file) before returning confirmation that the file has been uploaded. The default handling of timeouts in rclone is to assume a 5 minute pause is an error and close the connection - you'll see net/http: timeout awaiting response headers errors in the logs if this is happening. Setting the timeout to twice the max size of file in GiB should be enough, so if you want to upload a 30 GiB file set a timeout of 2 * 30 = 60m, that is --timeout 60m.
      
 
      Having a Yandex Mail account is mandatory to use the Yandex.Disk subscription. Token generation will work without a mail account, but Rclone won't be able to complete any actions.
      
 
      [403 - DiskUnsupportedUserAccountTypeError] User account type is not supported.
      
 
      Zoho Workdrive
      
 
      Zoho WorkDrive is a cloud storage solution created by Zoho.
      
-
      Configuration
      
+
      Configuration
      
 
      Here is an example of making a zoho configuration. First run
      
 
      rclone config
      
 
      This will guide you through an interactive setup process:
      
@@ -27020,7 +28230,7 @@ 
      Usage information
      
 
      To view your current quota you can use the rclone about remote: command which will display your current usage.
      
 
      Restricted filename characters
      
 
      Only control characters and invalid UTF-8 are replaced. In addition most Unicode full-width characters are not supported at all and will be removed from filenames during upload.
      
-
      Standard options
      
+
      Standard options
      
 
      Here are the Standard options specific to zoho (Zoho).
      
 
      --zoho-client-id
      
 
      OAuth Client Id.
      
@@ -27079,7 +28289,7 @@ 
      --zoho-region
      
 
 
 
-
      Advanced options
      
+
      Advanced options
      
 
      Here are the Advanced options specific to zoho (Zoho).
      
 
      --zoho-token
      
 
      OAuth Access Token as a JSON blob.
      
@@ -27132,7 +28342,7 @@ 
      Local Filesystem
      
 
      Local paths are specified as normal filesystem paths, e.g. /path/to/wherever, so
      
 
      rclone sync -i /home/source /tmp/destination
      
 
      Will sync /home/source to /tmp/destination.
      
-
      Configuration
      
+
      Configuration
      
 
      For consistencies sake one can also configure a remote of type local in the config file, and access the local filesystem using rclone remote paths, e.g. remote:path/to/wherever, but it is probably easier not to.
      
 
      Modified time
      
 
      Rclone reads and writes the modified time using an accuracy determined by the OS. Typically this is 1ns on Linux, 10 ns on Windows and 1 Second on OS X.
      
@@ -27503,7 +28713,7 @@ 
      Restricting filesystems
         0 file2
      
 
      NB Rclone (like most unix tools such as du, rsync and tar) treats a bind mount to the same device as being on the same filesystem.
      
 
      NB This flag is only available on Unix based systems. On systems where it isn't supported (e.g. Windows) it will be ignored.
      
-
      Advanced options
      
+
      Advanced options
      
 
      Here are the Advanced options specific to local (Local Disk).
      
 
      --local-nounc
      
 
      Disable UNC (long path names) conversion on Windows.
      
@@ -27740,7 +28950,7 @@ 
      Metadata
      
 
 
 
      See the metadata docs for more info.
      
-
      Backend commands
      
+
      Backend commands
      
 
      Here are the commands specific to the local backend.
      
 
      Run them with
      
 
      rclone backend COMMAND remote:
      
@@ -27757,6 +28967,201 @@ 
      noop
      
 
    82. "error": return an error based on option value
      83. 
 
 
      Changelog
      
+
      v1.60.0 - 2022-10-21
      
+
      See commits
      
+
        
+
      • New backends
+
        • 
+
      • New Features
+
          
+
        • build
+
            
+
          • Update to go1.19 and make go1.17 the minimum required version (Nick Craig-Wood)
            • 
+
          • Install.sh: fix arm-v7 download (Ole Frost)
            • 
+
          • 
+
        • fs: Warn the user when using an existing remote name without a colon (Nick Craig-Wood)
          • 
+
        • httplib: Add --xxx-min-tls-version option to select minimum TLS version for HTTP servers (Robert Newson)
          • 
+
        • librclone: Add PHP bindings and test program (Jordi Gonzalez Muñoz)
          • 
+
        • operations
+
            
+
          • Add --server-side-across-configs global flag for any backend (Nick Craig-Wood)
            • 
+
          • Optimise --copy-dest and --compare-dest (Nick Craig-Wood)
            • 
+
          • 
+
        • rc: add job/stopgroup to stop group (Evan Spensley)
          • 
+
        • serve dlna
+
            
+
          • Add --announce-interval to control SSDP Announce Interval (YanceyChiew)
            • 
+
          • Add --interface to Specify SSDP interface names line (Simon Bos)
            • 
+
          • Add support for more external subtitles (YanceyChiew)
            • 
+
          • Add verification of addresses (YanceyChiew)
            • 
+
          • 
+
        • sync: Optimise --copy-dest and --compare-dest (Nick Craig-Wood)
          • 
+
        • doc updates (albertony, Alexander Knorr, anonion, João Henrique Franco, Josh Soref, Lorenzo Milesi, Marco Molteni, Mark Trolley, Ole Frost, partev, Ryan Morey, Tom Mombourquette, YFdyh000)
          • 
+
        • 
+
      • Bug Fixes
+
          
+
        • filter
+
            
+
          • Fix incorrect filtering with UseFilter context flag and wrapping backends (Nick Craig-Wood)
            • 
+
          • Make sure we check --files-from when looking for a single file (Nick Craig-Wood)
            • 
+
          • 
+
        • rc
+
            
+
          • Fix mount/listmounts not returning the full Fs entered in mount/mount (Tom Mombourquette)
            • 
+
          • Handle external unmount when mounting (Isaac Aymerich)
            • 
+
          • Validate Daemon option is not set when mounting a volume via RC (Isaac Aymerich)
            • 
+
          • 
+
        • sync: Update docs and error messages to reflect fixes to overlap checks (Nick Naumann)
          • 
+
        • 
+
      • VFS
+
          
+
        • Reduce memory use by embedding sync.Cond (Nick Craig-Wood)
          • 
+
        • Reduce memory usage by re-ordering commonly used structures (Nick Craig-Wood)
          • 
+
        • Fix excess CPU used by VFS cache cleaner looping (Nick Craig-Wood)
          • 
+
        • 
+
      • Local
+
          
+
        • Obey file filters in listing to fix errors on excluded files (Nick Craig-Wood)
          • 
+
        • Fix "Failed to read metadata: function not implemented" on old Linux kernels (Nick Craig-Wood)
          • 
+
        • 
+
      • Compress
+
          
+
        • Fix crash due to nil metadata (Nick Craig-Wood)
          • 
+
        • Fix error handling to not use or return nil objects (Nick Craig-Wood)
          • 
+
        • 
+
      • Drive
+
          
+
        • Make --drive-stop-on-upload-limit obey quota exceeded error (Steve Kowalik)
          • 
+
        • 
+
      • FTP
+
          
+
        • Add --ftp-force-list-hidden option to show hidden items (Øyvind Heddeland Instefjord)
          • 
+
        • Fix hang when using ExplicitTLS to certain servers. (Nick Craig-Wood)
          • 
+
        • 
+
      • Google Cloud Storage
+
          
+
        • Add --gcs-endpoint flag and config parameter (Nick Craig-Wood)
          • 
+
        • 
+
      • Hubic
+
          
+
        • Remove backend as service has now shut down (Nick Craig-Wood)
          • 
+
        • 
+
      • Onedrive
+
          
+
        • Rename Onedrive(cn) 21Vianet to Vnet Group (Yen Hu)
          • 
+
        • Disable change notify in China region since it is not supported (Nick Craig-Wood)
          • 
+
        • 
+
      • S3
+
          
+
        • Implement --s3-versions flag to show old versions of objects if enabled (Nick Craig-Wood)
          • 
+
        • Implement --s3-version-at flag to show versions of objects at a particular time (Nick Craig-Wood)
          • 
+
        • Implement backend versioning command to get/set bucket versioning (Nick Craig-Wood)
          • 
+
        • Implement Purge to purge versions and backend cleanup-hidden (Nick Craig-Wood)
          • 
+
        • Add --s3-decompress flag to decompress gzip-encoded files (Nick Craig-Wood)
          • 
+
        • Add --s3-sse-customer-key-base64 to supply keys with binary data (Richard Bateman)
          • 
+
        • Try to keep the maximum precision in ModTime with --user-server-modtime (Nick Craig-Wood)
          • 
+
        • Drop binary metadata with an ERROR message as it can't be stored (Nick Craig-Wood)
          • 
+
        • Add --s3-no-system-metadata to suppress read and write of system metadata (Nick Craig-Wood)
          • 
+
        • 
+
      • SFTP
+
          
+
        • Fix directory creation races (Lesmiscore)
          • 
+
        • 
+
      • Swift
+
          
+
        • Add --swift-no-large-objects to reduce HEAD requests (Nick Craig-Wood)
          • 
+
        • 
+
      • Union
+
          
+
        • Propagate SlowHash feature to fix hasher interaction (Lesmiscore)
          • 
+
        • 
+
      
+
      v1.59.2 - 2022-09-15
      
+
      See commits
      
+
        
+
      • Bug Fixes
+
          
+
        • config: Move locking to fix fatal error: concurrent map read and map write (Nick Craig-Wood)
          • 
+
        • 
+
      • Local
+
          
+
        • Disable xattr support if the filesystems indicates it is not supported (Nick Craig-Wood)
          • 
+
        • 
+
      • Azure Blob
+
          
+
        • Fix chunksize calculations producing too many parts (Nick Craig-Wood)
          • 
+
        • 
+
      • B2
+
          
+
        • Fix chunksize calculations producing too many parts (Nick Craig-Wood)
          • 
+
        • 
+
      • S3
+
          
+
        • Fix chunksize calculations producing too many parts (Nick Craig-Wood)
          • 
+
        • 
+
      
+
      v1.59.1 - 2022-08-08
      
+
      See commits
      
+
        
+
      • Bug Fixes
+
          
+
        • accounting: Fix panic in core/stats-reset with unknown group (Nick Craig-Wood)
          • 
+
        • build: Fix android build after GitHub actions change (Nick Craig-Wood)
          • 
+
        • dlna: Fix SOAP action header parsing (Joram Schrijver)
          • 
+
        • docs: Fix links to mount command from install docs (albertony)
          • 
+
        • dropbox: Fix ChangeNotify was unable to decrypt errors (Nick Craig-Wood)
          • 
+
        • fs: Fix parsing of times and durations of the form "YYYY-MM-DD HH:MM:SS" (Nick Craig-Wood)
          • 
+
        • serve sftp: Fix checksum detection (Nick Craig-Wood)
          • 
+
        • sync: Add accidentally missed filter-sensitivity to --backup-dir option (Nick Naumann)
          • 
+
        • 
+
      • Combine
+
          
+
        • Fix docs showing remote= instead of upstreams= (Nick Craig-Wood)
          • 
+
        • Throw error if duplicate directory name is specified (Nick Craig-Wood)
          • 
+
        • Fix errors with backends shutting down while in use (Nick Craig-Wood)
          • 
+
        • 
+
      • Dropbox
+
          
+
        • Fix hang on quit with --dropbox-batch-mode off (Nick Craig-Wood)
          • 
+
        • Fix infinite loop on uploading a corrupted file (Nick Craig-Wood)
          • 
+
        • 
+
      • Internetarchive
+
          
+
        • Ignore checksums for files using the different method (Lesmiscore)
          • 
+
        • Handle hash symbol in the middle of filename (Lesmiscore)
          • 
+
        • 
+
      • Jottacloud
+
          
+
        • Fix working with whitelabel Elgiganten Cloud
          • 
+
        • Do not store username in config when using standard auth (albertony)
          • 
+
        • 
+
      • Mega
+
          
+
        • Fix nil pointer exception when bad node received (Nick Craig-Wood)
          • 
+
        • 
+
      • S3
+
          
+
        • Fix --s3-no-head panic: reflect: Elem of invalid type s3.PutObjectInput (Nick Craig-Wood)
          • 
+
        • 
+
      • SFTP
+
          
+
        • Fix issue with WS_FTP by working around failing RealPath (albertony)
          • 
+
        • 
+
      • Union
+
          
+
        • Fix duplicated files when using directories with leading / (Nick Craig-Wood)
          • 
+
        • Fix multiple files being uploaded when roots don't exist (Nick Craig-Wood)
          • 
+
        • Fix panic due to misalignment of struct field in 32 bit architectures (r-ricci)
          • 
+
        • 
+
      
 
      v1.59.0 - 2022-07-09
      
 
      See commits
      
 
        
@@ -28109,7 +29514,7 @@ 
        v1.58.0 - 2022-03-18
        
 
      • Fix ARM architecture version in .deb packages after nfpm change (Nick Craig-Wood)
        • 
 
      • Hard fork github.com/jlaffaye/ftp to fix go get github.com/rclone/rclone (Nick Craig-Wood)
        • 
 
      
-
    83. oauthutil: Fix crash when webrowser requests /robots.txt (Nick Craig-Wood)
      84. 
+
    84. oauthutil: Fix crash when webbrowser requests /robots.txt (Nick Craig-Wood)
      85. 
 
    85. operations: Fix goroutine leak in case of copy retry (Ankur Gupta)
      86. 
 
    86. rc:
 
        
@@ -28253,7 +29658,7 @@ 
        v1.58.0 - 2022-03-18
        
 
      • Add rclone to list of supported md5sum/sha1sum commands to look for (albertony)
        • 
 
      • Refactor so we only have one way of running remote commands (Nick Craig-Wood)
        • 
 
      • Fix timeout on hashing large files by sending keepalives (Nick Craig-Wood)
        • 
-
      • Fix unecessary seeking when uploading and downloading files (Nick Craig-Wood)
        • 
+
      • Fix unnecessary seeking when uploading and downloading files (Nick Craig-Wood)
        • 
 
      • Update docs on how to create known_hosts file (Nick Craig-Wood)
        • 
 
      87. 
 
    87. Storj
@@ -29147,9 +30552,9 @@ 
      v1.54.0 - 2021-02-02
      
 
    88. Add sort by average size in directory (Adam Plánský)
      89. 
 
    89. Add toggle option for average s3ize in directory - key 'a' (Adam Plánský)
      90. 
 
    90. Add empty folder flag into ncdu browser (Adam Plánský)
      91. 
-
    91. Add ! (errror) and . (unreadable) file flags to go with e (empty) (Nick Craig-Wood)
      92. 
+
    92. Add ! (error) and . (unreadable) file flags to go with e (empty) (Nick Craig-Wood)
      93. 
 
-
    93. obscure: Make rclone osbcure - ignore newline at end of line (Nick Craig-Wood)
      94. 
+
    94. obscure: Make rclone obscure - ignore newline at end of line (Nick Craig-Wood)
      95. 
 
    95. operations
 
        
 
      • Add logs when need to upload files to set mod times (Nick Craig-Wood)
        • 
@@ -29187,7 +30592,7 @@ 
        v1.54.0 - 2021-02-02
        
 
      • move: Fix data loss when source and destination are the same object (Nick Craig-Wood)
        • 
 
      • operations
 
          
-
        • Fix --cutof-mode hard not cutting off immediately (Nick Craig-Wood)
          • 
+
        • Fix --cutoff-mode hard not cutting off immediately (Nick Craig-Wood)
          • 
 
        • Fix --immutable error message (Nick Craig-Wood)
          • 
 
        • 
 
      • sync
@@ -29258,7 +30663,7 @@ 
        v1.54.0 - 2021-02-02
        
 
      • Box
 
          
 
        • Fix NewObject for files that differ in case (Nick Craig-Wood)
          • 
-
        • Fix finding directories in a case insentive way (Nick Craig-Wood)
          • 
+
        • Fix finding directories in a case insensitive way (Nick Craig-Wood)
          • 
 
        • 
 
      • Chunker
 
          
@@ -29377,7 +30782,7 @@ 
          v1.54.0 - 2021-02-02
          
 
        • Sugarsync
 
            
 
          • Fix NewObject for files that differ in case (Nick Craig-Wood)
            • 
-
          • Fix finding directories in a case insentive way (Nick Craig-Wood)
            • 
+
          • Fix finding directories in a case insensitive way (Nick Craig-Wood)
            • 
 
          • 
 
        • Swift
 
            
@@ -29488,7 +30893,7 @@ 
            v1.53.2 - 2020-10-26
            
 
              
 
            • Bug Fixes
 
                
-
              • acounting
+
              • accounting
 
                  
 
                • Fix incorrect speed and transferTime in core/stats (Nick Craig-Wood)
                  • 
 
                • Stabilize display order of transfers on Windows (Nick Craig-Wood)
                  • 
@@ -30880,7 +32285,7 @@ 
                  v1.49.0 - 2019-08-26
                  
 
                • 
 
              • Mount
 
                  
-
                • Default --daemon-timout to 15 minutes on macOS and FreeBSD (Nick Craig-Wood)
                  • 
+
                • Default --daemon-timeout to 15 minutes on macOS and FreeBSD (Nick Craig-Wood)
                  • 
 
                • Update docs to show mounting from root OK for bucket-based (Nick Craig-Wood)
                  • 
 
                • Remove nonseekable flag from write files (Nick Craig-Wood)
                  • 
 
                • 
@@ -31352,7 +32757,7 @@ 
                v1.46 - 2019-02-09
                
 
              • HTTP
 
                  
 
                • Add an example with username and password which is supported but wasn't documented (Nick Craig-Wood)
                  • 
-
                • Fix backend with --files-from and non-existent files (Nick Craig-Wood)
                  • 
+
                • Fix backend with --files-from and nonexistent files (Nick Craig-Wood)
                  • 
 
                • 
 
              • Hubic
 
                  
@@ -32083,7 +33488,7 @@ 
                  v1.41 - 2018-04-28
                  
 
                    
 
                  • Work around strange response from box FTP server
                    • 
 
                  • More workarounds for FTP servers to fix mkParentDir error
                    • 
-
                  • Fix no error on listing non-existent directory
                    • 
+
                  • Fix no error on listing nonexistent directory
                    • 
 
                  
 
                • Google Cloud Storage
 
                    
@@ -32196,7 +33601,7 @@ 
                    v1.40 - 2018-03-19
                    
 
                  • 
 
                • Bug Fixes
 
 
                  Contact the rclone project
                  
 
                  Forum
                  
diff --git a/MANUAL.md b/MANUAL.md
index 6cd4e96044ec6..c7030525ba61d 100644
--- a/MANUAL.md
+++ b/MANUAL.md
@@ -1,6 +1,6 @@
 % rclone(1) User Manual
 % Nick Craig-Wood
-% Jul 09, 2022
+% Oct 21, 2022
 
 # Rclone syncs your files to cloud storage
 
@@ -112,7 +112,6 @@ WebDAV or S3, that work out of the box.)
 - China Mobile Ecloud Elastic Object Storage (EOS)
 - Arvan Cloud Object Storage (AOS)
 - Citrix ShareFile
-- C14
 - Cloudflare R2
 - DigitalOcean Spaces
 - Digi Storage
@@ -127,11 +126,11 @@ WebDAV or S3, that work out of the box.)
 - Hetzner Storage Box
 - HiDrive
 - HTTP
-- Hubic
 - Internet Archive
 - Jottacloud
 - IBM COS S3
 - IDrive e2
+- IONOS Cloud
 - Koofr
 - Mail.ru Cloud
 - Memset Memstore
@@ -144,12 +143,14 @@ WebDAV or S3, that work out of the box.)
 - OVH
 - OpenDrive
 - OpenStack Swift
-- Oracle Cloud Storage
+- Oracle Cloud Storage Swift
+- Oracle Object Storage
 - ownCloud
 - pCloud
 - premiumize.me
 - put.io
 - QingStor
+- Qiniu Cloud Object Storage (Kodo)
 - Rackspace Cloud Files
 - rsync.net
 - Scaleway
@@ -158,6 +159,7 @@ WebDAV or S3, that work out of the box.)
 - SeaweedFS
 - SFTP
 - Sia
+- SMB / CIFS
 - StackPath
 - Storj
 - SugarSync
@@ -202,7 +204,7 @@ Rclone is a Go program and comes as a single binary file.
   * Run `rclone config` to setup. See [rclone config docs](https://rclone.org/docs/) for more details.
   * Optionally configure [automatic execution](#autostart).
 
-See below for some expanded Linux / macOS instructions.
+See below for some expanded Linux / macOS / Windows instructions.
 
 See the [usage](https://rclone.org/docs/) docs for how to use rclone, or
 run `rclone -h`.
@@ -223,7 +225,9 @@ For beta installation, run:
 Note that this script checks the version of rclone installed first and
 won't re-download if not needed.
 
-## Linux installation from precompiled binary
+## Linux installation {#linux}
+
+### Precompiled binary {#linux-precompiled}
 
 Fetch and unpack
 
@@ -247,7 +251,9 @@ Run `rclone config` to setup. See [rclone config docs](https://rclone.org/docs/)
 
     rclone config
 
-## macOS installation with brew
+## macOS installation {#macos}
+
+### Installation with brew {#macos-brew}
 
     brew install rclone
 
@@ -256,7 +262,12 @@ NOTE: This version of rclone will not support `mount` any more (see
 on macOS, either install a precompiled binary or enable the relevant option
 when [installing from source](#install-from-source).
 
-## macOS installation from precompiled binary, using curl
+Note that this is a third party installer not controlled by the rclone
+developers so it may be out of date. Its current version is as below.
+
+[![Homebrew package](https://repology.org/badge/version-for-repo/homebrew/rclone.svg)](https://repology.org/project/rclone/versions)
+
+### Precompiled binary, using curl {#macos-precompiled}
 
 To avoid problems with macOS gatekeeper enforcing the binary to be signed and
 notarized it is enough to download with `curl`.
@@ -284,7 +295,7 @@ Run `rclone config` to setup. See [rclone config docs](https://rclone.org/docs/)
 
     rclone config
 
-## macOS installation from precompiled binary, using a web browser
+### Precompiled binary, using a web browser  {#macos-precompiled-web}
 
 When downloading a binary with a web browser, the browser will set the macOS
 gatekeeper quarantine attribute. Starting from Catalina, when attempting to run
@@ -297,11 +308,73 @@ The simplest fix is to run
 
     xattr -d com.apple.quarantine rclone
 
-## Install with docker
+## Windows installation {#windows}
+
+### Precompiled binary {#windows-precompiled}
+
+Fetch the correct binary for your processor type by clicking on these
+links. If not sure, use the first link.
+
+- [Intel/AMD - 64 Bit](https://downloads.rclone.org/rclone-current-linux-amd64.zip)
+- [Intel/AMD - 32 Bit](https://downloads.rclone.org/rclone-current-linux-386.zip)
+- [ARM - 64 Bit](https://downloads.rclone.org/rclone-current-linux-arm64.zip)
+
+Open this file in the Explorer and extract `rclone.exe`. Rclone is a
+portable executable so you can place it wherever is convenient.
+
+Open a CMD window (or powershell) and run the binary. Note that rclone
+does not launch a GUI by default, it runs in the CMD Window.
+
+- Run `rclone.exe config` to setup. See [rclone config docs](https://rclone.org/docs/) for more details.
+- Optionally configure [automatic execution](#autostart).
+
+If you are planning to use the [rclone mount](https://rclone.org/commands/rclone_mount/)
+feature then you will need to install the third party utility
+[WinFsp](https://winfsp.dev/) also.
+
+### Chocolatey package manager {#windows-chocolatey}
+
+Make sure you have [Choco](https://chocolatey.org/) installed
+
+```
+choco search rclone
+choco install rclone
+```
+
+This will install rclone on your Windows machine. If you are planning
+to use [rclone mount](https://rclone.org/commands/rclone_mount/) then
+
+```
+choco install winfsp
+```
+
+will install that too.
+
+Note that this is a third party installer not controlled by the rclone
+developers so it may be out of date. Its current version is as below.
+
+[![Chocolatey package](https://repology.org/badge/version-for-repo/chocolatey/rclone.svg)](https://repology.org/project/rclone/versions)
+
+## Package manager installation {#package-manager}
+
+Many Linux, Windows, macOS and other OS distributions package and
+distribute rclone.
+
+The distributed versions of rclone are often quite out of date and for
+this reason we recommend one of the other installation methods if
+possible.
+
+You can get an idea of how up to date or not your OS distribution's
+package is here.
+
+[![Packaging status](https://repology.org/badge/vertical-allrepos/rclone.svg?columns=3)](https://repology.org/project/rclone/versions)
 
-The rclone maintains a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
-These images are autobuilt by docker hub from the rclone source based
-on a minimal Alpine linux image.
+## Docker installation {#docker}
+
+The rclone developers maintain a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
+
+These images are built as part of the release process based on a
+minimal Alpine Linux.
 
 The `:latest` tag will always point to the latest stable release.  You
 can use the `:beta` tag to get the latest build from master.  You can
@@ -376,10 +449,10 @@ ls ~/data/mount
 kill %1
 ```
 
-## Install from source
+## Source installation {#source}
 
 Make sure you have git and [Go](https://golang.org/) installed.
-Go version 1.16 or newer is required, latest release is recommended.
+Go version 1.17 or newer is required, latest release is recommended.
 You can get it from your package manager, or download it from
 [golang.org/dl](https://golang.org/dl/). Then you can run the following:
 
@@ -395,7 +468,7 @@ in the same folder. As an initial check you can now run `./rclone version`
 (`.\rclone version` on Windows).
 
 Note that on macOS and Windows the [mount](https://rclone.org/commands/rclone_mount/)
-command will not be available unless you specify additional build tag `cmount`.
+command will not be available unless you specify an additional build tag `cmount`.
 
 ```
 go build -tags cmount
@@ -414,7 +487,7 @@ distribution (make sure you install it in the classic mingw64 subsystem, the
 ucrt64 version is not compatible).
 
 Additionally, on Windows, you must install the third party utility
-[WinFsp](http://www.secfs.net/winfsp/), with the "Developer" feature selected.
+[WinFsp](https://winfsp.dev/), with the "Developer" feature selected.
 If building with cgo, you must also set environment variable CPATH pointing to
 the fuse include directory within the WinFsp installation
 (normally `C:\Program Files (x86)\WinFsp\inc\fuse`).
@@ -429,9 +502,10 @@ go build -trimpath -ldflags -s -tags cmount
 ```
 
 Instead of executing the `go build` command directly, you can run it via the
-Makefile, which also sets version information and copies the resulting rclone
-executable into your GOPATH bin folder (`$(go env GOPATH)/bin`, which
-corresponds to `~/go/bin/rclone` by default).
+Makefile. It changes the version number suffix from "-DEV" to "-beta" and
+appends commit details. It also copies the resulting rclone executable into
+your GOPATH bin folder (`$(go env GOPATH)/bin`, which corresponds to
+`~/go/bin/rclone` by default).
 
 ```
 make
@@ -443,7 +517,15 @@ To include mount command on macOS and Windows with Makefile build:
 make GOTAGS=cmount
 ```
 
-As an alternative you can download the source, build and install rclone in one
+There are other make targets that can be used for more advanced builds,
+such as cross-compiling for all supported os/architectures, embedding
+icon and version info resources into windows executable, and packaging
+results into release artifacts.
+See [Makefile](https://github.com/rclone/rclone/blob/master/Makefile)
+and [cross-compile.go](https://github.com/rclone/rclone/blob/master/bin/cross-compile.go)
+for details.
+
+Another alternative is to download the source, build and install rclone in one
 operation, as a regular Go package. The source will be stored it in the Go
 module cache, and the resulting executable will be in your GOPATH bin folder
 (`$(go env GOPATH)/bin`, which corresponds to `~/go/bin/rclone` by default).
@@ -462,7 +544,7 @@ with the current version):
 go get github.com/rclone/rclone
 ```
 
-## Installation with Ansible
+## Ansible installation {#ansible}
 
 This can be done with [Stefan Weichinger's ansible
 role](https://github.com/stefangweichinger/ansible-rclone).
@@ -478,7 +560,7 @@ Instructions
           - rclone
 ```
 
-## Portable installation
+## Portable installation {#portable}
 
 As mentioned [above](https://rclone.org/install/#quickstart), rclone is single
 executable (`rclone`, or `rclone.exe` on Windows) that you can download as a
@@ -506,7 +588,7 @@ such as a regular [sync](https://rclone.org/commands/rclone_sync/), you will pro
 to configure your rclone command in your operating system's scheduler. If you need to
 expose *service*-like features, such as [remote control](https://rclone.org/rc/),
 [GUI](https://rclone.org/gui/), [serve](https://rclone.org/commands/rclone_serve/)
-or [mount](https://rclone.org/commands/rclone_move/), you will often want an rclone
+or [mount](https://rclone.org/commands/rclone_mount/), you will often want an rclone
 command always running in the background, and configuring it to run in a service infrastructure
 may be a better option. Below are some alternatives on how to achieve this on
 different operating systems.
@@ -539,7 +621,7 @@ c:\rclone\rclone.exe sync c:\files remote:/files --no-console --log-file c:\rclo
 
 #### User account
 
-As mentioned in the [mount](https://rclone.org/commands/rclone_move/) documentation,
+As mentioned in the [mount](https://rclone.org/commands/rclone_mount/) documentation,
 mounted drives created as Administrator are not visible to other accounts, not even the
 account that was elevated as Administrator. By running the mount command as the
 built-in `SYSTEM` user account, it will create drives accessible for everyone on
@@ -548,7 +630,7 @@ the system. Both scheduled task and Windows service can be used to achieve this.
 NOTE: Remember that when rclone runs as the `SYSTEM` user, the user profile
 that it sees will not be yours. This means that if you normally run rclone with
 configuration file in the default location, to be able to use the same configuration
-when running as the system user you must explicitely tell rclone where to find
+when running as the system user you must explicitly tell rclone where to find
 it with the [`--config`](https://rclone.org/docs/#config-config-file) option,
 or else it will look in the system users profile path (`C:\Windows\System32\config\systemprofile`).
 To test your command manually from a Command Prompt, you can run it with
@@ -612,7 +694,7 @@ it should be possible through path rewriting as described [here](https://github.
 
 To Windows service running any rclone command, the excellent third-party utility
 [NSSM](http://nssm.cc), the "Non-Sucking Service Manager", can be used.
-It includes some advanced features such as adjusting process periority, defining
+It includes some advanced features such as adjusting process priority, defining
 process environment variables, redirect to file anything written to stdout, and
 customized response to different exit codes, with a GUI to configure everything from
 (although it can also be used from command line ).
@@ -690,7 +772,6 @@ See the following for detailed instructions for
   * [HDFS](https://rclone.org/hdfs/)
   * [HiDrive](https://rclone.org/hidrive/)
   * [HTTP](https://rclone.org/http/)
-  * [Hubic](https://rclone.org/hubic/)
   * [Internet Archive](https://rclone.org/internetarchive/)
   * [Jottacloud](https://rclone.org/jottacloud/)
   * [Koofr](https://rclone.org/koofr/)
@@ -701,6 +782,7 @@ See the following for detailed instructions for
   * [Microsoft OneDrive](https://rclone.org/onedrive/)
   * [OpenStack Swift / Rackspace Cloudfiles / Memset Memstore](https://rclone.org/swift/)
   * [OpenDrive](https://rclone.org/opendrive/)
+  * [Oracle Object Storage](https://rclone.org/oracleobjectstorage/)
   * [Pcloud](https://rclone.org/pcloud/)
   * [premiumize.me](https://rclone.org/premiumizeme/)
   * [put.io](https://rclone.org/putio/)
@@ -708,6 +790,7 @@ See the following for detailed instructions for
   * [Seafile](https://rclone.org/seafile/)
   * [SFTP](https://rclone.org/sftp/)
   * [Sia](https://rclone.org/sia/)
+  * [SMB](https://rclone.org/smb/)
   * [Storj](https://rclone.org/storj/)
   * [SugarSync](https://rclone.org/sugarsync/)
   * [Union](https://rclone.org/union/)
@@ -897,6 +980,11 @@ extended explanation in the [copy](https://rclone.org/commands/rclone_copy/) com
 If dest:path doesn't exist, it is created and the source:path contents
 go there.
 
+It is not possible to sync overlapping remotes. However, you may exclude
+the destination from the sync with a filter rule or by putting an 
+exclude-if-present file inside the destination directory and sync to a
+destination that is inside the source directory.
+
 **Note**: Use the `-P`/`--progress` flag to view real-time transfer statistics
 
 **Note**: Use the `rclone dedupe` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
@@ -1222,7 +1310,7 @@ Note that `ls` and `lsl` recurse by default - use `--max-depth 1` to stop the re
 
 The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use `-R` to make them recurse.
 
-Listing a non-existent directory will produce an error except for
+Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).
 
@@ -1290,7 +1378,7 @@ Note that `ls` and `lsl` recurse by default - use `--max-depth 1` to stop the re
 
 The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use `-R` to make them recurse.
 
-Listing a non-existent directory will produce an error except for
+Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).
 
@@ -1349,7 +1437,7 @@ Note that `ls` and `lsl` recurse by default - use `--max-depth 1` to stop the re
 
 The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use `-R` to make them recurse.
 
-Listing a non-existent directory will produce an error except for
+Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).
 
@@ -1391,7 +1479,7 @@ to running `rclone hashsum MD5 remote:path`.
 
 This command can also hash data received on standard input (stdin),
 by not passing a remote:path, or by passing a hyphen as remote:path
-when there is data to read (if not, the hypen will be treated literaly,
+when there is data to read (if not, the hyphen will be treated literally,
 as a relative path).
 
 
@@ -1436,7 +1524,7 @@ to running `rclone hashsum SHA1 remote:path`.
 
 This command can also hash data received on standard input (stdin),
 by not passing a remote:path, or by passing a hyphen as remote:path
-when there is data to read (if not, the hypen will be treated literaly,
+when there is data to read (if not, the hyphen will be treated literally,
 as a relative path).
 
 This command can also hash data received on STDIN, if not passing
@@ -1884,11 +1972,11 @@ See the [global flags page](https://rclone.org/flags/) for global options not li
 
 # rclone bisync
 
-Perform bidirectonal synchronization between two paths.
+Perform bidirectional synchronization between two paths.
 
 ## Synopsis
 
-Perform bidirectonal synchronization between two paths.
+Perform bidirectional synchronization between two paths.
 
 [Bisync](https://rclone.org/bisync/) provides a
 bidirectional cloud sync solution in rclone.
@@ -2087,7 +2175,7 @@ To load completions for every new session, execute once:
 
 ### macOS:
 
-	rclone completion bash > /usr/local/etc/bash_completion.d/rclone
+	rclone completion bash > $(brew --prefix)/etc/bash_completion.d/rclone
 
 You will need to start a new shell for this setup to take effect.
 
@@ -2191,6 +2279,10 @@ to enable it.  You can execute the following once:
 
 	echo "autoload -U compinit; compinit" >> ~/.zshrc
 
+To load completions in your current shell session:
+
+	source <(rclone completion zsh); compdef _rclone rclone
+
 To load completions for every new session, execute once:
 
 ### Linux:
@@ -2199,7 +2291,7 @@ To load completions for every new session, execute once:
 
 ### macOS:
 
-	rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone
+	rclone completion zsh > $(brew --prefix)/share/zsh/site-functions/_rclone
 
 You will need to start a new shell for this setup to take effect.
 
@@ -2261,7 +2353,7 @@ are 100% certain you are already passing obscured passwords then use
 `rclone config password` command.
 
 The flag `--non-interactive` is for use by applications that wish to
-configure rclone themeselves, rather than using rclone's text based
+configure rclone themselves, rather than using rclone's text based
 configuration questions. If this flag is set, and rclone needs to ask
 the user a question, a JSON blob will be returned with the question in
 it.
@@ -2656,7 +2748,7 @@ are 100% certain you are already passing obscured passwords then use
 `rclone config password` command.
 
 The flag `--non-interactive` is for use by applications that wish to
-configure rclone themeselves, rather than using rclone's text based
+configure rclone themselves, rather than using rclone's text based
 configuration questions. If this flag is set, and rclone needs to ask
 the user a question, a JSON blob will be returned with the question in
 it.
@@ -3211,7 +3303,7 @@ For the MD5 and SHA1 algorithms there are also dedicated commands,
 
 This command can also hash data received on standard input (stdin),
 by not passing a remote:path, or by passing a hyphen as remote:path
-when there is data to read (if not, the hypen will be treated literaly,
+when there is data to read (if not, the hyphen will be treated literally,
 as a relative path).
 
 Run without a hash to see the list of all supported hashes, e.g.
@@ -3452,7 +3544,7 @@ Note that `ls` and `lsl` recurse by default - use `--max-depth 1` to stop the re
 
 The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use `-R` to make them recurse.
 
-Listing a non-existent directory will produce an error except for
+Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).
 
@@ -3533,7 +3625,7 @@ If `--files-only` is not specified directories in addition to the files
 will be returned.
 
 If `--metadata` is set then an additional Metadata key will be returned.
-This will have metdata in rclone standard format as a JSON object.
+This will have metadata in rclone standard format as a JSON object.
 
 if `--stat` is set then a single JSON blob will be returned about the
 item pointed to. This will return an error if the item isn't found.
@@ -3579,7 +3671,7 @@ Note that `ls` and `lsl` recurse by default - use `--max-depth 1` to stop the re
 
 The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use `-R` to make them recurse.
 
-Listing a non-existent directory will produce an error except for
+Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).
 
@@ -3703,7 +3795,7 @@ and experience unexpected program errors, freezes or other issues, consider moun
 as a network drive instead.
 
 When mounting as a fixed disk drive you can either mount to an unused drive letter,
-or to a path representing a **non-existent** subdirectory of an **existing** parent
+or to a path representing a **nonexistent** subdirectory of an **existing** parent
 directory or drive. Using the special value `*` will tell rclone to
 automatically assign the next available drive letter, starting with Z: and moving backward.
 Examples:
@@ -3734,7 +3826,7 @@ the mapped drive, shown in Windows Explorer etc, while the complete
 `\\server\share` will be reported as the remote UNC path by
 `net use` etc, just like a normal network drive mapping.
 
-If you specify a full network share UNC path with `--volname`, this will implicitely
+If you specify a full network share UNC path with `--volname`, this will implicitly
 set the `--network-mode` option, so the following two examples have same result:
 
     rclone mount remote:path/to/files X: --network-mode
@@ -3743,7 +3835,7 @@ set the `--network-mode` option, so the following two examples have same result:
 You may also specify the network share UNC path as the mountpoint itself. Then rclone
 will automatically assign a drive letter, same as with `*` and use that as
 mountpoint, and instead use the UNC path specified as the volume name, as if it were
-specified with the `--volname` option. This will also implicitely set
+specified with the `--volname` option. This will also implicitly set
 the `--network-mode` option. This means the following two examples have same result:
 
     rclone mount remote:path/to/files \\cloud\remote
@@ -3779,7 +3871,7 @@ The permissions on each entry will be set according to [options](#options)
 
 The default permissions corresponds to `--file-perms 0666 --dir-perms 0777`,
 i.e. read and write permissions to everyone. This means you will not be able
-to start any programs from the the mount. To be able to do that you must add
+to start any programs from the mount. To be able to do that you must add
 execute permissions, e.g. `--file-perms 0777 --dir-perms 0777` to add it
 to everyone. If the program needs to write files, chances are you will have
 to enable [VFS File Caching](#vfs-file-caching) as well (see also [limitations](#limitations)).
@@ -3850,8 +3942,8 @@ applications won't work with their files on an rclone mount without
 `--vfs-cache-mode writes` or `--vfs-cache-mode full`.
 See the [VFS File Caching](#vfs-file-caching) section for more info.
 
-The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2,
-Hubic) do not support the concept of empty directories, so empty
+The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2)
+do not support the concept of empty directories, so empty
 directories will have a tendency to disappear once they fall out of
 the directory cache.
 
@@ -4471,7 +4563,7 @@ press '?' to toggle the help on and off. The supported keys are:
      q/ESC/^c to quit
 
 Listed files/directories may be prefixed by a one-character flag,
-some of them combined with a description in brackes at end of line.
+some of them combined with a description in brackets at end of line.
 These flags have the following meaning:
 
     e means this is an empty directory, i.e. contains no files (but
@@ -5240,11 +5332,13 @@ rclone serve dlna remote:path [flags]
 
 ```
       --addr string                            The ip:port or :port to bind the DLNA http server to (default ":7879")
+      --announce-interval duration             The interval between SSDP announcements (default 12m0s)
       --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
       --dir-perms FileMode                     Directory permissions (default 0777)
       --file-perms FileMode                    File permissions (default 0666)
       --gid uint32                             Override the gid field set by the filesystem (not supported on Windows) (default 1000)
   -h, --help                                   help for dlna
+      --interface stringArray                  The interface to use for SSDP (repeat as necessary)
       --log-trace                              Enable trace logging of SOAP traffic
       --name string                            Name of DLNA server
       --no-checksum                            Don't compare checksums on up/download
@@ -6237,6 +6331,10 @@ of that with the CA certificate.  `--key` should be the PEM encoded
 private key and `--client-ca` should be the PEM encoded client
 certificate authority certificate.
 
+--min-tls-version is minimum TLS version that is acceptable. Valid
+  values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
+  "tls1.0").
+
 ### Template
 
 `--template` allows a user to specify a custom markup template for HTTP
@@ -6623,6 +6721,7 @@ rclone serve http remote:path [flags]
       --htpasswd string                        A htpasswd file - if not provided no authentication is done
       --key string                             SSL PEM Private key
       --max-header-bytes int                   Maximum size of request header (default 4096)
+      --min-tls-version string                 Minimum TLS version that is acceptable (default "tls1.0")
       --no-checksum                            Don't compare checksums on up/download
       --no-modtime                             Don't read/write the modification time (can speed things up)
       --no-seek                                Don't allow seeking in files
@@ -6828,6 +6927,10 @@ of that with the CA certificate.  `--key` should be the PEM encoded
 private key and `--client-ca` should be the PEM encoded client
 certificate authority certificate.
 
+--min-tls-version is minimum TLS version that is acceptable. Valid
+  values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
+  "tls1.0").
+
 
 ```
 rclone serve restic remote:path [flags]
@@ -6846,6 +6949,7 @@ rclone serve restic remote:path [flags]
       --htpasswd string                 htpasswd file - if not provided no authentication is done
       --key string                      SSL PEM Private key
       --max-header-bytes int            Maximum size of request header (default 4096)
+      --min-tls-version string          Minimum TLS version that is acceptable (default "tls1.0")
       --pass string                     Password for authentication
       --private-repos                   Users can only access their private repo
       --realm string                    Realm for authentication (default "rclone")
@@ -6868,11 +6972,19 @@ Serve the remote over SFTP.
 
 ## Synopsis
 
-Run a SFTP server to serve a remote over SFTP. This can be used
-with an SFTP client or you can make a remote of type sftp to use with it.
+Run an SFTP server to serve a remote over SFTP. This can be used
+with an SFTP client or you can make a remote of type [sftp](/sftp) to use with it.
 
-You can use the filter flags (e.g. `--include`, `--exclude`) to control what
-is served.
+You can use the [filter](/filtering) flags (e.g. `--include`, `--exclude`)
+to control what is served.
+
+The server will respond to a small number of shell commands, mainly
+md5sum, sha1sum and df, which enable it to provide support for checksums
+and the about feature when accessed from an sftp remote.
+
+Note that this server uses standard 32 KiB packet payload size, which
+means you must not configure the client to expect anything else, e.g.
+with the [chunk_size](https://rclone.org/sftp/#sftp-chunk-size) option on an sftp remote.
 
 The server will log errors.  Use `-v` to see access logs.
 
@@ -6885,11 +6997,6 @@ You must provide some means of authentication, either with
 `--auth-proxy`, or set the `--no-auth` flag for no
 authentication when logging in.
 
-Note that this also implements a small number of shell commands so
-that it can provide md5sum/sha1sum/df information for the rclone sftp
-backend.  This means that is can support SHA1SUMs, MD5SUMs and the
-about command when paired with the rclone sftp backend.
-
 If you don't supply a host `--key` then rclone will generate rsa, ecdsa
 and ed25519 variants, and cache them for later use in rclone's cache
 directory (see `rclone help flags cache-dir`) in the "serve-sftp"
@@ -7341,7 +7448,7 @@ rclone serve sftp remote:path [flags]
       --pass string                            Password for authentication
       --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --read-only                              Only allow read-only access
-      --stdio                                  Run an sftp server on run stdin/stdout
+      --stdio                                  Run an sftp server on stdin/stdout
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
       --user string                            User name for authentication
@@ -7471,6 +7578,10 @@ of that with the CA certificate.  `--key` should be the PEM encoded
 private key and `--client-ca` should be the PEM encoded client
 certificate authority certificate.
 
+--min-tls-version is minimum TLS version that is acceptable. Valid
+  values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
+  "tls1.0").
+
 ## VFS - Virtual File System
 
 This command uses the VFS layer. This adapts the cloud storage objects
@@ -7893,6 +8004,7 @@ rclone serve webdav remote:path [flags]
       --htpasswd string                        htpasswd file - if not provided no authentication is done
       --key string                             SSL PEM Private key
       --max-header-bytes int                   Maximum size of request header (default 4096)
+      --min-tls-version string                 Minimum TLS version that is acceptable (default "tls1.0")
       --no-checksum                            Don't compare checksums on up/download
       --no-modtime                             Don't read/write the modification time (can speed things up)
       --no-seek                                Don't allow seeking in files
@@ -8651,7 +8763,7 @@ backends can also store arbitrary user metadata.
 
 Where possible the key names are standardized, so, for example, it is
 possible to copy object metadata from s3 to azureblob for example and
-metadata will be translated apropriately.
+metadata will be translated appropriately.
 
 Some backends have limits on the size of the metadata and rclone will
 give errors on upload if they are exceeded.
@@ -8713,10 +8825,34 @@ it to `false`.  It is also possible to specify `--boolean=false` or
 parsed as `--boolean` and the `false` is parsed as an extra command
 line argument for rclone.
 
-Options which use TIME use the go time parser.  A duration string is a
-possibly signed sequence of decimal numbers, each with optional
-fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid
-time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+### Time or duration options {#time-option}
+
+TIME or DURATION options can be specified as a duration string or a
+time string.
+
+A duration string is a possibly signed sequence of decimal numbers,
+each with optional fraction and a unit suffix, such as "300ms",
+"-1.5h" or "2h45m". Default units are seconds or the following
+abbreviations are valid:
+
+  * `ms` - Milliseconds
+  * `s`  - Seconds
+  * `m`  - Minutes
+  * `h`  - Hours
+  * `d`  - Days
+  * `w`  - Weeks
+  * `M`  - Months
+  * `y`  - Years
+
+These can also be specified as an absolute time in the following
+formats:
+
+- RFC3339 - e.g. `2006-01-02T15:04:05Z` or `2006-01-02T15:04:05+07:00`
+- ISO8601 Date and time, local timezone - `2006-01-02T15:04:05`
+- ISO8601 Date and time, local timezone - `2006-01-02 15:04:05`
+- ISO8601 Date - `2006-01-02` (YYYY-MM-DD)
+
+### Size options {#size-option}
 
 Options which use SIZE use KiB (multiples of 1024 bytes) by default.
 However, a suffix of `B` for Byte, `K` for KiB, `M` for MiB,
@@ -8735,7 +8871,8 @@ been added) in DIR, then it will be overwritten.
 
 The remote in use must support server-side move or copy and you must
 use the same remote as the destination of the sync.  The backup
-directory must not overlap the destination directory.
+directory must not overlap the destination directory without it being
+excluded by a filter rule.
 
 For example
 
@@ -8769,7 +8906,7 @@ would mean limit the upload and download bandwidth to 10 MiB/s.
 single limit, specify the desired bandwidth in KiB/s, or use a
 suffix B|K|M|G|T|P. The default is `0` which means to not limit bandwidth.
 
-The upload and download bandwidth can be specified seperately, as
+The upload and download bandwidth can be specified separately, as
 `--bwlimit UP:DOWN`, so
 
     --bwlimit 10M:100k
@@ -9796,6 +9933,18 @@ This sets the interval between each retry specified by `--retries`
 
 The default is `0`. Use `0` to disable.
 
+### --server-side-across-configs ###
+
+Allow server-side operations (e.g. copy or move) to work across
+different configurations.
+
+This can be useful if you wish to do a server-side copy or move
+between two remotes which use the same backend but are configured
+differently.
+
+Note that this isn't enabled by default because it isn't easy for
+rclone to tell if it will work between any two configurations.
+
 ### --size-only ###
 
 Normally rclone will look at modification time and size of files to
@@ -9984,13 +10133,22 @@ By default, rclone doesn't keep track of renamed files, so if you
 rename a file locally then sync it to a remote, rclone will delete the
 old file on the remote and upload a new copy.
 
-If you use this flag, and the remote supports server-side copy or
-server-side move, and the source and destination have a compatible
-hash, then this will track renames during `sync`
-operations and perform renaming server-side.
+An rclone sync with `--track-renames` runs like a normal sync, but keeps
+track of objects which exist in the destination but not in the source
+(which would normally be deleted), and which objects exist in the
+source but not the destination (which would normally be transferred).
+These objects are then candidates for renaming.
+
+After the sync, rclone matches up the source only and destination only
+objects using the `--track-renames-strategy` specified and either
+renames the destination object or transfers the source and deletes the
+destination object. `--track-renames` is stateless like all of
+rclone's syncs.
 
-Files will be matched by size and hash - if both match then a rename
-will be considered.
+To use this flag the destination must support server-side copy or
+server-side move, and to use a hash based `--track-renames-strategy`
+(the default) the source and the destination must have a compatible
+hash.
 
 If the destination does not support server-side copy or move, rclone
 will fall back to the default behaviour and log an error level message
@@ -10008,7 +10166,7 @@ Note also that `--track-renames` is incompatible with
 
 ### --track-renames-strategy (hash,modtime,leaf,size) ###
 
-This option changes the matching criteria for `--track-renames`.
+This option changes the file matching criteria for `--track-renames`.
 
 The matching is controlled by a comma separated selection of these tokens:
 
@@ -10017,15 +10175,15 @@ The matching is controlled by a comma separated selection of these tokens:
 - `leaf` - the name of the file not including its directory name
 - `size` - the size of the file (this is always enabled)
 
-So using `--track-renames-strategy modtime,leaf` would match files
+The default option is `hash`.
+
+Using `--track-renames-strategy modtime,leaf` would match files
 based on modification time, the leaf of the file name and the size
 only.
 
 Using `--track-renames-strategy modtime` or `leaf` can enable
 `--track-renames` support for encrypted destinations.
 
-If nothing is specified, the default option is matching by `hash`es.
-
 Note that the `hash` strategy is not supported with encrypted destinations.
 
 ### --delete-(before,during,after) ###
@@ -10061,7 +10219,7 @@ quickly using the least amount of memory.
 
 However, some remotes have a way of listing all files beneath a
 directory in one (or a small number) of transactions.  These tend to
-be the bucket-based remotes (e.g. S3, B2, GCS, Swift, Hubic).
+be the bucket-based remotes (e.g. S3, B2, GCS, Swift).
 
 If you use the `--fast-list` flag then rclone will use this method for
 listing directories.  This will have the following consequences for
@@ -10127,7 +10285,7 @@ In all other cases the file will not be updated.
 Consider using the `--modify-window` flag to compensate for time skews
 between the source and the backend, for backends that do not support
 mod times, and instead use uploaded times. However, if the backend
-does not support checksums, note that sync'ing or copying within the
+does not support checksums, note that syncing or copying within the
 time skew window may still result in additional transfers for safety.
 
 ### --use-mmap ###
@@ -10920,7 +11078,7 @@ them into regular expressions.
 | Rooted      | `/*.jpg` | `/file.jpg`    | `/file.png`    |
 |             |          | `/file2.jpg`    | `/dir/file.jpg` |
 | Alternates  | `*.{jpg,png}` | `/file.jpg`     | `/file.gif`    |
-|             |         | `/dir/file.gif` | `/dir/file.gif` |
+|             |         | `/dir/file.png` | `/dir/file.gif` |
 | Path Wildcard | `dir/**` | `/dir/anyfile`     | `file.png`    |
 |             |          | `/subdir/dir/subsubdir/anyfile` | `/subdir/file.png` |
 | Any Char    | `*.t?t` | `/file.txt`     | `/file.qxt`    |
@@ -11420,6 +11578,8 @@ Default units are `KiB` but abbreviations `K`, `M`, `G`, `T` or `P` are valid.
 E.g. `rclone ls remote: --min-size 50k` lists files on `remote:` of 50 KiB
 size or larger.
 
+See [the size option docs](https://rclone.org/docs/#size-option) for more info.
+
 ### `--max-size` - Don't transfer any file larger than this
 
 Controls the maximum size file within the scope of an rclone command.
@@ -11428,33 +11588,19 @@ Default units are `KiB` but abbreviations `K`, `M`, `G`, `T` or `P` are valid.
 E.g. `rclone ls remote: --max-size 1G` lists files on `remote:` of 1 GiB
 size or smaller.
 
+See [the size option docs](https://rclone.org/docs/#size-option) for more info.
+
 ### `--max-age` - Don't transfer any file older than this
 
 Controls the maximum age of files within the scope of an rclone command.
-Default units are seconds or the following abbreviations are valid:
-
-  * `ms` - Milliseconds
-  * `s`  - Seconds
-  * `m`  - Minutes
-  * `h`  - Hours
-  * `d`  - Days
-  * `w`  - Weeks
-  * `M`  - Months
-  * `y`  - Years
-
-`--max-age` can also be specified as an absolute time in the following
-formats:
-
-- RFC3339 - e.g. `2006-01-02T15:04:05Z` or `2006-01-02T15:04:05+07:00`
-- ISO8601 Date and time, local timezone - `2006-01-02T15:04:05`
-- ISO8601 Date and time, local timezone - `2006-01-02 15:04:05`
-- ISO8601 Date - `2006-01-02` (YYYY-MM-DD)
 
 `--max-age` applies only to files and not to directories.
 
 E.g. `rclone ls remote: --max-age 2d` lists files on `remote:` of 2 days
 old or less.
 
+See [the time option docs](https://rclone.org/docs/#time-option) for valid formats.
+
 ### `--min-age` - Don't transfer any file younger than this
 
 Controls the minimum age of files within the scope of an rclone command.
@@ -11465,6 +11611,8 @@ Controls the minimum age of files within the scope of an rclone command.
 E.g. `rclone ls remote: --min-age 2d` lists files on `remote:` of 2 days
 old or more.
 
+See [the time option docs](https://rclone.org/docs/#time-option) for valid formats.
+
 ## Other flags
 
 ### `--delete-excluded` - Delete files on dest excluded from sync
@@ -11660,6 +11808,11 @@ SSL PEM Private key
 
 Maximum size of request header (default 4096)
 
+### --rc-min-tls-version=VALUE
+
+The minimum TLS version that is acceptable. Valid values are "tls1.0",
+"tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").
+
 ### --rc-user=VALUE
 
 User name for authentication.
@@ -12016,7 +12169,7 @@ The parameters can be a string as per the rest of rclone, eg
 `s3:bucket/path` or `:sftp:/my/dir`. They can also be specified as
 JSON blobs.
 
-If specifyng a JSON blob it should be a object mapping strings to
+If specifying a JSON blob it should be a object mapping strings to
 strings. These values will be used to configure the remote. There are
 3 special values which may be set:
 
@@ -12596,6 +12749,12 @@ Parameters:
 
 - jobid - id of the job (integer).
 
+### job/stopgroup: Stop all running jobs in a group {#job-stopgroup}
+
+Parameters:
+
+- group - name of the group (string).
+
 ### mount/listmounts: Show current mount points {#mount-listmounts}
 
 This shows currently mounted points, which can be used for performing an unmount.
@@ -12671,9 +12830,11 @@ Example:
 
 **Authentication is required for this call.**
 
-### mount/unmountall: Show current mount points {#mount-unmountall}
+### mount/unmountall: Unmount all active mounts {#mount-unmountall}
 
-This shows currently mounted points, which can be used for performing an unmount.
+rclone allows Linux, FreeBSD, macOS and Windows to
+mount any of Rclone's cloud storage systems as a file system with
+FUSE.
 
 This takes no parameters and returns error if unmount does not succeed.
 
@@ -13187,7 +13348,7 @@ check that parameter passing is working properly.
 
 **Authentication is required for this call.**
 
-### sync/bisync: Perform bidirectonal synchronization between two paths. {#sync-bisync}
+### sync/bisync: Perform bidirectional synchronization between two paths. {#sync-bisync}
 
 This takes the following parameters
 
@@ -13618,7 +13779,6 @@ Here is an overview of the major features of each cloud storage system.
 | HDFS                         | -                | R/W     | No               | No              | -         | -        |
 | HiDrive                      | HiDrive ¹²       | R/W     | No               | No              | -         | -        |
 | HTTP                         | -                | R       | No               | No              | R         | -        |
-| Hubic                        | MD5              | R/W     | No               | No              | R/W       | -        |
 | Internet Archive             | MD5, SHA1, CRC32 | R/W ¹¹  | No               | No              | -         | RWU      |
 | Jottacloud                   | MD5              | R/W     | Yes              | No              | R         | -        |
 | Koofr                        | MD5              | -       | Yes              | No              | -         | -        |
@@ -13629,6 +13789,7 @@ Here is an overview of the major features of each cloud storage system.
 | Microsoft OneDrive           | SHA1 ⁵           | R/W     | Yes              | No              | R         | -        |
 | OpenDrive                    | MD5              | R/W     | Yes              | Partial ⁸       | -         | -        |
 | OpenStack Swift              | MD5              | R/W     | No               | No              | R/W       | -        |
+| Oracle Object Storage        | MD5              | R/W     | No               | No              | R/W       | -        |
 | pCloud                       | MD5, SHA1 ⁷      | R       | No               | No              | W         | -        |
 | premiumize.me                | -                | -       | Yes              | No              | R         | -        |
 | put.io                       | CRC-32           | R/W     | No               | Yes             | R         | -        |
@@ -13636,6 +13797,7 @@ Here is an overview of the major features of each cloud storage system.
 | Seafile                      | -                | -       | No               | No              | -         | -        |
 | SFTP                         | MD5, SHA1 ²      | R/W     | Depends          | No              | -         | -        |
 | Sia                          | -                | -       | No               | No              | -         | -        |
+| SMB                          | -                | -       | Yes              | No              | -         | -        |
 | SugarSync                    | -                | -       | No               | No              | -         | -        |
 | Storj                        | -                | R       | No               | No              | -         | -        |
 | Uptobox                      | -                | -       | No               | Yes             | -         | -        |
@@ -13697,7 +13859,7 @@ systems they must support a common hash type.
 
 ### ModTime ###
 
-Allmost all cloud storage systems store some sort of timestamp
+Almost all cloud storage systems store some sort of timestamp
 on objects, but several of them not something that is appropriate
 to use for syncing. E.g. some backends will only write a timestamp
 that represent the time of the upload. To be relevant for syncing
@@ -14069,7 +14231,6 @@ upon backend-specific capabilities.
 | HDFS                         | Yes   | No   | Yes  | Yes     | No      | No    | Yes          | No           | Yes   | Yes      |
 | HiDrive                      | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes          | No           | No    | Yes      |
 | HTTP                         | No    | No   | No   | No      | No      | No    | No           | No           | No    | Yes      |
-| Hubic                        | Yes † | Yes  | No   | No      | No      | Yes   | Yes          | No           | Yes   | No       |
 | Internet Archive             | No    | Yes  | No   | No      | Yes     | Yes   | No           | Yes          | Yes   | No       |
 | Jottacloud                   | Yes   | Yes  | Yes  | Yes     | Yes     | Yes   | No           | Yes          | Yes   | Yes      |
 | Koofr                        | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes          | Yes          | Yes   | Yes      |
@@ -14080,6 +14241,7 @@ upon backend-specific capabilities.
 | Microsoft OneDrive           | Yes   | Yes  | Yes  | Yes     | Yes     | No    | No           | Yes          | Yes   | Yes      |
 | OpenDrive                    | Yes   | Yes  | Yes  | Yes     | No      | No    | No           | No           | No    | Yes      |
 | OpenStack Swift              | Yes † | Yes  | No   | No      | No      | Yes   | Yes          | No           | Yes   | No       |
+| Oracle Object Storage        | Yes   | Yes  | No   | No      | Yes     | Yes   | No           | No           | No    | No       |
 | pCloud                       | Yes   | Yes  | Yes  | Yes     | Yes     | No    | No           | Yes          | Yes   | Yes      |
 | premiumize.me                | Yes   | No   | Yes  | Yes     | No      | No    | No           | Yes          | Yes   | Yes      |
 | put.io                       | Yes   | No   | Yes  | Yes     | Yes     | No    | Yes          | No           | Yes   | Yes      |
@@ -14087,6 +14249,7 @@ upon backend-specific capabilities.
 | Seafile                      | Yes   | Yes  | Yes  | Yes     | Yes     | Yes   | Yes          | Yes          | Yes   | Yes      |
 | SFTP                         | No    | No   | Yes  | Yes     | No      | No    | Yes          | No           | Yes   | Yes      |
 | Sia                          | No    | No   | No   | No      | No      | No    | Yes          | No           | No    | Yes      |
+| SMB                          | No    | No   | Yes  | Yes     | No      | No    | Yes          | No           | No    | Yes      |
 | SugarSync                    | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes          | Yes          | No    | Yes      |
 | Storj                        | Yes † | No   | Yes  | No      | No      | Yes   | Yes          | No           | No    | No       |
 | Uptobox                      | No    | Yes  | Yes  | Yes     | No      | No    | No           | No           | No    | No       |
@@ -14100,7 +14263,7 @@ upon backend-specific capabilities.
 This deletes a directory quicker than just deleting all the files in
 the directory.
 
-† Note Swift, Hubic, and Storj implement this in order to delete
+† Note Swift and Storj implement this in order to delete
 directory markers but they don't actually have a quicker way of deleting
 files other than deleting them individually.
 
@@ -14295,6 +14458,7 @@ These flags are available for every command.
       --rc-job-expire-interval duration      Interval to check for expired async jobs (default 10s)
       --rc-key string                        SSL PEM Private key
       --rc-max-header-bytes int              Maximum size of request header (default 4096)
+      --rc-min-tls-version string            Minimum TLS version that is acceptable (default "tls1.0")
       --rc-no-auth                           Don't require auth for certain methods
       --rc-pass string                       Password for authentication
       --rc-realm string                      Realm for authentication (default "rclone")
@@ -14311,6 +14475,7 @@ These flags are available for every command.
       --refresh-times                        Refresh the modtime of remote files
       --retries int                          Retry operations this many times if they fail (default 3)
       --retries-sleep duration               Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)
+      --server-side-across-configs           Allow server-side operations (e.g. copy) to work across different configs
       --size-only                            Skip based on size only, not mod-time or checksum
       --stats duration                       Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s)
       --stats-file-name-length int           Max file name length in stats (0 for no limit) (default 45)
@@ -14336,7 +14501,7 @@ These flags are available for every command.
       --use-json-log                         Use json log format
       --use-mmap                             Use mmap allocator (see docs)
       --use-server-modtime                   Use server modified time instead of object metadata
-      --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.59.0")
+      --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.60.0")
   -v, --verbose count                        Print lots more stuff (repeat for more)
 ```
 
@@ -14489,7 +14654,7 @@ and may be set in the config file.
       --drive-use-trash                              Send files to the trash instead of deleting permanently (default true)
       --drive-v2-download-min-size SizeSuffix        If Object's are greater, use drive v2 API to download (default off)
       --dropbox-auth-url string                      Auth server URL
-      --dropbox-batch-commit-timeout Duration        Max time to wait for a batch to finish comitting (default 10m0s)
+      --dropbox-batch-commit-timeout Duration        Max time to wait for a batch to finish committing (default 10m0s)
       --dropbox-batch-mode string                    Upload file batching sync|async|off (default "sync")
       --dropbox-batch-size int                       Max number of files in upload batch
       --dropbox-batch-timeout Duration               Max time to allow an idle upload batch before uploading (default 0s)
@@ -14523,6 +14688,7 @@ and may be set in the config file.
       --ftp-disable-utf8                             Disable using UTF-8 even if server advertises support
       --ftp-encoding MultiEncoder                    The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
       --ftp-explicit-tls                             Use Explicit FTPS (FTP over TLS)
+      --ftp-force-list-hidden                        Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
       --ftp-host string                              FTP host to connect to
       --ftp-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
       --ftp-no-check-certificate                     Do not verify the TLS certificate of the server
@@ -14541,6 +14707,7 @@ and may be set in the config file.
       --gcs-client-secret string                     OAuth Client Secret
       --gcs-decompress                               If set this will decompress gzip encoded objects
       --gcs-encoding MultiEncoder                    The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot)
+      --gcs-endpoint string                          Endpoint for the service
       --gcs-location string                          Location for the newly created buckets
       --gcs-no-check-bucket                          If set, don't attempt to check the bucket exists or create it
       --gcs-object-acl string                        Access Control List for new objects
@@ -14586,14 +14753,6 @@ and may be set in the config file.
       --http-no-head                                 Don't use HEAD requests
       --http-no-slash                                Set this if the site doesn't end directories with /
       --http-url string                              URL of HTTP host to connect to
-      --hubic-auth-url string                        Auth server URL
-      --hubic-chunk-size SizeSuffix                  Above this size files will be chunked into a _segments container (default 5Gi)
-      --hubic-client-id string                       OAuth Client Id
-      --hubic-client-secret string                   OAuth Client Secret
-      --hubic-encoding MultiEncoder                  The encoding for the backend (default Slash,InvalidUtf8)
-      --hubic-no-chunk                               Don't chunk files during streaming upload
-      --hubic-token string                           OAuth Access Token as a JSON blob
-      --hubic-token-url string                       Token server url
       --internetarchive-access-key-id string         IAS3 Access Key
       --internetarchive-disable-checksum             Don't ask the server to test against MD5 checksum calculated by rclone (default true)
       --internetarchive-encoding MultiEncoder        The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot)
@@ -14662,6 +14821,22 @@ and may be set in the config file.
       --onedrive-server-side-across-configs          Allow server-side operations (e.g. copy) to work across different onedrive configs
       --onedrive-token string                        OAuth Access Token as a JSON blob
       --onedrive-token-url string                    Token server url
+      --oos-chunk-size SizeSuffix                    Chunk size to use for uploading (default 5Mi)
+      --oos-compartment string                       Object storage compartment OCID
+      --oos-config-file string                       Path to OCI config file (default "~/.oci/config")
+      --oos-config-profile string                    Profile name inside the oci config file (default "Default")
+      --oos-copy-cutoff SizeSuffix                   Cutoff for switching to multipart copy (default 4.656Gi)
+      --oos-copy-timeout Duration                    Timeout for copy (default 1m0s)
+      --oos-disable-checksum                         Don't store MD5 checksum with object metadata
+      --oos-encoding MultiEncoder                    The encoding for the backend (default Slash,InvalidUtf8,Dot)
+      --oos-endpoint string                          Endpoint for Object storage API
+      --oos-leave-parts-on-error                     If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery
+      --oos-namespace string                         Object storage namespace
+      --oos-no-check-bucket                          If set, don't attempt to check the bucket exists or create it
+      --oos-provider string                          Choose your Auth Provider (default "env_auth")
+      --oos-region string                            Object storage Region
+      --oos-upload-concurrency int                   Concurrency for multipart uploads (default 10)
+      --oos-upload-cutoff SizeSuffix                 Cutoff for switching to chunked upload (default 200Mi)
       --opendrive-chunk-size SizeSuffix              Files will be uploaded in chunks this size (default 10Mi)
       --opendrive-encoding MultiEncoder              The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot)
       --opendrive-password string                    Password (obscured)
@@ -14693,6 +14868,7 @@ and may be set in the config file.
       --s3-bucket-acl string                         Canned ACL used when creating buckets
       --s3-chunk-size SizeSuffix                     Chunk size to use for uploading (default 5Mi)
       --s3-copy-cutoff SizeSuffix                    Cutoff for switching to multipart copy (default 4.656Gi)
+      --s3-decompress                                If set this will decompress gzip encoded objects
       --s3-disable-checksum                          Don't store MD5 checksum with object metadata
       --s3-disable-http2                             Disable usage of http2 for S3 backends
       --s3-download-url string                       Custom endpoint for downloads
@@ -14711,6 +14887,7 @@ and may be set in the config file.
       --s3-no-check-bucket                           If set, don't attempt to check the bucket exists or create it
       --s3-no-head                                   If set, don't HEAD uploaded objects to check integrity
       --s3-no-head-object                            If set, do not do HEAD before GET when getting objects
+      --s3-no-system-metadata                        Suppress setting and reading of system metadata
       --s3-profile string                            Profile to use in the shared credentials file
       --s3-provider string                           Choose your S3 provider
       --s3-region string                             Region to connect to
@@ -14720,7 +14897,8 @@ and may be set in the config file.
       --s3-session-token string                      An AWS session token
       --s3-shared-credentials-file string            Path to the shared credentials file
       --s3-sse-customer-algorithm string             If using SSE-C, the server-side encryption algorithm used when storing this object in S3
-      --s3-sse-customer-key string                   If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data
+      --s3-sse-customer-key string                   To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data
+      --s3-sse-customer-key-base64 string            If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data
       --s3-sse-customer-key-md5 string               If using SSE-C you may provide the secret encryption key MD5 checksum (optional)
       --s3-sse-kms-key-id string                     If using KMS ID you must provide the ARN of Key
       --s3-storage-class string                      The storage class to use when storing new objects in S3
@@ -14730,6 +14908,8 @@ and may be set in the config file.
       --s3-use-multipart-etag Tristate               Whether to use ETag in multipart uploads for verification (default unset)
       --s3-use-presigned-request                     Whether to use a presigned request or PutObject for single part uploads
       --s3-v2-auth                                   If true use v2 authentication
+      --s3-version-at Time                           Show file versions as they were at the specified time (default off)
+      --s3-versions                                  Include old versions in directory listings
       --seafile-2fa                                  Two-factor authentication ('true' if the account has 2FA enabled)
       --seafile-create-library                       Should rclone create a library if it doesn't exist
       --seafile-encoding MultiEncoder                The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
@@ -14776,6 +14956,15 @@ and may be set in the config file.
       --sia-encoding MultiEncoder                    The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot)
       --sia-user-agent string                        Siad User Agent (default "Sia-Agent")
       --skip-links                                   Don't warn about skipped symlinks
+      --smb-case-insensitive                         Whether the server is configured to be case-insensitive (default true)
+      --smb-domain string                            Domain name for NTLM authentication (default "WORKGROUP")
+      --smb-encoding MultiEncoder                    The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot)
+      --smb-hide-special-share                       Hide special shares (e.g. print$) which users aren't supposed to access (default true)
+      --smb-host string                              SMB server hostname to connect to
+      --smb-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
+      --smb-pass string                              SMB password (obscured)
+      --smb-port int                                 SMB port number (default 445)
+      --smb-user string                              SMB username (default "$USER")
       --storj-access-grant string                    Access grant
       --storj-api-key string                         API key
       --storj-passphrase string                      Encryption passphrase
@@ -14806,6 +14995,7 @@ and may be set in the config file.
       --swift-key string                             API key or password (OS_PASSWORD)
       --swift-leave-parts-on-error                   If true avoid calling abort upload on a failure
       --swift-no-chunk                               Don't chunk files during streaming upload
+      --swift-no-large-objects                       Disable support for static and dynamic large objects
       --swift-region string                          Region name - optional (OS_REGION_NAME)
       --swift-storage-policy string                  The storage policy to use when creating a new container
       --swift-storage-url string                     Storage URL - optional (OS_STORAGE_URL)
@@ -15722,7 +15912,7 @@ Most of these events come up due to a error status from an internal call.
 On such a critical error the `{...}.path1.lst` and `{...}.path2.lst`
 listing files are renamed to extension `.lst-err`, which blocks any future
 bisync runs (since the normal `.lst` files are not found).
-Bisync keeps them under `bisync` subdirectory of the rclone cache direcory,
+Bisync keeps them under `bisync` subdirectory of the rclone cache directory,
 typically at `${HOME}/.cache/rclone/bisync/` on Linux.
 
 Some errors are considered temporary and re-running the bisync is not blocked.
@@ -15820,7 +16010,7 @@ don't have spelling case differences (`Smile.jpg` vs. `smile.jpg`).
 ## Windows support {#windows}
 
 Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on Windows
-Github runners.
+GitHub runners.
 
 Drive letters are allowed, including drive letters mapped to network drives
 (`rclone bisync J:\localsync GDrive:`).
@@ -16328,7 +16518,7 @@ test command flags can be equally prefixed by a single `-` or double dash.
   synched tree even if there are check file mismatches in the test tree.
 - Some Dropbox tests can fail, notably printing the following message:
   `src and dst identical but can't set mod time without deleting and re-uploading`
-  This is expected and happens due a way Dropbox handles modificaion times.
+  This is expected and happens due a way Dropbox handles modification times.
   You should use the `-refresh-times` test flag to make up for this.
 - If Dropbox tests hit request limit for you and print error message
   `too_many_requests/...: Too many requests or write operations.`
@@ -16338,7 +16528,7 @@ test command flags can be equally prefixed by a single `-` or double dash.
 ### Updating golden results
 
 Sometimes even a slight change in the bisync source can cause little changes
-spread around many log files. Updating them manually would be a nighmare.
+spread around many log files. Updating them manually would be a nightmare.
 
 The `-golden` flag will store the `test.log` and `*.lst` listings from each
 test case into respective golden directories. Golden results will
@@ -16699,6 +16889,11 @@ Invoking `rclone mkdir backup:../desktop` is exactly the same as invoking
 The empty path is not allowed as a remote. To alias the current directory
 use `.` instead.
 
+The target remote can also be a [connection string](https://rclone.org/docs/#connection-strings). 
+This can be used to modify the config of a remote for different uses, e.g.
+the alias  `myDriveTrash` with the target remote `myDrive,trashed_only:` 
+can be used to only show the trashed files in `myDrive`.
+
 ## Configuration
 
 Here is an example of how to make an alias called `remote` for local folder.
@@ -17133,7 +17328,9 @@ The S3 backend can be used with a number of different providers:
 - Huawei OBS
 - IBM COS S3
 - IDrive e2
+- IONOS Cloud
 - Minio
+- Qiniu Cloud Object Storage (Kodo)
 - RackCorp Object Storage
 - Scaleway
 - Seagate Lyve Cloud
@@ -17446,7 +17643,7 @@ upload.
 
 Rclone's default directory traversal is to process each directory
 individually.  This takes one API call per directory.  Using the
-`--fast-list` flag will read all info about the the objects into
+`--fast-list` flag will read all info about the objects into
 memory first using a smaller number of API calls (one per 1000
 objects). See the [rclone docs](https://rclone.org/docs/#fast-list) for more details.
 
@@ -17498,6 +17695,74 @@ This will mean that these objects do not have an MD5 checksum.
 Note that reading this from the object takes an additional `HEAD`
 request as the metadata isn't returned in object listings.
 
+### Versions
+
+When bucket versioning is enabled (this can be done with rclone with
+the [`rclone backend versioning`](#versioning) command) when rclone
+uploads a new version of a file it creates a
+[new version of it](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html)
+Likewise when you delete a file, the old version will be marked hidden
+and still be available.
+
+Old versions of files, where available, are visible using the
+[`--s3-versions`](#s3-versions) flag.
+
+It is also possible to view a bucket as it was at a certain point in
+time, using the [`--s3-version-at`](#s3-version-at) flag. This will
+show the file versions as they were at that time, showing files that
+have been deleted afterwards, and hiding files that were created
+since.
+
+If you wish to remove all the old versions then you can use the
+[`rclone backend cleanup-hidden remote:bucket`](#cleanup-hidden)
+command which will delete all the old hidden versions of files,
+leaving the current ones intact. You can also supply a path and only
+old versions under that path will be deleted, e.g.
+`rclone backend cleanup-hidden remote:bucket/path/to/stuff`.
+
+When you `purge` a bucket, the current and the old versions will be
+deleted then the bucket will be deleted.
+
+However `delete` will cause the current versions of the files to
+become hidden old versions.
+
+Here is a session showing the listing and retrieval of an old
+version followed by a `cleanup` of the old versions.
+
+Show current version and all the versions with `--s3-versions` flag.
+
+```
+$ rclone -q ls s3:cleanup-test
+        9 one.txt
+
+$ rclone -q --s3-versions ls s3:cleanup-test
+        9 one.txt
+        8 one-v2016-07-04-141032-000.txt
+       16 one-v2016-07-04-141003-000.txt
+       15 one-v2016-07-02-155621-000.txt
+```
+
+Retrieve an old version
+
+```
+$ rclone -q --s3-versions copy s3:cleanup-test/one-v2016-07-04-141003-000.txt /tmp
+
+$ ls -l /tmp/one-v2016-07-04-141003-000.txt
+-rw-rw-r-- 1 ncw ncw 16 Jul  2 17:46 /tmp/one-v2016-07-04-141003-000.txt
+```
+
+Clean up all the old versions and show that they've gone.
+
+```
+$ rclone -q backend cleanup-hidden s3:cleanup-test
+
+$ rclone -q ls s3:cleanup-test
+        9 one.txt
+
+$ rclone -q --s3-versions ls s3:cleanup-test
+        9 one.txt
+```
+
 ### Cleanup
 
 If you run `rclone cleanup s3:bucket` then it will remove all pending
@@ -17685,7 +17950,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t
 
 ### Standard options
 
-Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi).
+Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
 
 #### --s3-provider
 
@@ -17720,6 +17985,8 @@ Properties:
         - IBM COS S3
     - "IDrive"
         - IDrive e2
+    - "IONOS"
+        - IONOS Cloud
     - "LyveCloud"
         - Seagate Lyve Cloud
     - "Minio"
@@ -17740,6 +18007,8 @@ Properties:
         - Tencent Cloud Object Storage (COS)
     - "Wasabi"
         - Wasabi Object Storage
+    - "Qiniu"
+        - Qiniu Object Storage (Kodo)
     - "Other"
         - Any other S3 compatible provider
 
@@ -18010,13 +18279,68 @@ Properties:
 
 Region to connect to.
 
+Properties:
+
+- Config:      region
+- Env Var:     RCLONE_S3_REGION
+- Provider:    Qiniu
+- Type:        string
+- Required:    false
+- Examples:
+    - "cn-east-1"
+        - The default endpoint - a good choice if you are unsure.
+        - East China Region 1.
+        - Needs location constraint cn-east-1.
+    - "cn-east-2"
+        - East China Region 2.
+        - Needs location constraint cn-east-2.
+    - "cn-north-1"
+        - North China Region 1.
+        - Needs location constraint cn-north-1.
+    - "cn-south-1"
+        - South China Region 1.
+        - Needs location constraint cn-south-1.
+    - "us-north-1"
+        - North America Region.
+        - Needs location constraint us-north-1.
+    - "ap-southeast-1"
+        - Southeast Asia Region 1.
+        - Needs location constraint ap-southeast-1.
+    - "ap-northeast-1"
+        - Northeast Asia Region 1.
+        - Needs location constraint ap-northeast-1.
+
+#### --s3-region
+
+Region where your bucket will be created and your data stored.
+
+
+Properties:
+
+- Config:      region
+- Env Var:     RCLONE_S3_REGION
+- Provider:    IONOS
+- Type:        string
+- Required:    false
+- Examples:
+    - "de"
+        - Frankfurt, Germany
+    - "eu-central-2"
+        - Berlin, Germany
+    - "eu-south-2"
+        - Logrono, Spain
+
+#### --s3-region
+
+Region to connect to.
+
 Leave blank if you are using an S3 clone and you don't have a region.
 
 Properties:
 
 - Config:      region
 - Env Var:     RCLONE_S3_REGION
-- Provider:    !AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
+- Provider:    !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
 - Type:        string
 - Required:    false
 - Examples:
@@ -18274,6 +18598,27 @@ Properties:
 
 #### --s3-endpoint
 
+Endpoint for IONOS S3 Object Storage.
+
+Specify the endpoint from the same region.
+
+Properties:
+
+- Config:      endpoint
+- Env Var:     RCLONE_S3_ENDPOINT
+- Provider:    IONOS
+- Type:        string
+- Required:    false
+- Examples:
+    - "s3-eu-central-1.ionoscloud.com"
+        - Frankfurt, Germany
+    - "s3-eu-central-2.ionoscloud.com"
+        - Berlin, Germany
+    - "s3-eu-south-2.ionoscloud.com"
+        - Logrono, Spain
+
+#### --s3-endpoint
+
 Endpoint for OSS API.
 
 Properties:
@@ -18539,6 +18884,33 @@ Properties:
 
 #### --s3-endpoint
 
+Endpoint for Qiniu Object Storage.
+
+Properties:
+
+- Config:      endpoint
+- Env Var:     RCLONE_S3_ENDPOINT
+- Provider:    Qiniu
+- Type:        string
+- Required:    false
+- Examples:
+    - "s3-cn-east-1.qiniucs.com"
+        - East China Endpoint 1
+    - "s3-cn-east-2.qiniucs.com"
+        - East China Endpoint 2
+    - "s3-cn-north-1.qiniucs.com"
+        - North China Endpoint 1
+    - "s3-cn-south-1.qiniucs.com"
+        - South China Endpoint 1
+    - "s3-us-north-1.qiniucs.com"
+        - North America Endpoint 1
+    - "s3-ap-southeast-1.qiniucs.com"
+        - Southeast Asia Endpoint 1
+    - "s3-ap-northeast-1.qiniucs.com"
+        - Northeast Asia Endpoint 1
+
+#### --s3-endpoint
+
 Endpoint for S3 API.
 
 Required when using an S3 clone.
@@ -18547,7 +18919,7 @@ Properties:
 
 - Config:      endpoint
 - Env Var:     RCLONE_S3_ENDPOINT
-- Provider:    !AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp
+- Provider:    !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu
 - Type:        string
 - Required:    false
 - Examples:
@@ -18874,13 +19246,42 @@ Properties:
 
 Location constraint - must be set to match the Region.
 
+Used when creating buckets only.
+
+Properties:
+
+- Config:      location_constraint
+- Env Var:     RCLONE_S3_LOCATION_CONSTRAINT
+- Provider:    Qiniu
+- Type:        string
+- Required:    false
+- Examples:
+    - "cn-east-1"
+        - East China Region 1
+    - "cn-east-2"
+        - East China Region 2
+    - "cn-north-1"
+        - North China Region 1
+    - "cn-south-1"
+        - South China Region 1
+    - "us-north-1"
+        - North America Region 1
+    - "ap-southeast-1"
+        - Southeast Asia Region 1
+    - "ap-northeast-1"
+        - Northeast Asia Region 1
+
+#### --s3-location-constraint
+
+Location constraint - must be set to match the Region.
+
 Leave blank if not sure. Used when creating buckets only.
 
 Properties:
 
 - Config:      location_constraint
 - Env Var:     RCLONE_S3_LOCATION_CONSTRAINT
-- Provider:    !AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS
+- Provider:    !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS
 - Type:        string
 - Required:    false
 
@@ -19110,9 +19511,30 @@ Properties:
         - Archived storage.
         - Prices are lower, but it needs to be restored first to be accessed.
 
+#### --s3-storage-class
+
+The storage class to use when storing new objects in Qiniu.
+
+Properties:
+
+- Config:      storage_class
+- Env Var:     RCLONE_S3_STORAGE_CLASS
+- Provider:    Qiniu
+- Type:        string
+- Required:    false
+- Examples:
+    - "STANDARD"
+        - Standard storage class
+    - "LINE"
+        - Infrequent access storage mode
+    - "GLACIER"
+        - Archive storage mode
+    - "DEEP_ARCHIVE"
+        - Deep archive storage mode
+
 ### Advanced options
 
-Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi).
+Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
 
 #### --s3-bucket-acl
 
@@ -19175,7 +19597,9 @@ Properties:
 
 #### --s3-sse-customer-key
 
-If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.
+To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data.
+
+Alternatively you can provide --sse-customer-key-base64.
 
 Properties:
 
@@ -19188,6 +19612,23 @@ Properties:
     - ""
         - None
 
+#### --s3-sse-customer-key-base64
+
+If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data.
+
+Alternatively you can provide --sse-customer-key.
+
+Properties:
+
+- Config:      sse_customer_key_base64
+- Env Var:     RCLONE_S3_SSE_CUSTOMER_KEY_BASE64
+- Provider:    AWS,Ceph,ChinaMobile,Minio
+- Type:        string
+- Required:    false
+- Examples:
+    - ""
+        - None
+
 #### --s3-sse-customer-key-md5
 
 If using SSE-C you may provide the secret encryption key MD5 checksum (optional).
@@ -19676,6 +20117,67 @@ Properties:
 - Type:        bool
 - Default:     false
 
+#### --s3-versions
+
+Include old versions in directory listings.
+
+Properties:
+
+- Config:      versions
+- Env Var:     RCLONE_S3_VERSIONS
+- Type:        bool
+- Default:     false
+
+#### --s3-version-at
+
+Show file versions as they were at the specified time.
+
+The parameter should be a date, "2006-01-02", datetime "2006-01-02
+15:04:05" or a duration for that long ago, eg "100d" or "1h".
+
+Note that when using this no file write operations are permitted,
+so you can't upload files or delete them.
+
+See [the time option docs](https://rclone.org/docs/#time-option) for valid formats.
+
+
+Properties:
+
+- Config:      version_at
+- Env Var:     RCLONE_S3_VERSION_AT
+- Type:        Time
+- Default:     off
+
+#### --s3-decompress
+
+If set this will decompress gzip encoded objects.
+
+It is possible to upload objects to S3 with "Content-Encoding: gzip"
+set. Normally rclone will download these files as compressed objects.
+
+If this flag is set then rclone will decompress these files with
+"Content-Encoding: gzip" as they are received. This means that rclone
+can't check the size and hash but the file contents will be decompressed.
+
+
+Properties:
+
+- Config:      decompress
+- Env Var:     RCLONE_S3_DECOMPRESS
+- Type:        bool
+- Default:     false
+
+#### --s3-no-system-metadata
+
+Suppress setting and reading of system metadata
+
+Properties:
+
+- Config:      no_system_metadata
+- Env Var:     RCLONE_S3_NO_SYSTEM_METADATA
+- Type:        bool
+- Default:     false
+
 ### Metadata
 
 User metadata is stored as x-amz-meta- keys. S3 metadata keys are case insensitive and are always returned in lower case.
@@ -19818,6 +20320,39 @@ Options:
 
 - "max-age": Max age of upload to delete
 
+### cleanup-hidden
+
+Remove old versions of files.
+
+    rclone backend cleanup-hidden remote: [options] [+]
+
+This command removes any old hidden versions of files
+on a versions enabled bucket.
+
+Note that you can use -i/--dry-run with this command to see what it
+would do.
+
+    rclone backend cleanup-hidden s3:bucket/path/to/dir
+
+
+### versioning
+
+Set/get versioning support for a bucket.
+
+    rclone backend versioning remote: [options] [+]
+
+This command sets versioning support if a parameter is
+passed and then returns the current versioning status for the bucket
+supplied.
+
+    rclone backend versioning s3:bucket # read status only
+    rclone backend versioning s3:bucket Enabled
+    rclone backend versioning s3:bucket Suspended
+
+It may return "Enabled", "Suspended" or "Unversioned". Note that once versioning
+has been enabled the status can't be set back to "Unversioned".
+
+
 
 
 ### Anonymous access to public buckets
@@ -20515,6 +21050,169 @@ d) Delete this remote
 y/e/d> y
 ```
 
+### IONOS Cloud {#ionos}
+
+[IONOS S3 Object Storage](https://cloud.ionos.com/storage/object-storage) is a service offered by IONOS for storing and accessing unstructured data.
+To connect to the service, you will need an access key and a secret key. These can be found in the [Data Center Designer](https://dcd.ionos.com/), by selecting **Manager resources** > **Object Storage Key Manager**.
+
+
+Here is an example of a configuration. First, run `rclone config`. This will walk you through an interactive setup process. Type `n` to add the new remote, and then enter a name:
+
+```
+Enter name for new remote.
+name> ionos-fra
+```
+
+Type `s3` to choose the connection type:
+```
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+   \ (s3)
+[snip]
+Storage> s3
+```
+
+Type `IONOS`:
+```
+Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+[snip]
+XX / IONOS Cloud
+   \ (IONOS)
+[snip]
+provider> IONOS
+```
+
+Press Enter to choose the default option `Enter AWS credentials in the next step`:
+```
+Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth>
+```
+
+Enter your Access Key and Secret key. These can be retrieved in the [Data Center Designer](https://dcd.ionos.com/), click on the menu “Manager resources”  / "Object Storage Key Manager".
+```
+Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> YOUR_ACCESS_KEY
+
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> YOUR_SECRET_KEY
+```
+
+Choose the region where your bucket is located:
+```
+Option region.
+Region where your bucket will be created and your data stored.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / Frankfurt, Germany
+   \ (de)
+ 2 / Berlin, Germany
+   \ (eu-central-2)
+ 3 / Logrono, Spain
+   \ (eu-south-2)
+region> 2
+```
+
+Choose the endpoint from the same region:
+```
+Option endpoint.
+Endpoint for IONOS S3 Object Storage.
+Specify the endpoint from the same region.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / Frankfurt, Germany
+   \ (s3-eu-central-1.ionoscloud.com)
+ 2 / Berlin, Germany
+   \ (s3-eu-central-2.ionoscloud.com)
+ 3 / Logrono, Spain
+   \ (s3-eu-south-2.ionoscloud.com)
+endpoint> 1
+```
+
+Press Enter to choose the default option or choose the desired ACL setting:
+```
+Option acl.
+Canned ACL used when creating buckets and storing or copying objects.
+This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+   / Owner gets FULL_CONTROL.
+[snip]
+acl>
+```
+
+Press Enter to skip the advanced config:
+```
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n>
+```
+
+Press Enter to save the configuration, and then `q` to quit the configuration process:
+```
+Configuration complete.
+Options:
+- type: s3
+- provider: IONOS
+- access_key_id: YOUR_ACCESS_KEY
+- secret_access_key: YOUR_SECRET_KEY
+- endpoint: s3-eu-central-1.ionoscloud.com
+Keep this "ionos-fra" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Done! Now you can try some commands (for macOS, use `./rclone` instead of `rclone`).
+
+1)  Create a bucket (the name must be unique within the whole IONOS S3)
+```
+rclone mkdir ionos-fra:my-bucket
+```
+2)  List available buckets
+```
+rclone lsd ionos-fra:
+```
+4) Copy a file from local to remote
+```
+rclone copy /Users/file.txt ionos-fra:my-bucket
+```
+3)  List contents of a bucket
+```
+rclone ls ionos-fra:my-bucket
+```
+5)  Copy a file from remote to local
+```
+rclone copy ionos-fra:my-bucket/file.txt
+```
+
 ### Minio
 
 [Minio](https://minio.io/) is an object storage server built for cloud application developers and devops.
@@ -20582,6 +21280,207 @@ So once set up, for example, to copy files into a bucket
 rclone copy /path/to/files minio:bucket
 ```
 
+### Qiniu Cloud Object Storage (Kodo) {#qiniu}
+
+[Qiniu Cloud Object Storage (Kodo)](https://www.qiniu.com/en/products/kodo), a completely independent-researched core technology which is proven by repeated customer experience has occupied absolute leading market leader position. Kodo can be widely applied to mass data management.
+
+To configure access to Qiniu Kodo, follow the steps below:
+
+1. Run `rclone config` and select `n` for a new remote.
+
+```
+rclone config
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+```
+
+2. Give the name of the configuration. For example, name it 'qiniu'.
+
+```
+name> qiniu
+```
+
+3. Select `s3` storage.
+
+```
+Choose a number from below, or type in your own value
+ 1 / 1Fichier
+   \ (fichier)
+ 2 / Akamai NetStorage
+   \ (netstorage)
+ 3 / Alias for an existing remote
+   \ (alias)
+ 4 / Amazon Drive
+   \ (amazon cloud drive)
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi
+   \ (s3)
+[snip]
+Storage> s3
+```
+
+4. Select `Qiniu` provider.
+```
+Choose a number from below, or type in your own value
+1 / Amazon Web Services (AWS) S3
+   \ "AWS"
+[snip]
+22 / Qiniu Object Storage (Kodo)
+   \ (Qiniu)
+[snip]
+provider> Qiniu
+```
+
+5. Enter your SecretId and SecretKey of Qiniu Kodo.
+
+```
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Enter a boolean value (true or false). Press Enter for the default ("false").
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+access_key_id> AKIDxxxxxxxxxx
+AWS Secret Access Key (password)
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+secret_access_key> xxxxxxxxxxx
+```
+
+6. Select endpoint for Qiniu Kodo. This is the standard endpoint for different region.
+
+```
+   / The default endpoint - a good choice if you are unsure.
+ 1 | East China Region 1.
+   | Needs location constraint cn-east-1.
+   \ (cn-east-1)
+   / East China Region 2.
+ 2 | Needs location constraint cn-east-2.
+   \ (cn-east-2)
+   / North China Region 1.
+ 3 | Needs location constraint cn-north-1.
+   \ (cn-north-1)
+   / South China Region 1.
+ 4 | Needs location constraint cn-south-1.
+   \ (cn-south-1)
+   / North America Region.
+ 5 | Needs location constraint us-north-1.
+   \ (us-north-1)
+   / Southeast Asia Region 1.
+ 6 | Needs location constraint ap-southeast-1.
+   \ (ap-southeast-1)
+   / Northeast Asia Region 1.
+ 7 | Needs location constraint ap-northeast-1.
+   \ (ap-northeast-1)
+[snip]
+endpoint> 1
+
+Option endpoint.
+Endpoint for Qiniu Object Storage.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / East China Endpoint 1
+   \ (s3-cn-east-1.qiniucs.com)
+ 2 / East China Endpoint 2
+   \ (s3-cn-east-2.qiniucs.com)
+ 3 / North China Endpoint 1
+   \ (s3-cn-north-1.qiniucs.com)
+ 4 / South China Endpoint 1
+   \ (s3-cn-south-1.qiniucs.com)
+ 5 / North America Endpoint 1
+   \ (s3-us-north-1.qiniucs.com)
+ 6 / Southeast Asia Endpoint 1
+   \ (s3-ap-southeast-1.qiniucs.com)
+ 7 / Northeast Asia Endpoint 1
+   \ (s3-ap-northeast-1.qiniucs.com)
+endpoint> 1
+
+Option location_constraint.
+Location constraint - must be set to match the Region.
+Used when creating buckets only.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / East China Region 1
+   \ (cn-east-1)
+ 2 / East China Region 2
+   \ (cn-east-2)
+ 3 / North China Region 1
+   \ (cn-north-1)
+ 4 / South China Region 1
+   \ (cn-south-1)
+ 5 / North America Region 1
+   \ (us-north-1)
+ 6 / Southeast Asia Region 1
+   \ (ap-southeast-1)
+ 7 / Northeast Asia Region 1
+   \ (ap-northeast-1)
+location_constraint> 1
+```
+
+7. Choose acl and storage class.
+
+```
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+   / Owner gets FULL_CONTROL.
+ 2 | The AllUsers group gets READ access.
+   \ (public-read)
+[snip]
+acl> 2
+The storage class to use when storing new objects in Tencent COS.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 1 / Standard storage class
+   \ (STANDARD)
+ 2 / Infrequent access storage mode
+   \ (LINE)
+ 3 / Archive storage mode
+   \ (GLACIER)
+ 4 / Deep archive storage mode
+   \ (DEEP_ARCHIVE)
+[snip]
+storage_class> 1
+Edit advanced config? (y/n)
+y) Yes
+n) No (default)
+y/n> n
+Remote config
+--------------------
+[qiniu]
+- type: s3
+- provider: Qiniu
+- access_key_id: xxx
+- secret_access_key: xxx
+- region: cn-east-1
+- endpoint: s3-cn-east-1.qiniucs.com
+- location_constraint: cn-east-1
+- acl: public-read
+- storage_class: STANDARD
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+Current remotes:
+
+Name                 Type
+====                 ====
+qiniu                s3
+```
+
 ### RackCorp {#RackCorp}
 
 [RackCorp Object Storage](https://www.rackcorp.com/storage/s3storage) is an S3 compatible object storage platform from your friendly cloud provider RackCorp. 
@@ -24318,7 +25217,7 @@ If you intend to use the wrapped remote both directly for keeping
 unencrypted content, as well as through a crypt remote for encrypted
 content, it is recommended to point the crypt remote to a separate
 directory within the wrapped remote. If you use a bucket-based storage
-system (e.g. Swift, S3, Google Compute Storage, B2, Hubic) it is generally
+system (e.g. Swift, S3, Google Compute Storage, B2) it is generally
 advisable to wrap the crypt remote around a specific bucket (`s3:bucket`).
 If wrapping around the entire root of the storage (`s3:`), and use the
 optional file name encryption, rclone will encrypt the bucket name.
@@ -24334,7 +25233,7 @@ the password configured for an existing crypt remote means you will no longer
 able to decrypt any of the previously encrypted content. The only possibility
 is to re-upload everything via a crypt remote configured with your new password.
 
-Depending on the size of your data, your bandwith, storage quota etc, there are
+Depending on the size of your data, your bandwidth, storage quota etc, there are
 different approaches you can take:
 - If you have everything in a different location, for example on your local system,
 you could remove all of the prior encrypted files, change the password for your
@@ -24347,7 +25246,7 @@ effectively decrypting everything on the fly using the old password and
 re-encrypting using the new password. When done, delete the original crypt
 remote directory and finally the rclone crypt configuration with the old password.
 All data will be streamed from the storage system and back, so you will
-get half the bandwith and be charged twice if you have upload and download quota
+get half the bandwidth and be charged twice if you have upload and download quota
 on the storage system.
 
 **Note**: A security problem related to the random password generator
@@ -24660,7 +25559,7 @@ How to encode the encrypted filename to text string.
 
 This option could help with shortening the encrypted filename. The 
 suitable option would depend on the way your remote count the filename
-length and if it's case sensitve.
+length and if it's case sensitive.
 
 Properties:
 
@@ -24988,7 +25887,7 @@ Generally -1 (default, equivalent to 5) is recommended.
 Levels 1 to 9 increase compression at the cost of speed. Going past 6 
 generally offers very little return.
 
-Level -2 uses Huffmann encoding only. Only use if you know what you
+Level -2 uses Huffman encoding only. Only use if you know what you
 are doing.
 Level 0 turns off compression.
 
@@ -25136,7 +26035,7 @@ This would produce something like this:
 
     [AllDrives]
     type = combine
-    remote = "My Drive=My Drive:" "Test Drive=Test Drive:"
+    upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
 
 If you then add that config to your config file (find it with `rclone
 config file`) then you can access all the shared drives in one place
@@ -25587,7 +26486,7 @@ Properties:
 
 #### --dropbox-batch-commit-timeout
 
-Max time to wait for a batch to finish comitting
+Max time to wait for a batch to finish committing
 
 Properties:
 
@@ -25672,7 +26571,7 @@ through a global file system.
 ## Configuration
 
 The initial setup for the Enterprise File Fabric backend involves
-getting a token from the the Enterprise File Fabric which you need to
+getting a token from the Enterprise File Fabric which you need to
 do in your browser.  `rclone config` walks you through it.
 
 Here is an example of how to make a remote called `remote`.  First run:
@@ -25954,8 +26853,7 @@ To create an FTP configuration named `remote`, run
 
 Rclone config guides you through an interactive setup process. A minimal
 rclone FTP remote definition only requires host, username and password.
-For an anonymous FTP server, use `anonymous` as username and your email
-address as password.
+For an anonymous FTP server, see [below](#anonymous-ftp).
 
 ```
 No remotes found, make a new one?
@@ -26032,11 +26930,33 @@ excess files in the directory.
 
     rclone sync -i /home/local/directory remote:directory
 
-### Example without a config file ###
+### Anonymous FTP
+
+When connecting to a FTP server that allows anonymous login, you can use the
+special "anonymous" username. Traditionally, this user account accepts any
+string as a password, although it is common to use either the password
+"anonymous" or "guest". Some servers require the use of a valid e-mail
+address as password.
+
+Using [on-the-fly](#backend-path-to-dir) or
+[connection string](https://rclone.org/docs/#connection-strings) remotes makes it easy to access
+such servers, without requiring any configuration in advance. The following
+are examples of that:
 
-    rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=`rclone obscure dummy`
+    rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=$(rclone obscure dummy)
+    rclone lsf :ftp,host=speedtest.tele2.net,user=anonymous,pass=$(rclone obscure dummy):
 
-### Implicit TLS ###
+The above examples work in Linux shells and in PowerShell, but not Windows
+Command Prompt. They execute the [rclone obscure](https://rclone.org/commands/rclone_obscure/)
+command to create a password string in the format required by the
+[pass](#ftp-pass) option. The following examples are exactly the same, except use
+an already obscured string representation of the same password "dummy", and
+therefore works even in Windows Command Prompt:
+
+    rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=IXs2wc8OJOz7SYLBk47Ji1rHTmxM
+    rclone lsf :ftp,host=speedtest.tele2.net,user=anonymous,pass=IXs2wc8OJOz7SYLBk47Ji1rHTmxM:
+
+### Implicit TLS
 
 Rlone FTP supports implicit FTP over TLS servers (FTPS). This has to
 be enabled in the FTP backend config for the remote, or with
@@ -26048,7 +26968,7 @@ can be set with [`--ftp-port`](#ftp-port).
 In addition to the [default restricted characters set](https://rclone.org/overview/#restricted-characters)
 the following characters are also replaced:
 
-File names cannot end with the following characters. Repacement is
+File names cannot end with the following characters. Replacement is
 limited to the last character in a file name:
 
 | Character | Value | Replacement |
@@ -26158,6 +27078,20 @@ Here are the Advanced options specific to ftp (FTP).
 
 Maximum number of FTP simultaneous connections, 0 for unlimited.
 
+Note that setting this is very likely to cause deadlocks so it should
+be used with care.
+
+If you are doing a sync or copy then make sure concurrency is one more
+than the sum of `--transfers` and `--checkers`.
+
+If you use `--check-first` then it just needs to be one more than the
+maximum of `--checkers` and `--transfers`.
+
+So for `concurrency 3` you'd use `--checkers 2 --transfers 2
+--check-first` or `--checkers 1 --transfers 1`.
+
+
+
 Properties:
 
 - Config:      concurrency
@@ -26220,6 +27154,17 @@ Properties:
 - Type:        bool
 - Default:     false
 
+#### --ftp-force-list-hidden
+
+Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD.
+
+Properties:
+
+- Config:      force_list_hidden
+- Env Var:     RCLONE_FTP_FORCE_LIST_HIDDEN
+- Type:        bool
+- Default:     false
+
 #### --ftp-idle-timeout
 
 Max time before closing idle connections.
@@ -26976,7 +27921,7 @@ Properties:
 If set this will decompress gzip encoded objects.
 
 It is possible to upload objects to GCS with "Content-Encoding: gzip"
-set. Normally rclone will download these files files as compressed objects.
+set. Normally rclone will download these files as compressed objects.
 
 If this flag is set then rclone will decompress these files with
 "Content-Encoding: gzip" as they are received. This means that rclone
@@ -26990,6 +27935,19 @@ Properties:
 - Type:        bool
 - Default:     false
 
+#### --gcs-endpoint
+
+Endpoint for the service.
+
+Leave blank normally.
+
+Properties:
+
+- Config:      endpoint
+- Env Var:     RCLONE_GCS_ENDPOINT
+- Type:        string
+- Required:    false
+
 #### --gcs-encoding
 
 The encoding for the backend.
@@ -28343,10 +29301,10 @@ drives found and a combined drive.
 
     [AllDrives]
     type = combine
-    remote = "My Drive=My Drive:" "Test Drive=Test Drive:"
+    upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
 
 Adding this to the rclone config file will cause those team drives to
-be accessible with the aliases shown. Any illegal charactes will be
+be accessible with the aliases shown. Any illegal characters will be
 substituted with "_" and duplicate names will have numbers suffixed.
 It will also add a remote called AllDrives which shows all the shared
 drives combined into one directory tree.
@@ -29716,7 +30674,7 @@ the process is very similar to the process of initial setup exemplified before.
 HiDrive allows modification times to be set on objects accurate to 1 second.
 
 HiDrive supports [its own hash type](https://static.hidrive.com/dev/0001)
-which is used to verify the integrety of file contents after successful transfers.
+which is used to verify the integrity of file contents after successful transfers.
 
 ### Restricted filename characters
 
@@ -30252,240 +31210,6 @@ remote.
 
 See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
 
-#  Hubic
-
-Paths are specified as `remote:path`
-
-Paths are specified as `remote:container` (or `remote:` for the `lsd`
-command.)  You may put subdirectories in too, e.g. `remote:container/path/to/dir`.
-
-## Configuration
-
-The initial setup for Hubic involves getting a token from Hubic which
-you need to do in your browser.  `rclone config` walks you through it.
-
-Here is an example of how to make a remote called `remote`.  First run:
-
-     rclone config
-
-This will guide you through an interactive setup process:
-
-```
-n) New remote
-s) Set configuration password
-n/s> n
-name> remote
-Type of storage to configure.
-Choose a number from below, or type in your own value
-[snip]
-XX / Hubic
-   \ "hubic"
-[snip]
-Storage> hubic
-Hubic Client Id - leave blank normally.
-client_id>
-Hubic Client Secret - leave blank normally.
-client_secret>
-Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
-y) Yes
-n) No
-y/n> y
-If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
-Log in and authorize rclone for access
-Waiting for code...
-Got code
---------------------
-[remote]
-client_id =
-client_secret =
-token = {"access_token":"XXXXXX"}
---------------------
-y) Yes this is OK
-e) Edit this remote
-d) Delete this remote
-y/e/d> y
-```
-
-See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a
-machine with no Internet browser available.
-
-Note that rclone runs a webserver on your local machine to collect the
-token as returned from Hubic. This only runs from the moment it opens
-your browser to the moment you get back the verification code.  This
-is on `http://127.0.0.1:53682/` and this it may require you to unblock
-it temporarily if you are running a host firewall.
-
-Once configured you can then use `rclone` like this,
-
-List containers in the top level of your Hubic
-
-    rclone lsd remote:
-
-List all the files in your Hubic
-
-    rclone ls remote:
-
-To copy a local directory to an Hubic directory called backup
-
-    rclone copy /home/source remote:backup
-
-If you want the directory to be visible in the official *Hubic
-browser*, you need to copy your files to the `default` directory
-
-    rclone copy /home/source remote:default/backup
-
-### --fast-list ###
-
-This remote supports `--fast-list` which allows you to use fewer
-transactions in exchange for more memory. See the [rclone
-docs](https://rclone.org/docs/#fast-list) for more details.
-
-### Modified time ###
-
-The modified time is stored as metadata on the object as
-`X-Object-Meta-Mtime` as floating point since the epoch accurate to 1
-ns.
-
-This is a de facto standard (used in the official python-swiftclient
-amongst others) for storing the modification time for an object.
-
-Note that Hubic wraps the Swift backend, so most of the properties of
-are the same.
-
-
-### Standard options
-
-Here are the Standard options specific to hubic (Hubic).
-
-#### --hubic-client-id
-
-OAuth Client Id.
-
-Leave blank normally.
-
-Properties:
-
-- Config:      client_id
-- Env Var:     RCLONE_HUBIC_CLIENT_ID
-- Type:        string
-- Required:    false
-
-#### --hubic-client-secret
-
-OAuth Client Secret.
-
-Leave blank normally.
-
-Properties:
-
-- Config:      client_secret
-- Env Var:     RCLONE_HUBIC_CLIENT_SECRET
-- Type:        string
-- Required:    false
-
-### Advanced options
-
-Here are the Advanced options specific to hubic (Hubic).
-
-#### --hubic-token
-
-OAuth Access Token as a JSON blob.
-
-Properties:
-
-- Config:      token
-- Env Var:     RCLONE_HUBIC_TOKEN
-- Type:        string
-- Required:    false
-
-#### --hubic-auth-url
-
-Auth server URL.
-
-Leave blank to use the provider defaults.
-
-Properties:
-
-- Config:      auth_url
-- Env Var:     RCLONE_HUBIC_AUTH_URL
-- Type:        string
-- Required:    false
-
-#### --hubic-token-url
-
-Token server url.
-
-Leave blank to use the provider defaults.
-
-Properties:
-
-- Config:      token_url
-- Env Var:     RCLONE_HUBIC_TOKEN_URL
-- Type:        string
-- Required:    false
-
-#### --hubic-chunk-size
-
-Above this size files will be chunked into a _segments container.
-
-Above this size files will be chunked into a _segments container.  The
-default for this is 5 GiB which is its maximum value.
-
-Properties:
-
-- Config:      chunk_size
-- Env Var:     RCLONE_HUBIC_CHUNK_SIZE
-- Type:        SizeSuffix
-- Default:     5Gi
-
-#### --hubic-no-chunk
-
-Don't chunk files during streaming upload.
-
-When doing streaming uploads (e.g. using rcat or mount) setting this
-flag will cause the swift backend to not upload chunked files.
-
-This will limit the maximum upload size to 5 GiB. However non chunked
-files are easier to deal with and have an MD5SUM.
-
-Rclone will still chunk files bigger than chunk_size when doing normal
-copy operations.
-
-Properties:
-
-- Config:      no_chunk
-- Env Var:     RCLONE_HUBIC_NO_CHUNK
-- Type:        bool
-- Default:     false
-
-#### --hubic-encoding
-
-The encoding for the backend.
-
-See the [encoding section in the overview](https://rclone.org/overview/#encoding) for more info.
-
-Properties:
-
-- Config:      encoding
-- Env Var:     RCLONE_HUBIC_ENCODING
-- Type:        MultiEncoder
-- Default:     Slash,InvalidUtf8
-
-
-
-## Limitations
-
-This uses the normal OpenStack Swift mechanism to refresh the Swift
-API credentials and ignores the expires field returned by the Hubic
-API.
-
-The Swift API doesn't return a correct MD5SUM for segmented files
-(Dynamic or Static Large Objects) so rclone won't check or use the
-MD5SUM for these.
-
 #  Internet Archive
 
 The Internet Archive backend utilizes Items on [archive.org](https://archive.org/)
@@ -30495,11 +31219,10 @@ Refer to [IAS3 API documentation](https://archive.org/services/docs/api/ias3.htm
 Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
 command.)  You may put subdirectories in too, e.g. `remote:item/path/to/dir`.
 
-Once you have made a remote (see the provider specific section above)
-you can use it like this:
-
 Unlike S3, listing up all items uploaded by you isn't supported.
 
+Once you have made a remote, you can use it like this:
+
 Make a new item
 
     rclone mkdir remote:item
@@ -30536,6 +31259,7 @@ The following are reserved by Internet Archive:
 - `format`
 - `old_version`
 - `viruscheck`
+- `summation`
 
 Trying to set values to these keys is ignored with a warning.
 Only setting `mtime` is an exception. Doing so make it the identical behavior as setting ModTime.
@@ -30741,19 +31465,20 @@ Here are the possible system metadata items for the internetarchive backend.
 
 | Name | Help | Type | Example | Read Only |
 |------|------|------|---------|-----------|
-| crc32 | CRC32 calculated by Internet Archive | string | 01234567 | N |
-| format | Name of format identified by Internet Archive | string | Comma-Separated Values | N |
-| md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | N |
-| mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N |
-| name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | N |
-| old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | N |
+| crc32 | CRC32 calculated by Internet Archive | string | 01234567 | **Y** |
+| format | Name of format identified by Internet Archive | string | Comma-Separated Values | **Y** |
+| md5 | MD5 hash calculated by Internet Archive | string | 01234567012345670123456701234567 | **Y** |
+| mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | **Y** |
+| name | Full file path, without the bucket part | filename | backend/internetarchive/internetarchive.go | **Y** |
+| old_version | Whether the file was replaced and moved by keep-old-version flag | boolean | true | **Y** |
 | rclone-ia-mtime | Time of last modification, managed by Internet Archive | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N |
 | rclone-mtime | Time of last modification, managed by Rclone | RFC 3339 | 2006-01-02T15:04:05.999999999Z | N |
 | rclone-update-track | Random value used by Rclone for tracking changes inside Internet Archive | string | aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa | N |
-| sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | N |
-| size | File size in bytes | decimal number | 123456 | N |
-| source | The source of the file | string | original | N |
-| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | N |
+| sha1 | SHA1 hash calculated by Internet Archive | string | 0123456701234567012345670123456701234567 | **Y** |
+| size | File size in bytes | decimal number | 123456 | **Y** |
+| source | The source of the file | string | original | **Y** |
+| summation | Check https://forum.rclone.org/t/31922 for how it is used | string | md5 | **Y** |
+| viruscheck | The last time viruscheck process was run for the file (?) | unixtime | 1654191352 | **Y** |
 
 See the [metadata](https://rclone.org/docs/#metadata) docs for more info.
 
@@ -30774,7 +31499,7 @@ it also provides white-label solutions to different companies, such as:
   * Elgiganten Sweden (cloud.elgiganten.se)
   * Elgiganten Denmark (cloud.elgiganten.dk)
   * Giganti Cloud  (cloud.gigantti.fi)
-  * ELKO Clouud (cloud.elko.is)
+  * ELKO Cloud (cloud.elko.is)
 
 Most of the white-label versions are supported by this backend, although may require different
 authentication setup - described below.
@@ -30790,10 +31515,33 @@ and you have to choose the correct one when setting up the remote.
 
 ### Standard authentication
 
-To configure Jottacloud you will need to generate a personal security token in the Jottacloud web interface.
-You will the option to do in your [account security settings](https://www.jottacloud.com/web/secure)
-(for whitelabel version you need to find this page in its web interface).
-Note that the web interface may refer to this token as a JottaCli token.
+The standard authentication method used by the official service (jottacloud.com), as well as
+some of the whitelabel services, requires you to generate a single-use personal login token
+from the account security settings in the service's web interface. Log in to your account,
+go to "Settings" and then "Security", or use the direct link presented to you by rclone when
+configuring the remote: . Scroll down to the section
+"Personal login token", and click the "Generate" button. Note that if you are using a
+whitelabel service you probably can't use the direct link, you need to find the same page in
+their dedicated web interface, and also it may be in a different location than described above.
+
+To access your account from multiple instances of rclone, you need to configure each of them
+with a separate personal login token. E.g. you create a Jottacloud remote with rclone in one
+location, and copy the configuration file to a second location where you also want to run
+rclone and access the same remote. Then you need to replace the token for one of them, using
+the [config reconnect](https://rclone.org/commands/rclone_config_reconnect/) command, which
+requires you to generate a new personal login token and supply as input. If you do not
+do this, the token may easily end up being invalidated, resulting in both instances failing
+with an error message something along the lines of:
+
+    oauth2: cannot fetch token: 400 Bad Request
+    Response: {"error":"invalid_grant","error_description":"Stale token"}
+
+When this happens, you need to replace the token as described above to be able to use your
+remote again.
+
+All personal login tokens you have taken into use will be listed in the web interface under
+"My logged in devices", and from the right side of that list you can click the "X" button to
+revoke individual tokens.
 
 ### Legacy authentication
 
@@ -31947,7 +32695,7 @@ Use `rclone dedupe` to fix duplicated files.
 #### Object not found
 
 If you are connecting to your Mega remote for the first time, 
-to test access and syncronisation, you may receive an error such as 
+to test access and synchronization, you may receive an error such as 
 
 ```
 Failed to create file system for "my-mega-remote:": 
@@ -32323,7 +33071,7 @@ Individual symlink files on the remote can be used with the commands like "cat"
 With NetStorage, directories can exist in one of two forms:
 
 1. **Explicit Directory**. This is an actual, physical directory that you have created in a storage group.
-2. **Implicit Directory**. This refers to a directory within a path that has not been physically created. For example, during upload of a file, non-existent subdirectories can be specified in the target path. NetStorage creates these as "implicit." While the directories aren't physically created, they exist implicitly and the noted path is connected with the uploaded file.
+2. **Implicit Directory**. This refers to a directory within a path that has not been physically created. For example, during upload of a file, nonexistent subdirectories can be specified in the target path. NetStorage creates these as "implicit." While the directories aren't physically created, they exist implicitly and the noted path is connected with the uploaded file.
 
 Rclone will intercept all file uploads and mkdir commands for the NetStorage remote and will explicitly issue the mkdir command for each directory in the uploading path. This will help with the interoperability with the other Akamai services such as SFTP and the Content Management Shell (CMShell). Rclone will not guarantee correctness of operations with implicit directories which might have been created as a result of using an upload API directly.
 
@@ -33100,7 +33848,7 @@ rclone uses a default Client ID when talking to OneDrive, unless a custom `clien
 The default Client ID and Key are shared by all rclone users when performing requests.
 
 You may choose to create and use your own Client ID, in case the default one does not work well for you. 
-For example, you might see throtting.
+For example, you might see throttling.
 
 #### Creating Client ID for OneDrive Personal
 
@@ -33128,7 +33876,7 @@ A common error is that the publisher of the App is not verified.
 You may try to [verify you account](https://docs.microsoft.com/en-us/azure/active-directory/develop/publisher-verification-overview), or try to limit the App to your organization only, as shown below.
 
 1. Make sure to create the App with your business account.
-2. Follow the steps above to create an App. However, we need a different account type here: `Accounts in this organizational directory only (*** - Single tenant)`. Note that you can also change the account type aftering creating the App.
+2. Follow the steps above to create an App. However, we need a different account type here: `Accounts in this organizational directory only (*** - Single tenant)`. Note that you can also change the account type after creating the App.
 3. Find the [tenant ID](https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/active-directory-how-to-find-tenant) of your organization.
 4. In the rclone config, set `auth_url` to `https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/authorize`.
 5. In the rclone config, set `token_url` to `https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/token`.
@@ -33239,7 +33987,7 @@ Properties:
     - "de"
         - Microsoft Cloud Germany
     - "cn"
-        - Azure and Office 365 operated by 21Vianet in China
+        - Azure and Office 365 operated by Vnet Group in China
 
 ### Advanced options
 
@@ -33544,7 +34292,7 @@ An official document about the limitations for different types of OneDrive can b
 ## Versions
 
 Every change in a file OneDrive causes the service to create a new
-version of the the file.  This counts against a users quota.  For
+version of the file.  This counts against a users quota.  For
 example changing the modification time of a file creates a second
 version, so the file apparently uses twice the space.
 
@@ -33865,6 +34613,535 @@ remote.
 
 See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
 
+#  Oracle Object Storage
+
+[Oracle Object Storage Overview](https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/objectstorageoverview.htm)
+
+[Oracle Object Storage FAQ](https://www.oracle.com/cloud/storage/object-storage/faq/)
+
+Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
+command.)  You may put subdirectories in too, e.g. `remote:bucket/path/to/dir`.
+
+## Configuration
+
+Here is an example of making an oracle object storage configuration. `rclone config` walks you 
+through it.
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+
+```
+n) New remote
+d) Delete remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+e/n/d/r/c/s/q> n
+
+Enter name for new remote.
+name> remote
+
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+XX / Oracle Cloud Infrastructure Object Storage
+   \ (oracleobjectstorage)
+Storage> oracleobjectstorage
+
+Option provider.
+Choose your Auth Provider
+Choose a number from below, or type in your own string value.
+Press Enter for the default (env_auth).
+ 1 / automatically pickup the credentials from runtime(env), first one to provide auth wins
+   \ (env_auth)
+   / use an OCI user and an API key for authentication.
+ 2 | you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key.
+   | https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm
+   \ (user_principal_auth)
+   / use instance principals to authorize an instance to make API calls. 
+ 3 | each instance has its own identity, and authenticates using the certificates that are read from instance metadata. 
+   | https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm
+   \ (instance_principal_auth)
+ 4 / use resource principals to make API calls
+   \ (resource_principal_auth)
+ 5 / no credentials needed, this is typically for reading public buckets
+   \ (no_auth)
+provider> 2
+
+Option namespace.
+Object storage namespace
+Enter a value.
+namespace> idbamagbg734
+
+Option compartment.
+Object storage compartment OCID
+Enter a value.
+compartment> ocid1.compartment.oc1..aaaaaaaapufkxc7ame3sthry5i7ujrwfc7ejnthhu6bhanm5oqfjpyasjkba
+
+Option region.
+Object storage Region
+Enter a value.
+region> us-ashburn-1
+
+Option endpoint.
+Endpoint for Object storage API.
+Leave blank to use the default endpoint for the region.
+Enter a value. Press Enter to leave empty.
+endpoint> 
+
+Option config_file.
+Path to OCI config file
+Choose a number from below, or type in your own string value.
+Press Enter for the default (~/.oci/config).
+ 1 / oci configuration file location
+   \ (~/.oci/config)
+config_file> /etc/oci/dev.conf
+
+Option config_profile.
+Profile name inside OCI config file
+Choose a number from below, or type in your own string value.
+Press Enter for the default (Default).
+ 1 / Use the default profile
+   \ (Default)
+config_profile> Test
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: oracleobjectstorage
+- namespace: idbamagbg734
+- compartment: ocid1.compartment.oc1..aaaaaaaapufkxc7ame3sthry5i7ujrwfc7ejnthhu6bhanm5oqfjpyasjkba
+- region: us-ashburn-1
+- provider: user_principal_auth
+- config_file: /etc/oci/dev.conf
+- config_profile: Test
+Keep this "remote" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+See all buckets
+
+    rclone lsd remote:
+
+Create a new bucket
+
+    rclone mkdir remote:bucket
+
+List the contents of a bucket
+
+    rclone ls remote:bucket
+    rclone ls remote:bucket --max-depth 1
+
+### Modified time
+
+The modified time is stored as metadata on the object as
+`opc-meta-mtime` as floating point since the epoch, accurate to 1 ns.
+
+If the modification time needs to be updated rclone will attempt to perform a server
+side copy to update the modification if the object can be copied in a single part.
+In the case the object is larger than 5Gb, the object will be uploaded rather than copied.
+
+Note that reading this from the object takes an additional `HEAD` request as the metadata
+isn't returned in object listings.
+
+### Multipart uploads
+
+rclone supports multipart uploads with OOS which means that it can
+upload files bigger than 5 GiB.
+
+Note that files uploaded *both* with multipart upload *and* through
+crypt remotes do not have MD5 sums.
+
+rclone switches from single part uploads to multipart uploads at the
+point specified by `--oos-upload-cutoff`.  This can be a maximum of 5 GiB
+and a minimum of 0 (ie always upload multipart files).
+
+The chunk sizes used in the multipart upload are specified by
+`--oos-chunk-size` and the number of chunks uploaded concurrently is
+specified by `--oos-upload-concurrency`.
+
+Multipart uploads will use `--transfers` * `--oos-upload-concurrency` *
+`--oos-chunk-size` extra memory.  Single part uploads to not use extra
+memory.
+
+Single part transfers can be faster than multipart transfers or slower
+depending on your latency from oos - the more latency, the more likely
+single part transfers will be faster.
+
+Increasing `--oos-upload-concurrency` will increase throughput (8 would
+be a sensible value) and increasing `--oos-chunk-size` also increases
+throughput (16M would be sensible).  Increasing either of these will
+use more memory.  The default values are high enough to gain most of
+the possible performance without using too much memory.
+
+
+### Standard options
+
+Here are the Standard options specific to oracleobjectstorage (Oracle Cloud Infrastructure Object Storage).
+
+#### --oos-provider
+
+Choose your Auth Provider
+
+Properties:
+
+- Config:      provider
+- Env Var:     RCLONE_OOS_PROVIDER
+- Type:        string
+- Default:     "env_auth"
+- Examples:
+    - "env_auth"
+        - automatically pickup the credentials from runtime(env), first one to provide auth wins
+    - "user_principal_auth"
+        - use an OCI user and an API key for authentication.
+        - you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key.
+        - https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm
+    - "instance_principal_auth"
+        - use instance principals to authorize an instance to make API calls. 
+        - each instance has its own identity, and authenticates using the certificates that are read from instance metadata. 
+        - https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm
+    - "resource_principal_auth"
+        - use resource principals to make API calls
+    - "no_auth"
+        - no credentials needed, this is typically for reading public buckets
+
+#### --oos-namespace
+
+Object storage namespace
+
+Properties:
+
+- Config:      namespace
+- Env Var:     RCLONE_OOS_NAMESPACE
+- Type:        string
+- Required:    true
+
+#### --oos-compartment
+
+Object storage compartment OCID
+
+Properties:
+
+- Config:      compartment
+- Env Var:     RCLONE_OOS_COMPARTMENT
+- Provider:    !no_auth
+- Type:        string
+- Required:    true
+
+#### --oos-region
+
+Object storage Region
+
+Properties:
+
+- Config:      region
+- Env Var:     RCLONE_OOS_REGION
+- Type:        string
+- Required:    true
+
+#### --oos-endpoint
+
+Endpoint for Object storage API.
+
+Leave blank to use the default endpoint for the region.
+
+Properties:
+
+- Config:      endpoint
+- Env Var:     RCLONE_OOS_ENDPOINT
+- Type:        string
+- Required:    false
+
+#### --oos-config-file
+
+Path to OCI config file
+
+Properties:
+
+- Config:      config_file
+- Env Var:     RCLONE_OOS_CONFIG_FILE
+- Provider:    user_principal_auth
+- Type:        string
+- Default:     "~/.oci/config"
+- Examples:
+    - "~/.oci/config"
+        - oci configuration file location
+
+#### --oos-config-profile
+
+Profile name inside the oci config file
+
+Properties:
+
+- Config:      config_profile
+- Env Var:     RCLONE_OOS_CONFIG_PROFILE
+- Provider:    user_principal_auth
+- Type:        string
+- Default:     "Default"
+- Examples:
+    - "Default"
+        - Use the default profile
+
+### Advanced options
+
+Here are the Advanced options specific to oracleobjectstorage (Oracle Cloud Infrastructure Object Storage).
+
+#### --oos-upload-cutoff
+
+Cutoff for switching to chunked upload.
+
+Any files larger than this will be uploaded in chunks of chunk_size.
+The minimum is 0 and the maximum is 5 GiB.
+
+Properties:
+
+- Config:      upload_cutoff
+- Env Var:     RCLONE_OOS_UPLOAD_CUTOFF
+- Type:        SizeSuffix
+- Default:     200Mi
+
+#### --oos-chunk-size
+
+Chunk size to use for uploading.
+
+When uploading files larger than upload_cutoff or files with unknown
+size (e.g. from "rclone rcat" or uploaded with "rclone mount" or google
+photos or google docs) they will be uploaded as multipart uploads
+using this chunk size.
+
+Note that "upload_concurrency" chunks of this size are buffered
+in memory per transfer.
+
+If you are transferring large files over high-speed links and you have
+enough memory, then increasing this will speed up the transfers.
+
+Rclone will automatically increase the chunk size when uploading a
+large file of known size to stay below the 10,000 chunks limit.
+
+Files of unknown size are uploaded with the configured
+chunk_size. Since the default chunk size is 5 MiB and there can be at
+most 10,000 chunks, this means that by default the maximum size of
+a file you can stream upload is 48 GiB.  If you wish to stream upload
+larger files then you will need to increase chunk_size.
+
+Increasing the chunk size decreases the accuracy of the progress
+statistics displayed with "-P" flag.
+
+
+Properties:
+
+- Config:      chunk_size
+- Env Var:     RCLONE_OOS_CHUNK_SIZE
+- Type:        SizeSuffix
+- Default:     5Mi
+
+#### --oos-upload-concurrency
+
+Concurrency for multipart uploads.
+
+This is the number of chunks of the same file that are uploaded
+concurrently.
+
+If you are uploading small numbers of large files over high-speed links
+and these uploads do not fully utilize your bandwidth, then increasing
+this may help to speed up the transfers.
+
+Properties:
+
+- Config:      upload_concurrency
+- Env Var:     RCLONE_OOS_UPLOAD_CONCURRENCY
+- Type:        int
+- Default:     10
+
+#### --oos-copy-cutoff
+
+Cutoff for switching to multipart copy.
+
+Any files larger than this that need to be server-side copied will be
+copied in chunks of this size.
+
+The minimum is 0 and the maximum is 5 GiB.
+
+Properties:
+
+- Config:      copy_cutoff
+- Env Var:     RCLONE_OOS_COPY_CUTOFF
+- Type:        SizeSuffix
+- Default:     4.656Gi
+
+#### --oos-copy-timeout
+
+Timeout for copy.
+
+Copy is an asynchronous operation, specify timeout to wait for copy to succeed
+
+
+Properties:
+
+- Config:      copy_timeout
+- Env Var:     RCLONE_OOS_COPY_TIMEOUT
+- Type:        Duration
+- Default:     1m0s
+
+#### --oos-disable-checksum
+
+Don't store MD5 checksum with object metadata.
+
+Normally rclone will calculate the MD5 checksum of the input before
+uploading it so it can add it to metadata on the object. This is great
+for data integrity checking but can cause long delays for large files
+to start uploading.
+
+Properties:
+
+- Config:      disable_checksum
+- Env Var:     RCLONE_OOS_DISABLE_CHECKSUM
+- Type:        bool
+- Default:     false
+
+#### --oos-encoding
+
+The encoding for the backend.
+
+See the [encoding section in the overview](https://rclone.org/overview/#encoding) for more info.
+
+Properties:
+
+- Config:      encoding
+- Env Var:     RCLONE_OOS_ENCODING
+- Type:        MultiEncoder
+- Default:     Slash,InvalidUtf8,Dot
+
+#### --oos-leave-parts-on-error
+
+If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery.
+
+It should be set to true for resuming uploads across different sessions.
+
+WARNING: Storing parts of an incomplete multipart upload counts towards space usage on object storage and will add
+additional costs if not cleaned up.
+
+
+Properties:
+
+- Config:      leave_parts_on_error
+- Env Var:     RCLONE_OOS_LEAVE_PARTS_ON_ERROR
+- Type:        bool
+- Default:     false
+
+#### --oos-no-check-bucket
+
+If set, don't attempt to check the bucket exists or create it.
+
+This can be useful when trying to minimise the number of transactions
+rclone does if you know the bucket exists already.
+
+It can also be needed if the user you are using does not have bucket
+creation permissions.
+
+
+Properties:
+
+- Config:      no_check_bucket
+- Env Var:     RCLONE_OOS_NO_CHECK_BUCKET
+- Type:        bool
+- Default:     false
+
+## Backend commands
+
+Here are the commands specific to the oracleobjectstorage backend.
+
+Run them with
+
+    rclone backend COMMAND remote:
+
+The help below will explain what arguments each command takes.
+
+See the [backend](https://rclone.org/commands/rclone_backend/) command for more
+info on how to pass options and arguments.
+
+These can be run on a running backend using the rc command
+[backend/command](https://rclone.org/rc/#backend-command).
+
+### rename
+
+change the name of an object
+
+    rclone backend rename remote: [options] [+]
+
+This command can be used to rename a object.
+
+Usage Examples:
+
+    rclone backend rename oos:bucket relative-object-path-under-bucket object-new-name
+
+
+### list-multipart-uploads
+
+List the unfinished multipart uploads
+
+    rclone backend list-multipart-uploads remote: [options] [+]
+
+This command lists the unfinished multipart uploads in JSON format.
+
+    rclone backend list-multipart-uploads oos:bucket/path/to/object
+
+It returns a dictionary of buckets with values as lists of unfinished
+multipart uploads.
+
+You can call it with no bucket in which case it lists all bucket, with
+a bucket or with a bucket and path.
+
+    {
+      "test-bucket": [
+                {
+                        "namespace": "test-namespace",
+                        "bucket": "test-bucket",
+                        "object": "600m.bin",
+                        "uploadId": "51dd8114-52a4-b2f2-c42f-5291f05eb3c8",
+                        "timeCreated": "2022-07-29T06:21:16.595Z",
+                        "storageTier": "Standard"
+                }
+        ]
+
+
+### cleanup
+
+Remove unfinished multipart uploads.
+
+    rclone backend cleanup remote: [options] [+]
+
+This command removes unfinished multipart uploads of age greater than
+max-age which defaults to 24 hours.
+
+Note that you can use -i/--dry-run with this command to see what it
+would do.
+
+    rclone backend cleanup oos:bucket/path/to/object
+    rclone backend cleanup -o max-age=7w oos:bucket/path/to/object
+
+Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
+
+
+Options:
+
+- "max-age": Max age of upload to delete
+
+
+
 #  QingStor
 
 Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
@@ -34392,7 +35669,7 @@ Commercial implementations of that being:
   * [Rackspace Cloud Files](https://www.rackspace.com/cloud/files/)
   * [Memset Memstore](https://www.memset.com/cloud/storage/)
   * [OVH Object Storage](https://www.ovh.co.uk/public-cloud/storage/object-storage/)
-  * [Oracle Cloud Storage](https://cloud.oracle.com/object-storage/buckets)
+  * [Oracle Cloud Storage](https://docs.oracle.com/en-us/iaas/integration/doc/configure-object-storage.html)
   * [IBM Bluemix Cloud ObjectStorage Swift](https://console.bluemix.net/docs/infrastructure/objectstorage-swift/index.html)
 
 Paths are specified as `remote:container` (or `remote:` for the `lsd`
@@ -34915,6 +36192,38 @@ Properties:
 - Type:        bool
 - Default:     false
 
+#### --swift-no-large-objects
+
+Disable support for static and dynamic large objects
+
+Swift cannot transparently store files bigger than 5 GiB. There are
+two schemes for doing that, static or dynamic large objects, and the
+API does not allow rclone to determine whether a file is a static or
+dynamic large object without doing a HEAD on the object. Since these
+need to be treated differently, this means rclone has to issue HEAD
+requests for objects for example when reading checksums.
+
+When `no_large_objects` is set, rclone will assume that there are no
+static or dynamic large objects stored. This means it can stop doing
+the extra HEAD calls which in turn increases performance greatly
+especially when doing a swift to swift transfer with `--checksum` set.
+
+Setting this option implies `no_chunk` and also that no files will be
+uploaded in chunks, so files bigger than 5 GiB will just fail on
+upload.
+
+If you set this option and there *are* static or dynamic large objects,
+then this will give incorrect hashes for them. Downloads will succeed,
+but other operations such as Remove and Copy will fail.
+
+
+Properties:
+
+- Config:      no_large_objects
+- Env Var:     RCLONE_SWIFT_NO_LARGE_OBJECTS
+- Type:        bool
+- Default:     false
+
 #### --swift-encoding
 
 The encoding for the backend.
@@ -35938,7 +37247,7 @@ SSH installations.
 Paths are specified as `remote:path`. If the path does not begin with
 a `/` it is relative to the home directory of the user.  An empty path
 `remote:` refers to the user's home directory. For example, `rclone lsd remote:` 
-would list the home directory of the user cofigured in the rclone remote config 
+would list the home directory of the user configured in the rclone remote config 
 (`i.e /home/sftpuser`). However, `rclone lsd remote:/` would list the root 
 directory for remote machine (i.e. `/`)
 
@@ -36181,7 +37490,7 @@ can also run a SSH server, which is a port of OpenSSH (see official
 [installation guide](https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse)). On a Windows server the shell handling is different: Although it can also
 be set up to use a Unix type shell, e.g. Cygwin bash, the default is to
 use Windows Command Prompt (cmd.exe), and PowerShell is a recommended
-alternative. All of these have bahave differently, which rclone must handle.
+alternative. All of these have behave differently, which rclone must handle.
 
 Rclone tries to auto-detect what type of shell is used on the server,
 first time you access the SFTP remote. If a remote shell session is
@@ -36213,7 +37522,7 @@ a new sftp remote is accessed. If you configure a sftp remote
 without a config file, e.g. an [on the fly](https://rclone.org/docs/#backend-path-to-dir])
 remote, rclone will have nowhere to store the result, and it
 will re-run the command on every access. To avoid this you should
-explicitely set the `shell_type` option to the correct value,
+explicitly set the `shell_type` option to the correct value,
 or to `none` if you want to prevent rclone from executing any
 remote shell commands.
 
@@ -36221,7 +37530,7 @@ It is also important to note that, since the shell type decides
 how quoting and escaping of file paths used as command-line arguments
 are performed, configuring the wrong shell type may leave you exposed
 to command injection exploits. Make sure to confirm the auto-detected
-shell type, or explicitely set the shell type you know is correct,
+shell type, or explicitly set the shell type you know is correct,
 or disable shell access until you know.
 
 ### Checksum
@@ -36706,19 +38015,24 @@ Properties:
 
 Upload and download chunk size.
 
-This controls the maximum packet size used in the SFTP protocol. The
-RFC limits this to 32768 bytes (32k), however a lot of servers
-support larger sizes and setting it larger will increase transfer
-speed dramatically on high latency links.
-
-Only use a setting higher than 32k if you always connect to the same
-server or after sufficiently broad testing.
-
-For example using the value of 252k with OpenSSH works well with its
-maximum packet size of 256k.
+This controls the maximum size of payload in SFTP protocol packets.
+The RFC limits this to 32768 bytes (32k), which is the default. However,
+a lot of servers support larger sizes, typically limited to a maximum
+total package size of 256k, and setting it larger will increase transfer
+speed dramatically on high latency links. This includes OpenSSH, and,
+for example, using the value of 255k works well, leaving plenty of room
+for overhead while still being within a total packet size of 256k.
 
-If you get the error "failed to send packet header: EOF" when copying
-a large file, try lowering this number.
+Make sure to test thoroughly before using a value higher than 32k,
+and only use it if you always connect to the same server or after
+sufficiently broad testing. If you get errors such as
+"failed to send packet payload: EOF", lots of "connection lost",
+or "corrupted on transfer", when copying a larger file, try lowering
+the value. The server run by [rclone serve sftp](/commands/rclone_serve_sftp)
+sends packets with standard 32k maximum payload so you must not
+set a different chunk_size when downloading files, but it accepts
+packets up to the 256k total size, so for uploads the chunk_size
+can be set as for the OpenSSH example above.
 
 
 Properties:
@@ -36808,6 +38122,233 @@ Hetzner Storage Boxes are supported through the SFTP backend on port 23.
 
 See [Hetzner's documentation for details](https://docs.hetzner.com/robot/storage-box/access/access-ssh-rsync-borg#rclone)
 
+#  SMB
+
+SMB is [a communication protocol to share files over network](https://en.wikipedia.org/wiki/Server_Message_Block).
+
+This relies on [go-smb2 library](https://github.com/hirochachacha/go-smb2/) for communication with SMB protocol.
+
+Paths are specified as `remote:sharename` (or `remote:` for the `lsd`
+command.)  You may put subdirectories in too, e.g. `remote:item/path/to/dir`.
+
+## Notes
+
+The first path segment must be the name of the share, which you entered when you started to share on Windows. On smbd, it's the section title in `smb.conf` (usually in `/etc/samba/`) file.
+You can find shares by quering the root if you're unsure (e.g. `rclone lsd remote:`).
+
+You can't access to the shared printers from rclone, obviously.
+
+You can't use Anonymous access for logging in. You have to use the `guest` user with an empty password instead.
+The rclone client tries to avoid 8.3 names when uploading files by encoding trailing spaces and periods.
+Alternatively, [the local backend](https://rclone.org/local/#paths-on-windows) on Windows can access SMB servers using UNC paths, by `\\server\share`. This doesn't apply to non-Windows OSes, such as Linux and macOS.
+
+## Configuration
+
+Here is an example of making a SMB configuration.
+
+First run
+
+    rclone config
+
+This will guide you through an interactive setup process.
+
+```
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+XX / SMB / CIFS
+   \ (smb)
+Storage> smb
+
+Option host.
+Samba hostname to connect to.
+E.g. "example.com".
+Enter a value.
+host> localhost
+
+Option user.
+Samba username.
+Enter a string value. Press Enter for the default (lesmi).
+user> guest
+
+Option port.
+Samba port number.
+Enter a signed integer. Press Enter for the default (445).
+port> 
+
+Option pass.
+Samba password.
+Choose an alternative below. Press Enter for the default (n).
+y) Yes, type in my own password
+g) Generate random password
+n) No, leave this optional password blank (default)
+y/g/n> g
+Password strength in bits.
+64 is just about memorable
+128 is secure
+1024 is the maximum
+Bits> 64
+Your password is: XXXX
+Use this password? Please note that an obscured version of this 
+password (and not the password itself) will be stored under your 
+configuration file, so keep this generated password in a safe place.
+y) Yes (default)
+n) No
+y/n> y
+
+Option domain.
+Domain name for NTLM authentication.
+Enter a string value. Press Enter for the default (WORKGROUP).
+domain> 
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: samba
+- host: localhost
+- user: guest
+- pass: *** ENCRYPTED ***
+Keep this "remote" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> d
+```
+
+
+### Standard options
+
+Here are the Standard options specific to smb (SMB / CIFS).
+
+#### --smb-host
+
+SMB server hostname to connect to.
+
+E.g. "example.com".
+
+Properties:
+
+- Config:      host
+- Env Var:     RCLONE_SMB_HOST
+- Type:        string
+- Required:    true
+
+#### --smb-user
+
+SMB username.
+
+Properties:
+
+- Config:      user
+- Env Var:     RCLONE_SMB_USER
+- Type:        string
+- Default:     "$USER"
+
+#### --smb-port
+
+SMB port number.
+
+Properties:
+
+- Config:      port
+- Env Var:     RCLONE_SMB_PORT
+- Type:        int
+- Default:     445
+
+#### --smb-pass
+
+SMB password.
+
+**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
+
+Properties:
+
+- Config:      pass
+- Env Var:     RCLONE_SMB_PASS
+- Type:        string
+- Required:    false
+
+#### --smb-domain
+
+Domain name for NTLM authentication.
+
+Properties:
+
+- Config:      domain
+- Env Var:     RCLONE_SMB_DOMAIN
+- Type:        string
+- Default:     "WORKGROUP"
+
+### Advanced options
+
+Here are the Advanced options specific to smb (SMB / CIFS).
+
+#### --smb-idle-timeout
+
+Max time before closing idle connections.
+
+If no connections have been returned to the connection pool in the time
+given, rclone will empty the connection pool.
+
+Set to 0 to keep connections indefinitely.
+
+
+Properties:
+
+- Config:      idle_timeout
+- Env Var:     RCLONE_SMB_IDLE_TIMEOUT
+- Type:        Duration
+- Default:     1m0s
+
+#### --smb-hide-special-share
+
+Hide special shares (e.g. print$) which users aren't supposed to access.
+
+Properties:
+
+- Config:      hide_special_share
+- Env Var:     RCLONE_SMB_HIDE_SPECIAL_SHARE
+- Type:        bool
+- Default:     true
+
+#### --smb-case-insensitive
+
+Whether the server is configured to be case-insensitive.
+
+Always true on Windows shares.
+
+Properties:
+
+- Config:      case_insensitive
+- Env Var:     RCLONE_SMB_CASE_INSENSITIVE
+- Type:        bool
+- Default:     true
+
+#### --smb-encoding
+
+The encoding for the backend.
+
+See the [encoding section in the overview](https://rclone.org/overview/#encoding) for more info.
+
+Properties:
+
+- Config:      encoding
+- Env Var:     RCLONE_SMB_ENCODING
+- Type:        MultiEncoder
+- Default:     Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot
+
+
+
 #  Storj
 
 [Storj](https://storj.io) is an encrypted, secure, and 
@@ -39413,6 +40954,134 @@ Options:
 
 # Changelog
 
+## v1.60.0 - 2022-10-21
+
+[See commits](https://github.com/rclone/rclone/compare/v1.59.0...v1.60.0)
+
+* New backends
+    * [Oracle object storage](https://rclone.org/oracleobjectstorage/) (Manoj Ghosh)
+    * [SMB](https://rclone.org/smb/) / CIFS (Windows file sharing) (Lesmiscore)
+    * New S3 providers
+        * [IONOS Cloud Storage](https://rclone.org/s3/#ionos) (Dmitry Deniskin)
+        * [Qiniu KODO](https://rclone.org/s3/#qiniu) (Bachue Zhou)
+* New Features
+    * build
+        * Update to go1.19 and make go1.17 the minimum required version (Nick Craig-Wood)
+        * Install.sh: fix arm-v7 download (Ole Frost)
+    * fs: Warn the user when using an existing remote name without a colon (Nick Craig-Wood)
+    * httplib: Add `--xxx-min-tls-version` option to select minimum TLS version for HTTP servers (Robert Newson)
+    * librclone: Add PHP bindings and test program (Jordi Gonzalez Muñoz)
+    * operations
+        * Add `--server-side-across-configs` global flag for any backend (Nick Craig-Wood)
+        * Optimise `--copy-dest` and `--compare-dest` (Nick Craig-Wood)
+    * rc: add `job/stopgroup` to stop group (Evan Spensley)
+    * serve dlna
+        * Add `--announce-interval` to control SSDP Announce Interval (YanceyChiew)
+        * Add `--interface` to Specify SSDP interface names line (Simon Bos)
+        * Add support for more external subtitles (YanceyChiew)
+        * Add verification of addresses (YanceyChiew)
+    * sync: Optimise `--copy-dest` and `--compare-dest` (Nick Craig-Wood)
+    * doc updates (albertony, Alexander Knorr, anonion, João Henrique Franco, Josh Soref, Lorenzo Milesi, Marco Molteni, Mark Trolley, Ole Frost, partev, Ryan Morey, Tom Mombourquette, YFdyh000)
+* Bug Fixes
+    * filter
+        * Fix incorrect filtering with `UseFilter` context flag and wrapping backends (Nick Craig-Wood)
+        * Make sure we check `--files-from` when looking for a single file (Nick Craig-Wood)
+    * rc
+        * Fix `mount/listmounts` not returning the full Fs entered in `mount/mount` (Tom Mombourquette)
+        * Handle external unmount when mounting (Isaac Aymerich)
+        * Validate Daemon option is not set when mounting a volume via RC (Isaac Aymerich)
+    * sync: Update docs and error messages to reflect fixes to overlap checks (Nick Naumann)
+* VFS
+    * Reduce memory use by embedding `sync.Cond` (Nick Craig-Wood)
+    * Reduce memory usage by re-ordering commonly used structures (Nick Craig-Wood)
+    * Fix excess CPU used by VFS cache cleaner looping (Nick Craig-Wood)
+* Local
+    * Obey file filters in listing to fix errors on excluded files (Nick Craig-Wood)
+    * Fix "Failed to read metadata: function not implemented" on old Linux kernels (Nick Craig-Wood)
+* Compress
+    * Fix crash due to nil metadata (Nick Craig-Wood)
+    * Fix error handling to not use or return nil objects (Nick Craig-Wood)
+* Drive
+    * Make `--drive-stop-on-upload-limit` obey quota exceeded error (Steve Kowalik)
+* FTP
+    * Add `--ftp-force-list-hidden` option to show hidden items (Øyvind Heddeland Instefjord)
+    * Fix hang when using ExplicitTLS to certain servers. (Nick Craig-Wood)
+* Google Cloud Storage
+    * Add `--gcs-endpoint` flag and config parameter (Nick Craig-Wood)
+* Hubic
+    * Remove backend as service has now shut down (Nick Craig-Wood)
+* Onedrive
+    * Rename Onedrive(cn) 21Vianet to Vnet Group (Yen Hu)
+    * Disable change notify in China region since it is not supported (Nick Craig-Wood)
+* S3
+    * Implement `--s3-versions` flag to show old versions of objects if enabled (Nick Craig-Wood)
+    * Implement `--s3-version-at` flag to show versions of objects at a particular time (Nick Craig-Wood)
+    * Implement `backend versioning` command to get/set bucket versioning (Nick Craig-Wood)
+    * Implement `Purge` to purge versions and `backend cleanup-hidden` (Nick Craig-Wood)
+    * Add `--s3-decompress` flag to decompress gzip-encoded files (Nick Craig-Wood)
+    * Add `--s3-sse-customer-key-base64` to supply keys with binary data (Richard Bateman)
+    * Try to keep the maximum precision in ModTime with `--user-server-modtime` (Nick Craig-Wood)
+    * Drop binary metadata with an ERROR message as it can't be stored (Nick Craig-Wood)
+    * Add `--s3-no-system-metadata` to suppress read and write of system metadata (Nick Craig-Wood)
+* SFTP
+    * Fix directory creation races (Lesmiscore)
+* Swift
+    * Add `--swift-no-large-objects` to reduce HEAD requests (Nick Craig-Wood)
+* Union
+    * Propagate SlowHash feature to fix hasher interaction (Lesmiscore)
+
+## v1.59.2 - 2022-09-15
+
+[See commits](https://github.com/rclone/rclone/compare/v1.59.1...v1.59.2)
+
+* Bug Fixes
+    * config: Move locking to fix fatal error: concurrent map read and map write (Nick Craig-Wood)
+* Local
+    * Disable xattr support if the filesystems indicates it is not supported (Nick Craig-Wood)
+* Azure Blob
+    * Fix chunksize calculations producing too many parts (Nick Craig-Wood)
+* B2
+    * Fix chunksize calculations producing too many parts (Nick Craig-Wood)
+* S3
+    * Fix chunksize calculations producing too many parts (Nick Craig-Wood)
+
+## v1.59.1 - 2022-08-08
+
+[See commits](https://github.com/rclone/rclone/compare/v1.59.0...v1.59.1)
+
+* Bug Fixes
+    * accounting: Fix panic in core/stats-reset with unknown group (Nick Craig-Wood)
+    * build: Fix android build after GitHub actions change (Nick Craig-Wood)
+    * dlna: Fix SOAP action header parsing (Joram Schrijver)
+    * docs: Fix links to mount command from install docs (albertony)
+    * dropbox: Fix ChangeNotify was unable to decrypt errors (Nick Craig-Wood)
+    * fs: Fix parsing of times and durations of the form "YYYY-MM-DD HH:MM:SS" (Nick Craig-Wood)
+    * serve sftp: Fix checksum detection (Nick Craig-Wood)
+    * sync: Add accidentally missed filter-sensitivity to --backup-dir option (Nick Naumann)
+* Combine
+    * Fix docs showing `remote=` instead of `upstreams=` (Nick Craig-Wood)
+    * Throw error if duplicate directory name is specified (Nick Craig-Wood)
+    * Fix errors with backends shutting down while in use (Nick Craig-Wood)
+* Dropbox
+    * Fix hang on quit with --dropbox-batch-mode off (Nick Craig-Wood)
+    * Fix infinite loop on uploading a corrupted file (Nick Craig-Wood)
+* Internetarchive
+    * Ignore checksums for files using the different method (Lesmiscore)
+    * Handle hash symbol in the middle of filename (Lesmiscore)
+* Jottacloud
+    * Fix working with whitelabel Elgiganten Cloud
+    * Do not store username in config when using standard auth (albertony)
+* Mega
+    * Fix nil pointer exception when bad node received (Nick Craig-Wood)
+* S3
+    * Fix --s3-no-head panic: reflect: Elem of invalid type s3.PutObjectInput (Nick Craig-Wood)
+* SFTP
+    * Fix issue with WS_FTP by working around failing RealPath (albertony)
+* Union
+    * Fix duplicated files when using directories with leading / (Nick Craig-Wood)
+    * Fix multiple files being uploaded when roots don't exist (Nick Craig-Wood)
+    * Fix panic due to misalignment of struct field in 32 bit architectures (r-ricci)
+
 ## v1.59.0 - 2022-07-09
 
 [See commits](https://github.com/rclone/rclone/compare/v1.58.0...v1.59.0)
@@ -39645,7 +41314,7 @@ Options:
     * build
         * Fix ARM architecture version in .deb packages after nfpm change (Nick Craig-Wood)
         * Hard fork `github.com/jlaffaye/ftp` to fix `go get github.com/rclone/rclone` (Nick Craig-Wood)
-    * oauthutil: Fix crash when webrowser requests `/robots.txt` (Nick Craig-Wood)
+    * oauthutil: Fix crash when webbrowser requests `/robots.txt` (Nick Craig-Wood)
     * operations: Fix goroutine leak in case of copy retry (Ankur Gupta)
     * rc:
         * Fix `operations/publiclink` default for `expires` parameter (Nick Craig-Wood)
@@ -39731,7 +41400,7 @@ Options:
     * Add rclone to list of supported `md5sum`/`sha1sum` commands to look for (albertony)
     * Refactor so we only have one way of running remote commands (Nick Craig-Wood)
     * Fix timeout on hashing large files by sending keepalives (Nick Craig-Wood)
-    * Fix unecessary seeking when uploading and downloading files (Nick Craig-Wood)
+    * Fix unnecessary seeking when uploading and downloading files (Nick Craig-Wood)
     * Update docs on how to create `known_hosts` file (Nick Craig-Wood)
 * Storj
     * Rename tardigrade backend to storj backend (Nick Craig-Wood)
@@ -40332,8 +42001,8 @@ Options:
         * Add sort by average size in directory (Adam Plánský)
         * Add toggle option for average s3ize in directory - key 'a' (Adam Plánský)
         * Add empty folder flag into ncdu browser (Adam Plánský)
-        * Add `!` (errror) and `.` (unreadable) file flags to go with `e` (empty) (Nick Craig-Wood)
-    * obscure: Make `rclone osbcure -` ignore newline at end of line (Nick Craig-Wood)
+        * Add `!` (error) and `.` (unreadable) file flags to go with `e` (empty) (Nick Craig-Wood)
+    * obscure: Make `rclone obscure -` ignore newline at end of line (Nick Craig-Wood)
     * operations
         * Add logs when need to upload files to set mod times (Nick Craig-Wood)
         * Move and copy log name of the destination object in verbose (Adam Plánský)
@@ -40358,7 +42027,7 @@ Options:
         * Make the error count match up in the log message (Nick Craig-Wood)
     * move: Fix data loss when source and destination are the same object (Nick Craig-Wood)
     * operations
-        * Fix `--cutof-mode` hard not cutting off immediately (Nick Craig-Wood)
+        * Fix `--cutoff-mode` hard not cutting off immediately (Nick Craig-Wood)
         * Fix `--immutable` error message (Nick Craig-Wood)
     * sync
         * Fix `--cutoff-mode` soft & cautious so it doesn't end the transfer early (Nick Craig-Wood)
@@ -40406,7 +42075,7 @@ Options:
     * Fixed crash on an empty file name (lluuaapp)
 * Box
     * Fix NewObject for files that differ in case (Nick Craig-Wood)
-    * Fix finding directories in a case insentive way (Nick Craig-Wood)
+    * Fix finding directories in a case insensitive way (Nick Craig-Wood)
 * Chunker
     * Skip long local hashing, hash in-transit (fixes) (Ivan Andreev)
     * Set Features ReadMimeType to false as Object.MimeType not supported (Nick Craig-Wood)
@@ -40487,7 +42156,7 @@ Options:
     * Implement `--sftp-use-fstat` for unusual SFTP servers (Nick Craig-Wood)
 * Sugarsync
     * Fix NewObject for files that differ in case (Nick Craig-Wood)
-    * Fix finding directories in a case insentive way (Nick Craig-Wood)
+    * Fix finding directories in a case insensitive way (Nick Craig-Wood)
 * Swift
     * Fix deletion of parts of Static Large Object (SLO) (Nguyễn Hữu Luân)
     * Ensure partially uploaded large files are uploaded unless `--swift-leave-parts-on-error` (Nguyễn Hữu Luân)
@@ -40561,7 +42230,7 @@ Options:
 [See commits](https://github.com/rclone/rclone/compare/v1.53.1...v1.53.2)
 
 * Bug Fixes
-    * acounting
+    * accounting
         * Fix incorrect speed and transferTime in core/stats (Nick Craig-Wood)
         * Stabilize display order of transfers on Windows (Nick Craig-Wood)
     * operations
@@ -41531,7 +43200,7 @@ all the docs and Edward Barker for helping re-write the front page.
     * rcat: Fix slowdown on systems with multiple hashes (Nick Craig-Wood)
     * rcd: Fix permissions problems on cache directory with web gui download (Nick Craig-Wood)
 * Mount
-    * Default `--daemon-timout` to 15 minutes on macOS and FreeBSD (Nick Craig-Wood)
+    * Default `--daemon-timeout` to 15 minutes on macOS and FreeBSD (Nick Craig-Wood)
     * Update docs to show mounting from root OK for bucket-based (Nick Craig-Wood)
     * Remove nonseekable flag from write files (Nick Craig-Wood)
 * VFS
@@ -41839,7 +43508,7 @@ all the docs and Edward Barker for helping re-write the front page.
     * Update google cloud storage endpoints (weetmuts)
 * HTTP
     * Add an example with username and password which is supported but wasn't documented (Nick Craig-Wood)
-    * Fix backend with `--files-from` and non-existent files (Nick Craig-Wood)
+    * Fix backend with `--files-from` and nonexistent files (Nick Craig-Wood)
 * Hubic
     * Make error message more informative if authentication fails (Nick Craig-Wood)
 * Jottacloud
@@ -42323,7 +43992,7 @@ Point release to fix hubic and azureblob backends.
 * FTP
     * Work around strange response from box FTP server
     * More workarounds for FTP servers to fix mkParentDir error
-    * Fix no error on listing non-existent directory
+    * Fix no error on listing nonexistent directory
 * Google Cloud Storage
     * Add service_account_credentials (Matt Holt)
     * Detect bucket presence by listing it - minimises permissions needed
@@ -42396,7 +44065,7 @@ Point release to fix hubic and azureblob backends.
     * Add .deb and .rpm packages as part of the build
     * Make a beta release for all branches on the main repo (but not pull requests)
 * Bug Fixes
-    * config: fixes errors on non existing config by loading config file only on first access
+    * config: fixes errors on nonexistent config by loading config file only on first access
     * config: retry saving the config after failure (Mateusz)
     * sync: when using `--backup-dir` don't delete files if we can't set their modtime
         * this fixes odd behaviour with Dropbox and `--backup-dir`
@@ -42931,7 +44600,7 @@ Point release to fix hubic and azureblob backends.
     * Update B2 docs with Data usage, and Crypt section - thanks Tomasz Mazur
 * S3
     * Command line and config file support for
-        * Setting/overriding ACL  - thanks Radek Senfeld
+        * Setting/overriding ACL  - thanks Radek Šenfeld
         * Setting storage class - thanks Asko Tamm
 * Drive
     * Make exponential backoff work exactly as per Google specification
@@ -44364,6 +46033,30 @@ put them back in again.` >}}
   * Lorenzo Maiorfi 
   * Claudio Maradonna 
   * Ovidiu Victor Tatar 
+  * Evan Spensley 
+  * Yen Hu <61753151+0x59656e@users.noreply.github.com>
+  * Steve Kowalik 
+  * Jordi Gonzalez Muñoz 
+  * Joram Schrijver 
+  * Mark Trolley 
+  * João Henrique Franco 
+  * anonion 
+  * Ryan Morey <4590343+rmorey@users.noreply.github.com>
+  * Simon Bos 
+  * YFdyh000   * Josh Soref <2119212+jsoref@users.noreply.github.com>
+  * Øyvind Heddeland Instefjord 
+  * Dmitry Deniskin <110819396+ddeniskin@users.noreply.github.com>
+  * Alexander Knorr <106825+opexxx@users.noreply.github.com>
+  * Richard Bateman 
+  * Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com>
+  * Lorenzo Milesi 
+  * Isaac Aymerich 
+  * YanceyChiew <35898533+YanceyChiew@users.noreply.github.com>
+  * Manoj Ghosh 
+  * Bachue Zhou 
+  * Manoj Ghosh 
+  * Tom Mombourquette 
+  * Robert Newson 
 
 # Contact the rclone project #
 
diff --git a/MANUAL.txt b/MANUAL.txt
index d8828a0656d31..6cdd6ee0a3a09 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -1,6 +1,6 @@
 rclone(1) User Manual
 Nick Craig-Wood
-Jul 09, 2022
+Oct 21, 2022
 
 Rclone syncs your files to cloud storage
 
@@ -101,7 +101,6 @@ S3, that work out of the box.)
 -   China Mobile Ecloud Elastic Object Storage (EOS)
 -   Arvan Cloud Object Storage (AOS)
 -   Citrix ShareFile
--   C14
 -   Cloudflare R2
 -   DigitalOcean Spaces
 -   Digi Storage
@@ -116,11 +115,11 @@ S3, that work out of the box.)
 -   Hetzner Storage Box
 -   HiDrive
 -   HTTP
--   Hubic
 -   Internet Archive
 -   Jottacloud
 -   IBM COS S3
 -   IDrive e2
+-   IONOS Cloud
 -   Koofr
 -   Mail.ru Cloud
 -   Memset Memstore
@@ -133,12 +132,14 @@ S3, that work out of the box.)
 -   OVH
 -   OpenDrive
 -   OpenStack Swift
--   Oracle Cloud Storage
+-   Oracle Cloud Storage Swift
+-   Oracle Object Storage
 -   ownCloud
 -   pCloud
 -   premiumize.me
 -   put.io
 -   QingStor
+-   Qiniu Cloud Object Storage (Kodo)
 -   Rackspace Cloud Files
 -   rsync.net
 -   Scaleway
@@ -147,6 +148,7 @@ S3, that work out of the box.)
 -   SeaweedFS
 -   SFTP
 -   Sia
+-   SMB / CIFS
 -   StackPath
 -   Storj
 -   SugarSync
@@ -190,7 +192,7 @@ Quickstart
 -   Run rclone config to setup. See rclone config docs for more details.
 -   Optionally configure automatic execution.
 
-See below for some expanded Linux / macOS instructions.
+See below for some expanded Linux / macOS / Windows instructions.
 
 See the usage docs for how to use rclone, or run rclone -h.
 
@@ -210,7 +212,9 @@ For beta installation, run:
 Note that this script checks the version of rclone installed first and
 won't re-download if not needed.
 
-Linux installation from precompiled binary
+Linux installation
+
+Precompiled binary
 
 Fetch and unpack
 
@@ -234,7 +238,9 @@ Run rclone config to setup. See rclone config docs for more details.
 
     rclone config
 
-macOS installation with brew
+macOS installation
+
+Installation with brew
 
     brew install rclone
 
@@ -242,7 +248,12 @@ NOTE: This version of rclone will not support mount any more (see
 #5373). If mounting is wanted on macOS, either install a precompiled
 binary or enable the relevant option when installing from source.
 
-macOS installation from precompiled binary, using curl
+Note that this is a third party installer not controlled by the rclone
+developers so it may be out of date. Its current version is as below.
+
+[Homebrew package]
+
+Precompiled binary, using curl
 
 To avoid problems with macOS gatekeeper enforcing the binary to be
 signed and notarized it is enough to download with curl.
@@ -271,7 +282,7 @@ Run rclone config to setup. See rclone config docs for more details.
 
     rclone config
 
-macOS installation from precompiled binary, using a web browser
+Precompiled binary, using a web browser
 
 When downloading a binary with a web browser, the browser will set the
 macOS gatekeeper quarantine attribute. Starting from Catalina, when
@@ -284,11 +295,69 @@ The simplest fix is to run
 
     xattr -d com.apple.quarantine rclone
 
-Install with docker
+Windows installation
+
+Precompiled binary
+
+Fetch the correct binary for your processor type by clicking on these
+links. If not sure, use the first link.
+
+-   Intel/AMD - 64 Bit
+-   Intel/AMD - 32 Bit
+-   ARM - 64 Bit
+
+Open this file in the Explorer and extract rclone.exe. Rclone is a
+portable executable so you can place it wherever is convenient.
+
+Open a CMD window (or powershell) and run the binary. Note that rclone
+does not launch a GUI by default, it runs in the CMD Window.
+
+-   Run rclone.exe config to setup. See rclone config docs for more
+    details.
+-   Optionally configure automatic execution.
+
+If you are planning to use the rclone mount feature then you will need
+to install the third party utility WinFsp also.
+
+Chocolatey package manager
+
+Make sure you have Choco installed
+
+    choco search rclone
+    choco install rclone
+
+This will install rclone on your Windows machine. If you are planning to
+use rclone mount then
+
+    choco install winfsp
 
-The rclone maintains a docker image for rclone. These images are
-autobuilt by docker hub from the rclone source based on a minimal Alpine
-linux image.
+will install that too.
+
+Note that this is a third party installer not controlled by the rclone
+developers so it may be out of date. Its current version is as below.
+
+[Chocolatey package]
+
+Package manager installation
+
+Many Linux, Windows, macOS and other OS distributions package and
+distribute rclone.
+
+The distributed versions of rclone are often quite out of date and for
+this reason we recommend one of the other installation methods if
+possible.
+
+You can get an idea of how up to date or not your OS distribution's
+package is here.
+
+[Packaging status]
+
+Docker installation
+
+The rclone developers maintain a docker image for rclone.
+
+These images are built as part of the release process based on a minimal
+Alpine Linux.
 
 The :latest tag will always point to the latest stable release. You can
 use the :beta tag to get the latest build from master. You can also use
@@ -364,9 +433,9 @@ Here are some commands tested on an Ubuntu 18.04.3 host:
     ls ~/data/mount
     kill %1
 
-Install from source
+Source installation
 
-Make sure you have git and Go installed. Go version 1.16 or newer is
+Make sure you have git and Go installed. Go version 1.17 or newer is
 required, latest release is recommended. You can get it from your
 package manager, or download it from golang.org/dl. Then you can run the
 following:
@@ -381,7 +450,7 @@ executable in the same folder. As an initial check you can now run
 ./rclone version (.\rclone version on Windows).
 
 Note that on macOS and Windows the mount command will not be available
-unless you specify additional build tag cmount.
+unless you specify an additional build tag cmount.
 
     go build -tags cmount
 
@@ -409,9 +478,10 @@ This is how the official rclone releases are built.
     go build -trimpath -ldflags -s -tags cmount
 
 Instead of executing the go build command directly, you can run it via
-the Makefile, which also sets version information and copies the
-resulting rclone executable into your GOPATH bin folder
-($(go env GOPATH)/bin, which corresponds to ~/go/bin/rclone by default).
+the Makefile. It changes the version number suffix from "-DEV" to
+"-beta" and appends commit details. It also copies the resulting rclone
+executable into your GOPATH bin folder ($(go env GOPATH)/bin, which
+corresponds to ~/go/bin/rclone by default).
 
     make
 
@@ -419,7 +489,13 @@ To include mount command on macOS and Windows with Makefile build:
 
     make GOTAGS=cmount
 
-As an alternative you can download the source, build and install rclone
+There are other make targets that can be used for more advanced builds,
+such as cross-compiling for all supported os/architectures, embedding
+icon and version info resources into windows executable, and packaging
+results into release artifacts. See Makefile and cross-compile.go for
+details.
+
+Another alternative is to download the source, build and install rclone
 in one operation, as a regular Go package. The source will be stored it
 in the Go module cache, and the resulting executable will be in your
 GOPATH bin folder ($(go env GOPATH)/bin, which corresponds to
@@ -435,7 +511,7 @@ don't work with the current version):
 
     go get github.com/rclone/rclone
 
-Installation with Ansible
+Ansible installation
 
 This can be done with Stefan Weichinger's ansible role.
 
@@ -518,7 +594,7 @@ NOTE: Remember that when rclone runs as the SYSTEM user, the user
 profile that it sees will not be yours. This means that if you normally
 run rclone with configuration file in the default location, to be able
 to use the same configuration when running as the system user you must
-explicitely tell rclone where to find it with the --config option, or
+explicitly tell rclone where to find it with the --config option, or
 else it will look in the system users profile path
 (C:\Windows\System32\config\systemprofile). To test your command
 manually from a Command Prompt, you can run it with the PsExec utility
@@ -583,7 +659,7 @@ Third-party service integration
 
 To Windows service running any rclone command, the excellent third-party
 utility NSSM, the "Non-Sucking Service Manager", can be used. It
-includes some advanced features such as adjusting process periority,
+includes some advanced features such as adjusting process priority,
 defining process environment variables, redirect to file anything
 written to stdout, and customized response to different exit codes, with
 a GUI to configure everything from (although it can also be used from
@@ -662,7 +738,6 @@ See the following for detailed instructions for
 -   HDFS
 -   HiDrive
 -   HTTP
--   Hubic
 -   Internet Archive
 -   Jottacloud
 -   Koofr
@@ -673,6 +748,7 @@ See the following for detailed instructions for
 -   Microsoft OneDrive
 -   OpenStack Swift / Rackspace Cloudfiles / Memset Memstore
 -   OpenDrive
+-   Oracle Object Storage
 -   Pcloud
 -   premiumize.me
 -   put.io
@@ -680,6 +756,7 @@ See the following for detailed instructions for
 -   Seafile
 -   SFTP
 -   Sia
+-   SMB
 -   Storj
 -   SugarSync
 -   Union
@@ -857,6 +934,11 @@ extended explanation in the copy command if unsure.
 If dest:path doesn't exist, it is created and the source:path contents
 go there.
 
+It is not possible to sync overlapping remotes. However, you may exclude
+the destination from the sync with a filter rule or by putting an
+exclude-if-present file inside the destination directory and sync to a
+destination that is inside the source directory.
+
 Note: Use the -P/--progress flag to view real-time transfer statistics
 
 Note: Use the rclone dedupe command to deal with "Duplicate
@@ -1141,8 +1223,8 @@ recursion.
 The other list commands lsd,lsf,lsjson do not recurse by default - use
 -R to make them recurse.
 
-Listing a non-existent directory will produce an error except for
-remotes which can't have empty directories (e.g. s3, swift, or gcs - the
+Listing a nonexistent directory will produce an error except for remotes
+which can't have empty directories (e.g. s3, swift, or gcs - the
 bucket-based remotes).
 
     rclone ls remote:path [flags]
@@ -1203,8 +1285,8 @@ recursion.
 The other list commands lsd,lsf,lsjson do not recurse by default - use
 -R to make them recurse.
 
-Listing a non-existent directory will produce an error except for
-remotes which can't have empty directories (e.g. s3, swift, or gcs - the
+Listing a nonexistent directory will produce an error except for remotes
+which can't have empty directories (e.g. s3, swift, or gcs - the
 bucket-based remotes).
 
     rclone lsd remote:path [flags]
@@ -1257,8 +1339,8 @@ recursion.
 The other list commands lsd,lsf,lsjson do not recurse by default - use
 -R to make them recurse.
 
-Listing a non-existent directory will produce an error except for
-remotes which can't have empty directories (e.g. s3, swift, or gcs - the
+Listing a nonexistent directory will produce an error except for remotes
+which can't have empty directories (e.g. s3, swift, or gcs - the
 bucket-based remotes).
 
     rclone lsl remote:path [flags]
@@ -1293,8 +1375,8 @@ rclone hashsum MD5 remote:path.
 
 This command can also hash data received on standard input (stdin), by
 not passing a remote:path, or by passing a hyphen as remote:path when
-there is data to read (if not, the hypen will be treated literaly, as a
-relative path).
+there is data to read (if not, the hyphen will be treated literally, as
+a relative path).
 
     rclone md5sum remote:path [flags]
 
@@ -1332,8 +1414,8 @@ rclone hashsum SHA1 remote:path.
 
 This command can also hash data received on standard input (stdin), by
 not passing a remote:path, or by passing a hyphen as remote:path when
-there is data to read (if not, the hypen will be treated literaly, as a
-relative path).
+there is data to read (if not, the hyphen will be treated literally, as
+a relative path).
 
 This command can also hash data received on STDIN, if not passing a
 remote:path.
@@ -1739,11 +1821,11 @@ SEE ALSO
 
 rclone bisync
 
-Perform bidirectonal synchronization between two paths.
+Perform bidirectional synchronization between two paths.
 
 Synopsis
 
-Perform bidirectonal synchronization between two paths.
+Perform bidirectional synchronization between two paths.
 
 Bisync provides a bidirectional cloud sync solution in rclone. It
 retains the Path1 and Path2 filesystem listings from the prior run. On
@@ -1928,7 +2010,7 @@ Linux:
 
 macOS:
 
-    rclone completion bash > /usr/local/etc/bash_completion.d/rclone
+    rclone completion bash > $(brew --prefix)/etc/bash_completion.d/rclone
 
 You will need to start a new shell for this setup to take effect.
 
@@ -2020,6 +2102,10 @@ need to enable it. You can execute the following once:
 
     echo "autoload -U compinit; compinit" >> ~/.zshrc
 
+To load completions in your current shell session:
+
+    source <(rclone completion zsh); compdef _rclone rclone
+
 To load completions for every new session, execute once:
 
 Linux:
@@ -2028,7 +2114,7 @@ Linux:
 
 macOS:
 
-    rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone
+    rclone completion zsh > $(brew --prefix)/share/zsh/site-functions/_rclone
 
 You will need to start a new shell for this setup to take effect.
 
@@ -2084,7 +2170,7 @@ already passing obscured passwords then use --no-obscure. You can also
 set obscured passwords using the rclone config password command.
 
 The flag --non-interactive is for use by applications that wish to
-configure rclone themeselves, rather than using rclone's text based
+configure rclone themselves, rather than using rclone's text based
 configuration questions. If this flag is set, and rclone needs to ask
 the user a question, a JSON blob will be returned with the question in
 it.
@@ -2421,7 +2507,7 @@ already passing obscured passwords then use --no-obscure. You can also
 set obscured passwords using the rclone config password command.
 
 The flag --non-interactive is for use by applications that wish to
-configure rclone themeselves, rather than using rclone's text based
+configure rclone themselves, rather than using rclone's text based
 configuration questions. If this flag is set, and rclone needs to ask
 the user a question, a JSON blob will be returned with the question in
 it.
@@ -2914,8 +3000,8 @@ md5sum and sha1sum.
 
 This command can also hash data received on standard input (stdin), by
 not passing a remote:path, or by passing a hyphen as remote:path when
-there is data to read (if not, the hypen will be treated literaly, as a
-relative path).
+there is data to read (if not, the hyphen will be treated literally, as
+a relative path).
 
 Run without a hash to see the list of all supported hashes, e.g.
 
@@ -3137,8 +3223,8 @@ recursion.
 The other list commands lsd,lsf,lsjson do not recurse by default - use
 -R to make them recurse.
 
-Listing a non-existent directory will produce an error except for
-remotes which can't have empty directories (e.g. s3, swift, or gcs - the
+Listing a nonexistent directory will produce an error except for remotes
+which can't have empty directories (e.g. s3, swift, or gcs - the
 bucket-based remotes).
 
     rclone lsf remote:path [flags]
@@ -3213,7 +3299,7 @@ If --files-only is not specified directories in addition to the files
 will be returned.
 
 If --metadata is set then an additional Metadata key will be returned.
-This will have metdata in rclone standard format as a JSON object.
+This will have metadata in rclone standard format as a JSON object.
 
 if --stat is set then a single JSON blob will be returned about the item
 pointed to. This will return an error if the item isn't found. However
@@ -3260,8 +3346,8 @@ recursion.
 The other list commands lsd,lsf,lsjson do not recurse by default - use
 -R to make them recurse.
 
-Listing a non-existent directory will produce an error except for
-remotes which can't have empty directories (e.g. s3, swift, or gcs - the
+Listing a nonexistent directory will produce an error except for remotes
+which can't have empty directories (e.g. s3, swift, or gcs - the
 bucket-based remotes).
 
     rclone lsjson remote:path [flags]
@@ -3383,10 +3469,10 @@ unexpected program errors, freezes or other issues, consider mounting as
 a network drive instead.
 
 When mounting as a fixed disk drive you can either mount to an unused
-drive letter, or to a path representing a non-existent subdirectory of
-an existing parent directory or drive. Using the special value * will
-tell rclone to automatically assign the next available drive letter,
-starting with Z: and moving backward. Examples:
+drive letter, or to a path representing a nonexistent subdirectory of an
+existing parent directory or drive. Using the special value * will tell
+rclone to automatically assign the next available drive letter, starting
+with Z: and moving backward. Examples:
 
     rclone mount remote:path/to/files *
     rclone mount remote:path/to/files X:
@@ -3416,7 +3502,7 @@ remote UNC path by net use etc, just like a normal network drive
 mapping.
 
 If you specify a full network share UNC path with --volname, this will
-implicitely set the --network-mode option, so the following two examples
+implicitly set the --network-mode option, so the following two examples
 have same result:
 
     rclone mount remote:path/to/files X: --network-mode
@@ -3426,7 +3512,7 @@ You may also specify the network share UNC path as the mountpoint
 itself. Then rclone will automatically assign a drive letter, same as
 with * and use that as mountpoint, and instead use the UNC path
 specified as the volume name, as if it were specified with the --volname
-option. This will also implicitely set the --network-mode option. This
+option. This will also implicitly set the --network-mode option. This
 means the following two examples have same result:
 
     rclone mount remote:path/to/files \\cloud\remote
@@ -3462,7 +3548,7 @@ on each entry will be set according to options --dir-perms and
 The default permissions corresponds to
 --file-perms 0666 --dir-perms 0777, i.e. read and write permissions to
 everyone. This means you will not be able to start any programs from the
-the mount. To be able to do that you must add execute permissions, e.g.
+mount. To be able to do that you must add execute permissions, e.g.
 --file-perms 0777 --dir-perms 0777 to add it to everyone. If the program
 needs to write files, chances are you will have to enable VFS File
 Caching as well (see also limitations).
@@ -3532,10 +3618,9 @@ applications won't work with their files on an rclone mount without
 --vfs-cache-mode writes or --vfs-cache-mode full. See the VFS File
 Caching section for more info.
 
-The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2,
-Hubic) do not support the concept of empty directories, so empty
-directories will have a tendency to disappear once they fall out of the
-directory cache.
+The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2) do
+not support the concept of empty directories, so empty directories will
+have a tendency to disappear once they fall out of the directory cache.
 
 When rclone mount is invoked on Unix with --daemon flag, the main rclone
 program will wait for the background mount to become ready or until the
@@ -4135,7 +4220,7 @@ toggle the help on and off. The supported keys are:
      q/ESC/^c to quit
 
 Listed files/directories may be prefixed by a one-character flag, some
-of them combined with a description in brackes at end of line. These
+of them combined with a description in brackets at end of line. These
 flags have the following meaning:
 
     e means this is an empty directory, i.e. contains no files (but
@@ -4852,11 +4937,13 @@ only with caching.
 Options
 
           --addr string                            The ip:port or :port to bind the DLNA http server to (default ":7879")
+          --announce-interval duration             The interval between SSDP announcements (default 12m0s)
           --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
           --dir-perms FileMode                     Directory permissions (default 0777)
           --file-perms FileMode                    File permissions (default 0666)
           --gid uint32                             Override the gid field set by the filesystem (not supported on Windows) (default 1000)
       -h, --help                                   help for dlna
+          --interface stringArray                  The interface to use for SSDP (repeat as necessary)
           --log-trace                              Enable trace logging of SOAP traffic
           --name string                            Name of DLNA server
           --no-checksum                            Don't compare checksums on up/download
@@ -5827,6 +5914,9 @@ of that with the CA certificate. --key should be the PEM encoded private
 key and --client-ca should be the PEM encoded client certificate
 authority certificate.
 
+--min-tls-version is minimum TLS version that is acceptable. Valid
+values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").
+
 Template
 
 --template allows a user to specify a custom markup template for HTTP
@@ -6237,6 +6327,7 @@ Options
           --htpasswd string                        A htpasswd file - if not provided no authentication is done
           --key string                             SSL PEM Private key
           --max-header-bytes int                   Maximum size of request header (default 4096)
+          --min-tls-version string                 Minimum TLS version that is acceptable (default "tls1.0")
           --no-checksum                            Don't compare checksums on up/download
           --no-modtime                             Don't read/write the modification time (can speed things up)
           --no-seek                                Don't allow seeking in files
@@ -6465,6 +6556,9 @@ that with the CA certificate. --key should be the PEM encoded private
 key and --client-ca should be the PEM encoded client certificate
 authority certificate.
 
+--min-tls-version is minimum TLS version that is acceptable. Valid
+values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").
+
     rclone serve restic remote:path [flags]
 
 Options
@@ -6479,6 +6573,7 @@ Options
           --htpasswd string                 htpasswd file - if not provided no authentication is done
           --key string                      SSL PEM Private key
           --max-header-bytes int            Maximum size of request header (default 4096)
+          --min-tls-version string          Minimum TLS version that is acceptable (default "tls1.0")
           --pass string                     Password for authentication
           --private-repos                   Users can only access their private repo
           --realm string                    Realm for authentication (default "rclone")
@@ -6500,12 +6595,20 @@ Serve the remote over SFTP.
 
 Synopsis
 
-Run a SFTP server to serve a remote over SFTP. This can be used with an
+Run an SFTP server to serve a remote over SFTP. This can be used with an
 SFTP client or you can make a remote of type sftp to use with it.
 
 You can use the filter flags (e.g. --include, --exclude) to control what
 is served.
 
+The server will respond to a small number of shell commands, mainly
+md5sum, sha1sum and df, which enable it to provide support for checksums
+and the about feature when accessed from an sftp remote.
+
+Note that this server uses standard 32 KiB packet payload size, which
+means you must not configure the client to expect anything else, e.g.
+with the chunk_size option on an sftp remote.
+
 The server will log errors. Use -v to see access logs.
 
 --bwlimit will be respected for file transfers. Use --stats to control
@@ -6516,11 +6619,6 @@ You must provide some means of authentication, either with
 --authorized-keys - the default is the same as ssh), an --auth-proxy, or
 set the --no-auth flag for no authentication when logging in.
 
-Note that this also implements a small number of shell commands so that
-it can provide md5sum/sha1sum/df information for the rclone sftp
-backend. This means that is can support SHA1SUMs, MD5SUMs and the about
-command when paired with the rclone sftp backend.
-
 If you don't supply a host --key then rclone will generate rsa, ecdsa
 and ed25519 variants, and cache them for later use in rclone's cache
 directory (see rclone help flags cache-dir) in the "serve-sftp"
@@ -6959,7 +7057,7 @@ Options
           --pass string                            Password for authentication
           --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
           --read-only                              Only allow read-only access
-          --stdio                                  Run an sftp server on run stdin/stdout
+          --stdio                                  Run an sftp server on stdin/stdout
           --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
           --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
           --user string                            User name for authentication
@@ -7113,6 +7211,9 @@ that with the CA certificate. --key should be the PEM encoded private
 key and --client-ca should be the PEM encoded client certificate
 authority certificate.
 
+--min-tls-version is minimum TLS version that is acceptable. Valid
+values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").
+
 VFS - Virtual File System
 
 This command uses the VFS layer. This adapts the cloud storage objects
@@ -7522,6 +7623,7 @@ Options
           --htpasswd string                        htpasswd file - if not provided no authentication is done
           --key string                             SSL PEM Private key
           --max-header-bytes int                   Maximum size of request header (default 4096)
+          --min-tls-version string                 Minimum TLS version that is acceptable (default "tls1.0")
           --no-checksum                            Don't compare checksums on up/download
           --no-modtime                             Don't read/write the modification time (can speed things up)
           --no-seek                                Don't allow seeking in files
@@ -8227,7 +8329,7 @@ backends can also store arbitrary user metadata.
 
 Where possible the key names are standardized, so, for example, it is
 possible to copy object metadata from s3 to azureblob for example and
-metadata will be translated apropriately.
+metadata will be translated appropriately.
 
 Some backends have limits on the size of the metadata and rclone will
 give errors on upload if they are exceeded.
@@ -8310,10 +8412,34 @@ also possible to specify --boolean=false or --boolean=true. Note that
 --boolean false is not valid - this is parsed as --boolean and the false
 is parsed as an extra command line argument for rclone.
 
-Options which use TIME use the go time parser. A duration string is a
-possibly signed sequence of decimal numbers, each with optional fraction
-and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units
-are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+Time or duration options
+
+TIME or DURATION options can be specified as a duration string or a time
+string.
+
+A duration string is a possibly signed sequence of decimal numbers, each
+with optional fraction and a unit suffix, such as "300ms", "-1.5h" or
+"2h45m". Default units are seconds or the following abbreviations are
+valid:
+
+-   ms - Milliseconds
+-   s - Seconds
+-   m - Minutes
+-   h - Hours
+-   d - Days
+-   w - Weeks
+-   M - Months
+-   y - Years
+
+These can also be specified as an absolute time in the following
+formats:
+
+-   RFC3339 - e.g. 2006-01-02T15:04:05Z or 2006-01-02T15:04:05+07:00
+-   ISO8601 Date and time, local timezone - 2006-01-02T15:04:05
+-   ISO8601 Date and time, local timezone - 2006-01-02 15:04:05
+-   ISO8601 Date - 2006-01-02 (YYYY-MM-DD)
+
+Size options
 
 Options which use SIZE use KiB (multiples of 1024 bytes) by default.
 However, a suffix of B for Byte, K for KiB, M for MiB, G for GiB, T for
@@ -8332,7 +8458,8 @@ added) in DIR, then it will be overwritten.
 
 The remote in use must support server-side move or copy and you must use
 the same remote as the destination of the sync. The backup directory
-must not overlap the destination directory.
+must not overlap the destination directory without it being excluded by
+a filter rule.
 
 For example
 
@@ -8365,7 +8492,7 @@ is bytes per second not bits per second. To use a single limit, specify
 the desired bandwidth in KiB/s, or use a suffix B|K|M|G|T|P. The default
 is 0 which means to not limit bandwidth.
 
-The upload and download bandwidth can be specified seperately, as
+The upload and download bandwidth can be specified separately, as
 --bwlimit UP:DOWN, so
 
     --bwlimit 10M:100k
@@ -9380,6 +9507,17 @@ This sets the interval between each retry specified by --retries
 
 The default is 0. Use 0 to disable.
 
+--server-side-across-configs
+
+Allow server-side operations (e.g. copy or move) to work across
+different configurations.
+
+This can be useful if you wish to do a server-side copy or move between
+two remotes which use the same backend but are configured differently.
+
+Note that this isn't enabled by default because it isn't easy for rclone
+to tell if it will work between any two configurations.
+
 --size-only
 
 Normally rclone will look at modification time and size of files to see
@@ -9567,13 +9705,21 @@ By default, rclone doesn't keep track of renamed files, so if you rename
 a file locally then sync it to a remote, rclone will delete the old file
 on the remote and upload a new copy.
 
-If you use this flag, and the remote supports server-side copy or
-server-side move, and the source and destination have a compatible hash,
-then this will track renames during sync operations and perform renaming
-server-side.
+An rclone sync with --track-renames runs like a normal sync, but keeps
+track of objects which exist in the destination but not in the source
+(which would normally be deleted), and which objects exist in the source
+but not the destination (which would normally be transferred). These
+objects are then candidates for renaming.
+
+After the sync, rclone matches up the source only and destination only
+objects using the --track-renames-strategy specified and either renames
+the destination object or transfers the source and deletes the
+destination object. --track-renames is stateless like all of rclone's
+syncs.
 
-Files will be matched by size and hash - if both match then a rename
-will be considered.
+To use this flag the destination must support server-side copy or
+server-side move, and to use a hash based --track-renames-strategy (the
+default) the source and the destination must have a compatible hash.
 
 If the destination does not support server-side copy or move, rclone
 will fall back to the default behaviour and log an error level message
@@ -9590,7 +9736,7 @@ will select --delete-after instead of --delete-during.
 
 --track-renames-strategy (hash,modtime,leaf,size)
 
-This option changes the matching criteria for --track-renames.
+This option changes the file matching criteria for --track-renames.
 
 The matching is controlled by a comma separated selection of these
 tokens:
@@ -9601,14 +9747,14 @@ tokens:
 -   leaf - the name of the file not including its directory name
 -   size - the size of the file (this is always enabled)
 
-So using --track-renames-strategy modtime,leaf would match files based
-on modification time, the leaf of the file name and the size only.
+The default option is hash.
+
+Using --track-renames-strategy modtime,leaf would match files based on
+modification time, the leaf of the file name and the size only.
 
 Using --track-renames-strategy modtime or leaf can enable
 --track-renames support for encrypted destinations.
 
-If nothing is specified, the default option is matching by hashes.
-
 Note that the hash strategy is not supported with encrypted
 destinations.
 
@@ -9645,7 +9791,7 @@ the least amount of memory.
 
 However, some remotes have a way of listing all files beneath a
 directory in one (or a small number) of transactions. These tend to be
-the bucket-based remotes (e.g. S3, B2, GCS, Swift, Hubic).
+the bucket-based remotes (e.g. S3, B2, GCS, Swift).
 
 If you use the --fast-list flag then rclone will use this method for
 listing directories. This will have the following consequences for the
@@ -9712,7 +9858,7 @@ In all other cases the file will not be updated.
 Consider using the --modify-window flag to compensate for time skews
 between the source and the backend, for backends that do not support mod
 times, and instead use uploaded times. However, if the backend does not
-support checksums, note that sync'ing or copying within the time skew
+support checksums, note that syncing or copying within the time skew
 window may still result in additional transfers for safety.
 
 --use-mmap
@@ -10488,7 +10634,7 @@ Filter pattern examples
   Rooted          /*.jpg           /file.jpg                       /file.png
                                    /file2.jpg                      /dir/file.jpg
   Alternates      *.{jpg,png}      /file.jpg                       /file.gif
-                                   /dir/file.gif                   /dir/file.gif
+                                   /dir/file.png                   /dir/file.gif
   Path Wildcard   dir/**           /dir/anyfile                    file.png
                                    /subdir/dir/subsubdir/anyfile   /subdir/file.png
   Any Char        *.t?t            /file.txt                       /file.qxt
@@ -10983,6 +11129,8 @@ Default units are KiB but abbreviations K, M, G, T or P are valid.
 E.g. rclone ls remote: --min-size 50k lists files on remote: of 50 KiB
 size or larger.
 
+See the size option docs for more info.
+
 --max-size - Don't transfer any file larger than this
 
 Controls the maximum size file within the scope of an rclone command.
@@ -10991,33 +11139,19 @@ Default units are KiB but abbreviations K, M, G, T or P are valid.
 E.g. rclone ls remote: --max-size 1G lists files on remote: of 1 GiB
 size or smaller.
 
+See the size option docs for more info.
+
 --max-age - Don't transfer any file older than this
 
 Controls the maximum age of files within the scope of an rclone command.
-Default units are seconds or the following abbreviations are valid:
-
--   ms - Milliseconds
--   s - Seconds
--   m - Minutes
--   h - Hours
--   d - Days
--   w - Weeks
--   M - Months
--   y - Years
-
---max-age can also be specified as an absolute time in the following
-formats:
-
--   RFC3339 - e.g. 2006-01-02T15:04:05Z or 2006-01-02T15:04:05+07:00
--   ISO8601 Date and time, local timezone - 2006-01-02T15:04:05
--   ISO8601 Date and time, local timezone - 2006-01-02 15:04:05
--   ISO8601 Date - 2006-01-02 (YYYY-MM-DD)
 
 --max-age applies only to files and not to directories.
 
 E.g. rclone ls remote: --max-age 2d lists files on remote: of 2 days old
 or less.
 
+See the time option docs for valid formats.
+
 --min-age - Don't transfer any file younger than this
 
 Controls the minimum age of files within the scope of an rclone command.
@@ -11028,6 +11162,8 @@ Controls the minimum age of files within the scope of an rclone command.
 E.g. rclone ls remote: --min-age 2d lists files on remote: of 2 days old
 or more.
 
+See the time option docs for valid formats.
+
 Other flags
 
 --delete-excluded - Delete files on dest excluded from sync
@@ -11228,6 +11364,11 @@ SSL PEM Private key
 
 Maximum size of request header (default 4096)
 
+--rc-min-tls-version=VALUE
+
+The minimum TLS version that is acceptable. Valid values are "tls1.0",
+"tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").
+
 --rc-user=VALUE
 
 User name for authentication.
@@ -11571,7 +11712,7 @@ The parameters can be a string as per the rest of rclone, eg
 s3:bucket/path or :sftp:/my/dir. They can also be specified as JSON
 blobs.
 
-If specifyng a JSON blob it should be a object mapping strings to
+If specifying a JSON blob it should be a object mapping strings to
 strings. These values will be used to configure the remote. There are 3
 special values which may be set:
 
@@ -12135,6 +12276,12 @@ Parameters:
 
 -   jobid - id of the job (integer).
 
+job/stopgroup: Stop all running jobs in a group
+
+Parameters:
+
+-   group - name of the group (string).
+
 mount/listmounts: Show current mount points
 
 This shows currently mounted points, which can be used for performing an
@@ -12214,10 +12361,10 @@ Example:
 
 Authentication is required for this call.
 
-mount/unmountall: Show current mount points
+mount/unmountall: Unmount all active mounts
 
-This shows currently mounted points, which can be used for performing an
-unmount.
+rclone allows Linux, FreeBSD, macOS and Windows to mount any of Rclone's
+cloud storage systems as a file system with FUSE.
 
 This takes no parameters and returns error if unmount does not succeed.
 
@@ -12742,7 +12889,7 @@ check that parameter passing is working properly.
 
 Authentication is required for this call.
 
-sync/bisync: Perform bidirectonal synchronization between two paths.
+sync/bisync: Perform bidirectional synchronization between two paths.
 
 This takes the following parameters
 
@@ -13142,7 +13289,6 @@ Here is an overview of the major features of each cloud storage system.
   HDFS                                   -             R/W            No                No              -          -
   HiDrive                            HiDrive ¹²        R/W            No                No              -          -
   HTTP                                   -              R             No                No              R          -
-  Hubic                                 MD5            R/W            No                No             R/W         -
   Internet Archive                MD5, SHA1, CRC32   R/W ¹¹           No                No              -         RWU
   Jottacloud                            MD5            R/W           Yes                No              R          -
   Koofr                                 MD5             -            Yes                No              -          -
@@ -13153,6 +13299,7 @@ Here is an overview of the major features of each cloud storage system.
   Microsoft OneDrive                   SHA1 ⁵          R/W           Yes                No              R          -
   OpenDrive                             MD5            R/W           Yes             Partial ⁸          -          -
   OpenStack Swift                       MD5            R/W            No                No             R/W         -
+  Oracle Object Storage                 MD5            R/W            No                No             R/W         -
   pCloud                            MD5, SHA1 ⁷         R             No                No              W          -
   premiumize.me                          -              -            Yes                No              R          -
   put.io                               CRC-32          R/W            No                Yes             R          -
@@ -13160,6 +13307,7 @@ Here is an overview of the major features of each cloud storage system.
   Seafile                                -              -             No                No              -          -
   SFTP                              MD5, SHA1 ²        R/W         Depends              No              -          -
   Sia                                    -              -             No                No              -          -
+  SMB                                    -              -            Yes                No              -          -
   SugarSync                              -              -             No                No              -          -
   Storj                                  -              R             No                No              -          -
   Uptobox                                -              -             No                Yes             -          -
@@ -13216,7 +13364,7 @@ systems they must support a common hash type.
 
 ModTime
 
-Allmost all cloud storage systems store some sort of timestamp on
+Almost all cloud storage systems store some sort of timestamp on
 objects, but several of them not something that is appropriate to use
 for syncing. E.g. some backends will only write a timestamp that
 represent the time of the upload. To be relevant for syncing it should
@@ -13623,7 +13771,6 @@ upon backend-specific capabilities.
   HDFS                             Yes     No    Yes      Yes       No       No         Yes            No         Yes      Yes
   HiDrive                          Yes    Yes    Yes      Yes       No       No         Yes            No         No       Yes
   HTTP                             No      No     No      No        No       No          No            No         No       Yes
-  Hubic                           Yes †   Yes     No      No        No       Yes        Yes            No         Yes       No
   Internet Archive                 No     Yes     No      No        Yes      Yes         No            Yes        Yes       No
   Jottacloud                       Yes    Yes    Yes      Yes       Yes      Yes         No            Yes        Yes      Yes
   Koofr                            Yes    Yes    Yes      Yes       No       No         Yes            Yes        Yes      Yes
@@ -13634,6 +13781,7 @@ upon backend-specific capabilities.
   Microsoft OneDrive               Yes    Yes    Yes      Yes       Yes      No          No            Yes        Yes      Yes
   OpenDrive                        Yes    Yes    Yes      Yes       No       No          No            No         No       Yes
   OpenStack Swift                 Yes †   Yes     No      No        No       Yes        Yes            No         Yes       No
+  Oracle Object Storage            Yes    Yes     No      No        Yes      Yes         No            No         No        No
   pCloud                           Yes    Yes    Yes      Yes       Yes      No          No            Yes        Yes      Yes
   premiumize.me                    Yes     No    Yes      Yes       No       No          No            Yes        Yes      Yes
   put.io                           Yes     No    Yes      Yes       Yes      No         Yes            No         Yes      Yes
@@ -13641,6 +13789,7 @@ upon backend-specific capabilities.
   Seafile                          Yes    Yes    Yes      Yes       Yes      Yes        Yes            Yes        Yes      Yes
   SFTP                             No      No    Yes      Yes       No       No         Yes            No         Yes      Yes
   Sia                              No      No     No      No        No       No         Yes            No         No       Yes
+  SMB                              No      No    Yes      Yes       No       No         Yes            No         No       Yes
   SugarSync                        Yes    Yes    Yes      Yes       No       No         Yes            Yes        No       Yes
   Storj                           Yes †    No    Yes      No        No       Yes        Yes            No         No        No
   Uptobox                          No     Yes    Yes      Yes       No       No          No            No         No        No
@@ -13654,9 +13803,9 @@ Purge
 This deletes a directory quicker than just deleting all the files in the
 directory.
 
-† Note Swift, Hubic, and Storj implement this in order to delete
-directory markers but they don't actually have a quicker way of deleting
-files other than deleting them individually.
+† Note Swift and Storj implement this in order to delete directory
+markers but they don't actually have a quicker way of deleting files
+other than deleting them individually.
 
 ‡ StreamUpload is not supported with Nextcloud
 
@@ -13846,6 +13995,7 @@ These flags are available for every command.
           --rc-job-expire-interval duration      Interval to check for expired async jobs (default 10s)
           --rc-key string                        SSL PEM Private key
           --rc-max-header-bytes int              Maximum size of request header (default 4096)
+          --rc-min-tls-version string            Minimum TLS version that is acceptable (default "tls1.0")
           --rc-no-auth                           Don't require auth for certain methods
           --rc-pass string                       Password for authentication
           --rc-realm string                      Realm for authentication (default "rclone")
@@ -13862,6 +14012,7 @@ These flags are available for every command.
           --refresh-times                        Refresh the modtime of remote files
           --retries int                          Retry operations this many times if they fail (default 3)
           --retries-sleep duration               Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)
+          --server-side-across-configs           Allow server-side operations (e.g. copy) to work across different configs
           --size-only                            Skip based on size only, not mod-time or checksum
           --stats duration                       Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s)
           --stats-file-name-length int           Max file name length in stats (0 for no limit) (default 45)
@@ -13887,7 +14038,7 @@ These flags are available for every command.
           --use-json-log                         Use json log format
           --use-mmap                             Use mmap allocator (see docs)
           --use-server-modtime                   Use server modified time instead of object metadata
-          --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.59.0")
+          --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.60.0")
       -v, --verbose count                        Print lots more stuff (repeat for more)
 
 Backend Flags
@@ -14038,7 +14189,7 @@ and may be set in the config file.
           --drive-use-trash                              Send files to the trash instead of deleting permanently (default true)
           --drive-v2-download-min-size SizeSuffix        If Object's are greater, use drive v2 API to download (default off)
           --dropbox-auth-url string                      Auth server URL
-          --dropbox-batch-commit-timeout Duration        Max time to wait for a batch to finish comitting (default 10m0s)
+          --dropbox-batch-commit-timeout Duration        Max time to wait for a batch to finish committing (default 10m0s)
           --dropbox-batch-mode string                    Upload file batching sync|async|off (default "sync")
           --dropbox-batch-size int                       Max number of files in upload batch
           --dropbox-batch-timeout Duration               Max time to allow an idle upload batch before uploading (default 0s)
@@ -14072,6 +14223,7 @@ and may be set in the config file.
           --ftp-disable-utf8                             Disable using UTF-8 even if server advertises support
           --ftp-encoding MultiEncoder                    The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
           --ftp-explicit-tls                             Use Explicit FTPS (FTP over TLS)
+          --ftp-force-list-hidden                        Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
           --ftp-host string                              FTP host to connect to
           --ftp-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
           --ftp-no-check-certificate                     Do not verify the TLS certificate of the server
@@ -14090,6 +14242,7 @@ and may be set in the config file.
           --gcs-client-secret string                     OAuth Client Secret
           --gcs-decompress                               If set this will decompress gzip encoded objects
           --gcs-encoding MultiEncoder                    The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot)
+          --gcs-endpoint string                          Endpoint for the service
           --gcs-location string                          Location for the newly created buckets
           --gcs-no-check-bucket                          If set, don't attempt to check the bucket exists or create it
           --gcs-object-acl string                        Access Control List for new objects
@@ -14135,14 +14288,6 @@ and may be set in the config file.
           --http-no-head                                 Don't use HEAD requests
           --http-no-slash                                Set this if the site doesn't end directories with /
           --http-url string                              URL of HTTP host to connect to
-          --hubic-auth-url string                        Auth server URL
-          --hubic-chunk-size SizeSuffix                  Above this size files will be chunked into a _segments container (default 5Gi)
-          --hubic-client-id string                       OAuth Client Id
-          --hubic-client-secret string                   OAuth Client Secret
-          --hubic-encoding MultiEncoder                  The encoding for the backend (default Slash,InvalidUtf8)
-          --hubic-no-chunk                               Don't chunk files during streaming upload
-          --hubic-token string                           OAuth Access Token as a JSON blob
-          --hubic-token-url string                       Token server url
           --internetarchive-access-key-id string         IAS3 Access Key
           --internetarchive-disable-checksum             Don't ask the server to test against MD5 checksum calculated by rclone (default true)
           --internetarchive-encoding MultiEncoder        The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot)
@@ -14211,6 +14356,22 @@ and may be set in the config file.
           --onedrive-server-side-across-configs          Allow server-side operations (e.g. copy) to work across different onedrive configs
           --onedrive-token string                        OAuth Access Token as a JSON blob
           --onedrive-token-url string                    Token server url
+          --oos-chunk-size SizeSuffix                    Chunk size to use for uploading (default 5Mi)
+          --oos-compartment string                       Object storage compartment OCID
+          --oos-config-file string                       Path to OCI config file (default "~/.oci/config")
+          --oos-config-profile string                    Profile name inside the oci config file (default "Default")
+          --oos-copy-cutoff SizeSuffix                   Cutoff for switching to multipart copy (default 4.656Gi)
+          --oos-copy-timeout Duration                    Timeout for copy (default 1m0s)
+          --oos-disable-checksum                         Don't store MD5 checksum with object metadata
+          --oos-encoding MultiEncoder                    The encoding for the backend (default Slash,InvalidUtf8,Dot)
+          --oos-endpoint string                          Endpoint for Object storage API
+          --oos-leave-parts-on-error                     If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery
+          --oos-namespace string                         Object storage namespace
+          --oos-no-check-bucket                          If set, don't attempt to check the bucket exists or create it
+          --oos-provider string                          Choose your Auth Provider (default "env_auth")
+          --oos-region string                            Object storage Region
+          --oos-upload-concurrency int                   Concurrency for multipart uploads (default 10)
+          --oos-upload-cutoff SizeSuffix                 Cutoff for switching to chunked upload (default 200Mi)
           --opendrive-chunk-size SizeSuffix              Files will be uploaded in chunks this size (default 10Mi)
           --opendrive-encoding MultiEncoder              The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot)
           --opendrive-password string                    Password (obscured)
@@ -14242,6 +14403,7 @@ and may be set in the config file.
           --s3-bucket-acl string                         Canned ACL used when creating buckets
           --s3-chunk-size SizeSuffix                     Chunk size to use for uploading (default 5Mi)
           --s3-copy-cutoff SizeSuffix                    Cutoff for switching to multipart copy (default 4.656Gi)
+          --s3-decompress                                If set this will decompress gzip encoded objects
           --s3-disable-checksum                          Don't store MD5 checksum with object metadata
           --s3-disable-http2                             Disable usage of http2 for S3 backends
           --s3-download-url string                       Custom endpoint for downloads
@@ -14260,6 +14422,7 @@ and may be set in the config file.
           --s3-no-check-bucket                           If set, don't attempt to check the bucket exists or create it
           --s3-no-head                                   If set, don't HEAD uploaded objects to check integrity
           --s3-no-head-object                            If set, do not do HEAD before GET when getting objects
+          --s3-no-system-metadata                        Suppress setting and reading of system metadata
           --s3-profile string                            Profile to use in the shared credentials file
           --s3-provider string                           Choose your S3 provider
           --s3-region string                             Region to connect to
@@ -14269,7 +14432,8 @@ and may be set in the config file.
           --s3-session-token string                      An AWS session token
           --s3-shared-credentials-file string            Path to the shared credentials file
           --s3-sse-customer-algorithm string             If using SSE-C, the server-side encryption algorithm used when storing this object in S3
-          --s3-sse-customer-key string                   If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data
+          --s3-sse-customer-key string                   To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data
+          --s3-sse-customer-key-base64 string            If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data
           --s3-sse-customer-key-md5 string               If using SSE-C you may provide the secret encryption key MD5 checksum (optional)
           --s3-sse-kms-key-id string                     If using KMS ID you must provide the ARN of Key
           --s3-storage-class string                      The storage class to use when storing new objects in S3
@@ -14279,6 +14443,8 @@ and may be set in the config file.
           --s3-use-multipart-etag Tristate               Whether to use ETag in multipart uploads for verification (default unset)
           --s3-use-presigned-request                     Whether to use a presigned request or PutObject for single part uploads
           --s3-v2-auth                                   If true use v2 authentication
+          --s3-version-at Time                           Show file versions as they were at the specified time (default off)
+          --s3-versions                                  Include old versions in directory listings
           --seafile-2fa                                  Two-factor authentication ('true' if the account has 2FA enabled)
           --seafile-create-library                       Should rclone create a library if it doesn't exist
           --seafile-encoding MultiEncoder                The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
@@ -14325,6 +14491,15 @@ and may be set in the config file.
           --sia-encoding MultiEncoder                    The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot)
           --sia-user-agent string                        Siad User Agent (default "Sia-Agent")
           --skip-links                                   Don't warn about skipped symlinks
+          --smb-case-insensitive                         Whether the server is configured to be case-insensitive (default true)
+          --smb-domain string                            Domain name for NTLM authentication (default "WORKGROUP")
+          --smb-encoding MultiEncoder                    The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot)
+          --smb-hide-special-share                       Hide special shares (e.g. print$) which users aren't supposed to access (default true)
+          --smb-host string                              SMB server hostname to connect to
+          --smb-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
+          --smb-pass string                              SMB password (obscured)
+          --smb-port int                                 SMB port number (default 445)
+          --smb-user string                              SMB username (default "$USER")
           --storj-access-grant string                    Access grant
           --storj-api-key string                         API key
           --storj-passphrase string                      Encryption passphrase
@@ -14355,6 +14530,7 @@ and may be set in the config file.
           --swift-key string                             API key or password (OS_PASSWORD)
           --swift-leave-parts-on-error                   If true avoid calling abort upload on a failure
           --swift-no-chunk                               Don't chunk files during streaming upload
+          --swift-no-large-objects                       Disable support for static and dynamic large objects
           --swift-region string                          Region name - optional (OS_REGION_NAME)
           --swift-storage-policy string                  The storage policy to use when creating a new container
           --swift-storage-url string                     Storage URL - optional (OS_STORAGE_URL)
@@ -15251,7 +15427,7 @@ Most of these events come up due to a error status from an internal
 call. On such a critical error the {...}.path1.lst and {...}.path2.lst
 listing files are renamed to extension .lst-err, which blocks any future
 bisync runs (since the normal .lst files are not found). Bisync keeps
-them under bisync subdirectory of the rclone cache direcory, typically
+them under bisync subdirectory of the rclone cache directory, typically
 at ${HOME}/.cache/rclone/bisync/ on Linux.
 
 Some errors are considered temporary and re-running the bisync is not
@@ -15341,7 +15517,7 @@ have spelling case differences (Smile.jpg vs. smile.jpg).
 Windows support
 
 Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on
-Windows Github runners.
+Windows GitHub runners.
 
 Drive letters are allowed, including drive letters mapped to network
 drives (rclone bisync J:\localsync GDrive:). If a drive letter is
@@ -15838,7 +16014,7 @@ Notes about testing
     check file mismatches in the test tree.
 -   Some Dropbox tests can fail, notably printing the following message:
     src and dst identical but can't set mod time without deleting and re-uploading
-    This is expected and happens due a way Dropbox handles modificaion
+    This is expected and happens due a way Dropbox handles modification
     times. You should use the -refresh-times test flag to make up for
     this.
 -   If Dropbox tests hit request limit for you and print error message
@@ -15849,7 +16025,7 @@ Updating golden results
 
 Sometimes even a slight change in the bisync source can cause little
 changes spread around many log files. Updating them manually would be a
-nighmare.
+nightmare.
 
 The -golden flag will store the test.log and *.lst listings from each
 test case into respective golden directories. Golden results will
@@ -16209,6 +16385,11 @@ Invoking rclone mkdir backup:../desktop is exactly the same as invoking
 rclone mkdir mydrive:private/backup/../desktop. The empty path is not
 allowed as a remote. To alias the current directory use . instead.
 
+The target remote can also be a connection string. This can be used to
+modify the config of a remote for different uses, e.g. the alias
+myDriveTrash with the target remote myDrive,trashed_only: can be used to
+only show the trashed files in myDrive.
+
 Configuration
 
 Here is an example of how to make an alias called remote for local
@@ -16633,7 +16814,9 @@ The S3 backend can be used with a number of different providers:
 -   Huawei OBS
 -   IBM COS S3
 -   IDrive e2
+-   IONOS Cloud
 -   Minio
+-   Qiniu Cloud Object Storage (Kodo)
 -   RackCorp Object Storage
 -   Scaleway
 -   Seagate Lyve Cloud
@@ -16944,9 +17127,9 @@ Avoiding GET requests to read directory listings
 
 Rclone's default directory traversal is to process each directory
 individually. This takes one API call per directory. Using the
---fast-list flag will read all info about the the objects into memory
-first using a smaller number of API calls (one per 1000 objects). See
-the rclone docs for more details.
+--fast-list flag will read all info about the objects into memory first
+using a smaller number of API calls (one per 1000 objects). See the
+rclone docs for more details.
 
     rclone sync --fast-list --checksum /path/to/source s3:bucket
 
@@ -16995,6 +17178,64 @@ will mean that these objects do not have an MD5 checksum.
 Note that reading this from the object takes an additional HEAD request
 as the metadata isn't returned in object listings.
 
+Versions
+
+When bucket versioning is enabled (this can be done with rclone with the
+rclone backend versioning command) when rclone uploads a new version of
+a file it creates a new version of it Likewise when you delete a file,
+the old version will be marked hidden and still be available.
+
+Old versions of files, where available, are visible using the
+--s3-versions flag.
+
+It is also possible to view a bucket as it was at a certain point in
+time, using the --s3-version-at flag. This will show the file versions
+as they were at that time, showing files that have been deleted
+afterwards, and hiding files that were created since.
+
+If you wish to remove all the old versions then you can use the
+rclone backend cleanup-hidden remote:bucket command which will delete
+all the old hidden versions of files, leaving the current ones intact.
+You can also supply a path and only old versions under that path will be
+deleted, e.g. rclone backend cleanup-hidden remote:bucket/path/to/stuff.
+
+When you purge a bucket, the current and the old versions will be
+deleted then the bucket will be deleted.
+
+However delete will cause the current versions of the files to become
+hidden old versions.
+
+Here is a session showing the listing and retrieval of an old version
+followed by a cleanup of the old versions.
+
+Show current version and all the versions with --s3-versions flag.
+
+    $ rclone -q ls s3:cleanup-test
+            9 one.txt
+
+    $ rclone -q --s3-versions ls s3:cleanup-test
+            9 one.txt
+            8 one-v2016-07-04-141032-000.txt
+           16 one-v2016-07-04-141003-000.txt
+           15 one-v2016-07-02-155621-000.txt
+
+Retrieve an old version
+
+    $ rclone -q --s3-versions copy s3:cleanup-test/one-v2016-07-04-141003-000.txt /tmp
+
+    $ ls -l /tmp/one-v2016-07-04-141003-000.txt
+    -rw-rw-r-- 1 ncw ncw 16 Jul  2 17:46 /tmp/one-v2016-07-04-141003-000.txt
+
+Clean up all the old versions and show that they've gone.
+
+    $ rclone -q backend cleanup-hidden s3:cleanup-test
+
+    $ rclone -q ls s3:cleanup-test
+            9 one.txt
+
+    $ rclone -q --s3-versions ls s3:cleanup-test
+            9 one.txt
+
 Cleanup
 
 If you run rclone cleanup s3:bucket then it will remove all pending
@@ -17190,8 +17431,8 @@ Standard options
 Here are the Standard options specific to s3 (Amazon S3 Compliant
 Storage Providers including AWS, Alibaba, Ceph, China Mobile,
 Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS,
-IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS,
-StackPath, Storj, Tencent COS and Wasabi).
+IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway,
+SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
 
 --s3-provider
 
@@ -17226,6 +17467,8 @@ Properties:
         -   IBM COS S3
     -   "IDrive"
         -   IDrive e2
+    -   "IONOS"
+        -   IONOS Cloud
     -   "LyveCloud"
         -   Seagate Lyve Cloud
     -   "Minio"
@@ -17246,6 +17489,8 @@ Properties:
         -   Tencent Cloud Object Storage (COS)
     -   "Wasabi"
         -   Wasabi Object Storage
+    -   "Qiniu"
+        -   Qiniu Object Storage (Kodo)
     -   "Other"
         -   Any other S3 compatible provider
 
@@ -17518,6 +17763,60 @@ Properties:
 
 Region to connect to.
 
+Properties:
+
+-   Config: region
+-   Env Var: RCLONE_S3_REGION
+-   Provider: Qiniu
+-   Type: string
+-   Required: false
+-   Examples:
+    -   "cn-east-1"
+        -   The default endpoint - a good choice if you are unsure.
+        -   East China Region 1.
+        -   Needs location constraint cn-east-1.
+    -   "cn-east-2"
+        -   East China Region 2.
+        -   Needs location constraint cn-east-2.
+    -   "cn-north-1"
+        -   North China Region 1.
+        -   Needs location constraint cn-north-1.
+    -   "cn-south-1"
+        -   South China Region 1.
+        -   Needs location constraint cn-south-1.
+    -   "us-north-1"
+        -   North America Region.
+        -   Needs location constraint us-north-1.
+    -   "ap-southeast-1"
+        -   Southeast Asia Region 1.
+        -   Needs location constraint ap-southeast-1.
+    -   "ap-northeast-1"
+        -   Northeast Asia Region 1.
+        -   Needs location constraint ap-northeast-1.
+
+--s3-region
+
+Region where your bucket will be created and your data stored.
+
+Properties:
+
+-   Config: region
+-   Env Var: RCLONE_S3_REGION
+-   Provider: IONOS
+-   Type: string
+-   Required: false
+-   Examples:
+    -   "de"
+        -   Frankfurt, Germany
+    -   "eu-central-2"
+        -   Berlin, Germany
+    -   "eu-south-2"
+        -   Logrono, Spain
+
+--s3-region
+
+Region to connect to.
+
 Leave blank if you are using an S3 clone and you don't have a region.
 
 Properties:
@@ -17525,7 +17824,7 @@ Properties:
 -   Config: region
 -   Env Var: RCLONE_S3_REGION
 -   Provider:
-    !AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
+    !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
 -   Type: string
 -   Required: false
 -   Examples:
@@ -17783,6 +18082,27 @@ Properties:
 
 --s3-endpoint
 
+Endpoint for IONOS S3 Object Storage.
+
+Specify the endpoint from the same region.
+
+Properties:
+
+-   Config: endpoint
+-   Env Var: RCLONE_S3_ENDPOINT
+-   Provider: IONOS
+-   Type: string
+-   Required: false
+-   Examples:
+    -   "s3-eu-central-1.ionoscloud.com"
+        -   Frankfurt, Germany
+    -   "s3-eu-central-2.ionoscloud.com"
+        -   Berlin, Germany
+    -   "s3-eu-south-2.ionoscloud.com"
+        -   Logrono, Spain
+
+--s3-endpoint
+
 Endpoint for OSS API.
 
 Properties:
@@ -18048,6 +18368,33 @@ Properties:
 
 --s3-endpoint
 
+Endpoint for Qiniu Object Storage.
+
+Properties:
+
+-   Config: endpoint
+-   Env Var: RCLONE_S3_ENDPOINT
+-   Provider: Qiniu
+-   Type: string
+-   Required: false
+-   Examples:
+    -   "s3-cn-east-1.qiniucs.com"
+        -   East China Endpoint 1
+    -   "s3-cn-east-2.qiniucs.com"
+        -   East China Endpoint 2
+    -   "s3-cn-north-1.qiniucs.com"
+        -   North China Endpoint 1
+    -   "s3-cn-south-1.qiniucs.com"
+        -   South China Endpoint 1
+    -   "s3-us-north-1.qiniucs.com"
+        -   North America Endpoint 1
+    -   "s3-ap-southeast-1.qiniucs.com"
+        -   Southeast Asia Endpoint 1
+    -   "s3-ap-northeast-1.qiniucs.com"
+        -   Northeast Asia Endpoint 1
+
+--s3-endpoint
+
 Endpoint for S3 API.
 
 Required when using an S3 clone.
@@ -18057,7 +18404,7 @@ Properties:
 -   Config: endpoint
 -   Env Var: RCLONE_S3_ENDPOINT
 -   Provider:
-    !AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp
+    !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu
 -   Type: string
 -   Required: false
 -   Examples:
@@ -18384,6 +18731,35 @@ Properties:
 
 Location constraint - must be set to match the Region.
 
+Used when creating buckets only.
+
+Properties:
+
+-   Config: location_constraint
+-   Env Var: RCLONE_S3_LOCATION_CONSTRAINT
+-   Provider: Qiniu
+-   Type: string
+-   Required: false
+-   Examples:
+    -   "cn-east-1"
+        -   East China Region 1
+    -   "cn-east-2"
+        -   East China Region 2
+    -   "cn-north-1"
+        -   North China Region 1
+    -   "cn-south-1"
+        -   South China Region 1
+    -   "us-north-1"
+        -   North America Region 1
+    -   "ap-southeast-1"
+        -   Southeast Asia Region 1
+    -   "ap-northeast-1"
+        -   Northeast Asia Region 1
+
+--s3-location-constraint
+
+Location constraint - must be set to match the Region.
+
 Leave blank if not sure. Used when creating buckets only.
 
 Properties:
@@ -18391,7 +18767,7 @@ Properties:
 -   Config: location_constraint
 -   Env Var: RCLONE_S3_LOCATION_CONSTRAINT
 -   Provider:
-    !AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS
+    !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS
 -   Type: string
 -   Required: false
 
@@ -18632,13 +19008,34 @@ Properties:
         -   Prices are lower, but it needs to be restored first to be
             accessed.
 
+--s3-storage-class
+
+The storage class to use when storing new objects in Qiniu.
+
+Properties:
+
+-   Config: storage_class
+-   Env Var: RCLONE_S3_STORAGE_CLASS
+-   Provider: Qiniu
+-   Type: string
+-   Required: false
+-   Examples:
+    -   "STANDARD"
+        -   Standard storage class
+    -   "LINE"
+        -   Infrequent access storage mode
+    -   "GLACIER"
+        -   Archive storage mode
+    -   "DEEP_ARCHIVE"
+        -   Deep archive storage mode
+
 Advanced options
 
 Here are the Advanced options specific to s3 (Amazon S3 Compliant
 Storage Providers including AWS, Alibaba, Ceph, China Mobile,
 Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS,
-IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS,
-StackPath, Storj, Tencent COS and Wasabi).
+IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway,
+SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
 
 --s3-bucket-acl
 
@@ -18703,9 +19100,11 @@ Properties:
 
 --s3-sse-customer-key
 
-If using SSE-C you must provide the secret encryption key used to
+To use SSE-C you may provide the secret encryption key used to
 encrypt/decrypt your data.
 
+Alternatively you can provide --sse-customer-key-base64.
+
 Properties:
 
 -   Config: sse_customer_key
@@ -18717,6 +19116,24 @@ Properties:
     -   ""
         -   None
 
+--s3-sse-customer-key-base64
+
+If using SSE-C you must provide the secret encryption key encoded in
+base64 format to encrypt/decrypt your data.
+
+Alternatively you can provide --sse-customer-key.
+
+Properties:
+
+-   Config: sse_customer_key_base64
+-   Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_BASE64
+-   Provider: AWS,Ceph,ChinaMobile,Minio
+-   Type: string
+-   Required: false
+-   Examples:
+    -   ""
+        -   None
+
 --s3-sse-customer-key-md5
 
 If using SSE-C you may provide the secret encryption key MD5 checksum
@@ -19199,6 +19616,66 @@ Properties:
 -   Type: bool
 -   Default: false
 
+--s3-versions
+
+Include old versions in directory listings.
+
+Properties:
+
+-   Config: versions
+-   Env Var: RCLONE_S3_VERSIONS
+-   Type: bool
+-   Default: false
+
+--s3-version-at
+
+Show file versions as they were at the specified time.
+
+The parameter should be a date, "2006-01-02", datetime "2006-01-02
+15:04:05" or a duration for that long ago, eg "100d" or "1h".
+
+Note that when using this no file write operations are permitted, so you
+can't upload files or delete them.
+
+See the time option docs for valid formats.
+
+Properties:
+
+-   Config: version_at
+-   Env Var: RCLONE_S3_VERSION_AT
+-   Type: Time
+-   Default: off
+
+--s3-decompress
+
+If set this will decompress gzip encoded objects.
+
+It is possible to upload objects to S3 with "Content-Encoding: gzip"
+set. Normally rclone will download these files as compressed objects.
+
+If this flag is set then rclone will decompress these files with
+"Content-Encoding: gzip" as they are received. This means that rclone
+can't check the size and hash but the file contents will be
+decompressed.
+
+Properties:
+
+-   Config: decompress
+-   Env Var: RCLONE_S3_DECOMPRESS
+-   Type: bool
+-   Default: false
+
+--s3-no-system-metadata
+
+Suppress setting and reading of system metadata
+
+Properties:
+
+-   Config: no_system_metadata
+-   Env Var: RCLONE_S3_NO_SYSTEM_METADATA
+-   Type: bool
+-   Default: false
+
 Metadata
 
 User metadata is stored as x-amz-meta- keys. S3 metadata keys are case
@@ -19353,6 +19830,37 @@ Options:
 
 -   "max-age": Max age of upload to delete
 
+cleanup-hidden
+
+Remove old versions of files.
+
+    rclone backend cleanup-hidden remote: [options] [+]
+
+This command removes any old hidden versions of files on a versions
+enabled bucket.
+
+Note that you can use -i/--dry-run with this command to see what it
+would do.
+
+    rclone backend cleanup-hidden s3:bucket/path/to/dir
+
+versioning
+
+Set/get versioning support for a bucket.
+
+    rclone backend versioning remote: [options] [+]
+
+This command sets versioning support if a parameter is passed and then
+returns the current versioning status for the bucket supplied.
+
+    rclone backend versioning s3:bucket # read status only
+    rclone backend versioning s3:bucket Enabled
+    rclone backend versioning s3:bucket Suspended
+
+It may return "Enabled", "Suspended" or "Unversioned". Note that once
+versioning has been enabled the status can't be set back to
+"Unversioned".
+
 Anonymous access to public buckets
 
 If you want to use rclone to access a public bucket, configure with a
@@ -20033,6 +20541,166 @@ This will guide you through an interactive setup process.
     d) Delete this remote
     y/e/d> y
 
+IONOS Cloud
+
+IONOS S3 Object Storage is a service offered by IONOS for storing and
+accessing unstructured data. To connect to the service, you will need an
+access key and a secret key. These can be found in the Data Center
+Designer, by selecting Manager resources > Object Storage Key Manager.
+
+Here is an example of a configuration. First, run rclone config. This
+will walk you through an interactive setup process. Type n to add the
+new remote, and then enter a name:
+
+    Enter name for new remote.
+    name> ionos-fra
+
+Type s3 to choose the connection type:
+
+    Option Storage.
+    Type of storage to configure.
+    Choose a number from below, or type in your own value.
+    [snip]
+    XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+       \ (s3)
+    [snip]
+    Storage> s3
+
+Type IONOS:
+
+    Option provider.
+    Choose your S3 provider.
+    Choose a number from below, or type in your own value.
+    Press Enter to leave empty.
+    [snip]
+    XX / IONOS Cloud
+       \ (IONOS)
+    [snip]
+    provider> IONOS
+
+Press Enter to choose the default option
+Enter AWS credentials in the next step:
+
+    Option env_auth.
+    Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+    Only applies if access_key_id and secret_access_key is blank.
+    Choose a number from below, or type in your own boolean value (true or false).
+    Press Enter for the default (false).
+     1 / Enter AWS credentials in the next step.
+       \ (false)
+     2 / Get AWS credentials from the environment (env vars or IAM).
+       \ (true)
+    env_auth>
+
+Enter your Access Key and Secret key. These can be retrieved in the Data
+Center Designer, click on the menu “Manager resources” / "Object Storage
+Key Manager".
+
+    Option access_key_id.
+    AWS Access Key ID.
+    Leave blank for anonymous access or runtime credentials.
+    Enter a value. Press Enter to leave empty.
+    access_key_id> YOUR_ACCESS_KEY
+
+    Option secret_access_key.
+    AWS Secret Access Key (password).
+    Leave blank for anonymous access or runtime credentials.
+    Enter a value. Press Enter to leave empty.
+    secret_access_key> YOUR_SECRET_KEY
+
+Choose the region where your bucket is located:
+
+    Option region.
+    Region where your bucket will be created and your data stored.
+    Choose a number from below, or type in your own value.
+    Press Enter to leave empty.
+     1 / Frankfurt, Germany
+       \ (de)
+     2 / Berlin, Germany
+       \ (eu-central-2)
+     3 / Logrono, Spain
+       \ (eu-south-2)
+    region> 2
+
+Choose the endpoint from the same region:
+
+    Option endpoint.
+    Endpoint for IONOS S3 Object Storage.
+    Specify the endpoint from the same region.
+    Choose a number from below, or type in your own value.
+    Press Enter to leave empty.
+     1 / Frankfurt, Germany
+       \ (s3-eu-central-1.ionoscloud.com)
+     2 / Berlin, Germany
+       \ (s3-eu-central-2.ionoscloud.com)
+     3 / Logrono, Spain
+       \ (s3-eu-south-2.ionoscloud.com)
+    endpoint> 1
+
+Press Enter to choose the default option or choose the desired ACL
+setting:
+
+    Option acl.
+    Canned ACL used when creating buckets and storing or copying objects.
+    This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
+    For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+    Note that this ACL is applied when server-side copying objects as S3
+    doesn't copy the ACL from the source but rather writes a fresh one.
+    Choose a number from below, or type in your own value.
+    Press Enter to leave empty.
+       / Owner gets FULL_CONTROL.
+     1 | No one else has access rights (default).
+       \ (private)
+       / Owner gets FULL_CONTROL.
+    [snip]
+    acl>
+
+Press Enter to skip the advanced config:
+
+    Edit advanced config?
+    y) Yes
+    n) No (default)
+    y/n>
+
+Press Enter to save the configuration, and then q to quit the
+configuration process:
+
+    Configuration complete.
+    Options:
+    - type: s3
+    - provider: IONOS
+    - access_key_id: YOUR_ACCESS_KEY
+    - secret_access_key: YOUR_SECRET_KEY
+    - endpoint: s3-eu-central-1.ionoscloud.com
+    Keep this "ionos-fra" remote?
+    y) Yes this is OK (default)
+    e) Edit this remote
+    d) Delete this remote
+    y/e/d> y
+
+Done! Now you can try some commands (for macOS, use ./rclone instead of
+rclone).
+
+1)  Create a bucket (the name must be unique within the whole IONOS S3)
+
+    rclone mkdir ionos-fra:my-bucket
+
+2)  List available buckets
+
+    rclone lsd ionos-fra:
+
+4)  Copy a file from local to remote
+
+    rclone copy /Users/file.txt ionos-fra:my-bucket
+
+3)  List contents of a bucket
+
+    rclone ls ionos-fra:my-bucket
+
+5)  Copy a file from remote to local
+
+    rclone copy ionos-fra:my-bucket/file.txt
+
 Minio
 
 Minio is an object storage server built for cloud application developers
@@ -20094,6 +20762,198 @@ So once set up, for example, to copy files into a bucket
 
     rclone copy /path/to/files minio:bucket
 
+Qiniu Cloud Object Storage (Kodo)
+
+Qiniu Cloud Object Storage (Kodo), a completely independent-researched
+core technology which is proven by repeated customer experience has
+occupied absolute leading market leader position. Kodo can be widely
+applied to mass data management.
+
+To configure access to Qiniu Kodo, follow the steps below:
+
+1.  Run rclone config and select n for a new remote.
+
+    rclone config
+    No remotes found, make a new one?
+    n) New remote
+    s) Set configuration password
+    q) Quit config
+    n/s/q> n
+
+2.  Give the name of the configuration. For example, name it 'qiniu'.
+
+    name> qiniu
+
+3.  Select s3 storage.
+
+    Choose a number from below, or type in your own value
+     1 / 1Fichier
+       \ (fichier)
+     2 / Akamai NetStorage
+       \ (netstorage)
+     3 / Alias for an existing remote
+       \ (alias)
+     4 / Amazon Drive
+       \ (amazon cloud drive)
+     5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi
+       \ (s3)
+    [snip]
+    Storage> s3
+
+4.  Select Qiniu provider.
+
+    Choose a number from below, or type in your own value
+    1 / Amazon Web Services (AWS) S3
+       \ "AWS"
+    [snip]
+    22 / Qiniu Object Storage (Kodo)
+       \ (Qiniu)
+    [snip]
+    provider> Qiniu
+
+5.  Enter your SecretId and SecretKey of Qiniu Kodo.
+
+    Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+    Only applies if access_key_id and secret_access_key is blank.
+    Enter a boolean value (true or false). Press Enter for the default ("false").
+    Choose a number from below, or type in your own value
+     1 / Enter AWS credentials in the next step
+       \ "false"
+     2 / Get AWS credentials from the environment (env vars or IAM)
+       \ "true"
+    env_auth> 1
+    AWS Access Key ID.
+    Leave blank for anonymous access or runtime credentials.
+    Enter a string value. Press Enter for the default ("").
+    access_key_id> AKIDxxxxxxxxxx
+    AWS Secret Access Key (password)
+    Leave blank for anonymous access or runtime credentials.
+    Enter a string value. Press Enter for the default ("").
+    secret_access_key> xxxxxxxxxxx
+
+6.  Select endpoint for Qiniu Kodo. This is the standard endpoint for
+    different region.
+
+       / The default endpoint - a good choice if you are unsure.
+     1 | East China Region 1.
+       | Needs location constraint cn-east-1.
+       \ (cn-east-1)
+       / East China Region 2.
+     2 | Needs location constraint cn-east-2.
+       \ (cn-east-2)
+       / North China Region 1.
+     3 | Needs location constraint cn-north-1.
+       \ (cn-north-1)
+       / South China Region 1.
+     4 | Needs location constraint cn-south-1.
+       \ (cn-south-1)
+       / North America Region.
+     5 | Needs location constraint us-north-1.
+       \ (us-north-1)
+       / Southeast Asia Region 1.
+     6 | Needs location constraint ap-southeast-1.
+       \ (ap-southeast-1)
+       / Northeast Asia Region 1.
+     7 | Needs location constraint ap-northeast-1.
+       \ (ap-northeast-1)
+    [snip]
+    endpoint> 1
+
+    Option endpoint.
+    Endpoint for Qiniu Object Storage.
+    Choose a number from below, or type in your own value.
+    Press Enter to leave empty.
+     1 / East China Endpoint 1
+       \ (s3-cn-east-1.qiniucs.com)
+     2 / East China Endpoint 2
+       \ (s3-cn-east-2.qiniucs.com)
+     3 / North China Endpoint 1
+       \ (s3-cn-north-1.qiniucs.com)
+     4 / South China Endpoint 1
+       \ (s3-cn-south-1.qiniucs.com)
+     5 / North America Endpoint 1
+       \ (s3-us-north-1.qiniucs.com)
+     6 / Southeast Asia Endpoint 1
+       \ (s3-ap-southeast-1.qiniucs.com)
+     7 / Northeast Asia Endpoint 1
+       \ (s3-ap-northeast-1.qiniucs.com)
+    endpoint> 1
+
+    Option location_constraint.
+    Location constraint - must be set to match the Region.
+    Used when creating buckets only.
+    Choose a number from below, or type in your own value.
+    Press Enter to leave empty.
+     1 / East China Region 1
+       \ (cn-east-1)
+     2 / East China Region 2
+       \ (cn-east-2)
+     3 / North China Region 1
+       \ (cn-north-1)
+     4 / South China Region 1
+       \ (cn-south-1)
+     5 / North America Region 1
+       \ (us-north-1)
+     6 / Southeast Asia Region 1
+       \ (ap-southeast-1)
+     7 / Northeast Asia Region 1
+       \ (ap-northeast-1)
+    location_constraint> 1
+
+7.  Choose acl and storage class.
+
+    Note that this ACL is applied when server-side copying objects as S3
+    doesn't copy the ACL from the source but rather writes a fresh one.
+    Enter a string value. Press Enter for the default ("").
+    Choose a number from below, or type in your own value
+       / Owner gets FULL_CONTROL.
+     1 | No one else has access rights (default).
+       \ (private)
+       / Owner gets FULL_CONTROL.
+     2 | The AllUsers group gets READ access.
+       \ (public-read)
+    [snip]
+    acl> 2
+    The storage class to use when storing new objects in Tencent COS.
+    Enter a string value. Press Enter for the default ("").
+    Choose a number from below, or type in your own value
+     1 / Standard storage class
+       \ (STANDARD)
+     2 / Infrequent access storage mode
+       \ (LINE)
+     3 / Archive storage mode
+       \ (GLACIER)
+     4 / Deep archive storage mode
+       \ (DEEP_ARCHIVE)
+    [snip]
+    storage_class> 1
+    Edit advanced config? (y/n)
+    y) Yes
+    n) No (default)
+    y/n> n
+    Remote config
+    --------------------
+    [qiniu]
+    - type: s3
+    - provider: Qiniu
+    - access_key_id: xxx
+    - secret_access_key: xxx
+    - region: cn-east-1
+    - endpoint: s3-cn-east-1.qiniucs.com
+    - location_constraint: cn-east-1
+    - acl: public-read
+    - storage_class: STANDARD
+    --------------------
+    y) Yes this is OK (default)
+    e) Edit this remote
+    d) Delete this remote
+    y/e/d> y
+    Current remotes:
+
+    Name                 Type
+    ====                 ====
+    qiniu                s3
+
 RackCorp
 
 RackCorp Object Storage is an S3 compatible object storage platform from
@@ -23749,11 +24609,10 @@ If you intend to use the wrapped remote both directly for keeping
 unencrypted content, as well as through a crypt remote for encrypted
 content, it is recommended to point the crypt remote to a separate
 directory within the wrapped remote. If you use a bucket-based storage
-system (e.g. Swift, S3, Google Compute Storage, B2, Hubic) it is
-generally advisable to wrap the crypt remote around a specific bucket
-(s3:bucket). If wrapping around the entire root of the storage (s3:),
-and use the optional file name encryption, rclone will encrypt the
-bucket name.
+system (e.g. Swift, S3, Google Compute Storage, B2) it is generally
+advisable to wrap the crypt remote around a specific bucket (s3:bucket).
+If wrapping around the entire root of the storage (s3:), and use the
+optional file name encryption, rclone will encrypt the bucket name.
 
 Changing password
 
@@ -23767,7 +24626,7 @@ crypt remote means you will no longer able to decrypt any of the
 previously encrypted content. The only possibility is to re-upload
 everything via a crypt remote configured with your new password.
 
-Depending on the size of your data, your bandwith, storage quota etc,
+Depending on the size of your data, your bandwidth, storage quota etc,
 there are different approaches you can take: - If you have everything in
 a different location, for example on your local system, you could remove
 all of the prior encrypted files, change the password for your
@@ -23780,7 +24639,7 @@ remote to the new, effectively decrypting everything on the fly using
 the old password and re-encrypting using the new password. When done,
 delete the original crypt remote directory and finally the rclone crypt
 configuration with the old password. All data will be streamed from the
-storage system and back, so you will get half the bandwith and be
+storage system and back, so you will get half the bandwidth and be
 charged twice if you have upload and download quota on the storage
 system.
 
@@ -24078,7 +24937,7 @@ How to encode the encrypted filename to text string.
 
 This option could help with shortening the encrypted filename. The
 suitable option would depend on the way your remote count the filename
-length and if it's case sensitve.
+length and if it's case sensitive.
 
 Properties:
 
@@ -24409,7 +25268,7 @@ Generally -1 (default, equivalent to 5) is recommended. Levels 1 to 9
 increase compression at the cost of speed. Going past 6 generally offers
 very little return.
 
-Level -2 uses Huffmann encoding only. Only use if you know what you are
+Level -2 uses Huffman encoding only. Only use if you know what you are
 doing. Level 0 turns off compression.
 
 Properties:
@@ -24544,7 +25403,7 @@ This would produce something like this:
 
     [AllDrives]
     type = combine
-    remote = "My Drive=My Drive:" "Test Drive=Test Drive:"
+    upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
 
 If you then add that config to your config file (find it with
 rclone config file) then you can access all the shared drives in one
@@ -24979,7 +25838,7 @@ Properties:
 
 --dropbox-batch-commit-timeout
 
-Max time to wait for a batch to finish comitting
+Max time to wait for a batch to finish committing
 
 Properties:
 
@@ -25071,8 +25930,8 @@ Storage accessible through a global file system.
 Configuration
 
 The initial setup for the Enterprise File Fabric backend involves
-getting a token from the the Enterprise File Fabric which you need to do
-in your browser. rclone config walks you through it.
+getting a token from the Enterprise File Fabric which you need to do in
+your browser. rclone config walks you through it.
 
 Here is an example of how to make a remote called remote. First run:
 
@@ -25344,8 +26203,7 @@ To create an FTP configuration named remote, run
 
 Rclone config guides you through an interactive setup process. A minimal
 rclone FTP remote definition only requires host, username and password.
-For an anonymous FTP server, use anonymous as username and your email
-address as password.
+For an anonymous FTP server, see below.
 
     No remotes found, make a new one?
     n) New remote
@@ -25420,9 +26278,30 @@ files in the directory.
 
     rclone sync -i /home/local/directory remote:directory
 
-Example without a config file
+Anonymous FTP
+
+When connecting to a FTP server that allows anonymous login, you can use
+the special "anonymous" username. Traditionally, this user account
+accepts any string as a password, although it is common to use either
+the password "anonymous" or "guest". Some servers require the use of a
+valid e-mail address as password.
 
-    rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=`rclone obscure dummy`
+Using on-the-fly or connection string remotes makes it easy to access
+such servers, without requiring any configuration in advance. The
+following are examples of that:
+
+    rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=$(rclone obscure dummy)
+    rclone lsf :ftp,host=speedtest.tele2.net,user=anonymous,pass=$(rclone obscure dummy):
+
+The above examples work in Linux shells and in PowerShell, but not
+Windows Command Prompt. They execute the rclone obscure command to
+create a password string in the format required by the pass option. The
+following examples are exactly the same, except use an already obscured
+string representation of the same password "dummy", and therefore works
+even in Windows Command Prompt:
+
+    rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=IXs2wc8OJOz7SYLBk47Ji1rHTmxM
+    rclone lsf :ftp,host=speedtest.tele2.net,user=anonymous,pass=IXs2wc8OJOz7SYLBk47Ji1rHTmxM:
 
 Implicit TLS
 
@@ -25435,7 +26314,7 @@ Restricted filename characters
 In addition to the default restricted characters set the following
 characters are also replaced:
 
-File names cannot end with the following characters. Repacement is
+File names cannot end with the following characters. Replacement is
 limited to the last character in a file name:
 
   Character    Value   Replacement
@@ -25544,6 +26423,18 @@ Here are the Advanced options specific to ftp (FTP).
 
 Maximum number of FTP simultaneous connections, 0 for unlimited.
 
+Note that setting this is very likely to cause deadlocks so it should be
+used with care.
+
+If you are doing a sync or copy then make sure concurrency is one more
+than the sum of --transfers and --checkers.
+
+If you use --check-first then it just needs to be one more than the
+maximum of --checkers and --transfers.
+
+So for concurrency 3 you'd use --checkers 2 --transfers 2 --check-first
+or --checkers 1 --transfers 1.
+
 Properties:
 
 -   Config: concurrency
@@ -25606,6 +26497,18 @@ Properties:
 -   Type: bool
 -   Default: false
 
+--ftp-force-list-hidden
+
+Use LIST -a to force listing of hidden files and folders. This will
+disable the use of MLSD.
+
+Properties:
+
+-   Config: force_list_hidden
+-   Env Var: RCLONE_FTP_FORCE_LIST_HIDDEN
+-   Type: bool
+-   Default: false
+
 --ftp-idle-timeout
 
 Max time before closing idle connections.
@@ -26356,8 +27259,7 @@ Properties:
 If set this will decompress gzip encoded objects.
 
 It is possible to upload objects to GCS with "Content-Encoding: gzip"
-set. Normally rclone will download these files files as compressed
-objects.
+set. Normally rclone will download these files as compressed objects.
 
 If this flag is set then rclone will decompress these files with
 "Content-Encoding: gzip" as they are received. This means that rclone
@@ -26371,6 +27273,19 @@ Properties:
 -   Type: bool
 -   Default: false
 
+--gcs-endpoint
+
+Endpoint for the service.
+
+Leave blank normally.
+
+Properties:
+
+-   Config: endpoint
+-   Env Var: RCLONE_GCS_ENDPOINT
+-   Type: string
+-   Required: false
+
 --gcs-encoding
 
 The encoding for the backend.
@@ -27752,10 +28667,10 @@ found and a combined drive.
 
     [AllDrives]
     type = combine
-    remote = "My Drive=My Drive:" "Test Drive=Test Drive:"
+    upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
 
 Adding this to the rclone config file will cause those team drives to be
-accessible with the aliases shown. Any illegal charactes will be
+accessible with the aliases shown. Any illegal characters will be
 substituted with "_" and duplicate names will have numbers suffixed. It
 will also add a remote called AllDrives which shows all the shared
 drives combined into one directory tree.
@@ -29102,7 +30017,7 @@ Modified time and hashes
 HiDrive allows modification times to be set on objects accurate to 1
 second.
 
-HiDrive supports its own hash type which is used to verify the integrety
+HiDrive supports its own hash type which is used to verify the integrity
 of file contents after successful transfers.
 
 Restricted filename characters
@@ -29649,234 +30564,6 @@ mfs (most free space) as a member of an rclone union remote.
 
 See List of backends that do not support rclone about and rclone about
 
-Hubic
-
-Paths are specified as remote:path
-
-Paths are specified as remote:container (or remote: for the lsd
-command.) You may put subdirectories in too, e.g.
-remote:container/path/to/dir.
-
-Configuration
-
-The initial setup for Hubic involves getting a token from Hubic which
-you need to do in your browser. rclone config walks you through it.
-
-Here is an example of how to make a remote called remote. First run:
-
-     rclone config
-
-This will guide you through an interactive setup process:
-
-    n) New remote
-    s) Set configuration password
-    n/s> n
-    name> remote
-    Type of storage to configure.
-    Choose a number from below, or type in your own value
-    [snip]
-    XX / Hubic
-       \ "hubic"
-    [snip]
-    Storage> hubic
-    Hubic Client Id - leave blank normally.
-    client_id>
-    Hubic Client Secret - leave blank normally.
-    client_secret>
-    Remote config
-    Use auto config?
-     * Say Y if not sure
-     * Say N if you are working on a remote or headless machine
-    y) Yes
-    n) No
-    y/n> y
-    If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth
-    Log in and authorize rclone for access
-    Waiting for code...
-    Got code
-    --------------------
-    [remote]
-    client_id =
-    client_secret =
-    token = {"access_token":"XXXXXX"}
-    --------------------
-    y) Yes this is OK
-    e) Edit this remote
-    d) Delete this remote
-    y/e/d> y
-
-See the remote setup docs for how to set it up on a machine with no
-Internet browser available.
-
-Note that rclone runs a webserver on your local machine to collect the
-token as returned from Hubic. This only runs from the moment it opens
-your browser to the moment you get back the verification code. This is
-on http://127.0.0.1:53682/ and this it may require you to unblock it
-temporarily if you are running a host firewall.
-
-Once configured you can then use rclone like this,
-
-List containers in the top level of your Hubic
-
-    rclone lsd remote:
-
-List all the files in your Hubic
-
-    rclone ls remote:
-
-To copy a local directory to an Hubic directory called backup
-
-    rclone copy /home/source remote:backup
-
-If you want the directory to be visible in the official Hubic browser,
-you need to copy your files to the default directory
-
-    rclone copy /home/source remote:default/backup
-
---fast-list
-
-This remote supports --fast-list which allows you to use fewer
-transactions in exchange for more memory. See the rclone docs for more
-details.
-
-Modified time
-
-The modified time is stored as metadata on the object as
-X-Object-Meta-Mtime as floating point since the epoch accurate to 1 ns.
-
-This is a de facto standard (used in the official python-swiftclient
-amongst others) for storing the modification time for an object.
-
-Note that Hubic wraps the Swift backend, so most of the properties of
-are the same.
-
-Standard options
-
-Here are the Standard options specific to hubic (Hubic).
-
---hubic-client-id
-
-OAuth Client Id.
-
-Leave blank normally.
-
-Properties:
-
--   Config: client_id
--   Env Var: RCLONE_HUBIC_CLIENT_ID
--   Type: string
--   Required: false
-
---hubic-client-secret
-
-OAuth Client Secret.
-
-Leave blank normally.
-
-Properties:
-
--   Config: client_secret
--   Env Var: RCLONE_HUBIC_CLIENT_SECRET
--   Type: string
--   Required: false
-
-Advanced options
-
-Here are the Advanced options specific to hubic (Hubic).
-
---hubic-token
-
-OAuth Access Token as a JSON blob.
-
-Properties:
-
--   Config: token
--   Env Var: RCLONE_HUBIC_TOKEN
--   Type: string
--   Required: false
-
---hubic-auth-url
-
-Auth server URL.
-
-Leave blank to use the provider defaults.
-
-Properties:
-
--   Config: auth_url
--   Env Var: RCLONE_HUBIC_AUTH_URL
--   Type: string
--   Required: false
-
---hubic-token-url
-
-Token server url.
-
-Leave blank to use the provider defaults.
-
-Properties:
-
--   Config: token_url
--   Env Var: RCLONE_HUBIC_TOKEN_URL
--   Type: string
--   Required: false
-
---hubic-chunk-size
-
-Above this size files will be chunked into a _segments container.
-
-Above this size files will be chunked into a _segments container. The
-default for this is 5 GiB which is its maximum value.
-
-Properties:
-
--   Config: chunk_size
--   Env Var: RCLONE_HUBIC_CHUNK_SIZE
--   Type: SizeSuffix
--   Default: 5Gi
-
---hubic-no-chunk
-
-Don't chunk files during streaming upload.
-
-When doing streaming uploads (e.g. using rcat or mount) setting this
-flag will cause the swift backend to not upload chunked files.
-
-This will limit the maximum upload size to 5 GiB. However non chunked
-files are easier to deal with and have an MD5SUM.
-
-Rclone will still chunk files bigger than chunk_size when doing normal
-copy operations.
-
-Properties:
-
--   Config: no_chunk
--   Env Var: RCLONE_HUBIC_NO_CHUNK
--   Type: bool
--   Default: false
-
---hubic-encoding
-
-The encoding for the backend.
-
-See the encoding section in the overview for more info.
-
-Properties:
-
--   Config: encoding
--   Env Var: RCLONE_HUBIC_ENCODING
--   Type: MultiEncoder
--   Default: Slash,InvalidUtf8
-
-Limitations
-
-This uses the normal OpenStack Swift mechanism to refresh the Swift API
-credentials and ignores the expires field returned by the Hubic API.
-
-The Swift API doesn't return a correct MD5SUM for segmented files
-(Dynamic or Static Large Objects) so rclone won't check or use the
-MD5SUM for these.
-
 Internet Archive
 
 The Internet Archive backend utilizes Items on archive.org
@@ -29886,11 +30573,10 @@ Refer to IAS3 API documentation for the API this backend uses.
 Paths are specified as remote:bucket (or remote: for the lsd command.)
 You may put subdirectories in too, e.g. remote:item/path/to/dir.
 
-Once you have made a remote (see the provider specific section above)
-you can use it like this:
-
 Unlike S3, listing up all items uploaded by you isn't supported.
 
+Once you have made a remote, you can use it like this:
+
 Make a new item
 
     rclone mkdir remote:item
@@ -29929,7 +30615,7 @@ file. The metadata will appear as file metadata on Internet Archive.
 However, some fields are reserved by both Internet Archive and rclone.
 
 The following are reserved by Internet Archive: - name - source - size -
-md5 - crc32 - sha1 - format - old_version - viruscheck
+md5 - crc32 - sha1 - format - old_version - viruscheck - summation
 
 Trying to set values to these keys is ignored with a warning. Only
 setting mtime is an exception. Doing so make it the identical behavior
@@ -30140,65 +30826,52 @@ including them.
 Here are the possible system metadata items for the internetarchive
 backend.
 
-  ----------------------------------------------------------------------------------------------------------------------
-  Name                  Help               Type        Example                                      Read Only
-  --------------------- ------------------ ----------- -------------------------------------------- --------------------
-  crc32                 CRC32 calculated   string      01234567                                     N
-                        by Internet                                                                 
-                        Archive                                                                     
-
-  format                Name of format     string      Comma-Separated Values                       N
-                        identified by                                                               
-                        Internet Archive                                                            
-
-  md5                   MD5 hash           string      01234567012345670123456701234567             N
-                        calculated by                                                               
-                        Internet Archive                                                            
-
-  mtime                 Time of last       RFC 3339    2006-01-02T15:04:05.999999999Z               N
-                        modification,                                                               
-                        managed by Rclone                                                           
-
-  name                  Full file path,    filename    backend/internetarchive/internetarchive.go   N
-                        without the bucket                                                          
-                        part                                                                        
-
-  old_version           Whether the file   boolean     true                                         N
-                        was replaced and                                                            
-                        moved by                                                                    
-                        keep-old-version                                                            
-                        flag                                                                        
-
-  rclone-ia-mtime       Time of last       RFC 3339    2006-01-02T15:04:05.999999999Z               N
-                        modification,                                                               
-                        managed by                                                                  
-                        Internet Archive                                                            
-
-  rclone-mtime          Time of last       RFC 3339    2006-01-02T15:04:05.999999999Z               N
-                        modification,                                                               
-                        managed by Rclone                                                           
-
-  rclone-update-track   Random value used  string      aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa             N
-                        by Rclone for                                                               
-                        tracking changes                                                            
-                        inside Internet                                                             
-                        Archive                                                                     
-
-  sha1                  SHA1 hash          string      0123456701234567012345670123456701234567     N
-                        calculated by                                                               
-                        Internet Archive                                                            
-
-  size                  File size in bytes decimal     123456                                       N
-                                           number                                                   
-
-  source                The source of the  string      original                                     N
-                        file                                                                        
-
-  viruscheck            The last time      unixtime    1654191352                                   N
-                        viruscheck process                                                          
-                        was run for the                                                             
-                        file (?)                                                                    
-  ----------------------------------------------------------------------------------------------------------------------
+  --------------------------------------------------------------------------------------------------------------------------------------
+  Name                  Help                               Type        Example                                      Read Only
+  --------------------- ---------------------------------- ----------- -------------------------------------------- --------------------
+  crc32                 CRC32 calculated by Internet       string      01234567                                     Y
+                        Archive                                                                                     
+
+  format                Name of format identified by       string      Comma-Separated Values                       Y
+                        Internet Archive                                                                            
+
+  md5                   MD5 hash calculated by Internet    string      01234567012345670123456701234567             Y
+                        Archive                                                                                     
+
+  mtime                 Time of last modification, managed RFC 3339    2006-01-02T15:04:05.999999999Z               Y
+                        by Rclone                                                                                   
+
+  name                  Full file path, without the bucket filename    backend/internetarchive/internetarchive.go   Y
+                        part                                                                                        
+
+  old_version           Whether the file was replaced and  boolean     true                                         Y
+                        moved by keep-old-version flag                                                              
+
+  rclone-ia-mtime       Time of last modification, managed RFC 3339    2006-01-02T15:04:05.999999999Z               N
+                        by Internet Archive                                                                         
+
+  rclone-mtime          Time of last modification, managed RFC 3339    2006-01-02T15:04:05.999999999Z               N
+                        by Rclone                                                                                   
+
+  rclone-update-track   Random value used by Rclone for    string      aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa             N
+                        tracking changes inside Internet                                                            
+                        Archive                                                                                     
+
+  sha1                  SHA1 hash calculated by Internet   string      0123456701234567012345670123456701234567     Y
+                        Archive                                                                                     
+
+  size                  File size in bytes                 decimal     123456                                       Y
+                                                           number                                                   
+
+  source                The source of the file             string      original                                     Y
+
+  summation             Check                              string      md5                                          Y
+                        https://forum.rclone.org/t/31922                                                            
+                        for how it is used                                                                          
+
+  viruscheck            The last time viruscheck process   unixtime    1654191352                                   Y
+                        was run for the file (?)                                                                    
+  --------------------------------------------------------------------------------------------------------------------------------------
 
 See the metadata docs for more info.
 
@@ -30211,7 +30884,7 @@ companies, such as: * Telia * Telia Cloud (cloud.telia.se) * Telia Sky
 (sky.telia.no) * Tele2 * Tele2 Cloud (mittcloud.tele2.se) * Elkjøp (with
 subsidiaries): * Elkjøp Cloud (cloud.elkjop.no) * Elgiganten Sweden
 (cloud.elgiganten.se) * Elgiganten Denmark (cloud.elgiganten.dk) *
-Giganti Cloud (cloud.gigantti.fi) * ELKO Clouud (cloud.elko.is)
+Giganti Cloud (cloud.gigantti.fi) * ELKO Cloud (cloud.elko.is)
 
 Most of the white-label versions are supported by this backend, although
 may require different authentication setup - described below.
@@ -30228,11 +30901,38 @@ setting up the remote.
 
 Standard authentication
 
-To configure Jottacloud you will need to generate a personal security
-token in the Jottacloud web interface. You will the option to do in your
-account security settings (for whitelabel version you need to find this
-page in its web interface). Note that the web interface may refer to
-this token as a JottaCli token.
+The standard authentication method used by the official service
+(jottacloud.com), as well as some of the whitelabel services, requires
+you to generate a single-use personal login token from the account
+security settings in the service's web interface. Log in to your
+account, go to "Settings" and then "Security", or use the direct link
+presented to you by rclone when configuring the remote:
+https://www.jottacloud.com/web/secure. Scroll down to the section
+"Personal login token", and click the "Generate" button. Note that if
+you are using a whitelabel service you probably can't use the direct
+link, you need to find the same page in their dedicated web interface,
+and also it may be in a different location than described above.
+
+To access your account from multiple instances of rclone, you need to
+configure each of them with a separate personal login token. E.g. you
+create a Jottacloud remote with rclone in one location, and copy the
+configuration file to a second location where you also want to run
+rclone and access the same remote. Then you need to replace the token
+for one of them, using the config reconnect command, which requires you
+to generate a new personal login token and supply as input. If you do
+not do this, the token may easily end up being invalidated, resulting in
+both instances failing with an error message something along the lines
+of:
+
+    oauth2: cannot fetch token: 400 Bad Request
+    Response: {"error":"invalid_grant","error_description":"Stale token"}
+
+When this happens, you need to replace the token as described above to
+be able to use your remote again.
+
+All personal login tokens you have taken into use will be listed in the
+web interface under "My logged in devices", and from the right side of
+that list you can click the "X" button to revoke individual tokens.
 
 Legacy authentication
 
@@ -31403,7 +32103,7 @@ Failure to log-in
 Object not found
 
 If you are connecting to your Mega remote for the first time, to test
-access and syncronisation, you may receive an error such as
+access and synchronization, you may receive an error such as
 
     Failed to create file system for "my-mega-remote:": 
     couldn't login: Object (typically, node or user) not found
@@ -31777,7 +32477,7 @@ With NetStorage, directories can exist in one of two forms:
     have created in a storage group.
 2.  Implicit Directory. This refers to a directory within a path that
     has not been physically created. For example, during upload of a
-    file, non-existent subdirectories can be specified in the target
+    file, nonexistent subdirectories can be specified in the target
     path. NetStorage creates these as "implicit." While the directories
     aren't physically created, they exist implicitly and the noted path
     is connected with the uploaded file.
@@ -32589,7 +33289,7 @@ custom client_id is specified in the config. The default Client ID and
 Key are shared by all rclone users when performing requests.
 
 You may choose to create and use your own Client ID, in case the default
-one does not work well for you. For example, you might see throtting.
+one does not work well for you. For example, you might see throttling.
 
 Creating Client ID for OneDrive Personal
 
@@ -32642,7 +33342,7 @@ organization only, as shown below.
 2.  Follow the steps above to create an App. However, we need a
     different account type here:
     Accounts in this organizational directory only (*** - Single tenant).
-    Note that you can also change the account type aftering creating the
+    Note that you can also change the account type after creating the
     App.
 3.  Find the tenant ID of your organization.
 4.  In the rclone config, set auth_url to
@@ -32754,7 +33454,7 @@ Properties:
     -   "de"
         -   Microsoft Cloud Germany
     -   "cn"
-        -   Azure and Office 365 operated by 21Vianet in China
+        -   Azure and Office 365 operated by Vnet Group in China
 
 Advanced options
 
@@ -33072,7 +33772,7 @@ OneDrive can be found here.
 Versions
 
 Every change in a file OneDrive causes the service to create a new
-version of the the file. This counts against a users quota. For example
+version of the file. This counts against a users quota. For example
 changing the modification time of a file creates a second version, so
 the file apparently uses twice the space.
 
@@ -33397,6 +34097,533 @@ policy mfs (most free space) as a member of an rclone union remote.
 
 See List of backends that do not support rclone about and rclone about
 
+Oracle Object Storage
+
+Oracle Object Storage Overview
+
+Oracle Object Storage FAQ
+
+Paths are specified as remote:bucket (or remote: for the lsd command.)
+You may put subdirectories in too, e.g. remote:bucket/path/to/dir.
+
+Configuration
+
+Here is an example of making an oracle object storage configuration.
+rclone config walks you through it.
+
+Here is an example of how to make a remote called remote. First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+    n) New remote
+    d) Delete remote
+    r) Rename remote
+    c) Copy remote
+    s) Set configuration password
+    q) Quit config
+    e/n/d/r/c/s/q> n
+
+    Enter name for new remote.
+    name> remote
+
+    Option Storage.
+    Type of storage to configure.
+    Choose a number from below, or type in your own value.
+    [snip]
+    XX / Oracle Cloud Infrastructure Object Storage
+       \ (oracleobjectstorage)
+    Storage> oracleobjectstorage
+
+    Option provider.
+    Choose your Auth Provider
+    Choose a number from below, or type in your own string value.
+    Press Enter for the default (env_auth).
+     1 / automatically pickup the credentials from runtime(env), first one to provide auth wins
+       \ (env_auth)
+       / use an OCI user and an API key for authentication.
+     2 | you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key.
+       | https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm
+       \ (user_principal_auth)
+       / use instance principals to authorize an instance to make API calls. 
+     3 | each instance has its own identity, and authenticates using the certificates that are read from instance metadata. 
+       | https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm
+       \ (instance_principal_auth)
+     4 / use resource principals to make API calls
+       \ (resource_principal_auth)
+     5 / no credentials needed, this is typically for reading public buckets
+       \ (no_auth)
+    provider> 2
+
+    Option namespace.
+    Object storage namespace
+    Enter a value.
+    namespace> idbamagbg734
+
+    Option compartment.
+    Object storage compartment OCID
+    Enter a value.
+    compartment> ocid1.compartment.oc1..aaaaaaaapufkxc7ame3sthry5i7ujrwfc7ejnthhu6bhanm5oqfjpyasjkba
+
+    Option region.
+    Object storage Region
+    Enter a value.
+    region> us-ashburn-1
+
+    Option endpoint.
+    Endpoint for Object storage API.
+    Leave blank to use the default endpoint for the region.
+    Enter a value. Press Enter to leave empty.
+    endpoint> 
+
+    Option config_file.
+    Path to OCI config file
+    Choose a number from below, or type in your own string value.
+    Press Enter for the default (~/.oci/config).
+     1 / oci configuration file location
+       \ (~/.oci/config)
+    config_file> /etc/oci/dev.conf
+
+    Option config_profile.
+    Profile name inside OCI config file
+    Choose a number from below, or type in your own string value.
+    Press Enter for the default (Default).
+     1 / Use the default profile
+       \ (Default)
+    config_profile> Test
+
+    Edit advanced config?
+    y) Yes
+    n) No (default)
+    y/n> n
+
+    Configuration complete.
+    Options:
+    - type: oracleobjectstorage
+    - namespace: idbamagbg734
+    - compartment: ocid1.compartment.oc1..aaaaaaaapufkxc7ame3sthry5i7ujrwfc7ejnthhu6bhanm5oqfjpyasjkba
+    - region: us-ashburn-1
+    - provider: user_principal_auth
+    - config_file: /etc/oci/dev.conf
+    - config_profile: Test
+    Keep this "remote" remote?
+    y) Yes this is OK (default)
+    e) Edit this remote
+    d) Delete this remote
+    y/e/d> y
+
+See all buckets
+
+    rclone lsd remote:
+
+Create a new bucket
+
+    rclone mkdir remote:bucket
+
+List the contents of a bucket
+
+    rclone ls remote:bucket
+    rclone ls remote:bucket --max-depth 1
+
+Modified time
+
+The modified time is stored as metadata on the object as opc-meta-mtime
+as floating point since the epoch, accurate to 1 ns.
+
+If the modification time needs to be updated rclone will attempt to
+perform a server side copy to update the modification if the object can
+be copied in a single part. In the case the object is larger than 5Gb,
+the object will be uploaded rather than copied.
+
+Note that reading this from the object takes an additional HEAD request
+as the metadata isn't returned in object listings.
+
+Multipart uploads
+
+rclone supports multipart uploads with OOS which means that it can
+upload files bigger than 5 GiB.
+
+Note that files uploaded both with multipart upload and through crypt
+remotes do not have MD5 sums.
+
+rclone switches from single part uploads to multipart uploads at the
+point specified by --oos-upload-cutoff. This can be a maximum of 5 GiB
+and a minimum of 0 (ie always upload multipart files).
+
+The chunk sizes used in the multipart upload are specified by
+--oos-chunk-size and the number of chunks uploaded concurrently is
+specified by --oos-upload-concurrency.
+
+Multipart uploads will use --transfers * --oos-upload-concurrency *
+--oos-chunk-size extra memory. Single part uploads to not use extra
+memory.
+
+Single part transfers can be faster than multipart transfers or slower
+depending on your latency from oos - the more latency, the more likely
+single part transfers will be faster.
+
+Increasing --oos-upload-concurrency will increase throughput (8 would be
+a sensible value) and increasing --oos-chunk-size also increases
+throughput (16M would be sensible). Increasing either of these will use
+more memory. The default values are high enough to gain most of the
+possible performance without using too much memory.
+
+Standard options
+
+Here are the Standard options specific to oracleobjectstorage (Oracle
+Cloud Infrastructure Object Storage).
+
+--oos-provider
+
+Choose your Auth Provider
+
+Properties:
+
+-   Config: provider
+-   Env Var: RCLONE_OOS_PROVIDER
+-   Type: string
+-   Default: "env_auth"
+-   Examples:
+    -   "env_auth"
+        -   automatically pickup the credentials from runtime(env),
+            first one to provide auth wins
+    -   "user_principal_auth"
+        -   use an OCI user and an API key for authentication.
+        -   you’ll need to put in a config file your tenancy OCID, user
+            OCID, region, the path, fingerprint to an API key.
+        -   https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm
+    -   "instance_principal_auth"
+        -   use instance principals to authorize an instance to make API
+            calls.
+        -   each instance has its own identity, and authenticates using
+            the certificates that are read from instance metadata.
+        -   https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm
+    -   "resource_principal_auth"
+        -   use resource principals to make API calls
+    -   "no_auth"
+        -   no credentials needed, this is typically for reading public
+            buckets
+
+--oos-namespace
+
+Object storage namespace
+
+Properties:
+
+-   Config: namespace
+-   Env Var: RCLONE_OOS_NAMESPACE
+-   Type: string
+-   Required: true
+
+--oos-compartment
+
+Object storage compartment OCID
+
+Properties:
+
+-   Config: compartment
+-   Env Var: RCLONE_OOS_COMPARTMENT
+-   Provider: !no_auth
+-   Type: string
+-   Required: true
+
+--oos-region
+
+Object storage Region
+
+Properties:
+
+-   Config: region
+-   Env Var: RCLONE_OOS_REGION
+-   Type: string
+-   Required: true
+
+--oos-endpoint
+
+Endpoint for Object storage API.
+
+Leave blank to use the default endpoint for the region.
+
+Properties:
+
+-   Config: endpoint
+-   Env Var: RCLONE_OOS_ENDPOINT
+-   Type: string
+-   Required: false
+
+--oos-config-file
+
+Path to OCI config file
+
+Properties:
+
+-   Config: config_file
+-   Env Var: RCLONE_OOS_CONFIG_FILE
+-   Provider: user_principal_auth
+-   Type: string
+-   Default: "~/.oci/config"
+-   Examples:
+    -   "~/.oci/config"
+        -   oci configuration file location
+
+--oos-config-profile
+
+Profile name inside the oci config file
+
+Properties:
+
+-   Config: config_profile
+-   Env Var: RCLONE_OOS_CONFIG_PROFILE
+-   Provider: user_principal_auth
+-   Type: string
+-   Default: "Default"
+-   Examples:
+    -   "Default"
+        -   Use the default profile
+
+Advanced options
+
+Here are the Advanced options specific to oracleobjectstorage (Oracle
+Cloud Infrastructure Object Storage).
+
+--oos-upload-cutoff
+
+Cutoff for switching to chunked upload.
+
+Any files larger than this will be uploaded in chunks of chunk_size. The
+minimum is 0 and the maximum is 5 GiB.
+
+Properties:
+
+-   Config: upload_cutoff
+-   Env Var: RCLONE_OOS_UPLOAD_CUTOFF
+-   Type: SizeSuffix
+-   Default: 200Mi
+
+--oos-chunk-size
+
+Chunk size to use for uploading.
+
+When uploading files larger than upload_cutoff or files with unknown
+size (e.g. from "rclone rcat" or uploaded with "rclone mount" or google
+photos or google docs) they will be uploaded as multipart uploads using
+this chunk size.
+
+Note that "upload_concurrency" chunks of this size are buffered in
+memory per transfer.
+
+If you are transferring large files over high-speed links and you have
+enough memory, then increasing this will speed up the transfers.
+
+Rclone will automatically increase the chunk size when uploading a large
+file of known size to stay below the 10,000 chunks limit.
+
+Files of unknown size are uploaded with the configured chunk_size. Since
+the default chunk size is 5 MiB and there can be at most 10,000 chunks,
+this means that by default the maximum size of a file you can stream
+upload is 48 GiB. If you wish to stream upload larger files then you
+will need to increase chunk_size.
+
+Increasing the chunk size decreases the accuracy of the progress
+statistics displayed with "-P" flag.
+
+Properties:
+
+-   Config: chunk_size
+-   Env Var: RCLONE_OOS_CHUNK_SIZE
+-   Type: SizeSuffix
+-   Default: 5Mi
+
+--oos-upload-concurrency
+
+Concurrency for multipart uploads.
+
+This is the number of chunks of the same file that are uploaded
+concurrently.
+
+If you are uploading small numbers of large files over high-speed links
+and these uploads do not fully utilize your bandwidth, then increasing
+this may help to speed up the transfers.
+
+Properties:
+
+-   Config: upload_concurrency
+-   Env Var: RCLONE_OOS_UPLOAD_CONCURRENCY
+-   Type: int
+-   Default: 10
+
+--oos-copy-cutoff
+
+Cutoff for switching to multipart copy.
+
+Any files larger than this that need to be server-side copied will be
+copied in chunks of this size.
+
+The minimum is 0 and the maximum is 5 GiB.
+
+Properties:
+
+-   Config: copy_cutoff
+-   Env Var: RCLONE_OOS_COPY_CUTOFF
+-   Type: SizeSuffix
+-   Default: 4.656Gi
+
+--oos-copy-timeout
+
+Timeout for copy.
+
+Copy is an asynchronous operation, specify timeout to wait for copy to
+succeed
+
+Properties:
+
+-   Config: copy_timeout
+-   Env Var: RCLONE_OOS_COPY_TIMEOUT
+-   Type: Duration
+-   Default: 1m0s
+
+--oos-disable-checksum
+
+Don't store MD5 checksum with object metadata.
+
+Normally rclone will calculate the MD5 checksum of the input before
+uploading it so it can add it to metadata on the object. This is great
+for data integrity checking but can cause long delays for large files to
+start uploading.
+
+Properties:
+
+-   Config: disable_checksum
+-   Env Var: RCLONE_OOS_DISABLE_CHECKSUM
+-   Type: bool
+-   Default: false
+
+--oos-encoding
+
+The encoding for the backend.
+
+See the encoding section in the overview for more info.
+
+Properties:
+
+-   Config: encoding
+-   Env Var: RCLONE_OOS_ENCODING
+-   Type: MultiEncoder
+-   Default: Slash,InvalidUtf8,Dot
+
+--oos-leave-parts-on-error
+
+If true avoid calling abort upload on a failure, leaving all
+successfully uploaded parts on S3 for manual recovery.
+
+It should be set to true for resuming uploads across different sessions.
+
+WARNING: Storing parts of an incomplete multipart upload counts towards
+space usage on object storage and will add additional costs if not
+cleaned up.
+
+Properties:
+
+-   Config: leave_parts_on_error
+-   Env Var: RCLONE_OOS_LEAVE_PARTS_ON_ERROR
+-   Type: bool
+-   Default: false
+
+--oos-no-check-bucket
+
+If set, don't attempt to check the bucket exists or create it.
+
+This can be useful when trying to minimise the number of transactions
+rclone does if you know the bucket exists already.
+
+It can also be needed if the user you are using does not have bucket
+creation permissions.
+
+Properties:
+
+-   Config: no_check_bucket
+-   Env Var: RCLONE_OOS_NO_CHECK_BUCKET
+-   Type: bool
+-   Default: false
+
+Backend commands
+
+Here are the commands specific to the oracleobjectstorage backend.
+
+Run them with
+
+    rclone backend COMMAND remote:
+
+The help below will explain what arguments each command takes.
+
+See the backend command for more info on how to pass options and
+arguments.
+
+These can be run on a running backend using the rc command
+backend/command.
+
+rename
+
+change the name of an object
+
+    rclone backend rename remote: [options] [+]
+
+This command can be used to rename a object.
+
+Usage Examples:
+
+    rclone backend rename oos:bucket relative-object-path-under-bucket object-new-name
+
+list-multipart-uploads
+
+List the unfinished multipart uploads
+
+    rclone backend list-multipart-uploads remote: [options] [+]
+
+This command lists the unfinished multipart uploads in JSON format.
+
+    rclone backend list-multipart-uploads oos:bucket/path/to/object
+
+It returns a dictionary of buckets with values as lists of unfinished
+multipart uploads.
+
+You can call it with no bucket in which case it lists all bucket, with a
+bucket or with a bucket and path.
+
+    {
+      "test-bucket": [
+                {
+                        "namespace": "test-namespace",
+                        "bucket": "test-bucket",
+                        "object": "600m.bin",
+                        "uploadId": "51dd8114-52a4-b2f2-c42f-5291f05eb3c8",
+                        "timeCreated": "2022-07-29T06:21:16.595Z",
+                        "storageTier": "Standard"
+                }
+        ]
+
+cleanup
+
+Remove unfinished multipart uploads.
+
+    rclone backend cleanup remote: [options] [+]
+
+This command removes unfinished multipart uploads of age greater than
+max-age which defaults to 24 hours.
+
+Note that you can use -i/--dry-run with this command to see what it
+would do.
+
+    rclone backend cleanup oos:bucket/path/to/object
+    rclone backend cleanup -o max-age=7w oos:bucket/path/to/object
+
+Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
+
+Options:
+
+-   "max-age": Max age of upload to delete
+
 QingStor
 
 Paths are specified as remote:bucket (or remote: for the lsd command.)
@@ -34430,6 +35657,36 @@ Properties:
 -   Type: bool
 -   Default: false
 
+--swift-no-large-objects
+
+Disable support for static and dynamic large objects
+
+Swift cannot transparently store files bigger than 5 GiB. There are two
+schemes for doing that, static or dynamic large objects, and the API
+does not allow rclone to determine whether a file is a static or dynamic
+large object without doing a HEAD on the object. Since these need to be
+treated differently, this means rclone has to issue HEAD requests for
+objects for example when reading checksums.
+
+When no_large_objects is set, rclone will assume that there are no
+static or dynamic large objects stored. This means it can stop doing the
+extra HEAD calls which in turn increases performance greatly especially
+when doing a swift to swift transfer with --checksum set.
+
+Setting this option implies no_chunk and also that no files will be
+uploaded in chunks, so files bigger than 5 GiB will just fail on upload.
+
+If you set this option and there are static or dynamic large objects,
+then this will give incorrect hashes for them. Downloads will succeed,
+but other operations such as Remove and Copy will fail.
+
+Properties:
+
+-   Config: no_large_objects
+-   Env Var: RCLONE_SWIFT_NO_LARGE_OBJECTS
+-   Type: bool
+-   Default: false
+
 --swift-encoding
 
 The encoding for the backend.
@@ -35431,9 +36688,9 @@ installations.
 Paths are specified as remote:path. If the path does not begin with a /
 it is relative to the home directory of the user. An empty path remote:
 refers to the user's home directory. For example, rclone lsd remote:
-would list the home directory of the user cofigured in the rclone remote
-config (i.e /home/sftpuser). However, rclone lsd remote:/ would list the
-root directory for remote machine (i.e. /)
+would list the home directory of the user configured in the rclone
+remote config (i.e /home/sftpuser). However, rclone lsd remote:/ would
+list the root directory for remote machine (i.e. /)
 
 Note that some SFTP servers will need the leading / - Synology is a good
 example of this. rsync.net and Hetzner, on the other hand, requires
@@ -35669,7 +36926,7 @@ and later can also run a SSH server, which is a port of OpenSSH (see
 official installation guide). On a Windows server the shell handling is
 different: Although it can also be set up to use a Unix type shell, e.g.
 Cygwin bash, the default is to use Windows Command Prompt (cmd.exe), and
-PowerShell is a recommended alternative. All of these have bahave
+PowerShell is a recommended alternative. All of these have behave
 differently, which rclone must handle.
 
 Rclone tries to auto-detect what type of shell is used on the server,
@@ -35700,15 +36957,15 @@ default rclone will try to run a shell command the first time a new sftp
 remote is accessed. If you configure a sftp remote without a config
 file, e.g. an on the fly remote, rclone will have nowhere to store the
 result, and it will re-run the command on every access. To avoid this
-you should explicitely set the shell_type option to the correct value,
-or to none if you want to prevent rclone from executing any remote shell
+you should explicitly set the shell_type option to the correct value, or
+to none if you want to prevent rclone from executing any remote shell
 commands.
 
 It is also important to note that, since the shell type decides how
 quoting and escaping of file paths used as command-line arguments are
 performed, configuring the wrong shell type may leave you exposed to
 command injection exploits. Make sure to confirm the auto-detected shell
-type, or explicitely set the shell type you know is correct, or disable
+type, or explicitly set the shell type you know is correct, or disable
 shell access until you know.
 
 Checksum
@@ -36197,19 +37454,23 @@ Properties:
 
 Upload and download chunk size.
 
-This controls the maximum packet size used in the SFTP protocol. The RFC
-limits this to 32768 bytes (32k), however a lot of servers support
-larger sizes and setting it larger will increase transfer speed
-dramatically on high latency links.
-
-Only use a setting higher than 32k if you always connect to the same
-server or after sufficiently broad testing.
+This controls the maximum size of payload in SFTP protocol packets. The
+RFC limits this to 32768 bytes (32k), which is the default. However, a
+lot of servers support larger sizes, typically limited to a maximum
+total package size of 256k, and setting it larger will increase transfer
+speed dramatically on high latency links. This includes OpenSSH, and,
+for example, using the value of 255k works well, leaving plenty of room
+for overhead while still being within a total packet size of 256k.
 
-For example using the value of 252k with OpenSSH works well with its
-maximum packet size of 256k.
-
-If you get the error "failed to send packet header: EOF" when copying a
-large file, try lowering this number.
+Make sure to test thoroughly before using a value higher than 32k, and
+only use it if you always connect to the same server or after
+sufficiently broad testing. If you get errors such as "failed to send
+packet payload: EOF", lots of "connection lost", or "corrupted on
+transfer", when copying a larger file, try lowering the value. The
+server run by rclone serve sftp sends packets with standard 32k maximum
+payload so you must not set a different chunk_size when downloading
+files, but it accepts packets up to the 256k total size, so for uploads
+the chunk_size can be set as for the OpenSSH example above.
 
 Properties:
 
@@ -36291,6 +37552,234 @@ Hetzner Storage Boxes are supported through the SFTP backend on port 23.
 
 See Hetzner's documentation for details
 
+SMB
+
+SMB is a communication protocol to share files over network.
+
+This relies on go-smb2 library for communication with SMB protocol.
+
+Paths are specified as remote:sharename (or remote: for the lsd
+command.) You may put subdirectories in too, e.g.
+remote:item/path/to/dir.
+
+Notes
+
+The first path segment must be the name of the share, which you entered
+when you started to share on Windows. On smbd, it's the section title in
+smb.conf (usually in /etc/samba/) file. You can find shares by quering
+the root if you're unsure (e.g. rclone lsd remote:).
+
+You can't access to the shared printers from rclone, obviously.
+
+You can't use Anonymous access for logging in. You have to use the guest
+user with an empty password instead. The rclone client tries to avoid
+8.3 names when uploading files by encoding trailing spaces and periods.
+Alternatively, the local backend on Windows can access SMB servers using
+UNC paths, by \\server\share. This doesn't apply to non-Windows OSes,
+such as Linux and macOS.
+
+Configuration
+
+Here is an example of making a SMB configuration.
+
+First run
+
+    rclone config
+
+This will guide you through an interactive setup process.
+
+    No remotes found, make a new one?
+    n) New remote
+    s) Set configuration password
+    q) Quit config
+    n/s/q> n
+    name> remote
+    Option Storage.
+    Type of storage to configure.
+    Choose a number from below, or type in your own value.
+    XX / SMB / CIFS
+       \ (smb)
+    Storage> smb
+
+    Option host.
+    Samba hostname to connect to.
+    E.g. "example.com".
+    Enter a value.
+    host> localhost
+
+    Option user.
+    Samba username.
+    Enter a string value. Press Enter for the default (lesmi).
+    user> guest
+
+    Option port.
+    Samba port number.
+    Enter a signed integer. Press Enter for the default (445).
+    port> 
+
+    Option pass.
+    Samba password.
+    Choose an alternative below. Press Enter for the default (n).
+    y) Yes, type in my own password
+    g) Generate random password
+    n) No, leave this optional password blank (default)
+    y/g/n> g
+    Password strength in bits.
+    64 is just about memorable
+    128 is secure
+    1024 is the maximum
+    Bits> 64
+    Your password is: XXXX
+    Use this password? Please note that an obscured version of this 
+    password (and not the password itself) will be stored under your 
+    configuration file, so keep this generated password in a safe place.
+    y) Yes (default)
+    n) No
+    y/n> y
+
+    Option domain.
+    Domain name for NTLM authentication.
+    Enter a string value. Press Enter for the default (WORKGROUP).
+    domain> 
+
+    Edit advanced config?
+    y) Yes
+    n) No (default)
+    y/n> n
+
+    Configuration complete.
+    Options:
+    - type: samba
+    - host: localhost
+    - user: guest
+    - pass: *** ENCRYPTED ***
+    Keep this "remote" remote?
+    y) Yes this is OK (default)
+    e) Edit this remote
+    d) Delete this remote
+    y/e/d> d
+
+Standard options
+
+Here are the Standard options specific to smb (SMB / CIFS).
+
+--smb-host
+
+SMB server hostname to connect to.
+
+E.g. "example.com".
+
+Properties:
+
+-   Config: host
+-   Env Var: RCLONE_SMB_HOST
+-   Type: string
+-   Required: true
+
+--smb-user
+
+SMB username.
+
+Properties:
+
+-   Config: user
+-   Env Var: RCLONE_SMB_USER
+-   Type: string
+-   Default: "$USER"
+
+--smb-port
+
+SMB port number.
+
+Properties:
+
+-   Config: port
+-   Env Var: RCLONE_SMB_PORT
+-   Type: int
+-   Default: 445
+
+--smb-pass
+
+SMB password.
+
+NB Input to this must be obscured - see rclone obscure.
+
+Properties:
+
+-   Config: pass
+-   Env Var: RCLONE_SMB_PASS
+-   Type: string
+-   Required: false
+
+--smb-domain
+
+Domain name for NTLM authentication.
+
+Properties:
+
+-   Config: domain
+-   Env Var: RCLONE_SMB_DOMAIN
+-   Type: string
+-   Default: "WORKGROUP"
+
+Advanced options
+
+Here are the Advanced options specific to smb (SMB / CIFS).
+
+--smb-idle-timeout
+
+Max time before closing idle connections.
+
+If no connections have been returned to the connection pool in the time
+given, rclone will empty the connection pool.
+
+Set to 0 to keep connections indefinitely.
+
+Properties:
+
+-   Config: idle_timeout
+-   Env Var: RCLONE_SMB_IDLE_TIMEOUT
+-   Type: Duration
+-   Default: 1m0s
+
+--smb-hide-special-share
+
+Hide special shares (e.g. print$) which users aren't supposed to access.
+
+Properties:
+
+-   Config: hide_special_share
+-   Env Var: RCLONE_SMB_HIDE_SPECIAL_SHARE
+-   Type: bool
+-   Default: true
+
+--smb-case-insensitive
+
+Whether the server is configured to be case-insensitive.
+
+Always true on Windows shares.
+
+Properties:
+
+-   Config: case_insensitive
+-   Env Var: RCLONE_SMB_CASE_INSENSITIVE
+-   Type: bool
+-   Default: true
+
+--smb-encoding
+
+The encoding for the backend.
+
+See the encoding section in the overview for more info.
+
+Properties:
+
+-   Config: encoding
+-   Env Var: RCLONE_SMB_ENCODING
+-   Type: MultiEncoder
+-   Default:
+    Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot
+
 Storj
 
 Storj is an encrypted, secure, and cost-effective object storage service
@@ -38939,6 +40428,189 @@ Options:
 
 Changelog
 
+v1.60.0 - 2022-10-21
+
+See commits
+
+-   New backends
+    -   Oracle object storage (Manoj Ghosh)
+    -   SMB / CIFS (Windows file sharing) (Lesmiscore)
+    -   New S3 providers
+        -   IONOS Cloud Storage (Dmitry Deniskin)
+        -   Qiniu KODO (Bachue Zhou)
+-   New Features
+    -   build
+        -   Update to go1.19 and make go1.17 the minimum required
+            version (Nick Craig-Wood)
+        -   Install.sh: fix arm-v7 download (Ole Frost)
+    -   fs: Warn the user when using an existing remote name without a
+        colon (Nick Craig-Wood)
+    -   httplib: Add --xxx-min-tls-version option to select minimum TLS
+        version for HTTP servers (Robert Newson)
+    -   librclone: Add PHP bindings and test program (Jordi Gonzalez
+        Muñoz)
+    -   operations
+        -   Add --server-side-across-configs global flag for any backend
+            (Nick Craig-Wood)
+        -   Optimise --copy-dest and --compare-dest (Nick Craig-Wood)
+    -   rc: add job/stopgroup to stop group (Evan Spensley)
+    -   serve dlna
+        -   Add --announce-interval to control SSDP Announce Interval
+            (YanceyChiew)
+        -   Add --interface to Specify SSDP interface names line (Simon
+            Bos)
+        -   Add support for more external subtitles (YanceyChiew)
+        -   Add verification of addresses (YanceyChiew)
+    -   sync: Optimise --copy-dest and --compare-dest (Nick Craig-Wood)
+    -   doc updates (albertony, Alexander Knorr, anonion, João Henrique
+        Franco, Josh Soref, Lorenzo Milesi, Marco Molteni, Mark Trolley,
+        Ole Frost, partev, Ryan Morey, Tom Mombourquette, YFdyh000)
+-   Bug Fixes
+    -   filter
+        -   Fix incorrect filtering with UseFilter context flag and
+            wrapping backends (Nick Craig-Wood)
+        -   Make sure we check --files-from when looking for a single
+            file (Nick Craig-Wood)
+    -   rc
+        -   Fix mount/listmounts not returning the full Fs entered in
+            mount/mount (Tom Mombourquette)
+        -   Handle external unmount when mounting (Isaac Aymerich)
+        -   Validate Daemon option is not set when mounting a volume via
+            RC (Isaac Aymerich)
+    -   sync: Update docs and error messages to reflect fixes to overlap
+        checks (Nick Naumann)
+-   VFS
+    -   Reduce memory use by embedding sync.Cond (Nick Craig-Wood)
+    -   Reduce memory usage by re-ordering commonly used structures
+        (Nick Craig-Wood)
+    -   Fix excess CPU used by VFS cache cleaner looping (Nick
+        Craig-Wood)
+-   Local
+    -   Obey file filters in listing to fix errors on excluded files
+        (Nick Craig-Wood)
+    -   Fix "Failed to read metadata: function not implemented" on old
+        Linux kernels (Nick Craig-Wood)
+-   Compress
+    -   Fix crash due to nil metadata (Nick Craig-Wood)
+    -   Fix error handling to not use or return nil objects (Nick
+        Craig-Wood)
+-   Drive
+    -   Make --drive-stop-on-upload-limit obey quota exceeded error
+        (Steve Kowalik)
+-   FTP
+    -   Add --ftp-force-list-hidden option to show hidden items (Øyvind
+        Heddeland Instefjord)
+    -   Fix hang when using ExplicitTLS to certain servers. (Nick
+        Craig-Wood)
+-   Google Cloud Storage
+    -   Add --gcs-endpoint flag and config parameter (Nick Craig-Wood)
+-   Hubic
+    -   Remove backend as service has now shut down (Nick Craig-Wood)
+-   Onedrive
+    -   Rename Onedrive(cn) 21Vianet to Vnet Group (Yen Hu)
+    -   Disable change notify in China region since it is not supported
+        (Nick Craig-Wood)
+-   S3
+    -   Implement --s3-versions flag to show old versions of objects if
+        enabled (Nick Craig-Wood)
+    -   Implement --s3-version-at flag to show versions of objects at a
+        particular time (Nick Craig-Wood)
+    -   Implement backend versioning command to get/set bucket
+        versioning (Nick Craig-Wood)
+    -   Implement Purge to purge versions and backend cleanup-hidden
+        (Nick Craig-Wood)
+    -   Add --s3-decompress flag to decompress gzip-encoded files (Nick
+        Craig-Wood)
+    -   Add --s3-sse-customer-key-base64 to supply keys with binary data
+        (Richard Bateman)
+    -   Try to keep the maximum precision in ModTime with
+        --user-server-modtime (Nick Craig-Wood)
+    -   Drop binary metadata with an ERROR message as it can't be stored
+        (Nick Craig-Wood)
+    -   Add --s3-no-system-metadata to suppress read and write of system
+        metadata (Nick Craig-Wood)
+-   SFTP
+    -   Fix directory creation races (Lesmiscore)
+-   Swift
+    -   Add --swift-no-large-objects to reduce HEAD requests (Nick
+        Craig-Wood)
+-   Union
+    -   Propagate SlowHash feature to fix hasher interaction
+        (Lesmiscore)
+
+v1.59.2 - 2022-09-15
+
+See commits
+
+-   Bug Fixes
+    -   config: Move locking to fix fatal error: concurrent map read and
+        map write (Nick Craig-Wood)
+-   Local
+    -   Disable xattr support if the filesystems indicates it is not
+        supported (Nick Craig-Wood)
+-   Azure Blob
+    -   Fix chunksize calculations producing too many parts (Nick
+        Craig-Wood)
+-   B2
+    -   Fix chunksize calculations producing too many parts (Nick
+        Craig-Wood)
+-   S3
+    -   Fix chunksize calculations producing too many parts (Nick
+        Craig-Wood)
+
+v1.59.1 - 2022-08-08
+
+See commits
+
+-   Bug Fixes
+    -   accounting: Fix panic in core/stats-reset with unknown group
+        (Nick Craig-Wood)
+    -   build: Fix android build after GitHub actions change (Nick
+        Craig-Wood)
+    -   dlna: Fix SOAP action header parsing (Joram Schrijver)
+    -   docs: Fix links to mount command from install docs (albertony)
+    -   dropbox: Fix ChangeNotify was unable to decrypt errors (Nick
+        Craig-Wood)
+    -   fs: Fix parsing of times and durations of the form "YYYY-MM-DD
+        HH:MM:SS" (Nick Craig-Wood)
+    -   serve sftp: Fix checksum detection (Nick Craig-Wood)
+    -   sync: Add accidentally missed filter-sensitivity to --backup-dir
+        option (Nick Naumann)
+-   Combine
+    -   Fix docs showing remote= instead of upstreams= (Nick Craig-Wood)
+    -   Throw error if duplicate directory name is specified (Nick
+        Craig-Wood)
+    -   Fix errors with backends shutting down while in use (Nick
+        Craig-Wood)
+-   Dropbox
+    -   Fix hang on quit with --dropbox-batch-mode off (Nick Craig-Wood)
+    -   Fix infinite loop on uploading a corrupted file (Nick
+        Craig-Wood)
+-   Internetarchive
+    -   Ignore checksums for files using the different method
+        (Lesmiscore)
+    -   Handle hash symbol in the middle of filename (Lesmiscore)
+-   Jottacloud
+    -   Fix working with whitelabel Elgiganten Cloud
+    -   Do not store username in config when using standard auth
+        (albertony)
+-   Mega
+    -   Fix nil pointer exception when bad node received (Nick
+        Craig-Wood)
+-   S3
+    -   Fix --s3-no-head panic: reflect: Elem of invalid type
+        s3.PutObjectInput (Nick Craig-Wood)
+-   SFTP
+    -   Fix issue with WS_FTP by working around failing RealPath
+        (albertony)
+-   Union
+    -   Fix duplicated files when using directories with leading / (Nick
+        Craig-Wood)
+    -   Fix multiple files being uploaded when roots don't exist (Nick
+        Craig-Wood)
+    -   Fix panic due to misalignment of struct field in 32 bit
+        architectures (r-ricci)
+
 v1.59.0 - 2022-07-09
 
 See commits
@@ -39252,7 +40924,7 @@ See commits
             change (Nick Craig-Wood)
         -   Hard fork github.com/jlaffaye/ftp to fix
             go get github.com/rclone/rclone (Nick Craig-Wood)
-    -   oauthutil: Fix crash when webrowser requests /robots.txt (Nick
+    -   oauthutil: Fix crash when webbrowser requests /robots.txt (Nick
         Craig-Wood)
     -   operations: Fix goroutine leak in case of copy retry (Ankur
         Gupta)
@@ -39370,7 +41042,7 @@ See commits
         (Nick Craig-Wood)
     -   Fix timeout on hashing large files by sending keepalives (Nick
         Craig-Wood)
-    -   Fix unecessary seeking when uploading and downloading files
+    -   Fix unnecessary seeking when uploading and downloading files
         (Nick Craig-Wood)
     -   Update docs on how to create known_hosts file (Nick Craig-Wood)
 -   Storj
@@ -40170,9 +41842,9 @@ See commits
         -   Add toggle option for average s3ize in directory - key 'a'
             (Adam Plánský)
         -   Add empty folder flag into ncdu browser (Adam Plánský)
-        -   Add ! (errror) and . (unreadable) file flags to go with e
+        -   Add ! (error) and . (unreadable) file flags to go with e
             (empty) (Nick Craig-Wood)
-    -   obscure: Make rclone osbcure - ignore newline at end of line
+    -   obscure: Make rclone obscure - ignore newline at end of line
         (Nick Craig-Wood)
     -   operations
         -   Add logs when need to upload files to set mod times (Nick
@@ -40209,7 +41881,7 @@ See commits
     -   move: Fix data loss when source and destination are the same
         object (Nick Craig-Wood)
     -   operations
-        -   Fix --cutof-mode hard not cutting off immediately (Nick
+        -   Fix --cutoff-mode hard not cutting off immediately (Nick
             Craig-Wood)
         -   Fix --immutable error message (Nick Craig-Wood)
     -   sync
@@ -40278,7 +41950,7 @@ See commits
     -   Fixed crash on an empty file name (lluuaapp)
 -   Box
     -   Fix NewObject for files that differ in case (Nick Craig-Wood)
-    -   Fix finding directories in a case insentive way (Nick
+    -   Fix finding directories in a case insensitive way (Nick
         Craig-Wood)
 -   Chunker
     -   Skip long local hashing, hash in-transit (fixes) (Ivan Andreev)
@@ -40394,7 +42066,7 @@ See commits
         Craig-Wood)
 -   Sugarsync
     -   Fix NewObject for files that differ in case (Nick Craig-Wood)
-    -   Fix finding directories in a case insentive way (Nick
+    -   Fix finding directories in a case insensitive way (Nick
         Craig-Wood)
 -   Swift
     -   Fix deletion of parts of Static Large Object (SLO) (Nguyễn Hữu
@@ -40490,7 +42162,7 @@ v1.53.2 - 2020-10-26
 See commits
 
 -   Bug Fixes
-    -   acounting
+    -   accounting
         -   Fix incorrect speed and transferTime in core/stats (Nick
             Craig-Wood)
         -   Stabilize display order of transfers on Windows (Nick
@@ -41854,8 +43526,8 @@ v1.49.0 - 2019-08-26
     -   rcd: Fix permissions problems on cache directory with web gui
         download (Nick Craig-Wood)
 -   Mount
-    -   Default --daemon-timout to 15 minutes on macOS and FreeBSD (Nick
-        Craig-Wood)
+    -   Default --daemon-timeout to 15 minutes on macOS and FreeBSD
+        (Nick Craig-Wood)
     -   Update docs to show mounting from root OK for bucket-based (Nick
         Craig-Wood)
     -   Remove nonseekable flag from write files (Nick Craig-Wood)
@@ -42286,7 +43958,7 @@ v1.46 - 2019-02-09
 -   HTTP
     -   Add an example with username and password which is supported but
         wasn't documented (Nick Craig-Wood)
-    -   Fix backend with --files-from and non-existent files (Nick
+    -   Fix backend with --files-from and nonexistent files (Nick
         Craig-Wood)
 -   Hubic
     -   Make error message more informative if authentication fails
@@ -42882,7 +44554,7 @@ v1.41 - 2018-04-28
 -   FTP
     -   Work around strange response from box FTP server
     -   More workarounds for FTP servers to fix mkParentDir error
-    -   Fix no error on listing non-existent directory
+    -   Fix no error on listing nonexistent directory
 -   Google Cloud Storage
     -   Add service_account_credentials (Matt Holt)
     -   Detect bucket presence by listing it - minimises permissions
@@ -42973,7 +44645,7 @@ v1.40 - 2018-03-19
     -   Make a beta release for all branches on the main repo (but not
         pull requests)
 -   Bug Fixes
-    -   config: fixes errors on non existing config by loading config
+    -   config: fixes errors on nonexistent config by loading config
         file only on first access
     -   config: retry saving the config after failure (Mateusz)
     -   sync: when using --backup-dir don't delete files if we can't set
@@ -43601,7 +45273,7 @@ v1.34 - 2016-11-06
         Tomasz Mazur
 -   S3
     -   Command line and config file support for
-        -   Setting/overriding ACL - thanks Radek Senfeld
+        -   Setting/overriding ACL - thanks Radek Šenfeld
         -   Setting storage class - thanks Asko Tamm
 -   Drive
     -   Make exponential backoff work exactly as per Google
@@ -45089,6 +46761,32 @@ email addresses removed from here need to be addeed to bin/.ignore-emails to mak
 -   Lorenzo Maiorfi maiorfi@gmail.com
 -   Claudio Maradonna penguyman@stronzi.org
 -   Ovidiu Victor Tatar ovi.tatar@googlemail.com
+-   Evan Spensley epspensley@gmail.com
+-   Yen Hu 61753151+0x59656e@users.noreply.github.com
+-   Steve Kowalik steven@wedontsleep.org
+-   Jordi Gonzalez Muñoz jordigonzm@gmail.com
+-   Joram Schrijver i@joram.io
+-   Mark Trolley marktrolley@gmail.com
+-   João Henrique Franco joaohenrique.franco@gmail.com
+-   anonion aman207@users.noreply.github.com
+-   Ryan Morey 4590343+rmorey@users.noreply.github.com
+-   Simon Bos simonbos9@gmail.com
+-   YFdyh000 yfdyh000@gmail.com * Josh Soref
+    2119212+jsoref@users.noreply.github.com
+-   Øyvind Heddeland Instefjord instefjord@outlook.com
+-   Dmitry Deniskin 110819396+ddeniskin@users.noreply.github.com
+-   Alexander Knorr 106825+opexxx@users.noreply.github.com
+-   Richard Bateman richard@batemansr.us
+-   Dimitri Papadopoulos Orfanos
+    3234522+DimitriPapadopoulos@users.noreply.github.com
+-   Lorenzo Milesi lorenzo.milesi@yetopen.com
+-   Isaac Aymerich isaac.aymerich@gmail.com
+-   YanceyChiew 35898533+YanceyChiew@users.noreply.github.com
+-   Manoj Ghosh msays2000@gmail.com
+-   Bachue Zhou bachue.shu@gmail.com
+-   Manoj Ghosh manoj.ghosh@oracle.com
+-   Tom Mombourquette tom@devnode.com
+-   Robert Newson rnewson@apache.org
 
 Contact the rclone project
 
diff --git a/docs/content/changelog.md b/docs/content/changelog.md
index c11146b4f55c4..f68c8f5a0e492 100644
--- a/docs/content/changelog.md
+++ b/docs/content/changelog.md
@@ -5,6 +5,82 @@ description: "Rclone Changelog"
 
 # Changelog
 
+## v1.60.0 - 2022-10-21
+
+[See commits](https://github.com/rclone/rclone/compare/v1.59.0...v1.60.0)
+
+* New backends
+    * [Oracle object storage](/oracleobjectstorage/) (Manoj Ghosh)
+    * [SMB](/smb/) / CIFS (Windows file sharing) (Lesmiscore)
+    * New S3 providers
+        * [IONOS Cloud Storage](/s3/#ionos) (Dmitry Deniskin)
+        * [Qiniu KODO](/s3/#qiniu) (Bachue Zhou)
+* New Features
+    * build
+        * Update to go1.19 and make go1.17 the minimum required version (Nick Craig-Wood)
+        * Install.sh: fix arm-v7 download (Ole Frost)
+    * fs: Warn the user when using an existing remote name without a colon (Nick Craig-Wood)
+    * httplib: Add `--xxx-min-tls-version` option to select minimum TLS version for HTTP servers (Robert Newson)
+    * librclone: Add PHP bindings and test program (Jordi Gonzalez Muñoz)
+    * operations
+        * Add `--server-side-across-configs` global flag for any backend (Nick Craig-Wood)
+        * Optimise `--copy-dest` and `--compare-dest` (Nick Craig-Wood)
+    * rc: add `job/stopgroup` to stop group (Evan Spensley)
+    * serve dlna
+        * Add `--announce-interval` to control SSDP Announce Interval (YanceyChiew)
+        * Add `--interface` to Specify SSDP interface names line (Simon Bos)
+        * Add support for more external subtitles (YanceyChiew)
+        * Add verification of addresses (YanceyChiew)
+    * sync: Optimise `--copy-dest` and `--compare-dest` (Nick Craig-Wood)
+    * doc updates (albertony, Alexander Knorr, anonion, João Henrique Franco, Josh Soref, Lorenzo Milesi, Marco Molteni, Mark Trolley, Ole Frost, partev, Ryan Morey, Tom Mombourquette, YFdyh000)
+* Bug Fixes
+    * filter
+        * Fix incorrect filtering with `UseFilter` context flag and wrapping backends (Nick Craig-Wood)
+        * Make sure we check `--files-from` when looking for a single file (Nick Craig-Wood)
+    * rc
+        * Fix `mount/listmounts` not returning the full Fs entered in `mount/mount` (Tom Mombourquette)
+        * Handle external unmount when mounting (Isaac Aymerich)
+        * Validate Daemon option is not set when mounting a volume via RC (Isaac Aymerich)
+    * sync: Update docs and error messages to reflect fixes to overlap checks (Nick Naumann)
+* VFS
+    * Reduce memory use by embedding `sync.Cond` (Nick Craig-Wood)
+    * Reduce memory usage by re-ordering commonly used structures (Nick Craig-Wood)
+    * Fix excess CPU used by VFS cache cleaner looping (Nick Craig-Wood)
+* Local
+    * Obey file filters in listing to fix errors on excluded files (Nick Craig-Wood)
+    * Fix "Failed to read metadata: function not implemented" on old Linux kernels (Nick Craig-Wood)
+* Compress
+    * Fix crash due to nil metadata (Nick Craig-Wood)
+    * Fix error handling to not use or return nil objects (Nick Craig-Wood)
+* Drive
+    * Make `--drive-stop-on-upload-limit` obey quota exceeded error (Steve Kowalik)
+* FTP
+    * Add `--ftp-force-list-hidden` option to show hidden items (Øyvind Heddeland Instefjord)
+    * Fix hang when using ExplicitTLS to certain servers. (Nick Craig-Wood)
+* Google Cloud Storage
+    * Add `--gcs-endpoint` flag and config parameter (Nick Craig-Wood)
+* Hubic
+    * Remove backend as service has now shut down (Nick Craig-Wood)
+* Onedrive
+    * Rename Onedrive(cn) 21Vianet to Vnet Group (Yen Hu)
+    * Disable change notify in China region since it is not supported (Nick Craig-Wood)
+* S3
+    * Implement `--s3-versions` flag to show old versions of objects if enabled (Nick Craig-Wood)
+    * Implement `--s3-version-at` flag to show versions of objects at a particular time (Nick Craig-Wood)
+    * Implement `backend versioning` command to get/set bucket versioning (Nick Craig-Wood)
+    * Implement `Purge` to purge versions and `backend cleanup-hidden` (Nick Craig-Wood)
+    * Add `--s3-decompress` flag to decompress gzip-encoded files (Nick Craig-Wood)
+    * Add `--s3-sse-customer-key-base64` to supply keys with binary data (Richard Bateman)
+    * Try to keep the maximum precision in ModTime with `--user-server-modtime` (Nick Craig-Wood)
+    * Drop binary metadata with an ERROR message as it can't be stored (Nick Craig-Wood)
+    * Add `--s3-no-system-metadata` to suppress read and write of system metadata (Nick Craig-Wood)
+* SFTP
+    * Fix directory creation races (Lesmiscore)
+* Swift
+    * Add `--swift-no-large-objects` to reduce HEAD requests (Nick Craig-Wood)
+* Union
+    * Propagate SlowHash feature to fix hasher interaction (Lesmiscore)
+
 ## v1.59.2 - 2022-09-15
 
 [See commits](https://github.com/rclone/rclone/compare/v1.59.1...v1.59.2)
diff --git a/docs/content/commands/rclone.md b/docs/content/commands/rclone.md
index e98dc744369ed..f256d6f4f4891 100644
--- a/docs/content/commands/rclone.md
+++ b/docs/content/commands/rclone.md
@@ -37,7 +37,7 @@ See the [global flags page](/flags/) for global options not listed here.
 * [rclone about](/commands/rclone_about/)	 - Get quota information from the remote.
 * [rclone authorize](/commands/rclone_authorize/)	 - Remote authorization.
 * [rclone backend](/commands/rclone_backend/)	 - Run a backend-specific command.
-* [rclone bisync](/commands/rclone_bisync/)	 - Perform bidirectonal synchronization between two paths.
+* [rclone bisync](/commands/rclone_bisync/)	 - Perform bidirectional synchronization between two paths.
 * [rclone cat](/commands/rclone_cat/)	 - Concatenates any files and sends them to stdout.
 * [rclone check](/commands/rclone_check/)	 - Checks the files in the source and destination match.
 * [rclone checksum](/commands/rclone_checksum/)	 - Checks the files in the source against a SUM file.
diff --git a/docs/content/commands/rclone_bisync.md b/docs/content/commands/rclone_bisync.md
index 76fcca809e61d..4a7127329b32c 100644
--- a/docs/content/commands/rclone_bisync.md
+++ b/docs/content/commands/rclone_bisync.md
@@ -1,17 +1,17 @@
 ---
 title: "rclone bisync"
-description: "Perform bidirectonal synchronization between two paths."
+description: "Perform bidirectional synchronization between two paths."
 slug: rclone_bisync
 url: /commands/rclone_bisync/
 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/bisync/ and as part of making a release run "make commanddocs"
 ---
 # rclone bisync
 
-Perform bidirectonal synchronization between two paths.
+Perform bidirectional synchronization between two paths.
 
 ## Synopsis
 
-Perform bidirectonal synchronization between two paths.
+Perform bidirectional synchronization between two paths.
 
 [Bisync](https://rclone.org/bisync/) provides a
 bidirectional cloud sync solution in rclone.
diff --git a/docs/content/commands/rclone_completion_bash.md b/docs/content/commands/rclone_completion_bash.md
index b3c24a6e44682..b5772be3ec1a1 100644
--- a/docs/content/commands/rclone_completion_bash.md
+++ b/docs/content/commands/rclone_completion_bash.md
@@ -28,7 +28,7 @@ To load completions for every new session, execute once:
 
 ### macOS:
 
-	rclone completion bash > /usr/local/etc/bash_completion.d/rclone
+	rclone completion bash > $(brew --prefix)/etc/bash_completion.d/rclone
 
 You will need to start a new shell for this setup to take effect.
 
diff --git a/docs/content/commands/rclone_completion_zsh.md b/docs/content/commands/rclone_completion_zsh.md
index b48faa25a43fd..1490817f71326 100644
--- a/docs/content/commands/rclone_completion_zsh.md
+++ b/docs/content/commands/rclone_completion_zsh.md
@@ -18,6 +18,10 @@ to enable it.  You can execute the following once:
 
 	echo "autoload -U compinit; compinit" >> ~/.zshrc
 
+To load completions in your current shell session:
+
+	source <(rclone completion zsh); compdef _rclone rclone
+
 To load completions for every new session, execute once:
 
 ### Linux:
@@ -26,7 +30,7 @@ To load completions for every new session, execute once:
 
 ### macOS:
 
-	rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone
+	rclone completion zsh > $(brew --prefix)/share/zsh/site-functions/_rclone
 
 You will need to start a new shell for this setup to take effect.
 
diff --git a/docs/content/commands/rclone_config_create.md b/docs/content/commands/rclone_config_create.md
index e1b5572c1d8f3..8d12ea86b512a 100644
--- a/docs/content/commands/rclone_config_create.md
+++ b/docs/content/commands/rclone_config_create.md
@@ -45,7 +45,7 @@ are 100% certain you are already passing obscured passwords then use
 `rclone config password` command.
 
 The flag `--non-interactive` is for use by applications that wish to
-configure rclone themeselves, rather than using rclone's text based
+configure rclone themselves, rather than using rclone's text based
 configuration questions. If this flag is set, and rclone needs to ask
 the user a question, a JSON blob will be returned with the question in
 it.
diff --git a/docs/content/commands/rclone_config_update.md b/docs/content/commands/rclone_config_update.md
index c4fc4e26edb3c..468a683076cfb 100644
--- a/docs/content/commands/rclone_config_update.md
+++ b/docs/content/commands/rclone_config_update.md
@@ -45,7 +45,7 @@ are 100% certain you are already passing obscured passwords then use
 `rclone config password` command.
 
 The flag `--non-interactive` is for use by applications that wish to
-configure rclone themeselves, rather than using rclone's text based
+configure rclone themselves, rather than using rclone's text based
 configuration questions. If this flag is set, and rclone needs to ask
 the user a question, a JSON blob will be returned with the question in
 it.
diff --git a/docs/content/commands/rclone_hashsum.md b/docs/content/commands/rclone_hashsum.md
index 0c34c0cd65bd0..f9b96e6a4af4e 100644
--- a/docs/content/commands/rclone_hashsum.md
+++ b/docs/content/commands/rclone_hashsum.md
@@ -26,7 +26,7 @@ For the MD5 and SHA1 algorithms there are also dedicated commands,
 
 This command can also hash data received on standard input (stdin),
 by not passing a remote:path, or by passing a hyphen as remote:path
-when there is data to read (if not, the hypen will be treated literaly,
+when there is data to read (if not, the hyphen will be treated literally,
 as a relative path).
 
 Run without a hash to see the list of all supported hashes, e.g.
diff --git a/docs/content/commands/rclone_ls.md b/docs/content/commands/rclone_ls.md
index 6a47d0c39a2fe..a1bfc8cfe4c7f 100644
--- a/docs/content/commands/rclone_ls.md
+++ b/docs/content/commands/rclone_ls.md
@@ -42,7 +42,7 @@ Note that `ls` and `lsl` recurse by default - use `--max-depth 1` to stop the re
 
 The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use `-R` to make them recurse.
 
-Listing a non-existent directory will produce an error except for
+Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).
 
diff --git a/docs/content/commands/rclone_lsd.md b/docs/content/commands/rclone_lsd.md
index fbd9b2c92f1fc..86a1f47c7b5cd 100644
--- a/docs/content/commands/rclone_lsd.md
+++ b/docs/content/commands/rclone_lsd.md
@@ -52,7 +52,7 @@ Note that `ls` and `lsl` recurse by default - use `--max-depth 1` to stop the re
 
 The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use `-R` to make them recurse.
 
-Listing a non-existent directory will produce an error except for
+Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).
 
diff --git a/docs/content/commands/rclone_lsf.md b/docs/content/commands/rclone_lsf.md
index 2cdd3ce5cf2c8..0a34277fae3fb 100644
--- a/docs/content/commands/rclone_lsf.md
+++ b/docs/content/commands/rclone_lsf.md
@@ -126,7 +126,7 @@ Note that `ls` and `lsl` recurse by default - use `--max-depth 1` to stop the re
 
 The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use `-R` to make them recurse.
 
-Listing a non-existent directory will produce an error except for
+Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).
 
diff --git a/docs/content/commands/rclone_lsjson.md b/docs/content/commands/rclone_lsjson.md
index abf8a39cacd45..e3cac5d21165a 100644
--- a/docs/content/commands/rclone_lsjson.md
+++ b/docs/content/commands/rclone_lsjson.md
@@ -56,7 +56,7 @@ If `--files-only` is not specified directories in addition to the files
 will be returned.
 
 If `--metadata` is set then an additional Metadata key will be returned.
-This will have metdata in rclone standard format as a JSON object.
+This will have metadata in rclone standard format as a JSON object.
 
 if `--stat` is set then a single JSON blob will be returned about the
 item pointed to. This will return an error if the item isn't found.
@@ -102,7 +102,7 @@ Note that `ls` and `lsl` recurse by default - use `--max-depth 1` to stop the re
 
 The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use `-R` to make them recurse.
 
-Listing a non-existent directory will produce an error except for
+Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).
 
diff --git a/docs/content/commands/rclone_lsl.md b/docs/content/commands/rclone_lsl.md
index 8fbaaba67f16c..f493916a92d03 100644
--- a/docs/content/commands/rclone_lsl.md
+++ b/docs/content/commands/rclone_lsl.md
@@ -42,7 +42,7 @@ Note that `ls` and `lsl` recurse by default - use `--max-depth 1` to stop the re
 
 The other list commands `lsd`,`lsf`,`lsjson` do not recurse by default - use `-R` to make them recurse.
 
-Listing a non-existent directory will produce an error except for
+Listing a nonexistent directory will produce an error except for
 remotes which can't have empty directories (e.g. s3, swift, or gcs -
 the bucket-based remotes).
 
diff --git a/docs/content/commands/rclone_md5sum.md b/docs/content/commands/rclone_md5sum.md
index de2b68fb0baed..9cd53cad08d20 100644
--- a/docs/content/commands/rclone_md5sum.md
+++ b/docs/content/commands/rclone_md5sum.md
@@ -26,7 +26,7 @@ to running `rclone hashsum MD5 remote:path`.
 
 This command can also hash data received on standard input (stdin),
 by not passing a remote:path, or by passing a hyphen as remote:path
-when there is data to read (if not, the hypen will be treated literaly,
+when there is data to read (if not, the hyphen will be treated literally,
 as a relative path).
 
 
diff --git a/docs/content/commands/rclone_mount.md b/docs/content/commands/rclone_mount.md
index fa132d24c689e..b322b23f89646 100644
--- a/docs/content/commands/rclone_mount.md
+++ b/docs/content/commands/rclone_mount.md
@@ -98,7 +98,7 @@ and experience unexpected program errors, freezes or other issues, consider moun
 as a network drive instead.
 
 When mounting as a fixed disk drive you can either mount to an unused drive letter,
-or to a path representing a **non-existent** subdirectory of an **existing** parent
+or to a path representing a **nonexistent** subdirectory of an **existing** parent
 directory or drive. Using the special value `*` will tell rclone to
 automatically assign the next available drive letter, starting with Z: and moving backward.
 Examples:
@@ -129,7 +129,7 @@ the mapped drive, shown in Windows Explorer etc, while the complete
 `\\server\share` will be reported as the remote UNC path by
 `net use` etc, just like a normal network drive mapping.
 
-If you specify a full network share UNC path with `--volname`, this will implicitely
+If you specify a full network share UNC path with `--volname`, this will implicitly
 set the `--network-mode` option, so the following two examples have same result:
 
     rclone mount remote:path/to/files X: --network-mode
@@ -138,7 +138,7 @@ set the `--network-mode` option, so the following two examples have same result:
 You may also specify the network share UNC path as the mountpoint itself. Then rclone
 will automatically assign a drive letter, same as with `*` and use that as
 mountpoint, and instead use the UNC path specified as the volume name, as if it were
-specified with the `--volname` option. This will also implicitely set
+specified with the `--volname` option. This will also implicitly set
 the `--network-mode` option. This means the following two examples have same result:
 
     rclone mount remote:path/to/files \\cloud\remote
@@ -174,7 +174,7 @@ The permissions on each entry will be set according to [options](#options)
 
 The default permissions corresponds to `--file-perms 0666 --dir-perms 0777`,
 i.e. read and write permissions to everyone. This means you will not be able
-to start any programs from the the mount. To be able to do that you must add
+to start any programs from the mount. To be able to do that you must add
 execute permissions, e.g. `--file-perms 0777 --dir-perms 0777` to add it
 to everyone. If the program needs to write files, chances are you will have
 to enable [VFS File Caching](#vfs-file-caching) as well (see also [limitations](#limitations)).
@@ -245,8 +245,8 @@ applications won't work with their files on an rclone mount without
 `--vfs-cache-mode writes` or `--vfs-cache-mode full`.
 See the [VFS File Caching](#vfs-file-caching) section for more info.
 
-The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2,
-Hubic) do not support the concept of empty directories, so empty
+The bucket-based remotes (e.g. Swift, S3, Google Compute Storage, B2)
+do not support the concept of empty directories, so empty
 directories will have a tendency to disappear once they fall out of
 the directory cache.
 
@@ -341,6 +341,8 @@ mount sftp1:subdir /mnt/data -t rclone -o vfs_cache_mode=writes,sftp_key_file=/p
 or create systemd mount units:
 ```
 # /etc/systemd/system/mnt-data.mount
+[Unit]
+After=network-online.target
 [Mount]
 Type=rclone
 What=sftp1:subdir
@@ -352,6 +354,7 @@ optionally accompanied by systemd automount unit
 ```
 # /etc/systemd/system/mnt-data.automount
 [Unit]
+After=network-online.target
 Before=remote-fs.target
 [Automount]
 Where=/mnt/data
diff --git a/docs/content/commands/rclone_ncdu.md b/docs/content/commands/rclone_ncdu.md
index e1f51e6049176..154eb219b3c2b 100644
--- a/docs/content/commands/rclone_ncdu.md
+++ b/docs/content/commands/rclone_ncdu.md
@@ -45,7 +45,7 @@ press '?' to toggle the help on and off. The supported keys are:
      q/ESC/^c to quit
 
 Listed files/directories may be prefixed by a one-character flag,
-some of them combined with a description in brackes at end of line.
+some of them combined with a description in brackets at end of line.
 These flags have the following meaning:
 
     e means this is an empty directory, i.e. contains no files (but
diff --git a/docs/content/commands/rclone_serve_dlna.md b/docs/content/commands/rclone_serve_dlna.md
index 9ab1e83b942bd..0cb85400e51fb 100644
--- a/docs/content/commands/rclone_serve_dlna.md
+++ b/docs/content/commands/rclone_serve_dlna.md
@@ -32,11 +32,6 @@ IPs.
 Use `--name` to choose the friendly server name, which is by
 default "rclone (hostname)".
 
-Use `--announce-interval` to specify the interval at which SSDP server
-announce devices and services. Larger active announcement intervals help
-keep the multicast domain clean, this value does not affect unicast
-responses to `M-SEARCH` requests from other devices.
-
 Use `--log-trace` in conjunction with `-vv` to enable additional debug
 logging of all UPNP traffic.
 
@@ -367,11 +362,13 @@ rclone serve dlna remote:path [flags]
 
 ```
       --addr string                            The ip:port or :port to bind the DLNA http server to (default ":7879")
+      --announce-interval duration             The interval between SSDP announcements (default 12m0s)
       --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
       --dir-perms FileMode                     Directory permissions (default 0777)
       --file-perms FileMode                    File permissions (default 0666)
       --gid uint32                             Override the gid field set by the filesystem (not supported on Windows) (default 1000)
   -h, --help                                   help for dlna
+      --interface stringArray                  The interface to use for SSDP (repeat as necessary)
       --log-trace                              Enable trace logging of SOAP traffic
       --name string                            Name of DLNA server
       --no-checksum                            Don't compare checksums on up/download
diff --git a/docs/content/commands/rclone_serve_http.md b/docs/content/commands/rclone_serve_http.md
index 329bc14207f58..5ad079b41d007 100644
--- a/docs/content/commands/rclone_serve_http.md
+++ b/docs/content/commands/rclone_serve_http.md
@@ -60,6 +60,10 @@ of that with the CA certificate.  `--key` should be the PEM encoded
 private key and `--client-ca` should be the PEM encoded client
 certificate authority certificate.
 
+--min-tls-version is minimum TLS version that is acceptable. Valid
+  values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
+  "tls1.0").
+
 ### Template
 
 `--template` allows a user to specify a custom markup template for HTTP
@@ -446,6 +450,7 @@ rclone serve http remote:path [flags]
       --htpasswd string                        A htpasswd file - if not provided no authentication is done
       --key string                             SSL PEM Private key
       --max-header-bytes int                   Maximum size of request header (default 4096)
+      --min-tls-version string                 Minimum TLS version that is acceptable (default "tls1.0")
       --no-checksum                            Don't compare checksums on up/download
       --no-modtime                             Don't read/write the modification time (can speed things up)
       --no-seek                                Don't allow seeking in files
diff --git a/docs/content/commands/rclone_serve_restic.md b/docs/content/commands/rclone_serve_restic.md
index 881e697f00615..fff9d2f2071ef 100644
--- a/docs/content/commands/rclone_serve_restic.md
+++ b/docs/content/commands/rclone_serve_restic.md
@@ -174,6 +174,10 @@ of that with the CA certificate.  `--key` should be the PEM encoded
 private key and `--client-ca` should be the PEM encoded client
 certificate authority certificate.
 
+--min-tls-version is minimum TLS version that is acceptable. Valid
+  values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
+  "tls1.0").
+
 
 ```
 rclone serve restic remote:path [flags]
@@ -192,6 +196,7 @@ rclone serve restic remote:path [flags]
       --htpasswd string                 htpasswd file - if not provided no authentication is done
       --key string                      SSL PEM Private key
       --max-header-bytes int            Maximum size of request header (default 4096)
+      --min-tls-version string          Minimum TLS version that is acceptable (default "tls1.0")
       --pass string                     Password for authentication
       --private-repos                   Users can only access their private repo
       --realm string                    Realm for authentication (default "rclone")
diff --git a/docs/content/commands/rclone_serve_sftp.md b/docs/content/commands/rclone_serve_sftp.md
index 226e2976946c2..9bed264fbebbf 100644
--- a/docs/content/commands/rclone_serve_sftp.md
+++ b/docs/content/commands/rclone_serve_sftp.md
@@ -11,11 +11,19 @@ Serve the remote over SFTP.
 
 ## Synopsis
 
-Run a SFTP server to serve a remote over SFTP. This can be used
-with an SFTP client or you can make a remote of type sftp to use with it.
+Run an SFTP server to serve a remote over SFTP. This can be used
+with an SFTP client or you can make a remote of type [sftp](/sftp) to use with it.
 
-You can use the filter flags (e.g. `--include`, `--exclude`) to control what
-is served.
+You can use the [filter](/filtering) flags (e.g. `--include`, `--exclude`)
+to control what is served.
+
+The server will respond to a small number of shell commands, mainly
+md5sum, sha1sum and df, which enable it to provide support for checksums
+and the about feature when accessed from an sftp remote.
+
+Note that this server uses standard 32 KiB packet payload size, which
+means you must not configure the client to expect anything else, e.g.
+with the [chunk_size](/sftp/#sftp-chunk-size) option on an sftp remote.
 
 The server will log errors.  Use `-v` to see access logs.
 
@@ -28,11 +36,6 @@ You must provide some means of authentication, either with
 `--auth-proxy`, or set the `--no-auth` flag for no
 authentication when logging in.
 
-Note that this also implements a small number of shell commands so
-that it can provide md5sum/sha1sum/df information for the rclone sftp
-backend.  This means that is can support SHA1SUMs, MD5SUMs and the
-about command when paired with the rclone sftp backend.
-
 If you don't supply a host `--key` then rclone will generate rsa, ecdsa
 and ed25519 variants, and cache them for later use in rclone's cache
 directory (see `rclone help flags cache-dir`) in the "serve-sftp"
@@ -484,7 +487,7 @@ rclone serve sftp remote:path [flags]
       --pass string                            Password for authentication
       --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --read-only                              Only allow read-only access
-      --stdio                                  Run an sftp server on run stdin/stdout
+      --stdio                                  Run an sftp server on stdin/stdout
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
       --user string                            User name for authentication
diff --git a/docs/content/commands/rclone_serve_webdav.md b/docs/content/commands/rclone_serve_webdav.md
index 5209719f43f4c..21c2491ea06b9 100644
--- a/docs/content/commands/rclone_serve_webdav.md
+++ b/docs/content/commands/rclone_serve_webdav.md
@@ -109,6 +109,10 @@ of that with the CA certificate.  `--key` should be the PEM encoded
 private key and `--client-ca` should be the PEM encoded client
 certificate authority certificate.
 
+--min-tls-version is minimum TLS version that is acceptable. Valid
+  values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
+  "tls1.0").
+
 ## VFS - Virtual File System
 
 This command uses the VFS layer. This adapts the cloud storage objects
@@ -531,6 +535,7 @@ rclone serve webdav remote:path [flags]
       --htpasswd string                        htpasswd file - if not provided no authentication is done
       --key string                             SSL PEM Private key
       --max-header-bytes int                   Maximum size of request header (default 4096)
+      --min-tls-version string                 Minimum TLS version that is acceptable (default "tls1.0")
       --no-checksum                            Don't compare checksums on up/download
       --no-modtime                             Don't read/write the modification time (can speed things up)
       --no-seek                                Don't allow seeking in files
diff --git a/docs/content/commands/rclone_sha1sum.md b/docs/content/commands/rclone_sha1sum.md
index a61b15e4593c0..49a09ec29ef35 100644
--- a/docs/content/commands/rclone_sha1sum.md
+++ b/docs/content/commands/rclone_sha1sum.md
@@ -26,7 +26,7 @@ to running `rclone hashsum SHA1 remote:path`.
 
 This command can also hash data received on standard input (stdin),
 by not passing a remote:path, or by passing a hyphen as remote:path
-when there is data to read (if not, the hypen will be treated literaly,
+when there is data to read (if not, the hyphen will be treated literally,
 as a relative path).
 
 This command can also hash data received on STDIN, if not passing
diff --git a/docs/content/commands/rclone_sync.md b/docs/content/commands/rclone_sync.md
index 0f7da84a244c1..173714f21d1ea 100644
--- a/docs/content/commands/rclone_sync.md
+++ b/docs/content/commands/rclone_sync.md
@@ -37,6 +37,11 @@ extended explanation in the [copy](/commands/rclone_copy/) command if unsure.
 If dest:path doesn't exist, it is created and the source:path contents
 go there.
 
+It is not possible to sync overlapping remotes. However, you may exclude
+the destination from the sync with a filter rule or by putting an 
+exclude-if-present file inside the destination directory and sync to a
+destination that is inside the source directory.
+
 **Note**: Use the `-P`/`--progress` flag to view real-time transfer statistics
 
 **Note**: Use the `rclone dedupe` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
diff --git a/docs/content/flags.md b/docs/content/flags.md
index acf93f6c7735e..1cdce0e1dd582 100644
--- a/docs/content/flags.md
+++ b/docs/content/flags.md
@@ -119,7 +119,7 @@ These flags are available for every command.
       --rc-job-expire-interval duration      Interval to check for expired async jobs (default 10s)
       --rc-key string                        SSL PEM Private key
       --rc-max-header-bytes int              Maximum size of request header (default 4096)
-      --rc-min-tls-version string            Minimum TLS version that is acceptable
+      --rc-min-tls-version string            Minimum TLS version that is acceptable (default "tls1.0")
       --rc-no-auth                           Don't require auth for certain methods
       --rc-pass string                       Password for authentication
       --rc-realm string                      Realm for authentication (default "rclone")
@@ -136,6 +136,7 @@ These flags are available for every command.
       --refresh-times                        Refresh the modtime of remote files
       --retries int                          Retry operations this many times if they fail (default 3)
       --retries-sleep duration               Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)
+      --server-side-across-configs           Allow server-side operations (e.g. copy) to work across different configs
       --size-only                            Skip based on size only, not mod-time or checksum
       --stats duration                       Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s)
       --stats-file-name-length int           Max file name length in stats (0 for no limit) (default 45)
@@ -161,7 +162,7 @@ These flags are available for every command.
       --use-json-log                         Use json log format
       --use-mmap                             Use mmap allocator (see docs)
       --use-server-modtime                   Use server modified time instead of object metadata
-      --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.59.0")
+      --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.60.0")
   -v, --verbose count                        Print lots more stuff (repeat for more)
 ```
 
@@ -348,6 +349,7 @@ and may be set in the config file.
       --ftp-disable-utf8                             Disable using UTF-8 even if server advertises support
       --ftp-encoding MultiEncoder                    The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
       --ftp-explicit-tls                             Use Explicit FTPS (FTP over TLS)
+      --ftp-force-list-hidden                        Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
       --ftp-host string                              FTP host to connect to
       --ftp-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
       --ftp-no-check-certificate                     Do not verify the TLS certificate of the server
@@ -358,7 +360,6 @@ and may be set in the config file.
       --ftp-tls-cache-size int                       Size of TLS session cache for all control and data connections (default 32)
       --ftp-user string                              FTP username (default "$USER")
       --ftp-writing-mdtm                             Use MDTM to set modification time (VsFtpd quirk)
-      --ftp-force-list-hidden                        Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD.
       --gcs-anonymous                                Access public buckets and objects without credentials
       --gcs-auth-url string                          Auth server URL
       --gcs-bucket-acl string                        Access Control List for new buckets
@@ -367,6 +368,7 @@ and may be set in the config file.
       --gcs-client-secret string                     OAuth Client Secret
       --gcs-decompress                               If set this will decompress gzip encoded objects
       --gcs-encoding MultiEncoder                    The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot)
+      --gcs-endpoint string                          Endpoint for the service
       --gcs-location string                          Location for the newly created buckets
       --gcs-no-check-bucket                          If set, don't attempt to check the bucket exists or create it
       --gcs-object-acl string                        Access Control List for new objects
@@ -412,14 +414,6 @@ and may be set in the config file.
       --http-no-head                                 Don't use HEAD requests
       --http-no-slash                                Set this if the site doesn't end directories with /
       --http-url string                              URL of HTTP host to connect to
-      --hubic-auth-url string                        Auth server URL
-      --hubic-chunk-size SizeSuffix                  Above this size files will be chunked into a _segments container (default 5Gi)
-      --hubic-client-id string                       OAuth Client Id
-      --hubic-client-secret string                   OAuth Client Secret
-      --hubic-encoding MultiEncoder                  The encoding for the backend (default Slash,InvalidUtf8)
-      --hubic-no-chunk                               Don't chunk files during streaming upload
-      --hubic-token string                           OAuth Access Token as a JSON blob
-      --hubic-token-url string                       Token server url
       --internetarchive-access-key-id string         IAS3 Access Key
       --internetarchive-disable-checksum             Don't ask the server to test against MD5 checksum calculated by rclone (default true)
       --internetarchive-encoding MultiEncoder        The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot)
@@ -535,6 +529,7 @@ and may be set in the config file.
       --s3-bucket-acl string                         Canned ACL used when creating buckets
       --s3-chunk-size SizeSuffix                     Chunk size to use for uploading (default 5Mi)
       --s3-copy-cutoff SizeSuffix                    Cutoff for switching to multipart copy (default 4.656Gi)
+      --s3-decompress                                If set this will decompress gzip encoded objects
       --s3-disable-checksum                          Don't store MD5 checksum with object metadata
       --s3-disable-http2                             Disable usage of http2 for S3 backends
       --s3-download-url string                       Custom endpoint for downloads
@@ -553,6 +548,7 @@ and may be set in the config file.
       --s3-no-check-bucket                           If set, don't attempt to check the bucket exists or create it
       --s3-no-head                                   If set, don't HEAD uploaded objects to check integrity
       --s3-no-head-object                            If set, do not do HEAD before GET when getting objects
+      --s3-no-system-metadata                        Suppress setting and reading of system metadata
       --s3-profile string                            Profile to use in the shared credentials file
       --s3-provider string                           Choose your S3 provider
       --s3-region string                             Region to connect to
@@ -562,7 +558,8 @@ and may be set in the config file.
       --s3-session-token string                      An AWS session token
       --s3-shared-credentials-file string            Path to the shared credentials file
       --s3-sse-customer-algorithm string             If using SSE-C, the server-side encryption algorithm used when storing this object in S3
-      --s3-sse-customer-key string                   If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data
+      --s3-sse-customer-key string                   To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data
+      --s3-sse-customer-key-base64 string            If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data
       --s3-sse-customer-key-md5 string               If using SSE-C you may provide the secret encryption key MD5 checksum (optional)
       --s3-sse-kms-key-id string                     If using KMS ID you must provide the ARN of Key
       --s3-storage-class string                      The storage class to use when storing new objects in S3
@@ -572,6 +569,8 @@ and may be set in the config file.
       --s3-use-multipart-etag Tristate               Whether to use ETag in multipart uploads for verification (default unset)
       --s3-use-presigned-request                     Whether to use a presigned request or PutObject for single part uploads
       --s3-v2-auth                                   If true use v2 authentication
+      --s3-version-at Time                           Show file versions as they were at the specified time (default off)
+      --s3-versions                                  Include old versions in directory listings
       --seafile-2fa                                  Two-factor authentication ('true' if the account has 2FA enabled)
       --seafile-create-library                       Should rclone create a library if it doesn't exist
       --seafile-encoding MultiEncoder                The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
@@ -618,6 +617,15 @@ and may be set in the config file.
       --sia-encoding MultiEncoder                    The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot)
       --sia-user-agent string                        Siad User Agent (default "Sia-Agent")
       --skip-links                                   Don't warn about skipped symlinks
+      --smb-case-insensitive                         Whether the server is configured to be case-insensitive (default true)
+      --smb-domain string                            Domain name for NTLM authentication (default "WORKGROUP")
+      --smb-encoding MultiEncoder                    The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot)
+      --smb-hide-special-share                       Hide special shares (e.g. print$) which users aren't supposed to access (default true)
+      --smb-host string                              SMB server hostname to connect to
+      --smb-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
+      --smb-pass string                              SMB password (obscured)
+      --smb-port int                                 SMB port number (default 445)
+      --smb-user string                              SMB username (default "$USER")
       --storj-access-grant string                    Access grant
       --storj-api-key string                         API key
       --storj-passphrase string                      Encryption passphrase
@@ -648,6 +656,7 @@ and may be set in the config file.
       --swift-key string                             API key or password (OS_PASSWORD)
       --swift-leave-parts-on-error                   If true avoid calling abort upload on a failure
       --swift-no-chunk                               Don't chunk files during streaming upload
+      --swift-no-large-objects                       Disable support for static and dynamic large objects
       --swift-region string                          Region name - optional (OS_REGION_NAME)
       --swift-storage-policy string                  The storage policy to use when creating a new container
       --swift-storage-url string                     Storage URL - optional (OS_STORAGE_URL)
diff --git a/docs/content/ftp.md b/docs/content/ftp.md
index 7d4da7d7b1e6b..0ccd0f915446b 100644
--- a/docs/content/ftp.md
+++ b/docs/content/ftp.md
@@ -248,6 +248,20 @@ Here are the Advanced options specific to ftp (FTP).
 
 Maximum number of FTP simultaneous connections, 0 for unlimited.
 
+Note that setting this is very likely to cause deadlocks so it should
+be used with care.
+
+If you are doing a sync or copy then make sure concurrency is one more
+than the sum of `--transfers` and `--checkers`.
+
+If you use `--check-first` then it just needs to be one more than the
+maximum of `--checkers` and `--transfers`.
+
+So for `concurrency 3` you'd use `--checkers 2 --transfers 2
+--check-first` or `--checkers 1 --transfers 1`.
+
+
+
 Properties:
 
 - Config:      concurrency
diff --git a/docs/content/googlecloudstorage.md b/docs/content/googlecloudstorage.md
index e2309058539cb..b98f890352bc1 100644
--- a/docs/content/googlecloudstorage.md
+++ b/docs/content/googlecloudstorage.md
@@ -621,6 +621,19 @@ Properties:
 - Type:        bool
 - Default:     false
 
+#### --gcs-endpoint
+
+Endpoint for the service.
+
+Leave blank normally.
+
+Properties:
+
+- Config:      endpoint
+- Env Var:     RCLONE_GCS_ENDPOINT
+- Type:        string
+- Required:    false
+
 #### --gcs-encoding
 
 The encoding for the backend.
diff --git a/docs/content/rc.md b/docs/content/rc.md
index 51d291720bcfb..4b966635e578c 100644
--- a/docs/content/rc.md
+++ b/docs/content/rc.md
@@ -982,6 +982,12 @@ Parameters:
 
 - jobid - id of the job (integer).
 
+### job/stopgroup: Stop all running jobs in a group {#job-stopgroup}
+
+Parameters:
+
+- group - name of the group (string).
+
 ### mount/listmounts: Show current mount points {#mount-listmounts}
 
 This shows currently mounted points, which can be used for performing an unmount.
@@ -1057,9 +1063,11 @@ Example:
 
 **Authentication is required for this call.**
 
-### mount/unmountall: Show current mount points {#mount-unmountall}
+### mount/unmountall: Unmount all active mounts {#mount-unmountall}
 
-This shows currently mounted points, which can be used for performing an unmount.
+rclone allows Linux, FreeBSD, macOS and Windows to
+mount any of Rclone's cloud storage systems as a file system with
+FUSE.
 
 This takes no parameters and returns error if unmount does not succeed.
 
diff --git a/docs/content/s3.md b/docs/content/s3.md
index a07e2dc27ded0..4ffa23922e4b9 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -641,7 +641,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t
 {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}}
 ### Standard options
 
-Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi).
+Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
 
 #### --s3-provider
 
@@ -676,6 +676,8 @@ Properties:
         - IBM COS S3
     - "IDrive"
         - IDrive e2
+    - "IONOS"
+        - IONOS Cloud
     - "LyveCloud"
         - Seagate Lyve Cloud
     - "Minio"
@@ -696,6 +698,8 @@ Properties:
         - Tencent Cloud Object Storage (COS)
     - "Wasabi"
         - Wasabi Object Storage
+    - "Qiniu"
+        - Qiniu Object Storage (Kodo)
     - "Other"
         - Any other S3 compatible provider
 
@@ -966,13 +970,68 @@ Properties:
 
 Region to connect to.
 
+Properties:
+
+- Config:      region
+- Env Var:     RCLONE_S3_REGION
+- Provider:    Qiniu
+- Type:        string
+- Required:    false
+- Examples:
+    - "cn-east-1"
+        - The default endpoint - a good choice if you are unsure.
+        - East China Region 1.
+        - Needs location constraint cn-east-1.
+    - "cn-east-2"
+        - East China Region 2.
+        - Needs location constraint cn-east-2.
+    - "cn-north-1"
+        - North China Region 1.
+        - Needs location constraint cn-north-1.
+    - "cn-south-1"
+        - South China Region 1.
+        - Needs location constraint cn-south-1.
+    - "us-north-1"
+        - North America Region.
+        - Needs location constraint us-north-1.
+    - "ap-southeast-1"
+        - Southeast Asia Region 1.
+        - Needs location constraint ap-southeast-1.
+    - "ap-northeast-1"
+        - Northeast Asia Region 1.
+        - Needs location constraint ap-northeast-1.
+
+#### --s3-region
+
+Region where your bucket will be created and your data stored.
+
+
+Properties:
+
+- Config:      region
+- Env Var:     RCLONE_S3_REGION
+- Provider:    IONOS
+- Type:        string
+- Required:    false
+- Examples:
+    - "de"
+        - Frankfurt, Germany
+    - "eu-central-2"
+        - Berlin, Germany
+    - "eu-south-2"
+        - Logrono, Spain
+
+#### --s3-region
+
+Region to connect to.
+
 Leave blank if you are using an S3 clone and you don't have a region.
 
 Properties:
 
 - Config:      region
 - Env Var:     RCLONE_S3_REGION
-- Provider:    !AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
+- Provider:    !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
 - Type:        string
 - Required:    false
 - Examples:
@@ -1230,6 +1289,27 @@ Properties:
 
 #### --s3-endpoint
 
+Endpoint for IONOS S3 Object Storage.
+
+Specify the endpoint from the same region.
+
+Properties:
+
+- Config:      endpoint
+- Env Var:     RCLONE_S3_ENDPOINT
+- Provider:    IONOS
+- Type:        string
+- Required:    false
+- Examples:
+    - "s3-eu-central-1.ionoscloud.com"
+        - Frankfurt, Germany
+    - "s3-eu-central-2.ionoscloud.com"
+        - Berlin, Germany
+    - "s3-eu-south-2.ionoscloud.com"
+        - Logrono, Spain
+
+#### --s3-endpoint
+
 Endpoint for OSS API.
 
 Properties:
@@ -1495,6 +1575,33 @@ Properties:
 
 #### --s3-endpoint
 
+Endpoint for Qiniu Object Storage.
+
+Properties:
+
+- Config:      endpoint
+- Env Var:     RCLONE_S3_ENDPOINT
+- Provider:    Qiniu
+- Type:        string
+- Required:    false
+- Examples:
+    - "s3-cn-east-1.qiniucs.com"
+        - East China Endpoint 1
+    - "s3-cn-east-2.qiniucs.com"
+        - East China Endpoint 2
+    - "s3-cn-north-1.qiniucs.com"
+        - North China Endpoint 1
+    - "s3-cn-south-1.qiniucs.com"
+        - South China Endpoint 1
+    - "s3-us-north-1.qiniucs.com"
+        - North America Endpoint 1
+    - "s3-ap-southeast-1.qiniucs.com"
+        - Southeast Asia Endpoint 1
+    - "s3-ap-northeast-1.qiniucs.com"
+        - Northeast Asia Endpoint 1
+
+#### --s3-endpoint
+
 Endpoint for S3 API.
 
 Required when using an S3 clone.
@@ -1503,7 +1610,7 @@ Properties:
 
 - Config:      endpoint
 - Env Var:     RCLONE_S3_ENDPOINT
-- Provider:    !AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp
+- Provider:    !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu
 - Type:        string
 - Required:    false
 - Examples:
@@ -1830,13 +1937,42 @@ Properties:
 
 Location constraint - must be set to match the Region.
 
+Used when creating buckets only.
+
+Properties:
+
+- Config:      location_constraint
+- Env Var:     RCLONE_S3_LOCATION_CONSTRAINT
+- Provider:    Qiniu
+- Type:        string
+- Required:    false
+- Examples:
+    - "cn-east-1"
+        - East China Region 1
+    - "cn-east-2"
+        - East China Region 2
+    - "cn-north-1"
+        - North China Region 1
+    - "cn-south-1"
+        - South China Region 1
+    - "us-north-1"
+        - North America Region 1
+    - "ap-southeast-1"
+        - Southeast Asia Region 1
+    - "ap-northeast-1"
+        - Northeast Asia Region 1
+
+#### --s3-location-constraint
+
+Location constraint - must be set to match the Region.
+
 Leave blank if not sure. Used when creating buckets only.
 
 Properties:
 
 - Config:      location_constraint
 - Env Var:     RCLONE_S3_LOCATION_CONSTRAINT
-- Provider:    !AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS
+- Provider:    !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS
 - Type:        string
 - Required:    false
 
@@ -2066,9 +2202,30 @@ Properties:
         - Archived storage.
         - Prices are lower, but it needs to be restored first to be accessed.
 
+#### --s3-storage-class
+
+The storage class to use when storing new objects in Qiniu.
+
+Properties:
+
+- Config:      storage_class
+- Env Var:     RCLONE_S3_STORAGE_CLASS
+- Provider:    Qiniu
+- Type:        string
+- Required:    false
+- Examples:
+    - "STANDARD"
+        - Standard storage class
+    - "LINE"
+        - Infrequent access storage mode
+    - "GLACIER"
+        - Archive storage mode
+    - "DEEP_ARCHIVE"
+        - Deep archive storage mode
+
 ### Advanced options
 
-Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi).
+Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
 
 #### --s3-bucket-acl
 
@@ -2131,7 +2288,9 @@ Properties:
 
 #### --s3-sse-customer-key
 
-If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.
+To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data.
+
+Alternatively you can provide --sse-customer-key-base64.
 
 Properties:
 
@@ -2144,6 +2303,23 @@ Properties:
     - ""
         - None
 
+#### --s3-sse-customer-key-base64
+
+If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data.
+
+Alternatively you can provide --sse-customer-key.
+
+Properties:
+
+- Config:      sse_customer_key_base64
+- Env Var:     RCLONE_S3_SSE_CUSTOMER_KEY_BASE64
+- Provider:    AWS,Ceph,ChinaMobile,Minio
+- Type:        string
+- Required:    false
+- Examples:
+    - ""
+        - None
+
 #### --s3-sse-customer-key-md5
 
 If using SSE-C you may provide the secret encryption key MD5 checksum (optional).
@@ -2663,6 +2839,36 @@ Properties:
 - Type:        Time
 - Default:     off
 
+#### --s3-decompress
+
+If set this will decompress gzip encoded objects.
+
+It is possible to upload objects to S3 with "Content-Encoding: gzip"
+set. Normally rclone will download these files as compressed objects.
+
+If this flag is set then rclone will decompress these files with
+"Content-Encoding: gzip" as they are received. This means that rclone
+can't check the size and hash but the file contents will be decompressed.
+
+
+Properties:
+
+- Config:      decompress
+- Env Var:     RCLONE_S3_DECOMPRESS
+- Type:        bool
+- Default:     false
+
+#### --s3-no-system-metadata
+
+Suppress setting and reading of system metadata
+
+Properties:
+
+- Config:      no_system_metadata
+- Env Var:     RCLONE_S3_NO_SYSTEM_METADATA
+- Type:        bool
+- Default:     false
+
 ### Metadata
 
 User metadata is stored as x-amz-meta- keys. S3 metadata keys are case insensitive and are always returned in lower case.
diff --git a/docs/content/sftp.md b/docs/content/sftp.md
index 48ebfb3a3af72..dfaf179b81582 100644
--- a/docs/content/sftp.md
+++ b/docs/content/sftp.md
@@ -789,19 +789,24 @@ Properties:
 
 Upload and download chunk size.
 
-This controls the maximum packet size used in the SFTP protocol. The
-RFC limits this to 32768 bytes (32k), however a lot of servers
-support larger sizes and setting it larger will increase transfer
-speed dramatically on high latency links.
-
-Only use a setting higher than 32k if you always connect to the same
-server or after sufficiently broad testing.
-
-For example using the value of 252k with OpenSSH works well with its
-maximum packet size of 256k.
-
-If you get the error "failed to send packet header: EOF" when copying
-a large file, try lowering this number.
+This controls the maximum size of payload in SFTP protocol packets.
+The RFC limits this to 32768 bytes (32k), which is the default. However,
+a lot of servers support larger sizes, typically limited to a maximum
+total package size of 256k, and setting it larger will increase transfer
+speed dramatically on high latency links. This includes OpenSSH, and,
+for example, using the value of 255k works well, leaving plenty of room
+for overhead while still being within a total packet size of 256k.
+
+Make sure to test thoroughly before using a value higher than 32k,
+and only use it if you always connect to the same server or after
+sufficiently broad testing. If you get errors such as
+"failed to send packet payload: EOF", lots of "connection lost",
+or "corrupted on transfer", when copying a larger file, try lowering
+the value. The server run by [rclone serve sftp](/commands/rclone_serve_sftp)
+sends packets with standard 32k maximum payload so you must not
+set a different chunk_size when downloading files, but it accepts
+packets up to the 256k total size, so for uploads the chunk_size
+can be set as for the OpenSSH example above.
 
 
 Properties:
diff --git a/docs/content/swift.md b/docs/content/swift.md
index f75e4f6ddb868..c58706efc067b 100644
--- a/docs/content/swift.md
+++ b/docs/content/swift.md
@@ -534,6 +534,38 @@ Properties:
 - Type:        bool
 - Default:     false
 
+#### --swift-no-large-objects
+
+Disable support for static and dynamic large objects
+
+Swift cannot transparently store files bigger than 5 GiB. There are
+two schemes for doing that, static or dynamic large objects, and the
+API does not allow rclone to determine whether a file is a static or
+dynamic large object without doing a HEAD on the object. Since these
+need to be treated differently, this means rclone has to issue HEAD
+requests for objects for example when reading checksums.
+
+When `no_large_objects` is set, rclone will assume that there are no
+static or dynamic large objects stored. This means it can stop doing
+the extra HEAD calls which in turn increases performance greatly
+especially when doing a swift to swift transfer with `--checksum` set.
+
+Setting this option implies `no_chunk` and also that no files will be
+uploaded in chunks, so files bigger than 5 GiB will just fail on
+upload.
+
+If you set this option and there *are* static or dynamic large objects,
+then this will give incorrect hashes for them. Downloads will succeed,
+but other operations such as Remove and Copy will fail.
+
+
+Properties:
+
+- Config:      no_large_objects
+- Env Var:     RCLONE_SWIFT_NO_LARGE_OBJECTS
+- Type:        bool
+- Default:     false
+
 #### --swift-encoding
 
 The encoding for the backend.
diff --git a/rclone.1 b/rclone.1
index 6b3c7fd376a86..fd0219cddcc37 100644
--- a/rclone.1
+++ b/rclone.1
@@ -1,7 +1,7 @@
 .\"t
 .\" Automatically generated by Pandoc 2.9.2.1
 .\"
-.TH "rclone" "1" "Jul 09, 2022" "User Manual" ""
+.TH "rclone" "1" "Oct 21, 2022" "User Manual" ""
 .hy
 .SH Rclone syncs your files to cloud storage
 .PP
@@ -163,8 +163,6 @@ Arvan Cloud Object Storage (AOS)
 .IP \[bu] 2
 Citrix ShareFile
 .IP \[bu] 2
-C14
-.IP \[bu] 2
 Cloudflare R2
 .IP \[bu] 2
 DigitalOcean Spaces
@@ -193,8 +191,6 @@ HiDrive
 .IP \[bu] 2
 HTTP
 .IP \[bu] 2
-Hubic
-.IP \[bu] 2
 Internet Archive
 .IP \[bu] 2
 Jottacloud
@@ -203,6 +199,8 @@ IBM COS S3
 .IP \[bu] 2
 IDrive e2
 .IP \[bu] 2
+IONOS Cloud
+.IP \[bu] 2
 Koofr
 .IP \[bu] 2
 Mail.ru Cloud
@@ -227,7 +225,9 @@ OpenDrive
 .IP \[bu] 2
 OpenStack Swift
 .IP \[bu] 2
-Oracle Cloud Storage
+Oracle Cloud Storage Swift
+.IP \[bu] 2
+Oracle Object Storage
 .IP \[bu] 2
 ownCloud
 .IP \[bu] 2
@@ -239,6 +239,8 @@ put.io
 .IP \[bu] 2
 QingStor
 .IP \[bu] 2
+Qiniu Cloud Object Storage (Kodo)
+.IP \[bu] 2
 Rackspace Cloud Files
 .IP \[bu] 2
 rsync.net
@@ -255,6 +257,8 @@ SFTP
 .IP \[bu] 2
 Sia
 .IP \[bu] 2
+SMB / CIFS
+.IP \[bu] 2
 StackPath
 .IP \[bu] 2
 Storj
@@ -318,7 +322,7 @@ See rclone config docs (https://rclone.org/docs/) for more details.
 .IP \[bu] 2
 Optionally configure automatic execution.
 .PP
-See below for some expanded Linux / macOS instructions.
+See below for some expanded Linux / macOS / Windows instructions.
 .PP
 See the usage (https://rclone.org/docs/) docs for how to use rclone, or
 run \f[C]rclone -h\f[R].
@@ -346,7 +350,8 @@ sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta
 .PP
 Note that this script checks the version of rclone installed first and
 won\[aq]t re-download if not needed.
-.SS Linux installation from precompiled binary
+.SS Linux installation
+.SS Precompiled binary
 .PP
 Fetch and unpack
 .IP
@@ -386,7 +391,8 @@ See rclone config docs (https://rclone.org/docs/) for more details.
 rclone config
 \f[R]
 .fi
-.SS macOS installation with brew
+.SS macOS installation
+.SS Installation with brew
 .IP
 .nf
 \f[C]
@@ -398,7 +404,14 @@ NOTE: This version of rclone will not support \f[C]mount\f[R] any more
 (see #5373 (https://github.com/rclone/rclone/issues/5373)).
 If mounting is wanted on macOS, either install a precompiled binary or
 enable the relevant option when installing from source.
-.SS macOS installation from precompiled binary, using curl
+.PP
+Note that this is a third party installer not controlled by the rclone
+developers so it may be out of date.
+Its current version is as below.
+.PP
+[IMAGE: Homebrew
+package (https://repology.org/badge/version-for-repo/homebrew/rclone.svg)] (https://repology.org/project/rclone/versions)
+.SS Precompiled binary, using curl
 .PP
 To avoid problems with macOS gatekeeper enforcing the binary to be
 signed and notarized it is enough to download with \f[C]curl\f[R].
@@ -448,7 +461,7 @@ See rclone config docs (https://rclone.org/docs/) for more details.
 rclone config
 \f[R]
 .fi
-.SS macOS installation from precompiled binary, using a web browser
+.SS Precompiled binary, using a web browser
 .PP
 When downloading a binary with a web browser, the browser will set the
 macOS gatekeeper quarantine attribute.
@@ -469,12 +482,89 @@ The simplest fix is to run
 xattr -d com.apple.quarantine rclone
 \f[R]
 .fi
-.SS Install with docker
+.SS Windows installation
+.SS Precompiled binary
+.PP
+Fetch the correct binary for your processor type by clicking on these
+links.
+If not sure, use the first link.
+.IP \[bu] 2
+Intel/AMD - 64
+Bit (https://downloads.rclone.org/rclone-current-linux-amd64.zip)
+.IP \[bu] 2
+Intel/AMD - 32
+Bit (https://downloads.rclone.org/rclone-current-linux-386.zip)
+.IP \[bu] 2
+ARM - 64
+Bit (https://downloads.rclone.org/rclone-current-linux-arm64.zip)
+.PP
+Open this file in the Explorer and extract \f[C]rclone.exe\f[R].
+Rclone is a portable executable so you can place it wherever is
+convenient.
+.PP
+Open a CMD window (or powershell) and run the binary.
+Note that rclone does not launch a GUI by default, it runs in the CMD
+Window.
+.IP \[bu] 2
+Run \f[C]rclone.exe config\f[R] to setup.
+See rclone config docs (https://rclone.org/docs/) for more details.
+.IP \[bu] 2
+Optionally configure automatic execution.
+.PP
+If you are planning to use the rclone
+mount (https://rclone.org/commands/rclone_mount/) feature then you will
+need to install the third party utility WinFsp (https://winfsp.dev/)
+also.
+.SS Chocolatey package manager
+.PP
+Make sure you have Choco (https://chocolatey.org/) installed
+.IP
+.nf
+\f[C]
+choco search rclone
+choco install rclone
+\f[R]
+.fi
+.PP
+This will install rclone on your Windows machine.
+If you are planning to use rclone
+mount (https://rclone.org/commands/rclone_mount/) then
+.IP
+.nf
+\f[C]
+choco install winfsp
+\f[R]
+.fi
 .PP
-The rclone maintains a docker image for
+will install that too.
+.PP
+Note that this is a third party installer not controlled by the rclone
+developers so it may be out of date.
+Its current version is as below.
+.PP
+[IMAGE: Chocolatey
+package (https://repology.org/badge/version-for-repo/chocolatey/rclone.svg)] (https://repology.org/project/rclone/versions)
+.SS Package manager installation
+.PP
+Many Linux, Windows, macOS and other OS distributions package and
+distribute rclone.
+.PP
+The distributed versions of rclone are often quite out of date and for
+this reason we recommend one of the other installation methods if
+possible.
+.PP
+You can get an idea of how up to date or not your OS distribution\[aq]s
+package is here.
+.PP
+[IMAGE: Packaging
+status (https://repology.org/badge/vertical-allrepos/rclone.svg?columns=3)] (https://repology.org/project/rclone/versions)
+.SS Docker installation
+.PP
+The rclone developers maintain a docker image for
 rclone (https://hub.docker.com/r/rclone/rclone).
-These images are autobuilt by docker hub from the rclone source based on
-a minimal Alpine linux image.
+.PP
+These images are built as part of the release process based on a minimal
+Alpine Linux.
 .PP
 The \f[C]:latest\f[R] tag will always point to the latest stable
 release.
@@ -568,10 +658,10 @@ ls \[ti]/data/mount
 kill %1
 \f[R]
 .fi
-.SS Install from source
+.SS Source installation
 .PP
 Make sure you have git and Go (https://golang.org/) installed.
-Go version 1.16 or newer is required, latest release is recommended.
+Go version 1.17 or newer is required, latest release is recommended.
 You can get it from your package manager, or download it from
 golang.org/dl (https://golang.org/dl/).
 Then you can run the following:
@@ -592,7 +682,7 @@ As an initial check you can now run \f[C]./rclone version\f[R]
 .PP
 Note that on macOS and Windows the
 mount (https://rclone.org/commands/rclone_mount/) command will not be
-available unless you specify additional build tag \f[C]cmount\f[R].
+available unless you specify an additional build tag \f[C]cmount\f[R].
 .IP
 .nf
 \f[C]
@@ -615,8 +705,8 @@ sure you install it in the classic mingw64 subsystem, the ucrt64 version
 is not compatible).
 .PP
 Additionally, on Windows, you must install the third party utility
-WinFsp (http://www.secfs.net/winfsp/), with the \[dq]Developer\[dq]
-feature selected.
+WinFsp (https://winfsp.dev/), with the \[dq]Developer\[dq] feature
+selected.
 If building with cgo, you must also set environment variable CPATH
 pointing to the fuse include directory within the WinFsp installation
 (normally
@@ -635,9 +725,11 @@ go build -trimpath -ldflags -s -tags cmount
 .fi
 .PP
 Instead of executing the \f[C]go build\f[R] command directly, you can
-run it via the Makefile, which also sets version information and copies
-the resulting rclone executable into your GOPATH bin folder
-(\f[C]$(go env GOPATH)/bin\f[R], which corresponds to
+run it via the Makefile.
+It changes the version number suffix from \[dq]-DEV\[dq] to
+\[dq]-beta\[dq] and appends commit details.
+It also copies the resulting rclone executable into your GOPATH bin
+folder (\f[C]$(go env GOPATH)/bin\f[R], which corresponds to
 \f[C]\[ti]/go/bin/rclone\f[R] by default).
 .IP
 .nf
@@ -654,7 +746,15 @@ make GOTAGS=cmount
 \f[R]
 .fi
 .PP
-As an alternative you can download the source, build and install rclone
+There are other make targets that can be used for more advanced builds,
+such as cross-compiling for all supported os/architectures, embedding
+icon and version info resources into windows executable, and packaging
+results into release artifacts.
+See Makefile (https://github.com/rclone/rclone/blob/master/Makefile) and
+cross-compile.go (https://github.com/rclone/rclone/blob/master/bin/cross-compile.go)
+for details.
+.PP
+Another alternative is to download the source, build and install rclone
 in one operation, as a regular Go package.
 The source will be stored it in the Go module cache, and the resulting
 executable will be in your GOPATH bin folder
@@ -678,7 +778,7 @@ and sometimes these don\[aq]t work with the current version):
 go get github.com/rclone/rclone
 \f[R]
 .fi
-.SS Installation with Ansible
+.SS Ansible installation
 .PP
 This can be done with Stefan Weichinger\[aq]s ansible
 role (https://github.com/stefangweichinger/ansible-rclone).
@@ -732,9 +832,9 @@ system\[aq]s scheduler.
 If you need to expose \f[I]service\f[R]-like features, such as remote
 control (https://rclone.org/rc/), GUI (https://rclone.org/gui/),
 serve (https://rclone.org/commands/rclone_serve/) or
-mount (https://rclone.org/commands/rclone_move/), you will often want an
-rclone command always running in the background, and configuring it to
-run in a service infrastructure may be a better option.
+mount (https://rclone.org/commands/rclone_mount/), you will often want
+an rclone command always running in the background, and configuring it
+to run in a service infrastructure may be a better option.
 Below are some alternatives on how to achieve this on different
 operating systems.
 .PP
@@ -770,7 +870,7 @@ c:\[rs]rclone\[rs]rclone.exe sync c:\[rs]files remote:/files --no-console --log-
 .fi
 .SS User account
 .PP
-As mentioned in the mount (https://rclone.org/commands/rclone_move/)
+As mentioned in the mount (https://rclone.org/commands/rclone_mount/)
 documentation, mounted drives created as Administrator are not visible
 to other accounts, not even the account that was elevated as
 Administrator.
@@ -782,8 +882,8 @@ NOTE: Remember that when rclone runs as the \f[C]SYSTEM\f[R] user, the
 user profile that it sees will not be yours.
 This means that if you normally run rclone with configuration file in
 the default location, to be able to use the same configuration when
-running as the system user you must explicitely tell rclone where to
-find it with the
+running as the system user you must explicitly tell rclone where to find
+it with the
 \f[C]--config\f[R] (https://rclone.org/docs/#config-config-file) option,
 or else it will look in the system users profile path
 (\f[C]C:\[rs]Windows\[rs]System32\[rs]config\[rs]systemprofile\f[R]).
@@ -862,7 +962,7 @@ here (https://github.com/rclone/rclone/issues/3340).
 To Windows service running any rclone command, the excellent third-party
 utility NSSM (http://nssm.cc), the \[dq]Non-Sucking Service
 Manager\[dq], can be used.
-It includes some advanced features such as adjusting process periority,
+It includes some advanced features such as adjusting process priority,
 defining process environment variables, redirect to file anything
 written to stdout, and customized response to different exit codes, with
 a GUI to configure everything from (although it can also be used from
@@ -971,8 +1071,6 @@ HiDrive (https://rclone.org/hidrive/)
 .IP \[bu] 2
 HTTP (https://rclone.org/http/)
 .IP \[bu] 2
-Hubic (https://rclone.org/hubic/)
-.IP \[bu] 2
 Internet Archive (https://rclone.org/internetarchive/)
 .IP \[bu] 2
 Jottacloud (https://rclone.org/jottacloud/)
@@ -994,6 +1092,8 @@ Memstore (https://rclone.org/swift/)
 .IP \[bu] 2
 OpenDrive (https://rclone.org/opendrive/)
 .IP \[bu] 2
+Oracle Object Storage (https://rclone.org/oracleobjectstorage/)
+.IP \[bu] 2
 Pcloud (https://rclone.org/pcloud/)
 .IP \[bu] 2
 premiumize.me (https://rclone.org/premiumizeme/)
@@ -1008,6 +1108,8 @@ SFTP (https://rclone.org/sftp/)
 .IP \[bu] 2
 Sia (https://rclone.org/sia/)
 .IP \[bu] 2
+SMB (https://rclone.org/smb/)
+.IP \[bu] 2
 Storj (https://rclone.org/storj/)
 .IP \[bu] 2
 SugarSync (https://rclone.org/sugarsync/)
@@ -1271,6 +1373,11 @@ copy (https://rclone.org/commands/rclone_copy/) command if unsure.
 If dest:path doesn\[aq]t exist, it is created and the source:path
 contents go there.
 .PP
+It is not possible to sync overlapping remotes.
+However, you may exclude the destination from the sync with a filter
+rule or by putting an exclude-if-present file inside the destination
+directory and sync to a destination that is inside the source directory.
+.PP
 \f[B]Note\f[R]: Use the \f[C]-P\f[R]/\f[C]--progress\f[R] flag to view
 real-time transfer statistics
 .PP
@@ -1651,8 +1758,8 @@ Note that \f[C]ls\f[R] and \f[C]lsl\f[R] recurse by default - use
 The other list commands \f[C]lsd\f[R],\f[C]lsf\f[R],\f[C]lsjson\f[R] do
 not recurse by default - use \f[C]-R\f[R] to make them recurse.
 .PP
-Listing a non-existent directory will produce an error except for
-remotes which can\[aq]t have empty directories (e.g.
+Listing a nonexistent directory will produce an error except for remotes
+which can\[aq]t have empty directories (e.g.
 s3, swift, or gcs - the bucket-based remotes).
 .IP
 .nf
@@ -1735,8 +1842,8 @@ Note that \f[C]ls\f[R] and \f[C]lsl\f[R] recurse by default - use
 The other list commands \f[C]lsd\f[R],\f[C]lsf\f[R],\f[C]lsjson\f[R] do
 not recurse by default - use \f[C]-R\f[R] to make them recurse.
 .PP
-Listing a non-existent directory will produce an error except for
-remotes which can\[aq]t have empty directories (e.g.
+Listing a nonexistent directory will produce an error except for remotes
+which can\[aq]t have empty directories (e.g.
 s3, swift, or gcs - the bucket-based remotes).
 .IP
 .nf
@@ -1805,8 +1912,8 @@ Note that \f[C]ls\f[R] and \f[C]lsl\f[R] recurse by default - use
 The other list commands \f[C]lsd\f[R],\f[C]lsf\f[R],\f[C]lsjson\f[R] do
 not recurse by default - use \f[C]-R\f[R] to make them recurse.
 .PP
-Listing a non-existent directory will produce an error except for
-remotes which can\[aq]t have empty directories (e.g.
+Listing a nonexistent directory will produce an error except for remotes
+which can\[aq]t have empty directories (e.g.
 s3, swift, or gcs - the bucket-based remotes).
 .IP
 .nf
@@ -1848,8 +1955,8 @@ Running \f[C]rclone md5sum remote:path\f[R] is equivalent to running
 .PP
 This command can also hash data received on standard input (stdin), by
 not passing a remote:path, or by passing a hyphen as remote:path when
-there is data to read (if not, the hypen will be treated literaly, as a
-relative path).
+there is data to read (if not, the hyphen will be treated literally, as
+a relative path).
 .IP
 .nf
 \f[C]
@@ -1894,8 +2001,8 @@ Running \f[C]rclone sha1sum remote:path\f[R] is equivalent to running
 .PP
 This command can also hash data received on standard input (stdin), by
 not passing a remote:path, or by passing a hyphen as remote:path when
-there is data to read (if not, the hypen will be treated literaly, as a
-relative path).
+there is data to read (if not, the hyphen will be treated literally, as
+a relative path).
 .PP
 This command can also hash data received on STDIN, if not passing a
 remote:path.
@@ -2442,10 +2549,10 @@ rclone (https://rclone.org/commands/rclone/) - Show help for rclone
 commands, flags and backends.
 .SH rclone bisync
 .PP
-Perform bidirectonal synchronization between two paths.
+Perform bidirectional synchronization between two paths.
 .SS Synopsis
 .PP
-Perform bidirectonal synchronization between two paths.
+Perform bidirectional synchronization between two paths.
 .PP
 Bisync (https://rclone.org/bisync/) provides a bidirectional cloud sync
 solution in rclone.
@@ -2695,7 +2802,7 @@ rclone completion bash > /etc/bash_completion.d/rclone
 .IP
 .nf
 \f[C]
-rclone completion bash > /usr/local/etc/bash_completion.d/rclone
+rclone completion bash > $(brew --prefix)/etc/bash_completion.d/rclone
 \f[R]
 .fi
 .PP
@@ -2821,6 +2928,14 @@ echo \[dq]autoload -U compinit; compinit\[dq] >> \[ti]/.zshrc
 \f[R]
 .fi
 .PP
+To load completions in your current shell session:
+.IP
+.nf
+\f[C]
+source <(rclone completion zsh); compdef _rclone rclone
+\f[R]
+.fi
+.PP
 To load completions for every new session, execute once:
 .SS Linux:
 .IP
@@ -2833,7 +2948,7 @@ rclone completion zsh > \[dq]${fpath[1]}/_rclone\[dq]
 .IP
 .nf
 \f[C]
-rclone completion zsh > /usr/local/share/zsh/site-functions/_rclone
+rclone completion zsh > $(brew --prefix)/share/zsh/site-functions/_rclone
 \f[R]
 .fi
 .PP
@@ -2907,8 +3022,8 @@ You can also set obscured passwords using the
 \f[C]rclone config password\f[R] command.
 .PP
 The flag \f[C]--non-interactive\f[R] is for use by applications that
-wish to configure rclone themeselves, rather than using rclone\[aq]s
-text based configuration questions.
+wish to configure rclone themselves, rather than using rclone\[aq]s text
+based configuration questions.
 If this flag is set, and rclone needs to ask the user a question, a JSON
 blob will be returned with the question in it.
 .PP
@@ -3362,8 +3477,8 @@ You can also set obscured passwords using the
 \f[C]rclone config password\f[R] command.
 .PP
 The flag \f[C]--non-interactive\f[R] is for use by applications that
-wish to configure rclone themeselves, rather than using rclone\[aq]s
-text based configuration questions.
+wish to configure rclone themselves, rather than using rclone\[aq]s text
+based configuration questions.
 If this flag is set, and rclone needs to ask the user a question, a JSON
 blob will be returned with the question in it.
 .PP
@@ -4022,8 +4137,8 @@ sha1sum (https://rclone.org/commands/rclone_sha1sum/).
 .PP
 This command can also hash data received on standard input (stdin), by
 not passing a remote:path, or by passing a hyphen as remote:path when
-there is data to read (if not, the hypen will be treated literaly, as a
-relative path).
+there is data to read (if not, the hyphen will be treated literally, as
+a relative path).
 .PP
 Run without a hash to see the list of all supported hashes, e.g.
 .IP
@@ -4320,8 +4435,8 @@ Note that \f[C]ls\f[R] and \f[C]lsl\f[R] recurse by default - use
 The other list commands \f[C]lsd\f[R],\f[C]lsf\f[R],\f[C]lsjson\f[R] do
 not recurse by default - use \f[C]-R\f[R] to make them recurse.
 .PP
-Listing a non-existent directory will produce an error except for
-remotes which can\[aq]t have empty directories (e.g.
+Listing a nonexistent directory will produce an error except for remotes
+which can\[aq]t have empty directories (e.g.
 s3, swift, or gcs - the bucket-based remotes).
 .IP
 .nf
@@ -4412,7 +4527,7 @@ the files will be returned.
 .PP
 If \f[C]--metadata\f[R] is set then an additional Metadata key will be
 returned.
-This will have metdata in rclone standard format as a JSON object.
+This will have metadata in rclone standard format as a JSON object.
 .PP
 if \f[C]--stat\f[R] is set then a single JSON blob will be returned
 about the item pointed to.
@@ -4471,8 +4586,8 @@ Note that \f[C]ls\f[R] and \f[C]lsl\f[R] recurse by default - use
 The other list commands \f[C]lsd\f[R],\f[C]lsf\f[R],\f[C]lsjson\f[R] do
 not recurse by default - use \f[C]-R\f[R] to make them recurse.
 .PP
-Listing a non-existent directory will produce an error except for
-remotes which can\[aq]t have empty directories (e.g.
+Listing a nonexistent directory will produce an error except for remotes
+which can\[aq]t have empty directories (e.g.
 s3, swift, or gcs - the bucket-based remotes).
 .IP
 .nf
@@ -4620,7 +4735,7 @@ experience unexpected program errors, freezes or other issues, consider
 mounting as a network drive instead.
 .PP
 When mounting as a fixed disk drive you can either mount to an unused
-drive letter, or to a path representing a \f[B]non-existent\f[R]
+drive letter, or to a path representing a \f[B]nonexistent\f[R]
 subdirectory of an \f[B]existing\f[R] parent directory or drive.
 Using the special value \f[C]*\f[R] will tell rclone to automatically
 assign the next available drive letter, starting with Z: and moving
@@ -4670,7 +4785,7 @@ shown in Windows Explorer etc, while the complete
 path by \f[C]net use\f[R] etc, just like a normal network drive mapping.
 .PP
 If you specify a full network share UNC path with \f[C]--volname\f[R],
-this will implicitely set the \f[C]--network-mode\f[R] option, so the
+this will implicitly set the \f[C]--network-mode\f[R] option, so the
 following two examples have same result:
 .IP
 .nf
@@ -4686,7 +4801,7 @@ Then rclone will automatically assign a drive letter, same as with
 \f[C]*\f[R] and use that as mountpoint, and instead use the UNC path
 specified as the volume name, as if it were specified with the
 \f[C]--volname\f[R] option.
-This will also implicitely set the \f[C]--network-mode\f[R] option.
+This will also implicitly set the \f[C]--network-mode\f[R] option.
 This means the following two examples have same result:
 .IP
 .nf
@@ -4731,8 +4846,7 @@ notation (https://en.wikipedia.org/wiki/File-system_permissions#Numeric_notation
 The default permissions corresponds to
 \f[C]--file-perms 0666 --dir-perms 0777\f[R], i.e.
 read and write permissions to everyone.
-This means you will not be able to start any programs from the the
-mount.
+This means you will not be able to start any programs from the mount.
 To be able to do that you must add execute permissions, e.g.
 \f[C]--file-perms 0777 --dir-perms 0777\f[R] to add it to everyone.
 If the program needs to write files, chances are you will have to enable
@@ -4818,8 +4932,8 @@ rclone mount without \f[C]--vfs-cache-mode writes\f[R] or
 See the VFS File Caching section for more info.
 .PP
 The bucket-based remotes (e.g.
-Swift, S3, Google Compute Storage, B2, Hubic) do not support the concept
-of empty directories, so empty directories will have a tendency to
+Swift, S3, Google Compute Storage, B2) do not support the concept of
+empty directories, so empty directories will have a tendency to
 disappear once they fall out of the directory cache.
 .PP
 When \f[C]rclone mount\f[R] is invoked on Unix with \f[C]--daemon\f[R]
@@ -5561,7 +5675,7 @@ The supported keys are:
 .fi
 .PP
 Listed files/directories may be prefixed by a one-character flag, some
-of them combined with a description in brackes at end of line.
+of them combined with a description in brackets at end of line.
 These flags have the following meaning:
 .IP
 .nf
@@ -6496,11 +6610,13 @@ rclone serve dlna remote:path [flags]
 .nf
 \f[C]
       --addr string                            The ip:port or :port to bind the DLNA http server to (default \[dq]:7879\[dq])
+      --announce-interval duration             The interval between SSDP announcements (default 12m0s)
       --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
       --dir-perms FileMode                     Directory permissions (default 0777)
       --file-perms FileMode                    File permissions (default 0666)
       --gid uint32                             Override the gid field set by the filesystem (not supported on Windows) (default 1000)
   -h, --help                                   help for dlna
+      --interface stringArray                  The interface to use for SSDP (repeat as necessary)
       --log-trace                              Enable trace logging of SOAP traffic
       --name string                            Name of DLNA server
       --no-checksum                            Don\[aq]t compare checksums on up/download
@@ -7668,6 +7784,10 @@ concatenation of that with the CA certificate.
 \f[C]--key\f[R] should be the PEM encoded private key and
 \f[C]--client-ca\f[R] should be the PEM encoded client certificate
 authority certificate.
+.PP
+--min-tls-version is minimum TLS version that is acceptable.
+Valid values are \[dq]tls1.0\[dq], \[dq]tls1.1\[dq], \[dq]tls1.2\[dq]
+and \[dq]tls1.3\[dq] (default \[dq]tls1.0\[dq]).
 .SS Template
 .PP
 \f[C]--template\f[R] allows a user to specify a custom markup template
@@ -8209,6 +8329,7 @@ rclone serve http remote:path [flags]
       --htpasswd string                        A htpasswd file - if not provided no authentication is done
       --key string                             SSL PEM Private key
       --max-header-bytes int                   Maximum size of request header (default 4096)
+      --min-tls-version string                 Minimum TLS version that is acceptable (default \[dq]tls1.0\[dq])
       --no-checksum                            Don\[aq]t compare checksums on up/download
       --no-modtime                             Don\[aq]t read/write the modification time (can speed things up)
       --no-seek                                Don\[aq]t allow seeking in files
@@ -8510,6 +8631,10 @@ concatenation of that with the CA certificate.
 \f[C]--key\f[R] should be the PEM encoded private key and
 \f[C]--client-ca\f[R] should be the PEM encoded client certificate
 authority certificate.
+.PP
+--min-tls-version is minimum TLS version that is acceptable.
+Valid values are \[dq]tls1.0\[dq], \[dq]tls1.1\[dq], \[dq]tls1.2\[dq]
+and \[dq]tls1.3\[dq] (default \[dq]tls1.0\[dq]).
 .IP
 .nf
 \f[C]
@@ -8530,6 +8655,7 @@ rclone serve restic remote:path [flags]
       --htpasswd string                 htpasswd file - if not provided no authentication is done
       --key string                      SSL PEM Private key
       --max-header-bytes int            Maximum size of request header (default 4096)
+      --min-tls-version string          Minimum TLS version that is acceptable (default \[dq]tls1.0\[dq])
       --pass string                     Password for authentication
       --private-repos                   Users can only access their private repo
       --realm string                    Realm for authentication (default \[dq]rclone\[dq])
@@ -8552,13 +8678,22 @@ remote over a protocol.
 Serve the remote over SFTP.
 .SS Synopsis
 .PP
-Run a SFTP server to serve a remote over SFTP.
+Run an SFTP server to serve a remote over SFTP.
 This can be used with an SFTP client or you can make a remote of type
 sftp to use with it.
 .PP
 You can use the filter flags (e.g.
 \f[C]--include\f[R], \f[C]--exclude\f[R]) to control what is served.
 .PP
+The server will respond to a small number of shell commands, mainly
+md5sum, sha1sum and df, which enable it to provide support for checksums
+and the about feature when accessed from an sftp remote.
+.PP
+Note that this server uses standard 32 KiB packet payload size, which
+means you must not configure the client to expect anything else, e.g.
+with the chunk_size (https://rclone.org/sftp/#sftp-chunk-size) option on
+an sftp remote.
+.PP
 The server will log errors.
 Use \f[C]-v\f[R] to see access logs.
 .PP
@@ -8571,12 +8706,6 @@ location with \f[C]--authorized-keys\f[R] - the default is the same as
 ssh), an \f[C]--auth-proxy\f[R], or set the \f[C]--no-auth\f[R] flag for
 no authentication when logging in.
 .PP
-Note that this also implements a small number of shell commands so that
-it can provide md5sum/sha1sum/df information for the rclone sftp
-backend.
-This means that is can support SHA1SUMs, MD5SUMs and the about command
-when paired with the rclone sftp backend.
-.PP
 If you don\[aq]t supply a host \f[C]--key\f[R] then rclone will generate
 rsa, ecdsa and ed25519 variants, and cache them for later use in
 rclone\[aq]s cache directory (see \f[C]rclone help flags cache-dir\f[R])
@@ -9121,7 +9250,7 @@ rclone serve sftp remote:path [flags]
       --pass string                            Password for authentication
       --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --read-only                              Only allow read-only access
-      --stdio                                  Run an sftp server on run stdin/stdout
+      --stdio                                  Run an sftp server on stdin/stdout
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
       --user string                            User name for authentication
@@ -9334,6 +9463,10 @@ concatenation of that with the CA certificate.
 \f[C]--key\f[R] should be the PEM encoded private key and
 \f[C]--client-ca\f[R] should be the PEM encoded client certificate
 authority certificate.
+.PP
+--min-tls-version is minimum TLS version that is acceptable.
+Valid values are \[dq]tls1.0\[dq], \[dq]tls1.1\[dq], \[dq]tls1.2\[dq]
+and \[dq]tls1.3\[dq] (default \[dq]tls1.0\[dq]).
 .SS VFS - Virtual File System
 .PP
 This command uses the VFS layer.
@@ -9844,6 +9977,7 @@ rclone serve webdav remote:path [flags]
       --htpasswd string                        htpasswd file - if not provided no authentication is done
       --key string                             SSL PEM Private key
       --max-header-bytes int                   Maximum size of request header (default 4096)
+      --min-tls-version string                 Minimum TLS version that is acceptable (default \[dq]tls1.0\[dq])
       --no-checksum                            Don\[aq]t compare checksums on up/download
       --no-modtime                             Don\[aq]t read/write the modification time (can speed things up)
       --no-seek                                Don\[aq]t allow seeking in files
@@ -10804,7 +10938,7 @@ Some backends can also store arbitrary user metadata.
 .PP
 Where possible the key names are standardized, so, for example, it is
 possible to copy object metadata from s3 to azureblob for example and
-metadata will be translated apropriately.
+metadata will be translated appropriately.
 .PP
 Some backends have limits on the size of the metadata and rclone will
 give errors on upload if they are exceeded.
@@ -10946,13 +11080,44 @@ It is also possible to specify \f[C]--boolean=false\f[R] or
 Note that \f[C]--boolean false\f[R] is not valid - this is parsed as
 \f[C]--boolean\f[R] and the \f[C]false\f[R] is parsed as an extra
 command line argument for rclone.
+.SS Time or duration options
+.PP
+TIME or DURATION options can be specified as a duration string or a time
+string.
 .PP
-Options which use TIME use the go time parser.
 A duration string is a possibly signed sequence of decimal numbers, each
 with optional fraction and a unit suffix, such as \[dq]300ms\[dq],
 \[dq]-1.5h\[dq] or \[dq]2h45m\[dq].
-Valid time units are \[dq]ns\[dq], \[dq]us\[dq] (or \[dq]\[mc]s\[dq]),
-\[dq]ms\[dq], \[dq]s\[dq], \[dq]m\[dq], \[dq]h\[dq].
+Default units are seconds or the following abbreviations are valid:
+.IP \[bu] 2
+\f[C]ms\f[R] - Milliseconds
+.IP \[bu] 2
+\f[C]s\f[R] - Seconds
+.IP \[bu] 2
+\f[C]m\f[R] - Minutes
+.IP \[bu] 2
+\f[C]h\f[R] - Hours
+.IP \[bu] 2
+\f[C]d\f[R] - Days
+.IP \[bu] 2
+\f[C]w\f[R] - Weeks
+.IP \[bu] 2
+\f[C]M\f[R] - Months
+.IP \[bu] 2
+\f[C]y\f[R] - Years
+.PP
+These can also be specified as an absolute time in the following
+formats:
+.IP \[bu] 2
+RFC3339 - e.g.
+\f[C]2006-01-02T15:04:05Z\f[R] or \f[C]2006-01-02T15:04:05+07:00\f[R]
+.IP \[bu] 2
+ISO8601 Date and time, local timezone - \f[C]2006-01-02T15:04:05\f[R]
+.IP \[bu] 2
+ISO8601 Date and time, local timezone - \f[C]2006-01-02 15:04:05\f[R]
+.IP \[bu] 2
+ISO8601 Date - \f[C]2006-01-02\f[R] (YYYY-MM-DD)
+.SS Size options
 .PP
 Options which use SIZE use KiB (multiples of 1024 bytes) by default.
 However, a suffix of \f[C]B\f[R] for Byte, \f[C]K\f[R] for KiB,
@@ -10973,7 +11138,8 @@ in DIR, then it will be overwritten.
 .PP
 The remote in use must support server-side move or copy and you must use
 the same remote as the destination of the sync.
-The backup directory must not overlap the destination directory.
+The backup directory must not overlap the destination directory without
+it being excluded by a filter rule.
 .PP
 For example
 .IP
@@ -11018,7 +11184,7 @@ To use a single limit, specify the desired bandwidth in KiB/s, or use a
 suffix B|K|M|G|T|P.
 The default is \f[C]0\f[R] which means to not limit bandwidth.
 .PP
-The upload and download bandwidth can be specified seperately, as
+The upload and download bandwidth can be specified separately, as
 \f[C]--bwlimit UP:DOWN\f[R], so
 .IP
 .nf
@@ -12207,6 +12373,16 @@ This sets the interval between each retry specified by
 .PP
 The default is \f[C]0\f[R].
 Use \f[C]0\f[R] to disable.
+.SS --server-side-across-configs
+.PP
+Allow server-side operations (e.g.
+copy or move) to work across different configurations.
+.PP
+This can be useful if you wish to do a server-side copy or move between
+two remotes which use the same backend but are configured differently.
+.PP
+Note that this isn\[aq]t enabled by default because it isn\[aq]t easy
+for rclone to tell if it will work between any two configurations.
 .SS --size-only
 .PP
 Normally rclone will look at modification time and size of files to see
@@ -12409,13 +12585,22 @@ By default, rclone doesn\[aq]t keep track of renamed files, so if you
 rename a file locally then sync it to a remote, rclone will delete the
 old file on the remote and upload a new copy.
 .PP
-If you use this flag, and the remote supports server-side copy or
-server-side move, and the source and destination have a compatible hash,
-then this will track renames during \f[C]sync\f[R] operations and
-perform renaming server-side.
+An rclone sync with \f[C]--track-renames\f[R] runs like a normal sync,
+but keeps track of objects which exist in the destination but not in the
+source (which would normally be deleted), and which objects exist in the
+source but not the destination (which would normally be transferred).
+These objects are then candidates for renaming.
+.PP
+After the sync, rclone matches up the source only and destination only
+objects using the \f[C]--track-renames-strategy\f[R] specified and
+either renames the destination object or transfers the source and
+deletes the destination object.
+\f[C]--track-renames\f[R] is stateless like all of rclone\[aq]s syncs.
 .PP
-Files will be matched by size and hash - if both match then a rename
-will be considered.
+To use this flag the destination must support server-side copy or
+server-side move, and to use a hash based
+\f[C]--track-renames-strategy\f[R] (the default) the source and the
+destination must have a compatible hash.
 .PP
 If the destination does not support server-side copy or move, rclone
 will fall back to the default behaviour and log an error level message
@@ -12434,7 +12619,8 @@ Note also that \f[C]--track-renames\f[R] is incompatible with
 instead of \f[C]--delete-during\f[R].
 .SS --track-renames-strategy (hash,modtime,leaf,size)
 .PP
-This option changes the matching criteria for \f[C]--track-renames\f[R].
+This option changes the file matching criteria for
+\f[C]--track-renames\f[R].
 .PP
 The matching is controlled by a comma separated selection of these
 tokens:
@@ -12449,16 +12635,14 @@ backends
 .IP \[bu] 2
 \f[C]size\f[R] - the size of the file (this is always enabled)
 .PP
-So using \f[C]--track-renames-strategy modtime,leaf\f[R] would match
-files based on modification time, the leaf of the file name and the size
-only.
+The default option is \f[C]hash\f[R].
+.PP
+Using \f[C]--track-renames-strategy modtime,leaf\f[R] would match files
+based on modification time, the leaf of the file name and the size only.
 .PP
 Using \f[C]--track-renames-strategy modtime\f[R] or \f[C]leaf\f[R] can
 enable \f[C]--track-renames\f[R] support for encrypted destinations.
 .PP
-If nothing is specified, the default option is matching by
-\f[C]hash\f[R]es.
-.PP
 Note that the \f[C]hash\f[R] strategy is not supported with encrypted
 destinations.
 .SS --delete-(before,during,after)
@@ -12499,7 +12683,7 @@ of memory.
 However, some remotes have a way of listing all files beneath a
 directory in one (or a small number) of transactions.
 These tend to be the bucket-based remotes (e.g.
-S3, B2, GCS, Swift, Hubic).
+S3, B2, GCS, Swift).
 .PP
 If you use the \f[C]--fast-list\f[R] flag then rclone will use this
 method for listing directories.
@@ -12572,9 +12756,9 @@ In all other cases the file will not be updated.
 Consider using the \f[C]--modify-window\f[R] flag to compensate for time
 skews between the source and the backend, for backends that do not
 support mod times, and instead use uploaded times.
-However, if the backend does not support checksums, note that
-sync\[aq]ing or copying within the time skew window may still result in
-additional transfers for safety.
+However, if the backend does not support checksums, note that syncing or
+copying within the time skew window may still result in additional
+transfers for safety.
 .SS --use-mmap
 .PP
 If this flag is set then rclone will use anonymous memory allocated by
@@ -13593,7 +13777,7 @@ T}
 T{
 T}@T{
 T}@T{
-\f[C]/dir/file.gif\f[R]
+\f[C]/dir/file.png\f[R]
 T}@T{
 \f[C]/dir/file.gif\f[R]
 T}
@@ -14343,6 +14527,9 @@ Default units are \f[C]KiB\f[R] but abbreviations \f[C]K\f[R],
 E.g.
 \f[C]rclone ls remote: --min-size 50k\f[R] lists files on
 \f[C]remote:\f[R] of 50 KiB size or larger.
+.PP
+See the size option docs (https://rclone.org/docs/#size-option) for more
+info.
 .SS \f[C]--max-size\f[R] - Don\[aq]t transfer any file larger than this
 .PP
 Controls the maximum size file within the scope of an rclone command.
@@ -14352,44 +14539,21 @@ Default units are \f[C]KiB\f[R] but abbreviations \f[C]K\f[R],
 E.g.
 \f[C]rclone ls remote: --max-size 1G\f[R] lists files on
 \f[C]remote:\f[R] of 1 GiB size or smaller.
+.PP
+See the size option docs (https://rclone.org/docs/#size-option) for more
+info.
 .SS \f[C]--max-age\f[R] - Don\[aq]t transfer any file older than this
 .PP
 Controls the maximum age of files within the scope of an rclone command.
-Default units are seconds or the following abbreviations are valid:
-.IP \[bu] 2
-\f[C]ms\f[R] - Milliseconds
-.IP \[bu] 2
-\f[C]s\f[R] - Seconds
-.IP \[bu] 2
-\f[C]m\f[R] - Minutes
-.IP \[bu] 2
-\f[C]h\f[R] - Hours
-.IP \[bu] 2
-\f[C]d\f[R] - Days
-.IP \[bu] 2
-\f[C]w\f[R] - Weeks
-.IP \[bu] 2
-\f[C]M\f[R] - Months
-.IP \[bu] 2
-\f[C]y\f[R] - Years
-.PP
-\f[C]--max-age\f[R] can also be specified as an absolute time in the
-following formats:
-.IP \[bu] 2
-RFC3339 - e.g.
-\f[C]2006-01-02T15:04:05Z\f[R] or \f[C]2006-01-02T15:04:05+07:00\f[R]
-.IP \[bu] 2
-ISO8601 Date and time, local timezone - \f[C]2006-01-02T15:04:05\f[R]
-.IP \[bu] 2
-ISO8601 Date and time, local timezone - \f[C]2006-01-02 15:04:05\f[R]
-.IP \[bu] 2
-ISO8601 Date - \f[C]2006-01-02\f[R] (YYYY-MM-DD)
 .PP
 \f[C]--max-age\f[R] applies only to files and not to directories.
 .PP
 E.g.
 \f[C]rclone ls remote: --max-age 2d\f[R] lists files on
 \f[C]remote:\f[R] of 2 days old or less.
+.PP
+See the time option docs (https://rclone.org/docs/#time-option) for
+valid formats.
 .SS \f[C]--min-age\f[R] - Don\[aq]t transfer any file younger than this
 .PP
 Controls the minimum age of files within the scope of an rclone command.
@@ -14400,6 +14564,9 @@ Controls the minimum age of files within the scope of an rclone command.
 E.g.
 \f[C]rclone ls remote: --min-age 2d\f[R] lists files on
 \f[C]remote:\f[R] of 2 days old or more.
+.PP
+See the time option docs (https://rclone.org/docs/#time-option) for
+valid formats.
 .SS Other flags
 .SS \f[C]--delete-excluded\f[R] - Delete files on dest excluded from sync
 .PP
@@ -14628,6 +14795,11 @@ SSL PEM Private key
 .SS --rc-max-header-bytes=VALUE
 .PP
 Maximum size of request header (default 4096)
+.SS --rc-min-tls-version=VALUE
+.PP
+The minimum TLS version that is acceptable.
+Valid values are \[dq]tls1.0\[dq], \[dq]tls1.1\[dq], \[dq]tls1.2\[dq]
+and \[dq]tls1.3\[dq] (default \[dq]tls1.0\[dq]).
 .SS --rc-user=VALUE
 .PP
 User name for authentication.
@@ -15030,7 +15202,7 @@ The parameters can be a string as per the rest of rclone, eg
 \f[C]s3:bucket/path\f[R] or \f[C]:sftp:/my/dir\f[R].
 They can also be specified as JSON blobs.
 .PP
-If specifyng a JSON blob it should be a object mapping strings to
+If specifying a JSON blob it should be a object mapping strings to
 strings.
 These values will be used to configure the remote.
 There are 3 special values which may be set:
@@ -15731,6 +15903,11 @@ progress - output of the progress related to the underlying job
 Parameters:
 .IP \[bu] 2
 jobid - id of the job (integer).
+.SS job/stopgroup: Stop all running jobs in a group
+.PP
+Parameters:
+.IP \[bu] 2
+group - name of the group (string).
 .SS mount/listmounts: Show current mount points
 .PP
 This shows currently mounted points, which can be used for performing an
@@ -15831,10 +16008,10 @@ rclone rc mount/unmount mountPoint=/home//mountPoint
 .fi
 .PP
 \f[B]Authentication is required for this call.\f[R]
-.SS mount/unmountall: Show current mount points
+.SS mount/unmountall: Unmount all active mounts
 .PP
-This shows currently mounted points, which can be used for performing an
-unmount.
+rclone allows Linux, FreeBSD, macOS and Windows to mount any of
+Rclone\[aq]s cloud storage systems as a file system with FUSE.
 .PP
 This takes no parameters and returns error if unmount does not succeed.
 .PP
@@ -16455,7 +16632,7 @@ It can be used to check that rclone is still alive and to check that
 parameter passing is working properly.
 .PP
 \f[B]Authentication is required for this call.\f[R]
-.SS sync/bisync: Perform bidirectonal synchronization between two paths.
+.SS sync/bisync: Perform bidirectional synchronization between two paths.
 .PP
 This takes the following parameters
 .IP \[bu] 2
@@ -17223,21 +17400,6 @@ T}@T{
 -
 T}
 T{
-Hubic
-T}@T{
-MD5
-T}@T{
-R/W
-T}@T{
-No
-T}@T{
-No
-T}@T{
-R/W
-T}@T{
--
-T}
-T{
 Internet Archive
 T}@T{
 MD5, SHA1, CRC32
@@ -17388,6 +17550,21 @@ T}@T{
 -
 T}
 T{
+Oracle Object Storage
+T}@T{
+MD5
+T}@T{
+R/W
+T}@T{
+No
+T}@T{
+No
+T}@T{
+R/W
+T}@T{
+-
+T}
+T{
 pCloud
 T}@T{
 MD5, SHA1 \[u2077]
@@ -17493,6 +17670,21 @@ T}@T{
 -
 T}
 T{
+SMB
+T}@T{
+-
+T}@T{
+-
+T}@T{
+Yes
+T}@T{
+No
+T}@T{
+-
+T}@T{
+-
+T}
+T{
 SugarSync
 T}@T{
 -
@@ -17652,7 +17844,7 @@ To use the verify checksums when transferring between cloud storage
 systems they must support a common hash type.
 .SS ModTime
 .PP
-Allmost all cloud storage systems store some sort of timestamp on
+Almost all cloud storage systems store some sort of timestamp on
 objects, but several of them not something that is appropriate to use
 for syncing.
 E.g.
@@ -18889,29 +19081,6 @@ T}@T{
 Yes
 T}
 T{
-Hubic
-T}@T{
-Yes \[dg]
-T}@T{
-Yes
-T}@T{
-No
-T}@T{
-No
-T}@T{
-No
-T}@T{
-Yes
-T}@T{
-Yes
-T}@T{
-No
-T}@T{
-Yes
-T}@T{
-No
-T}
-T{
 Internet Archive
 T}@T{
 No
@@ -19142,6 +19311,29 @@ T}@T{
 No
 T}
 T{
+Oracle Object Storage
+T}@T{
+Yes
+T}@T{
+Yes
+T}@T{
+No
+T}@T{
+No
+T}@T{
+Yes
+T}@T{
+Yes
+T}@T{
+No
+T}@T{
+No
+T}@T{
+No
+T}@T{
+No
+T}
+T{
 pCloud
 T}@T{
 Yes
@@ -19303,6 +19495,29 @@ T}@T{
 Yes
 T}
 T{
+SMB
+T}@T{
+No
+T}@T{
+No
+T}@T{
+Yes
+T}@T{
+Yes
+T}@T{
+No
+T}@T{
+No
+T}@T{
+Yes
+T}@T{
+No
+T}@T{
+No
+T}@T{
+Yes
+T}
+T{
 SugarSync
 T}@T{
 Yes
@@ -19469,9 +19684,9 @@ T}
 This deletes a directory quicker than just deleting all the files in the
 directory.
 .PP
-\[dg] Note Swift, Hubic, and Storj implement this in order to delete
-directory markers but they don\[aq]t actually have a quicker way of
-deleting files other than deleting them individually.
+\[dg] Note Swift and Storj implement this in order to delete directory
+markers but they don\[aq]t actually have a quicker way of deleting files
+other than deleting them individually.
 .PP
 \[dd] StreamUpload is not supported with Nextcloud
 .SS Copy
@@ -19666,6 +19881,7 @@ These flags are available for every command.
       --rc-job-expire-interval duration      Interval to check for expired async jobs (default 10s)
       --rc-key string                        SSL PEM Private key
       --rc-max-header-bytes int              Maximum size of request header (default 4096)
+      --rc-min-tls-version string            Minimum TLS version that is acceptable (default \[dq]tls1.0\[dq])
       --rc-no-auth                           Don\[aq]t require auth for certain methods
       --rc-pass string                       Password for authentication
       --rc-realm string                      Realm for authentication (default \[dq]rclone\[dq])
@@ -19682,6 +19898,7 @@ These flags are available for every command.
       --refresh-times                        Refresh the modtime of remote files
       --retries int                          Retry operations this many times if they fail (default 3)
       --retries-sleep duration               Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)
+      --server-side-across-configs           Allow server-side operations (e.g. copy) to work across different configs
       --size-only                            Skip based on size only, not mod-time or checksum
       --stats duration                       Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s)
       --stats-file-name-length int           Max file name length in stats (0 for no limit) (default 45)
@@ -19707,7 +19924,7 @@ These flags are available for every command.
       --use-json-log                         Use json log format
       --use-mmap                             Use mmap allocator (see docs)
       --use-server-modtime                   Use server modified time instead of object metadata
-      --user-agent string                    Set the user-agent to a specified string (default \[dq]rclone/v1.59.0\[dq])
+      --user-agent string                    Set the user-agent to a specified string (default \[dq]rclone/v1.60.0\[dq])
   -v, --verbose count                        Print lots more stuff (repeat for more)
 \f[R]
 .fi
@@ -19861,7 +20078,7 @@ They control the backends and may be set in the config file.
       --drive-use-trash                              Send files to the trash instead of deleting permanently (default true)
       --drive-v2-download-min-size SizeSuffix        If Object\[aq]s are greater, use drive v2 API to download (default off)
       --dropbox-auth-url string                      Auth server URL
-      --dropbox-batch-commit-timeout Duration        Max time to wait for a batch to finish comitting (default 10m0s)
+      --dropbox-batch-commit-timeout Duration        Max time to wait for a batch to finish committing (default 10m0s)
       --dropbox-batch-mode string                    Upload file batching sync|async|off (default \[dq]sync\[dq])
       --dropbox-batch-size int                       Max number of files in upload batch
       --dropbox-batch-timeout Duration               Max time to allow an idle upload batch before uploading (default 0s)
@@ -19895,6 +20112,7 @@ They control the backends and may be set in the config file.
       --ftp-disable-utf8                             Disable using UTF-8 even if server advertises support
       --ftp-encoding MultiEncoder                    The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
       --ftp-explicit-tls                             Use Explicit FTPS (FTP over TLS)
+      --ftp-force-list-hidden                        Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
       --ftp-host string                              FTP host to connect to
       --ftp-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
       --ftp-no-check-certificate                     Do not verify the TLS certificate of the server
@@ -19913,6 +20131,7 @@ They control the backends and may be set in the config file.
       --gcs-client-secret string                     OAuth Client Secret
       --gcs-decompress                               If set this will decompress gzip encoded objects
       --gcs-encoding MultiEncoder                    The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot)
+      --gcs-endpoint string                          Endpoint for the service
       --gcs-location string                          Location for the newly created buckets
       --gcs-no-check-bucket                          If set, don\[aq]t attempt to check the bucket exists or create it
       --gcs-object-acl string                        Access Control List for new objects
@@ -19958,14 +20177,6 @@ They control the backends and may be set in the config file.
       --http-no-head                                 Don\[aq]t use HEAD requests
       --http-no-slash                                Set this if the site doesn\[aq]t end directories with /
       --http-url string                              URL of HTTP host to connect to
-      --hubic-auth-url string                        Auth server URL
-      --hubic-chunk-size SizeSuffix                  Above this size files will be chunked into a _segments container (default 5Gi)
-      --hubic-client-id string                       OAuth Client Id
-      --hubic-client-secret string                   OAuth Client Secret
-      --hubic-encoding MultiEncoder                  The encoding for the backend (default Slash,InvalidUtf8)
-      --hubic-no-chunk                               Don\[aq]t chunk files during streaming upload
-      --hubic-token string                           OAuth Access Token as a JSON blob
-      --hubic-token-url string                       Token server url
       --internetarchive-access-key-id string         IAS3 Access Key
       --internetarchive-disable-checksum             Don\[aq]t ask the server to test against MD5 checksum calculated by rclone (default true)
       --internetarchive-encoding MultiEncoder        The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot)
@@ -20034,6 +20245,22 @@ They control the backends and may be set in the config file.
       --onedrive-server-side-across-configs          Allow server-side operations (e.g. copy) to work across different onedrive configs
       --onedrive-token string                        OAuth Access Token as a JSON blob
       --onedrive-token-url string                    Token server url
+      --oos-chunk-size SizeSuffix                    Chunk size to use for uploading (default 5Mi)
+      --oos-compartment string                       Object storage compartment OCID
+      --oos-config-file string                       Path to OCI config file (default \[dq]\[ti]/.oci/config\[dq])
+      --oos-config-profile string                    Profile name inside the oci config file (default \[dq]Default\[dq])
+      --oos-copy-cutoff SizeSuffix                   Cutoff for switching to multipart copy (default 4.656Gi)
+      --oos-copy-timeout Duration                    Timeout for copy (default 1m0s)
+      --oos-disable-checksum                         Don\[aq]t store MD5 checksum with object metadata
+      --oos-encoding MultiEncoder                    The encoding for the backend (default Slash,InvalidUtf8,Dot)
+      --oos-endpoint string                          Endpoint for Object storage API
+      --oos-leave-parts-on-error                     If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery
+      --oos-namespace string                         Object storage namespace
+      --oos-no-check-bucket                          If set, don\[aq]t attempt to check the bucket exists or create it
+      --oos-provider string                          Choose your Auth Provider (default \[dq]env_auth\[dq])
+      --oos-region string                            Object storage Region
+      --oos-upload-concurrency int                   Concurrency for multipart uploads (default 10)
+      --oos-upload-cutoff SizeSuffix                 Cutoff for switching to chunked upload (default 200Mi)
       --opendrive-chunk-size SizeSuffix              Files will be uploaded in chunks this size (default 10Mi)
       --opendrive-encoding MultiEncoder              The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot)
       --opendrive-password string                    Password (obscured)
@@ -20065,6 +20292,7 @@ They control the backends and may be set in the config file.
       --s3-bucket-acl string                         Canned ACL used when creating buckets
       --s3-chunk-size SizeSuffix                     Chunk size to use for uploading (default 5Mi)
       --s3-copy-cutoff SizeSuffix                    Cutoff for switching to multipart copy (default 4.656Gi)
+      --s3-decompress                                If set this will decompress gzip encoded objects
       --s3-disable-checksum                          Don\[aq]t store MD5 checksum with object metadata
       --s3-disable-http2                             Disable usage of http2 for S3 backends
       --s3-download-url string                       Custom endpoint for downloads
@@ -20083,6 +20311,7 @@ They control the backends and may be set in the config file.
       --s3-no-check-bucket                           If set, don\[aq]t attempt to check the bucket exists or create it
       --s3-no-head                                   If set, don\[aq]t HEAD uploaded objects to check integrity
       --s3-no-head-object                            If set, do not do HEAD before GET when getting objects
+      --s3-no-system-metadata                        Suppress setting and reading of system metadata
       --s3-profile string                            Profile to use in the shared credentials file
       --s3-provider string                           Choose your S3 provider
       --s3-region string                             Region to connect to
@@ -20092,7 +20321,8 @@ They control the backends and may be set in the config file.
       --s3-session-token string                      An AWS session token
       --s3-shared-credentials-file string            Path to the shared credentials file
       --s3-sse-customer-algorithm string             If using SSE-C, the server-side encryption algorithm used when storing this object in S3
-      --s3-sse-customer-key string                   If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data
+      --s3-sse-customer-key string                   To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data
+      --s3-sse-customer-key-base64 string            If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data
       --s3-sse-customer-key-md5 string               If using SSE-C you may provide the secret encryption key MD5 checksum (optional)
       --s3-sse-kms-key-id string                     If using KMS ID you must provide the ARN of Key
       --s3-storage-class string                      The storage class to use when storing new objects in S3
@@ -20102,6 +20332,8 @@ They control the backends and may be set in the config file.
       --s3-use-multipart-etag Tristate               Whether to use ETag in multipart uploads for verification (default unset)
       --s3-use-presigned-request                     Whether to use a presigned request or PutObject for single part uploads
       --s3-v2-auth                                   If true use v2 authentication
+      --s3-version-at Time                           Show file versions as they were at the specified time (default off)
+      --s3-versions                                  Include old versions in directory listings
       --seafile-2fa                                  Two-factor authentication (\[aq]true\[aq] if the account has 2FA enabled)
       --seafile-create-library                       Should rclone create a library if it doesn\[aq]t exist
       --seafile-encoding MultiEncoder                The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
@@ -20148,6 +20380,15 @@ They control the backends and may be set in the config file.
       --sia-encoding MultiEncoder                    The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot)
       --sia-user-agent string                        Siad User Agent (default \[dq]Sia-Agent\[dq])
       --skip-links                                   Don\[aq]t warn about skipped symlinks
+      --smb-case-insensitive                         Whether the server is configured to be case-insensitive (default true)
+      --smb-domain string                            Domain name for NTLM authentication (default \[dq]WORKGROUP\[dq])
+      --smb-encoding MultiEncoder                    The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot)
+      --smb-hide-special-share                       Hide special shares (e.g. print$) which users aren\[aq]t supposed to access (default true)
+      --smb-host string                              SMB server hostname to connect to
+      --smb-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
+      --smb-pass string                              SMB password (obscured)
+      --smb-port int                                 SMB port number (default 445)
+      --smb-user string                              SMB username (default \[dq]$USER\[dq])
       --storj-access-grant string                    Access grant
       --storj-api-key string                         API key
       --storj-passphrase string                      Encryption passphrase
@@ -20178,6 +20419,7 @@ They control the backends and may be set in the config file.
       --swift-key string                             API key or password (OS_PASSWORD)
       --swift-leave-parts-on-error                   If true avoid calling abort upload on a failure
       --swift-no-chunk                               Don\[aq]t chunk files during streaming upload
+      --swift-no-large-objects                       Disable support for static and dynamic large objects
       --swift-region string                          Region name - optional (OS_REGION_NAME)
       --swift-storage-policy string                  The storage policy to use when creating a new container
       --swift-storage-url string                     Storage URL - optional (OS_STORAGE_URL)
@@ -21422,7 +21664,7 @@ On such a critical error the \f[C]{...}.path1.lst\f[R] and
 \f[C].lst-err\f[R], which blocks any future bisync runs (since the
 normal \f[C].lst\f[R] files are not found).
 Bisync keeps them under \f[C]bisync\f[R] subdirectory of the rclone
-cache direcory, typically at \f[C]${HOME}/.cache/rclone/bisync/\f[R] on
+cache directory, typically at \f[C]${HOME}/.cache/rclone/bisync/\f[R] on
 Linux.
 .PP
 Some errors are considered temporary and re-running the bisync is not
@@ -21521,7 +21763,7 @@ don\[aq]t have spelling case differences (\f[C]Smile.jpg\f[R] vs.
 .SS Windows support
 .PP
 Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on
-Windows Github runners.
+Windows GitHub runners.
 .PP
 Drive letters are allowed, including drive letters mapped to network
 drives (\f[C]rclone bisync J:\[rs]localsync GDrive:\f[R]).
@@ -22144,7 +22386,7 @@ file mismatches in the test tree.
 .IP \[bu] 2
 Some Dropbox tests can fail, notably printing the following message:
 \f[C]src and dst identical but can\[aq]t set mod time without deleting and re-uploading\f[R]
-This is expected and happens due a way Dropbox handles modificaion
+This is expected and happens due a way Dropbox handles modification
 times.
 You should use the \f[C]-refresh-times\f[R] test flag to make up for
 this.
@@ -22157,7 +22399,7 @@ instructions (https://rclone.org/dropbox/#get-your-own-dropbox-app-id).
 .PP
 Sometimes even a slight change in the bisync source can cause little
 changes spread around many log files.
-Updating them manually would be a nighmare.
+Updating them manually would be a nightmare.
 .PP
 The \f[C]-golden\f[R] flag will store the \f[C]test.log\f[R] and
 \f[C]*.lst\f[R] listings from each test case into respective golden
@@ -22701,6 +22943,14 @@ Invoking \f[C]rclone mkdir backup:../desktop\f[R] is exactly the same as
 invoking \f[C]rclone mkdir mydrive:private/backup/../desktop\f[R].
 The empty path is not allowed as a remote.
 To alias the current directory use \f[C].\f[R] instead.
+.PP
+The target remote can also be a connection
+string (https://rclone.org/docs/#connection-strings).
+This can be used to modify the config of a remote for different uses,
+e.g.
+the alias \f[C]myDriveTrash\f[R] with the target remote
+\f[C]myDrive,trashed_only:\f[R] can be used to only show the trashed
+files in \f[C]myDrive\f[R].
 .SS Configuration
 .PP
 Here is an example of how to make an alias called \f[C]remote\f[R] for
@@ -23227,8 +23477,12 @@ IBM COS S3
 .IP \[bu] 2
 IDrive e2
 .IP \[bu] 2
+IONOS Cloud
+.IP \[bu] 2
 Minio
 .IP \[bu] 2
+Qiniu Cloud Object Storage (Kodo)
+.IP \[bu] 2
 RackCorp Object Storage
 .IP \[bu] 2
 Scaleway
@@ -23596,7 +23850,7 @@ the modification times of the objects being the time of upload.
 Rclone\[aq]s default directory traversal is to process each directory
 individually.
 This takes one API call per directory.
-Using the \f[C]--fast-list\f[R] flag will read all info about the the
+Using the \f[C]--fast-list\f[R] flag will read all info about the
 objects into memory first using a smaller number of API calls (one per
 1000 objects).
 See the rclone docs (https://rclone.org/docs/#fast-list) for more
@@ -23658,6 +23912,81 @@ This will mean that these objects do not have an MD5 checksum.
 Note that reading this from the object takes an additional
 \f[C]HEAD\f[R] request as the metadata isn\[aq]t returned in object
 listings.
+.SS Versions
+.PP
+When bucket versioning is enabled (this can be done with rclone with the
+\f[C]rclone backend versioning\f[R] command) when rclone uploads a new
+version of a file it creates a new version of
+it (https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html)
+Likewise when you delete a file, the old version will be marked hidden
+and still be available.
+.PP
+Old versions of files, where available, are visible using the
+\f[C]--s3-versions\f[R] flag.
+.PP
+It is also possible to view a bucket as it was at a certain point in
+time, using the \f[C]--s3-version-at\f[R] flag.
+This will show the file versions as they were at that time, showing
+files that have been deleted afterwards, and hiding files that were
+created since.
+.PP
+If you wish to remove all the old versions then you can use the
+\f[C]rclone backend cleanup-hidden remote:bucket\f[R] command which will
+delete all the old hidden versions of files, leaving the current ones
+intact.
+You can also supply a path and only old versions under that path will be
+deleted, e.g.
+\f[C]rclone backend cleanup-hidden remote:bucket/path/to/stuff\f[R].
+.PP
+When you \f[C]purge\f[R] a bucket, the current and the old versions will
+be deleted then the bucket will be deleted.
+.PP
+However \f[C]delete\f[R] will cause the current versions of the files to
+become hidden old versions.
+.PP
+Here is a session showing the listing and retrieval of an old version
+followed by a \f[C]cleanup\f[R] of the old versions.
+.PP
+Show current version and all the versions with \f[C]--s3-versions\f[R]
+flag.
+.IP
+.nf
+\f[C]
+$ rclone -q ls s3:cleanup-test
+        9 one.txt
+
+$ rclone -q --s3-versions ls s3:cleanup-test
+        9 one.txt
+        8 one-v2016-07-04-141032-000.txt
+       16 one-v2016-07-04-141003-000.txt
+       15 one-v2016-07-02-155621-000.txt
+\f[R]
+.fi
+.PP
+Retrieve an old version
+.IP
+.nf
+\f[C]
+$ rclone -q --s3-versions copy s3:cleanup-test/one-v2016-07-04-141003-000.txt /tmp
+
+$ ls -l /tmp/one-v2016-07-04-141003-000.txt
+-rw-rw-r-- 1 ncw ncw 16 Jul  2 17:46 /tmp/one-v2016-07-04-141003-000.txt
+\f[R]
+.fi
+.PP
+Clean up all the old versions and show that they\[aq]ve gone.
+.IP
+.nf
+\f[C]
+$ rclone -q backend cleanup-hidden s3:cleanup-test
+
+$ rclone -q ls s3:cleanup-test
+        9 one.txt
+
+$ rclone -q --s3-versions ls s3:cleanup-test
+        9 one.txt
+\f[R]
+.fi
 .SS Cleanup
 .PP
 If you run \f[C]rclone cleanup s3:bucket\f[R] then it will remove all
@@ -23939,8 +24268,8 @@ all the files to be uploaded as multipart.
 Here are the Standard options specific to s3 (Amazon S3 Compliant
 Storage Providers including AWS, Alibaba, Ceph, China Mobile,
 Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS,
-IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS,
-StackPath, Storj, Tencent COS and Wasabi).
+IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway,
+SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
 .SS --s3-provider
 .PP
 Choose your S3 provider.
@@ -24024,6 +24353,12 @@ IBM COS S3
 IDrive e2
 .RE
 .IP \[bu] 2
+\[dq]IONOS\[dq]
+.RS 2
+.IP \[bu] 2
+IONOS Cloud
+.RE
+.IP \[bu] 2
 \[dq]LyveCloud\[dq]
 .RS 2
 .IP \[bu] 2
@@ -24084,6 +24419,12 @@ Tencent Cloud Object Storage (COS)
 Wasabi Object Storage
 .RE
 .IP \[bu] 2
+\[dq]Qiniu\[dq]
+.RS 2
+.IP \[bu] 2
+Qiniu Object Storage (Kodo)
+.RE
+.IP \[bu] 2
 \[dq]Other\[dq]
 .RS 2
 .IP \[bu] 2
@@ -24685,6 +25026,120 @@ centers for low latency.
 .PP
 Region to connect to.
 .PP
+Properties:
+.IP \[bu] 2
+Config: region
+.IP \[bu] 2
+Env Var: RCLONE_S3_REGION
+.IP \[bu] 2
+Provider: Qiniu
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: false
+.IP \[bu] 2
+Examples:
+.RS 2
+.IP \[bu] 2
+\[dq]cn-east-1\[dq]
+.RS 2
+.IP \[bu] 2
+The default endpoint - a good choice if you are unsure.
+.IP \[bu] 2
+East China Region 1.
+.IP \[bu] 2
+Needs location constraint cn-east-1.
+.RE
+.IP \[bu] 2
+\[dq]cn-east-2\[dq]
+.RS 2
+.IP \[bu] 2
+East China Region 2.
+.IP \[bu] 2
+Needs location constraint cn-east-2.
+.RE
+.IP \[bu] 2
+\[dq]cn-north-1\[dq]
+.RS 2
+.IP \[bu] 2
+North China Region 1.
+.IP \[bu] 2
+Needs location constraint cn-north-1.
+.RE
+.IP \[bu] 2
+\[dq]cn-south-1\[dq]
+.RS 2
+.IP \[bu] 2
+South China Region 1.
+.IP \[bu] 2
+Needs location constraint cn-south-1.
+.RE
+.IP \[bu] 2
+\[dq]us-north-1\[dq]
+.RS 2
+.IP \[bu] 2
+North America Region.
+.IP \[bu] 2
+Needs location constraint us-north-1.
+.RE
+.IP \[bu] 2
+\[dq]ap-southeast-1\[dq]
+.RS 2
+.IP \[bu] 2
+Southeast Asia Region 1.
+.IP \[bu] 2
+Needs location constraint ap-southeast-1.
+.RE
+.IP \[bu] 2
+\[dq]ap-northeast-1\[dq]
+.RS 2
+.IP \[bu] 2
+Northeast Asia Region 1.
+.IP \[bu] 2
+Needs location constraint ap-northeast-1.
+.RE
+.RE
+.SS --s3-region
+.PP
+Region where your bucket will be created and your data stored.
+.PP
+Properties:
+.IP \[bu] 2
+Config: region
+.IP \[bu] 2
+Env Var: RCLONE_S3_REGION
+.IP \[bu] 2
+Provider: IONOS
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: false
+.IP \[bu] 2
+Examples:
+.RS 2
+.IP \[bu] 2
+\[dq]de\[dq]
+.RS 2
+.IP \[bu] 2
+Frankfurt, Germany
+.RE
+.IP \[bu] 2
+\[dq]eu-central-2\[dq]
+.RS 2
+.IP \[bu] 2
+Berlin, Germany
+.RE
+.IP \[bu] 2
+\[dq]eu-south-2\[dq]
+.RS 2
+.IP \[bu] 2
+Logrono, Spain
+.RE
+.RE
+.SS --s3-region
+.PP
+Region to connect to.
+.PP
 Leave blank if you are using an S3 clone and you don\[aq]t have a
 region.
 .PP
@@ -24695,7 +25150,7 @@ Config: region
 Env Var: RCLONE_S3_REGION
 .IP \[bu] 2
 Provider:
-!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
+!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
 .IP \[bu] 2
 Type: string
 .IP \[bu] 2
@@ -25367,6 +25822,45 @@ Singapore Single Site Private Endpoint
 .RE
 .SS --s3-endpoint
 .PP
+Endpoint for IONOS S3 Object Storage.
+.PP
+Specify the endpoint from the same region.
+.PP
+Properties:
+.IP \[bu] 2
+Config: endpoint
+.IP \[bu] 2
+Env Var: RCLONE_S3_ENDPOINT
+.IP \[bu] 2
+Provider: IONOS
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: false
+.IP \[bu] 2
+Examples:
+.RS 2
+.IP \[bu] 2
+\[dq]s3-eu-central-1.ionoscloud.com\[dq]
+.RS 2
+.IP \[bu] 2
+Frankfurt, Germany
+.RE
+.IP \[bu] 2
+\[dq]s3-eu-central-2.ionoscloud.com\[dq]
+.RS 2
+.IP \[bu] 2
+Berlin, Germany
+.RE
+.IP \[bu] 2
+\[dq]s3-eu-south-2.ionoscloud.com\[dq]
+.RS 2
+.IP \[bu] 2
+Logrono, Spain
+.RE
+.RE
+.SS --s3-endpoint
+.PP
 Endpoint for OSS API.
 .PP
 Properties:
@@ -26022,6 +26516,67 @@ Auckland (New Zealand) Endpoint
 .RE
 .SS --s3-endpoint
 .PP
+Endpoint for Qiniu Object Storage.
+.PP
+Properties:
+.IP \[bu] 2
+Config: endpoint
+.IP \[bu] 2
+Env Var: RCLONE_S3_ENDPOINT
+.IP \[bu] 2
+Provider: Qiniu
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: false
+.IP \[bu] 2
+Examples:
+.RS 2
+.IP \[bu] 2
+\[dq]s3-cn-east-1.qiniucs.com\[dq]
+.RS 2
+.IP \[bu] 2
+East China Endpoint 1
+.RE
+.IP \[bu] 2
+\[dq]s3-cn-east-2.qiniucs.com\[dq]
+.RS 2
+.IP \[bu] 2
+East China Endpoint 2
+.RE
+.IP \[bu] 2
+\[dq]s3-cn-north-1.qiniucs.com\[dq]
+.RS 2
+.IP \[bu] 2
+North China Endpoint 1
+.RE
+.IP \[bu] 2
+\[dq]s3-cn-south-1.qiniucs.com\[dq]
+.RS 2
+.IP \[bu] 2
+South China Endpoint 1
+.RE
+.IP \[bu] 2
+\[dq]s3-us-north-1.qiniucs.com\[dq]
+.RS 2
+.IP \[bu] 2
+North America Endpoint 1
+.RE
+.IP \[bu] 2
+\[dq]s3-ap-southeast-1.qiniucs.com\[dq]
+.RS 2
+.IP \[bu] 2
+Southeast Asia Endpoint 1
+.RE
+.IP \[bu] 2
+\[dq]s3-ap-northeast-1.qiniucs.com\[dq]
+.RS 2
+.IP \[bu] 2
+Northeast Asia Endpoint 1
+.RE
+.RE
+.SS --s3-endpoint
+.PP
 Endpoint for S3 API.
 .PP
 Required when using an S3 clone.
@@ -26033,7 +26588,7 @@ Config: endpoint
 Env Var: RCLONE_S3_ENDPOINT
 .IP \[bu] 2
 Provider:
-!AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp
+!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu
 .IP \[bu] 2
 Type: string
 .IP \[bu] 2
@@ -26882,6 +27437,69 @@ Auckland (New Zealand) Region
 .PP
 Location constraint - must be set to match the Region.
 .PP
+Used when creating buckets only.
+.PP
+Properties:
+.IP \[bu] 2
+Config: location_constraint
+.IP \[bu] 2
+Env Var: RCLONE_S3_LOCATION_CONSTRAINT
+.IP \[bu] 2
+Provider: Qiniu
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: false
+.IP \[bu] 2
+Examples:
+.RS 2
+.IP \[bu] 2
+\[dq]cn-east-1\[dq]
+.RS 2
+.IP \[bu] 2
+East China Region 1
+.RE
+.IP \[bu] 2
+\[dq]cn-east-2\[dq]
+.RS 2
+.IP \[bu] 2
+East China Region 2
+.RE
+.IP \[bu] 2
+\[dq]cn-north-1\[dq]
+.RS 2
+.IP \[bu] 2
+North China Region 1
+.RE
+.IP \[bu] 2
+\[dq]cn-south-1\[dq]
+.RS 2
+.IP \[bu] 2
+South China Region 1
+.RE
+.IP \[bu] 2
+\[dq]us-north-1\[dq]
+.RS 2
+.IP \[bu] 2
+North America Region 1
+.RE
+.IP \[bu] 2
+\[dq]ap-southeast-1\[dq]
+.RS 2
+.IP \[bu] 2
+Southeast Asia Region 1
+.RE
+.IP \[bu] 2
+\[dq]ap-northeast-1\[dq]
+.RS 2
+.IP \[bu] 2
+Northeast Asia Region 1
+.RE
+.RE
+.SS --s3-location-constraint
+.PP
+Location constraint - must be set to match the Region.
+.PP
 Leave blank if not sure.
 Used when creating buckets only.
 .PP
@@ -26892,7 +27510,7 @@ Config: location_constraint
 Env Var: RCLONE_S3_LOCATION_CONSTRAINT
 .IP \[bu] 2
 Provider:
-!AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS
+!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS
 .IP \[bu] 2
 Type: string
 .IP \[bu] 2
@@ -27369,13 +27987,56 @@ Archived storage.
 Prices are lower, but it needs to be restored first to be accessed.
 .RE
 .RE
+.SS --s3-storage-class
+.PP
+The storage class to use when storing new objects in Qiniu.
+.PP
+Properties:
+.IP \[bu] 2
+Config: storage_class
+.IP \[bu] 2
+Env Var: RCLONE_S3_STORAGE_CLASS
+.IP \[bu] 2
+Provider: Qiniu
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: false
+.IP \[bu] 2
+Examples:
+.RS 2
+.IP \[bu] 2
+\[dq]STANDARD\[dq]
+.RS 2
+.IP \[bu] 2
+Standard storage class
+.RE
+.IP \[bu] 2
+\[dq]LINE\[dq]
+.RS 2
+.IP \[bu] 2
+Infrequent access storage mode
+.RE
+.IP \[bu] 2
+\[dq]GLACIER\[dq]
+.RS 2
+.IP \[bu] 2
+Archive storage mode
+.RE
+.IP \[bu] 2
+\[dq]DEEP_ARCHIVE\[dq]
+.RS 2
+.IP \[bu] 2
+Deep archive storage mode
+.RE
+.RE
 .SS Advanced options
 .PP
 Here are the Advanced options specific to s3 (Amazon S3 Compliant
 Storage Providers including AWS, Alibaba, Ceph, China Mobile,
 Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS,
-IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS,
-StackPath, Storj, Tencent COS and Wasabi).
+IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway,
+SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
 .SS --s3-bucket-acl
 .PP
 Canned ACL used when creating buckets.
@@ -27482,9 +28143,11 @@ AES256
 .RE
 .SS --s3-sse-customer-key
 .PP
-If using SSE-C you must provide the secret encryption key used to
+To use SSE-C you may provide the secret encryption key used to
 encrypt/decrypt your data.
 .PP
+Alternatively you can provide --sse-customer-key-base64.
+.PP
 Properties:
 .IP \[bu] 2
 Config: sse_customer_key
@@ -27506,6 +28169,34 @@ Examples:
 None
 .RE
 .RE
+.SS --s3-sse-customer-key-base64
+.PP
+If using SSE-C you must provide the secret encryption key encoded in
+base64 format to encrypt/decrypt your data.
+.PP
+Alternatively you can provide --sse-customer-key.
+.PP
+Properties:
+.IP \[bu] 2
+Config: sse_customer_key_base64
+.IP \[bu] 2
+Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_BASE64
+.IP \[bu] 2
+Provider: AWS,Ceph,ChinaMobile,Minio
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: false
+.IP \[bu] 2
+Examples:
+.RS 2
+.IP \[bu] 2
+\[dq]\[dq]
+.RS 2
+.IP \[bu] 2
+None
+.RE
+.RE
 .SS --s3-sse-customer-key-md5
 .PP
 If using SSE-C you may provide the secret encryption key MD5 checksum
@@ -28074,6 +28765,77 @@ Env Var: RCLONE_S3_USE_PRESIGNED_REQUEST
 Type: bool
 .IP \[bu] 2
 Default: false
+.SS --s3-versions
+.PP
+Include old versions in directory listings.
+.PP
+Properties:
+.IP \[bu] 2
+Config: versions
+.IP \[bu] 2
+Env Var: RCLONE_S3_VERSIONS
+.IP \[bu] 2
+Type: bool
+.IP \[bu] 2
+Default: false
+.SS --s3-version-at
+.PP
+Show file versions as they were at the specified time.
+.PP
+The parameter should be a date, \[dq]2006-01-02\[dq], datetime
+\[dq]2006-01-02 15:04:05\[dq] or a duration for that long ago, eg
+\[dq]100d\[dq] or \[dq]1h\[dq].
+.PP
+Note that when using this no file write operations are permitted, so you
+can\[aq]t upload files or delete them.
+.PP
+See the time option docs (https://rclone.org/docs/#time-option) for
+valid formats.
+.PP
+Properties:
+.IP \[bu] 2
+Config: version_at
+.IP \[bu] 2
+Env Var: RCLONE_S3_VERSION_AT
+.IP \[bu] 2
+Type: Time
+.IP \[bu] 2
+Default: off
+.SS --s3-decompress
+.PP
+If set this will decompress gzip encoded objects.
+.PP
+It is possible to upload objects to S3 with \[dq]Content-Encoding:
+gzip\[dq] set.
+Normally rclone will download these files as compressed objects.
+.PP
+If this flag is set then rclone will decompress these files with
+\[dq]Content-Encoding: gzip\[dq] as they are received.
+This means that rclone can\[aq]t check the size and hash but the file
+contents will be decompressed.
+.PP
+Properties:
+.IP \[bu] 2
+Config: decompress
+.IP \[bu] 2
+Env Var: RCLONE_S3_DECOMPRESS
+.IP \[bu] 2
+Type: bool
+.IP \[bu] 2
+Default: false
+.SS --s3-no-system-metadata
+.PP
+Suppress setting and reading of system metadata
+.PP
+Properties:
+.IP \[bu] 2
+Config: no_system_metadata
+.IP \[bu] 2
+Env Var: RCLONE_S3_NO_SYSTEM_METADATA
+.IP \[bu] 2
+Type: bool
+.IP \[bu] 2
+Default: false
 .SS Metadata
 .PP
 User metadata is stored as x-amz-meta- keys.
@@ -28348,6 +29110,52 @@ Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
 Options:
 .IP \[bu] 2
 \[dq]max-age\[dq]: Max age of upload to delete
+.SS cleanup-hidden
+.PP
+Remove old versions of files.
+.IP
+.nf
+\f[C]
+rclone backend cleanup-hidden remote: [options] [+]
+\f[R]
+.fi
+.PP
+This command removes any old hidden versions of files on a versions
+enabled bucket.
+.PP
+Note that you can use -i/--dry-run with this command to see what it
+would do.
+.IP
+.nf
+\f[C]
+rclone backend cleanup-hidden s3:bucket/path/to/dir
+\f[R]
+.fi
+.SS versioning
+.PP
+Set/get versioning support for a bucket.
+.IP
+.nf
+\f[C]
+rclone backend versioning remote: [options] [+]
+\f[R]
+.fi
+.PP
+This command sets versioning support if a parameter is passed and then
+returns the current versioning status for the bucket supplied.
+.IP
+.nf
+\f[C]
+rclone backend versioning s3:bucket # read status only
+rclone backend versioning s3:bucket Enabled
+rclone backend versioning s3:bucket Suspended
+\f[R]
+.fi
+.PP
+It may return \[dq]Enabled\[dq], \[dq]Suspended\[dq] or
+\[dq]Unversioned\[dq].
+Note that once versioning has been enabled the status can\[aq]t be set
+back to \[dq]Unversioned\[dq].
 .SS Anonymous access to public buckets
 .PP
 If you want to use rclone to access a public bucket, configure with a
@@ -29140,6 +29948,230 @@ d) Delete this remote
 y/e/d> y
 \f[R]
 .fi
+.SS IONOS Cloud
+.PP
+IONOS S3 Object Storage (https://cloud.ionos.com/storage/object-storage)
+is a service offered by IONOS for storing and accessing unstructured
+data.
+To connect to the service, you will need an access key and a secret key.
+These can be found in the Data Center Designer (https://dcd.ionos.com/),
+by selecting \f[B]Manager resources\f[R] > \f[B]Object Storage Key
+Manager\f[R].
+.PP
+Here is an example of a configuration.
+First, run \f[C]rclone config\f[R].
+This will walk you through an interactive setup process.
+Type \f[C]n\f[R] to add the new remote, and then enter a name:
+.IP
+.nf
+\f[C]
+Enter name for new remote.
+name> ionos-fra
+\f[R]
+.fi
+.PP
+Type \f[C]s3\f[R] to choose the connection type:
+.IP
+.nf
+\f[C]
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+   \[rs] (s3)
+[snip]
+Storage> s3
+\f[R]
+.fi
+.PP
+Type \f[C]IONOS\f[R]:
+.IP
+.nf
+\f[C]
+Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+[snip]
+XX / IONOS Cloud
+   \[rs] (IONOS)
+[snip]
+provider> IONOS
+\f[R]
+.fi
+.PP
+Press Enter to choose the default option
+\f[C]Enter AWS credentials in the next step\f[R]:
+.IP
+.nf
+\f[C]
+Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \[rs] (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \[rs] (true)
+env_auth>
+\f[R]
+.fi
+.PP
+Enter your Access Key and Secret key.
+These can be retrieved in the Data Center
+Designer (https://dcd.ionos.com/), click on the menu \[lq]Manager
+resources\[rq] / \[dq]Object Storage Key Manager\[dq].
+.IP
+.nf
+\f[C]
+Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> YOUR_ACCESS_KEY
+
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> YOUR_SECRET_KEY
+\f[R]
+.fi
+.PP
+Choose the region where your bucket is located:
+.IP
+.nf
+\f[C]
+Option region.
+Region where your bucket will be created and your data stored.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / Frankfurt, Germany
+   \[rs] (de)
+ 2 / Berlin, Germany
+   \[rs] (eu-central-2)
+ 3 / Logrono, Spain
+   \[rs] (eu-south-2)
+region> 2
+\f[R]
+.fi
+.PP
+Choose the endpoint from the same region:
+.IP
+.nf
+\f[C]
+Option endpoint.
+Endpoint for IONOS S3 Object Storage.
+Specify the endpoint from the same region.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / Frankfurt, Germany
+   \[rs] (s3-eu-central-1.ionoscloud.com)
+ 2 / Berlin, Germany
+   \[rs] (s3-eu-central-2.ionoscloud.com)
+ 3 / Logrono, Spain
+   \[rs] (s3-eu-south-2.ionoscloud.com)
+endpoint> 1
+\f[R]
+.fi
+.PP
+Press Enter to choose the default option or choose the desired ACL
+setting:
+.IP
+.nf
+\f[C]
+Option acl.
+Canned ACL used when creating buckets and storing or copying objects.
+This ACL is used for creating objects and if bucket_acl isn\[aq]t set, for creating buckets too.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Note that this ACL is applied when server-side copying objects as S3
+doesn\[aq]t copy the ACL from the source but rather writes a fresh one.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \[rs] (private)
+   / Owner gets FULL_CONTROL.
+[snip]
+acl>
+\f[R]
+.fi
+.PP
+Press Enter to skip the advanced config:
+.IP
+.nf
+\f[C]
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n>
+\f[R]
+.fi
+.PP
+Press Enter to save the configuration, and then \f[C]q\f[R] to quit the
+configuration process:
+.IP
+.nf
+\f[C]
+Configuration complete.
+Options:
+- type: s3
+- provider: IONOS
+- access_key_id: YOUR_ACCESS_KEY
+- secret_access_key: YOUR_SECRET_KEY
+- endpoint: s3-eu-central-1.ionoscloud.com
+Keep this \[dq]ionos-fra\[dq] remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+\f[R]
+.fi
+.PP
+Done! Now you can try some commands (for macOS, use \f[C]./rclone\f[R]
+instead of \f[C]rclone\f[R]).
+.IP "1)" 3
+Create a bucket (the name must be unique within the whole IONOS S3)
+.IP
+.nf
+\f[C]
+rclone mkdir ionos-fra:my-bucket
+\f[R]
+.fi
+.IP "2)" 3
+List available buckets
+.IP
+.nf
+\f[C]
+rclone lsd ionos-fra:
+\f[R]
+.fi
+.IP "4)" 3
+Copy a file from local to remote
+.IP
+.nf
+\f[C]
+rclone copy /Users/file.txt ionos-fra:my-bucket
+\f[R]
+.fi
+.IP "3)" 3
+List contents of a bucket
+.IP
+.nf
+\f[C]
+rclone ls ionos-fra:my-bucket
+\f[R]
+.fi
+.IP "5)" 3
+Copy a file from remote to local
+.IP
+.nf
+\f[C]
+rclone copy ionos-fra:my-bucket/file.txt
+\f[R]
+.fi
 .SS Minio
 .PP
 Minio (https://minio.io/) is an object storage server built for cloud
@@ -29217,6 +30249,228 @@ So once set up, for example, to copy files into a bucket
 rclone copy /path/to/files minio:bucket
 \f[R]
 .fi
+.SS Qiniu Cloud Object Storage (Kodo)
+.PP
+Qiniu Cloud Object Storage
+(Kodo) (https://www.qiniu.com/en/products/kodo), a completely
+independent-researched core technology which is proven by repeated
+customer experience has occupied absolute leading market leader
+position.
+Kodo can be widely applied to mass data management.
+.PP
+To configure access to Qiniu Kodo, follow the steps below:
+.IP "1." 3
+Run \f[C]rclone config\f[R] and select \f[C]n\f[R] for a new remote.
+.IP
+.nf
+\f[C]
+rclone config
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+\f[R]
+.fi
+.IP "2." 3
+Give the name of the configuration.
+For example, name it \[aq]qiniu\[aq].
+.IP
+.nf
+\f[C]
+name> qiniu
+\f[R]
+.fi
+.IP "3." 3
+Select \f[C]s3\f[R] storage.
+.IP
+.nf
+\f[C]
+Choose a number from below, or type in your own value
+ 1 / 1Fichier
+   \[rs] (fichier)
+ 2 / Akamai NetStorage
+   \[rs] (netstorage)
+ 3 / Alias for an existing remote
+   \[rs] (alias)
+ 4 / Amazon Drive
+   \[rs] (amazon cloud drive)
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi
+   \[rs] (s3)
+[snip]
+Storage> s3
+\f[R]
+.fi
+.IP "4." 3
+Select \f[C]Qiniu\f[R] provider.
+.IP
+.nf
+\f[C]
+Choose a number from below, or type in your own value
+1 / Amazon Web Services (AWS) S3
+   \[rs] \[dq]AWS\[dq]
+[snip]
+22 / Qiniu Object Storage (Kodo)
+   \[rs] (Qiniu)
+[snip]
+provider> Qiniu
+\f[R]
+.fi
+.IP "5." 3
+Enter your SecretId and SecretKey of Qiniu Kodo.
+.IP
+.nf
+\f[C]
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Enter a boolean value (true or false). Press Enter for the default (\[dq]false\[dq]).
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \[rs] \[dq]false\[dq]
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \[rs] \[dq]true\[dq]
+env_auth> 1
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default (\[dq]\[dq]).
+access_key_id> AKIDxxxxxxxxxx
+AWS Secret Access Key (password)
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default (\[dq]\[dq]).
+secret_access_key> xxxxxxxxxxx
+\f[R]
+.fi
+.IP "6." 3
+Select endpoint for Qiniu Kodo.
+This is the standard endpoint for different region.
+.IP
+.nf
+\f[C]
+   / The default endpoint - a good choice if you are unsure.
+ 1 | East China Region 1.
+   | Needs location constraint cn-east-1.
+   \[rs] (cn-east-1)
+   / East China Region 2.
+ 2 | Needs location constraint cn-east-2.
+   \[rs] (cn-east-2)
+   / North China Region 1.
+ 3 | Needs location constraint cn-north-1.
+   \[rs] (cn-north-1)
+   / South China Region 1.
+ 4 | Needs location constraint cn-south-1.
+   \[rs] (cn-south-1)
+   / North America Region.
+ 5 | Needs location constraint us-north-1.
+   \[rs] (us-north-1)
+   / Southeast Asia Region 1.
+ 6 | Needs location constraint ap-southeast-1.
+   \[rs] (ap-southeast-1)
+   / Northeast Asia Region 1.
+ 7 | Needs location constraint ap-northeast-1.
+   \[rs] (ap-northeast-1)
+[snip]
+endpoint> 1
+
+Option endpoint.
+Endpoint for Qiniu Object Storage.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / East China Endpoint 1
+   \[rs] (s3-cn-east-1.qiniucs.com)
+ 2 / East China Endpoint 2
+   \[rs] (s3-cn-east-2.qiniucs.com)
+ 3 / North China Endpoint 1
+   \[rs] (s3-cn-north-1.qiniucs.com)
+ 4 / South China Endpoint 1
+   \[rs] (s3-cn-south-1.qiniucs.com)
+ 5 / North America Endpoint 1
+   \[rs] (s3-us-north-1.qiniucs.com)
+ 6 / Southeast Asia Endpoint 1
+   \[rs] (s3-ap-southeast-1.qiniucs.com)
+ 7 / Northeast Asia Endpoint 1
+   \[rs] (s3-ap-northeast-1.qiniucs.com)
+endpoint> 1
+
+Option location_constraint.
+Location constraint - must be set to match the Region.
+Used when creating buckets only.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / East China Region 1
+   \[rs] (cn-east-1)
+ 2 / East China Region 2
+   \[rs] (cn-east-2)
+ 3 / North China Region 1
+   \[rs] (cn-north-1)
+ 4 / South China Region 1
+   \[rs] (cn-south-1)
+ 5 / North America Region 1
+   \[rs] (us-north-1)
+ 6 / Southeast Asia Region 1
+   \[rs] (ap-southeast-1)
+ 7 / Northeast Asia Region 1
+   \[rs] (ap-northeast-1)
+location_constraint> 1
+\f[R]
+.fi
+.IP "7." 3
+Choose acl and storage class.
+.IP
+.nf
+\f[C]
+Note that this ACL is applied when server-side copying objects as S3
+doesn\[aq]t copy the ACL from the source but rather writes a fresh one.
+Enter a string value. Press Enter for the default (\[dq]\[dq]).
+Choose a number from below, or type in your own value
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \[rs] (private)
+   / Owner gets FULL_CONTROL.
+ 2 | The AllUsers group gets READ access.
+   \[rs] (public-read)
+[snip]
+acl> 2
+The storage class to use when storing new objects in Tencent COS.
+Enter a string value. Press Enter for the default (\[dq]\[dq]).
+Choose a number from below, or type in your own value
+ 1 / Standard storage class
+   \[rs] (STANDARD)
+ 2 / Infrequent access storage mode
+   \[rs] (LINE)
+ 3 / Archive storage mode
+   \[rs] (GLACIER)
+ 4 / Deep archive storage mode
+   \[rs] (DEEP_ARCHIVE)
+[snip]
+storage_class> 1
+Edit advanced config? (y/n)
+y) Yes
+n) No (default)
+y/n> n
+Remote config
+--------------------
+[qiniu]
+- type: s3
+- provider: Qiniu
+- access_key_id: xxx
+- secret_access_key: xxx
+- region: cn-east-1
+- endpoint: s3-cn-east-1.qiniucs.com
+- location_constraint: cn-east-1
+- acl: public-read
+- storage_class: STANDARD
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+Current remotes:
+
+Name                 Type
+====                 ====
+qiniu                s3
+\f[R]
+.fi
 .SS RackCorp
 .PP
 RackCorp Object Storage (https://www.rackcorp.com/storage/s3storage) is
@@ -33714,8 +34968,8 @@ unencrypted content, as well as through a crypt remote for encrypted
 content, it is recommended to point the crypt remote to a separate
 directory within the wrapped remote.
 If you use a bucket-based storage system (e.g.
-Swift, S3, Google Compute Storage, B2, Hubic) it is generally advisable
-to wrap the crypt remote around a specific bucket (\f[C]s3:bucket\f[R]).
+Swift, S3, Google Compute Storage, B2) it is generally advisable to wrap
+the crypt remote around a specific bucket (\f[C]s3:bucket\f[R]).
 If wrapping around the entire root of the storage (\f[C]s3:\f[R]), and
 use the optional file name encryption, rclone will encrypt the bucket
 name.
@@ -33733,7 +34987,7 @@ content.
 The only possibility is to re-upload everything via a crypt remote
 configured with your new password.
 .PP
-Depending on the size of your data, your bandwith, storage quota etc,
+Depending on the size of your data, your bandwidth, storage quota etc,
 there are different approaches you can take: - If you have everything in
 a different location, for example on your local system, you could remove
 all of the prior encrypted files, change the password for your
@@ -33748,7 +35002,7 @@ and re-encrypting using the new password.
 When done, delete the original crypt remote directory and finally the
 rclone crypt configuration with the old password.
 All data will be streamed from the storage system and back, so you will
-get half the bandwith and be charged twice if you have upload and
+get half the bandwidth and be charged twice if you have upload and
 download quota on the storage system.
 .PP
 \f[B]Note\f[R]: A security problem related to the random password
@@ -34138,7 +35392,7 @@ How to encode the encrypted filename to text string.
 .PP
 This option could help with shortening the encrypted filename.
 The suitable option would depend on the way your remote count the
-filename length and if it\[aq]s case sensitve.
+filename length and if it\[aq]s case sensitive.
 .PP
 Properties:
 .IP \[bu] 2
@@ -34531,7 +35785,7 @@ Generally -1 (default, equivalent to 5) is recommended.
 Levels 1 to 9 increase compression at the cost of speed.
 Going past 6 generally offers very little return.
 .PP
-Level -2 uses Huffmann encoding only.
+Level -2 uses Huffman encoding only.
 Only use if you know what you are doing.
 Level 0 turns off compression.
 .PP
@@ -34701,7 +35955,7 @@ remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=:
 
 [AllDrives]
 type = combine
-remote = \[dq]My Drive=My Drive:\[dq] \[dq]Test Drive=Test Drive:\[dq]
+upstreams = \[dq]My Drive=My Drive:\[dq] \[dq]Test Drive=Test Drive:\[dq]
 \f[R]
 .fi
 .PP
@@ -35256,7 +36510,7 @@ Type: Duration
 Default: 0s
 .SS --dropbox-batch-commit-timeout
 .PP
-Max time to wait for a batch to finish comitting
+Max time to wait for a batch to finish committing
 .PP
 Properties:
 .IP \[bu] 2
@@ -35369,8 +36623,8 @@ accessible through a global file system.
 .SS Configuration
 .PP
 The initial setup for the Enterprise File Fabric backend involves
-getting a token from the the Enterprise File Fabric which you need to do
-in your browser.
+getting a token from the Enterprise File Fabric which you need to do in
+your browser.
 \f[C]rclone config\f[R] walks you through it.
 .PP
 Here is an example of how to make a remote called \f[C]remote\f[R].
@@ -35702,8 +36956,7 @@ rclone config
 Rclone config guides you through an interactive setup process.
 A minimal rclone FTP remote definition only requires host, username and
 password.
-For an anonymous FTP server, use \f[C]anonymous\f[R] as username and
-your email address as password.
+For an anonymous FTP server, see below.
 .IP
 .nf
 \f[C]
@@ -35797,11 +37050,41 @@ any excess files in the directory.
 rclone sync -i /home/local/directory remote:directory
 \f[R]
 .fi
-.SS Example without a config file
+.SS Anonymous FTP
+.PP
+When connecting to a FTP server that allows anonymous login, you can use
+the special \[dq]anonymous\[dq] username.
+Traditionally, this user account accepts any string as a password,
+although it is common to use either the password \[dq]anonymous\[dq] or
+\[dq]guest\[dq].
+Some servers require the use of a valid e-mail address as password.
+.PP
+Using on-the-fly or connection
+string (https://rclone.org/docs/#connection-strings) remotes makes it
+easy to access such servers, without requiring any configuration in
+advance.
+The following are examples of that:
 .IP
 .nf
 \f[C]
-rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=\[ga]rclone obscure dummy\[ga]
+rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=$(rclone obscure dummy)
+rclone lsf :ftp,host=speedtest.tele2.net,user=anonymous,pass=$(rclone obscure dummy):
+\f[R]
+.fi
+.PP
+The above examples work in Linux shells and in PowerShell, but not
+Windows Command Prompt.
+They execute the rclone
+obscure (https://rclone.org/commands/rclone_obscure/) command to create
+a password string in the format required by the pass option.
+The following examples are exactly the same, except use an already
+obscured string representation of the same password \[dq]dummy\[dq], and
+therefore works even in Windows Command Prompt:
+.IP
+.nf
+\f[C]
+rclone lsf :ftp: --ftp-host=speedtest.tele2.net --ftp-user=anonymous --ftp-pass=IXs2wc8OJOz7SYLBk47Ji1rHTmxM
+rclone lsf :ftp,host=speedtest.tele2.net,user=anonymous,pass=IXs2wc8OJOz7SYLBk47Ji1rHTmxM:
 \f[R]
 .fi
 .SS Implicit TLS
@@ -35818,7 +37101,7 @@ set (https://rclone.org/overview/#restricted-characters) the following
 characters are also replaced:
 .PP
 File names cannot end with the following characters.
-Repacement is limited to the last character in a file name:
+Replacement is limited to the last character in a file name:
 .PP
 .TS
 tab(@);
@@ -35971,6 +37254,19 @@ Here are the Advanced options specific to ftp (FTP).
 .PP
 Maximum number of FTP simultaneous connections, 0 for unlimited.
 .PP
+Note that setting this is very likely to cause deadlocks so it should be
+used with care.
+.PP
+If you are doing a sync or copy then make sure concurrency is one more
+than the sum of \f[C]--transfers\f[R] and \f[C]--checkers\f[R].
+.PP
+If you use \f[C]--check-first\f[R] then it just needs to be one more
+than the maximum of \f[C]--checkers\f[R] and \f[C]--transfers\f[R].
+.PP
+So for \f[C]concurrency 3\f[R] you\[aq]d use
+\f[C]--checkers 2 --transfers 2 --check-first\f[R] or
+\f[C]--checkers 1 --transfers 1\f[R].
+.PP
 Properties:
 .IP \[bu] 2
 Config: concurrency
@@ -36045,6 +37341,20 @@ Env Var: RCLONE_FTP_WRITING_MDTM
 Type: bool
 .IP \[bu] 2
 Default: false
+.SS --ftp-force-list-hidden
+.PP
+Use LIST -a to force listing of hidden files and folders.
+This will disable the use of MLSD.
+.PP
+Properties:
+.IP \[bu] 2
+Config: force_list_hidden
+.IP \[bu] 2
+Env Var: RCLONE_FTP_FORCE_LIST_HIDDEN
+.IP \[bu] 2
+Type: bool
+.IP \[bu] 2
+Default: false
 .SS --ftp-idle-timeout
 .PP
 Max time before closing idle connections.
@@ -37185,7 +38495,7 @@ If set this will decompress gzip encoded objects.
 .PP
 It is possible to upload objects to GCS with \[dq]Content-Encoding:
 gzip\[dq] set.
-Normally rclone will download these files files as compressed objects.
+Normally rclone will download these files as compressed objects.
 .PP
 If this flag is set then rclone will decompress these files with
 \[dq]Content-Encoding: gzip\[dq] as they are received.
@@ -37201,6 +38511,21 @@ Env Var: RCLONE_GCS_DECOMPRESS
 Type: bool
 .IP \[bu] 2
 Default: false
+.SS --gcs-endpoint
+.PP
+Endpoint for the service.
+.PP
+Leave blank normally.
+.PP
+Properties:
+.IP \[bu] 2
+Config: endpoint
+.IP \[bu] 2
+Env Var: RCLONE_GCS_ENDPOINT
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: false
 .SS --gcs-encoding
 .PP
 The encoding for the backend.
@@ -39081,14 +40406,14 @@ remote = drive,team_drive=0ABCDEFabcdefghijkl,root_folder_id=:
 
 [AllDrives]
 type = combine
-remote = \[dq]My Drive=My Drive:\[dq] \[dq]Test Drive=Test Drive:\[dq]
+upstreams = \[dq]My Drive=My Drive:\[dq] \[dq]Test Drive=Test Drive:\[dq]
 \f[R]
 .fi
 .PP
 Adding this to the rclone config file will cause those team drives to be
 accessible with the aliases shown.
-Any illegal charactes will be substituted with \[dq]_\[dq] and duplicate
-names will have numbers suffixed.
+Any illegal characters will be substituted with \[dq]_\[dq] and
+duplicate names will have numbers suffixed.
 It will also add a remote called AllDrives which shows all the shared
 drives combined into one directory tree.
 .SS untrash
@@ -40729,7 +42054,7 @@ HiDrive allows modification times to be set on objects accurate to 1
 second.
 .PP
 HiDrive supports its own hash type (https://static.hidrive.com/dev/0001)
-which is used to verify the integrety of file contents after successful
+which is used to verify the integrity of file contents after successful
 transfers.
 .SS Restricted filename characters
 .PP
@@ -41400,275 +42725,6 @@ of an rclone union remote.
 See List of backends that do not support rclone
 about (https://rclone.org/overview/#optional-features) and rclone
 about (https://rclone.org/commands/rclone_about/)
-.SH Hubic
-.PP
-Paths are specified as \f[C]remote:path\f[R]
-.PP
-Paths are specified as \f[C]remote:container\f[R] (or \f[C]remote:\f[R]
-for the \f[C]lsd\f[R] command.) You may put subdirectories in too, e.g.
-\f[C]remote:container/path/to/dir\f[R].
-.SS Configuration
-.PP
-The initial setup for Hubic involves getting a token from Hubic which
-you need to do in your browser.
-\f[C]rclone config\f[R] walks you through it.
-.PP
-Here is an example of how to make a remote called \f[C]remote\f[R].
-First run:
-.IP
-.nf
-\f[C]
- rclone config
-\f[R]
-.fi
-.PP
-This will guide you through an interactive setup process:
-.IP
-.nf
-\f[C]
-n) New remote
-s) Set configuration password
-n/s> n
-name> remote
-Type of storage to configure.
-Choose a number from below, or type in your own value
-[snip]
-XX / Hubic
-   \[rs] \[dq]hubic\[dq]
-[snip]
-Storage> hubic
-Hubic Client Id - leave blank normally.
-client_id>
-Hubic Client Secret - leave blank normally.
-client_secret>
-Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
-y) Yes
-n) No
-y/n> y
-If your browser doesn\[aq]t open automatically go to the following link: http://127.0.0.1:53682/auth
-Log in and authorize rclone for access
-Waiting for code...
-Got code
---------------------
-[remote]
-client_id =
-client_secret =
-token = {\[dq]access_token\[dq]:\[dq]XXXXXX\[dq]}
---------------------
-y) Yes this is OK
-e) Edit this remote
-d) Delete this remote
-y/e/d> y
-\f[R]
-.fi
-.PP
-See the remote setup docs (https://rclone.org/remote_setup/) for how to
-set it up on a machine with no Internet browser available.
-.PP
-Note that rclone runs a webserver on your local machine to collect the
-token as returned from Hubic.
-This only runs from the moment it opens your browser to the moment you
-get back the verification code.
-This is on \f[C]http://127.0.0.1:53682/\f[R] and this it may require you
-to unblock it temporarily if you are running a host firewall.
-.PP
-Once configured you can then use \f[C]rclone\f[R] like this,
-.PP
-List containers in the top level of your Hubic
-.IP
-.nf
-\f[C]
-rclone lsd remote:
-\f[R]
-.fi
-.PP
-List all the files in your Hubic
-.IP
-.nf
-\f[C]
-rclone ls remote:
-\f[R]
-.fi
-.PP
-To copy a local directory to an Hubic directory called backup
-.IP
-.nf
-\f[C]
-rclone copy /home/source remote:backup
-\f[R]
-.fi
-.PP
-If you want the directory to be visible in the official \f[I]Hubic
-browser\f[R], you need to copy your files to the \f[C]default\f[R]
-directory
-.IP
-.nf
-\f[C]
-rclone copy /home/source remote:default/backup
-\f[R]
-.fi
-.SS --fast-list
-.PP
-This remote supports \f[C]--fast-list\f[R] which allows you to use fewer
-transactions in exchange for more memory.
-See the rclone docs (https://rclone.org/docs/#fast-list) for more
-details.
-.SS Modified time
-.PP
-The modified time is stored as metadata on the object as
-\f[C]X-Object-Meta-Mtime\f[R] as floating point since the epoch accurate
-to 1 ns.
-.PP
-This is a de facto standard (used in the official python-swiftclient
-amongst others) for storing the modification time for an object.
-.PP
-Note that Hubic wraps the Swift backend, so most of the properties of
-are the same.
-.SS Standard options
-.PP
-Here are the Standard options specific to hubic (Hubic).
-.SS --hubic-client-id
-.PP
-OAuth Client Id.
-.PP
-Leave blank normally.
-.PP
-Properties:
-.IP \[bu] 2
-Config: client_id
-.IP \[bu] 2
-Env Var: RCLONE_HUBIC_CLIENT_ID
-.IP \[bu] 2
-Type: string
-.IP \[bu] 2
-Required: false
-.SS --hubic-client-secret
-.PP
-OAuth Client Secret.
-.PP
-Leave blank normally.
-.PP
-Properties:
-.IP \[bu] 2
-Config: client_secret
-.IP \[bu] 2
-Env Var: RCLONE_HUBIC_CLIENT_SECRET
-.IP \[bu] 2
-Type: string
-.IP \[bu] 2
-Required: false
-.SS Advanced options
-.PP
-Here are the Advanced options specific to hubic (Hubic).
-.SS --hubic-token
-.PP
-OAuth Access Token as a JSON blob.
-.PP
-Properties:
-.IP \[bu] 2
-Config: token
-.IP \[bu] 2
-Env Var: RCLONE_HUBIC_TOKEN
-.IP \[bu] 2
-Type: string
-.IP \[bu] 2
-Required: false
-.SS --hubic-auth-url
-.PP
-Auth server URL.
-.PP
-Leave blank to use the provider defaults.
-.PP
-Properties:
-.IP \[bu] 2
-Config: auth_url
-.IP \[bu] 2
-Env Var: RCLONE_HUBIC_AUTH_URL
-.IP \[bu] 2
-Type: string
-.IP \[bu] 2
-Required: false
-.SS --hubic-token-url
-.PP
-Token server url.
-.PP
-Leave blank to use the provider defaults.
-.PP
-Properties:
-.IP \[bu] 2
-Config: token_url
-.IP \[bu] 2
-Env Var: RCLONE_HUBIC_TOKEN_URL
-.IP \[bu] 2
-Type: string
-.IP \[bu] 2
-Required: false
-.SS --hubic-chunk-size
-.PP
-Above this size files will be chunked into a _segments container.
-.PP
-Above this size files will be chunked into a _segments container.
-The default for this is 5 GiB which is its maximum value.
-.PP
-Properties:
-.IP \[bu] 2
-Config: chunk_size
-.IP \[bu] 2
-Env Var: RCLONE_HUBIC_CHUNK_SIZE
-.IP \[bu] 2
-Type: SizeSuffix
-.IP \[bu] 2
-Default: 5Gi
-.SS --hubic-no-chunk
-.PP
-Don\[aq]t chunk files during streaming upload.
-.PP
-When doing streaming uploads (e.g.
-using rcat or mount) setting this flag will cause the swift backend to
-not upload chunked files.
-.PP
-This will limit the maximum upload size to 5 GiB.
-However non chunked files are easier to deal with and have an MD5SUM.
-.PP
-Rclone will still chunk files bigger than chunk_size when doing normal
-copy operations.
-.PP
-Properties:
-.IP \[bu] 2
-Config: no_chunk
-.IP \[bu] 2
-Env Var: RCLONE_HUBIC_NO_CHUNK
-.IP \[bu] 2
-Type: bool
-.IP \[bu] 2
-Default: false
-.SS --hubic-encoding
-.PP
-The encoding for the backend.
-.PP
-See the encoding section in the
-overview (https://rclone.org/overview/#encoding) for more info.
-.PP
-Properties:
-.IP \[bu] 2
-Config: encoding
-.IP \[bu] 2
-Env Var: RCLONE_HUBIC_ENCODING
-.IP \[bu] 2
-Type: MultiEncoder
-.IP \[bu] 2
-Default: Slash,InvalidUtf8
-.SS Limitations
-.PP
-This uses the normal OpenStack Swift mechanism to refresh the Swift API
-credentials and ignores the expires field returned by the Hubic API.
-.PP
-The Swift API doesn\[aq]t return a correct MD5SUM for segmented files
-(Dynamic or Static Large Objects) so rclone won\[aq]t check or use the
-MD5SUM for these.
 .SH Internet Archive
 .PP
 The Internet Archive backend utilizes Items on
@@ -41682,11 +42738,10 @@ Paths are specified as \f[C]remote:bucket\f[R] (or \f[C]remote:\f[R] for
 the \f[C]lsd\f[R] command.) You may put subdirectories in too, e.g.
 \f[C]remote:item/path/to/dir\f[R].
 .PP
-Once you have made a remote (see the provider specific section above)
-you can use it like this:
-.PP
 Unlike S3, listing up all items uploaded by you isn\[aq]t supported.
 .PP
+Once you have made a remote, you can use it like this:
+.PP
 Make a new item
 .IP
 .nf
@@ -41741,7 +42796,7 @@ However, some fields are reserved by both Internet Archive and rclone.
 The following are reserved by Internet Archive: - \f[C]name\f[R] -
 \f[C]source\f[R] - \f[C]size\f[R] - \f[C]md5\f[R] - \f[C]crc32\f[R] -
 \f[C]sha1\f[R] - \f[C]format\f[R] - \f[C]old_version\f[R] -
-\f[C]viruscheck\f[R]
+\f[C]viruscheck\f[R] - \f[C]summation\f[R]
 .PP
 Trying to set values to these keys is ignored with a warning.
 Only setting \f[C]mtime\f[R] is an exception.
@@ -41999,7 +43054,7 @@ string
 T}@T{
 01234567
 T}@T{
-N
+\f[B]Y\f[R]
 T}
 T{
 format
@@ -42010,7 +43065,7 @@ string
 T}@T{
 Comma-Separated Values
 T}@T{
-N
+\f[B]Y\f[R]
 T}
 T{
 md5
@@ -42021,7 +43076,7 @@ string
 T}@T{
 01234567012345670123456701234567
 T}@T{
-N
+\f[B]Y\f[R]
 T}
 T{
 mtime
@@ -42032,7 +43087,7 @@ RFC 3339
 T}@T{
 2006-01-02T15:04:05.999999999Z
 T}@T{
-N
+\f[B]Y\f[R]
 T}
 T{
 name
@@ -42043,7 +43098,7 @@ filename
 T}@T{
 backend/internetarchive/internetarchive.go
 T}@T{
-N
+\f[B]Y\f[R]
 T}
 T{
 old_version
@@ -42054,7 +43109,7 @@ boolean
 T}@T{
 true
 T}@T{
-N
+\f[B]Y\f[R]
 T}
 T{
 rclone-ia-mtime
@@ -42098,7 +43153,7 @@ string
 T}@T{
 0123456701234567012345670123456701234567
 T}@T{
-N
+\f[B]Y\f[R]
 T}
 T{
 size
@@ -42109,7 +43164,7 @@ decimal number
 T}@T{
 123456
 T}@T{
-N
+\f[B]Y\f[R]
 T}
 T{
 source
@@ -42120,7 +43175,18 @@ string
 T}@T{
 original
 T}@T{
-N
+\f[B]Y\f[R]
+T}
+T{
+summation
+T}@T{
+Check https://forum.rclone.org/t/31922 for how it is used
+T}@T{
+string
+T}@T{
+md5
+T}@T{
+\f[B]Y\f[R]
 T}
 T{
 viruscheck
@@ -42131,7 +43197,7 @@ unixtime
 T}@T{
 1654191352
 T}@T{
-N
+\f[B]Y\f[R]
 T}
 .TE
 .PP
@@ -42147,7 +43213,7 @@ Cloud (cloud.telia.se) * Telia Sky (sky.telia.no) * Tele2 * Tele2 Cloud
 (mittcloud.tele2.se) * Elkj\[/o]p (with subsidiaries): * Elkj\[/o]p
 Cloud (cloud.elkjop.no) * Elgiganten Sweden (cloud.elgiganten.se) *
 Elgiganten Denmark (cloud.elgiganten.dk) * Giganti Cloud
-(cloud.gigantti.fi) * ELKO Clouud (cloud.elko.is)
+(cloud.gigantti.fi) * ELKO Cloud (cloud.elko.is)
 .PP
 Most of the white-label versions are supported by this backend, although
 may require different authentication setup - described below.
@@ -42163,12 +43229,48 @@ than the official service, and you have to choose the correct one when
 setting up the remote.
 .SS Standard authentication
 .PP
-To configure Jottacloud you will need to generate a personal security
-token in the Jottacloud web interface.
-You will the option to do in your account security
-settings (https://www.jottacloud.com/web/secure) (for whitelabel version
-you need to find this page in its web interface).
-Note that the web interface may refer to this token as a JottaCli token.
+The standard authentication method used by the official service
+(jottacloud.com), as well as some of the whitelabel services, requires
+you to generate a single-use personal login token from the account
+security settings in the service\[aq]s web interface.
+Log in to your account, go to \[dq]Settings\[dq] and then
+\[dq]Security\[dq], or use the direct link presented to you by rclone
+when configuring the remote: .
+Scroll down to the section \[dq]Personal login token\[dq], and click the
+\[dq]Generate\[dq] button.
+Note that if you are using a whitelabel service you probably can\[aq]t
+use the direct link, you need to find the same page in their dedicated
+web interface, and also it may be in a different location than described
+above.
+.PP
+To access your account from multiple instances of rclone, you need to
+configure each of them with a separate personal login token.
+E.g.
+you create a Jottacloud remote with rclone in one location, and copy the
+configuration file to a second location where you also want to run
+rclone and access the same remote.
+Then you need to replace the token for one of them, using the config
+reconnect (https://rclone.org/commands/rclone_config_reconnect/)
+command, which requires you to generate a new personal login token and
+supply as input.
+If you do not do this, the token may easily end up being invalidated,
+resulting in both instances failing with an error message something
+along the lines of:
+.IP
+.nf
+\f[C]
+oauth2: cannot fetch token: 400 Bad Request
+Response: {\[dq]error\[dq]:\[dq]invalid_grant\[dq],\[dq]error_description\[dq]:\[dq]Stale token\[dq]}
+\f[R]
+.fi
+.PP
+When this happens, you need to replace the token as described above to
+be able to use your remote again.
+.PP
+All personal login tokens you have taken into use will be listed in the
+web interface under \[dq]My logged in devices\[dq], and from the right
+side of that list you can click the \[dq]X\[dq] button to revoke
+individual tokens.
 .SS Legacy authentication
 .PP
 If you are using one of the whitelabel versions (e.g.
@@ -43750,7 +44852,7 @@ Use \f[C]rclone dedupe\f[R] to fix duplicated files.
 .SS Object not found
 .PP
 If you are connecting to your Mega remote for the first time, to test
-access and syncronisation, you may receive an error such as
+access and synchronization, you may receive an error such as
 .IP
 .nf
 \f[C]
@@ -44214,7 +45316,7 @@ group.
 \f[B]Implicit Directory\f[R].
 This refers to a directory within a path that has not been physically
 created.
-For example, during upload of a file, non-existent subdirectories can be
+For example, during upload of a file, nonexistent subdirectories can be
 specified in the target path.
 NetStorage creates these as \[dq]implicit.\[dq] While the directories
 aren\[aq]t physically created, they exist implicitly and the noted path
@@ -45245,7 +46347,7 @@ performing requests.
 .PP
 You may choose to create and use your own Client ID, in case the default
 one does not work well for you.
-For example, you might see throtting.
+For example, you might see throttling.
 .SS Creating Client ID for OneDrive Personal
 .PP
 To create your own Client ID, please follow these steps:
@@ -45312,8 +46414,7 @@ Make sure to create the App with your business account.
 Follow the steps above to create an App.
 However, we need a different account type here:
 \f[C]Accounts in this organizational directory only (*** - Single tenant)\f[R].
-Note that you can also change the account type aftering creating the
-App.
+Note that you can also change the account type after creating the App.
 .IP "3." 3
 Find the tenant
 ID (https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/active-directory-how-to-find-tenant)
@@ -45555,7 +46656,7 @@ Microsoft Cloud Germany
 \[dq]cn\[dq]
 .RS 2
 .IP \[bu] 2
-Azure and Office 365 operated by 21Vianet in China
+Azure and Office 365 operated by Vnet Group in China
 .RE
 .RE
 .SS Advanced options
@@ -45958,7 +47059,7 @@ here (https://support.office.com/en-us/article/invalid-file-names-and-file-types
 .SS Versions
 .PP
 Every change in a file OneDrive causes the service to create a new
-version of the the file.
+version of the file.
 This counts against a users quota.
 For example changing the modification time of a file creates a second
 version, so the file apparently uses twice the space.
@@ -46471,6 +47572,662 @@ of an rclone union remote.
 See List of backends that do not support rclone
 about (https://rclone.org/overview/#optional-features) and rclone
 about (https://rclone.org/commands/rclone_about/)
+.SH Oracle Object Storage
+.PP
+Oracle Object Storage
+Overview (https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/objectstorageoverview.htm)
+.PP
+Oracle Object Storage
+FAQ (https://www.oracle.com/cloud/storage/object-storage/faq/)
+.PP
+Paths are specified as \f[C]remote:bucket\f[R] (or \f[C]remote:\f[R] for
+the \f[C]lsd\f[R] command.) You may put subdirectories in too, e.g.
+\f[C]remote:bucket/path/to/dir\f[R].
+.SS Configuration
+.PP
+Here is an example of making an oracle object storage configuration.
+\f[C]rclone config\f[R] walks you through it.
+.PP
+Here is an example of how to make a remote called \f[C]remote\f[R].
+First run:
+.IP
+.nf
+\f[C]
+ rclone config
+\f[R]
+.fi
+.PP
+This will guide you through an interactive setup process:
+.IP
+.nf
+\f[C]
+n) New remote
+d) Delete remote
+r) Rename remote
+c) Copy remote
+s) Set configuration password
+q) Quit config
+e/n/d/r/c/s/q> n
+
+Enter name for new remote.
+name> remote
+
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+XX / Oracle Cloud Infrastructure Object Storage
+   \[rs] (oracleobjectstorage)
+Storage> oracleobjectstorage
+
+Option provider.
+Choose your Auth Provider
+Choose a number from below, or type in your own string value.
+Press Enter for the default (env_auth).
+ 1 / automatically pickup the credentials from runtime(env), first one to provide auth wins
+   \[rs] (env_auth)
+   / use an OCI user and an API key for authentication.
+ 2 | you\[cq]ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key.
+   | https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm
+   \[rs] (user_principal_auth)
+   / use instance principals to authorize an instance to make API calls. 
+ 3 | each instance has its own identity, and authenticates using the certificates that are read from instance metadata. 
+   | https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm
+   \[rs] (instance_principal_auth)
+ 4 / use resource principals to make API calls
+   \[rs] (resource_principal_auth)
+ 5 / no credentials needed, this is typically for reading public buckets
+   \[rs] (no_auth)
+provider> 2
+
+Option namespace.
+Object storage namespace
+Enter a value.
+namespace> idbamagbg734
+
+Option compartment.
+Object storage compartment OCID
+Enter a value.
+compartment> ocid1.compartment.oc1..aaaaaaaapufkxc7ame3sthry5i7ujrwfc7ejnthhu6bhanm5oqfjpyasjkba
+
+Option region.
+Object storage Region
+Enter a value.
+region> us-ashburn-1
+
+Option endpoint.
+Endpoint for Object storage API.
+Leave blank to use the default endpoint for the region.
+Enter a value. Press Enter to leave empty.
+endpoint> 
+
+Option config_file.
+Path to OCI config file
+Choose a number from below, or type in your own string value.
+Press Enter for the default (\[ti]/.oci/config).
+ 1 / oci configuration file location
+   \[rs] (\[ti]/.oci/config)
+config_file> /etc/oci/dev.conf
+
+Option config_profile.
+Profile name inside OCI config file
+Choose a number from below, or type in your own string value.
+Press Enter for the default (Default).
+ 1 / Use the default profile
+   \[rs] (Default)
+config_profile> Test
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: oracleobjectstorage
+- namespace: idbamagbg734
+- compartment: ocid1.compartment.oc1..aaaaaaaapufkxc7ame3sthry5i7ujrwfc7ejnthhu6bhanm5oqfjpyasjkba
+- region: us-ashburn-1
+- provider: user_principal_auth
+- config_file: /etc/oci/dev.conf
+- config_profile: Test
+Keep this \[dq]remote\[dq] remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+\f[R]
+.fi
+.PP
+See all buckets
+.IP
+.nf
+\f[C]
+rclone lsd remote:
+\f[R]
+.fi
+.PP
+Create a new bucket
+.IP
+.nf
+\f[C]
+rclone mkdir remote:bucket
+\f[R]
+.fi
+.PP
+List the contents of a bucket
+.IP
+.nf
+\f[C]
+rclone ls remote:bucket
+rclone ls remote:bucket --max-depth 1
+\f[R]
+.fi
+.SS Modified time
+.PP
+The modified time is stored as metadata on the object as
+\f[C]opc-meta-mtime\f[R] as floating point since the epoch, accurate to
+1 ns.
+.PP
+If the modification time needs to be updated rclone will attempt to
+perform a server side copy to update the modification if the object can
+be copied in a single part.
+In the case the object is larger than 5Gb, the object will be uploaded
+rather than copied.
+.PP
+Note that reading this from the object takes an additional
+\f[C]HEAD\f[R] request as the metadata isn\[aq]t returned in object
+listings.
+.SS Multipart uploads
+.PP
+rclone supports multipart uploads with OOS which means that it can
+upload files bigger than 5 GiB.
+.PP
+Note that files uploaded \f[I]both\f[R] with multipart upload
+\f[I]and\f[R] through crypt remotes do not have MD5 sums.
+.PP
+rclone switches from single part uploads to multipart uploads at the
+point specified by \f[C]--oos-upload-cutoff\f[R].
+This can be a maximum of 5 GiB and a minimum of 0 (ie always upload
+multipart files).
+.PP
+The chunk sizes used in the multipart upload are specified by
+\f[C]--oos-chunk-size\f[R] and the number of chunks uploaded
+concurrently is specified by \f[C]--oos-upload-concurrency\f[R].
+.PP
+Multipart uploads will use \f[C]--transfers\f[R] *
+\f[C]--oos-upload-concurrency\f[R] * \f[C]--oos-chunk-size\f[R] extra
+memory.
+Single part uploads to not use extra memory.
+.PP
+Single part transfers can be faster than multipart transfers or slower
+depending on your latency from oos - the more latency, the more likely
+single part transfers will be faster.
+.PP
+Increasing \f[C]--oos-upload-concurrency\f[R] will increase throughput
+(8 would be a sensible value) and increasing \f[C]--oos-chunk-size\f[R]
+also increases throughput (16M would be sensible).
+Increasing either of these will use more memory.
+The default values are high enough to gain most of the possible
+performance without using too much memory.
+.SS Standard options
+.PP
+Here are the Standard options specific to oracleobjectstorage (Oracle
+Cloud Infrastructure Object Storage).
+.SS --oos-provider
+.PP
+Choose your Auth Provider
+.PP
+Properties:
+.IP \[bu] 2
+Config: provider
+.IP \[bu] 2
+Env Var: RCLONE_OOS_PROVIDER
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Default: \[dq]env_auth\[dq]
+.IP \[bu] 2
+Examples:
+.RS 2
+.IP \[bu] 2
+\[dq]env_auth\[dq]
+.RS 2
+.IP \[bu] 2
+automatically pickup the credentials from runtime(env), first one to
+provide auth wins
+.RE
+.IP \[bu] 2
+\[dq]user_principal_auth\[dq]
+.RS 2
+.IP \[bu] 2
+use an OCI user and an API key for authentication.
+.IP \[bu] 2
+you\[cq]ll need to put in a config file your tenancy OCID, user OCID,
+region, the path, fingerprint to an API key.
+.IP \[bu] 2
+https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm
+.RE
+.IP \[bu] 2
+\[dq]instance_principal_auth\[dq]
+.RS 2
+.IP \[bu] 2
+use instance principals to authorize an instance to make API calls.
+.IP \[bu] 2
+each instance has its own identity, and authenticates using the
+certificates that are read from instance metadata.
+.IP \[bu] 2
+https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm
+.RE
+.IP \[bu] 2
+\[dq]resource_principal_auth\[dq]
+.RS 2
+.IP \[bu] 2
+use resource principals to make API calls
+.RE
+.IP \[bu] 2
+\[dq]no_auth\[dq]
+.RS 2
+.IP \[bu] 2
+no credentials needed, this is typically for reading public buckets
+.RE
+.RE
+.SS --oos-namespace
+.PP
+Object storage namespace
+.PP
+Properties:
+.IP \[bu] 2
+Config: namespace
+.IP \[bu] 2
+Env Var: RCLONE_OOS_NAMESPACE
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: true
+.SS --oos-compartment
+.PP
+Object storage compartment OCID
+.PP
+Properties:
+.IP \[bu] 2
+Config: compartment
+.IP \[bu] 2
+Env Var: RCLONE_OOS_COMPARTMENT
+.IP \[bu] 2
+Provider: !no_auth
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: true
+.SS --oos-region
+.PP
+Object storage Region
+.PP
+Properties:
+.IP \[bu] 2
+Config: region
+.IP \[bu] 2
+Env Var: RCLONE_OOS_REGION
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: true
+.SS --oos-endpoint
+.PP
+Endpoint for Object storage API.
+.PP
+Leave blank to use the default endpoint for the region.
+.PP
+Properties:
+.IP \[bu] 2
+Config: endpoint
+.IP \[bu] 2
+Env Var: RCLONE_OOS_ENDPOINT
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: false
+.SS --oos-config-file
+.PP
+Path to OCI config file
+.PP
+Properties:
+.IP \[bu] 2
+Config: config_file
+.IP \[bu] 2
+Env Var: RCLONE_OOS_CONFIG_FILE
+.IP \[bu] 2
+Provider: user_principal_auth
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Default: \[dq]\[ti]/.oci/config\[dq]
+.IP \[bu] 2
+Examples:
+.RS 2
+.IP \[bu] 2
+\[dq]\[ti]/.oci/config\[dq]
+.RS 2
+.IP \[bu] 2
+oci configuration file location
+.RE
+.RE
+.SS --oos-config-profile
+.PP
+Profile name inside the oci config file
+.PP
+Properties:
+.IP \[bu] 2
+Config: config_profile
+.IP \[bu] 2
+Env Var: RCLONE_OOS_CONFIG_PROFILE
+.IP \[bu] 2
+Provider: user_principal_auth
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Default: \[dq]Default\[dq]
+.IP \[bu] 2
+Examples:
+.RS 2
+.IP \[bu] 2
+\[dq]Default\[dq]
+.RS 2
+.IP \[bu] 2
+Use the default profile
+.RE
+.RE
+.SS Advanced options
+.PP
+Here are the Advanced options specific to oracleobjectstorage (Oracle
+Cloud Infrastructure Object Storage).
+.SS --oos-upload-cutoff
+.PP
+Cutoff for switching to chunked upload.
+.PP
+Any files larger than this will be uploaded in chunks of chunk_size.
+The minimum is 0 and the maximum is 5 GiB.
+.PP
+Properties:
+.IP \[bu] 2
+Config: upload_cutoff
+.IP \[bu] 2
+Env Var: RCLONE_OOS_UPLOAD_CUTOFF
+.IP \[bu] 2
+Type: SizeSuffix
+.IP \[bu] 2
+Default: 200Mi
+.SS --oos-chunk-size
+.PP
+Chunk size to use for uploading.
+.PP
+When uploading files larger than upload_cutoff or files with unknown
+size (e.g.
+from \[dq]rclone rcat\[dq] or uploaded with \[dq]rclone mount\[dq] or
+google photos or google docs) they will be uploaded as multipart uploads
+using this chunk size.
+.PP
+Note that \[dq]upload_concurrency\[dq] chunks of this size are buffered
+in memory per transfer.
+.PP
+If you are transferring large files over high-speed links and you have
+enough memory, then increasing this will speed up the transfers.
+.PP
+Rclone will automatically increase the chunk size when uploading a large
+file of known size to stay below the 10,000 chunks limit.
+.PP
+Files of unknown size are uploaded with the configured chunk_size.
+Since the default chunk size is 5 MiB and there can be at most 10,000
+chunks, this means that by default the maximum size of a file you can
+stream upload is 48 GiB.
+If you wish to stream upload larger files then you will need to increase
+chunk_size.
+.PP
+Increasing the chunk size decreases the accuracy of the progress
+statistics displayed with \[dq]-P\[dq] flag.
+.PP
+Properties:
+.IP \[bu] 2
+Config: chunk_size
+.IP \[bu] 2
+Env Var: RCLONE_OOS_CHUNK_SIZE
+.IP \[bu] 2
+Type: SizeSuffix
+.IP \[bu] 2
+Default: 5Mi
+.SS --oos-upload-concurrency
+.PP
+Concurrency for multipart uploads.
+.PP
+This is the number of chunks of the same file that are uploaded
+concurrently.
+.PP
+If you are uploading small numbers of large files over high-speed links
+and these uploads do not fully utilize your bandwidth, then increasing
+this may help to speed up the transfers.
+.PP
+Properties:
+.IP \[bu] 2
+Config: upload_concurrency
+.IP \[bu] 2
+Env Var: RCLONE_OOS_UPLOAD_CONCURRENCY
+.IP \[bu] 2
+Type: int
+.IP \[bu] 2
+Default: 10
+.SS --oos-copy-cutoff
+.PP
+Cutoff for switching to multipart copy.
+.PP
+Any files larger than this that need to be server-side copied will be
+copied in chunks of this size.
+.PP
+The minimum is 0 and the maximum is 5 GiB.
+.PP
+Properties:
+.IP \[bu] 2
+Config: copy_cutoff
+.IP \[bu] 2
+Env Var: RCLONE_OOS_COPY_CUTOFF
+.IP \[bu] 2
+Type: SizeSuffix
+.IP \[bu] 2
+Default: 4.656Gi
+.SS --oos-copy-timeout
+.PP
+Timeout for copy.
+.PP
+Copy is an asynchronous operation, specify timeout to wait for copy to
+succeed
+.PP
+Properties:
+.IP \[bu] 2
+Config: copy_timeout
+.IP \[bu] 2
+Env Var: RCLONE_OOS_COPY_TIMEOUT
+.IP \[bu] 2
+Type: Duration
+.IP \[bu] 2
+Default: 1m0s
+.SS --oos-disable-checksum
+.PP
+Don\[aq]t store MD5 checksum with object metadata.
+.PP
+Normally rclone will calculate the MD5 checksum of the input before
+uploading it so it can add it to metadata on the object.
+This is great for data integrity checking but can cause long delays for
+large files to start uploading.
+.PP
+Properties:
+.IP \[bu] 2
+Config: disable_checksum
+.IP \[bu] 2
+Env Var: RCLONE_OOS_DISABLE_CHECKSUM
+.IP \[bu] 2
+Type: bool
+.IP \[bu] 2
+Default: false
+.SS --oos-encoding
+.PP
+The encoding for the backend.
+.PP
+See the encoding section in the
+overview (https://rclone.org/overview/#encoding) for more info.
+.PP
+Properties:
+.IP \[bu] 2
+Config: encoding
+.IP \[bu] 2
+Env Var: RCLONE_OOS_ENCODING
+.IP \[bu] 2
+Type: MultiEncoder
+.IP \[bu] 2
+Default: Slash,InvalidUtf8,Dot
+.SS --oos-leave-parts-on-error
+.PP
+If true avoid calling abort upload on a failure, leaving all
+successfully uploaded parts on S3 for manual recovery.
+.PP
+It should be set to true for resuming uploads across different sessions.
+.PP
+WARNING: Storing parts of an incomplete multipart upload counts towards
+space usage on object storage and will add additional costs if not
+cleaned up.
+.PP
+Properties:
+.IP \[bu] 2
+Config: leave_parts_on_error
+.IP \[bu] 2
+Env Var: RCLONE_OOS_LEAVE_PARTS_ON_ERROR
+.IP \[bu] 2
+Type: bool
+.IP \[bu] 2
+Default: false
+.SS --oos-no-check-bucket
+.PP
+If set, don\[aq]t attempt to check the bucket exists or create it.
+.PP
+This can be useful when trying to minimise the number of transactions
+rclone does if you know the bucket exists already.
+.PP
+It can also be needed if the user you are using does not have bucket
+creation permissions.
+.PP
+Properties:
+.IP \[bu] 2
+Config: no_check_bucket
+.IP \[bu] 2
+Env Var: RCLONE_OOS_NO_CHECK_BUCKET
+.IP \[bu] 2
+Type: bool
+.IP \[bu] 2
+Default: false
+.SS Backend commands
+.PP
+Here are the commands specific to the oracleobjectstorage backend.
+.PP
+Run them with
+.IP
+.nf
+\f[C]
+rclone backend COMMAND remote:
+\f[R]
+.fi
+.PP
+The help below will explain what arguments each command takes.
+.PP
+See the backend (https://rclone.org/commands/rclone_backend/) command
+for more info on how to pass options and arguments.
+.PP
+These can be run on a running backend using the rc command
+backend/command (https://rclone.org/rc/#backend-command).
+.SS rename
+.PP
+change the name of an object
+.IP
+.nf
+\f[C]
+rclone backend rename remote: [options] [+]
+\f[R]
+.fi
+.PP
+This command can be used to rename a object.
+.PP
+Usage Examples:
+.IP
+.nf
+\f[C]
+rclone backend rename oos:bucket relative-object-path-under-bucket object-new-name
+\f[R]
+.fi
+.SS list-multipart-uploads
+.PP
+List the unfinished multipart uploads
+.IP
+.nf
+\f[C]
+rclone backend list-multipart-uploads remote: [options] [+]
+\f[R]
+.fi
+.PP
+This command lists the unfinished multipart uploads in JSON format.
+.IP
+.nf
+\f[C]
+rclone backend list-multipart-uploads oos:bucket/path/to/object
+\f[R]
+.fi
+.PP
+It returns a dictionary of buckets with values as lists of unfinished
+multipart uploads.
+.PP
+You can call it with no bucket in which case it lists all bucket, with a
+bucket or with a bucket and path.
+.IP
+.nf
+\f[C]
+{
+  \[dq]test-bucket\[dq]: [
+            {
+                    \[dq]namespace\[dq]: \[dq]test-namespace\[dq],
+                    \[dq]bucket\[dq]: \[dq]test-bucket\[dq],
+                    \[dq]object\[dq]: \[dq]600m.bin\[dq],
+                    \[dq]uploadId\[dq]: \[dq]51dd8114-52a4-b2f2-c42f-5291f05eb3c8\[dq],
+                    \[dq]timeCreated\[dq]: \[dq]2022-07-29T06:21:16.595Z\[dq],
+                    \[dq]storageTier\[dq]: \[dq]Standard\[dq]
+            }
+    ]
+\f[R]
+.fi
+.SS cleanup
+.PP
+Remove unfinished multipart uploads.
+.IP
+.nf
+\f[C]
+rclone backend cleanup remote: [options] [+]
+\f[R]
+.fi
+.PP
+This command removes unfinished multipart uploads of age greater than
+max-age which defaults to 24 hours.
+.PP
+Note that you can use -i/--dry-run with this command to see what it
+would do.
+.IP
+.nf
+\f[C]
+rclone backend cleanup oos:bucket/path/to/object
+rclone backend cleanup -o max-age=7w oos:bucket/path/to/object
+\f[R]
+.fi
+.PP
+Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
+.PP
+Options:
+.IP \[bu] 2
+\[dq]max-age\[dq]: Max age of upload to delete
 .SH QingStor
 .PP
 Paths are specified as \f[C]remote:bucket\f[R] (or \f[C]remote:\f[R] for
@@ -47128,7 +48885,8 @@ Memset Memstore (https://www.memset.com/cloud/storage/)
 OVH Object
 Storage (https://www.ovh.co.uk/public-cloud/storage/object-storage/)
 .IP \[bu] 2
-Oracle Cloud Storage (https://cloud.oracle.com/object-storage/buckets)
+Oracle Cloud
+Storage (https://docs.oracle.com/en-us/iaas/integration/doc/configure-object-storage.html)
 .IP \[bu] 2
 IBM Bluemix Cloud ObjectStorage
 Swift (https://console.bluemix.net/docs/infrastructure/objectstorage-swift/index.html)
@@ -47818,6 +49576,41 @@ Env Var: RCLONE_SWIFT_NO_CHUNK
 Type: bool
 .IP \[bu] 2
 Default: false
+.SS --swift-no-large-objects
+.PP
+Disable support for static and dynamic large objects
+.PP
+Swift cannot transparently store files bigger than 5 GiB.
+There are two schemes for doing that, static or dynamic large objects,
+and the API does not allow rclone to determine whether a file is a
+static or dynamic large object without doing a HEAD on the object.
+Since these need to be treated differently, this means rclone has to
+issue HEAD requests for objects for example when reading checksums.
+.PP
+When \f[C]no_large_objects\f[R] is set, rclone will assume that there
+are no static or dynamic large objects stored.
+This means it can stop doing the extra HEAD calls which in turn
+increases performance greatly especially when doing a swift to swift
+transfer with \f[C]--checksum\f[R] set.
+.PP
+Setting this option implies \f[C]no_chunk\f[R] and also that no files
+will be uploaded in chunks, so files bigger than 5 GiB will just fail on
+upload.
+.PP
+If you set this option and there \f[I]are\f[R] static or dynamic large
+objects, then this will give incorrect hashes for them.
+Downloads will succeed, but other operations such as Remove and Copy
+will fail.
+.PP
+Properties:
+.IP \[bu] 2
+Config: no_large_objects
+.IP \[bu] 2
+Env Var: RCLONE_SWIFT_NO_LARGE_OBJECTS
+.IP \[bu] 2
+Type: bool
+.IP \[bu] 2
+Default: false
 .SS --swift-encoding
 .PP
 The encoding for the backend.
@@ -49093,7 +50886,7 @@ If the path does not begin with a \f[C]/\f[R] it is relative to the home
 directory of the user.
 An empty path \f[C]remote:\f[R] refers to the user\[aq]s home directory.
 For example, \f[C]rclone lsd remote:\f[R] would list the home directory
-of the user cofigured in the rclone remote config
+of the user configured in the rclone remote config
 (\f[C]i.e /home/sftpuser\f[R]).
 However, \f[C]rclone lsd remote:/\f[R] would list the root directory for
 remote machine (i.e.
@@ -49424,7 +51217,7 @@ On a Windows server the shell handling is different: Although it can
 also be set up to use a Unix type shell, e.g.
 Cygwin bash, the default is to use Windows Command Prompt (cmd.exe), and
 PowerShell is a recommended alternative.
-All of these have bahave differently, which rclone must handle.
+All of these have behave differently, which rclone must handle.
 .PP
 Rclone tries to auto-detect what type of shell is used on the server,
 first time you access the SFTP remote.
@@ -49463,7 +51256,7 @@ If you configure a sftp remote without a config file, e.g.
 an on the fly (https://rclone.org/docs/#backend-path-to-dir%5D) remote,
 rclone will have nowhere to store the result, and it will re-run the
 command on every access.
-To avoid this you should explicitely set the \f[C]shell_type\f[R] option
+To avoid this you should explicitly set the \f[C]shell_type\f[R] option
 to the correct value, or to \f[C]none\f[R] if you want to prevent rclone
 from executing any remote shell commands.
 .PP
@@ -49471,9 +51264,8 @@ It is also important to note that, since the shell type decides how
 quoting and escaping of file paths used as command-line arguments are
 performed, configuring the wrong shell type may leave you exposed to
 command injection exploits.
-Make sure to confirm the auto-detected shell type, or explicitely set
-the shell type you know is correct, or disable shell access until you
-know.
+Make sure to confirm the auto-detected shell type, or explicitly set the
+shell type you know is correct, or disable shell access until you know.
 .SS Checksum
 .PP
 SFTP does not natively support checksums (file hash), but rclone is able
@@ -50085,19 +51877,25 @@ Default: 1m0s
 .PP
 Upload and download chunk size.
 .PP
-This controls the maximum packet size used in the SFTP protocol.
-The RFC limits this to 32768 bytes (32k), however a lot of servers
-support larger sizes and setting it larger will increase transfer speed
-dramatically on high latency links.
+This controls the maximum size of payload in SFTP protocol packets.
+The RFC limits this to 32768 bytes (32k), which is the default.
+However, a lot of servers support larger sizes, typically limited to a
+maximum total package size of 256k, and setting it larger will increase
+transfer speed dramatically on high latency links.
+This includes OpenSSH, and, for example, using the value of 255k works
+well, leaving plenty of room for overhead while still being within a
+total packet size of 256k.
 .PP
-Only use a setting higher than 32k if you always connect to the same
-server or after sufficiently broad testing.
-.PP
-For example using the value of 252k with OpenSSH works well with its
-maximum packet size of 256k.
-.PP
-If you get the error \[dq]failed to send packet header: EOF\[dq] when
-copying a large file, try lowering this number.
+Make sure to test thoroughly before using a value higher than 32k, and
+only use it if you always connect to the same server or after
+sufficiently broad testing.
+If you get errors such as \[dq]failed to send packet payload: EOF\[dq],
+lots of \[dq]connection lost\[dq], or \[dq]corrupted on transfer\[dq],
+when copying a larger file, try lowering the value.
+The server run by rclone serve sftp sends packets with standard 32k
+maximum payload so you must not set a different chunk_size when
+downloading files, but it accepts packets up to the 256k total size, so
+for uploads the chunk_size can be set as for the OpenSSH example above.
 .PP
 Properties:
 .IP \[bu] 2
@@ -50202,6 +52000,267 @@ Hetzner Storage Boxes are supported through the SFTP backend on port 23.
 .PP
 See Hetzner\[aq]s documentation for
 details (https://docs.hetzner.com/robot/storage-box/access/access-ssh-rsync-borg#rclone)
+.SH SMB
+.PP
+SMB is a communication protocol to share files over
+network (https://en.wikipedia.org/wiki/Server_Message_Block).
+.PP
+This relies on go-smb2
+library (https://github.com/hirochachacha/go-smb2/) for communication
+with SMB protocol.
+.PP
+Paths are specified as \f[C]remote:sharename\f[R] (or \f[C]remote:\f[R]
+for the \f[C]lsd\f[R] command.) You may put subdirectories in too, e.g.
+\f[C]remote:item/path/to/dir\f[R].
+.SS Notes
+.PP
+The first path segment must be the name of the share, which you entered
+when you started to share on Windows.
+On smbd, it\[aq]s the section title in \f[C]smb.conf\f[R] (usually in
+\f[C]/etc/samba/\f[R]) file.
+You can find shares by quering the root if you\[aq]re unsure (e.g.
+\f[C]rclone lsd remote:\f[R]).
+.PP
+You can\[aq]t access to the shared printers from rclone, obviously.
+.PP
+You can\[aq]t use Anonymous access for logging in.
+You have to use the \f[C]guest\f[R] user with an empty password instead.
+The rclone client tries to avoid 8.3 names when uploading files by
+encoding trailing spaces and periods.
+Alternatively, the local
+backend (https://rclone.org/local/#paths-on-windows) on Windows can
+access SMB servers using UNC paths, by
+\f[C]\[rs]\[rs]server\[rs]share\f[R].
+This doesn\[aq]t apply to non-Windows OSes, such as Linux and macOS.
+.SS Configuration
+.PP
+Here is an example of making a SMB configuration.
+.PP
+First run
+.IP
+.nf
+\f[C]
+rclone config
+\f[R]
+.fi
+.PP
+This will guide you through an interactive setup process.
+.IP
+.nf
+\f[C]
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+XX / SMB / CIFS
+   \[rs] (smb)
+Storage> smb
+
+Option host.
+Samba hostname to connect to.
+E.g. \[dq]example.com\[dq].
+Enter a value.
+host> localhost
+
+Option user.
+Samba username.
+Enter a string value. Press Enter for the default (lesmi).
+user> guest
+
+Option port.
+Samba port number.
+Enter a signed integer. Press Enter for the default (445).
+port> 
+
+Option pass.
+Samba password.
+Choose an alternative below. Press Enter for the default (n).
+y) Yes, type in my own password
+g) Generate random password
+n) No, leave this optional password blank (default)
+y/g/n> g
+Password strength in bits.
+64 is just about memorable
+128 is secure
+1024 is the maximum
+Bits> 64
+Your password is: XXXX
+Use this password? Please note that an obscured version of this 
+password (and not the password itself) will be stored under your 
+configuration file, so keep this generated password in a safe place.
+y) Yes (default)
+n) No
+y/n> y
+
+Option domain.
+Domain name for NTLM authentication.
+Enter a string value. Press Enter for the default (WORKGROUP).
+domain> 
+
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n> n
+
+Configuration complete.
+Options:
+- type: samba
+- host: localhost
+- user: guest
+- pass: *** ENCRYPTED ***
+Keep this \[dq]remote\[dq] remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> d
+\f[R]
+.fi
+.SS Standard options
+.PP
+Here are the Standard options specific to smb (SMB / CIFS).
+.SS --smb-host
+.PP
+SMB server hostname to connect to.
+.PP
+E.g.
+\[dq]example.com\[dq].
+.PP
+Properties:
+.IP \[bu] 2
+Config: host
+.IP \[bu] 2
+Env Var: RCLONE_SMB_HOST
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: true
+.SS --smb-user
+.PP
+SMB username.
+.PP
+Properties:
+.IP \[bu] 2
+Config: user
+.IP \[bu] 2
+Env Var: RCLONE_SMB_USER
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Default: \[dq]$USER\[dq]
+.SS --smb-port
+.PP
+SMB port number.
+.PP
+Properties:
+.IP \[bu] 2
+Config: port
+.IP \[bu] 2
+Env Var: RCLONE_SMB_PORT
+.IP \[bu] 2
+Type: int
+.IP \[bu] 2
+Default: 445
+.SS --smb-pass
+.PP
+SMB password.
+.PP
+\f[B]NB\f[R] Input to this must be obscured - see rclone
+obscure (https://rclone.org/commands/rclone_obscure/).
+.PP
+Properties:
+.IP \[bu] 2
+Config: pass
+.IP \[bu] 2
+Env Var: RCLONE_SMB_PASS
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Required: false
+.SS --smb-domain
+.PP
+Domain name for NTLM authentication.
+.PP
+Properties:
+.IP \[bu] 2
+Config: domain
+.IP \[bu] 2
+Env Var: RCLONE_SMB_DOMAIN
+.IP \[bu] 2
+Type: string
+.IP \[bu] 2
+Default: \[dq]WORKGROUP\[dq]
+.SS Advanced options
+.PP
+Here are the Advanced options specific to smb (SMB / CIFS).
+.SS --smb-idle-timeout
+.PP
+Max time before closing idle connections.
+.PP
+If no connections have been returned to the connection pool in the time
+given, rclone will empty the connection pool.
+.PP
+Set to 0 to keep connections indefinitely.
+.PP
+Properties:
+.IP \[bu] 2
+Config: idle_timeout
+.IP \[bu] 2
+Env Var: RCLONE_SMB_IDLE_TIMEOUT
+.IP \[bu] 2
+Type: Duration
+.IP \[bu] 2
+Default: 1m0s
+.SS --smb-hide-special-share
+.PP
+Hide special shares (e.g.
+print$) which users aren\[aq]t supposed to access.
+.PP
+Properties:
+.IP \[bu] 2
+Config: hide_special_share
+.IP \[bu] 2
+Env Var: RCLONE_SMB_HIDE_SPECIAL_SHARE
+.IP \[bu] 2
+Type: bool
+.IP \[bu] 2
+Default: true
+.SS --smb-case-insensitive
+.PP
+Whether the server is configured to be case-insensitive.
+.PP
+Always true on Windows shares.
+.PP
+Properties:
+.IP \[bu] 2
+Config: case_insensitive
+.IP \[bu] 2
+Env Var: RCLONE_SMB_CASE_INSENSITIVE
+.IP \[bu] 2
+Type: bool
+.IP \[bu] 2
+Default: true
+.SS --smb-encoding
+.PP
+The encoding for the backend.
+.PP
+See the encoding section in the
+overview (https://rclone.org/overview/#encoding) for more info.
+.PP
+Properties:
+.IP \[bu] 2
+Config: encoding
+.IP \[bu] 2
+Env Var: RCLONE_SMB_ENCODING
+.IP \[bu] 2
+Type: MultiEncoder
+.IP \[bu] 2
+Default:
+Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot
 .SH Storj
 .PP
 Storj (https://storj.io) is an encrypted, secure, and cost-effective
@@ -53965,6 +56024,354 @@ Options:
 .IP \[bu] 2
 \[dq]error\[dq]: return an error based on option value
 .SH Changelog
+.SS v1.60.0 - 2022-10-21
+.PP
+See commits (https://github.com/rclone/rclone/compare/v1.59.0...v1.60.0)
+.IP \[bu] 2
+New backends
+.RS 2
+.IP \[bu] 2
+Oracle object storage (https://rclone.org/oracleobjectstorage/) (Manoj
+Ghosh)
+.IP \[bu] 2
+SMB (https://rclone.org/smb/) / CIFS (Windows file sharing) (Lesmiscore)
+.IP \[bu] 2
+New S3 providers
+.RS 2
+.IP \[bu] 2
+IONOS Cloud Storage (https://rclone.org/s3/#ionos) (Dmitry Deniskin)
+.IP \[bu] 2
+Qiniu KODO (https://rclone.org/s3/#qiniu) (Bachue Zhou)
+.RE
+.RE
+.IP \[bu] 2
+New Features
+.RS 2
+.IP \[bu] 2
+build
+.RS 2
+.IP \[bu] 2
+Update to go1.19 and make go1.17 the minimum required version (Nick
+Craig-Wood)
+.IP \[bu] 2
+Install.sh: fix arm-v7 download (Ole Frost)
+.RE
+.IP \[bu] 2
+fs: Warn the user when using an existing remote name without a colon
+(Nick Craig-Wood)
+.IP \[bu] 2
+httplib: Add \f[C]--xxx-min-tls-version\f[R] option to select minimum
+TLS version for HTTP servers (Robert Newson)
+.IP \[bu] 2
+librclone: Add PHP bindings and test program (Jordi Gonzalez Mu\[~n]oz)
+.IP \[bu] 2
+operations
+.RS 2
+.IP \[bu] 2
+Add \f[C]--server-side-across-configs\f[R] global flag for any backend
+(Nick Craig-Wood)
+.IP \[bu] 2
+Optimise \f[C]--copy-dest\f[R] and \f[C]--compare-dest\f[R] (Nick
+Craig-Wood)
+.RE
+.IP \[bu] 2
+rc: add \f[C]job/stopgroup\f[R] to stop group (Evan Spensley)
+.IP \[bu] 2
+serve dlna
+.RS 2
+.IP \[bu] 2
+Add \f[C]--announce-interval\f[R] to control SSDP Announce Interval
+(YanceyChiew)
+.IP \[bu] 2
+Add \f[C]--interface\f[R] to Specify SSDP interface names line (Simon
+Bos)
+.IP \[bu] 2
+Add support for more external subtitles (YanceyChiew)
+.IP \[bu] 2
+Add verification of addresses (YanceyChiew)
+.RE
+.IP \[bu] 2
+sync: Optimise \f[C]--copy-dest\f[R] and \f[C]--compare-dest\f[R] (Nick
+Craig-Wood)
+.IP \[bu] 2
+doc updates (albertony, Alexander Knorr, anonion, Jo\[~a]o Henrique
+Franco, Josh Soref, Lorenzo Milesi, Marco Molteni, Mark Trolley, Ole
+Frost, partev, Ryan Morey, Tom Mombourquette, YFdyh000)
+.RE
+.IP \[bu] 2
+Bug Fixes
+.RS 2
+.IP \[bu] 2
+filter
+.RS 2
+.IP \[bu] 2
+Fix incorrect filtering with \f[C]UseFilter\f[R] context flag and
+wrapping backends (Nick Craig-Wood)
+.IP \[bu] 2
+Make sure we check \f[C]--files-from\f[R] when looking for a single file
+(Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+rc
+.RS 2
+.IP \[bu] 2
+Fix \f[C]mount/listmounts\f[R] not returning the full Fs entered in
+\f[C]mount/mount\f[R] (Tom Mombourquette)
+.IP \[bu] 2
+Handle external unmount when mounting (Isaac Aymerich)
+.IP \[bu] 2
+Validate Daemon option is not set when mounting a volume via RC (Isaac
+Aymerich)
+.RE
+.IP \[bu] 2
+sync: Update docs and error messages to reflect fixes to overlap checks
+(Nick Naumann)
+.RE
+.IP \[bu] 2
+VFS
+.RS 2
+.IP \[bu] 2
+Reduce memory use by embedding \f[C]sync.Cond\f[R] (Nick Craig-Wood)
+.IP \[bu] 2
+Reduce memory usage by re-ordering commonly used structures (Nick
+Craig-Wood)
+.IP \[bu] 2
+Fix excess CPU used by VFS cache cleaner looping (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+Local
+.RS 2
+.IP \[bu] 2
+Obey file filters in listing to fix errors on excluded files (Nick
+Craig-Wood)
+.IP \[bu] 2
+Fix \[dq]Failed to read metadata: function not implemented\[dq] on old
+Linux kernels (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+Compress
+.RS 2
+.IP \[bu] 2
+Fix crash due to nil metadata (Nick Craig-Wood)
+.IP \[bu] 2
+Fix error handling to not use or return nil objects (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+Drive
+.RS 2
+.IP \[bu] 2
+Make \f[C]--drive-stop-on-upload-limit\f[R] obey quota exceeded error
+(Steve Kowalik)
+.RE
+.IP \[bu] 2
+FTP
+.RS 2
+.IP \[bu] 2
+Add \f[C]--ftp-force-list-hidden\f[R] option to show hidden items
+(\[/O]yvind Heddeland Instefjord)
+.IP \[bu] 2
+Fix hang when using ExplicitTLS to certain servers.
+(Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+Google Cloud Storage
+.RS 2
+.IP \[bu] 2
+Add \f[C]--gcs-endpoint\f[R] flag and config parameter (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+Hubic
+.RS 2
+.IP \[bu] 2
+Remove backend as service has now shut down (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+Onedrive
+.RS 2
+.IP \[bu] 2
+Rename Onedrive(cn) 21Vianet to Vnet Group (Yen Hu)
+.IP \[bu] 2
+Disable change notify in China region since it is not supported (Nick
+Craig-Wood)
+.RE
+.IP \[bu] 2
+S3
+.RS 2
+.IP \[bu] 2
+Implement \f[C]--s3-versions\f[R] flag to show old versions of objects
+if enabled (Nick Craig-Wood)
+.IP \[bu] 2
+Implement \f[C]--s3-version-at\f[R] flag to show versions of objects at
+a particular time (Nick Craig-Wood)
+.IP \[bu] 2
+Implement \f[C]backend versioning\f[R] command to get/set bucket
+versioning (Nick Craig-Wood)
+.IP \[bu] 2
+Implement \f[C]Purge\f[R] to purge versions and
+\f[C]backend cleanup-hidden\f[R] (Nick Craig-Wood)
+.IP \[bu] 2
+Add \f[C]--s3-decompress\f[R] flag to decompress gzip-encoded files
+(Nick Craig-Wood)
+.IP \[bu] 2
+Add \f[C]--s3-sse-customer-key-base64\f[R] to supply keys with binary
+data (Richard Bateman)
+.IP \[bu] 2
+Try to keep the maximum precision in ModTime with
+\f[C]--user-server-modtime\f[R] (Nick Craig-Wood)
+.IP \[bu] 2
+Drop binary metadata with an ERROR message as it can\[aq]t be stored
+(Nick Craig-Wood)
+.IP \[bu] 2
+Add \f[C]--s3-no-system-metadata\f[R] to suppress read and write of
+system metadata (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+SFTP
+.RS 2
+.IP \[bu] 2
+Fix directory creation races (Lesmiscore)
+.RE
+.IP \[bu] 2
+Swift
+.RS 2
+.IP \[bu] 2
+Add \f[C]--swift-no-large-objects\f[R] to reduce HEAD requests (Nick
+Craig-Wood)
+.RE
+.IP \[bu] 2
+Union
+.RS 2
+.IP \[bu] 2
+Propagate SlowHash feature to fix hasher interaction (Lesmiscore)
+.RE
+.SS v1.59.2 - 2022-09-15
+.PP
+See commits (https://github.com/rclone/rclone/compare/v1.59.1...v1.59.2)
+.IP \[bu] 2
+Bug Fixes
+.RS 2
+.IP \[bu] 2
+config: Move locking to fix fatal error: concurrent map read and map
+write (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+Local
+.RS 2
+.IP \[bu] 2
+Disable xattr support if the filesystems indicates it is not supported
+(Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+Azure Blob
+.RS 2
+.IP \[bu] 2
+Fix chunksize calculations producing too many parts (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+B2
+.RS 2
+.IP \[bu] 2
+Fix chunksize calculations producing too many parts (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+S3
+.RS 2
+.IP \[bu] 2
+Fix chunksize calculations producing too many parts (Nick Craig-Wood)
+.RE
+.SS v1.59.1 - 2022-08-08
+.PP
+See commits (https://github.com/rclone/rclone/compare/v1.59.0...v1.59.1)
+.IP \[bu] 2
+Bug Fixes
+.RS 2
+.IP \[bu] 2
+accounting: Fix panic in core/stats-reset with unknown group (Nick
+Craig-Wood)
+.IP \[bu] 2
+build: Fix android build after GitHub actions change (Nick Craig-Wood)
+.IP \[bu] 2
+dlna: Fix SOAP action header parsing (Joram Schrijver)
+.IP \[bu] 2
+docs: Fix links to mount command from install docs (albertony)
+.IP \[bu] 2
+dropbox: Fix ChangeNotify was unable to decrypt errors (Nick Craig-Wood)
+.IP \[bu] 2
+fs: Fix parsing of times and durations of the form \[dq]YYYY-MM-DD
+HH:MM:SS\[dq] (Nick Craig-Wood)
+.IP \[bu] 2
+serve sftp: Fix checksum detection (Nick Craig-Wood)
+.IP \[bu] 2
+sync: Add accidentally missed filter-sensitivity to --backup-dir option
+(Nick Naumann)
+.RE
+.IP \[bu] 2
+Combine
+.RS 2
+.IP \[bu] 2
+Fix docs showing \f[C]remote=\f[R] instead of \f[C]upstreams=\f[R] (Nick
+Craig-Wood)
+.IP \[bu] 2
+Throw error if duplicate directory name is specified (Nick Craig-Wood)
+.IP \[bu] 2
+Fix errors with backends shutting down while in use (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+Dropbox
+.RS 2
+.IP \[bu] 2
+Fix hang on quit with --dropbox-batch-mode off (Nick Craig-Wood)
+.IP \[bu] 2
+Fix infinite loop on uploading a corrupted file (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+Internetarchive
+.RS 2
+.IP \[bu] 2
+Ignore checksums for files using the different method (Lesmiscore)
+.IP \[bu] 2
+Handle hash symbol in the middle of filename (Lesmiscore)
+.RE
+.IP \[bu] 2
+Jottacloud
+.RS 2
+.IP \[bu] 2
+Fix working with whitelabel Elgiganten Cloud
+.IP \[bu] 2
+Do not store username in config when using standard auth (albertony)
+.RE
+.IP \[bu] 2
+Mega
+.RS 2
+.IP \[bu] 2
+Fix nil pointer exception when bad node received (Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+S3
+.RS 2
+.IP \[bu] 2
+Fix --s3-no-head panic: reflect: Elem of invalid type s3.PutObjectInput
+(Nick Craig-Wood)
+.RE
+.IP \[bu] 2
+SFTP
+.RS 2
+.IP \[bu] 2
+Fix issue with WS_FTP by working around failing RealPath (albertony)
+.RE
+.IP \[bu] 2
+Union
+.RS 2
+.IP \[bu] 2
+Fix duplicated files when using directories with leading / (Nick
+Craig-Wood)
+.IP \[bu] 2
+Fix multiple files being uploaded when roots don\[aq]t exist (Nick
+Craig-Wood)
+.IP \[bu] 2
+Fix panic due to misalignment of struct field in 32 bit architectures
+(r-ricci)
+.RE
 .SS v1.59.0 - 2022-07-09
 .PP
 See commits (https://github.com/rclone/rclone/compare/v1.58.0...v1.59.0)
@@ -54610,8 +57017,8 @@ Hard fork \f[C]github.com/jlaffaye/ftp\f[R] to fix
 \f[C]go get github.com/rclone/rclone\f[R] (Nick Craig-Wood)
 .RE
 .IP \[bu] 2
-oauthutil: Fix crash when webrowser requests \f[C]/robots.txt\f[R] (Nick
-Craig-Wood)
+oauthutil: Fix crash when webbrowser requests \f[C]/robots.txt\f[R]
+(Nick Craig-Wood)
 .IP \[bu] 2
 operations: Fix goroutine leak in case of copy retry (Ankur Gupta)
 .IP \[bu] 2
@@ -54874,7 +57281,7 @@ Craig-Wood)
 Fix timeout on hashing large files by sending keepalives (Nick
 Craig-Wood)
 .IP \[bu] 2
-Fix unecessary seeking when uploading and downloading files (Nick
+Fix unnecessary seeking when uploading and downloading files (Nick
 Craig-Wood)
 .IP \[bu] 2
 Update docs on how to create \f[C]known_hosts\f[R] file (Nick
@@ -56486,11 +58893,11 @@ Pl\['a]nsk\['y])
 .IP \[bu] 2
 Add empty folder flag into ncdu browser (Adam Pl\['a]nsk\['y])
 .IP \[bu] 2
-Add \f[C]!\f[R] (errror) and \f[C].\f[R] (unreadable) file flags to go
+Add \f[C]!\f[R] (error) and \f[C].\f[R] (unreadable) file flags to go
 with \f[C]e\f[R] (empty) (Nick Craig-Wood)
 .RE
 .IP \[bu] 2
-obscure: Make \f[C]rclone osbcure -\f[R] ignore newline at end of line
+obscure: Make \f[C]rclone obscure -\f[R] ignore newline at end of line
 (Nick Craig-Wood)
 .IP \[bu] 2
 operations
@@ -56563,7 +58970,7 @@ move: Fix data loss when source and destination are the same object
 operations
 .RS 2
 .IP \[bu] 2
-Fix \f[C]--cutof-mode\f[R] hard not cutting off immediately (Nick
+Fix \f[C]--cutoff-mode\f[R] hard not cutting off immediately (Nick
 Craig-Wood)
 .IP \[bu] 2
 Fix \f[C]--immutable\f[R] error message (Nick Craig-Wood)
@@ -56698,7 +59105,7 @@ Box
 .IP \[bu] 2
 Fix NewObject for files that differ in case (Nick Craig-Wood)
 .IP \[bu] 2
-Fix finding directories in a case insentive way (Nick Craig-Wood)
+Fix finding directories in a case insensitive way (Nick Craig-Wood)
 .RE
 .IP \[bu] 2
 Chunker
@@ -56920,7 +59327,7 @@ Sugarsync
 .IP \[bu] 2
 Fix NewObject for files that differ in case (Nick Craig-Wood)
 .IP \[bu] 2
-Fix finding directories in a case insentive way (Nick Craig-Wood)
+Fix finding directories in a case insensitive way (Nick Craig-Wood)
 .RE
 .IP \[bu] 2
 Swift
@@ -57102,7 +59509,7 @@ See commits (https://github.com/rclone/rclone/compare/v1.53.1...v1.53.2)
 Bug Fixes
 .RS 2
 .IP \[bu] 2
-acounting
+accounting
 .RS 2
 .IP \[bu] 2
 Fix incorrect speed and transferTime in core/stats (Nick Craig-Wood)
@@ -59710,7 +62117,7 @@ rcd: Fix permissions problems on cache directory with web gui download
 Mount
 .RS 2
 .IP \[bu] 2
-Default \f[C]--daemon-timout\f[R] to 15 minutes on macOS and FreeBSD
+Default \f[C]--daemon-timeout\f[R] to 15 minutes on macOS and FreeBSD
 (Nick Craig-Wood)
 .IP \[bu] 2
 Update docs to show mounting from root OK for bucket-based (Nick
@@ -60583,7 +62990,7 @@ HTTP
 Add an example with username and password which is supported but
 wasn\[aq]t documented (Nick Craig-Wood)
 .IP \[bu] 2
-Fix backend with \f[C]--files-from\f[R] and non-existent files (Nick
+Fix backend with \f[C]--files-from\f[R] and nonexistent files (Nick
 Craig-Wood)
 .RE
 .IP \[bu] 2
@@ -61860,7 +64267,7 @@ Work around strange response from box FTP server
 .IP \[bu] 2
 More workarounds for FTP servers to fix mkParentDir error
 .IP \[bu] 2
-Fix no error on listing non-existent directory
+Fix no error on listing nonexistent directory
 .RE
 .IP \[bu] 2
 Google Cloud Storage
@@ -62057,7 +64464,7 @@ requests)
 Bug Fixes
 .RS 2
 .IP \[bu] 2
-config: fixes errors on non existing config by loading config file only
+config: fixes errors on nonexistent config by loading config file only
 on first access
 .IP \[bu] 2
 config: retry saving the config after failure (Mateusz)
@@ -63440,7 +65847,7 @@ S3
 Command line and config file support for
 .RS 2
 .IP \[bu] 2
-Setting/overriding ACL - thanks Radek Senfeld
+Setting/overriding ACL - thanks Radek \[vS]enfeld
 .IP \[bu] 2
 Setting storage class - thanks Asko Tamm
 .RE
@@ -66077,6 +68484,56 @@ Lorenzo Maiorfi 
 Claudio Maradonna 
 .IP \[bu] 2
 Ovidiu Victor Tatar 
+.IP \[bu] 2
+Evan Spensley 
+.IP \[bu] 2
+Yen Hu <61753151+0x59656e@users.noreply.github.com>
+.IP \[bu] 2
+Steve Kowalik 
+.IP \[bu] 2
+Jordi Gonzalez Mu\[~n]oz 
+.IP \[bu] 2
+Joram Schrijver 
+.IP \[bu] 2
+Mark Trolley 
+.IP \[bu] 2
+Jo\[~a]o Henrique Franco 
+.IP \[bu] 2
+anonion 
+.IP \[bu] 2
+Ryan Morey <4590343+rmorey@users.noreply.github.com>
+.IP \[bu] 2
+Simon Bos 
+.IP \[bu] 2
+YFdyh000  * Josh Soref
+<2119212+jsoref@users.noreply.github.com>
+.IP \[bu] 2
+\[/O]yvind Heddeland Instefjord 
+.IP \[bu] 2
+Dmitry Deniskin <110819396+ddeniskin@users.noreply.github.com>
+.IP \[bu] 2
+Alexander Knorr <106825+opexxx@users.noreply.github.com>
+.IP \[bu] 2
+Richard Bateman 
+.IP \[bu] 2
+Dimitri Papadopoulos Orfanos
+<3234522+DimitriPapadopoulos@users.noreply.github.com>
+.IP \[bu] 2
+Lorenzo Milesi 
+.IP \[bu] 2
+Isaac Aymerich 
+.IP \[bu] 2
+YanceyChiew <35898533+YanceyChiew@users.noreply.github.com>
+.IP \[bu] 2
+Manoj Ghosh 
+.IP \[bu] 2
+Bachue Zhou 
+.IP \[bu] 2
+Manoj Ghosh 
+.IP \[bu] 2
+Tom Mombourquette 
+.IP \[bu] 2
+Robert Newson 
 .SH Contact the rclone project
 .SS Forum
 .PP

From db5d58240433b9546d1bd076be40f876768fde24 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Fri, 21 Oct 2022 16:15:53 +0100
Subject: [PATCH 328/560] Start v1.61.0-DEV development

---
 VERSION                            | 2 +-
 docs/layouts/partials/version.html | 2 +-
 fs/versiontag.go                   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/VERSION b/VERSION
index 9dd0095850226..137ade0bc6194 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-v1.60.0
+v1.61.0
diff --git a/docs/layouts/partials/version.html b/docs/layouts/partials/version.html
index 514c727161d3e..d9a35e5f7da0c 100644
--- a/docs/layouts/partials/version.html
+++ b/docs/layouts/partials/version.html
@@ -1 +1 @@
-v1.60.0
\ No newline at end of file
+v1.61.0
\ No newline at end of file
diff --git a/fs/versiontag.go b/fs/versiontag.go
index a9bc5f0e7ad82..15da27a842b47 100644
--- a/fs/versiontag.go
+++ b/fs/versiontag.go
@@ -1,4 +1,4 @@
 package fs
 
 // VersionTag of rclone
-var VersionTag = "v1.60.0"
+var VersionTag = "v1.61.0"

From 8114744bce62af2db93674a49a574b25fb5f2401 Mon Sep 17 00:00:00 2001
From: Samuel Johnson 
Date: Sun, 23 Oct 2022 15:45:09 +0530
Subject: [PATCH 329/560] docs: Update faq.md with bisync

Updated FAQ to clarify that experimental bi-sync is now available.
---
 docs/content/faq.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/content/faq.md b/docs/content/faq.md
index 00ed0e06d3950..a0a99c9302361 100644
--- a/docs/content/faq.md
+++ b/docs/content/faq.md
@@ -82,9 +82,8 @@ of metadata, which breaks the desired 1:1 mapping of files to objects.
 
 ### Can rclone do bi-directional sync? ###
 
-No, not at present.  rclone only does uni-directional sync from A ->
-B. It may do in the future though since it has all the primitives - it
-just requires writing the algorithm to do it.
+Yes, since rclone v1.58.0, [bidirectional cloud sync](/bisync/) is
+available.
 
 ### Can I use rclone with an HTTP proxy? ###
 

From 76eeca9eaeb3195da506652ae497662c26ed921b Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Sun, 23 Oct 2022 15:37:02 +0200
Subject: [PATCH 330/560] build: setup-go v3 dropped the stable input

---
 .github/workflows/build.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 79ffc62a6de99..2b349a3a90344 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -104,7 +104,6 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v3
         with:
-          stable: 'false'
           go-version: ${{ matrix.go }}
           check-latest: true
 

From c0a8ffcbef0f056f0f747d7c6afe3563136734df Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Sun, 23 Oct 2022 15:41:59 +0200
Subject: [PATCH 331/560] build: setup-go v3 improved semver notation

---
 .github/workflows/build.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2b349a3a90344..76dcafdb51f56 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -30,7 +30,7 @@ jobs:
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.19.x'
+            go: '1.19'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -41,14 +41,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '1.19.x'
+            go: '1.19'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-11
-            go: '1.19.x'
+            go: '1.19'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -57,14 +57,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-11
-            go: '1.19.x'
+            go: '1.19'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '1.19.x'
+            go: '1.19'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -74,20 +74,20 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.19.x'
+            go: '1.19'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
           - job_name: go1.17
             os: ubuntu-latest
-            go: '1.17.x'
+            go: '1.17'
             quicktest: true
             racequicktest: true
 
           - job_name: go1.18
             os: ubuntu-latest
-            go: '1.18.x'
+            go: '1.18'
             quicktest: true
             racequicktest: true
 
@@ -249,7 +249,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.x
+          go-version: 1.19
 
       - name: Go module cache
         uses: actions/cache@v3

From f4a571786ceed95867df1f1b7bdea033804f0681 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 21 Oct 2022 14:20:24 +0200
Subject: [PATCH 332/560] local: clean absolute paths - fixes #6493

---
 backend/local/local.go | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/backend/local/local.go b/backend/local/local.go
index 7d56d8012f0e5..654bb02d1b973 100644
--- a/backend/local/local.go
+++ b/backend/local/local.go
@@ -1400,30 +1400,27 @@ func (o *Object) writeMetadata(metadata fs.Metadata) (err error) {
 }
 
 func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
-	if runtime.GOOS == "windows" {
-		if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") {
+	if runtime.GOOS != "windows" || !strings.HasPrefix(s, "\\") {
+		if !filepath.IsAbs(s) {
 			s2, err := filepath.Abs(s)
 			if err == nil {
 				s = s2
 			}
+		} else {
+			s = filepath.Clean(s)
 		}
+	}
+	if runtime.GOOS == "windows" {
 		s = filepath.ToSlash(s)
 		vol := filepath.VolumeName(s)
 		s = vol + enc.FromStandardPath(s[len(vol):])
 		s = filepath.FromSlash(s)
-
 		if !noUNC {
 			// Convert to UNC
 			s = file.UNCPath(s)
 		}
 		return s
 	}
-	if !filepath.IsAbs(s) {
-		s2, err := filepath.Abs(s)
-		if err == nil {
-			s = s2
-		}
-	}
 	s = enc.FromStandardPath(s)
 	return s
 }

From 178cf821deb462faff6e5975bde95f4e77815cc4 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Sun, 23 Oct 2022 14:27:56 +0200
Subject: [PATCH 333/560] build: add vulnerability testing using govulncheck

---
 .github/workflows/build.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 76dcafdb51f56..29bdde22be5b6 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -233,6 +233,19 @@ jobs:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           version: latest
 
+      # Run govulncheck on the latest go version, the one we build binaries with
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+          check-latest: true
+
+      - name: Install govulncheck
+        run: go install golang.org/x/vuln/cmd/govulncheck@latest
+
+      - name: Scan for vulnerabilities
+        run: govulncheck ./...
+
   android:
     if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
     timeout-minutes: 30

From 22abcc9fd222c32cfbf3baae3f166689d22cbd60 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Tue, 25 Oct 2022 20:55:21 +0200
Subject: [PATCH 334/560] build: update golang.org/x/net dependency

This fixes vulnerability GO-2022-0969 reported by govulncheck:

HTTP/2 server connections can hang forever waiting for a clean
shutdown that was preempted by a fatal error. This condition can
be exploited by a malicious client to cause a denial of service.

Call stacks in your code:
Error: cmd/serve/restic/restic.go:150:22: github.com/rclone/rclone/cmd/serve/restic.init$1$1 calls golang.org/x/net/http2.Server.ServeConn

Found in: golang.org/x/net/http2@v0.0.0-20220805013720-a33c5aa5df48
Fixed in: golang.org/x/net/http2@v0.0.0-20220906165146-f3363e06e74c
More info: https://pkg.go.dev/vuln/GO-2022-0969
---
 go.mod | 8 ++++----
 go.sum | 8 ++++++++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/go.mod b/go.mod
index b7aecb2e7137d..5f073eff3cdd6 100644
--- a/go.mod
+++ b/go.mod
@@ -62,11 +62,11 @@ require (
 	go.etcd.io/bbolt v1.3.6
 	goftp.io/server v0.4.1
 	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
-	golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48
+	golang.org/x/net v0.1.0
 	golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c
 	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
-	golang.org/x/sys v0.0.0-20220804214406-8e32c043e418
-	golang.org/x/text v0.3.7
+	golang.org/x/sys v0.1.0
+	golang.org/x/text v0.4.0
 	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
 	google.golang.org/api v0.91.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -143,5 +143,5 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/pkg/xattr v0.4.7
 	golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105
-	golang.org/x/term v0.0.0-20220722155259-a9ba230a4035
+	golang.org/x/term v0.1.0
 )
diff --git a/go.sum b/go.sum
index be4dc9ecf2086..141df3fac81f7 100644
--- a/go.sum
+++ b/go.sum
@@ -796,6 +796,8 @@ golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM=
 golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 h1:N9Vc/rorQUDes6B9CNdIxAn5jODGj2wzfrei2x4wNj4=
 golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -929,11 +931,15 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220804214406-8e32c043e418 h1:9vYwv7OjYaky/tlAeD7C4oC9EsPTlaFl1H2jS++V+ME=
 golang.org/x/sys v0.0.0-20220804214406-8e32c043e418/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc=
 golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -944,6 +950,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

From 1fc864fb3217f06c90d04e207231fb1b63c89b28 Mon Sep 17 00:00:00 2001
From: Manoj Ghosh 
Date: Fri, 28 Oct 2022 11:32:17 -0700
Subject: [PATCH 335/560] oracle-object-storage: doc fix

See #6521
---
 backend/oracleobjectstorage/oracleobjectstorage.go | 1 +
 docs/content/overview.md                           | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.go
index d18a3467ac4b7..6f730f2587e1d 100644
--- a/backend/oracleobjectstorage/oracleobjectstorage.go
+++ b/backend/oracleobjectstorage/oracleobjectstorage.go
@@ -686,6 +686,7 @@ var (
 	_ fs.PutStreamer = &Fs{}
 	_ fs.ListRer     = &Fs{}
 	_ fs.Commander   = &Fs{}
+	_ fs.CleanUpper  = &Fs{}
 
 	_ fs.Object    = &Object{}
 	_ fs.MimeTyper = &Object{}
diff --git a/docs/content/overview.md b/docs/content/overview.md
index dbb15445cfb79..790dbf7fd9235 100644
--- a/docs/content/overview.md
+++ b/docs/content/overview.md
@@ -494,7 +494,7 @@ upon backend-specific capabilities.
 | Microsoft OneDrive           | Yes   | Yes  | Yes  | Yes     | Yes     | No    | No           | Yes          | Yes   | Yes      |
 | OpenDrive                    | Yes   | Yes  | Yes  | Yes     | No      | No    | No           | No           | No    | Yes      |
 | OpenStack Swift              | Yes † | Yes  | No   | No      | No      | Yes   | Yes          | No           | Yes   | No       |
-| Oracle Object Storage        | Yes   | Yes  | No   | No      | Yes     | Yes   | No           | No           | No    | No       |
+| Oracle Object Storage        | No    | Yes  | No   | No      | Yes     | Yes   | Yes          | No           | No    | No       |
 | pCloud                       | Yes   | Yes  | Yes  | Yes     | Yes     | No    | No           | Yes          | Yes   | Yes      |
 | premiumize.me                | Yes   | No   | Yes  | Yes     | No      | No    | No           | Yes          | Yes   | Yes      |
 | put.io                       | Yes   | No   | Yes  | Yes     | Yes     | No    | Yes          | No           | Yes   | Yes      |

From 65987f5970892f7fc8bc353f87fe8b7363e0b06d Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 28 Oct 2022 21:00:22 +0200
Subject: [PATCH 336/560] lib/file: improve error message for create dir on
 non-existent network host on windows (#6420)

---
 lib/file/mkdir_windows.go      | 16 +++++++++-------
 lib/file/mkdir_windows_test.go | 18 ++++++++++++++++++
 2 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/lib/file/mkdir_windows.go b/lib/file/mkdir_windows.go
index 36ccadd9c9ceb..73a82ac62dd8d 100644
--- a/lib/file/mkdir_windows.go
+++ b/lib/file/mkdir_windows.go
@@ -11,8 +11,9 @@ import (
 
 // MkdirAll creates a directory named path, along with any necessary parents.
 //
-// Improves os.MkdirAll by avoiding trying to create a folder \\? when the
-// volume of a given extended length path does not exist.
+// Improves os.MkdirAll by avoiding trying to create a folder `\\?` when the
+// volume of a given extended length path does not exist, and `\\?\UNC` when
+// a network host name does not exist.
 //
 // Based on source code from golang's os.MkdirAll
 // (https://github.com/golang/go/blob/master/src/os/path.go)
@@ -37,7 +38,6 @@ func MkdirAll(path string, perm os.FileMode) error {
 	}
 	if i > 0 {
 		path = path[:i]
-
 		if path == filepath.VolumeName(path) {
 			// Make reference to a drive's root directory include the trailing slash.
 			// In extended-length form without trailing slash ("\\?\C:"), os.Stat
@@ -53,10 +53,12 @@ func MkdirAll(path string, perm os.FileMode) error {
 				j--
 			}
 			if j > 1 {
-				// Create parent.
-				err = MkdirAll(path[:j-1], perm)
-				if err != nil {
-					return err
+				if path[:j-1] != `\\?\UNC` {
+					// Create parent.
+					err = MkdirAll(path[:j-1], perm)
+					if err != nil {
+						return err
+					}
 				}
 			}
 		}
diff --git a/lib/file/mkdir_windows_test.go b/lib/file/mkdir_windows_test.go
index e6daf1d4907dc..6b574b7921a10 100644
--- a/lib/file/mkdir_windows_test.go
+++ b/lib/file/mkdir_windows_test.go
@@ -128,3 +128,21 @@ func TestMkdirAllOnUnusedDrive(t *testing.T) {
 	errormsg = fmt.Sprintf("mkdir \\\\?\\%s\\: The system cannot find the path specified.", path)
 	checkMkdirAllSubdirs(t, `\\?\`+path, false, errormsg)
 }
+
+// Testing paths on unknown network host
+// This is an additional difference from golang's os.MkdirAll. With our
+// first fix, stopping it from recursing extended-length paths down to
+// the "\\?" prefix, it would now stop at `\\?\UNC`, because that is what
+// filepath.VolumeName returns (which is wrong, that is not a volume name!),
+// and still return a nonifnromative error:
+// "mkdir \\?\UNC\\: The filename, directory name, or volume label syntax is incorrect."
+// Our version stops the recursion at level before this, and reports:
+// "mkdir \\?\UNC\0.0.0.0: The specified path is invalid."
+func TestMkdirAllOnUnusedNetworkHost(t *testing.T) {
+	path := `\\0.0.0.0\share`
+	errormsg := fmt.Sprintf("mkdir %s\\: The format of the specified network name is invalid.", path)
+	checkMkdirAllSubdirs(t, path, false, errormsg)
+	path = `\\?\UNC\0.0.0.0\share`
+	errormsg = `mkdir \\?\UNC\0.0.0.0: The specified path is invalid.`
+	checkMkdirAllSubdirs(t, path, false, errormsg)
+}

From 53dd174f3d70cff00028c885b2afe13353dac475 Mon Sep 17 00:00:00 2001
From: coultonluke 
Date: Mon, 31 Oct 2022 20:09:53 +0000
Subject: [PATCH 337/560] docs: corrected download links in windows install
 docs

---
 docs/content/install.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/content/install.md b/docs/content/install.md
index 041010092a418..e04b85337b77b 100644
--- a/docs/content/install.md
+++ b/docs/content/install.md
@@ -125,9 +125,9 @@ The simplest fix is to run
 Fetch the correct binary for your processor type by clicking on these
 links. If not sure, use the first link.
 
-- [Intel/AMD - 64 Bit](https://downloads.rclone.org/rclone-current-linux-amd64.zip)
-- [Intel/AMD - 32 Bit](https://downloads.rclone.org/rclone-current-linux-386.zip)
-- [ARM - 64 Bit](https://downloads.rclone.org/rclone-current-linux-arm64.zip)
+- [Intel/AMD - 64 Bit](https://downloads.rclone.org/rclone-current-windows-amd64.zip)
+- [Intel/AMD - 32 Bit](https://downloads.rclone.org/rclone-current-windows-386.zip)
+- [ARM - 64 Bit](https://downloads.rclone.org/rclone-current-windows-arm64.zip)
 
 Open this file in the Explorer and extract `rclone.exe`. Rclone is a
 portable executable so you can place it wherever is convenient.

From 26176107416fe27fc22d61b4aff7aa8c9843bd72 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Mon, 31 Oct 2022 21:12:14 +0100
Subject: [PATCH 338/560] docs: add direct download link for windows arm64

---
 docs/content/downloads.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/content/downloads.md b/docs/content/downloads.md
index f365dc9e56751..378fbadedeed2 100644
--- a/docs/content/downloads.md
+++ b/docs/content/downloads.md
@@ -85,7 +85,7 @@ script) from a URL which doesn't change then you can use these links.
 | Intel/AMD - 32 Bit | {{< cdownload windows 386 >}} | - | {{< cdownload linux 386 >}} | {{< cdownload linux 386 deb >}} | {{< cdownload linux 386 rpm >}} | {{< cdownload freebsd 386 >}} | {{< cdownload netbsd 386 >}} | {{< cdownload openbsd 386 >}} | {{< cdownload plan9 386 >}} | - |
 | ARMv6 - 32 Bit | - | - | {{< cdownload linux arm >}} | {{< cdownload linux arm deb >}} | {{< cdownload linux arm rpm >}} | {{< cdownload freebsd arm >}} | {{< cdownload netbsd arm >}} | - | - | - |
 | ARMv7 - 32 Bit | - | - | {{< cdownload linux arm-v7 >}} | {{< cdownload linux arm-v7 deb >}} | {{< cdownload linux arm-v7 rpm >}} | {{< cdownload freebsd arm-v7 >}} | {{< cdownload netbsd arm-v7 >}} | - | - | - |
-| ARM - 64 Bit | - | {{< cdownload osx arm64 >}} | {{< cdownload linux arm64 >}} | {{< cdownload linux arm64 deb >}} | {{< cdownload linux arm64 rpm >}} | - | - | - | - | - |
+| ARM - 64 Bit | {{< cdownload windows arm64 >}} | {{< cdownload osx arm64 >}} | {{< cdownload linux arm64 >}} | {{< cdownload linux arm64 deb >}} | {{< cdownload linux arm64 rpm >}} | - | - | - | - | - |
 | MIPS - Big Endian | - | - | {{< cdownload linux mips >}} | {{< cdownload linux mips deb >}} | {{< cdownload linux mips rpm >}} | - | - | - | - | - |
 | MIPS - Little Endian | - | - | {{< cdownload linux mipsle >}} | {{< cdownload linux mipsle deb >}} | {{< cdownload linux mipsle rpm >}} | - | - | - | - | - |
 

From 10c884552caab3d3d8484ed3401e94f983a23f93 Mon Sep 17 00:00:00 2001
From: Anthony Pessy 
Date: Sat, 29 Oct 2022 09:37:08 +0200
Subject: [PATCH 339/560] s3: use different strategy to resolve s3 region

The API endpoint GetBucketLocation requires
top level permission.

If we do an authenticated head request to a bucket, the bucket location will be returned in the HTTP headers.

Fixes #5066
---
 backend/s3/s3.go | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index c0fc888fc007d..c13a78a37a212 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -26,6 +26,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/corehandlers"
@@ -2992,19 +2994,13 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Gets the bucket location
 func (f *Fs) getBucketLocation(ctx context.Context, bucket string) (string, error) {
-	req := s3.GetBucketLocationInput{
-		Bucket: &bucket,
-	}
-	var resp *s3.GetBucketLocationOutput
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.c.GetBucketLocation(&req)
-		return f.shouldRetry(ctx, err)
+	region, err := s3manager.GetBucketRegion(ctx, f.ses, bucket, "", func(r *request.Request) {
+		r.Config.S3ForcePathStyle = aws.Bool(f.opt.ForcePathStyle)
 	})
 	if err != nil {
 		return "", err
 	}
-	return s3.NormalizeBucketLocation(aws.StringValue(resp.LocationConstraint)), nil
+	return region, nil
 }
 
 // Updates the region for the bucket by reading the region from the

From 6aa8f7409ac7d851841728ae42361dfe3f07454f Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 3 Nov 2022 17:06:03 +0000
Subject: [PATCH 340/560] Add Samuel Johnson to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 012cd07b63927..5d546908f6e16 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -650,3 +650,4 @@ put them back in again.` >}}
   * Manoj Ghosh 
   * Tom Mombourquette 
   * Robert Newson 
+  * Samuel Johnson 

From df9be72a82de5b2cffa5abd21ba89251b6a6f283 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 3 Nov 2022 17:06:03 +0000
Subject: [PATCH 341/560] Add coultonluke to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 5d546908f6e16..f3ce6f9988d44 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -651,3 +651,4 @@ put them back in again.` >}}
   * Tom Mombourquette 
   * Robert Newson 
   * Samuel Johnson 
+  * coultonluke 

From 126d71b332bf6e95ba6f94b4f6c3c4ddfcfc114d Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 3 Nov 2022 17:06:03 +0000
Subject: [PATCH 342/560] Add Anthony Pessy to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index f3ce6f9988d44..7b41f5e4e0c39 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -652,3 +652,4 @@ put them back in again.` >}}
   * Robert Newson 
   * Samuel Johnson 
   * coultonluke 
+  * Anthony Pessy 

From 3292c112c5b81b3eb2a00378ad0a2318da5b8034 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 3 Nov 2022 17:06:03 +0000
Subject: [PATCH 343/560] Add Philip Harvey to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 7b41f5e4e0c39..84b42fd19c1c1 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -653,3 +653,4 @@ put them back in again.` >}}
   * Samuel Johnson 
   * coultonluke 
   * Anthony Pessy 
+  * Philip Harvey 

From c7c9356af5f9b3be84c9c9f220aa0181c176ccd7 Mon Sep 17 00:00:00 2001
From: Philip Harvey 
Date: Tue, 6 Sep 2022 15:39:48 -0600
Subject: [PATCH 344/560] s3: stop setting object and bucket ACL to "private"
 if it is an empty string #5730

---
 backend/s3/s3.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index c13a78a37a212..962640c1ccb60 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -2801,9 +2801,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.Versions && opt.VersionAt.IsSet() {
 		return nil, errors.New("s3: cant use --s3-versions and --s3-version-at at the same time")
 	}
-	if opt.ACL == "" {
-		opt.ACL = "private"
-	}
 	if opt.BucketACL == "" {
 		opt.BucketACL = opt.ACL
 	}

From 028832ce73099e387e81533f21007b95e9e99b33 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Wed, 14 Sep 2022 16:24:57 +0100
Subject: [PATCH 345/560] s3: if bucket or object ACL is empty string then
 don't add X-Amz-Acl: header - fixes #5730

Before this fix it was impossible to stop rclone generating an
X-Amx-Acl: header which is incompatible with GCS with uniform access
control and is generally deprecated at AWS.
---
 backend/s3/s3.go | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 962640c1ccb60..388adeb4be4db 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -1539,7 +1539,11 @@ This ACL is used for creating objects and if bucket_acl isn't set, for creating
 For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 Note that this ACL is applied when server-side copying objects as S3
-doesn't copy the ACL from the source but rather writes a fresh one.`,
+doesn't copy the ACL from the source but rather writes a fresh one.
+
+If the acl is an empty string then no X-Amz-Acl: header is added and
+the default (private) will be used.
+`,
 			Provider: "!Storj,Cloudflare",
 			Examples: []fs.OptionExample{{
 				Value:    "default",
@@ -1593,7 +1597,11 @@ doesn't copy the ACL from the source but rather writes a fresh one.`,
 For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 Note that this ACL is applied when only when creating buckets.  If it
-isn't set then "acl" is used instead.`,
+isn't set then "acl" is used instead.
+
+If the "acl" and "bucket_acl" are empty strings then no X-Amz-Acl:
+header is added and the default (private) will be used.
+`,
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "private",
@@ -2782,6 +2790,14 @@ func (f *Fs) setRoot(root string) {
 	f.rootBucket, f.rootDirectory = bucket.Split(f.root)
 }
 
+// return a pointer to the string if non empty or nil if it is empty
+func stringPointerOrNil(s string) *string {
+	if s == "" {
+		return nil
+	}
+	return &s
+}
+
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -3669,7 +3685,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
 	return f.cache.Create(bucket, func() error {
 		req := s3.CreateBucketInput{
 			Bucket: &bucket,
-			ACL:    &f.opt.BucketACL,
+			ACL:    stringPointerOrNil(f.opt.BucketACL),
 		}
 		if f.opt.LocationConstraint != "" {
 			req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
@@ -3734,7 +3750,7 @@ func pathEscape(s string) string {
 // method
 func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPath, srcBucket, srcPath string, src *Object) error {
 	req.Bucket = &dstBucket
-	req.ACL = &f.opt.ACL
+	req.ACL = stringPointerOrNil(f.opt.ACL)
 	req.Key = &dstPath
 	source := pathEscape(path.Join(srcBucket, srcPath))
 	if src.versionID != nil {
@@ -5212,7 +5228,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	req := s3.PutObjectInput{
 		Bucket: &bucket,
-		ACL:    &o.fs.opt.ACL,
+		ACL:    stringPointerOrNil(o.fs.opt.ACL),
 		Key:    &bucketPath,
 	}
 

From 7042a11875b6011f7cd5e847c6d14c19b89979e5 Mon Sep 17 00:00:00 2001
From: dgouju 
Date: Thu, 3 Nov 2022 18:11:28 +0100
Subject: [PATCH 346/560] sftp: add configuration options to set ssh Ciphers /
 MACs / KeyExchange

---
 backend/sftp/sftp.go | 65 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 63 insertions(+), 2 deletions(-)

diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go
index 19596e59d3cde..eaafec787c58b 100644
--- a/backend/sftp/sftp.go
+++ b/backend/sftp/sftp.go
@@ -123,7 +123,10 @@ This enables the use of the following insecure ciphers and key exchange methods:
 - diffie-hellman-group-exchange-sha256
 - diffie-hellman-group-exchange-sha1
 
-Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.`,
+Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.
+
+This must be false if you use either ciphers or key_exchange advanced options.
+`,
 			Default: false,
 			Examples: []fs.OptionExample{
 				{
@@ -325,6 +328,46 @@ and pass variables with spaces in in quotes, eg
 
     "VAR3=value with space" "VAR4=value with space" VAR5=nospacehere
 
+`,
+			Advanced: true,
+		}, {
+			Name:    "ciphers",
+			Default: fs.SpaceSepList{},
+			Help: `Space separated list of ciphers to be used for session encryption, ordered by preference.
+
+At least one must match with server configuration. This can be checked for example using ssh -Q cipher.
+
+This must not be set if use_insecure_cipher is true.
+
+Example:
+
+    aes128-ctr aes192-ctr aes256-ctr aes128-gcm@openssh.com aes256-gcm@openssh.com
+`,
+			Advanced: true,
+		}, {
+			Name:    "key_exchange",
+			Default: fs.SpaceSepList{},
+			Help: `Space separated list of key exchange algorithms, ordered by preference.
+
+At least one must match with server configuration. This can be checked for example using ssh -Q kex.
+
+This must not be set if use_insecure_cipher is true.
+
+Example:
+
+    sntrup761x25519-sha512@openssh.com curve25519-sha256 curve25519-sha256@libssh.org ecdh-sha2-nistp256
+`,
+			Advanced: true,
+		}, {
+			Name:    "macs",
+			Default: fs.SpaceSepList{},
+			Help: `Space separated list of MACs (message authentication code) algorithms, ordered by preference.
+
+At least one must match with server configuration. This can be checked for example using ssh -Q mac.
+
+Example:
+
+    umac-64-etm@openssh.com umac-128-etm@openssh.com hmac-sha2-256-etm@openssh.com
 `,
 			Advanced: true,
 		}},
@@ -362,6 +405,9 @@ type Options struct {
 	ChunkSize               fs.SizeSuffix   `config:"chunk_size"`
 	Concurrency             int             `config:"concurrency"`
 	SetEnv                  fs.SpaceSepList `config:"set_env"`
+	Ciphers                 fs.SpaceSepList `config:"ciphers"`
+	KeyExchange             fs.SpaceSepList `config:"key_exchange"`
+	MACs                    fs.SpaceSepList `config:"macs"`
 }
 
 // Fs stores the interface to the remote SFTP files
@@ -702,10 +748,25 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		sshConfig.HostKeyCallback = hostcallback
 	}
 
+	if opt.UseInsecureCipher && (opt.Ciphers != nil || opt.KeyExchange != nil) {
+		return nil, fmt.Errorf("use_insecure_cipher must be false if ciphers or key_exchange are set in advanced configuration")
+	}
+
+	sshConfig.Config.SetDefaults()
 	if opt.UseInsecureCipher {
-		sshConfig.Config.SetDefaults()
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc", "aes192-cbc", "aes256-cbc", "3des-cbc")
 		sshConfig.Config.KeyExchanges = append(sshConfig.Config.KeyExchanges, "diffie-hellman-group-exchange-sha1", "diffie-hellman-group-exchange-sha256")
+	} else {
+		if opt.Ciphers != nil {
+			sshConfig.Config.Ciphers = opt.Ciphers
+		}
+		if opt.KeyExchange != nil {
+			sshConfig.Config.KeyExchanges = opt.KeyExchange
+		}
+	}
+
+	if opt.MACs != nil {
+		sshConfig.Config.MACs = opt.MACs
 	}
 
 	keyFile := env.ShellExpand(opt.KeyFile)

From 0de74864b6e951554c50b94a9eec9d36c6de1ebe Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Fri, 4 Nov 2022 16:53:24 +0000
Subject: [PATCH 347/560] Add dgouju to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 84b42fd19c1c1..ef9a5c4efd66e 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -654,3 +654,4 @@ put them back in again.` >}}
   * coultonluke 
   * Anthony Pessy 
   * Philip Harvey 
+  * dgouju 

From 5b5fdc6bc5df29fd85cd1449c43425f4e0810d02 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Fri, 28 Oct 2022 23:23:29 +0100
Subject: [PATCH 348/560] s3: add provider quirk --s3-might-gzip to fix
 corrupted on transfer: sizes differ

Before this change, some files were giving this error when downloaded
from Cloudflare and other providers.

    ERROR corrupted on transfer: sizes differ NNN vs MMM

This is because these providers auto gzips the object when rclone
wasn't expecting it to. (AWS does not gzip objects without their being
uploaded gzipped).

This patch adds a quirk to for fix the problem and a flag to control
it. The quirk `might_gzip` is set to `true` for all providers except
AWS.

See: https://forum.rclone.org/t/s3-error-corrupted-on-transfer-sizes-differ-nnn-vs-mmm/33694/
Fixes: #6533
---
 backend/s3/s3.go | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 388adeb4be4db..8be8307ce81a0 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -2174,6 +2174,31 @@ can't check the size and hash but the file contents will be decompressed.
 `,
 			Advanced: true,
 			Default:  false,
+		}, {
+			Name: "might_gzip",
+			Help: strings.ReplaceAll(`Set this if the backend might gzip objects.
+
+Normally providers will not alter objects when they are downloaded. If
+an object was not uploaded with |Content-Encoding: gzip| then it won't
+be set on download.
+
+However some providers may gzip objects even if they weren't uploaded
+with |Content-Encoding: gzip| (eg Cloudflare).
+
+A symptom of this would be receiving errors like
+
+    ERROR corrupted on transfer: sizes differ NNN vs MMM
+
+If you set this flag and rclone downloads an object with
+Content-Encoding: gzip set and chunked transfer encoding, then rclone
+will decompress the object on the fly.
+
+If this is set to unset (the default) then rclone will choose
+according to the provider setting what to apply, but you can override
+rclone's choice here.
+`, "|", "`"),
+			Default:  fs.Tristate{},
+			Advanced: true,
 		}, {
 			Name:     "no_system_metadata",
 			Help:     `Suppress setting and reading of system metadata`,
@@ -2305,6 +2330,7 @@ type Options struct {
 	Versions              bool                 `config:"versions"`
 	VersionAt             fs.Time              `config:"version_at"`
 	Decompress            bool                 `config:"decompress"`
+	MightGzip             fs.Tristate          `config:"might_gzip"`
 	NoSystemMetadata      bool                 `config:"no_system_metadata"`
 }
 
@@ -2665,10 +2691,12 @@ func setQuirks(opt *Options) {
 		virtualHostStyle  = true
 		urlEncodeListings = true
 		useMultipartEtag  = true
+		mightGzip         = true // assume all providers might gzip until proven otherwise
 	)
 	switch opt.Provider {
 	case "AWS":
 		// No quirks
+		mightGzip = false // Never auto gzips objects
 	case "Alibaba":
 		useMultipartEtag = false // Alibaba seems to calculate multipart Etags differently from AWS
 	case "HuaweiOBS":
@@ -2782,6 +2810,12 @@ func setQuirks(opt *Options) {
 		opt.UseMultipartEtag.Valid = true
 		opt.UseMultipartEtag.Value = useMultipartEtag
 	}
+
+	// set MightGzip if not manually set
+	if !opt.MightGzip.Valid {
+		opt.MightGzip.Valid = true
+		opt.MightGzip.Value = mightGzip
+	}
 }
 
 // setRoot changes the root of the Fs
@@ -4867,7 +4901,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 	// Decompress body if necessary
 	if aws.StringValue(resp.ContentEncoding) == "gzip" {
-		if o.fs.opt.Decompress {
+		if o.fs.opt.Decompress || (resp.ContentLength == nil && o.fs.opt.MightGzip.Value) {
 			return readers.NewGzipReader(resp.Body)
 		}
 		o.fs.warnCompressed.Do(func() {

From 210331bf61d9731c8af83e2a35012d29a187403f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joda=20St=C3=B6=C3=9Fer?= 
Date: Mon, 7 Nov 2022 04:38:40 +0100
Subject: [PATCH 349/560] docs: fix typo `remove` in rclone_serve_restic
 command

---
 docs/content/commands/rclone_serve_restic.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/content/commands/rclone_serve_restic.md b/docs/content/commands/rclone_serve_restic.md
index fff9d2f2071ef..40a4ed6d803f4 100644
--- a/docs/content/commands/rclone_serve_restic.md
+++ b/docs/content/commands/rclone_serve_restic.md
@@ -11,7 +11,7 @@ Serve the remote for restic's REST API.
 
 ## Synopsis
 
-Run a basic web server to serve a remove over restic's REST backend
+Run a basic web server to serve a remote over restic's REST backend
 API over HTTP.  This allows restic to use rclone as a data storage
 mechanism for cloud providers that restic does not support directly.
 

From a2dca02594dc5fbe0198c9fe5d593e94cc9a2eba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Notin?= 
Date: Mon, 7 Nov 2022 07:50:21 +0000
Subject: [PATCH 350/560] docs: fix character that was incorrectly interpreted
 as markdown

---
 docs/content/local.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/content/local.md b/docs/content/local.md
index f123aa74b1e26..8b7920c91a7a3 100644
--- a/docs/content/local.md
+++ b/docs/content/local.md
@@ -428,8 +428,8 @@ Properties:
 Don't check to see if the files change during upload.
 
 Normally rclone checks the size and modification time of files as they
-are being uploaded and aborts with a message which starts "can't copy
-- source file is being updated" if the file changes during upload.
+are being uploaded and aborts with a message which starts "can't copy - 
+source file is being updated" if the file changes during upload.
 
 However on some file systems this modification time check may fail (e.g.
 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this

From c9acc06a49aa2f4962327a8b17fbf7159148e912 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Mon, 7 Nov 2022 08:51:49 +0100
Subject: [PATCH 351/560] =?UTF-8?q?Add=20Cl=C3=A9ment=20Notin=20to=20contr?=
 =?UTF-8?q?ibutors?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index ef9a5c4efd66e..2e50aff32700b 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -655,3 +655,4 @@ put them back in again.` >}}
   * Anthony Pessy 
   * Philip Harvey 
   * dgouju 
+  * Clément Notin 

From 776e5ea83aa2b71bbc46bbc957a8d9b172767a1c Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Mon, 7 Nov 2022 08:59:40 +0100
Subject: [PATCH 352/560] docs: fix character that was incorrectly interpreted
 as markdown

---
 backend/local/local.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/local/local.go b/backend/local/local.go
index 654bb02d1b973..4a2fea5752982 100644
--- a/backend/local/local.go
+++ b/backend/local/local.go
@@ -124,8 +124,8 @@ routine so this flag shouldn't normally be used.`,
 			Help: `Don't check to see if the files change during upload.
 
 Normally rclone checks the size and modification time of files as they
-are being uploaded and aborts with a message which starts "can't copy
-- source file is being updated" if the file changes during upload.
+are being uploaded and aborts with a message which starts "can't copy -
+source file is being updated" if the file changes during upload.
 
 However on some file systems this modification time check may fail (e.g.
 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this

From 5d6b8141ec5cc4f56f2adf101024d583e17ac1a5 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Sat, 20 Aug 2022 16:38:02 +0200
Subject: [PATCH 353/560] Replace deprecated ioutil

As of Go 1.16, the same functionality is now provided by package io or
package os, and those implementations should be preferred in new code.
---
 backend/azureblob/azureblob.go                |  4 +--
 backend/azureblob/imds.go                     |  7 ++---
 backend/box/box.go                            |  4 +--
 backend/cache/cache_internal_test.go          |  7 ++---
 backend/cache/plex.go                         |  4 +--
 backend/cache/storage_persistent.go           |  5 ++-
 backend/chunker/chunker.go                    |  5 ++-
 backend/chunker/chunker_internal_test.go      | 14 ++++-----
 backend/compress/compress.go                  |  7 ++---
 backend/crypt/cipher_test.go                  | 31 +++++++++----------
 backend/drive/drive.go                        |  6 ++--
 backend/drive/drive_internal_test.go          |  3 +-
 backend/filefabric/filefabric.go              |  3 +-
 .../googlecloudstorage/googlecloudstorage.go  |  4 +--
 backend/googlephotos/googlephotos_test.go     |  4 +--
 backend/hasher/object.go                      |  3 +-
 backend/http/http_internal_test.go            | 10 +++---
 backend/jottacloud/jottacloud.go              | 11 +++----
 backend/local/local.go                        |  5 ++-
 backend/local/local_internal_test.go          |  6 ++--
 backend/local/remove_test.go                  |  3 +-
 backend/mailru/mailru.go                      |  7 ++---
 backend/memory/memory.go                      |  5 ++-
 backend/netstorage/netstorage.go              |  5 ++-
 backend/s3/s3.go                              |  3 +-
 backend/seafile/webapi.go                     |  3 +-
 backend/sftp/sftp.go                          |  7 ++---
 backend/sharefile/sharefile.go                |  3 +-
 backend/sharefile/tzdata_vfsdata.go           |  3 +-
 backend/sugarsync/sugarsync_internal_test.go  |  4 +--
 backend/swift/swift_test.go                   |  3 +-
 backend/union/entry.go                        |  3 +-
 backend/union/union.go                        |  3 +-
 backend/uptobox/uptobox.go                    |  3 +-
 backend/zoho/zoho.go                          |  3 +-
 bin/cross-compile.go                          |  5 ++-
 bin/get-github-release.go                     |  3 +-
 bin/not-in-stable.go                          |  3 +-
 cmd/bisync/bilib/files.go                     |  5 ++-
 cmd/bisync/bilib/names.go                     |  4 +--
 cmd/bisync/bisync_test.go                     | 13 ++++----
 cmd/bisync/cmd.go                             |  5 ++-
 cmd/bisync/operations.go                      |  3 +-
 cmd/cat/cat.go                                |  3 +-
 cmd/genautocomplete/genautocomplete_test.go   | 25 +++++++--------
 cmd/gendocs/gendocs.go                        |  7 ++---
 cmd/mount/test/seeker.go                      |  5 ++-
 cmd/mountlib/rc_test.go                       |  3 +-
 cmd/rc/rc.go                                  |  4 +--
 cmd/selfupdate/selfupdate.go                  |  7 ++---
 cmd/selfupdate/selfupdate_test.go             | 11 +++----
 cmd/serve/dlna/data/assets_vfsdata.go         |  3 +-
 cmd/serve/dlna/data/data.go                   |  4 +--
 cmd/serve/dlna/dlna_test.go                   | 16 +++++-----
 cmd/serve/docker/docker_test.go               |  8 ++---
 cmd/serve/docker/driver.go                    |  5 ++-
 cmd/serve/docker/serve.go                     |  3 +-
 cmd/serve/http/data/assets_vfsdata.go         |  3 +-
 cmd/serve/http/data/data.go                   |  7 +++--
 cmd/serve/http/http_test.go                   |  9 +++---
 cmd/serve/httplib/httplib.go                  |  4 +--
 cmd/serve/sftp/server.go                      | 11 +++----
 cmd/serve/webdav/webdav_test.go               |  8 ++---
 cmd/version/version.go                        |  4 +--
 cmd/version/version_test.go                   |  3 +-
 cmdtest/cmdtest_test.go                       |  3 +-
 fs/accounting/accounting_test.go              | 27 ++++++++--------
 fs/asyncreader/asyncreader_test.go            | 17 +++++-----
 fs/config/configfile/configfile.go            |  3 +-
 fs/config/configfile/configfile_test.go       |  5 ++-
 fs/config/crypt.go                            |  9 +++---
 fs/config/ui_test.go                          |  3 +-
 fs/filter/filter_test.go                      |  3 +-
 fs/fshttp/http.go                             |  4 +--
 fs/fspath/path_test.go                        |  3 +-
 fs/newfs.go                                   |  3 +-
 fs/object/object.go                           |  5 ++-
 fs/object/object_test.go                      |  3 +-
 fs/operations/operations.go                   |  9 +++---
 fs/operations/operations_test.go              | 13 ++++----
 fs/operations/reopen_test.go                  |  7 ++---
 fs/rc/rcserver/rcserver_test.go               |  5 ++-
 fs/rc/webgui/plugins.go                       |  7 ++---
 fs/rc/webgui/webgui.go                        |  5 ++-
 fstest/fstest.go                              |  3 +-
 fstest/fstests/fstests.go                     |  3 +-
 fstest/run.go                                 |  5 ++-
 fstest/test_all/config.go                     |  4 +--
 fstest/test_all/report.go                     |  5 ++-
 lib/encoder/filename/gentable.go              |  4 +--
 lib/file/file_test.go                         |  9 +++---
 lib/http/http.go                              |  4 +--
 lib/http/serve/dir_test.go                    |  6 ++--
 lib/http/serve/serve_test.go                  | 12 +++----
 lib/jwtutil/jwtutil.go                        |  3 +-
 lib/plugin/plugin.go                          |  3 +-
 lib/readers/pattern_reader_test.go            |  7 ++---
 lib/rest/rest.go                              |  3 +-
 vfs/file_test.go                              |  6 ++--
 vfs/make_open_tests.go                        |  3 +-
 vfs/read_write_test.go                        |  3 +-
 vfs/test_vfs/test_vfs.go                      |  3 +-
 vfs/vfs.go                                    |  4 +--
 vfs/vfscache/downloaders/downloaders_test.go  |  3 +-
 vfs/vfscache/item_test.go                     |  3 +-
 vfs/vfstest/os.go                             | 17 ++++++++--
 vfs/vfstest/read.go                           |  9 +++---
 vfs/vfstest/submount.go                       |  3 +-
 108 files changed, 295 insertions(+), 359 deletions(-)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 0db3592220d16..e9402ad0e10bc 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -12,9 +12,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -717,7 +717,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
 		}
 		// Try loading service principal credentials from file.
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
 		}
diff --git a/backend/azureblob/imds.go b/backend/azureblob/imds.go
index 84c6379e6d821..889e224dba707 100644
--- a/backend/azureblob/imds.go
+++ b/backend/azureblob/imds.go
@@ -9,7 +9,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 
 	"github.com/Azure/go-autorest/autorest/adal"
@@ -97,7 +96,7 @@ func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
 		return result, fmt.Errorf("MSI is not enabled on this VM: %w", err)
 	}
 	defer func() { // resp and Body should not be nil
-		_, err = io.Copy(ioutil.Discard, resp.Body)
+		_, err = io.Copy(io.Discard, resp.Body)
 		if err != nil {
 			fs.Debugf(nil, "Unable to drain IMDS response: %v", err)
 		}
@@ -112,12 +111,12 @@ func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
 	case 200, 201, 202:
 		break
 	default:
-		body, _ := ioutil.ReadAll(resp.Body)
+		body, _ := io.ReadAll(resp.Body)
 		fs.Errorf(nil, "Couldn't obtain OAuth token from IMDS; server returned status code %d and body: %v", resp.StatusCode, string(body))
 		return result, httpError{Response: resp}
 	}
 
-	b, err := ioutil.ReadAll(resp.Body)
+	b, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return result, fmt.Errorf("couldn't read IMDS response: %w", err)
 	}
diff --git a/backend/box/box.go b/backend/box/box.go
index a6a33efdb1ea6..e0cfbed07d6bf 100644
--- a/backend/box/box.go
+++ b/backend/box/box.go
@@ -17,9 +17,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -183,7 +183,7 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
 }
 
 func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
-	file, err := ioutil.ReadFile(configFile)
+	file, err := os.ReadFile(configFile)
 	if err != nil {
 		return nil, fmt.Errorf("box: failed to read Box config: %w", err)
 	}
diff --git a/backend/cache/cache_internal_test.go b/backend/cache/cache_internal_test.go
index 4bea042914856..849354274069e 100644
--- a/backend/cache/cache_internal_test.go
+++ b/backend/cache/cache_internal_test.go
@@ -11,7 +11,6 @@ import (
 	goflag "flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -167,7 +166,7 @@ func TestInternalVfsCache(t *testing.T) {
 			li2 := [2]string{path.Join("test", "one"), path.Join("test", "second")}
 			for _, r := range li2 {
 				var err error
-				ci, err := ioutil.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
+				ci, err := os.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
 				if err != nil || len(ci) == 0 {
 					log.Printf("========== '%v' not in cache", r)
 				} else {
@@ -841,7 +840,7 @@ func newRun() *run {
 	}
 
 	if uploadDir == "" {
-		r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
+		r.tmpUploadDir, err = os.MkdirTemp("", "rclonecache-tmp")
 		if err != nil {
 			panic(fmt.Sprintf("Failed to create temp dir: %v", err))
 		}
@@ -984,7 +983,7 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	chunk := int64(1024)
 	cnt := size / chunk
 	left := size % chunk
-	f, err := ioutil.TempFile("", "rclonecache-tempfile")
+	f, err := os.CreateTemp("", "rclonecache-tempfile")
 	require.NoError(t, err)
 
 	for i := 0; i < int(cnt); i++ {
diff --git a/backend/cache/plex.go b/backend/cache/plex.go
index 09ea472336886..8d57ecb17f72b 100644
--- a/backend/cache/plex.go
+++ b/backend/cache/plex.go
@@ -8,7 +8,7 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -167,7 +167,7 @@ func (p *plexConnector) listenWebsocket() {
 								continue
 							}
 							var data []byte
-							data, err = ioutil.ReadAll(resp.Body)
+							data, err = io.ReadAll(resp.Body)
 							if err != nil {
 								continue
 							}
diff --git a/backend/cache/storage_persistent.go b/backend/cache/storage_persistent.go
index eb36a08063e44..ac8fdfd57c976 100644
--- a/backend/cache/storage_persistent.go
+++ b/backend/cache/storage_persistent.go
@@ -9,7 +9,6 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"strconv"
@@ -473,7 +472,7 @@ func (b *Persistent) GetChunk(cachedObject *Object, offset int64) ([]byte, error
 	var data []byte
 
 	fp := path.Join(b.dataPath, cachedObject.abs(), strconv.FormatInt(offset, 10))
-	data, err := ioutil.ReadFile(fp)
+	data, err := os.ReadFile(fp)
 	if err != nil {
 		return nil, err
 	}
@@ -486,7 +485,7 @@ func (b *Persistent) AddChunk(fp string, data []byte, offset int64) error {
 	_ = os.MkdirAll(path.Join(b.dataPath, fp), os.ModePerm)
 
 	filePath := path.Join(b.dataPath, fp, strconv.FormatInt(offset, 10))
-	err := ioutil.WriteFile(filePath, data, os.ModePerm)
+	err := os.WriteFile(filePath, data, os.ModePerm)
 	if err != nil {
 		return err
 	}
diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.go
index 17da9393c20b7..bb4b498126a1e 100644
--- a/backend/chunker/chunker.go
+++ b/backend/chunker/chunker.go
@@ -12,7 +12,6 @@ import (
 	"fmt"
 	gohash "hash"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"path"
 	"regexp"
@@ -1038,7 +1037,7 @@ func (o *Object) readMetadata(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	metadata, err := ioutil.ReadAll(reader)
+	metadata, err := io.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return err
@@ -1097,7 +1096,7 @@ func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
 	if err != nil {
 		return "", err
 	}
-	data, err := ioutil.ReadAll(reader)
+	data, err := io.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return "", err
diff --git a/backend/chunker/chunker_internal_test.go b/backend/chunker/chunker_internal_test.go
index c875c6762e5ba..dee3657d94b4d 100644
--- a/backend/chunker/chunker_internal_test.go
+++ b/backend/chunker/chunker_internal_test.go
@@ -5,7 +5,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"path"
 	"regexp"
 	"strings"
@@ -413,7 +413,7 @@ func testSmallFileInternals(t *testing.T, f *Fs) {
 		if r == nil {
 			return
 		}
-		data, err := ioutil.ReadAll(r)
+		data, err := io.ReadAll(r)
 		assert.NoError(t, err)
 		assert.Equal(t, contents, string(data))
 		_ = r.Close()
@@ -538,7 +538,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	assert.NoError(t, err)
 	var chunkContents []byte
 	assert.NotPanics(t, func() {
-		chunkContents, err = ioutil.ReadAll(r)
+		chunkContents, err = io.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -573,7 +573,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	r, err = willyChunk.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotPanics(t, func() {
-		_, err = ioutil.ReadAll(r)
+		_, err = io.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -672,7 +672,7 @@ func testMetadataInput(t *testing.T, f *Fs) {
 		assert.NoError(t, err, "open "+description)
 		assert.NotNil(t, r, "open stream of "+description)
 		if err == nil && r != nil {
-			data, err := ioutil.ReadAll(r)
+			data, err := io.ReadAll(r)
 			assert.NoError(t, err, "read all of "+description)
 			assert.Equal(t, contents, string(data), description+" contents is ok")
 			_ = r.Close()
@@ -758,7 +758,7 @@ func testFutureProof(t *testing.T, f *Fs) {
 	assert.Error(t, err)
 
 	// Rcat must fail
-	in := ioutil.NopCloser(bytes.NewBufferString("abc"))
+	in := io.NopCloser(bytes.NewBufferString("abc"))
 	robj, err := operations.Rcat(ctx, f, file, in, modTime)
 	assert.Nil(t, robj)
 	assert.NotNil(t, err)
@@ -854,7 +854,7 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
 	r, err := dstFile.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotNil(t, r)
-	data, err := ioutil.ReadAll(r)
+	data, err := io.ReadAll(r)
 	assert.NoError(t, err)
 	assert.Equal(t, contents, string(data))
 	_ = r.Close()
diff --git a/backend/compress/compress.go b/backend/compress/compress.go
index 6bfe1210c6595..21ab99c03f50a 100644
--- a/backend/compress/compress.go
+++ b/backend/compress/compress.go
@@ -13,7 +13,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"regexp"
 	"strings"
@@ -468,7 +467,7 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
 	}
 
 	fs.Debugf(f, "Target remote doesn't support streaming uploads, creating temporary local file")
-	tempFile, err := ioutil.TempFile("", "rclone-press-")
+	tempFile, err := os.CreateTemp("", "rclone-press-")
 	defer func() {
 		// these errors should be relatively uncritical and the upload should've succeeded so it's okay-ish
 		// to ignore them
@@ -546,8 +545,8 @@ func (f *Fs) putCompress(ctx context.Context, in io.Reader, src fs.ObjectInfo, o
 	}
 
 	// Transfer the data
-	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx), options)
-	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx))
+	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
+	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx))
 	if err != nil {
 		if o != nil {
 			removeErr := o.Remove(ctx)
diff --git a/backend/crypt/cipher_test.go b/backend/crypt/cipher_test.go
index 565ef6af528d6..d0ba808be4678 100644
--- a/backend/crypt/cipher_test.go
+++ b/backend/crypt/cipher_test.go
@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"strings"
 	"testing"
 
@@ -1073,7 +1072,7 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
 	source := newRandomSource(copySize)
 	encrypted, err := c.newEncrypter(source, nil)
 	assert.NoError(t, err)
-	decrypted, err := c.newDecrypter(ioutil.NopCloser(encrypted))
+	decrypted, err := c.newDecrypter(io.NopCloser(encrypted))
 	assert.NoError(t, err)
 	sink := newRandomSource(copySize)
 	n, err := io.CopyBuffer(sink, decrypted, buf)
@@ -1144,15 +1143,15 @@ func TestEncryptData(t *testing.T) {
 		buf := bytes.NewBuffer(test.in)
 		encrypted, err := c.EncryptData(buf)
 		assert.NoError(t, err)
-		out, err := ioutil.ReadAll(encrypted)
+		out, err := io.ReadAll(encrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.expected, out)
 
 		// Check we can decode the data properly too...
 		buf = bytes.NewBuffer(out)
-		decrypted, err := c.DecryptData(ioutil.NopCloser(buf))
+		decrypted, err := c.DecryptData(io.NopCloser(buf))
 		assert.NoError(t, err)
-		out, err = ioutil.ReadAll(decrypted)
+		out, err = io.ReadAll(decrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.in, out)
 	}
@@ -1187,7 +1186,7 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
 	fh, err := c.newEncrypter(in, nil)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(io.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(32), n)
 }
@@ -1257,12 +1256,12 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
 
 	in2 := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
 	in1 := bytes.NewBuffer(file16)
-	in := ioutil.NopCloser(io.MultiReader(in1, in2))
+	in := io.NopCloser(io.MultiReader(in1, in2))
 
 	fh, err := c.newDecrypter(in)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(io.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(16), n)
 }
@@ -1274,14 +1273,14 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 
 	// Make random data
 	const dataSize = 150000
-	plaintext, err := ioutil.ReadAll(newRandomSource(dataSize))
+	plaintext, err := io.ReadAll(newRandomSource(dataSize))
 	assert.NoError(t, err)
 
 	// Encrypt the data
 	buf := bytes.NewBuffer(plaintext)
 	encrypted, err := c.EncryptData(buf)
 	assert.NoError(t, err)
-	ciphertext, err := ioutil.ReadAll(encrypted)
+	ciphertext, err := io.ReadAll(encrypted)
 	assert.NoError(t, err)
 
 	trials := []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65,
@@ -1300,7 +1299,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 				end = len(ciphertext)
 			}
 		}
-		reader = ioutil.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
+		reader = io.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
 		return reader, nil
 	}
 
@@ -1490,7 +1489,7 @@ func TestDecrypterRead(t *testing.T) {
 			assert.NoError(t, err, what)
 			continue
 		}
-		_, err = ioutil.ReadAll(fh)
+		_, err = io.ReadAll(fh)
 		var expectedErr error
 		switch {
 		case i == fileHeaderSize:
@@ -1514,7 +1513,7 @@ func TestDecrypterRead(t *testing.T) {
 	cd := newCloseDetector(in)
 	fh, err := c.newDecrypter(cd)
 	assert.NoError(t, err)
-	_, err = ioutil.ReadAll(fh)
+	_, err = io.ReadAll(fh)
 	assert.Error(t, err, "potato")
 	assert.Equal(t, 0, cd.closed)
 
@@ -1524,13 +1523,13 @@ func TestDecrypterRead(t *testing.T) {
 	copy(file16copy, file16)
 	for i := range file16copy {
 		file16copy[i] ^= 0xFF
-		fh, err := c.newDecrypter(ioutil.NopCloser(bytes.NewBuffer(file16copy)))
+		fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
 		if i < fileMagicSize {
 			assert.Error(t, err, ErrorEncryptedBadMagic.Error())
 			assert.Nil(t, fh)
 		} else {
 			assert.NoError(t, err)
-			_, err = ioutil.ReadAll(fh)
+			_, err = io.ReadAll(fh)
 			assert.Error(t, err, ErrorEncryptedFileBadHeader.Error())
 		}
 		file16copy[i] ^= 0xFF
@@ -1565,7 +1564,7 @@ func TestDecrypterClose(t *testing.T) {
 	assert.Equal(t, 0, cd.closed)
 
 	// close after reading
-	out, err := ioutil.ReadAll(fh)
+	out, err := io.ReadAll(fh)
 	assert.NoError(t, err)
 	assert.Equal(t, []byte{1}, out)
 	assert.Equal(t, io.EOF, fh.err)
diff --git a/backend/drive/drive.go b/backend/drive/drive.go
index e0b4abb61fc3c..cca4245c3636f 100644
--- a/backend/drive/drive.go
+++ b/backend/drive/drive.go
@@ -14,9 +14,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"net/http"
+	"os"
 	"path"
 	"regexp"
 	"sort"
@@ -1108,7 +1108,7 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
 
 	// try loading service account credentials from env variable, then from a file
 	if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
@@ -3800,7 +3800,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
 		data = data[:limit]
 	}
 
-	return ioutil.NopCloser(bytes.NewReader(data)), nil
+	return io.NopCloser(bytes.NewReader(data)), nil
 }
 
 func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
diff --git a/backend/drive/drive_internal_test.go b/backend/drive/drive_internal_test.go
index c17da706293c9..58bdb13138e4d 100644
--- a/backend/drive/drive_internal_test.go
+++ b/backend/drive/drive_internal_test.go
@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"os"
 	"path"
@@ -78,7 +77,7 @@ var additionalMimeTypes = map[string]string{
 // Load the example export formats into exportFormats for testing
 func TestInternalLoadExampleFormats(t *testing.T) {
 	fetchFormatsOnce.Do(func() {})
-	buf, err := ioutil.ReadFile(filepath.FromSlash("test/about.json"))
+	buf, err := os.ReadFile(filepath.FromSlash("test/about.json"))
 	var about struct {
 		ExportFormats map[string][]string `json:"exportFormats,omitempty"`
 		ImportFormats map[string][]string `json:"importFormats,omitempty"`
diff --git a/backend/filefabric/filefabric.go b/backend/filefabric/filefabric.go
index 7442814c51aab..3d157f8fbbea0 100644
--- a/backend/filefabric/filefabric.go
+++ b/backend/filefabric/filefabric.go
@@ -20,7 +20,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -1186,7 +1185,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, errors.New("can't download - no id")
 	}
 	if o.contentType == emptyMimeType {
-		return ioutil.NopCloser(bytes.NewReader([]byte{})), nil
+		return io.NopCloser(bytes.NewReader([]byte{})), nil
 	}
 	fs.FixRangeOption(options, o.size)
 	resp, err := o.fs.rpc(ctx, "getFile", params{
diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go
index dc233bfcd147e..f984fa79d14e3 100644
--- a/backend/googlecloudstorage/googlecloudstorage.go
+++ b/backend/googlecloudstorage/googlecloudstorage.go
@@ -19,8 +19,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -487,7 +487,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// try loading service account credentials from env variable, then from a file
 	if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
diff --git a/backend/googlephotos/googlephotos_test.go b/backend/googlephotos/googlephotos_test.go
index 11cb28e1bf271..3cc3a2706d018 100644
--- a/backend/googlephotos/googlephotos_test.go
+++ b/backend/googlephotos/googlephotos_test.go
@@ -3,7 +3,7 @@ package googlephotos
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"path"
 	"testing"
@@ -99,7 +99,7 @@ func TestIntegration(t *testing.T) {
 			t.Run("ObjectOpen", func(t *testing.T) {
 				in, err := dstObj.Open(ctx)
 				require.NoError(t, err)
-				buf, err := ioutil.ReadAll(in)
+				buf, err := io.ReadAll(in)
 				require.NoError(t, err)
 				require.NoError(t, in.Close())
 				assert.True(t, len(buf) > 1000)
diff --git a/backend/hasher/object.go b/backend/hasher/object.go
index 6b9a8e4d05d35..ef6349b0fe3c9 100644
--- a/backend/hasher/object.go
+++ b/backend/hasher/object.go
@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"path"
 	"time"
 
@@ -118,7 +117,7 @@ func (o *Object) updateHashes(ctx context.Context) error {
 	defer func() {
 		_ = r.Close()
 	}()
-	if _, err = io.Copy(ioutil.Discard, r); err != nil {
+	if _, err = io.Copy(io.Discard, r); err != nil {
 		fs.Infof(o, "update failed (copy): %v", err)
 		return err
 	}
diff --git a/backend/http/http_internal_test.go b/backend/http/http_internal_test.go
index 17c58737c0c2f..8bf6be6b66acd 100644
--- a/backend/http/http_internal_test.go
+++ b/backend/http/http_internal_test.go
@@ -3,7 +3,7 @@ package http
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -41,12 +41,12 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
 	// verify the file path is correct, and also check which line endings
 	// are used to get sizes right ("\n" except on Windows, but even there
 	// we may have "\n" or "\r\n" depending on git crlf setting)
-	fileList, err := ioutil.ReadDir(filesPath)
+	fileList, err := os.ReadDir(filesPath)
 	require.NoError(t, err)
 	require.Greater(t, len(fileList), 0)
 	for _, file := range fileList {
 		if !file.IsDir() {
-			data, _ := ioutil.ReadFile(filepath.Join(filesPath, file.Name()))
+			data, _ := os.ReadFile(filepath.Join(filesPath, file.Name()))
 			if strings.HasSuffix(string(data), "\r\n") {
 				lineEndSize = 2
 			}
@@ -203,7 +203,7 @@ func TestOpen(t *testing.T) {
 	// Test normal read
 	fd, err := o.Open(context.Background())
 	require.NoError(t, err)
-	data, err := ioutil.ReadAll(fd)
+	data, err := io.ReadAll(fd)
 	require.NoError(t, err)
 	require.NoError(t, fd.Close())
 	if lineEndSize == 2 {
@@ -215,7 +215,7 @@ func TestOpen(t *testing.T) {
 	// Test with range request
 	fd, err = o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
 	require.NoError(t, err)
-	data, err = ioutil.ReadAll(fd)
+	data, err = io.ReadAll(fd)
 	require.NoError(t, err)
 	require.NoError(t, fd.Close())
 	assert.Equal(t, "eetro", string(data))
diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go
index 45e8489eab0ea..6eb76d1ef00f5 100644
--- a/backend/jottacloud/jottacloud.go
+++ b/backend/jottacloud/jottacloud.go
@@ -12,7 +12,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -822,7 +821,7 @@ func (f *Fs) allocatePathRaw(file string, absolute bool) string {
 func grantTypeFilter(req *http.Request) {
 	if legacyTokenURL == req.URL.String() {
 		// read the entire body
-		refreshBody, err := ioutil.ReadAll(req.Body)
+		refreshBody, err := io.ReadAll(req.Body)
 		if err != nil {
 			return
 		}
@@ -832,7 +831,7 @@ func grantTypeFilter(req *http.Request) {
 		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
 
 		// set the new ReadCloser (with a dummy Close())
-		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
+		req.Body = io.NopCloser(bytes.NewReader(refreshBody))
 	}
 }
 
@@ -1789,7 +1788,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 		var tempFile *os.File
 
 		// create the cache file
-		tempFile, err = ioutil.TempFile("", cachePrefix)
+		tempFile, err = os.CreateTemp("", cachePrefix)
 		if err != nil {
 			return
 		}
@@ -1817,7 +1816,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 	} else {
 		// that's a small file, just read it into memory
 		var inData []byte
-		inData, err = ioutil.ReadAll(teeReader)
+		inData, err = io.ReadAll(teeReader)
 		if err != nil {
 			return
 		}
@@ -1914,7 +1913,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		// copy the already uploaded bytes into the trash :)
 		var result api.UploadResponse
-		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
+		_, err = io.CopyN(io.Discard, in, response.ResumePos)
 		if err != nil {
 			return err
 		}
diff --git a/backend/local/local.go b/backend/local/local.go
index 4a2fea5752982..8d16ad4a86c4c 100644
--- a/backend/local/local.go
+++ b/backend/local/local.go
@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -646,7 +645,7 @@ func (f *Fs) readPrecision() (precision time.Duration) {
 	precision = time.Second
 
 	// Create temporary file and test it
-	fd, err := ioutil.TempFile("", "rclone")
+	fd, err := os.CreateTemp("", "rclone")
 	if err != nil {
 		// If failed return 1s
 		// fmt.Println("Failed to create temp file", err)
@@ -1073,7 +1072,7 @@ func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err
 	if err != nil {
 		return nil, err
 	}
-	return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
+	return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
 }
 
 // Open an object for read
diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go
index 7ecc9986bf5fb..a8c48329a04e5 100644
--- a/backend/local/local_internal_test.go
+++ b/backend/local/local_internal_test.go
@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -150,7 +150,7 @@ func TestSymlink(t *testing.T) {
 	// Check reading the object
 	in, err := o.Open(ctx)
 	require.NoError(t, err)
-	contents, err := ioutil.ReadAll(in)
+	contents, err := io.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt", string(contents))
 	require.NoError(t, in.Close())
@@ -158,7 +158,7 @@ func TestSymlink(t *testing.T) {
 	// Check reading the object with range
 	in, err = o.Open(ctx, &fs.RangeOption{Start: 2, End: 5})
 	require.NoError(t, err)
-	contents, err = ioutil.ReadAll(in)
+	contents, err = io.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt"[2:5+1], string(contents))
 	require.NoError(t, in.Close())
diff --git a/backend/local/remove_test.go b/backend/local/remove_test.go
index b2e34cdc818c7..a8007500014b0 100644
--- a/backend/local/remove_test.go
+++ b/backend/local/remove_test.go
@@ -1,7 +1,6 @@
 package local
 
 import (
-	"io/ioutil"
 	"os"
 	"sync"
 	"testing"
@@ -13,7 +12,7 @@ import (
 
 // Check we can remove an open file
 func TestRemove(t *testing.T) {
-	fd, err := ioutil.TempFile("", "rclone-remove-test")
+	fd, err := os.CreateTemp("", "rclone-remove-test")
 	require.NoError(t, err)
 	name := fd.Name()
 	defer func() {
diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go
index 95507996c4f28..5a630a2d3c1a0 100644
--- a/backend/mailru/mailru.go
+++ b/backend/mailru/mailru.go
@@ -18,7 +18,6 @@ import (
 
 	"encoding/hex"
 	"encoding/json"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 
@@ -1660,7 +1659,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Attempt to put by calculating hash in memory
 	if trySpeedup && size <= int64(o.fs.opt.SpeedupMaxMem) {
-		fileBuf, err = ioutil.ReadAll(in)
+		fileBuf, err = io.ReadAll(in)
 		if err != nil {
 			return err
 		}
@@ -1703,7 +1702,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if size <= mrhash.Size {
 		// Optimize upload: skip extra request if data fits in the hash buffer.
 		if fileBuf == nil {
-			fileBuf, err = ioutil.ReadAll(wrapIn)
+			fileBuf, err = io.ReadAll(wrapIn)
 		}
 		if fileHash == nil && err == nil {
 			fileHash = mrhash.Sum(fileBuf)
@@ -2214,7 +2213,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		fs.Debugf(o, "Server returned full content instead of range")
 		if start > 0 {
 			// Discard the beginning of the data
-			_, err = io.CopyN(ioutil.Discard, wrapStream, start)
+			_, err = io.CopyN(io.Discard, wrapStream, start)
 			if err != nil {
 				closeBody(res)
 				return nil, err
diff --git a/backend/memory/memory.go b/backend/memory/memory.go
index 838a060d59c53..80fc321f97b39 100644
--- a/backend/memory/memory.go
+++ b/backend/memory/memory.go
@@ -8,7 +8,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"path"
 	"strings"
 	"sync"
@@ -575,7 +574,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 		data = data[:limit]
 	}
-	return ioutil.NopCloser(bytes.NewBuffer(data)), nil
+	return io.NopCloser(bytes.NewBuffer(data)), nil
 }
 
 // Update the object with the contents of the io.Reader, modTime and size
@@ -583,7 +582,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	bucket, bucketPath := o.split()
-	data, err := ioutil.ReadAll(in)
+	data, err := io.ReadAll(in)
 	if err != nil {
 		return fmt.Errorf("failed to update memory object: %w", err)
 	}
diff --git a/backend/netstorage/netstorage.go b/backend/netstorage/netstorage.go
index 31adcbe7a4fab..20a6401ab7d2a 100755
--- a/backend/netstorage/netstorage.go
+++ b/backend/netstorage/netstorage.go
@@ -12,7 +12,6 @@ import (
 	"fmt"
 	gohash "hash"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -972,7 +971,7 @@ func (o *Object) netStorageUploadRequest(ctx context.Context, in io.Reader, src
 		URL = o.fs.url(src.Remote())
 	}
 	if strings.HasSuffix(URL, ".rclonelink") {
-		bits, err := ioutil.ReadAll(in)
+		bits, err := io.ReadAll(in)
 		if err != nil {
 			return err
 		}
@@ -1058,7 +1057,7 @@ func (o *Object) netStorageDownloadRequest(ctx context.Context, options []fs.Ope
 	if strings.HasSuffix(URL, ".rclonelink") && o.target != "" {
 		fs.Infof(nil, "Converting a symlink to the rclonelink file on download %q", URL)
 		reader := strings.NewReader(o.target)
-		readcloser := ioutil.NopCloser(reader)
+		readcloser := io.NopCloser(reader)
 		return readcloser, nil
 	}
 
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 8be8307ce81a0..efea72410d4eb 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -15,7 +15,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -5148,7 +5147,7 @@ func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjec
 		// Can't upload zero length files like this for some reason
 		r.Body = bytes.NewReader([]byte{})
 	} else {
-		r.SetStreamingBody(ioutil.NopCloser(in))
+		r.SetStreamingBody(io.NopCloser(in))
 	}
 	r.SetContext(ctx)
 	r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
diff --git a/backend/seafile/webapi.go b/backend/seafile/webapi.go
index c1eb33da0a934..cf3e77bda55e7 100644
--- a/backend/seafile/webapi.go
+++ b/backend/seafile/webapi.go
@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -633,7 +632,7 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs
 		})
 		if start > 0 {
 			// We need to read and discard the beginning of the data...
-			_, err = io.CopyN(ioutil.Discard, resp.Body, start)
+			_, err = io.CopyN(io.Discard, resp.Body, start)
 			if err != nil {
 				return nil, err
 			}
diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go
index eaafec787c58b..b6afcd3558d3c 100644
--- a/backend/sftp/sftp.go
+++ b/backend/sftp/sftp.go
@@ -10,7 +10,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"regexp"
@@ -783,7 +782,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, fmt.Errorf("couldn't read ssh agent signers: %w", err)
 		}
 		if keyFile != "" {
-			pubBytes, err := ioutil.ReadFile(keyFile + ".pub")
+			pubBytes, err := os.ReadFile(keyFile + ".pub")
 			if err != nil {
 				return nil, fmt.Errorf("failed to read public key file: %w", err)
 			}
@@ -812,7 +811,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if keyFile != "" || opt.KeyPem != "" {
 		var key []byte
 		if opt.KeyPem == "" {
-			key, err = ioutil.ReadFile(keyFile)
+			key, err = os.ReadFile(keyFile)
 			if err != nil {
 				return nil, fmt.Errorf("failed to read private key file: %w", err)
 			}
@@ -843,7 +842,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 		// If a public key has been specified then use that
 		if pubkeyFile != "" {
-			certfile, err := ioutil.ReadFile(pubkeyFile)
+			certfile, err := os.ReadFile(pubkeyFile)
 			if err != nil {
 				return nil, fmt.Errorf("unable to read cert file: %w", err)
 			}
diff --git a/backend/sharefile/sharefile.go b/backend/sharefile/sharefile.go
index acb5d65c85b2c..11bbefa6a85c5 100644
--- a/backend/sharefile/sharefile.go
+++ b/backend/sharefile/sharefile.go
@@ -77,7 +77,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -479,7 +478,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, fmt.Errorf("failed to open timezone db: %w", err)
 	}
-	tzdata, err := ioutil.ReadAll(timezone)
+	tzdata, err := io.ReadAll(timezone)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read timezone: %w", err)
 	}
diff --git a/backend/sharefile/tzdata_vfsdata.go b/backend/sharefile/tzdata_vfsdata.go
index 601d82d45cdfa..23d2120f952fb 100644
--- a/backend/sharefile/tzdata_vfsdata.go
+++ b/backend/sharefile/tzdata_vfsdata.go
@@ -10,7 +10,6 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	pathpkg "path"
@@ -119,7 +118,7 @@ func (f *vfsgen۰CompressedFile) Read(p []byte) (n int, err error) {
 	}
 	if f.grPos < f.seekPos {
 		// Fast-forward.
-		_, err = io.CopyN(ioutil.Discard, f.gr, f.seekPos-f.grPos)
+		_, err = io.CopyN(io.Discard, f.gr, f.seekPos-f.grPos)
 		if err != nil {
 			return 0, err
 		}
diff --git a/backend/sugarsync/sugarsync_internal_test.go b/backend/sugarsync/sugarsync_internal_test.go
index 020970e041b6d..59093be2f70d0 100644
--- a/backend/sugarsync/sugarsync_internal_test.go
+++ b/backend/sugarsync/sugarsync_internal_test.go
@@ -2,7 +2,7 @@ package sugarsync
 
 import (
 	"bytes"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"testing"
 
@@ -48,7 +48,7 @@ func TestErrorHandler(t *testing.T) {
 	} {
 		t.Run(test.name, func(t *testing.T) {
 			resp := http.Response{
-				Body:       ioutil.NopCloser(bytes.NewBufferString(test.body)),
+				Body:       io.NopCloser(bytes.NewBufferString(test.body)),
 				StatusCode: test.code,
 				Status:     test.status,
 			}
diff --git a/backend/swift/swift_test.go b/backend/swift/swift_test.go
index 4669d9e901e57..528b86f464a03 100644
--- a/backend/swift/swift_test.go
+++ b/backend/swift/swift_test.go
@@ -6,7 +6,6 @@ import (
 	"context"
 	"errors"
 	"io"
-	"io/ioutil"
 	"testing"
 
 	"github.com/ncw/swift/v2"
@@ -136,7 +135,7 @@ func (f *Fs) testWithChunkFail(t *testing.T) {
 	buf := bytes.NewBufferString(contents[:errPosition])
 	errMessage := "potato"
 	er := &readers.ErrorReader{Err: errors.New(errMessage)}
-	in := ioutil.NopCloser(io.MultiReader(buf, er))
+	in := io.NopCloser(io.MultiReader(buf, er))
 
 	file.Size = contentSize
 	obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
diff --git a/backend/union/entry.go b/backend/union/entry.go
index 023f3bb456d7c..d794d62262ea7 100644
--- a/backend/union/entry.go
+++ b/backend/union/entry.go
@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"sync"
 	"time"
 
@@ -87,7 +86,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 				errs[i] = fmt.Errorf("%s: %w", o.UpstreamFs().Name(), err)
 				if len(entries) > 1 {
 					// Drain the input buffer to allow other uploads to continue
-					_, _ = io.Copy(ioutil.Discard, readers[i])
+					_, _ = io.Copy(io.Discard, readers[i])
 				}
 			}
 		} else {
diff --git a/backend/union/union.go b/backend/union/union.go
index fcfe4ac95884b..aeb0aeca63df1 100644
--- a/backend/union/union.go
+++ b/backend/union/union.go
@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"path"
 	"path/filepath"
 	"strings"
@@ -501,7 +500,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
 			errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
 			if len(upstreams) > 1 {
 				// Drain the input buffer to allow other uploads to continue
-				_, _ = io.Copy(ioutil.Discard, readers[i])
+				_, _ = io.Copy(io.Discard, readers[i])
 			}
 			return
 		}
diff --git a/backend/uptobox/uptobox.go b/backend/uptobox/uptobox.go
index d403341bf8416..172d50f756f63 100644
--- a/backend/uptobox/uptobox.go
+++ b/backend/uptobox/uptobox.go
@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -239,7 +238,7 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe
 func (f *Fs) decodeError(resp *http.Response, response interface{}) (err error) {
 	defer fs.CheckClose(resp.Body, &err)
 
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
diff --git a/backend/zoho/zoho.go b/backend/zoho/zoho.go
index 1e7556b03ac67..f9fc6eb6b8629 100644
--- a/backend/zoho/zoho.go
+++ b/backend/zoho/zoho.go
@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -1219,7 +1218,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if partialContent && resp.StatusCode == 200 {
 		if start > 0 {
 			// We need to read and discard the beginning of the data...
-			_, err = io.CopyN(ioutil.Discard, resp.Body, start)
+			_, err = io.CopyN(io.Discard, resp.Body, start)
 			if err != nil {
 				if resp != nil {
 					_ = resp.Body.Close()
diff --git a/bin/cross-compile.go b/bin/cross-compile.go
index c66b0eecc7c7f..0f0cb64665cbf 100644
--- a/bin/cross-compile.go
+++ b/bin/cross-compile.go
@@ -9,7 +9,6 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -240,7 +239,7 @@ func buildWindowsResourceSyso(goarch string, versionTag string) string {
 		log.Printf("Failed to resolve path: %v", err)
 		return ""
 	}
-	err = ioutil.WriteFile(jsonPath, bs, 0644)
+	err = os.WriteFile(jsonPath, bs, 0644)
 	if err != nil {
 		log.Printf("Failed to write %s: %v", jsonPath, err)
 		return ""
@@ -476,7 +475,7 @@ func main() {
 		run("mkdir", "build")
 	}
 	chdir("build")
-	err := ioutil.WriteFile("version.txt", []byte(fmt.Sprintf("rclone %s\n", version)), 0666)
+	err := os.WriteFile("version.txt", []byte(fmt.Sprintf("rclone %s\n", version)), 0666)
 	if err != nil {
 		log.Fatalf("Couldn't write version.txt: %v", err)
 	}
diff --git a/bin/get-github-release.go b/bin/get-github-release.go
index 1aa98f16a47fd..3aa97cd360640 100644
--- a/bin/get-github-release.go
+++ b/bin/get-github-release.go
@@ -16,7 +16,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"net/url"
@@ -168,7 +167,7 @@ func defaultBinDir() string {
 
 // read the body or an error message
 func readBody(in io.Reader) string {
-	data, err := ioutil.ReadAll(in)
+	data, err := io.ReadAll(in)
 	if err != nil {
 		return fmt.Sprintf("Error reading body: %v", err.Error())
 	}
diff --git a/bin/not-in-stable.go b/bin/not-in-stable.go
index f6fae822657d8..dc685cb6fedea 100644
--- a/bin/not-in-stable.go
+++ b/bin/not-in-stable.go
@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -56,7 +55,7 @@ func main() {
 		log.Fatalf("Syntax: %s", os.Args[0])
 	}
 	// v1.54.0
-	versionBytes, err := ioutil.ReadFile("VERSION")
+	versionBytes, err := os.ReadFile("VERSION")
 	if err != nil {
 		log.Fatalf("Failed to read version: %v", err)
 	}
diff --git a/cmd/bisync/bilib/files.go b/cmd/bisync/bilib/files.go
index 9171c970051e9..e2c7bb1e8d035 100644
--- a/cmd/bisync/bilib/files.go
+++ b/cmd/bisync/bilib/files.go
@@ -5,7 +5,6 @@ package bilib
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -106,7 +105,7 @@ func CopyDir(src string, dst string) (err error) {
 		return
 	}
 
-	entries, err := ioutil.ReadDir(src)
+	entries, err := os.ReadDir(src)
 	if err != nil {
 		return
 	}
@@ -122,7 +121,7 @@ func CopyDir(src string, dst string) (err error) {
 			}
 		} else {
 			// Skip symlinks.
-			if entry.Mode()&os.ModeSymlink != 0 {
+			if entry.Type()&os.ModeSymlink != 0 {
 				continue
 			}
 
diff --git a/cmd/bisync/bilib/names.go b/cmd/bisync/bilib/names.go
index 177a19f4a407c..41c4692b5f6b3 100644
--- a/cmd/bisync/bilib/names.go
+++ b/cmd/bisync/bilib/names.go
@@ -2,7 +2,7 @@ package bilib
 
 import (
 	"bytes"
-	"io/ioutil"
+	"os"
 	"sort"
 	"strconv"
 )
@@ -57,5 +57,5 @@ func SaveList(list []string, path string) error {
 		_, _ = buf.WriteString(strconv.Quote(s))
 		_ = buf.WriteByte('\n')
 	}
-	return ioutil.WriteFile(path, buf.Bytes(), PermSecure)
+	return os.WriteFile(path, buf.Bytes(), PermSecure)
 }
diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go
index 758d786eacf06..11299f012e686 100644
--- a/cmd/bisync/bisync_test.go
+++ b/cmd/bisync/bisync_test.go
@@ -10,7 +10,6 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"path"
@@ -303,7 +302,7 @@ func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase str
 
 	// Execute test scenario
 	scenFile := filepath.Join(b.testDir, "scenario.txt")
-	scenBuf, err := ioutil.ReadFile(scenFile)
+	scenBuf, err := os.ReadFile(scenFile)
 	scenReplacer := b.newReplacer(false)
 	require.NoError(b.t, err)
 	b.step = 0
@@ -903,8 +902,8 @@ func (b *bisyncTest) compareResults() int {
 			// save mangled logs so difference is easier on eyes
 			goldenFile := filepath.Join(b.logDir, "mangled.golden.log")
 			resultFile := filepath.Join(b.logDir, "mangled.result.log")
-			require.NoError(b.t, ioutil.WriteFile(goldenFile, []byte(goldenText), bilib.PermSecure))
-			require.NoError(b.t, ioutil.WriteFile(resultFile, []byte(resultText), bilib.PermSecure))
+			require.NoError(b.t, os.WriteFile(goldenFile, []byte(goldenText), bilib.PermSecure))
+			require.NoError(b.t, os.WriteFile(resultFile, []byte(resultText), bilib.PermSecure))
 		}
 
 		if goldenText == resultText {
@@ -974,7 +973,7 @@ func (b *bisyncTest) storeGolden() {
 
 		goldName := b.toGolden(fileName)
 		goldPath := filepath.Join(b.goldenDir, goldName)
-		err := ioutil.WriteFile(goldPath, []byte(text), bilib.PermSecure)
+		err := os.WriteFile(goldPath, []byte(text), bilib.PermSecure)
 		assert.NoError(b.t, err, "writing golden file %s", goldName)
 
 		if goldName != fileName {
@@ -986,7 +985,7 @@ func (b *bisyncTest) storeGolden() {
 
 // mangleResult prepares test logs or listings for comparison
 func (b *bisyncTest) mangleResult(dir, file string, golden bool) string {
-	buf, err := ioutil.ReadFile(filepath.Join(dir, file))
+	buf, err := os.ReadFile(filepath.Join(dir, file))
 	require.NoError(b.t, err)
 	text := string(buf)
 
@@ -1205,7 +1204,7 @@ func (b *bisyncTest) ensureDir(parent, dir string, optional bool) string {
 }
 
 func (b *bisyncTest) listDir(dir string) (names []string) {
-	files, err := ioutil.ReadDir(dir)
+	files, err := os.ReadDir(dir)
 	require.NoError(b.t, err)
 	for _, file := range files {
 		names = append(names, filepath.Base(file.Name()))
diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.go
index 42b6489bf90b7..c14edf2e72ace 100644
--- a/cmd/bisync/cmd.go
+++ b/cmd/bisync/cmd.go
@@ -9,7 +9,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -198,7 +197,7 @@ func (opt *Options) applyFilters(ctx context.Context) (context.Context, error) {
 	_ = f.Close()
 
 	hashFile := filtersFile + ".md5"
-	wantHash, err := ioutil.ReadFile(hashFile)
+	wantHash, err := os.ReadFile(hashFile)
 	if err != nil && !opt.Resync {
 		return ctx, fmt.Errorf("filters file md5 hash not found (must run --resync): %s", filtersFile)
 	}
@@ -209,7 +208,7 @@ func (opt *Options) applyFilters(ctx context.Context) (context.Context, error) {
 
 	if opt.Resync {
 		fs.Infof(nil, "Storing filters file hash to %s", hashFile)
-		if err := ioutil.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil {
+		if err := os.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil {
 			return ctx, err
 		}
 	}
diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go
index d32153edc3333..85b1e43d17182 100644
--- a/cmd/bisync/operations.go
+++ b/cmd/bisync/operations.go
@@ -7,7 +7,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -81,7 +80,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 		}
 
 		pidStr := []byte(strconv.Itoa(os.Getpid()))
-		if err = ioutil.WriteFile(lockFile, pidStr, bilib.PermSecure); err != nil {
+		if err = os.WriteFile(lockFile, pidStr, bilib.PermSecure); err != nil {
 			return fmt.Errorf("cannot create lock file: %s: %w", lockFile, err)
 		}
 		fs.Debugf(nil, "Lock file created: %s", lockFile)
diff --git a/cmd/cat/cat.go b/cmd/cat/cat.go
index 227ddb1f396c8..ead3614556592 100644
--- a/cmd/cat/cat.go
+++ b/cmd/cat/cat.go
@@ -4,7 +4,6 @@ package cat
 import (
 	"context"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"strings"
@@ -77,7 +76,7 @@ Note that if offset is negative it will count from the end, so
 		fsrc := cmd.NewFsSrc(args)
 		var w io.Writer = os.Stdout
 		if discard {
-			w = ioutil.Discard
+			w = io.Discard
 		}
 		cmd.Run(false, false, command, func() error {
 			return operations.Cat(context.Background(), fsrc, w, offset, count)
diff --git a/cmd/genautocomplete/genautocomplete_test.go b/cmd/genautocomplete/genautocomplete_test.go
index ce766c6fdca4f..fab32f3d17a3c 100644
--- a/cmd/genautocomplete/genautocomplete_test.go
+++ b/cmd/genautocomplete/genautocomplete_test.go
@@ -1,7 +1,6 @@
 package genautocomplete
 
 import (
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -9,7 +8,7 @@ import (
 )
 
 func TestCompletionBash(t *testing.T) {
-	tempFile, err := ioutil.TempFile("", "completion_bash")
+	tempFile, err := os.CreateTemp("", "completion_bash")
 	assert.NoError(t, err)
 	defer func() {
 		_ = tempFile.Close()
@@ -18,14 +17,14 @@ func TestCompletionBash(t *testing.T) {
 
 	bashCommandDefinition.Run(bashCommandDefinition, []string{tempFile.Name()})
 
-	bs, err := ioutil.ReadFile(tempFile.Name())
+	bs, err := os.ReadFile(tempFile.Name())
 	assert.NoError(t, err)
 	assert.NotEmpty(t, string(bs))
 }
 
 func TestCompletionBashStdout(t *testing.T) {
 	originalStdout := os.Stdout
-	tempFile, err := ioutil.TempFile("", "completion_zsh")
+	tempFile, err := os.CreateTemp("", "completion_zsh")
 	assert.NoError(t, err)
 	defer func() {
 		_ = tempFile.Close()
@@ -37,13 +36,13 @@ func TestCompletionBashStdout(t *testing.T) {
 
 	bashCommandDefinition.Run(bashCommandDefinition, []string{"-"})
 
-	output, err := ioutil.ReadFile(tempFile.Name())
+	output, err := os.ReadFile(tempFile.Name())
 	assert.NoError(t, err)
 	assert.NotEmpty(t, string(output))
 }
 
 func TestCompletionZsh(t *testing.T) {
-	tempFile, err := ioutil.TempFile("", "completion_zsh")
+	tempFile, err := os.CreateTemp("", "completion_zsh")
 	assert.NoError(t, err)
 	defer func() {
 		_ = tempFile.Close()
@@ -52,14 +51,14 @@ func TestCompletionZsh(t *testing.T) {
 
 	zshCommandDefinition.Run(zshCommandDefinition, []string{tempFile.Name()})
 
-	bs, err := ioutil.ReadFile(tempFile.Name())
+	bs, err := os.ReadFile(tempFile.Name())
 	assert.NoError(t, err)
 	assert.NotEmpty(t, string(bs))
 }
 
 func TestCompletionZshStdout(t *testing.T) {
 	originalStdout := os.Stdout
-	tempFile, err := ioutil.TempFile("", "completion_zsh")
+	tempFile, err := os.CreateTemp("", "completion_zsh")
 	assert.NoError(t, err)
 	defer func() {
 		_ = tempFile.Close()
@@ -70,13 +69,13 @@ func TestCompletionZshStdout(t *testing.T) {
 	defer func() { os.Stdout = originalStdout }()
 
 	zshCommandDefinition.Run(zshCommandDefinition, []string{"-"})
-	output, err := ioutil.ReadFile(tempFile.Name())
+	output, err := os.ReadFile(tempFile.Name())
 	assert.NoError(t, err)
 	assert.NotEmpty(t, string(output))
 }
 
 func TestCompletionFish(t *testing.T) {
-	tempFile, err := ioutil.TempFile("", "completion_fish")
+	tempFile, err := os.CreateTemp("", "completion_fish")
 	assert.NoError(t, err)
 	defer func() {
 		_ = tempFile.Close()
@@ -85,14 +84,14 @@ func TestCompletionFish(t *testing.T) {
 
 	fishCommandDefinition.Run(fishCommandDefinition, []string{tempFile.Name()})
 
-	bs, err := ioutil.ReadFile(tempFile.Name())
+	bs, err := os.ReadFile(tempFile.Name())
 	assert.NoError(t, err)
 	assert.NotEmpty(t, string(bs))
 }
 
 func TestCompletionFishStdout(t *testing.T) {
 	originalStdout := os.Stdout
-	tempFile, err := ioutil.TempFile("", "completion_zsh")
+	tempFile, err := os.CreateTemp("", "completion_zsh")
 	assert.NoError(t, err)
 	defer func() {
 		_ = tempFile.Close()
@@ -104,7 +103,7 @@ func TestCompletionFishStdout(t *testing.T) {
 
 	fishCommandDefinition.Run(fishCommandDefinition, []string{"-"})
 
-	output, err := ioutil.ReadFile(tempFile.Name())
+	output, err := os.ReadFile(tempFile.Name())
 	assert.NoError(t, err)
 	assert.NotEmpty(t, string(output))
 }
diff --git a/cmd/gendocs/gendocs.go b/cmd/gendocs/gendocs.go
index e80b8a924ba5b..4965aa7a38d56 100644
--- a/cmd/gendocs/gendocs.go
+++ b/cmd/gendocs/gendocs.go
@@ -3,7 +3,6 @@ package gendocs
 
 import (
 	"bytes"
-	"io/ioutil"
 	"log"
 	"os"
 	"path"
@@ -71,7 +70,7 @@ rclone.org website.`,
 		if err != nil {
 			return err
 		}
-		err = ioutil.WriteFile(filepath.Join(root, "flags.md"), buf.Bytes(), 0777)
+		err = os.WriteFile(filepath.Join(root, "flags.md"), buf.Bytes(), 0777)
 		if err != nil {
 			return err
 		}
@@ -129,7 +128,7 @@ rclone.org website.`,
 				return err
 			}
 			if !info.IsDir() {
-				b, err := ioutil.ReadFile(path)
+				b, err := os.ReadFile(path)
 				if err != nil {
 					return err
 				}
@@ -140,7 +139,7 @@ See the [global flags page](/flags/) for global options not listed here.
 ### SEE ALSO`, 1)
 				// outdent all the titles by one
 				doc = outdentTitle.ReplaceAllString(doc, `$1`)
-				err = ioutil.WriteFile(path, []byte(doc), 0777)
+				err = os.WriteFile(path, []byte(doc), 0777)
 				if err != nil {
 					return err
 				}
diff --git a/cmd/mount/test/seeker.go b/cmd/mount/test/seeker.go
index bcc9bb323aecc..42a2f0e9e72db 100644
--- a/cmd/mount/test/seeker.go
+++ b/cmd/mount/test/seeker.go
@@ -8,7 +8,6 @@ import (
 	"bytes"
 	"flag"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -60,11 +59,11 @@ func randomSeekTest(size int64, in1, in2 *os.File, file1, file2 string) {
 
 	if !bytes.Equal(buf1, buf2) {
 		log.Printf("Dumping different blocks")
-		err = ioutil.WriteFile("/tmp/z1", buf1, 0777)
+		err = os.WriteFile("/tmp/z1", buf1, 0777)
 		if err != nil {
 			log.Fatalf("Failed to write /tmp/z1: %v", err)
 		}
-		err = ioutil.WriteFile("/tmp/z2", buf2, 0777)
+		err = os.WriteFile("/tmp/z2", buf2, 0777)
 		if err != nil {
 			log.Fatalf("Failed to write /tmp/z2: %v", err)
 		}
diff --git a/cmd/mountlib/rc_test.go b/cmd/mountlib/rc_test.go
index 5513ce4a7c518..0ac4d6e537683 100644
--- a/cmd/mountlib/rc_test.go
+++ b/cmd/mountlib/rc_test.go
@@ -2,7 +2,6 @@ package mountlib_test
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -36,7 +35,7 @@ func TestRc(t *testing.T) {
 	assert.NotNil(t, getMountTypes)
 
 	localDir := t.TempDir()
-	err := ioutil.WriteFile(filepath.Join(localDir, "file.txt"), []byte("hello"), 0666)
+	err := os.WriteFile(filepath.Join(localDir, "file.txt"), []byte("hello"), 0666)
 	require.NoError(t, err)
 
 	mountPoint := t.TempDir()
diff --git a/cmd/rc/rc.go b/cmd/rc/rc.go
index 93c2e319eb24b..2bb163d036508 100644
--- a/cmd/rc/rc.go
+++ b/cmd/rc/rc.go
@@ -7,7 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"os"
 	"strings"
@@ -204,7 +204,7 @@ func doCall(ctx context.Context, path string, in rc.Params) (out rc.Params, err
 
 	if resp.StatusCode != http.StatusOK {
 		var body []byte
-		body, err = ioutil.ReadAll(resp.Body)
+		body, err = io.ReadAll(resp.Body)
 		var bodyString string
 		if err == nil {
 			bodyString = string(body)
diff --git a/cmd/selfupdate/selfupdate.go b/cmd/selfupdate/selfupdate.go
index ab24290b30ac5..c7f1a3cdacdeb 100644
--- a/cmd/selfupdate/selfupdate.go
+++ b/cmd/selfupdate/selfupdate.go
@@ -14,7 +14,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -227,7 +226,7 @@ func InstallUpdate(ctx context.Context, opt *Options) error {
 }
 
 func installPackage(ctx context.Context, beta bool, version, siteURL, packageFormat string) error {
-	tempFile, err := ioutil.TempFile("", "rclone.*."+packageFormat)
+	tempFile, err := os.CreateTemp("", "rclone.*."+packageFormat)
 	if err != nil {
 		return fmt.Errorf("unable to write temporary package: %w", err)
 	}
@@ -357,7 +356,7 @@ func downloadUpdate(ctx context.Context, beta bool, version, siteURL, newFile, p
 	}
 
 	if packageFormat == "deb" || packageFormat == "rpm" {
-		if err := ioutil.WriteFile(newFile, archiveBuf, 0644); err != nil {
+		if err := os.WriteFile(newFile, archiveBuf, 0644); err != nil {
 			return fmt.Errorf("cannot write temporary .%s: %w", packageFormat, err)
 		}
 		return nil
@@ -471,5 +470,5 @@ func downloadFile(ctx context.Context, url string) ([]byte, error) {
 	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("failed with %s downloading %s", resp.Status, url)
 	}
-	return ioutil.ReadAll(resp.Body)
+	return io.ReadAll(resp.Body)
 }
diff --git a/cmd/selfupdate/selfupdate_test.go b/cmd/selfupdate/selfupdate_test.go
index b8c52a349319f..d4ce2ec2ad279 100644
--- a/cmd/selfupdate/selfupdate_test.go
+++ b/cmd/selfupdate/selfupdate_test.go
@@ -5,7 +5,6 @@ package selfupdate
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -86,7 +85,7 @@ func TestInstallOnLinux(t *testing.T) {
 	assert.NoError(t, InstallUpdate(ctx, &Options{Beta: true, Output: path, Version: fs.Version}))
 
 	// Must fail on non-writable file
-	assert.NoError(t, ioutil.WriteFile(path, []byte("test"), 0644))
+	assert.NoError(t, os.WriteFile(path, []byte("test"), 0644))
 	assert.NoError(t, os.Chmod(path, 0000))
 	err = (InstallUpdate(ctx, &Options{Beta: true, Output: path}))
 	assert.Error(t, err)
@@ -101,7 +100,7 @@ func TestInstallOnLinux(t *testing.T) {
 	assert.Equal(t, os.FileMode(0644), info.Mode().Perm())
 
 	// Must remove temporary files
-	files, err := ioutil.ReadDir(testDir)
+	files, err := os.ReadDir(testDir)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(files))
 
@@ -141,7 +140,7 @@ func TestRenameOnWindows(t *testing.T) {
 	// Must not create temporary files when target doesn't exist
 	assert.NoError(t, InstallUpdate(ctx, &Options{Beta: true, Output: path}))
 
-	files, err := ioutil.ReadDir(testDir)
+	files, err := os.ReadDir(testDir)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(files))
 
@@ -152,7 +151,7 @@ func TestRenameOnWindows(t *testing.T) {
 	assert.NoError(t, cmdWait.Start())
 
 	assert.NoError(t, InstallUpdate(ctx, &Options{Beta: false, Output: path}))
-	files, err = ioutil.ReadDir(testDir)
+	files, err = os.ReadDir(testDir)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(files))
 
@@ -189,7 +188,7 @@ func TestRenameOnWindows(t *testing.T) {
 
 	// Updating when the "old" executable is running must produce a random "old" file
 	assert.NoError(t, InstallUpdate(ctx, &Options{Beta: true, Output: path}))
-	files, err = ioutil.ReadDir(testDir)
+	files, err = os.ReadDir(testDir)
 	assert.NoError(t, err)
 	assert.Equal(t, 3, len(files))
 
diff --git a/cmd/serve/dlna/data/assets_vfsdata.go b/cmd/serve/dlna/data/assets_vfsdata.go
index 642be7b01fa70..84fb623ad18ef 100644
--- a/cmd/serve/dlna/data/assets_vfsdata.go
+++ b/cmd/serve/dlna/data/assets_vfsdata.go
@@ -9,7 +9,6 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	pathpkg "path"
@@ -152,7 +151,7 @@ func (f *vfsgen۰CompressedFile) Read(p []byte) (n int, err error) {
 	}
 	if f.grPos < f.seekPos {
 		// Fast-forward.
-		_, err = io.CopyN(ioutil.Discard, f.gr, f.seekPos-f.grPos)
+		_, err = io.CopyN(io.Discard, f.gr, f.seekPos-f.grPos)
 		if err != nil {
 			return 0, err
 		}
diff --git a/cmd/serve/dlna/data/data.go b/cmd/serve/dlna/data/data.go
index a5a92880cb814..f0ee92f51f0e1 100644
--- a/cmd/serve/dlna/data/data.go
+++ b/cmd/serve/dlna/data/data.go
@@ -6,7 +6,7 @@ package data
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"text/template"
 
 	"github.com/rclone/rclone/fs"
@@ -21,7 +21,7 @@ func GetTemplate() (tpl *template.Template, err error) {
 
 	defer fs.CheckClose(templateFile, &err)
 
-	templateBytes, err := ioutil.ReadAll(templateFile)
+	templateBytes, err := io.ReadAll(templateFile)
 	if err != nil {
 		return nil, fmt.Errorf("get template read: %w", err)
 	}
diff --git a/cmd/serve/dlna/dlna_test.go b/cmd/serve/dlna/dlna_test.go
index 3a3d2fa4be417..389e3ba3c2c15 100644
--- a/cmd/serve/dlna/dlna_test.go
+++ b/cmd/serve/dlna/dlna_test.go
@@ -5,7 +5,7 @@ import (
 	"context"
 	"fmt"
 	"html"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"os"
 	"strings"
@@ -60,7 +60,7 @@ func TestRootSCPD(t *testing.T) {
 	resp, err := http.DefaultClient.Do(req)
 	require.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	// Make sure that the SCPD contains a CDS service.
 	require.Contains(t, string(body),
@@ -80,7 +80,7 @@ func TestServeContent(t *testing.T) {
 	require.NoError(t, err)
 	defer fs.CheckClose(resp.Body, &err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	actualContents, err := ioutil.ReadAll(resp.Body)
+	actualContents, err := io.ReadAll(resp.Body)
 	assert.NoError(t, err)
 
 	// Now compare the contents with the golden file.
@@ -90,7 +90,7 @@ func TestServeContent(t *testing.T) {
 	goldenReader, err := goldenFile.Open(os.O_RDONLY)
 	assert.NoError(t, err)
 	defer fs.CheckClose(goldenReader, &err)
-	goldenContents, err := ioutil.ReadAll(goldenReader)
+	goldenContents, err := io.ReadAll(goldenReader)
 	assert.NoError(t, err)
 
 	require.Equal(t, goldenContents, actualContents)
@@ -119,7 +119,7 @@ func TestContentDirectoryBrowseMetadata(t *testing.T) {
 	resp, err := http.DefaultClient.Do(req)
 	require.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	// should contain an appropriate URN
 	require.Contains(t, string(body), "urn:schemas-upnp-org:service:ContentDirectory:1")
@@ -145,7 +145,7 @@ func TestMediaReceiverRegistrarService(t *testing.T) {
 	resp, err := http.DefaultClient.Do(req)
 	require.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	require.Contains(t, string(body), "")
 }
@@ -173,7 +173,7 @@ func TestContentDirectoryBrowseDirectChildren(t *testing.T) {
 	resp, err := http.DefaultClient.Do(req)
 	require.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	// expect video.mp4, video.srt, video.en.srt URLs to be in the DIDL
 	require.Contains(t, string(body), "/r/video.mp4")
@@ -201,7 +201,7 @@ func TestContentDirectoryBrowseDirectChildren(t *testing.T) {
 	resp, err = http.DefaultClient.Do(req)
 	require.NoError(t, err)
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	body, err = ioutil.ReadAll(resp.Body)
+	body, err = io.ReadAll(resp.Body)
 	require.NoError(t, err)
 	// expect video.mp4, video.srt, URLs to be in the DIDL
 	require.Contains(t, string(body), "/r/subdir/video.mp4")
diff --git a/cmd/serve/docker/docker_test.go b/cmd/serve/docker/docker_test.go
index 82f2170160895..4346b2f9212cb 100644
--- a/cmd/serve/docker/docker_test.go
+++ b/cmd/serve/docker/docker_test.go
@@ -8,7 +8,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net"
 	"net/http"
 	"os"
@@ -280,7 +280,7 @@ func (a *APIClient) request(path string, in, out interface{}, wantErr bool) {
 	}
 	assert.Equal(t, wantStatus, res.StatusCode)
 
-	dataOut, err = ioutil.ReadAll(res.Body)
+	dataOut, err = io.ReadAll(res.Body)
 	require.NoError(t, err)
 	err = res.Body.Close()
 	require.NoError(t, err)
@@ -389,11 +389,11 @@ func testMountAPI(t *testing.T, sockAddr string) {
 	assert.Contains(t, res, "volume is in use")
 
 	text := []byte("banana")
-	err = ioutil.WriteFile(filepath.Join(mount1, "txt"), text, 0644)
+	err = os.WriteFile(filepath.Join(mount1, "txt"), text, 0644)
 	assert.NoError(t, err)
 	time.Sleep(tempDelay)
 
-	text2, err := ioutil.ReadFile(filepath.Join(path1, "txt"))
+	text2, err := os.ReadFile(filepath.Join(path1, "txt"))
 	assert.NoError(t, err)
 	if runtime.GOOS != "windows" {
 		// this check sometimes fails on windows - ignore
diff --git a/cmd/serve/docker/driver.go b/cmd/serve/docker/driver.go
index 6b1db258cd9b0..dfdb190f9953f 100644
--- a/cmd/serve/docker/driver.go
+++ b/cmd/serve/docker/driver.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -329,7 +328,7 @@ func (drv *Driver) saveState() error {
 	ctx := context.Background()
 	retries := fs.GetConfig(ctx).LowLevelRetries
 	for i := 0; i <= retries; i++ {
-		err = ioutil.WriteFile(drv.statePath, data, 0600)
+		err = os.WriteFile(drv.statePath, data, 0600)
 		if err == nil {
 			return nil
 		}
@@ -342,7 +341,7 @@ func (drv *Driver) saveState() error {
 func (drv *Driver) restoreState(ctx context.Context) error {
 	fs.Debugf(nil, "Restore state from %s", drv.statePath)
 
-	data, err := ioutil.ReadFile(drv.statePath)
+	data, err := os.ReadFile(drv.statePath)
 	if os.IsNotExist(err) {
 		return nil
 	}
diff --git a/cmd/serve/docker/serve.go b/cmd/serve/docker/serve.go
index 94a1ce45bae7c..9fa6e56421b2d 100644
--- a/cmd/serve/docker/serve.go
+++ b/cmd/serve/docker/serve.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
@@ -93,7 +92,7 @@ func writeSpecFile(addr, proto, specDir string) (string, error) {
 	}
 	specFile := filepath.Join(specDir, "rclone.spec")
 	url := fmt.Sprintf("%s://%s", proto, addr)
-	if err := ioutil.WriteFile(specFile, []byte(url), 0644); err != nil {
+	if err := os.WriteFile(specFile, []byte(url), 0644); err != nil {
 		return "", err
 	}
 	fs.Debugf(nil, "Plugin spec has been written to %s", specFile)
diff --git a/cmd/serve/http/data/assets_vfsdata.go b/cmd/serve/http/data/assets_vfsdata.go
index 1324e34e4d6a6..69bc52ef763f9 100644
--- a/cmd/serve/http/data/assets_vfsdata.go
+++ b/cmd/serve/http/data/assets_vfsdata.go
@@ -10,7 +10,6 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	pathpkg "path"
@@ -112,7 +111,7 @@ func (f *vfsgen۰CompressedFile) Read(p []byte) (n int, err error) {
 	}
 	if f.grPos < f.seekPos {
 		// Fast-forward.
-		_, err = io.CopyN(ioutil.Discard, f.gr, f.seekPos-f.grPos)
+		_, err = io.CopyN(io.Discard, f.gr, f.seekPos-f.grPos)
 		if err != nil {
 			return 0, err
 		}
diff --git a/cmd/serve/http/data/data.go b/cmd/serve/http/data/data.go
index 075acf8b79f19..d0ab355b5db2b 100644
--- a/cmd/serve/http/data/data.go
+++ b/cmd/serve/http/data/data.go
@@ -7,7 +7,8 @@ package data
 import (
 	"fmt"
 	"html/template"
-	"io/ioutil"
+	"io"
+	"os"
 	"time"
 
 	"github.com/spf13/pflag"
@@ -70,7 +71,7 @@ func GetTemplate(tmpl string) (tpl *template.Template, err error) {
 
 		defer fs.CheckClose(templateFile, &err)
 
-		templateBytes, err := ioutil.ReadAll(templateFile)
+		templateBytes, err := io.ReadAll(templateFile)
 		if err != nil {
 			return nil, fmt.Errorf("get template read: %w", err)
 		}
@@ -78,7 +79,7 @@ func GetTemplate(tmpl string) (tpl *template.Template, err error) {
 		templateString = string(templateBytes)
 
 	} else {
-		templateFile, err := ioutil.ReadFile(tmpl)
+		templateFile, err := os.ReadFile(tmpl)
 		if err != nil {
 			return nil, fmt.Errorf("get template open: %w", err)
 		}
diff --git a/cmd/serve/http/http_test.go b/cmd/serve/http/http_test.go
index 148b5d22adebc..0c77e4b61aeea 100644
--- a/cmd/serve/http/http_test.go
+++ b/cmd/serve/http/http_test.go
@@ -3,8 +3,9 @@ package http
 import (
 	"context"
 	"flag"
-	"io/ioutil"
+	"io"
 	"net/http"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -91,10 +92,10 @@ func TestInit(t *testing.T) {
 func checkGolden(t *testing.T, fileName string, got []byte) {
 	if *updateGolden {
 		t.Logf("Updating golden file %q", fileName)
-		err := ioutil.WriteFile(fileName, got, 0666)
+		err := os.WriteFile(fileName, got, 0666)
 		require.NoError(t, err)
 	} else {
-		want, err := ioutil.ReadFile(fileName)
+		want, err := os.ReadFile(fileName)
 		require.NoError(t, err)
 		wants := strings.Split(string(want), "\n")
 		gots := strings.Split(string(got), "\n")
@@ -210,7 +211,7 @@ func TestGET(t *testing.T) {
 		resp, err := http.DefaultClient.Do(req)
 		require.NoError(t, err)
 		assert.Equal(t, test.Status, resp.StatusCode, test.Golden)
-		body, err := ioutil.ReadAll(resp.Body)
+		body, err := io.ReadAll(resp.Body)
 		require.NoError(t, err)
 
 		// Check we got a Last-Modified header and that it is a valid date
diff --git a/cmd/serve/httplib/httplib.go b/cmd/serve/httplib/httplib.go
index 10d8eb1ecd597..d8a625ea0a892 100644
--- a/cmd/serve/httplib/httplib.go
+++ b/cmd/serve/httplib/httplib.go
@@ -10,10 +10,10 @@ import (
 	"encoding/base64"
 	"fmt"
 	"html/template"
-	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
@@ -315,7 +315,7 @@ func NewServer(handler http.Handler, opt *Options) *Server {
 			log.Fatalf("Can't use --client-ca without --cert and --key")
 		}
 		certpool := x509.NewCertPool()
-		pem, err := ioutil.ReadFile(s.Opt.ClientCA)
+		pem, err := os.ReadFile(s.Opt.ClientCA)
 		if err != nil {
 			log.Fatalf("Failed to read client certificate authority: %v", err)
 		}
diff --git a/cmd/serve/sftp/server.go b/cmd/serve/sftp/server.go
index d1fa92bf8369b..b094ad48f55f4 100644
--- a/cmd/serve/sftp/server.go
+++ b/cmd/serve/sftp/server.go
@@ -17,7 +17,6 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
@@ -311,7 +310,7 @@ func (s *server) Close() {
 }
 
 func loadPrivateKey(keyPath string) (ssh.Signer, error) {
-	privateBytes, err := ioutil.ReadFile(keyPath)
+	privateBytes, err := os.ReadFile(keyPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load private key: %w", err)
 	}
@@ -326,7 +325,7 @@ func loadPrivateKey(keyPath string) (ssh.Signer, error) {
 // the public key of a received connection
 // with the entries in the authorized_keys file.
 func loadAuthorizedKeys(authorizedKeysPath string) (authorizedKeysMap map[string]struct{}, err error) {
-	authorizedKeysBytes, err := ioutil.ReadFile(authorizedKeysPath)
+	authorizedKeysBytes, err := os.ReadFile(authorizedKeysPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load authorized keys: %w", err)
 	}
@@ -369,7 +368,7 @@ func makeRSASSHKeyPair(bits int, pubKeyPath, privateKeyPath string) (err error)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0644)
+	return os.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0644)
 }
 
 // makeECDSASSHKeyPair make a pair of public and private keys for ECDSA SSH access.
@@ -401,7 +400,7 @@ func makeECDSASSHKeyPair(pubKeyPath, privateKeyPath string) (err error) {
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0644)
+	return os.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0644)
 }
 
 // makeEd25519SSHKeyPair make a pair of public and private keys for Ed25519 SSH access.
@@ -433,5 +432,5 @@ func makeEd25519SSHKeyPair(pubKeyPath, privateKeyPath string) (err error) {
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0644)
+	return os.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0644)
 }
diff --git a/cmd/serve/webdav/webdav_test.go b/cmd/serve/webdav/webdav_test.go
index b6ee3259a0096..bb95b07931775 100644
--- a/cmd/serve/webdav/webdav_test.go
+++ b/cmd/serve/webdav/webdav_test.go
@@ -11,7 +11,7 @@ package webdav
 import (
 	"context"
 	"flag"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"os"
 	"strings"
@@ -134,10 +134,10 @@ func TestHTTPFunction(t *testing.T) {
 func checkGolden(t *testing.T, fileName string, got []byte) {
 	if *updateGolden {
 		t.Logf("Updating golden file %q", fileName)
-		err := ioutil.WriteFile(fileName, got, 0666)
+		err := os.WriteFile(fileName, got, 0666)
 		require.NoError(t, err)
 	} else {
-		want, err := ioutil.ReadFile(fileName)
+		want, err := os.ReadFile(fileName)
 		require.NoError(t, err, "problem")
 		wants := strings.Split(string(want), "\n")
 		gots := strings.Split(string(got), "\n")
@@ -253,7 +253,7 @@ func HelpTestGET(t *testing.T, testURL string) {
 		resp, err := http.DefaultClient.Do(req)
 		require.NoError(t, err)
 		assert.Equal(t, test.Status, resp.StatusCode, test.Golden)
-		body, err := ioutil.ReadAll(resp.Body)
+		body, err := io.ReadAll(resp.Body)
 		require.NoError(t, err)
 
 		checkGolden(t, test.Golden, body)
diff --git a/cmd/version/version.go b/cmd/version/version.go
index 25a17aa41b980..32416f89a7d8e 100644
--- a/cmd/version/version.go
+++ b/cmd/version/version.go
@@ -4,7 +4,7 @@ package version
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"strings"
 	"time"
@@ -95,7 +95,7 @@ func GetVersion(url string) (v *semver.Version, vs string, date time.Time, err e
 	if resp.StatusCode != http.StatusOK {
 		return v, vs, date, errors.New(resp.Status)
 	}
-	bodyBytes, err := ioutil.ReadAll(resp.Body)
+	bodyBytes, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return v, vs, date, err
 	}
diff --git a/cmd/version/version_test.go b/cmd/version/version_test.go
index d481105382cfe..fa20a0a9cde8e 100644
--- a/cmd/version/version_test.go
+++ b/cmd/version/version_test.go
@@ -1,7 +1,6 @@
 package version
 
 import (
-	"io/ioutil"
 	"os"
 	"runtime"
 	"testing"
@@ -13,7 +12,7 @@ import (
 
 func TestVersionWorksWithoutAccessibleConfigFile(t *testing.T) {
 	// create temp config file
-	tempFile, err := ioutil.TempFile("", "unreadable_config.conf")
+	tempFile, err := os.CreateTemp("", "unreadable_config.conf")
 	assert.NoError(t, err)
 	path := tempFile.Name()
 	defer func() {
diff --git a/cmdtest/cmdtest_test.go b/cmdtest/cmdtest_test.go
index 1d0c8487eff12..62f51f28a4978 100644
--- a/cmdtest/cmdtest_test.go
+++ b/cmdtest/cmdtest_test.go
@@ -6,7 +6,6 @@
 package cmdtest
 
 import (
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -121,7 +120,7 @@ func removeTestEnvironment(t *testing.T) {
 
 // createTestFile creates the file testFolder/name
 func createTestFile(name string, t *testing.T) string {
-	err := ioutil.WriteFile(testFolder+"/"+name, []byte("content_of_"+name), 0666)
+	err := os.WriteFile(testFolder+"/"+name, []byte("content_of_"+name), 0666)
 	require.NoError(t, err)
 	return testFolder + "/" + name
 }
diff --git a/fs/accounting/accounting_test.go b/fs/accounting/accounting_test.go
index 53cb09a7b9fe1..4b3a876df071b 100644
--- a/fs/accounting/accounting_test.go
+++ b/fs/accounting/accounting_test.go
@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"strings"
 	"testing"
 	"unicode/utf8"
@@ -29,7 +28,7 @@ var (
 
 func TestNewAccountSizeName(t *testing.T) {
 	ctx := context.Background()
-	in := ioutil.NopCloser(bytes.NewBuffer([]byte{1}))
+	in := io.NopCloser(bytes.NewBuffer([]byte{1}))
 	stats := NewStats(ctx)
 	acc := newAccountSizeName(context.Background(), stats, in, 1, "test")
 	assert.Equal(t, in, acc.in)
@@ -44,7 +43,7 @@ func TestNewAccountSizeName(t *testing.T) {
 
 func TestAccountWithBuffer(t *testing.T) {
 	ctx := context.Background()
-	in := ioutil.NopCloser(bytes.NewBuffer([]byte{1}))
+	in := io.NopCloser(bytes.NewBuffer([]byte{1}))
 
 	stats := NewStats(ctx)
 	acc := newAccountSizeName(ctx, stats, in, -1, "test")
@@ -68,7 +67,7 @@ func TestAccountGetUpdateReader(t *testing.T) {
 	ctx := context.Background()
 	test := func(doClose bool) func(t *testing.T) {
 		return func(t *testing.T) {
-			in := ioutil.NopCloser(bytes.NewBuffer([]byte{1}))
+			in := io.NopCloser(bytes.NewBuffer([]byte{1}))
 			stats := NewStats(ctx)
 			acc := newAccountSizeName(ctx, stats, in, 1, "test")
 
@@ -80,7 +79,7 @@ func TestAccountGetUpdateReader(t *testing.T) {
 				require.NoError(t, acc.Close())
 			}
 
-			in2 := ioutil.NopCloser(bytes.NewBuffer([]byte{1}))
+			in2 := io.NopCloser(bytes.NewBuffer([]byte{1}))
 			acc.UpdateReader(ctx, in2)
 
 			assert.Equal(t, in2, acc.GetReader())
@@ -95,7 +94,7 @@ func TestAccountGetUpdateReader(t *testing.T) {
 
 func TestAccountRead(t *testing.T) {
 	ctx := context.Background()
-	in := ioutil.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
+	in := io.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
 	stats := NewStats(ctx)
 	acc := newAccountSizeName(ctx, stats, in, 1, "test")
 
@@ -137,7 +136,7 @@ func testAccountWriteTo(t *testing.T, withBuffer bool) {
 	for i := range buf {
 		buf[i] = byte(i % 251)
 	}
-	in := ioutil.NopCloser(bytes.NewBuffer(buf))
+	in := io.NopCloser(bytes.NewBuffer(buf))
 	stats := NewStats(ctx)
 	acc := newAccountSizeName(ctx, stats, in, int64(len(buf)), "test")
 	if withBuffer {
@@ -178,7 +177,7 @@ func TestAccountWriteToWithBuffer(t *testing.T) {
 
 func TestAccountString(t *testing.T) {
 	ctx := context.Background()
-	in := ioutil.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
+	in := io.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
 	stats := NewStats(ctx)
 	acc := newAccountSizeName(ctx, stats, in, 3, "test")
 
@@ -199,13 +198,13 @@ func TestAccountString(t *testing.T) {
 // Test the Accounter interface methods on Account and accountStream
 func TestAccountAccounter(t *testing.T) {
 	ctx := context.Background()
-	in := ioutil.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
+	in := io.NopCloser(bytes.NewBuffer([]byte{1, 2, 3}))
 	stats := NewStats(ctx)
 	acc := newAccountSizeName(ctx, stats, in, 3, "test")
 
 	assert.True(t, in == acc.OldStream())
 
-	in2 := ioutil.NopCloser(bytes.NewBuffer([]byte{2, 3, 4}))
+	in2 := io.NopCloser(bytes.NewBuffer([]byte{2, 3, 4}))
 
 	acc.SetStream(in2)
 	assert.True(t, in2 == acc.OldStream())
@@ -228,7 +227,7 @@ func TestAccountAccounter(t *testing.T) {
 	assert.Equal(t, []byte{1, 2}, buf[:n])
 
 	// Test that we can get another accountstream out
-	in3 := ioutil.NopCloser(bytes.NewBuffer([]byte{3, 1, 2}))
+	in3 := io.NopCloser(bytes.NewBuffer([]byte{3, 1, 2}))
 	r2 := as.WrapStream(in3)
 	as2, ok := r2.(Accounter)
 	require.True(t, ok)
@@ -268,7 +267,7 @@ func TestAccountMaxTransfer(t *testing.T) {
 		ci.CutoffMode = oldMode
 	}()
 
-	in := ioutil.NopCloser(bytes.NewBuffer(make([]byte, 100)))
+	in := io.NopCloser(bytes.NewBuffer(make([]byte, 100)))
 	stats := NewStats(ctx)
 	acc := newAccountSizeName(ctx, stats, in, 1, "test")
 
@@ -312,7 +311,7 @@ func TestAccountMaxTransferWriteTo(t *testing.T) {
 		ci.CutoffMode = oldMode
 	}()
 
-	in := ioutil.NopCloser(readers.NewPatternReader(1024))
+	in := io.NopCloser(readers.NewPatternReader(1024))
 	stats := NewStats(ctx)
 	acc := newAccountSizeName(ctx, stats, in, 1, "test")
 
@@ -326,7 +325,7 @@ func TestAccountMaxTransferWriteTo(t *testing.T) {
 func TestAccountReadCtx(t *testing.T) {
 	ctx := context.Background()
 	ctx, cancel := context.WithCancel(ctx)
-	in := ioutil.NopCloser(bytes.NewBuffer(make([]byte, 100)))
+	in := io.NopCloser(bytes.NewBuffer(make([]byte, 100)))
 	stats := NewStats(ctx)
 	acc := newAccountSizeName(ctx, stats, in, 1, "test")
 
diff --git a/fs/asyncreader/asyncreader_test.go b/fs/asyncreader/asyncreader_test.go
index 342494d066f28..2cc3a83896ecd 100644
--- a/fs/asyncreader/asyncreader_test.go
+++ b/fs/asyncreader/asyncreader_test.go
@@ -6,7 +6,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"strings"
 	"sync"
@@ -23,7 +22,7 @@ import (
 func TestAsyncReader(t *testing.T) {
 	ctx := context.Background()
 
-	buf := ioutil.NopCloser(bytes.NewBufferString("Testbuffer"))
+	buf := io.NopCloser(bytes.NewBufferString("Testbuffer"))
 	ar, err := New(ctx, buf, 4)
 	require.NoError(t, err)
 
@@ -48,7 +47,7 @@ func TestAsyncReader(t *testing.T) {
 	require.NoError(t, err)
 
 	// Test Close without reading everything
-	buf = ioutil.NopCloser(bytes.NewBuffer(make([]byte, 50000)))
+	buf = io.NopCloser(bytes.NewBuffer(make([]byte, 50000)))
 	ar, err = New(ctx, buf, 4)
 	require.NoError(t, err)
 	err = ar.Close()
@@ -59,7 +58,7 @@ func TestAsyncReader(t *testing.T) {
 func TestAsyncWriteTo(t *testing.T) {
 	ctx := context.Background()
 
-	buf := ioutil.NopCloser(bytes.NewBufferString("Testbuffer"))
+	buf := io.NopCloser(bytes.NewBufferString("Testbuffer"))
 	ar, err := New(ctx, buf, 4)
 	require.NoError(t, err)
 
@@ -85,7 +84,7 @@ func TestAsyncReaderErrors(t *testing.T) {
 	require.Error(t, err)
 
 	// invalid buffer number
-	buf := ioutil.NopCloser(bytes.NewBufferString("Testbuffer"))
+	buf := io.NopCloser(bytes.NewBufferString("Testbuffer"))
 	_, err = New(ctx, buf, 0)
 	require.Error(t, err)
 	_, err = New(ctx, buf, -1)
@@ -170,7 +169,7 @@ func TestAsyncReaderSizes(t *testing.T) {
 						bufsize := bufsizes[k]
 						read := readmaker.fn(strings.NewReader(text))
 						buf := bufio.NewReaderSize(read, bufsize)
-						ar, _ := New(ctx, ioutil.NopCloser(buf), l)
+						ar, _ := New(ctx, io.NopCloser(buf), l)
 						s := bufreader.fn(ar)
 						// "timeout" expects the Reader to recover, AsyncReader does not.
 						if s != text && readmaker.name != "timeout" {
@@ -211,7 +210,7 @@ func TestAsyncReaderWriteTo(t *testing.T) {
 						bufsize := bufsizes[k]
 						read := readmaker.fn(strings.NewReader(text))
 						buf := bufio.NewReaderSize(read, bufsize)
-						ar, _ := New(ctx, ioutil.NopCloser(buf), l)
+						ar, _ := New(ctx, io.NopCloser(buf), l)
 						dst := &bytes.Buffer{}
 						_, err := ar.WriteTo(dst)
 						if err != nil && err != io.EOF && err != iotest.ErrTimeout {
@@ -272,7 +271,7 @@ func testAsyncReaderClose(t *testing.T, writeto bool) {
 		close(started)
 		if writeto {
 			// exercise the WriteTo path
-			copyN, copyErr = a.WriteTo(ioutil.Discard)
+			copyN, copyErr = a.WriteTo(io.Discard)
 		} else {
 			// exercise the Read path
 			buf := make([]byte, 64*1024)
@@ -327,7 +326,7 @@ func TestAsyncReaderSkipBytes(t *testing.T) {
 				t.Run(fmt.Sprintf("%d", initialRead), func(t *testing.T) {
 					for _, skip := range skips {
 						t.Run(fmt.Sprintf("%d", skip), func(t *testing.T) {
-							ar, err := New(ctx, ioutil.NopCloser(bytes.NewReader(data)), buffers)
+							ar, err := New(ctx, io.NopCloser(bytes.NewReader(data)), buffers)
 							require.NoError(t, err)
 
 							wantSkipFalse := false
diff --git a/fs/config/configfile/configfile.go b/fs/config/configfile/configfile.go
index c0bef0389d73a..11763fd2e7d03 100644
--- a/fs/config/configfile/configfile.go
+++ b/fs/config/configfile/configfile.go
@@ -4,7 +4,6 @@ package configfile
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -113,7 +112,7 @@ func (s *Storage) Save() error {
 	if err != nil {
 		return fmt.Errorf("failed to create config directory: %w", err)
 	}
-	f, err := ioutil.TempFile(dir, name)
+	f, err := os.CreateTemp(dir, name)
 	if err != nil {
 		return fmt.Errorf("failed to create temp file for new config: %w", err)
 	}
diff --git a/fs/config/configfile/configfile_test.go b/fs/config/configfile/configfile_test.go
index 2a85b64856e61..c7a325cce28b5 100644
--- a/fs/config/configfile/configfile_test.go
+++ b/fs/config/configfile/configfile_test.go
@@ -2,7 +2,6 @@ package configfile
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"runtime"
 	"strings"
@@ -30,7 +29,7 @@ fruit = banana
 
 // Fill up a temporary config file with the testdata filename passed in
 func setConfigFile(t *testing.T, data string) func() {
-	out, err := ioutil.TempFile("", "rclone-configfile-test")
+	out, err := os.CreateTemp("", "rclone-configfile-test")
 	require.NoError(t, err)
 	filePath := out.Name()
 
@@ -160,7 +159,7 @@ type = number3
 `, toUnix(buf))
 					t.Run("Save", func(t *testing.T) {
 						require.NoError(t, data.Save())
-						buf, err := ioutil.ReadFile(config.GetConfigPath())
+						buf, err := os.ReadFile(config.GetConfigPath())
 						require.NoError(t, err)
 						assert.Equal(t, `[one]
 fruit = potato
diff --git a/fs/config/crypt.go b/fs/config/crypt.go
index 6792811dcef5c..c1de69e7cf198 100644
--- a/fs/config/crypt.go
+++ b/fs/config/crypt.go
@@ -10,7 +10,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"strings"
@@ -128,7 +127,7 @@ func Decrypt(b io.ReadSeeker) (io.Reader, error) {
 
 	// Encrypted content is base64 encoded.
 	dec := base64.NewDecoder(base64.StdEncoding, r)
-	box, err := ioutil.ReadAll(dec)
+	box, err := io.ReadAll(dec)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load base64 encoded data: %w", err)
 	}
@@ -140,7 +139,7 @@ func Decrypt(b io.ReadSeeker) (io.Reader, error) {
 	for {
 		if envKeyFile := os.Getenv("_RCLONE_CONFIG_KEY_FILE"); len(envKeyFile) > 0 {
 			fs.Debugf(nil, "attempting to obtain configKey from temp file %s", envKeyFile)
-			obscuredKey, err := ioutil.ReadFile(envKeyFile)
+			obscuredKey, err := os.ReadFile(envKeyFile)
 			if err != nil {
 				errRemove := os.Remove(envKeyFile)
 				if errRemove != nil {
@@ -212,7 +211,7 @@ func Encrypt(src io.Reader, dst io.Writer) error {
 	var key [32]byte
 	copy(key[:], configKey[:32])
 
-	data, err := ioutil.ReadAll(src)
+	data, err := io.ReadAll(src)
 	if err != nil {
 		return err
 	}
@@ -256,7 +255,7 @@ func SetConfigPassword(password string) error {
 	}
 	configKey = sha.Sum(nil)
 	if PassConfigKeyForDaemonization {
-		tempFile, err := ioutil.TempFile("", "rclone")
+		tempFile, err := os.CreateTemp("", "rclone")
 		if err != nil {
 			return fmt.Errorf("cannot create temp file to store configKey: %w", err)
 		}
diff --git a/fs/config/ui_test.go b/fs/config/ui_test.go
index 5aec209042d91..0756481d11a8e 100644
--- a/fs/config/ui_test.go
+++ b/fs/config/ui_test.go
@@ -7,7 +7,6 @@ package config_test
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"testing"
 
@@ -37,7 +36,7 @@ func testConfigFile(t *testing.T, options []fs.Option, configFileName string) fu
 	_ = os.Unsetenv("_RCLONE_CONFIG_KEY_FILE")
 	_ = os.Unsetenv("RCLONE_CONFIG_PASS")
 	// create temp config file
-	tempFile, err := ioutil.TempFile("", configFileName)
+	tempFile, err := os.CreateTemp("", configFileName)
 	assert.NoError(t, err)
 	path := tempFile.Name()
 	assert.NoError(t, tempFile.Close())
diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go
index aada638fc5031..6feab8ecc6322 100644
--- a/fs/filter/filter_test.go
+++ b/fs/filter/filter_test.go
@@ -3,7 +3,6 @@ package filter
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 	"sync"
@@ -30,7 +29,7 @@ func TestNewFilterDefault(t *testing.T) {
 
 // testFile creates a temp file with the contents
 func testFile(t *testing.T, contents string) string {
-	out, err := ioutil.TempFile("", "filter_test")
+	out, err := os.CreateTemp("", "filter_test")
 	require.NoError(t, err)
 	defer func() {
 		err := out.Close()
diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go
index 953a313dd59f4..b37fd460c2d6d 100644
--- a/fs/fshttp/http.go
+++ b/fs/fshttp/http.go
@@ -6,12 +6,12 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
-	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
 	"net/http/cookiejar"
 	"net/http/httputil"
+	"os"
 	"sync"
 	"time"
 
@@ -74,7 +74,7 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht
 
 	// Load CA cert
 	if ci.CaCert != "" {
-		caCert, err := ioutil.ReadFile(ci.CaCert)
+		caCert, err := os.ReadFile(ci.CaCert)
 		if err != nil {
 			log.Fatalf("Failed to read --ca-cert: %v", err)
 		}
diff --git a/fs/fspath/path_test.go b/fs/fspath/path_test.go
index e0f72681f8d1e..e1ac7866d6f4b 100644
--- a/fs/fspath/path_test.go
+++ b/fs/fspath/path_test.go
@@ -3,7 +3,6 @@ package fspath
 import (
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -424,7 +423,7 @@ func TestParse(t *testing.T) {
 		if *makeCorpus {
 			// write the test corpus for fuzzing
 			require.NoError(t, os.MkdirAll("corpus", 0777))
-			require.NoError(t, ioutil.WriteFile(fmt.Sprintf("corpus/%02d", testNumber), []byte(test.in), 0666))
+			require.NoError(t, os.WriteFile(fmt.Sprintf("corpus/%02d", testNumber), []byte(test.in), 0666))
 		}
 
 	}
diff --git a/fs/newfs.go b/fs/newfs.go
index 927d278b49eb6..cc921bb650b2e 100644
--- a/fs/newfs.go
+++ b/fs/newfs.go
@@ -6,7 +6,6 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/base64"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -117,7 +116,7 @@ func ConfigString(f Fs) string {
 //
 // No cleanup is performed, the caller must call Purge on the Fs themselves.
 func TemporaryLocalFs(ctx context.Context) (Fs, error) {
-	path, err := ioutil.TempDir("", "rclone-spool")
+	path, err := os.MkdirTemp("", "rclone-spool")
 	if err == nil {
 		err = os.Remove(path)
 	}
diff --git a/fs/object/object.go b/fs/object/object.go
index c88b9da02199e..a2971941d9ea0 100644
--- a/fs/object/object.go
+++ b/fs/object/object.go
@@ -6,7 +6,6 @@ import (
 	"context"
 	"errors"
 	"io"
-	"io/ioutil"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -214,7 +213,7 @@ func (o *MemoryObject) Open(ctx context.Context, options ...fs.OpenOption) (io.R
 			}
 		}
 	}
-	return ioutil.NopCloser(bytes.NewBuffer(content)), nil
+	return io.NopCloser(bytes.NewBuffer(content)), nil
 }
 
 // Update in to the object with the modTime given of the given size
@@ -225,7 +224,7 @@ func (o *MemoryObject) Update(ctx context.Context, in io.Reader, src fs.ObjectIn
 	if size == 0 {
 		o.content = nil
 	} else if size < 0 || int64(cap(o.content)) < size {
-		o.content, err = ioutil.ReadAll(in)
+		o.content, err = io.ReadAll(in)
 	} else {
 		o.content = o.content[:size]
 		_, err = io.ReadFull(in, o.content)
diff --git a/fs/object/object_test.go b/fs/object/object_test.go
index d5cdfe6198fa0..a00cf29a0f446 100644
--- a/fs/object/object_test.go
+++ b/fs/object/object_test.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"io"
-	"io/ioutil"
 	"testing"
 	"time"
 
@@ -110,7 +109,7 @@ func TestMemoryObject(t *testing.T) {
 	assert.Equal(t, newNow, o.ModTime(context.Background()))
 
 	checkOpen := func(rc io.ReadCloser, expected string) {
-		actual, err := ioutil.ReadAll(rc)
+		actual, err := io.ReadAll(rc)
 		assert.NoError(t, err)
 		err = rc.Close()
 		assert.NoError(t, err)
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 59de5aac0c6c9..26db4cc0d46f7 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -11,7 +11,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"net/http"
 	"os"
@@ -1430,7 +1429,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
 
 	if SkipDestructive(ctx, dstFileName, "upload from pipe") {
 		// prevents "broken pipe" errors
-		_, err = io.Copy(ioutil.Discard, in)
+		_, err = io.Copy(io.Discard, in)
 		return nil, err
 	}
 
@@ -1735,12 +1734,12 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
 		defer func() {
 			tr.Done(ctx, err)
 		}()
-		body := ioutil.NopCloser(in) // we let the server close the body
-		in := tr.Account(ctx, body)  // account the transfer (no buffering)
+		body := io.NopCloser(in)    // we let the server close the body
+		in := tr.Account(ctx, body) // account the transfer (no buffering)
 
 		if SkipDestructive(ctx, dstFileName, "upload from pipe") {
 			// prevents "broken pipe" errors
-			_, err = io.Copy(ioutil.Discard, in)
+			_, err = io.Copy(io.Discard, in)
 			return nil, err
 		}
 
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 20fcc61a025ad..8f8229c4f72e1 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -26,7 +26,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -320,7 +319,7 @@ func TestHashSumsWithErrors(t *testing.T) {
 
 func TestHashStream(t *testing.T) {
 	reader := strings.NewReader("")
-	in := ioutil.NopCloser(reader)
+	in := io.NopCloser(reader)
 	out := &bytes.Buffer{}
 	for _, test := range []struct {
 		input      string
@@ -1590,11 +1589,11 @@ func TestRcat(t *testing.T) {
 		data2 := string(make([]byte, ci.StreamingUploadCutoff+1))
 		path2 := prefix + "big_file_from_pipe"
 
-		in := ioutil.NopCloser(strings.NewReader(data1))
+		in := io.NopCloser(strings.NewReader(data1))
 		_, err := operations.Rcat(ctx, r.Fremote, path1, in, t1)
 		require.NoError(t, err)
 
-		in = ioutil.NopCloser(strings.NewReader(data2))
+		in = io.NopCloser(strings.NewReader(data2))
 		_, err = operations.Rcat(ctx, r.Fremote, path2, in, t2)
 		require.NoError(t, err)
 
@@ -1621,15 +1620,15 @@ func TestRcatSize(t *testing.T) {
 	file1 := r.WriteFile("potato1", body, t1)
 	file2 := r.WriteFile("potato2", body, t2)
 	// Test with known length
-	bodyReader := ioutil.NopCloser(strings.NewReader(body))
+	bodyReader := io.NopCloser(strings.NewReader(body))
 	obj, err := operations.RcatSize(ctx, r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime)
 	require.NoError(t, err)
 	assert.Equal(t, int64(len(body)), obj.Size())
 	assert.Equal(t, file1.Path, obj.Remote())
 
 	// Test with unknown length
-	bodyReader = ioutil.NopCloser(strings.NewReader(body)) // reset Reader
-	ioutil.NopCloser(strings.NewReader(body))
+	bodyReader = io.NopCloser(strings.NewReader(body)) // reset Reader
+	io.NopCloser(strings.NewReader(body))
 	obj, err = operations.RcatSize(ctx, r.Fremote, file2.Path, bodyReader, -1, file2.ModTime)
 	require.NoError(t, err)
 	assert.Equal(t, int64(len(body)), obj.Size())
diff --git a/fs/operations/reopen_test.go b/fs/operations/reopen_test.go
index 3a75bc0bcdda0..2a7195b569dd1 100644
--- a/fs/operations/reopen_test.go
+++ b/fs/operations/reopen_test.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"io"
-	"io/ioutil"
 	"testing"
 
 	"github.com/rclone/rclone/fs"
@@ -83,7 +82,7 @@ func TestReOpen(t *testing.T) {
 				assert.NoError(t, err)
 
 				// Check contents read correctly
-				got, err := ioutil.ReadAll(h)
+				got, err := io.ReadAll(h)
 				assert.NoError(t, err)
 				assert.Equal(t, expectedRead, got)
 
@@ -118,7 +117,7 @@ func TestReOpen(t *testing.T) {
 				assert.NoError(t, err)
 
 				// check contents
-				got, err := ioutil.ReadAll(h)
+				got, err := io.ReadAll(h)
 				assert.NoError(t, err)
 				assert.Equal(t, expectedRead, got)
 
@@ -132,7 +131,7 @@ func TestReOpen(t *testing.T) {
 				assert.NoError(t, err)
 
 				// check contents
-				got, err := ioutil.ReadAll(h)
+				got, err := io.ReadAll(h)
 				assert.Equal(t, errorTestError, err)
 				assert.Equal(t, expectedRead[:6], got)
 
diff --git a/fs/rc/rcserver/rcserver_test.go b/fs/rc/rcserver/rcserver_test.go
index 34181b631ed0c..62c5e25374680 100644
--- a/fs/rc/rcserver/rcserver_test.go
+++ b/fs/rc/rcserver/rcserver_test.go
@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -77,7 +76,7 @@ func TestRcServer(t *testing.T) {
 	}
 
 	require.NoError(t, err)
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	_ = resp.Body.Close()
 
 	require.NoError(t, err)
@@ -132,7 +131,7 @@ func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
 			resp := w.Result()
 
 			assert.Equal(t, test.Status, resp.StatusCode)
-			body, err := ioutil.ReadAll(resp.Body)
+			body, err := io.ReadAll(resp.Body)
 			require.NoError(t, err)
 
 			if test.Contains == nil {
diff --git a/fs/rc/webgui/plugins.go b/fs/rc/webgui/plugins.go
index 859e6d9991df8..c51eeab57bfa3 100644
--- a/fs/rc/webgui/plugins.go
+++ b/fs/rc/webgui/plugins.go
@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -112,7 +111,7 @@ func (p *Plugins) readFromFile() (err error) {
 	availablePluginsJSON := filepath.Join(pluginsConfigPath, p.fileName)
 	_, err = os.Stat(availablePluginsJSON)
 	if err == nil {
-		data, err := ioutil.ReadFile(availablePluginsJSON)
+		data, err := os.ReadFile(availablePluginsJSON)
 		if err != nil {
 			return err
 		}
@@ -134,7 +133,7 @@ func (p *Plugins) readFromFile() (err error) {
 func (p *Plugins) addPlugin(pluginName string, packageJSONPath string) (err error) {
 	p.mutex.Lock()
 	defer p.mutex.Unlock()
-	data, err := ioutil.ReadFile(packageJSONPath)
+	data, err := os.ReadFile(packageJSONPath)
 	if err != nil {
 		return err
 	}
@@ -187,7 +186,7 @@ func (p *Plugins) writeToFile() (err error) {
 	if err != nil {
 		fs.Logf(nil, "%s", err)
 	}
-	err = ioutil.WriteFile(availablePluginsJSON, file, 0755)
+	err = os.WriteFile(availablePluginsJSON, file, 0755)
 	if err != nil {
 		fs.Logf(nil, "%s", err)
 	}
diff --git a/fs/rc/webgui/webgui.go b/fs/rc/webgui/webgui.go
index ee05730f7b61f..bcc231d3a6956 100644
--- a/fs/rc/webgui/webgui.go
+++ b/fs/rc/webgui/webgui.go
@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -64,7 +63,7 @@ func CheckAndDownloadWebGUIRelease(checkUpdate bool, forceUpdate bool, fetchURL
 	if err != nil {
 		return fmt.Errorf("error checking for web gui release update, skipping update: %w", err)
 	}
-	dat, err := ioutil.ReadFile(tagPath)
+	dat, err := os.ReadFile(tagPath)
 	tagsMatch := false
 	if err != nil {
 		fs.Errorf(nil, "Error reading tag file at %s ", tagPath)
@@ -129,7 +128,7 @@ func CheckAndDownloadWebGUIRelease(checkUpdate bool, forceUpdate bool, fetchURL
 			fs.Logf(nil, "Downloaded ZIP cannot be deleted")
 		}
 
-		err = ioutil.WriteFile(tagPath, []byte(tag), 0644)
+		err = os.WriteFile(tagPath, []byte(tag), 0644)
 		if err != nil {
 			fs.Infof(nil, "Cannot write tag file. You may be required to redownload the binary next time.")
 		}
diff --git a/fstest/fstest.go b/fstest/fstest.go
index c39ccdec7672c..1c23b537351b7 100644
--- a/fstest/fstest.go
+++ b/fstest/fstest.go
@@ -9,7 +9,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -411,7 +410,7 @@ func Time(timeString string) time.Time {
 
 // LocalRemote creates a temporary directory name for local remotes
 func LocalRemote() (path string, err error) {
-	path, err = ioutil.TempDir("", "rclone")
+	path, err = os.MkdirTemp("", "rclone")
 	if err == nil {
 		// Now remove the directory
 		err = os.Remove(path)
diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go
index 3b9767eaf0f91..f0aca7d3cf0e9 100644
--- a/fstest/fstests/fstests.go
+++ b/fstest/fstests/fstests.go
@@ -12,7 +12,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/bits"
 	"os"
 	"path"
@@ -303,7 +302,7 @@ func ReadObject(ctx context.Context, t *testing.T, obj fs.Object, limit int64, o
 	if limit >= 0 {
 		r = &io.LimitedReader{R: r, N: limit}
 	}
-	contents, err := ioutil.ReadAll(r)
+	contents, err := io.ReadAll(r)
 	require.NoError(t, err, what)
 	err = in.Close()
 	require.NoError(t, err, what)
diff --git a/fstest/run.go b/fstest/run.go
index 8751b3d23ef23..eb9b0c3b58f90 100644
--- a/fstest/run.go
+++ b/fstest/run.go
@@ -29,7 +29,6 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"path"
@@ -99,7 +98,7 @@ func newRun() *Run {
 		r.Fatalf("Failed to open remote %q: %v", *RemoteName, err)
 	}
 
-	r.LocalName, err = ioutil.TempDir("", "rclone")
+	r.LocalName, err = os.MkdirTemp("", "rclone")
 	if err != nil {
 		r.Fatalf("Failed to create temp dir: %v", err)
 	}
@@ -221,7 +220,7 @@ func (r *Run) WriteFile(filePath, content string, t time.Time) Item {
 	if err != nil {
 		r.Fatalf("Failed to make directories %q: %v", dirPath, err)
 	}
-	err = ioutil.WriteFile(filePath, []byte(content), 0600)
+	err = os.WriteFile(filePath, []byte(content), 0600)
 	if err != nil {
 		r.Fatalf("Failed to write file %q: %v", filePath, err)
 	}
diff --git a/fstest/test_all/config.go b/fstest/test_all/config.go
index 62d8c057c85e9..45920e25eaf34 100644
--- a/fstest/test_all/config.go
+++ b/fstest/test_all/config.go
@@ -4,8 +4,8 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
 	"log"
+	"os"
 	"path"
 
 	"github.com/rclone/rclone/fs"
@@ -110,7 +110,7 @@ type Config struct {
 
 // NewConfig reads the config file
 func NewConfig(configFile string) (*Config, error) {
-	d, err := ioutil.ReadFile(configFile)
+	d, err := os.ReadFile(configFile)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read config file: %w", err)
 	}
diff --git a/fstest/test_all/report.go b/fstest/test_all/report.go
index cbc48f5cb2be7..0dd4167d55490 100644
--- a/fstest/test_all/report.go
+++ b/fstest/test_all/report.go
@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"html/template"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -69,7 +68,7 @@ func NewReport() *Report {
 	r.DateTime = r.StartTime.Format(timeFormat)
 
 	// Find previous log directory if possible
-	names, err := ioutil.ReadDir(*outputDir)
+	names, err := os.ReadDir(*outputDir)
 	if err == nil && len(names) > 0 {
 		r.Previous = names[len(names)-1].Name()
 	}
@@ -152,7 +151,7 @@ func (r *Report) LogJSON() {
 	if err != nil {
 		log.Fatalf("Failed to marshal data for index.json: %v", err)
 	}
-	err = ioutil.WriteFile(path.Join(r.LogDir, "index.json"), out, 0666)
+	err = os.WriteFile(path.Join(r.LogDir, "index.json"), out, 0666)
 	if err != nil {
 		log.Fatalf("Failed to write index.json: %v", err)
 	}
diff --git a/lib/encoder/filename/gentable.go b/lib/encoder/filename/gentable.go
index 13c8a7d918cad..2fb5ed8a60d6d 100644
--- a/lib/encoder/filename/gentable.go
+++ b/lib/encoder/filename/gentable.go
@@ -9,8 +9,8 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"math"
+	"os"
 	"strings"
 	"unicode/utf8"
 
@@ -43,7 +43,7 @@ func main() {
 		for i := range histogram[:] {
 			histogram[i] = 0
 		}
-		b, err := ioutil.ReadFile(*indexFile)
+		b, err := os.ReadFile(*indexFile)
 		if err != nil {
 			panic(err)
 		}
diff --git a/lib/file/file_test.go b/lib/file/file_test.go
index 933fb4ca79e20..eab7bf7c146ec 100644
--- a/lib/file/file_test.go
+++ b/lib/file/file_test.go
@@ -3,7 +3,6 @@ package file
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"runtime"
@@ -16,7 +15,7 @@ import (
 // This lists dir and checks the listing is as expected without checking the size
 func checkListingNoSize(t *testing.T, dir string, want []string) {
 	var got []string
-	nodes, err := ioutil.ReadDir(dir)
+	nodes, err := os.ReadDir(dir)
 	require.NoError(t, err)
 	for _, node := range nodes {
 		got = append(got, fmt.Sprintf("%s,%v", node.Name(), node.IsDir()))
@@ -27,10 +26,12 @@ func checkListingNoSize(t *testing.T, dir string, want []string) {
 // This lists dir and checks the listing is as expected
 func checkListing(t *testing.T, dir string, want []string) {
 	var got []string
-	nodes, err := ioutil.ReadDir(dir)
+	nodes, err := os.ReadDir(dir)
 	require.NoError(t, err)
 	for _, node := range nodes {
-		got = append(got, fmt.Sprintf("%s,%d,%v", node.Name(), node.Size(), node.IsDir()))
+		info, err := node.Info()
+		assert.NoError(t, err)
+		got = append(got, fmt.Sprintf("%s,%d,%v", node.Name(), info.Size(), node.IsDir()))
 	}
 	assert.Equal(t, want, got)
 }
diff --git a/lib/http/http.go b/lib/http/http.go
index 527555504e034..0138b0fec8b79 100644
--- a/lib/http/http.go
+++ b/lib/http/http.go
@@ -7,10 +7,10 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -187,7 +187,7 @@ func NewServer(listeners, tlsListeners []net.Listener, opt Options) (Server, err
 			return nil, err
 		}
 		certpool := x509.NewCertPool()
-		pem, err := ioutil.ReadFile(opt.ClientCA)
+		pem, err := os.ReadFile(opt.ClientCA)
 		if err != nil {
 			log.Fatalf("Failed to read client certificate authority: %v", err)
 			return nil, err
diff --git a/lib/http/serve/dir_test.go b/lib/http/serve/dir_test.go
index bdcf3f9486986..be128ae33adc8 100644
--- a/lib/http/serve/dir_test.go
+++ b/lib/http/serve/dir_test.go
@@ -3,7 +3,7 @@ package serve
 import (
 	"errors"
 	"html/template"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -94,7 +94,7 @@ func TestError(t *testing.T) {
 	Error("potato", w, "sausage", err)
 	resp := w.Result()
 	assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "sausage.\n", string(body))
 }
 
@@ -108,7 +108,7 @@ func TestServe(t *testing.T) {
 	d.Serve(w, r)
 	resp := w.Result()
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, `
 
 
diff --git a/lib/http/serve/serve_test.go b/lib/http/serve/serve_test.go
index 934fabe658169..b6cc70976d32b 100644
--- a/lib/http/serve/serve_test.go
+++ b/lib/http/serve/serve_test.go
@@ -1,7 +1,7 @@
 package serve
 
 import (
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -17,7 +17,7 @@ func TestObjectBadMethod(t *testing.T) {
 	Object(w, r, o)
 	resp := w.Result()
 	assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "Method Not Allowed\n", string(body))
 }
 
@@ -30,7 +30,7 @@ func TestObjectHEAD(t *testing.T) {
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	assert.Equal(t, "5", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "", string(body))
 }
 
@@ -43,7 +43,7 @@ func TestObjectGET(t *testing.T) {
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	assert.Equal(t, "5", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "hello", string(body))
 }
 
@@ -58,7 +58,7 @@ func TestObjectRange(t *testing.T) {
 	assert.Equal(t, "3", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
 	assert.Equal(t, "bytes 3-5/10", resp.Header.Get("Content-Range"))
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "345", string(body))
 }
 
@@ -71,6 +71,6 @@ func TestObjectBadRange(t *testing.T) {
 	resp := w.Result()
 	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
 	assert.Equal(t, "10", resp.Header.Get("Content-Length"))
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "Bad Request\n", string(body))
 }
diff --git a/lib/jwtutil/jwtutil.go b/lib/jwtutil/jwtutil.go
index dfb4dc33be799..da3b0e7cb1267 100644
--- a/lib/jwtutil/jwtutil.go
+++ b/lib/jwtutil/jwtutil.go
@@ -10,7 +10,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"strings"
 	"time"
@@ -96,7 +95,7 @@ func Config(id, name string, claims *jws.ClaimSet, header *jws.Header, queryPara
 }
 
 func bodyToString(responseBody io.Reader) (bodyString string, err error) {
-	bodyBytes, err := ioutil.ReadAll(responseBody)
+	bodyBytes, err := io.ReadAll(responseBody)
 	if err != nil {
 		return "", err
 	}
diff --git a/lib/plugin/plugin.go b/lib/plugin/plugin.go
index fb08c942050f8..a57b83c0e1c48 100644
--- a/lib/plugin/plugin.go
+++ b/lib/plugin/plugin.go
@@ -6,7 +6,6 @@ package plugin
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"plugin"
@@ -19,7 +18,7 @@ func init() {
 		return
 	}
 	// Get file names of plugin dir
-	listing, err := ioutil.ReadDir(dir)
+	listing, err := os.ReadDir(dir)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Failed to open plugin directory:", err)
 	}
diff --git a/lib/readers/pattern_reader_test.go b/lib/readers/pattern_reader_test.go
index 10c21d0646747..36f57c9212892 100644
--- a/lib/readers/pattern_reader_test.go
+++ b/lib/readers/pattern_reader_test.go
@@ -2,7 +2,6 @@ package readers
 
 import (
 	"io"
-	"io/ioutil"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -13,7 +12,7 @@ func TestPatternReader(t *testing.T) {
 	b2 := make([]byte, 1)
 
 	r := NewPatternReader(0)
-	b, err := ioutil.ReadAll(r)
+	b, err := io.ReadAll(r)
 	require.NoError(t, err)
 	assert.Equal(t, []byte{}, b)
 	n, err := r.Read(b2)
@@ -21,7 +20,7 @@ func TestPatternReader(t *testing.T) {
 	require.Equal(t, 0, n)
 
 	r = NewPatternReader(10)
-	b, err = ioutil.ReadAll(r)
+	b, err = io.ReadAll(r)
 	require.NoError(t, err)
 	assert.Equal(t, []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, b)
 	n, err = r.Read(b2)
@@ -31,7 +30,7 @@ func TestPatternReader(t *testing.T) {
 
 func TestPatternReaderSeek(t *testing.T) {
 	r := NewPatternReader(1024)
-	b, err := ioutil.ReadAll(r)
+	b, err := io.ReadAll(r)
 	require.NoError(t, err)
 
 	for i := range b {
diff --git a/lib/rest/rest.go b/lib/rest/rest.go
index a424b9d3eeb27..b5f575ce79267 100644
--- a/lib/rest/rest.go
+++ b/lib/rest/rest.go
@@ -11,7 +11,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"net/url"
@@ -44,7 +43,7 @@ func NewClient(c *http.Client) *Client {
 // ReadBody reads resp.Body into result, closing the body
 func ReadBody(resp *http.Response) (result []byte, err error) {
 	defer fs.CheckClose(resp.Body, &err)
-	return ioutil.ReadAll(resp.Body)
+	return io.ReadAll(resp.Body)
 }
 
 // defaultErrorHandler doesn't attempt to parse the http body, just
diff --git a/vfs/file_test.go b/vfs/file_test.go
index 2143e34b6acdb..a2b7423150dc1 100644
--- a/vfs/file_test.go
+++ b/vfs/file_test.go
@@ -3,7 +3,7 @@ package vfs
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"testing"
 
@@ -168,7 +168,7 @@ func fileCheckContents(t *testing.T, file *File) {
 	fd, err := file.Open(os.O_RDONLY)
 	require.NoError(t, err)
 
-	contents, err := ioutil.ReadAll(fd)
+	contents, err := io.ReadAll(fd)
 	require.NoError(t, err)
 	assert.Equal(t, "file1 contents", string(contents))
 
@@ -217,7 +217,7 @@ func TestFileOpenReadUnknownSize(t *testing.T) {
 	assert.Equal(t, int64(0), fd.Size())
 
 	// check the contents are not empty even though size is empty
-	gotContents, err := ioutil.ReadAll(fd)
+	gotContents, err := io.ReadAll(fd)
 	require.NoError(t, err)
 	assert.Equal(t, contents, gotContents)
 	t.Logf("gotContents = %q", gotContents)
diff --git a/vfs/make_open_tests.go b/vfs/make_open_tests.go
index d20a0e290a4a2..d4e64f05838b9 100644
--- a/vfs/make_open_tests.go
+++ b/vfs/make_open_tests.go
@@ -17,7 +17,6 @@ package main
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"strings"
@@ -196,7 +195,7 @@ type openTest struct{
 // combination of flags.  This obeys Unix semantics even on Windows.
 var openTests = []openTest{
 `)
-	f, err := ioutil.TempFile("", "open-test")
+	f, err := os.CreateTemp("", "open-test")
 	if err != nil {
 		log.Fatal(err)
 	}
diff --git a/vfs/read_write_test.go b/vfs/read_write_test.go
index c43bc56ffe1ba..ee8051c86faae 100644
--- a/vfs/read_write_test.go
+++ b/vfs/read_write_test.go
@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"testing"
 	"time"
@@ -620,7 +619,7 @@ func testRWFileHandleOpenTest(t *testing.T, vfs *VFS, test *openTest) {
 	// read the file
 	f, err = vfs.OpenFile(fileName, os.O_RDONLY, 0)
 	require.NoError(t, err)
-	buf, err := ioutil.ReadAll(f)
+	buf, err := io.ReadAll(f)
 	require.NoError(t, err)
 	err = f.Close()
 	require.NoError(t, err)
diff --git a/vfs/test_vfs/test_vfs.go b/vfs/test_vfs/test_vfs.go
index bd11ac02cb63b..955362357a8f8 100644
--- a/vfs/test_vfs/test_vfs.go
+++ b/vfs/test_vfs/test_vfs.go
@@ -7,7 +7,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math"
 	"math/rand"
@@ -112,7 +111,7 @@ func (t *Test) errorf(format string, a ...interface{}) {
 // list test
 func (t *Test) list() {
 	t.logf("list")
-	fis, err := ioutil.ReadDir(t.dir)
+	fis, err := os.ReadDir(t.dir)
 	if err != nil {
 		t.errorf("%s: failed to read directory: %v", t.dir, err)
 		return
diff --git a/vfs/vfs.go b/vfs/vfs.go
index 77ccdf49998de..7e9b0043f4363 100644
--- a/vfs/vfs.go
+++ b/vfs/vfs.go
@@ -23,7 +23,7 @@ package vfs
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"path"
 	"sort"
@@ -697,7 +697,7 @@ func (vfs *VFS) ReadFile(filename string) (b []byte, err error) {
 		return nil, err
 	}
 	defer fs.CheckClose(f, &err)
-	return ioutil.ReadAll(f)
+	return io.ReadAll(f)
 }
 
 // AddVirtual adds the object (file or dir) to the directory cache
diff --git a/vfs/vfscache/downloaders/downloaders_test.go b/vfs/vfscache/downloaders/downloaders_test.go
index fdbd942dcdbbd..9fa42519266eb 100644
--- a/vfs/vfscache/downloaders/downloaders_test.go
+++ b/vfs/vfscache/downloaders/downloaders_test.go
@@ -3,7 +3,6 @@ package downloaders
 import (
 	"context"
 	"io"
-	"io/ioutil"
 	"sync"
 	"testing"
 	"time"
@@ -85,7 +84,7 @@ func TestDownloaders(t *testing.T) {
 	)
 
 	// Write the test file
-	in := ioutil.NopCloser(readers.NewPatternReader(size))
+	in := io.NopCloser(readers.NewPatternReader(size))
 	src, err := operations.RcatSize(ctx, r.Fremote, remote, in, size, time.Now())
 	require.NoError(t, err)
 	assert.Equal(t, size, src.Size())
diff --git a/vfs/vfscache/item_test.go b/vfs/vfscache/item_test.go
index 59561708b433f..676380f11e4e6 100644
--- a/vfs/vfscache/item_test.go
+++ b/vfs/vfscache/item_test.go
@@ -6,7 +6,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"os"
 	"sync"
@@ -42,7 +41,7 @@ func checkObject(t *testing.T, r *fstest.Run, remote string, contents string) {
 	require.NoError(t, err)
 	in, err := obj.Open(context.Background())
 	require.NoError(t, err)
-	buf, err := ioutil.ReadAll(in)
+	buf, err := io.ReadAll(in)
 	require.NoError(t, err)
 	require.NoError(t, in.Close())
 	assert.Equal(t, contents, string(buf))
diff --git a/vfs/vfstest/os.go b/vfs/vfstest/os.go
index a27f3ead2219e..d12f55ddb6b0a 100644
--- a/vfs/vfstest/os.go
+++ b/vfs/vfstest/os.go
@@ -1,7 +1,6 @@
 package vfstest
 
 import (
-	"io/ioutil"
 	"os"
 	"time"
 
@@ -88,12 +87,24 @@ func (r realOs) OpenFile(name string, flags int, perm os.FileMode) (vfs.Handle,
 
 // ReadDir
 func (r realOs) ReadDir(dirname string) ([]os.FileInfo, error) {
-	return ioutil.ReadDir(dirname)
+	entries, err := os.ReadDir(dirname)
+	if err != nil {
+		return nil, err
+	}
+	infos := make([]os.FileInfo, 0, len(entries))
+	for _, entry := range entries {
+		info, err := entry.Info()
+		if err != nil {
+			return nil, err
+		}
+		infos = append(infos, info)
+	}
+	return infos, nil
 }
 
 // ReadFile
 func (r realOs) ReadFile(filename string) (b []byte, err error) {
-	return ioutil.ReadFile(filename)
+	return os.ReadFile(filename)
 }
 
 // Remove
diff --git a/vfs/vfstest/read.go b/vfs/vfstest/read.go
index 469c5459f5391..6b34a771a4fa6 100644
--- a/vfs/vfstest/read.go
+++ b/vfs/vfstest/read.go
@@ -2,7 +2,6 @@ package vfstest
 
 import (
 	"io"
-	"io/ioutil"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -89,7 +88,7 @@ func TestReadSeek(t *testing.T) {
 	_, err = fd.Seek(5, io.SeekStart)
 	assert.NoError(t, err)
 
-	buf, err := ioutil.ReadAll(fd)
+	buf, err := io.ReadAll(fd)
 	assert.NoError(t, err)
 	assert.Equal(t, buf, []byte("HELLO"))
 
@@ -97,7 +96,7 @@ func TestReadSeek(t *testing.T) {
 	_, err = fd.Seek(10, io.SeekStart)
 	assert.NoError(t, err)
 
-	buf, err = ioutil.ReadAll(fd)
+	buf, err = io.ReadAll(fd)
 	assert.NoError(t, err)
 	assert.Equal(t, buf, []byte(""))
 
@@ -105,7 +104,7 @@ func TestReadSeek(t *testing.T) {
 	_, err = fd.Seek(1000000, io.SeekStart)
 	assert.NoError(t, err)
 
-	buf, err = ioutil.ReadAll(fd)
+	buf, err = io.ReadAll(fd)
 	assert.NoError(t, err)
 	assert.Equal(t, buf, []byte(""))
 
@@ -113,7 +112,7 @@ func TestReadSeek(t *testing.T) {
 	_, err = fd.Seek(0, io.SeekStart)
 	assert.NoError(t, err)
 
-	buf, err = ioutil.ReadAll(fd)
+	buf, err = io.ReadAll(fd)
 	assert.NoError(t, err)
 	assert.Equal(t, buf, []byte("helloHELLO"))
 
diff --git a/vfs/vfstest/submount.go b/vfs/vfstest/submount.go
index 7c28de8a9d38a..d8e8fb6ff6581 100644
--- a/vfs/vfstest/submount.go
+++ b/vfs/vfstest/submount.go
@@ -7,7 +7,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -100,7 +99,7 @@ func (r *Run) startMountSubProcess() {
 // Find a free path to run the mount on
 func findMountPath() string {
 	if runtime.GOOS != "windows" {
-		mountPath, err := ioutil.TempDir("", "rclonefs-mount")
+		mountPath, err := os.MkdirTemp("", "rclonefs-mount")
 		if err != nil {
 			log.Fatalf("Failed to create mount dir: %v", err)
 		}

From d452f502c392cc002158059599b6b69bc7354b21 Mon Sep 17 00:00:00 2001
From: x3-apptech <66947598+x3-apptech@users.noreply.github.com>
Date: Mon, 7 Nov 2022 19:45:04 +0800
Subject: [PATCH 354/560] cmd: Enable SIGINFO (Ctrl-T) handler on FreeBSD,
 NetBSD, OpenBSD and Dragonfly BSD

---
 cmd/{siginfo_darwin.go => siginfo_bsd.go} | 4 ++--
 cmd/siginfo_others.go                     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename cmd/{siginfo_darwin.go => siginfo_bsd.go} (75%)

diff --git a/cmd/siginfo_darwin.go b/cmd/siginfo_bsd.go
similarity index 75%
rename from cmd/siginfo_darwin.go
rename to cmd/siginfo_bsd.go
index 62e403277169d..fc4063913a9d4 100644
--- a/cmd/siginfo_darwin.go
+++ b/cmd/siginfo_bsd.go
@@ -1,5 +1,5 @@
-//go:build darwin
-// +build darwin
+//go:build darwin || freebsd || netbsd || dragonfly || openbsd
+// +build darwin freebsd netbsd dragonfly openbsd
 
 package cmd
 
diff --git a/cmd/siginfo_others.go b/cmd/siginfo_others.go
index a2266f6321b27..680dba1b50ff6 100644
--- a/cmd/siginfo_others.go
+++ b/cmd/siginfo_others.go
@@ -1,5 +1,5 @@
-//go:build !darwin
-// +build !darwin
+//go:build !darwin && !freebsd && !netbsd && !dragonfly && !openbsd
+// +build !darwin,!freebsd,!netbsd,!dragonfly,!openbsd
 
 package cmd
 

From 528fc899fb63c71c1e46f8b6ae189338ae693a25 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 4 Nov 2022 12:08:41 +0100
Subject: [PATCH 355/560] ncdu: fallback to sort by name also for sort by
 average size

---
 cmd/ncdu/ncdu.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go
index d56f69bfc527e..2fd26fd6f61ae 100644
--- a/cmd/ncdu/ncdu.go
+++ b/cmd/ncdu/ncdu.go
@@ -692,13 +692,17 @@ func (ds *ncduSort) Less(i, j int) bool {
 			return iAvgSize < jAvgSize
 		}
 		// if avgSize is equal, sort by size
-		return iattrs.Size < jattrs.Size
+		if iattrs.Size != jattrs.Size {
+			return iattrs.Size < jattrs.Size
+		}
 	case ds.u.sortByAverageSize > 0:
 		if iAvgSize != jAvgSize {
 			return iAvgSize > jAvgSize
 		}
 		// if avgSize is equal, sort by size
-		return iattrs.Size > jattrs.Size
+		if iattrs.Size != jattrs.Size {
+			return iattrs.Size > jattrs.Size
+		}
 	}
 	// if everything equal, sort by name
 	return iname < jname

From 64cdbb67b5abfb5516b4cf87857cb9480dc533c2 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 4 Nov 2022 14:19:25 +0100
Subject: [PATCH 356/560] ncdu: add support for modification time

---
 cmd/ncdu/ncdu.go      | 51 +++++++++++++++++++++++++++++++++++--------
 cmd/ncdu/scan/scan.go | 31 +++++++++++++++++++-------
 2 files changed, 65 insertions(+), 17 deletions(-)

diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go
index 2fd26fd6f61ae..9d89ae2374d73 100644
--- a/cmd/ncdu/ncdu.go
+++ b/cmd/ncdu/ncdu.go
@@ -89,11 +89,12 @@ func helpText() (tr []string) {
 		" ↑,↓ or k,j to Move",
 		" →,l to enter",
 		" ←,h to return",
-		" c toggle counts",
 		" g toggle graph",
+		" c toggle counts",
 		" a toggle average size in directory",
+		" m toggle modified time",
 		" u toggle human-readable format",
-		" n,s,C,A sort by name,size,count,average size",
+		" n,s,C,A,M sort by name,size,count,asize,mtime",
 		" d delete file/directory",
 		" v select file/directory",
 		" V enter visual select mode",
@@ -131,12 +132,14 @@ type UI struct {
 	showGraph          bool          // toggle showing graph
 	showCounts         bool          // toggle showing counts
 	showDirAverageSize bool          // toggle average size
+	showModTime        bool          // toggle showing timestamps
 	humanReadable      bool          // toggle human-readable format
 	visualSelectMode   bool          // toggle visual selection mode
-	sortByName         int8          // +1 for normal, 0 for off, -1 for reverse
-	sortBySize         int8
+	sortByName         int8          // +1 for normal (lexical), 0 for off, -1 for reverse
+	sortBySize         int8          // +1 for normal (largest first), 0 for off, -1 for reverse (smallest first)
 	sortByCount        int8
 	sortByAverageSize  int8
+	sortByModTime      int8              // +1 for normal (newest first), 0 for off, -1 for reverse (oldest first)
 	dirPosMap          map[string]dirPos // store for directory positions
 	selectedEntries    map[string]dirPos // selected entries of current directory
 }
@@ -332,6 +335,7 @@ func (u *UI) hasEmptyDir() bool {
 
 // Draw the current screen
 func (u *UI) Draw() error {
+	ctx := context.Background()
 	w, h := termbox.Size()
 	u.dirListHeight = h - 3
 
@@ -365,7 +369,13 @@ func (u *UI) Draw() error {
 			if y >= h-1 {
 				break
 			}
-			attrs, err := u.d.AttrI(u.sortPerm[n])
+			var attrs scan.Attrs
+			var err error
+			if u.showModTime {
+				attrs, err = u.d.AttrWithModTimeI(ctx, u.sortPerm[n])
+			} else {
+				attrs, err = u.d.AttrI(u.sortPerm[n])
+			}
 			_, isSelected := u.selectedEntries[entry.String()]
 			fg := termbox.ColorWhite
 			if attrs.EntriesHaveErrors {
@@ -421,6 +431,9 @@ func (u *UI) Draw() error {
 					extras += strings.Repeat(" ", len(ss))
 				}
 			}
+			if u.showModTime {
+				extras += attrs.ModTime.Local().Format("2006-01-02 15:04:05") + " "
+			}
 			if showEmptyDir {
 				if attrs.IsDir && attrs.Count == 0 && fileFlag == ' ' {
 					fileFlag = 'e'
@@ -656,8 +669,15 @@ type ncduSort struct {
 // Less is part of sort.Interface.
 func (ds *ncduSort) Less(i, j int) bool {
 	var iAvgSize, jAvgSize float64
-	iattrs, _ := ds.d.AttrI(ds.sortPerm[i])
-	jattrs, _ := ds.d.AttrI(ds.sortPerm[j])
+	var iattrs, jattrs scan.Attrs
+	if ds.u.sortByModTime != 0 {
+		ctx := context.Background()
+		iattrs, _ = ds.d.AttrWithModTimeI(ctx, ds.sortPerm[i])
+		jattrs, _ = ds.d.AttrWithModTimeI(ctx, ds.sortPerm[j])
+	} else {
+		iattrs, _ = ds.d.AttrI(ds.sortPerm[i])
+		jattrs, _ = ds.d.AttrI(ds.sortPerm[j])
+	}
 	iname, jname := ds.entries[ds.sortPerm[i]].Remote(), ds.entries[ds.sortPerm[j]].Remote()
 	if iattrs.Count > 0 {
 		iAvgSize = iattrs.AverageSize()
@@ -679,6 +699,14 @@ func (ds *ncduSort) Less(i, j int) bool {
 		if iattrs.Size != jattrs.Size {
 			return iattrs.Size > jattrs.Size
 		}
+	case ds.u.sortByModTime < 0:
+		if iattrs.ModTime != jattrs.ModTime {
+			return iattrs.ModTime.Before(jattrs.ModTime)
+		}
+	case ds.u.sortByModTime > 0:
+		if iattrs.ModTime != jattrs.ModTime {
+			return iattrs.ModTime.After(jattrs.ModTime)
+		}
 	case ds.u.sortByCount < 0:
 		if iattrs.Count != jattrs.Count {
 			return iattrs.Count < jattrs.Count
@@ -847,8 +875,9 @@ func NewUI(f fs.Fs) *UI {
 		showCounts:         false,
 		showDirAverageSize: false,
 		humanReadable:      true,
-		sortByName:         0, // +1 for normal, 0 for off, -1 for reverse
-		sortBySize:         1,
+		sortByName:         0,
+		sortBySize:         1, // Sort by largest first
+		sortByModTime:      0,
 		sortByCount:        0,
 		dirPosMap:          make(map[string]dirPos),
 		selectedEntries:    make(map[string]dirPos),
@@ -937,6 +966,8 @@ outer:
 					u.enter()
 				case 'c':
 					u.showCounts = !u.showCounts
+				case 'm':
+					u.showModTime = !u.showModTime
 				case 'g':
 					u.showGraph = !u.showGraph
 				case 'a':
@@ -945,6 +976,8 @@ outer:
 					u.toggleSort(&u.sortByName)
 				case 's':
 					u.toggleSort(&u.sortBySize)
+				case 'M':
+					u.toggleSort(&u.sortByModTime)
 				case 'v':
 					u.toggleSelectForCursor()
 				case 'V':
diff --git a/cmd/ncdu/scan/scan.go b/cmd/ncdu/scan/scan.go
index 3b391ff53c110..fb47b335c7b7f 100644
--- a/cmd/ncdu/scan/scan.go
+++ b/cmd/ncdu/scan/scan.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"path"
 	"sync"
+	"time"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/walk"
@@ -31,6 +32,7 @@ type Dir struct {
 // in the total count. They are not included in the size, i.e. treated
 // as empty files, which means the size may be underestimated.
 type Attrs struct {
+	ModTime           time.Time
 	Size              int64
 	Count             int64
 	CountUnknownSize  int64
@@ -193,20 +195,33 @@ func (d *Dir) Attr() (size int64, count int64) {
 	return d.size, d.count
 }
 
-// AttrI returns the size, count and flags for the i-th directory entry
-func (d *Dir) AttrI(i int) (attrs Attrs, err error) {
-	d.mu.Lock()
-	defer d.mu.Unlock()
+// attrI returns the size, count and flags for the i-th directory entry
+func (d *Dir) attrI(i int) (attrs Attrs, err error) {
 	subDir, isDir := d.getDir(i)
-
 	if !isDir {
-		return Attrs{d.entries[i].Size(), 0, 0, false, true, d.entriesHaveErrors}, d.readError
+		return Attrs{time.Time{}, d.entries[i].Size(), 0, 0, false, true, d.entriesHaveErrors}, d.readError
 	}
 	if subDir == nil {
-		return Attrs{0, 0, 0, true, false, false}, nil
+		return Attrs{time.Time{}, 0, 0, 0, true, false, false}, nil
 	}
 	size, count := subDir.Attr()
-	return Attrs{size, count, subDir.countUnknownSize, true, true, subDir.entriesHaveErrors}, subDir.readError
+	return Attrs{time.Time{}, size, count, subDir.countUnknownSize, true, true, subDir.entriesHaveErrors}, subDir.readError
+}
+
+// AttrI returns the size, count and flags for the i-th directory entry
+func (d *Dir) AttrI(i int) (attrs Attrs, err error) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.attrI(i)
+}
+
+// AttrWithModTimeI returns the modtime, size, count and flags for the i-th directory entry
+func (d *Dir) AttrWithModTimeI(ctx context.Context, i int) (attrs Attrs, err error) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	attrs, err = d.attrI(i)
+	attrs.ModTime = d.entries[i].ModTime(ctx)
+	return
 }
 
 // Scan the Fs passed in, returning a root directory channel and an

From 92ffcf9f8618bb3d24a3ada1ab508a2dbe682978 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jo=C3=A3o=20Henrique=20Franco?=
 
Date: Mon, 7 Nov 2022 09:13:23 -0300
Subject: [PATCH 357/560] wasm: fix walltime link error by adding up-to-date
 wasm_exec.js

Solves link error while running rclone's wasm version. Go's `walltime1` function was renamed to `walltime`. This commit updates wasm_exec.js with the new name.
---
 fs/rc/js/wasm_exec.js | 1235 ++++++++++++++++++++++-------------------
 1 file changed, 653 insertions(+), 582 deletions(-)

diff --git a/fs/rc/js/wasm_exec.js b/fs/rc/js/wasm_exec.js
index 8501ae7cd843a..52ce4f76dc0c6 100644
--- a/fs/rc/js/wasm_exec.js
+++ b/fs/rc/js/wasm_exec.js
@@ -2,587 +2,658 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+"use strict";
+
 (() => {
-	// Map multiple JavaScript environments to a single common API,
-	// preferring web standards over Node.js API.
-	//
-	// Environments considered:
-	// - Browsers
-	// - Node.js
-	// - Electron
-	// - Parcel
-
-	if (typeof global !== "undefined") {
-		// global already exists
-	} else if (typeof window !== "undefined") {
-		window.global = window;
-	} else if (typeof self !== "undefined") {
-		self.global = self;
-	} else {
-		throw new Error("cannot export Go (neither global, window nor self is defined)");
-	}
-
-	if (!global.require && typeof require !== "undefined") {
-		global.require = require;
-	}
-
-	if (!global.fs && global.require) {
-		const fs = require("fs");
-		if (Object.keys(fs) !== 0) {
-			global.fs = fs;
-		}
-	}
-
-	const enosys = () => {
-		const err = new Error("not implemented");
-		err.code = "ENOSYS";
-		return err;
-	};
-
-	if (!global.fs) {
-		let outputBuf = "";
-		global.fs = {
-			constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused
-			writeSync(fd, buf) {
-				outputBuf += decoder.decode(buf);
-				const nl = outputBuf.lastIndexOf("\n");
-				if (nl != -1) {
-					console.log(outputBuf.substr(0, nl));
-					outputBuf = outputBuf.substr(nl + 1);
-				}
-				return buf.length;
-			},
-			write(fd, buf, offset, length, position, callback) {
-				if (offset !== 0 || length !== buf.length || position !== null) {
-					callback(enosys());
-					return;
-				}
-				const n = this.writeSync(fd, buf);
-				callback(null, n);
-			},
-			chmod(path, mode, callback) { callback(enosys()); },
-			chown(path, uid, gid, callback) { callback(enosys()); },
-			close(fd, callback) { callback(enosys()); },
-			fchmod(fd, mode, callback) { callback(enosys()); },
-			fchown(fd, uid, gid, callback) { callback(enosys()); },
-			fstat(fd, callback) { callback(enosys()); },
-			fsync(fd, callback) { callback(null); },
-			ftruncate(fd, length, callback) { callback(enosys()); },
-			lchown(path, uid, gid, callback) { callback(enosys()); },
-			link(path, link, callback) { callback(enosys()); },
-			lstat(path, callback) { callback(enosys()); },
-			mkdir(path, perm, callback) { callback(enosys()); },
-			open(path, flags, mode, callback) { callback(enosys()); },
-			read(fd, buffer, offset, length, position, callback) { callback(enosys()); },
-			readdir(path, callback) { callback(enosys()); },
-			readlink(path, callback) { callback(enosys()); },
-			rename(from, to, callback) { callback(enosys()); },
-			rmdir(path, callback) { callback(enosys()); },
-			stat(path, callback) { callback(enosys()); },
-			symlink(path, link, callback) { callback(enosys()); },
-			truncate(path, length, callback) { callback(enosys()); },
-			unlink(path, callback) { callback(enosys()); },
-			utimes(path, atime, mtime, callback) { callback(enosys()); },
-		};
-	}
-
-	if (!global.process) {
-		global.process = {
-			getuid() { return -1; },
-			getgid() { return -1; },
-			geteuid() { return -1; },
-			getegid() { return -1; },
-			getgroups() { throw enosys(); },
-			pid: -1,
-			ppid: -1,
-			umask() { throw enosys(); },
-			cwd() { throw enosys(); },
-			chdir() { throw enosys(); },
-		}
-	}
-
-	if (!global.crypto) {
-		const nodeCrypto = require("crypto");
-		global.crypto = {
-			getRandomValues(b) {
-				nodeCrypto.randomFillSync(b);
-			},
-		};
-	}
-
-	if (!global.performance) {
-		global.performance = {
-			now() {
-				const [sec, nsec] = process.hrtime();
-				return sec * 1000 + nsec / 1000000;
-			},
-		};
-	}
-
-	if (!global.TextEncoder) {
-		global.TextEncoder = require("util").TextEncoder;
-	}
-
-	if (!global.TextDecoder) {
-		global.TextDecoder = require("util").TextDecoder;
-	}
-
-	// End of polyfills for common API.
-
-	const encoder = new TextEncoder("utf-8");
-	const decoder = new TextDecoder("utf-8");
-
-	global.Go = class {
-		constructor() {
-			this.argv = ["js"];
-			this.env = {};
-			this.exit = (code) => {
-				if (code !== 0) {
-					console.warn("exit code:", code);
-				}
-			};
-			this._exitPromise = new Promise((resolve) => {
-				this._resolveExitPromise = resolve;
-			});
-			this._pendingEvent = null;
-			this._scheduledTimeouts = new Map();
-			this._nextCallbackTimeoutID = 1;
-
-			const setInt64 = (addr, v) => {
-				this.mem.setUint32(addr + 0, v, true);
-				this.mem.setUint32(addr + 4, Math.floor(v / 4294967296), true);
-			}
-
-			const getInt64 = (addr) => {
-				const low = this.mem.getUint32(addr + 0, true);
-				const high = this.mem.getInt32(addr + 4, true);
-				return low + high * 4294967296;
-			}
-
-			const loadValue = (addr) => {
-				const f = this.mem.getFloat64(addr, true);
-				if (f === 0) {
-					return undefined;
-				}
-				if (!isNaN(f)) {
-					return f;
-				}
-
-				const id = this.mem.getUint32(addr, true);
-				return this._values[id];
-			}
-
-			const storeValue = (addr, v) => {
-				const nanHead = 0x7FF80000;
-
-				if (typeof v === "number" && v !== 0) {
-					if (isNaN(v)) {
-						this.mem.setUint32(addr + 4, nanHead, true);
-						this.mem.setUint32(addr, 0, true);
-						return;
-					}
-					this.mem.setFloat64(addr, v, true);
-					return;
-				}
-
-				if (v === undefined) {
-					this.mem.setFloat64(addr, 0, true);
-					return;
-				}
-
-				let id = this._ids.get(v);
-				if (id === undefined) {
-					id = this._idPool.pop();
-					if (id === undefined) {
-						id = this._values.length;
-					}
-					this._values[id] = v;
-					this._goRefCounts[id] = 0;
-					this._ids.set(v, id);
-				}
-				this._goRefCounts[id]++;
-				let typeFlag = 0;
-				switch (typeof v) {
-					case "object":
-						if (v !== null) {
-							typeFlag = 1;
-						}
-						break;
-					case "string":
-						typeFlag = 2;
-						break;
-					case "symbol":
-						typeFlag = 3;
-						break;
-					case "function":
-						typeFlag = 4;
-						break;
-				}
-				this.mem.setUint32(addr + 4, nanHead | typeFlag, true);
-				this.mem.setUint32(addr, id, true);
-			}
-
-			const loadSlice = (addr) => {
-				const array = getInt64(addr + 0);
-				const len = getInt64(addr + 8);
-				return new Uint8Array(this._inst.exports.mem.buffer, array, len);
-			}
-
-			const loadSliceOfValues = (addr) => {
-				const array = getInt64(addr + 0);
-				const len = getInt64(addr + 8);
-				const a = new Array(len);
-				for (let i = 0; i < len; i++) {
-					a[i] = loadValue(array + i * 8);
-				}
-				return a;
-			}
-
-			const loadString = (addr) => {
-				const saddr = getInt64(addr + 0);
-				const len = getInt64(addr + 8);
-				return decoder.decode(new DataView(this._inst.exports.mem.buffer, saddr, len));
-			}
-
-			const timeOrigin = Date.now() - performance.now();
-			this.importObject = {
-				go: {
-					// Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters)
-					// may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported
-					// function. A goroutine can switch to a new stack if the current stack is too small (see morestack function).
-					// This changes the SP, thus we have to update the SP used by the imported function.
-
-					// func wasmExit(code int32)
-					"runtime.wasmExit": (sp) => {
-						const code = this.mem.getInt32(sp + 8, true);
-						this.exited = true;
-						delete this._inst;
-						delete this._values;
-						delete this._goRefCounts;
-						delete this._ids;
-						delete this._idPool;
-						this.exit(code);
-					},
-
-					// func wasmWrite(fd uintptr, p unsafe.Pointer, n int32)
-					"runtime.wasmWrite": (sp) => {
-						const fd = getInt64(sp + 8);
-						const p = getInt64(sp + 16);
-						const n = this.mem.getInt32(sp + 24, true);
-						fs.writeSync(fd, new Uint8Array(this._inst.exports.mem.buffer, p, n));
-					},
-
-					// func resetMemoryDataView()
-					"runtime.resetMemoryDataView": (sp) => {
-						this.mem = new DataView(this._inst.exports.mem.buffer);
-					},
-
-					// func nanotime1() int64
-					"runtime.nanotime1": (sp) => {
-						setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000);
-					},
-
-					// func walltime1() (sec int64, nsec int32)
-					"runtime.walltime1": (sp) => {
-						const msec = (new Date).getTime();
-						setInt64(sp + 8, msec / 1000);
-						this.mem.setInt32(sp + 16, (msec % 1000) * 1000000, true);
-					},
-
-					// func scheduleTimeoutEvent(delay int64) int32
-					"runtime.scheduleTimeoutEvent": (sp) => {
-						const id = this._nextCallbackTimeoutID;
-						this._nextCallbackTimeoutID++;
-						this._scheduledTimeouts.set(id, setTimeout(
-							() => {
-								this._resume();
-								while (this._scheduledTimeouts.has(id)) {
-									// for some reason Go failed to register the timeout event, log and try again
-									// (temporary workaround for https://github.com/golang/go/issues/28975)
-									console.warn("scheduleTimeoutEvent: missed timeout event");
-									this._resume();
-								}
-							},
-							getInt64(sp + 8) + 1, // setTimeout has been seen to fire up to 1 millisecond early
-						));
-						this.mem.setInt32(sp + 16, id, true);
-					},
-
-					// func clearTimeoutEvent(id int32)
-					"runtime.clearTimeoutEvent": (sp) => {
-						const id = this.mem.getInt32(sp + 8, true);
-						clearTimeout(this._scheduledTimeouts.get(id));
-						this._scheduledTimeouts.delete(id);
-					},
-
-					// func getRandomData(r []byte)
-					"runtime.getRandomData": (sp) => {
-						crypto.getRandomValues(loadSlice(sp + 8));
-					},
-
-					// func finalizeRef(v ref)
-					"syscall/js.finalizeRef": (sp) => {
-						const id = this.mem.getUint32(sp + 8, true);
-						this._goRefCounts[id]--;
-						if (this._goRefCounts[id] === 0) {
-							const v = this._values[id];
-							this._values[id] = null;
-							this._ids.delete(v);
-							this._idPool.push(id);
-						}
-					},
-
-					// func stringVal(value string) ref
-					"syscall/js.stringVal": (sp) => {
-						storeValue(sp + 24, loadString(sp + 8));
-					},
-
-					// func valueGet(v ref, p string) ref
-					"syscall/js.valueGet": (sp) => {
-						const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16));
-						sp = this._inst.exports.getsp(); // see comment above
-						storeValue(sp + 32, result);
-					},
-
-					// func valueSet(v ref, p string, x ref)
-					"syscall/js.valueSet": (sp) => {
-						Reflect.set(loadValue(sp + 8), loadString(sp + 16), loadValue(sp + 32));
-					},
-
-					// func valueDelete(v ref, p string)
-					"syscall/js.valueDelete": (sp) => {
-						Reflect.deleteProperty(loadValue(sp + 8), loadString(sp + 16));
-					},
-
-					// func valueIndex(v ref, i int) ref
-					"syscall/js.valueIndex": (sp) => {
-						storeValue(sp + 24, Reflect.get(loadValue(sp + 8), getInt64(sp + 16)));
-					},
-
-					// valueSetIndex(v ref, i int, x ref)
-					"syscall/js.valueSetIndex": (sp) => {
-						Reflect.set(loadValue(sp + 8), getInt64(sp + 16), loadValue(sp + 24));
-					},
-
-					// func valueCall(v ref, m string, args []ref) (ref, bool)
-					"syscall/js.valueCall": (sp) => {
-						try {
-							const v = loadValue(sp + 8);
-							const m = Reflect.get(v, loadString(sp + 16));
-							const args = loadSliceOfValues(sp + 32);
-							const result = Reflect.apply(m, v, args);
-							sp = this._inst.exports.getsp(); // see comment above
-							storeValue(sp + 56, result);
-							this.mem.setUint8(sp + 64, 1);
-						} catch (err) {
-							storeValue(sp + 56, err);
-							this.mem.setUint8(sp + 64, 0);
-						}
-					},
-
-					// func valueInvoke(v ref, args []ref) (ref, bool)
-					"syscall/js.valueInvoke": (sp) => {
-						try {
-							const v = loadValue(sp + 8);
-							const args = loadSliceOfValues(sp + 16);
-							const result = Reflect.apply(v, undefined, args);
-							sp = this._inst.exports.getsp(); // see comment above
-							storeValue(sp + 40, result);
-							this.mem.setUint8(sp + 48, 1);
-						} catch (err) {
-							storeValue(sp + 40, err);
-							this.mem.setUint8(sp + 48, 0);
-						}
-					},
-
-					// func valueNew(v ref, args []ref) (ref, bool)
-					"syscall/js.valueNew": (sp) => {
-						try {
-							const v = loadValue(sp + 8);
-							const args = loadSliceOfValues(sp + 16);
-							const result = Reflect.construct(v, args);
-							sp = this._inst.exports.getsp(); // see comment above
-							storeValue(sp + 40, result);
-							this.mem.setUint8(sp + 48, 1);
-						} catch (err) {
-							storeValue(sp + 40, err);
-							this.mem.setUint8(sp + 48, 0);
-						}
-					},
-
-					// func valueLength(v ref) int
-					"syscall/js.valueLength": (sp) => {
-						setInt64(sp + 16, parseInt(loadValue(sp + 8).length));
-					},
-
-					// valuePrepareString(v ref) (ref, int)
-					"syscall/js.valuePrepareString": (sp) => {
-						const str = encoder.encode(String(loadValue(sp + 8)));
-						storeValue(sp + 16, str);
-						setInt64(sp + 24, str.length);
-					},
-
-					// valueLoadString(v ref, b []byte)
-					"syscall/js.valueLoadString": (sp) => {
-						const str = loadValue(sp + 8);
-						loadSlice(sp + 16).set(str);
-					},
-
-					// func valueInstanceOf(v ref, t ref) bool
-					"syscall/js.valueInstanceOf": (sp) => {
-						this.mem.setUint8(sp + 24, (loadValue(sp + 8) instanceof loadValue(sp + 16)) ? 1 : 0);
-					},
-
-					// func copyBytesToGo(dst []byte, src ref) (int, bool)
-					"syscall/js.copyBytesToGo": (sp) => {
-						const dst = loadSlice(sp + 8);
-						const src = loadValue(sp + 32);
-						if (!(src instanceof Uint8Array || src instanceof Uint8ClampedArray)) {
-							this.mem.setUint8(sp + 48, 0);
-							return;
-						}
-						const toCopy = src.subarray(0, dst.length);
-						dst.set(toCopy);
-						setInt64(sp + 40, toCopy.length);
-						this.mem.setUint8(sp + 48, 1);
-					},
-
-					// func copyBytesToJS(dst ref, src []byte) (int, bool)
-					"syscall/js.copyBytesToJS": (sp) => {
-						const dst = loadValue(sp + 8);
-						const src = loadSlice(sp + 16);
-						if (!(dst instanceof Uint8Array || dst instanceof Uint8ClampedArray)) {
-							this.mem.setUint8(sp + 48, 0);
-							return;
-						}
-						const toCopy = src.subarray(0, dst.length);
-						dst.set(toCopy);
-						setInt64(sp + 40, toCopy.length);
-						this.mem.setUint8(sp + 48, 1);
-					},
-
-					"debug": (value) => {
-						console.log(value);
-					},
-				}
-			};
-		}
-
-		async run(instance) {
-			this._inst = instance;
-			this.mem = new DataView(this._inst.exports.mem.buffer);
-			this._values = [ // JS values that Go currently has references to, indexed by reference id
-				NaN,
-				0,
-				null,
-				true,
-				false,
-				global,
-				this,
-			];
-			this._goRefCounts = new Array(this._values.length).fill(Infinity); // number of references that Go has to a JS value, indexed by reference id
-			this._ids = new Map([ // mapping from JS values to reference ids
-				[0, 1],
-				[null, 2],
-				[true, 3],
-				[false, 4],
-				[global, 5],
-				[this, 6],
-			]);
-			this._idPool = [];   // unused ids that have been garbage collected
-			this.exited = false; // whether the Go program has exited
-
-			// Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory.
-			let offset = 4096;
-
-			const strPtr = (str) => {
-				const ptr = offset;
-				const bytes = encoder.encode(str + "\0");
-				new Uint8Array(this.mem.buffer, offset, bytes.length).set(bytes);
-				offset += bytes.length;
-				if (offset % 8 !== 0) {
-					offset += 8 - (offset % 8);
-				}
-				return ptr;
-			};
-
-			const argc = this.argv.length;
-
-			const argvPtrs = [];
-			this.argv.forEach((arg) => {
-				argvPtrs.push(strPtr(arg));
-			});
-			argvPtrs.push(0);
-
-			const keys = Object.keys(this.env).sort();
-			keys.forEach((key) => {
-				argvPtrs.push(strPtr(`${key}=${this.env[key]}`));
-			});
-			argvPtrs.push(0);
-
-			const argv = offset;
-			argvPtrs.forEach((ptr) => {
-				this.mem.setUint32(offset, ptr, true);
-				this.mem.setUint32(offset + 4, 0, true);
-				offset += 8;
-			});
-
-			this._inst.exports.run(argc, argv);
-			if (this.exited) {
-				this._resolveExitPromise();
-			}
-			await this._exitPromise;
-		}
-
-		_resume() {
-			if (this.exited) {
-				throw new Error("Go program has already exited");
-			}
-			this._inst.exports.resume();
-			if (this.exited) {
-				this._resolveExitPromise();
-			}
-		}
-
-		_makeFuncWrapper(id) {
-			const go = this;
-			return function () {
-				const event = { id: id, this: this, args: arguments };
-				go._pendingEvent = event;
-				go._resume();
-				return event.result;
-			};
-		}
-	}
-
-	if (
-		global.require &&
-		global.require.main === module &&
-		global.process &&
-		global.process.versions &&
-		!global.process.versions.electron
-	) {
-		if (process.argv.length < 3) {
-			console.error("usage: go_js_wasm_exec [wasm binary] [arguments]");
-			process.exit(1);
-		}
-
-		const go = new Go();
-		go.argv = process.argv.slice(2);
-		go.env = Object.assign({ TMPDIR: require("os").tmpdir() }, process.env);
-		go.exit = process.exit;
-		WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject).then((result) => {
-			process.on("exit", (code) => { // Node.js exits if no event handler is pending
-				if (code === 0 && !go.exited) {
-					// deadlock, make Go print error and stack traces
-					go._pendingEvent = { id: 0 };
-					go._resume();
-				}
-			});
-			return go.run(result.instance);
-		}).catch((err) => {
-			console.error(err);
-			process.exit(1);
-		});
-	}
+  const enosys = () => {
+    const err = new Error("not implemented");
+    err.code = "ENOSYS";
+    return err;
+  };
+
+  if (!globalThis.fs) {
+    let outputBuf = "";
+    globalThis.fs = {
+      constants: {
+        O_WRONLY: -1,
+        O_RDWR: -1,
+        O_CREAT: -1,
+        O_TRUNC: -1,
+        O_APPEND: -1,
+        O_EXCL: -1,
+      }, // unused
+      writeSync(fd, buf) {
+        outputBuf += decoder.decode(buf);
+        const nl = outputBuf.lastIndexOf("\n");
+        if (nl != -1) {
+          console.log(outputBuf.substring(0, nl));
+          outputBuf = outputBuf.substring(nl + 1);
+        }
+        return buf.length;
+      },
+      write(fd, buf, offset, length, position, callback) {
+        if (offset !== 0 || length !== buf.length || position !== null) {
+          callback(enosys());
+          return;
+        }
+        const n = this.writeSync(fd, buf);
+        callback(null, n);
+      },
+      chmod(path, mode, callback) {
+        callback(enosys());
+      },
+      chown(path, uid, gid, callback) {
+        callback(enosys());
+      },
+      close(fd, callback) {
+        callback(enosys());
+      },
+      fchmod(fd, mode, callback) {
+        callback(enosys());
+      },
+      fchown(fd, uid, gid, callback) {
+        callback(enosys());
+      },
+      fstat(fd, callback) {
+        callback(enosys());
+      },
+      fsync(fd, callback) {
+        callback(null);
+      },
+      ftruncate(fd, length, callback) {
+        callback(enosys());
+      },
+      lchown(path, uid, gid, callback) {
+        callback(enosys());
+      },
+      link(path, link, callback) {
+        callback(enosys());
+      },
+      lstat(path, callback) {
+        callback(enosys());
+      },
+      mkdir(path, perm, callback) {
+        callback(enosys());
+      },
+      open(path, flags, mode, callback) {
+        callback(enosys());
+      },
+      read(fd, buffer, offset, length, position, callback) {
+        callback(enosys());
+      },
+      readdir(path, callback) {
+        callback(enosys());
+      },
+      readlink(path, callback) {
+        callback(enosys());
+      },
+      rename(from, to, callback) {
+        callback(enosys());
+      },
+      rmdir(path, callback) {
+        callback(enosys());
+      },
+      stat(path, callback) {
+        callback(enosys());
+      },
+      symlink(path, link, callback) {
+        callback(enosys());
+      },
+      truncate(path, length, callback) {
+        callback(enosys());
+      },
+      unlink(path, callback) {
+        callback(enosys());
+      },
+      utimes(path, atime, mtime, callback) {
+        callback(enosys());
+      },
+    };
+  }
+
+  if (!globalThis.process) {
+    globalThis.process = {
+      getuid() {
+        return -1;
+      },
+      getgid() {
+        return -1;
+      },
+      geteuid() {
+        return -1;
+      },
+      getegid() {
+        return -1;
+      },
+      getgroups() {
+        throw enosys();
+      },
+      pid: -1,
+      ppid: -1,
+      umask() {
+        throw enosys();
+      },
+      cwd() {
+        throw enosys();
+      },
+      chdir() {
+        throw enosys();
+      },
+    };
+  }
+
+  if (!globalThis.crypto) {
+    throw new Error(
+      "globalThis.crypto is not available, polyfill required (crypto.getRandomValues only)",
+    );
+  }
+
+  if (!globalThis.performance) {
+    throw new Error(
+      "globalThis.performance is not available, polyfill required (performance.now only)",
+    );
+  }
+
+  if (!globalThis.TextEncoder) {
+    throw new Error(
+      "globalThis.TextEncoder is not available, polyfill required",
+    );
+  }
+
+  if (!globalThis.TextDecoder) {
+    throw new Error(
+      "globalThis.TextDecoder is not available, polyfill required",
+    );
+  }
+
+  const encoder = new TextEncoder("utf-8");
+  const decoder = new TextDecoder("utf-8");
+
+  globalThis.Go = class {
+    constructor() {
+      this.argv = ["js"];
+      this.env = {};
+      this.exit = (code) => {
+        if (code !== 0) {
+          console.warn("exit code:", code);
+        }
+      };
+      this._exitPromise = new Promise((resolve) => {
+        this._resolveExitPromise = resolve;
+      });
+      this._pendingEvent = null;
+      this._scheduledTimeouts = new Map();
+      this._nextCallbackTimeoutID = 1;
+
+      const setInt64 = (addr, v) => {
+        this.mem.setUint32(addr + 0, v, true);
+        this.mem.setUint32(addr + 4, Math.floor(v / 4294967296), true);
+      };
+
+      const getInt64 = (addr) => {
+        const low = this.mem.getUint32(addr + 0, true);
+        const high = this.mem.getInt32(addr + 4, true);
+        return low + high * 4294967296;
+      };
+
+      const loadValue = (addr) => {
+        const f = this.mem.getFloat64(addr, true);
+        if (f === 0) {
+          return undefined;
+        }
+        if (!isNaN(f)) {
+          return f;
+        }
+
+        const id = this.mem.getUint32(addr, true);
+        return this._values[id];
+      };
+
+      const storeValue = (addr, v) => {
+        const nanHead = 0x7FF80000;
+
+        if (typeof v === "number" && v !== 0) {
+          if (isNaN(v)) {
+            this.mem.setUint32(addr + 4, nanHead, true);
+            this.mem.setUint32(addr, 0, true);
+            return;
+          }
+          this.mem.setFloat64(addr, v, true);
+          return;
+        }
+
+        if (v === undefined) {
+          this.mem.setFloat64(addr, 0, true);
+          return;
+        }
+
+        let id = this._ids.get(v);
+        if (id === undefined) {
+          id = this._idPool.pop();
+          if (id === undefined) {
+            id = this._values.length;
+          }
+          this._values[id] = v;
+          this._goRefCounts[id] = 0;
+          this._ids.set(v, id);
+        }
+        this._goRefCounts[id]++;
+        let typeFlag = 0;
+        switch (typeof v) {
+          case "object":
+            if (v !== null) {
+              typeFlag = 1;
+            }
+            break;
+          case "string":
+            typeFlag = 2;
+            break;
+          case "symbol":
+            typeFlag = 3;
+            break;
+          case "function":
+            typeFlag = 4;
+            break;
+        }
+        this.mem.setUint32(addr + 4, nanHead | typeFlag, true);
+        this.mem.setUint32(addr, id, true);
+      };
+
+      const loadSlice = (addr) => {
+        const array = getInt64(addr + 0);
+        const len = getInt64(addr + 8);
+        return new Uint8Array(this._inst.exports.mem.buffer, array, len);
+      };
+
+      const loadSliceOfValues = (addr) => {
+        const array = getInt64(addr + 0);
+        const len = getInt64(addr + 8);
+        const a = new Array(len);
+        for (let i = 0; i < len; i++) {
+          a[i] = loadValue(array + i * 8);
+        }
+        return a;
+      };
+
+      const loadString = (addr) => {
+        const saddr = getInt64(addr + 0);
+        const len = getInt64(addr + 8);
+        return decoder.decode(
+          new DataView(this._inst.exports.mem.buffer, saddr, len),
+        );
+      };
+
+      const timeOrigin = Date.now() - performance.now();
+      this.importObject = {
+        go: {
+          // Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters)
+          // may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported
+          // function. A goroutine can switch to a new stack if the current stack is too small (see morestack function).
+          // This changes the SP, thus we have to update the SP used by the imported function.
+
+          // func wasmExit(code int32)
+          "runtime.wasmExit": (sp) => {
+            sp >>>= 0;
+            const code = this.mem.getInt32(sp + 8, true);
+            this.exited = true;
+            delete this._inst;
+            delete this._values;
+            delete this._goRefCounts;
+            delete this._ids;
+            delete this._idPool;
+            this.exit(code);
+          },
+
+          // func wasmWrite(fd uintptr, p unsafe.Pointer, n int32)
+          "runtime.wasmWrite": (sp) => {
+            sp >>>= 0;
+            const fd = getInt64(sp + 8);
+            const p = getInt64(sp + 16);
+            const n = this.mem.getInt32(sp + 24, true);
+            fs.writeSync(
+              fd,
+              new Uint8Array(this._inst.exports.mem.buffer, p, n),
+            );
+          },
+
+          // func resetMemoryDataView()
+          "runtime.resetMemoryDataView": (sp) => {
+            sp >>>= 0;
+            this.mem = new DataView(this._inst.exports.mem.buffer);
+          },
+
+          // func nanotime1() int64
+          "runtime.nanotime1": (sp) => {
+            sp >>>= 0;
+            setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000);
+          },
+
+          // func walltime() (sec int64, nsec int32)
+          "runtime.walltime": (sp) => {
+            sp >>>= 0;
+            const msec = (new Date()).getTime();
+            setInt64(sp + 8, msec / 1000);
+            this.mem.setInt32(sp + 16, (msec % 1000) * 1000000, true);
+          },
+
+          // func scheduleTimeoutEvent(delay int64) int32
+          "runtime.scheduleTimeoutEvent": (sp) => {
+            sp >>>= 0;
+            const id = this._nextCallbackTimeoutID;
+            this._nextCallbackTimeoutID++;
+            this._scheduledTimeouts.set(
+              id,
+              setTimeout(
+                () => {
+                  this._resume();
+                  while (this._scheduledTimeouts.has(id)) {
+                    // for some reason Go failed to register the timeout event, log and try again
+                    // (temporary workaround for https://github.com/golang/go/issues/28975)
+                    console.warn("scheduleTimeoutEvent: missed timeout event");
+                    this._resume();
+                  }
+                },
+                getInt64(sp + 8) + 1, // setTimeout has been seen to fire up to 1 millisecond early
+              ),
+            );
+            this.mem.setInt32(sp + 16, id, true);
+          },
+
+          // func clearTimeoutEvent(id int32)
+          "runtime.clearTimeoutEvent": (sp) => {
+            sp >>>= 0;
+            const id = this.mem.getInt32(sp + 8, true);
+            clearTimeout(this._scheduledTimeouts.get(id));
+            this._scheduledTimeouts.delete(id);
+          },
+
+          // func getRandomData(r []byte)
+          "runtime.getRandomData": (sp) => {
+            sp >>>= 0;
+            crypto.getRandomValues(loadSlice(sp + 8));
+          },
+
+          // func finalizeRef(v ref)
+          "syscall/js.finalizeRef": (sp) => {
+            sp >>>= 0;
+            const id = this.mem.getUint32(sp + 8, true);
+            this._goRefCounts[id]--;
+            if (this._goRefCounts[id] === 0) {
+              const v = this._values[id];
+              this._values[id] = null;
+              this._ids.delete(v);
+              this._idPool.push(id);
+            }
+          },
+
+          // func stringVal(value string) ref
+          "syscall/js.stringVal": (sp) => {
+            sp >>>= 0;
+            storeValue(sp + 24, loadString(sp + 8));
+          },
+
+          // func valueGet(v ref, p string) ref
+          "syscall/js.valueGet": (sp) => {
+            sp >>>= 0;
+            const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16));
+            sp = this._inst.exports.getsp() >>> 0; // see comment above
+            storeValue(sp + 32, result);
+          },
+
+          // func valueSet(v ref, p string, x ref)
+          "syscall/js.valueSet": (sp) => {
+            sp >>>= 0;
+            Reflect.set(
+              loadValue(sp + 8),
+              loadString(sp + 16),
+              loadValue(sp + 32),
+            );
+          },
+
+          // func valueDelete(v ref, p string)
+          "syscall/js.valueDelete": (sp) => {
+            sp >>>= 0;
+            Reflect.deleteProperty(loadValue(sp + 8), loadString(sp + 16));
+          },
+
+          // func valueIndex(v ref, i int) ref
+          "syscall/js.valueIndex": (sp) => {
+            sp >>>= 0;
+            storeValue(
+              sp + 24,
+              Reflect.get(loadValue(sp + 8), getInt64(sp + 16)),
+            );
+          },
+
+          // valueSetIndex(v ref, i int, x ref)
+          "syscall/js.valueSetIndex": (sp) => {
+            sp >>>= 0;
+            Reflect.set(
+              loadValue(sp + 8),
+              getInt64(sp + 16),
+              loadValue(sp + 24),
+            );
+          },
+
+          // func valueCall(v ref, m string, args []ref) (ref, bool)
+          "syscall/js.valueCall": (sp) => {
+            sp >>>= 0;
+            try {
+              const v = loadValue(sp + 8);
+              const m = Reflect.get(v, loadString(sp + 16));
+              const args = loadSliceOfValues(sp + 32);
+              const result = Reflect.apply(m, v, args);
+              sp = this._inst.exports.getsp() >>> 0; // see comment above
+              storeValue(sp + 56, result);
+              this.mem.setUint8(sp + 64, 1);
+            } catch (err) {
+              sp = this._inst.exports.getsp() >>> 0; // see comment above
+              storeValue(sp + 56, err);
+              this.mem.setUint8(sp + 64, 0);
+            }
+          },
+
+          // func valueInvoke(v ref, args []ref) (ref, bool)
+          "syscall/js.valueInvoke": (sp) => {
+            sp >>>= 0;
+            try {
+              const v = loadValue(sp + 8);
+              const args = loadSliceOfValues(sp + 16);
+              const result = Reflect.apply(v, undefined, args);
+              sp = this._inst.exports.getsp() >>> 0; // see comment above
+              storeValue(sp + 40, result);
+              this.mem.setUint8(sp + 48, 1);
+            } catch (err) {
+              sp = this._inst.exports.getsp() >>> 0; // see comment above
+              storeValue(sp + 40, err);
+              this.mem.setUint8(sp + 48, 0);
+            }
+          },
+
+          // func valueNew(v ref, args []ref) (ref, bool)
+          "syscall/js.valueNew": (sp) => {
+            sp >>>= 0;
+            try {
+              const v = loadValue(sp + 8);
+              const args = loadSliceOfValues(sp + 16);
+              const result = Reflect.construct(v, args);
+              sp = this._inst.exports.getsp() >>> 0; // see comment above
+              storeValue(sp + 40, result);
+              this.mem.setUint8(sp + 48, 1);
+            } catch (err) {
+              sp = this._inst.exports.getsp() >>> 0; // see comment above
+              storeValue(sp + 40, err);
+              this.mem.setUint8(sp + 48, 0);
+            }
+          },
+
+          // func valueLength(v ref) int
+          "syscall/js.valueLength": (sp) => {
+            sp >>>= 0;
+            setInt64(sp + 16, parseInt(loadValue(sp + 8).length));
+          },
+
+          // valuePrepareString(v ref) (ref, int)
+          "syscall/js.valuePrepareString": (sp) => {
+            sp >>>= 0;
+            const str = encoder.encode(String(loadValue(sp + 8)));
+            storeValue(sp + 16, str);
+            setInt64(sp + 24, str.length);
+          },
+
+          // valueLoadString(v ref, b []byte)
+          "syscall/js.valueLoadString": (sp) => {
+            sp >>>= 0;
+            const str = loadValue(sp + 8);
+            loadSlice(sp + 16).set(str);
+          },
+
+          // func valueInstanceOf(v ref, t ref) bool
+          "syscall/js.valueInstanceOf": (sp) => {
+            sp >>>= 0;
+            this.mem.setUint8(
+              sp + 24,
+              (loadValue(sp + 8) instanceof loadValue(sp + 16)) ? 1 : 0,
+            );
+          },
+
+          // func copyBytesToGo(dst []byte, src ref) (int, bool)
+          "syscall/js.copyBytesToGo": (sp) => {
+            sp >>>= 0;
+            const dst = loadSlice(sp + 8);
+            const src = loadValue(sp + 32);
+            if (
+              !(src instanceof Uint8Array || src instanceof Uint8ClampedArray)
+            ) {
+              this.mem.setUint8(sp + 48, 0);
+              return;
+            }
+            const toCopy = src.subarray(0, dst.length);
+            dst.set(toCopy);
+            setInt64(sp + 40, toCopy.length);
+            this.mem.setUint8(sp + 48, 1);
+          },
+
+          // func copyBytesToJS(dst ref, src []byte) (int, bool)
+          "syscall/js.copyBytesToJS": (sp) => {
+            sp >>>= 0;
+            const dst = loadValue(sp + 8);
+            const src = loadSlice(sp + 16);
+            if (
+              !(dst instanceof Uint8Array || dst instanceof Uint8ClampedArray)
+            ) {
+              this.mem.setUint8(sp + 48, 0);
+              return;
+            }
+            const toCopy = src.subarray(0, dst.length);
+            dst.set(toCopy);
+            setInt64(sp + 40, toCopy.length);
+            this.mem.setUint8(sp + 48, 1);
+          },
+
+          "debug": (value) => {
+            console.log(value);
+          },
+        },
+      };
+    }
+
+    async run(instance) {
+      if (!(instance instanceof WebAssembly.Instance)) {
+        throw new Error("Go.run: WebAssembly.Instance expected");
+      }
+      this._inst = instance;
+      this.mem = new DataView(this._inst.exports.mem.buffer);
+      this._values = [ // JS values that Go currently has references to, indexed by reference id
+        NaN,
+        0,
+        null,
+        true,
+        false,
+        globalThis,
+        this,
+      ];
+      this._goRefCounts = new Array(this._values.length).fill(Infinity); // number of references that Go has to a JS value, indexed by reference id
+      this._ids = new Map([ // mapping from JS values to reference ids
+        [0, 1],
+        [null, 2],
+        [true, 3],
+        [false, 4],
+        [globalThis, 5],
+        [this, 6],
+      ]);
+      this._idPool = []; // unused ids that have been garbage collected
+      this.exited = false; // whether the Go program has exited
+
+      // Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory.
+      let offset = 4096;
+
+      const strPtr = (str) => {
+        const ptr = offset;
+        const bytes = encoder.encode(str + "\0");
+        new Uint8Array(this.mem.buffer, offset, bytes.length).set(bytes);
+        offset += bytes.length;
+        if (offset % 8 !== 0) {
+          offset += 8 - (offset % 8);
+        }
+        return ptr;
+      };
+
+      const argc = this.argv.length;
+
+      const argvPtrs = [];
+      this.argv.forEach((arg) => {
+        argvPtrs.push(strPtr(arg));
+      });
+      argvPtrs.push(0);
+
+      const keys = Object.keys(this.env).sort();
+      keys.forEach((key) => {
+        argvPtrs.push(strPtr(`${key}=${this.env[key]}`));
+      });
+      argvPtrs.push(0);
+
+      const argv = offset;
+      argvPtrs.forEach((ptr) => {
+        this.mem.setUint32(offset, ptr, true);
+        this.mem.setUint32(offset + 4, 0, true);
+        offset += 8;
+      });
+
+      // The linker guarantees global data starts from at least wasmMinDataAddr.
+      // Keep in sync with cmd/link/internal/ld/data.go:wasmMinDataAddr.
+      const wasmMinDataAddr = 4096 + 8192;
+      if (offset >= wasmMinDataAddr) {
+        throw new Error(
+          "total length of command line and environment variables exceeds limit",
+        );
+      }
+
+      this._inst.exports.run(argc, argv);
+      if (this.exited) {
+        this._resolveExitPromise();
+      }
+      await this._exitPromise;
+    }
+
+    _resume() {
+      if (this.exited) {
+        throw new Error("Go program has already exited");
+      }
+      this._inst.exports.resume();
+      if (this.exited) {
+        this._resolveExitPromise();
+      }
+    }
+
+    _makeFuncWrapper(id) {
+      const go = this;
+      return function () {
+        const event = { id: id, this: this, args: arguments };
+        go._pendingEvent = event;
+        go._resume();
+        return event.result;
+      };
+    }
+  };
 })();

From cafce9618521cc50f3bb94f78f5ea25b971c4eee Mon Sep 17 00:00:00 2001
From: Arnie97 
Date: Sun, 20 Mar 2022 15:40:20 +0800
Subject: [PATCH 358/560] backend/http: parse get responses when no_head is set

---
 backend/http/http.go | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/backend/http/http.go b/backend/http/http.go
index 3e6e83155426d..59a23916e0346 100644
--- a/backend/http/http.go
+++ b/backend/http/http.go
@@ -305,7 +305,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 		fs:     f,
 		remote: remote,
 	}
-	err := o.stat(ctx)
+	err := o.head(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -500,7 +500,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					fs:     f,
 					remote: remote,
 				}
-				switch err := file.stat(ctx); err {
+				switch err := file.head(ctx); err {
 				case nil:
 					add(file)
 				case fs.ErrorNotAFile:
@@ -579,8 +579,8 @@ func (o *Object) url() string {
 	return o.fs.url(o.remote)
 }
 
-// stat updates the info field in the Object
-func (o *Object) stat(ctx context.Context) error {
+// head sends a HEAD request to update info fields in the Object
+func (o *Object) head(ctx context.Context) error {
 	if o.fs.opt.NoHead {
 		o.size = -1
 		o.modTime = timeUnset
@@ -601,6 +601,11 @@ func (o *Object) stat(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to stat: %w", err)
 	}
+	return o.stat(ctx, res)
+}
+
+// stat updates info fields in the Object according to HTTP response headers
+func (o *Object) stat(ctx context.Context, res *http.Response) error {
 	t, err := http.ParseTime(res.Header.Get("Last-Modified"))
 	if err != nil {
 		t = timeUnset
@@ -653,6 +658,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if err != nil {
 		return nil, fmt.Errorf("Open failed: %w", err)
 	}
+
+	if o.fs.opt.NoHead {
+		if err = o.stat(ctx, res); err != nil {
+			return nil, fmt.Errorf("Stat failed: %w", err)
+		}
+	}
 	return res.Body, nil
 }
 

From ebac8545127f074f0a0f0b4d8d511b3466e908c1 Mon Sep 17 00:00:00 2001
From: Arnie97 
Date: Sun, 20 Mar 2022 16:30:46 +0800
Subject: [PATCH 359/560] backend/http: do not update object size based on
 range requests

---
 backend/http/http.go | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/backend/http/http.go b/backend/http/http.go
index 59a23916e0346..498d92e21df71 100644
--- a/backend/http/http.go
+++ b/backend/http/http.go
@@ -601,18 +601,24 @@ func (o *Object) head(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to stat: %w", err)
 	}
-	return o.stat(ctx, res)
+	return o.stat(ctx, res, true)
 }
 
 // stat updates info fields in the Object according to HTTP response headers
-func (o *Object) stat(ctx context.Context, res *http.Response) error {
+func (o *Object) stat(ctx context.Context, res *http.Response, isRangeRequest bool) error {
 	t, err := http.ParseTime(res.Header.Get("Last-Modified"))
 	if err != nil {
 		t = timeUnset
 	}
-	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
-	o.contentType = res.Header.Get("Content-Type")
+
+	// TODO: parse Content-Range for total size
+	// https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
+	if !isRangeRequest {
+		o.size = parseInt64(res.Header.Get("Content-Length"), -1)
+		o.contentType = res.Header.Get("Content-Type")
+	}
+
 	// If NoSlash is set then check ContentType to see if it is a directory
 	if o.fs.opt.NoSlash {
 		mediaType, _, err := mime.ParseMediaType(o.contentType)
@@ -660,7 +666,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 
 	if o.fs.opt.NoHead {
-		if err = o.stat(ctx, res); err != nil {
+		isRangeRequest := len(req.Header.Get("Range")) > 0
+		if err = o.stat(ctx, res, isRangeRequest); err != nil {
 			return nil, fmt.Errorf("Stat failed: %w", err)
 		}
 	}

From 6a5b7664f7d006f010c1d5c6e3bebda4a4cc2963 Mon Sep 17 00:00:00 2001
From: Arnie97 
Date: Sun, 20 Mar 2022 18:12:48 +0800
Subject: [PATCH 360/560] backend/http: support content-range response header

---
 backend/http/http.go               | 26 ++--------
 backend/http/http_internal_test.go | 77 ++++++++++++++++++++++--------
 backend/s3/s3.go                   | 21 ++------
 lib/rest/headers.go                | 39 +++++++++++++++
 lib/rest/headers_test.go           | 41 ++++++++++++++++
 5 files changed, 146 insertions(+), 58 deletions(-)
 create mode 100644 lib/rest/headers.go
 create mode 100644 lib/rest/headers_test.go

diff --git a/backend/http/http.go b/backend/http/http.go
index 498d92e21df71..867c59371bdae 100644
--- a/backend/http/http.go
+++ b/backend/http/http.go
@@ -13,7 +13,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -317,15 +316,6 @@ func (f *Fs) url(remote string) string {
 	return f.endpointURL + rest.URLPathEscape(remote)
 }
 
-// parse s into an int64, on failure return def
-func parseInt64(s string, def int64) int64 {
-	n, e := strconv.ParseInt(s, 10, 64)
-	if e != nil {
-		return def
-	}
-	return n
-}
-
 // Errors returned by parseName
 var (
 	errURLJoinFailed     = errors.New("URLJoin failed")
@@ -601,23 +591,18 @@ func (o *Object) head(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to stat: %w", err)
 	}
-	return o.stat(ctx, res, true)
+	return o.stat(ctx, res)
 }
 
 // stat updates info fields in the Object according to HTTP response headers
-func (o *Object) stat(ctx context.Context, res *http.Response, isRangeRequest bool) error {
+func (o *Object) stat(ctx context.Context, res *http.Response) error {
 	t, err := http.ParseTime(res.Header.Get("Last-Modified"))
 	if err != nil {
 		t = timeUnset
 	}
 	o.modTime = t
-
-	// TODO: parse Content-Range for total size
-	// https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
-	if !isRangeRequest {
-		o.size = parseInt64(res.Header.Get("Content-Length"), -1)
-		o.contentType = res.Header.Get("Content-Type")
-	}
+	o.contentType = res.Header.Get("Content-Type")
+	o.size = rest.ParseSizeFromHeaders(res.Header)
 
 	// If NoSlash is set then check ContentType to see if it is a directory
 	if o.fs.opt.NoSlash {
@@ -666,8 +651,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 
 	if o.fs.opt.NoHead {
-		isRangeRequest := len(req.Header.Get("Range")) > 0
-		if err = o.stat(ctx, res, isRangeRequest); err != nil {
+		if err = o.stat(ctx, res); err != nil {
 			return nil, fmt.Errorf("Stat failed: %w", err)
 		}
 	}
diff --git a/backend/http/http_internal_test.go b/backend/http/http_internal_test.go
index 8bf6be6b66acd..d0aba666e9bd4 100644
--- a/backend/http/http_internal_test.go
+++ b/backend/http/http_internal_test.go
@@ -194,31 +194,66 @@ func TestNewObject(t *testing.T) {
 }
 
 func TestOpen(t *testing.T) {
-	f, tidy := prepare(t)
+	m, tidy := prepareServer(t)
 	defer tidy()
 
-	o, err := f.NewObject(context.Background(), "four/under four.txt")
-	require.NoError(t, err)
+	for _, head := range []bool{false, true} {
+		if !head {
+			m.Set("no_head", "true")
+		}
+		f, err := NewFs(context.Background(), remoteName, "", m)
+		require.NoError(t, err)
 
-	// Test normal read
-	fd, err := o.Open(context.Background())
-	require.NoError(t, err)
-	data, err := io.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	if lineEndSize == 2 {
-		assert.Equal(t, "beetroot\r\n", string(data))
-	} else {
-		assert.Equal(t, "beetroot\n", string(data))
-	}
+		for _, rangeRead := range []bool{false, true} {
+			o, err := f.NewObject(context.Background(), "four/under four.txt")
+			require.NoError(t, err)
 
-	// Test with range request
-	fd, err = o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
-	require.NoError(t, err)
-	data, err = io.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	assert.Equal(t, "eetro", string(data))
+			if !head {
+				// Test mod time is still indeterminate
+				tObj := o.ModTime(context.Background())
+				assert.Equal(t, time.Duration(0), time.Unix(0, 0).Sub(tObj))
+
+				// Test file size is still indeterminate
+				assert.Equal(t, int64(-1), o.Size())
+			}
+
+			var data []byte
+			if !rangeRead {
+				// Test normal read
+				fd, err := o.Open(context.Background())
+				require.NoError(t, err)
+				data, err = io.ReadAll(fd)
+				require.NoError(t, err)
+				require.NoError(t, fd.Close())
+				if lineEndSize == 2 {
+					assert.Equal(t, "beetroot\r\n", string(data))
+				} else {
+					assert.Equal(t, "beetroot\n", string(data))
+				}
+			} else {
+				// Test with range request
+				fd, err := o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
+				require.NoError(t, err)
+				data, err = io.ReadAll(fd)
+				require.NoError(t, err)
+				require.NoError(t, fd.Close())
+				assert.Equal(t, "eetro", string(data))
+			}
+
+			fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt"))
+			require.NoError(t, err)
+			tFile := fi.ModTime()
+
+			// Test the time is always correct on the object after file open
+			tObj := o.ModTime(context.Background())
+			fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
+
+			if !rangeRead {
+				// Test the file size
+				assert.Equal(t, int64(len(data)), o.Size())
+			}
+		}
+	}
 }
 
 func TestMimeType(t *testing.T) {
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index efea72410d4eb..a1a2fdb3b720d 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -4761,23 +4761,12 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options
 		return nil, err
 	}
 
-	contentLength := &resp.ContentLength
-	if resp.Header.Get("Content-Range") != "" {
-		var contentRange = resp.Header.Get("Content-Range")
-		slash := strings.IndexRune(contentRange, '/')
-		if slash >= 0 {
-			i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
-			if err == nil {
-				contentLength = &i
-			} else {
-				fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
-			}
-		} else {
-			fs.Debugf(o, "Failed to find length in %q", contentRange)
-		}
+	contentLength := rest.ParseSizeFromHeaders(resp.Header)
+	if contentLength < 0 {
+		fs.Debugf(o, "Failed to parse file size from headers")
 	}
 
-	lastModified, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
+	lastModified, err := http.ParseTime(resp.Header.Get("Last-Modified"))
 	if err != nil {
 		fs.Debugf(o, "Failed to parse last modified from string %s, %v", resp.Header.Get("Last-Modified"), err)
 	}
@@ -4801,7 +4790,7 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options
 
 	var head = s3.HeadObjectOutput{
 		ETag:               header("Etag"),
-		ContentLength:      contentLength,
+		ContentLength:      &contentLength,
 		LastModified:       &lastModified,
 		Metadata:           metaData,
 		CacheControl:       header("Cache-Control"),
diff --git a/lib/rest/headers.go b/lib/rest/headers.go
new file mode 100644
index 0000000000000..3e52fbdaa1dd6
--- /dev/null
+++ b/lib/rest/headers.go
@@ -0,0 +1,39 @@
+package rest
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+// ParseSizeFromHeaders parses HTTP response headers to get the full file size.
+// Returns -1 if the headers did not exist or were invalid.
+func ParseSizeFromHeaders(headers http.Header) (size int64) {
+	size = -1
+
+	var contentLength = headers.Get("Content-Length")
+	if len(contentLength) != 0 {
+		var err error
+		if size, err = strconv.ParseInt(contentLength, 10, 64); err != nil {
+			return -1
+		}
+	}
+
+	var contentRange = headers.Get("Content-Range")
+	if len(contentRange) == 0 {
+		return size
+	}
+
+	if !strings.HasPrefix(contentRange, "bytes ") {
+		return -1
+	}
+	slash := strings.IndexRune(contentRange, '/')
+	if slash < 0 {
+		return -1
+	}
+	ret, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
+	if err != nil {
+		return -1
+	}
+	return ret
+}
diff --git a/lib/rest/headers_test.go b/lib/rest/headers_test.go
new file mode 100644
index 0000000000000..23400062da657
--- /dev/null
+++ b/lib/rest/headers_test.go
@@ -0,0 +1,41 @@
+package rest
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseSizeFromHeaders(t *testing.T) {
+	testCases := []struct {
+		ContentLength, ContentRange string
+		Size                        int64
+	}{{
+		"", "", -1,
+	}, {
+		"42", "", 42,
+	}, {
+		"42", "invalid", -1,
+	}, {
+		"", "bytes 22-33/42", 42,
+	}, {
+		"12", "bytes 22-33/42", 42,
+	}, {
+		"12", "otherUnit 22-33/42", -1,
+	}, {
+		"12", "bytes 22-33/*", -1,
+	}, {
+		"0", "bytes */42", 42,
+	}}
+	for _, testCase := range testCases {
+		headers := make(http.Header, 2)
+		if len(testCase.ContentLength) > 0 {
+			headers.Set("Content-Length", testCase.ContentLength)
+		}
+		if len(testCase.ContentRange) > 0 {
+			headers.Set("Content-Range", testCase.ContentRange)
+		}
+		assert.Equalf(t, testCase.Size, ParseSizeFromHeaders(headers), "%+v", testCase)
+	}
+}

From 36c37ffec150bbe00c6e7464f1cc172ede243370 Mon Sep 17 00:00:00 2001
From: Arnie97 
Date: Tue, 8 Nov 2022 13:36:22 +0800
Subject: [PATCH 361/560] backend/http: rename stat to decodeMetadata

---
 backend/http/http.go | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/backend/http/http.go b/backend/http/http.go
index 867c59371bdae..518ea411cd8d7 100644
--- a/backend/http/http.go
+++ b/backend/http/http.go
@@ -591,11 +591,11 @@ func (o *Object) head(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to stat: %w", err)
 	}
-	return o.stat(ctx, res)
+	return o.decodeMetadata(ctx, res)
 }
 
-// stat updates info fields in the Object according to HTTP response headers
-func (o *Object) stat(ctx context.Context, res *http.Response) error {
+// decodeMetadata updates info fields in the Object according to HTTP response headers
+func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error {
 	t, err := http.ParseTime(res.Header.Get("Last-Modified"))
 	if err != nil {
 		t = timeUnset
@@ -649,11 +649,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if err != nil {
 		return nil, fmt.Errorf("Open failed: %w", err)
 	}
-
-	if o.fs.opt.NoHead {
-		if err = o.stat(ctx, res); err != nil {
-			return nil, fmt.Errorf("Stat failed: %w", err)
-		}
+	if err = o.decodeMetadata(ctx, res); err != nil {
+		return nil, fmt.Errorf("decodeMetadata failed: %w", err)
 	}
 	return res.Body, nil
 }

From 5fd0abb2b9e936be3d2c726a4aa5dcc0df81ceb9 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 16:05:11 +0000
Subject: [PATCH 362/560] Add x3-apptech to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 2e50aff32700b..1176978bc210d 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -656,3 +656,4 @@ put them back in again.` >}}
   * Philip Harvey 
   * dgouju 
   * Clément Notin 
+  * x3-apptech <66947598+x3-apptech@users.noreply.github.com>

From 09858c0c5a51fe9fa6c8b1380d988616808cfb6d Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 16:05:11 +0000
Subject: [PATCH 363/560] Add Arnie97 to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 1176978bc210d..5bf1292a093cf 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -657,3 +657,4 @@ put them back in again.` >}}
   * dgouju 
   * Clément Notin 
   * x3-apptech <66947598+x3-apptech@users.noreply.github.com>
+  * Arnie97 

From 691159fe9402b965abda4e8a3e15923e71c62812 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 12:13:19 +0000
Subject: [PATCH 364/560] s3: allow Storj to server side copy since it seems to
 work now - fixes #6550

---
 backend/s3/s3.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index a1a2fdb3b720d..1549359068640 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -2937,7 +2937,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return f, fs.ErrorIsFile
 	}
 	if opt.Provider == "Storj" {
-		f.features.Copy = nil
 		f.features.SetTier = false
 		f.features.GetTier = false
 	}

From 65528fd0099ea303ac459f2958ef51a1a0ba2626 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 16:11:34 +0000
Subject: [PATCH 365/560] docs: remove link to rclone slack as it is no longer
 supported

---
 docs/layouts/chrome/menu.html | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/layouts/chrome/menu.html b/docs/layouts/chrome/menu.html
index 85ea1e5435df0..047708bc9493a 100644
--- a/docs/layouts/chrome/menu.html
+++ b/docs/layouts/chrome/menu.html
@@ -33,7 +33,6 @@
     
                  
        Rclone forum
                  
        GitHub project
                  
-       Rclone slack
                  
        Rclone Wiki
                  
        Donate
                  
        @njcw

From e455940f71085d51c551e5b239b7aa6d7c55aeb9 Mon Sep 17 00:00:00 2001
From: Roel Arents <2691308+roelarents@users.noreply.github.com>
Date: Tue, 8 Nov 2022 21:24:06 +0100
Subject: [PATCH 366/560] azureblob: allow emulator account/key override

---
 backend/azureblob/azureblob.go | 10 +++++++++-
 docs/content/azureblob.md      |  8 +++++---
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index e9402ad0e10bc..8c49a50dd9727 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -595,7 +595,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	)
 	switch {
 	case opt.UseEmulator:
-		credential, err := azblob.NewSharedKeyCredential(emulatorAccount, emulatorAccountKey)
+		var actualEmulatorAccount = emulatorAccount
+		if opt.Account != "" {
+			actualEmulatorAccount = opt.Account
+		}
+		var actualEmulatorKey = emulatorAccountKey
+		if opt.Key != "" {
+			actualEmulatorKey = opt.Key
+		}
+		credential, err := azblob.NewSharedKeyCredential(actualEmulatorAccount, actualEmulatorKey)
 		if err != nil {
 			return nil, fmt.Errorf("failed to parse credentials: %w", err)
 		}
diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md
index b9841e0e3256e..5a1e2663fca24 100644
--- a/docs/content/azureblob.md
+++ b/docs/content/azureblob.md
@@ -164,7 +164,7 @@ Here are the Standard options specific to azureblob (Microsoft Azure Blob Storag
 
 Storage Account Name.
 
-Leave blank to use SAS URL or Emulator.
+Leave blank to use SAS URL.
 
 Properties:
 
@@ -198,7 +198,7 @@ Properties:
 
 Storage Account Key.
 
-Leave blank to use SAS URL or Emulator.
+Leave blank to use SAS URL.
 
 Properties:
 
@@ -246,6 +246,8 @@ Uses local storage emulator if provided as 'true'.
 
 Leave blank if using real azure storage endpoint.
 
+Provide `account`, `key`, and `endpoint` if desired to override their _azurite_ defaults.
+
 Properties:
 
 - Config:      use_emulator
@@ -528,6 +530,6 @@ See [List of backends that do not support rclone about](https://rclone.org/overv
 
 You can run rclone with storage emulator (usually _azurite_).
 
-To do this, just set up a new remote with `rclone config` following instructions described in introduction and set `use_emulator` config as `true`. You do not need to provide default account name neither an account key.
+To do this, just set up a new remote with `rclone config` following instructions described in introduction and set `use_emulator` config as `true`. You do not need to provide default account name neither an account key. But you can override them in the _account_ and _key_ options. (Prior to v1.61 they were hard coded to _azurite_'s `devstoreaccount1`.)
 
 Also, if you want to access a storage emulator instance running on a different machine, you can override _Endpoint_ parameter in advanced settings, setting it to `http(s)://:/devstoreaccount1` (e.g. `http://10.254.2.5:10000/devstoreaccount1`).

From 133c006c37192d1a8e3a8f5dec91759822f7d73f Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 20:33:03 +0000
Subject: [PATCH 367/560] Add Roel Arents to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 5bf1292a093cf..a109bf390c3fb 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -658,3 +658,4 @@ put them back in again.` >}}
   * Clément Notin 
   * x3-apptech <66947598+x3-apptech@users.noreply.github.com>
   * Arnie97 
+  * Roel Arents <2691308+roelarents@users.noreply.github.com>

From 3a3bc5a1ae822b5430292b7ecc513045b71f70f8 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 20:31:26 +0000
Subject: [PATCH 368/560] mailru: note that an app password is now needed -
 fixes #6398

---
 backend/mailru/mailru.go |  9 +++++++--
 docs/content/mailru.md   | 21 +++++++++++++++++++--
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go
index 5a630a2d3c1a0..2df923c3c7268 100644
--- a/backend/mailru/mailru.go
+++ b/backend/mailru/mailru.go
@@ -90,8 +90,13 @@ func init() {
 			Help:     "User name (usually email).",
 			Required: true,
 		}, {
-			Name:       "pass",
-			Help:       "Password.",
+			Name: "pass",
+			Help: `Password.
+
+This must be an app password - rclone will not work with your normal
+password. See the Configuration section in the docs for how to make an
+app password.
+`,
 			Required:   true,
 			IsPassword: true,
 		}, {
diff --git a/docs/content/mailru.md b/docs/content/mailru.md
index 96bd5aabb1acd..5814283c8b291 100644
--- a/docs/content/mailru.md
+++ b/docs/content/mailru.md
@@ -24,8 +24,21 @@ Currently it is recommended to disable 2FA on Mail.ru accounts intended for rclo
 
 ## Configuration
 
-Here is an example of making a mailru configuration. First create a Mail.ru Cloud
-account and choose a tariff, then run
+Here is an example of making a mailru configuration.
+
+First create a Mail.ru Cloud account and choose a tariff.
+
+You will need to log in and create an app password for rclone. Rclone
+**will not work** with your normal username and password - it will
+give an error like `oauth2: server response missing access_token`.
+
+- Click on your user icon in the top right
+- Go to Security / "Пароль и безопасность"
+- Click password for apps / "Пароли для внешних приложений"
+- Add the password - give it a name - eg "rclone"
+- Copy the password and use this password below - your normal login password won't work.
+
+Now run
 
     rclone config
 
@@ -51,6 +64,10 @@ User name (usually email)
 Enter a string value. Press Enter for the default ("").
 user> username@mail.ru
 Password
+
+This must be an app password - rclone will not work with your normal
+password. See the Configuration section in the docs for how to make an
+app password.
 y) Yes type in my own password
 g) Generate random password
 y/g> y

From 919e28b8bf1b1da893aae606f11d57f9692d3a26 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 11:44:03 +0000
Subject: [PATCH 369/560] lib/cache: fix alias backend shutting down too soon

Before this patch, when an alias backend was created it would be
renamed to be canonical and in the process Shutdown would be called on
it. This was particularly noticeable with the dropbox backend which
gave this error when uploading files after the backend was Shutdown.

    Failed to copy: upload failed: batcher is shutting down

This patch fixes the cache Rename code not to finalize objects if the
object that is being overwritten is the same as the existing object.

See: https://forum.rclone.org/t/upload-failed-batcher-is-shutting-down/33900
---
 lib/cache/cache.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/cache/cache.go b/lib/cache/cache.go
index 8e03819bc2152..cbcf78796fccf 100644
--- a/lib/cache/cache.go
+++ b/lib/cache/cache.go
@@ -190,9 +190,11 @@ func (c *Cache) Rename(oldKey, newKey string) (value interface{}, found bool) {
 	c.mu.Lock()
 	if newEntry, newFound := c.cache[newKey]; newFound {
 		// If new entry is found use that
-		if _, oldFound := c.cache[oldKey]; oldFound {
-			// If there's an old entry, we drop it and also try shutdown.
-			c.finalize(c.cache[oldKey].value)
+		if oldEntry, oldFound := c.cache[oldKey]; oldFound {
+			// If there's an old entry that is different we must finalize it
+			if newEntry.value != oldEntry.value {
+				c.finalize(c.cache[oldKey].value)
+			}
 		}
 		delete(c.cache, oldKey)
 		value, found = newEntry.value, newFound

From 47b04580db00f805ab0703af62044c49d53db592 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 20:18:58 +0000
Subject: [PATCH 370/560] accounting: make it so we can account directories as
 well as files

---
 fs/accounting/stats.go    | 4 ++--
 fs/accounting/transfer.go | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index 3101ef993ba91..b029cb5901aee 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -675,7 +675,7 @@ func (s *StatsInfo) RetryAfter() time.Time {
 }
 
 // NewCheckingTransfer adds a checking transfer to the stats, from the object.
-func (s *StatsInfo) NewCheckingTransfer(obj fs.Object) *Transfer {
+func (s *StatsInfo) NewCheckingTransfer(obj fs.DirEntry) *Transfer {
 	tr := newCheckingTransfer(s, obj)
 	s.checking.add(tr)
 	return tr
@@ -697,7 +697,7 @@ func (s *StatsInfo) GetTransfers() int64 {
 }
 
 // NewTransfer adds a transfer to the stats from the object.
-func (s *StatsInfo) NewTransfer(obj fs.Object) *Transfer {
+func (s *StatsInfo) NewTransfer(obj fs.DirEntry) *Transfer {
 	tr := newTransfer(s, obj)
 	s.transferring.add(tr)
 	s.startAverageLoop()
diff --git a/fs/accounting/transfer.go b/fs/accounting/transfer.go
index 09d27cf177b47..e6dda29797d61 100644
--- a/fs/accounting/transfer.go
+++ b/fs/accounting/transfer.go
@@ -63,12 +63,12 @@ type Transfer struct {
 }
 
 // newCheckingTransfer instantiates new checking of the object.
-func newCheckingTransfer(stats *StatsInfo, obj fs.Object) *Transfer {
+func newCheckingTransfer(stats *StatsInfo, obj fs.DirEntry) *Transfer {
 	return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), true)
 }
 
 // newTransfer instantiates new transfer.
-func newTransfer(stats *StatsInfo, obj fs.Object) *Transfer {
+func newTransfer(stats *StatsInfo, obj fs.DirEntry) *Transfer {
 	return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), false)
 }
 

From 653bc2372845b58865c25c76ade835a51e1c1084 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 20:19:28 +0000
Subject: [PATCH 371/560] dedupe: count Checks in the stats while scanning for
 duplicates

This allows the user to see rclone has not hung.

See: https://forum.rclone.org/t/dial-tcp-lookup-api-pcloud-com-no-such-host/33910/
---
 fs/operations/dedupe.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/fs/operations/dedupe.go b/fs/operations/dedupe.go
index 32daa9d4d7cba..ca4376a2998fa 100644
--- a/fs/operations/dedupe.go
+++ b/fs/operations/dedupe.go
@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
@@ -285,6 +286,8 @@ func dedupeFindDuplicateDirs(ctx context.Context, f fs.Fs) (duplicateDirs [][]*d
 	ci := fs.GetConfig(ctx)
 	err = walk.ListR(ctx, f, "", true, ci.MaxDepth, walk.ListAll, func(entries fs.DirEntries) error {
 		for _, entry := range entries {
+			tr := accounting.Stats(ctx).NewCheckingTransfer(entry)
+
 			remote := entry.Remote()
 			parentRemote := path.Dir(remote)
 			if parentRemote == "." {
@@ -318,6 +321,7 @@ func dedupeFindDuplicateDirs(ctx context.Context, f fs.Fs) (duplicateDirs [][]*d
 			}
 
 			dirsByID.increment(parent)
+			tr.Done(ctx, nil)
 		}
 		return nil
 	})
@@ -434,6 +438,9 @@ func Deduplicate(ctx context.Context, f fs.Fs, mode DeduplicateMode, byHash bool
 	files := map[string][]fs.Object{}
 	err := walk.ListR(ctx, f, "", true, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
 		entries.ForObject(func(o fs.Object) {
+			tr := accounting.Stats(ctx).NewCheckingTransfer(o)
+			defer tr.Done(ctx, nil)
+
 			var remote string
 			var err error
 			if byHash {

From be6f29930b98566b36ea0a9b011682142d1a2e95 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Wed, 9 Nov 2022 16:36:09 +0000
Subject: [PATCH 372/560] dedupe: make dedupe obey the filters

See: https://forum.rclone.org/t/dial-tcp-lookup-api-pcloud-com-no-such-host/33910/
---
 fs/operations/dedupe.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fs/operations/dedupe.go b/fs/operations/dedupe.go
index ca4376a2998fa..1d3eff9c02f38 100644
--- a/fs/operations/dedupe.go
+++ b/fs/operations/dedupe.go
@@ -284,7 +284,7 @@ func dedupeFindDuplicateDirs(ctx context.Context, f fs.Fs) (duplicateDirs [][]*d
 	dirs := map[string][]*dedupeDir{}
 
 	ci := fs.GetConfig(ctx)
-	err = walk.ListR(ctx, f, "", true, ci.MaxDepth, walk.ListAll, func(entries fs.DirEntries) error {
+	err = walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListAll, func(entries fs.DirEntries) error {
 		for _, entry := range entries {
 			tr := accounting.Stats(ctx).NewCheckingTransfer(entry)
 
@@ -436,7 +436,7 @@ func Deduplicate(ctx context.Context, f fs.Fs, mode DeduplicateMode, byHash bool
 
 	// Now find duplicate files
 	files := map[string][]fs.Object{}
-	err := walk.ListR(ctx, f, "", true, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
+	err := walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
 		entries.ForObject(func(o fs.Object) {
 			tr := accounting.Stats(ctx).NewCheckingTransfer(o)
 			defer tr.Done(ctx, nil)

From b285efb476e5194f32145985e839014fb72bf322 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 15:47:10 +0000
Subject: [PATCH 373/560] mailru: allow timestamps to be before the epoch
 1970-01-01

Fixes #6547
---
 backend/mailru/api/helpers.go | 5 +++++
 backend/mailru/mailru.go      | 9 ++-------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/backend/mailru/api/helpers.go b/backend/mailru/api/helpers.go
index d7164185fe2dd..6ac2b5d9cb65a 100644
--- a/backend/mailru/api/helpers.go
+++ b/backend/mailru/api/helpers.go
@@ -69,6 +69,11 @@ func (w *BinWriter) WritePu64(val int64) {
 	w.b.Write(w.a[:binary.PutUvarint(w.a, uint64(val))])
 }
 
+// WriteP64 writes an signed long as unsigned varint
+func (w *BinWriter) WriteP64(val int64) {
+	w.b.Write(w.a[:binary.PutUvarint(w.a, uint64(val))])
+}
+
 // WriteString writes a zero-terminated string
 func (w *BinWriter) WriteString(str string) {
 	buf := []byte(str)
diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go
index 2df923c3c7268..28fafe7021da9 100644
--- a/backend/mailru/mailru.go
+++ b/backend/mailru/mailru.go
@@ -645,12 +645,7 @@ func (f *Fs) itemToDirEntry(ctx context.Context, item *api.ListItem) (entry fs.D
 		return nil, -1, err
 	}
 
-	mTime := int64(item.Mtime)
-	if mTime < 0 {
-		fs.Debugf(f, "Fixing invalid timestamp %d on mailru file %q", mTime, remote)
-		mTime = 0
-	}
-	modTime := time.Unix(mTime, 0)
+	modTime := time.Unix(int64(item.Mtime), 0)
 
 	isDir, err := f.isDir(item.Kind, remote)
 	if err != nil {
@@ -2062,7 +2057,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error {
 	req.WritePu16(0) // revision
 	req.WriteString(o.fs.opt.Enc.FromStandardPath(o.absPath()))
 	req.WritePu64(o.size)
-	req.WritePu64(o.modTime.Unix())
+	req.WriteP64(o.modTime.Unix())
 	req.WritePu32(0)
 	req.Write(o.mrHash)
 

From b0248e807001434dc05f558729ba39e4aa7d93dc Mon Sep 17 00:00:00 2001
From: Aaron Gokaslan 
Date: Thu, 10 Nov 2022 06:52:59 -0500
Subject: [PATCH 374/560] s3: fix for unchecked err value in s3 listv2

---
 backend/s3/s3.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 1549359068640..cb8b8cd9ea286 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -3159,8 +3159,11 @@ func (f *Fs) newV2List(req *s3.ListObjectsV2Input) bucketLister {
 // Do a V2 listing
 func (ls *v2List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versionIDs []*string, err error) {
 	resp, err = ls.f.c.ListObjectsV2WithContext(ctx, &ls.req)
+	if err != nil {
+		return nil, nil, err
+	}
 	ls.req.ContinuationToken = resp.NextContinuationToken
-	return resp, nil, err
+	return resp, nil, nil
 }
 
 // URL Encode the listings

From 57bde20acdf33617e719858dc8c20300b44156f7 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 10 Nov 2022 12:04:13 +0000
Subject: [PATCH 375/560] Add Aaron Gokaslan to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index a109bf390c3fb..69845ef29b775 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -659,3 +659,4 @@ put them back in again.` >}}
   * x3-apptech <66947598+x3-apptech@users.noreply.github.com>
   * Arnie97 
   * Roel Arents <2691308+roelarents@users.noreply.github.com>
+  * Aaron Gokaslan 

From 458845ce89891820856860162328811b4f6b9adb Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 17:41:27 +0000
Subject: [PATCH 376/560] fs/object: add WithMetadata and WithMimetype to
 static and memory objects

---
 fs/object/object.go | 114 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 95 insertions(+), 19 deletions(-)

diff --git a/fs/object/object.go b/fs/object/object.go
index a2971941d9ea0..93c9236814a2f 100644
--- a/fs/object/object.go
+++ b/fs/object/object.go
@@ -12,20 +12,32 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 )
 
+// StaticObjectInfo is an ObjectInfo which can be constructed from scratch
+type StaticObjectInfo struct {
+	remote   string
+	modTime  time.Time
+	size     int64
+	storable bool
+	hashes   map[hash.Type]string
+	fs       fs.Info
+	meta     fs.Metadata
+	mimeType string
+}
+
 // NewStaticObjectInfo returns a static ObjectInfo
 // If hashes is nil and fs is not nil, the hash map will be replaced with
 // empty hashes of the types supported by the fs.
-func NewStaticObjectInfo(remote string, modTime time.Time, size int64, storable bool, hashes map[hash.Type]string, fs fs.Info) fs.ObjectInfo {
-	info := &staticObjectInfo{
+func NewStaticObjectInfo(remote string, modTime time.Time, size int64, storable bool, hashes map[hash.Type]string, f fs.Info) *StaticObjectInfo {
+	info := &StaticObjectInfo{
 		remote:   remote,
 		modTime:  modTime,
 		size:     size,
 		storable: storable,
 		hashes:   hashes,
-		fs:       fs,
+		fs:       f,
 	}
-	if fs != nil && hashes == nil {
-		set := fs.Hashes().Array()
+	if f != nil && hashes == nil {
+		set := f.Hashes().Array()
 		info.hashes = make(map[hash.Type]string)
 		for _, ht := range set {
 			info.hashes[ht] = ""
@@ -34,22 +46,50 @@ func NewStaticObjectInfo(remote string, modTime time.Time, size int64, storable
 	return info
 }
 
-type staticObjectInfo struct {
-	remote   string
-	modTime  time.Time
-	size     int64
-	storable bool
-	hashes   map[hash.Type]string
-	fs       fs.Info
+// WithMetadata adds meta to the ObjectInfo
+func (i *StaticObjectInfo) WithMetadata(meta fs.Metadata) *StaticObjectInfo {
+	i.meta = meta
+	return i
+}
+
+// WithMimeType adds meta to the ObjectInfo
+func (i *StaticObjectInfo) WithMimeType(mimeType string) *StaticObjectInfo {
+	i.mimeType = mimeType
+	return i
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (i *StaticObjectInfo) Fs() fs.Info {
+	return i.fs
+}
+
+// Remote returns the remote path
+func (i *StaticObjectInfo) Remote() string {
+	return i.remote
 }
 
-func (i *staticObjectInfo) Fs() fs.Info                           { return i.fs }
-func (i *staticObjectInfo) Remote() string                        { return i.remote }
-func (i *staticObjectInfo) String() string                        { return i.remote }
-func (i *staticObjectInfo) ModTime(ctx context.Context) time.Time { return i.modTime }
-func (i *staticObjectInfo) Size() int64                           { return i.size }
-func (i *staticObjectInfo) Storable() bool                        { return i.storable }
-func (i *staticObjectInfo) Hash(ctx context.Context, h hash.Type) (string, error) {
+// String returns a description of the Object
+func (i *StaticObjectInfo) String() string {
+	return i.remote
+}
+
+// ModTime returns the modification date of the file
+func (i *StaticObjectInfo) ModTime(ctx context.Context) time.Time {
+	return i.modTime
+}
+
+// Size returns the size of the file
+func (i *StaticObjectInfo) Size() int64 {
+	return i.size
+}
+
+// Storable says whether this object can be stored
+func (i *StaticObjectInfo) Storable() bool {
+	return i.storable
+}
+
+// Hash returns the requested hash of the contents
+func (i *StaticObjectInfo) Hash(ctx context.Context, h hash.Type) (string, error) {
 	if len(i.hashes) == 0 {
 		return "", hash.ErrUnsupported
 	}
@@ -59,6 +99,24 @@ func (i *staticObjectInfo) Hash(ctx context.Context, h hash.Type) (string, error
 	return "", hash.ErrUnsupported
 }
 
+// Metadata on the object
+func (i *StaticObjectInfo) Metadata(ctx context.Context) (fs.Metadata, error) {
+	return i.meta, nil
+}
+
+// MimeType returns the content type of the Object if
+// known, or "" if not
+func (i *StaticObjectInfo) MimeType(ctx context.Context) string {
+	return i.mimeType
+}
+
+// Check interfaces
+var (
+	_ fs.ObjectInfo = (*StaticObjectInfo)(nil)
+	_ fs.Metadataer = (*StaticObjectInfo)(nil)
+	_ fs.MimeTyper  = (*StaticObjectInfo)(nil)
+)
+
 // MemoryFs is an in memory Fs, it only supports FsInfo and Put
 var MemoryFs memoryFs
 
@@ -133,6 +191,7 @@ type MemoryObject struct {
 	remote  string
 	modTime time.Time
 	content []byte
+	meta    fs.Metadata
 }
 
 // NewMemoryObject returns an in memory Object with the modTime and content passed in
@@ -144,6 +203,12 @@ func NewMemoryObject(remote string, modTime time.Time, content []byte) *MemoryOb
 	}
 }
 
+// WithMetadata adds meta to the MemoryObject
+func (o *MemoryObject) WithMetadata(meta fs.Metadata) *MemoryObject {
+	o.meta = meta
+	return o
+}
+
 // Content returns the underlying buffer
 func (o *MemoryObject) Content() []byte {
 	return o.content
@@ -237,3 +302,14 @@ func (o *MemoryObject) Update(ctx context.Context, in io.Reader, src fs.ObjectIn
 func (o *MemoryObject) Remove(ctx context.Context) error {
 	return errors.New("memoryObject.Remove not supported")
 }
+
+// Metadata on the object
+func (o *MemoryObject) Metadata(ctx context.Context) (fs.Metadata, error) {
+	return o.meta, nil
+}
+
+// Check interfaces
+var (
+	_ fs.Object     = (*MemoryObject)(nil)
+	_ fs.Metadataer = (*MemoryObject)(nil)
+)

From ec2024b9074d00b11c7c9cd411d64ef97402a20f Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 19:33:42 +0000
Subject: [PATCH 377/560] fstest: use WithMetadata / WithMimeType

---
 fstest/fstests/fstests.go | 34 +---------------------------------
 1 file changed, 1 insertion(+), 33 deletions(-)

diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go
index f0aca7d3cf0e9..40fb398513121 100644
--- a/fstest/fstests/fstests.go
+++ b/fstest/fstests/fstests.go
@@ -152,38 +152,6 @@ func retry(t *testing.T, what string, f func() error) {
 	require.NoError(t, err, what)
 }
 
-// An fs.ObjectInfo that can override mime type
-type objectInfoWithMimeType struct {
-	fs.ObjectInfo
-	mimeType string
-	metadata fs.Metadata
-}
-
-// Return a wrapped fs.ObjectInfo which returns the mime type given
-func overrideMimeType(o fs.ObjectInfo, mimeType string, metadata fs.Metadata) fs.ObjectInfo {
-	return &objectInfoWithMimeType{
-		ObjectInfo: o,
-		mimeType:   mimeType,
-		metadata:   metadata,
-	}
-}
-
-// MimeType that was overridden
-func (o *objectInfoWithMimeType) MimeType(ctx context.Context) string {
-	return o.mimeType
-}
-
-// Metadata that was overridden
-func (o *objectInfoWithMimeType) Metadata(ctx context.Context) (fs.Metadata, error) {
-	return o.metadata, nil
-}
-
-// check interfaces
-var (
-	_ fs.MimeTyper  = (*objectInfoWithMimeType)(nil)
-	_ fs.Metadataer = (*objectInfoWithMimeType)(nil)
-)
-
 // check interface
 
 // PutTestContentsMetadata puts file with given contents to the remote and checks it but unlike TestPutLarge doesn't remove
@@ -214,7 +182,7 @@ func PutTestContentsMetadata(ctx context.Context, t *testing.T, f fs.Fs, file *f
 					ci.Metadata = previousMetadata
 				}()
 			}
-			obji = overrideMimeType(obji, mimeType, metadata)
+			obji.WithMetadata(metadata).WithMimeType(mimeType)
 		}
 		obj, err = f.Put(ctx, in, obji)
 		return err

From 617c5d5e1b4c94f9c7f3c7ab72cc73898999ae58 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 8 Nov 2022 17:42:18 +0000
Subject: [PATCH 378/560] rcat: preserve metadata when Copy falls back to Rcat

Before this change if we copied files of unknown size, then they lost
their metadata.

This was particularly noticeable using --s3-decompress.

This change adds metadata to Rcat and RcatSized and changes Copy to
pass the metadata in when it calls Rcat for an unknown sized input.

Fixes #6546
---
 backend/chunker/chunker_internal_test.go     |   2 +-
 cmd/rcat/rcat.go                             |   2 +-
 cmd/serve/restic/restic.go                   |   2 +-
 fs/operations/operations.go                  |  43 +++++--
 fs/operations/operations_test.go             | 116 ++++++++++++++++++-
 fs/operations/rc.go                          |   2 +-
 vfs/vfscache/downloaders/downloaders_test.go |   2 +-
 vfs/write.go                                 |   2 +-
 8 files changed, 151 insertions(+), 20 deletions(-)

diff --git a/backend/chunker/chunker_internal_test.go b/backend/chunker/chunker_internal_test.go
index dee3657d94b4d..fc3e958b02c06 100644
--- a/backend/chunker/chunker_internal_test.go
+++ b/backend/chunker/chunker_internal_test.go
@@ -759,7 +759,7 @@ func testFutureProof(t *testing.T, f *Fs) {
 
 	// Rcat must fail
 	in := io.NopCloser(bytes.NewBufferString("abc"))
-	robj, err := operations.Rcat(ctx, f, file, in, modTime)
+	robj, err := operations.Rcat(ctx, f, file, in, modTime, nil)
 	assert.Nil(t, robj)
 	assert.NotNil(t, err)
 	if err != nil {
diff --git a/cmd/rcat/rcat.go b/cmd/rcat/rcat.go
index c7766265d897f..4328a8aa81848 100644
--- a/cmd/rcat/rcat.go
+++ b/cmd/rcat/rcat.go
@@ -66,7 +66,7 @@ a lot of data, you're better off caching locally and then
 
 		fdst, dstFileName := cmd.NewFsDstFile(args)
 		cmd.Run(false, false, command, func() error {
-			_, err := operations.RcatSize(context.Background(), fdst, dstFileName, os.Stdin, size, time.Now())
+			_, err := operations.RcatSize(context.Background(), fdst, dstFileName, os.Stdin, size, time.Now(), nil)
 			return err
 		})
 	},
diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go
index a0a521bb678b3..7dfed0e1e1ed6 100644
--- a/cmd/serve/restic/restic.go
+++ b/cmd/serve/restic/restic.go
@@ -295,7 +295,7 @@ func (s *Server) postObject(w http.ResponseWriter, r *http.Request, remote strin
 		}
 	}
 
-	o, err := operations.RcatSize(r.Context(), s.f, remote, r.Body, r.ContentLength, time.Now())
+	o, err := operations.RcatSize(r.Context(), s.f, remote, r.Body, r.ContentLength, time.Now(), nil)
 	if err != nil {
 		err = accounting.Stats(r.Context()).Error(err)
 		fs.Errorf(remote, "Post request rcat error: %v", err)
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 26db4cc0d46f7..4b4afd041b134 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -475,8 +475,16 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 						} else {
 							actionTaken = "Copied (Rcat, new)"
 						}
+						// Make any metadata to pass to rcat
+						var meta fs.Metadata
+						if ci.Metadata {
+							meta, err = fs.GetMetadata(ctx, src)
+							if err != nil {
+								fs.Errorf(src, "Failed to read metadata: %v", err)
+							}
+						}
 						// NB Rcat closes in0
-						dst, err = Rcat(ctx, f, remote, in0, src.ModTime(ctx))
+						dst, err = Rcat(ctx, f, remote, in0, src.ModTime(ctx), meta)
 						newDst = dst
 					} else {
 						in := tr.Account(ctx, in0).WithBuffer() // account and buffer the transfer
@@ -1347,7 +1355,7 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64) error {
 }
 
 // Rcat reads data from the Reader until EOF and uploads it to a file on remote
-func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time) (dst fs.Object, err error) {
+func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time, meta fs.Metadata) (dst fs.Object, err error) {
 	ci := fs.GetConfig(ctx)
 	tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, -1)
 	defer func() {
@@ -1386,7 +1394,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
 			opt.checkSum = true
 			sums = hasher.Sums()
 		}
-		src := object.NewStaticObjectInfo(dstFileName, modTime, int64(readCounter.BytesRead()), false, sums, fdst)
+		src := object.NewStaticObjectInfo(dstFileName, modTime, int64(readCounter.BytesRead()), false, sums, fdst).WithMetadata(meta)
 		if !equal(ctx, src, dst, opt) {
 			err = fmt.Errorf("corrupted on transfer")
 			err = fs.CountError(err)
@@ -1400,7 +1408,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
 	buf := make([]byte, ci.StreamingUploadCutoff)
 	if n, err := io.ReadFull(trackingIn, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
 		fs.Debugf(fdst, "File to upload is small (%d bytes), uploading instead of streaming", n)
-		src := object.NewMemoryObject(dstFileName, modTime, buf[:n])
+		src := object.NewMemoryObject(dstFileName, modTime, buf[:n]).WithMetadata(meta)
 		return Copy(ctx, fdst, nil, dstFileName, src)
 	}
 
@@ -1433,7 +1441,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
 		return nil, err
 	}
 
-	objInfo := object.NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, nil)
+	objInfo := object.NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, nil).WithMetadata(meta)
 	if dst, err = fStreamTo.Features().PutStream(ctx, in, objInfo, options...); err != nil {
 		return dst, err
 	}
@@ -1442,7 +1450,22 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
 	}
 	if !canStream {
 		// copy dst (which is the local object we have just streamed to) to the remote
-		return Copy(ctx, fdst, nil, dstFileName, dst)
+		newCtx := ctx
+		if ci.Metadata && len(meta) != 0 {
+			// If we have metadata and we are setting it then use
+			// the --metadataset mechanism to supply it to Copy
+			var newCi *fs.ConfigInfo
+			newCtx, newCi = fs.AddConfig(ctx)
+			if len(newCi.MetadataSet) == 0 {
+				newCi.MetadataSet = meta
+			} else {
+				var newMeta fs.Metadata
+				newMeta.Merge(meta)
+				newMeta.Merge(newCi.MetadataSet) // --metadata-set takes priority
+				newCi.MetadataSet = newMeta
+			}
+		}
+		return Copy(newCtx, fdst, nil, dstFileName, dst)
 	}
 	return dst, nil
 }
@@ -1724,7 +1747,7 @@ func NeedTransfer(ctx context.Context, dst, src fs.Object) bool {
 
 // RcatSize reads data from the Reader until EOF and uploads it to a file on remote.
 // Pass in size >=0 if known, <0 if not known
-func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modTime time.Time) (dst fs.Object, err error) {
+func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, size int64, modTime time.Time, meta fs.Metadata) (dst fs.Object, err error) {
 	var obj fs.Object
 
 	if size >= 0 {
@@ -1743,7 +1766,7 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
 			return nil, err
 		}
 
-		info := object.NewStaticObjectInfo(dstFileName, modTime, size, true, nil, fdst)
+		info := object.NewStaticObjectInfo(dstFileName, modTime, size, true, nil, fdst).WithMetadata(meta)
 		obj, err = fdst.Put(ctx, in, info)
 		if err != nil {
 			fs.Errorf(dstFileName, "Post request put error: %v", err)
@@ -1752,7 +1775,7 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
 		}
 	} else {
 		// Size unknown use Rcat
-		obj, err = Rcat(ctx, fdst, dstFileName, in, modTime)
+		obj, err = Rcat(ctx, fdst, dstFileName, in, modTime, meta)
 		if err != nil {
 			fs.Errorf(dstFileName, "Post request rcat error: %v", err)
 
@@ -1811,7 +1834,7 @@ func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, au
 				return errors.New("CopyURL failed: file already exist")
 			}
 		}
-		dst, err = RcatSize(ctx, fdst, dstFileName, in, size, modTime)
+		dst, err = RcatSize(ctx, fdst, dstFileName, in, size, modTime, nil)
 		return err
 	})
 	return dst, err
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 8f8229c4f72e1..b9be649027ba3 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -1590,11 +1590,11 @@ func TestRcat(t *testing.T) {
 		path2 := prefix + "big_file_from_pipe"
 
 		in := io.NopCloser(strings.NewReader(data1))
-		_, err := operations.Rcat(ctx, r.Fremote, path1, in, t1)
+		_, err := operations.Rcat(ctx, r.Fremote, path1, in, t1, nil)
 		require.NoError(t, err)
 
 		in = io.NopCloser(strings.NewReader(data2))
-		_, err = operations.Rcat(ctx, r.Fremote, path2, in, t2)
+		_, err = operations.Rcat(ctx, r.Fremote, path2, in, t2, nil)
 		require.NoError(t, err)
 
 		file1 := fstest.NewItem(path1, data1, t1)
@@ -1611,6 +1611,62 @@ func TestRcat(t *testing.T) {
 	}
 }
 
+func TestRcatMetadata(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	if !r.Fremote.Features().UserMetadata {
+		t.Skip("Skipping as destination doesn't support user metadata")
+	}
+
+	test := func(disableUploadCutoff bool) {
+		ctx := context.Background()
+		ctx, ci := fs.AddConfig(ctx)
+		ci.Metadata = true
+		data := "this is some really nice test data with metadata"
+		path := "rcat_metadata"
+
+		meta := fs.Metadata{
+			"key":     "value",
+			"sausage": "potato",
+		}
+
+		if disableUploadCutoff {
+			ci.StreamingUploadCutoff = 0
+			data += " uploadCutoff=0"
+			path += "_uploadcutoff0"
+		}
+
+		fstest.CheckListing(t, r.Fremote, []fstest.Item{})
+
+		in := io.NopCloser(strings.NewReader(data))
+		_, err := operations.Rcat(ctx, r.Fremote, path, in, t1, meta)
+		require.NoError(t, err)
+
+		file := fstest.NewItem(path, data, t1)
+		r.CheckRemoteItems(t, file)
+
+		o, err := r.Fremote.NewObject(ctx, path)
+		require.NoError(t, err)
+		gotMeta, err := fs.GetMetadata(ctx, o)
+		require.NoError(t, err)
+		// Check the specific user data we set is set
+		// Likey there will be other values
+		assert.Equal(t, "value", gotMeta["key"])
+		assert.Equal(t, "potato", gotMeta["sausage"])
+
+		// Delete the test file
+		require.NoError(t, o.Remove(ctx))
+	}
+
+	t.Run("Normal", func(t *testing.T) {
+		test(false)
+	})
+	t.Run("ViaDisk", func(t *testing.T) {
+		test(true)
+	})
+}
+
 func TestRcatSize(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
@@ -1621,7 +1677,7 @@ func TestRcatSize(t *testing.T) {
 	file2 := r.WriteFile("potato2", body, t2)
 	// Test with known length
 	bodyReader := io.NopCloser(strings.NewReader(body))
-	obj, err := operations.RcatSize(ctx, r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime)
+	obj, err := operations.RcatSize(ctx, r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime, nil)
 	require.NoError(t, err)
 	assert.Equal(t, int64(len(body)), obj.Size())
 	assert.Equal(t, file1.Path, obj.Remote())
@@ -1629,7 +1685,7 @@ func TestRcatSize(t *testing.T) {
 	// Test with unknown length
 	bodyReader = io.NopCloser(strings.NewReader(body)) // reset Reader
 	io.NopCloser(strings.NewReader(body))
-	obj, err = operations.RcatSize(ctx, r.Fremote, file2.Path, bodyReader, -1, file2.ModTime)
+	obj, err = operations.RcatSize(ctx, r.Fremote, file2.Path, bodyReader, -1, file2.ModTime, nil)
 	require.NoError(t, err)
 	assert.Equal(t, int64(len(body)), obj.Size())
 	assert.Equal(t, file2.Path, obj.Remote())
@@ -1638,6 +1694,58 @@ func TestRcatSize(t *testing.T) {
 	r.CheckRemoteItems(t, file1, file2)
 }
 
+func TestRcatSizeMetadata(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	if !r.Fremote.Features().UserMetadata {
+		t.Skip("Skipping as destination doesn't support user metadata")
+	}
+
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	ci.Metadata = true
+
+	meta := fs.Metadata{
+		"key":     "value",
+		"sausage": "potato",
+	}
+
+	const body = "------------------------------------------------------------"
+	file1 := r.WriteFile("potato1", body, t1)
+	file2 := r.WriteFile("potato2", body, t2)
+
+	// Test with known length
+	bodyReader := io.NopCloser(strings.NewReader(body))
+	obj, err := operations.RcatSize(ctx, r.Fremote, file1.Path, bodyReader, int64(len(body)), file1.ModTime, meta)
+	require.NoError(t, err)
+	assert.Equal(t, int64(len(body)), obj.Size())
+	assert.Equal(t, file1.Path, obj.Remote())
+
+	// Test with unknown length
+	bodyReader = io.NopCloser(strings.NewReader(body)) // reset Reader
+	io.NopCloser(strings.NewReader(body))
+	obj, err = operations.RcatSize(ctx, r.Fremote, file2.Path, bodyReader, -1, file2.ModTime, meta)
+	require.NoError(t, err)
+	assert.Equal(t, int64(len(body)), obj.Size())
+	assert.Equal(t, file2.Path, obj.Remote())
+
+	// Check files exist
+	r.CheckRemoteItems(t, file1, file2)
+
+	// Check metadata OK
+	for _, path := range []string{file1.Path, file2.Path} {
+		o, err := r.Fremote.NewObject(ctx, path)
+		require.NoError(t, err)
+		gotMeta, err := fs.GetMetadata(ctx, o)
+		require.NoError(t, err)
+		// Check the specific user data we set is set
+		// Likey there will be other values
+		assert.Equal(t, "value", gotMeta["key"])
+		assert.Equal(t, "potato", gotMeta["sausage"])
+	}
+}
+
 func TestCopyFileMaxTransfer(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
diff --git a/fs/operations/rc.go b/fs/operations/rc.go
index 39705b11958a5..a4e15ea096bbd 100644
--- a/fs/operations/rc.go
+++ b/fs/operations/rc.go
@@ -305,7 +305,7 @@ func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bo
 					return nil, err
 				}
 				if p.FileName() != "" {
-					obj, err := Rcat(ctx, f, path.Join(remote, p.FileName()), p, time.Now())
+					obj, err := Rcat(ctx, f, path.Join(remote, p.FileName()), p, time.Now(), nil)
 					if err != nil {
 						return nil, err
 					}
diff --git a/vfs/vfscache/downloaders/downloaders_test.go b/vfs/vfscache/downloaders/downloaders_test.go
index 9fa42519266eb..d079986b501bc 100644
--- a/vfs/vfscache/downloaders/downloaders_test.go
+++ b/vfs/vfscache/downloaders/downloaders_test.go
@@ -85,7 +85,7 @@ func TestDownloaders(t *testing.T) {
 
 	// Write the test file
 	in := io.NopCloser(readers.NewPatternReader(size))
-	src, err := operations.RcatSize(ctx, r.Fremote, remote, in, size, time.Now())
+	src, err := operations.RcatSize(ctx, r.Fremote, remote, in, size, time.Now(), nil)
 	require.NoError(t, err)
 	assert.Equal(t, size, src.Size())
 
diff --git a/vfs/write.go b/vfs/write.go
index 4556ebe3e2a56..e18f1d001012a 100644
--- a/vfs/write.go
+++ b/vfs/write.go
@@ -68,7 +68,7 @@ func (fh *WriteFileHandle) openPending() (err error) {
 	pipeReader, fh.pipeWriter = io.Pipe()
 	go func() {
 		// NB Rcat deals with Stats.Transferring, etc.
-		o, err := operations.Rcat(context.TODO(), fh.file.Fs(), fh.remote, pipeReader, time.Now())
+		o, err := operations.Rcat(context.TODO(), fh.file.Fs(), fh.remote, pipeReader, time.Now(), nil)
 		if err != nil {
 			fs.Errorf(fh.remote, "WriteFileHandle.New Rcat failed: %v", err)
 		}

From 0c56c46523b1fea148ac732d29a30d576188621b Mon Sep 17 00:00:00 2001
From: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>
Date: Thu, 27 Oct 2022 16:46:31 +0530
Subject: [PATCH 379/560] rc: Add commands to set GC Percent & Memory Limit
 (1.19+)

Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>
---
 fs/rc/internal.go          | 94 ++++++++++++++++++++++++++++++++++++--
 lib/debug/common.go        | 12 +++++
 lib/debug/go1.19.go        | 14 ++++++
 lib/debug/go1.19_compat.go | 14 ++++++
 4 files changed, 131 insertions(+), 3 deletions(-)
 create mode 100644 lib/debug/common.go
 create mode 100644 lib/debug/go1.19.go
 create mode 100644 lib/debug/go1.19_compat.go

diff --git a/fs/rc/internal.go b/fs/rc/internal.go
index 24e67eabb0a94..4dafbb5069418 100644
--- a/fs/rc/internal.go
+++ b/fs/rc/internal.go
@@ -18,6 +18,7 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/buildinfo"
+	"github.com/rclone/rclone/lib/debug"
 )
 
 func init() {
@@ -300,7 +301,6 @@ Results:
 	})
 }
 
-// Terminates app
 func rcSetMutexProfileFraction(ctx context.Context, in Params) (out Params, err error) {
 	rate, err := in.GetInt64("rate")
 	if err != nil {
@@ -336,7 +336,6 @@ Parameters:
 	})
 }
 
-// Terminates app
 func rcSetBlockProfileRate(ctx context.Context, in Params) (out Params, err error) {
 	rate, err := in.GetInt64("rate")
 	if err != nil {
@@ -346,6 +345,95 @@ func rcSetBlockProfileRate(ctx context.Context, in Params) (out Params, err erro
 	return nil, nil
 }
 
+func init() {
+	Add(Call{
+		Path:  "debug/set-soft-memory-limit",
+		Fn:    rcSetSoftMemoryLimit,
+		Title: "Call runtime/debug.SetMemoryLimit for setting a soft memory limit for the runtime.",
+		Help: `
+SetMemoryLimit provides the runtime with a soft memory limit.
+
+The runtime undertakes several processes to try to respect this memory limit, including
+adjustments to the frequency of garbage collections and returning memory to the underlying
+system more aggressively. This limit will be respected even if GOGC=off (or, if SetGCPercent(-1) is executed).
+
+The input limit is provided as bytes, and includes all memory mapped, managed, and not
+released by the Go runtime. Notably, it does not account for space used by the Go binary
+and memory external to Go, such as memory managed by the underlying system on behalf of
+the process, or memory managed by non-Go code inside the same process.
+Examples of excluded memory sources include: OS kernel memory held on behalf of the process,
+memory allocated by C code, and memory mapped by syscall.Mmap (because it is not managed by the Go runtime).
+
+A zero limit or a limit that's lower than the amount of memory used by the Go runtime may cause
+the garbage collector to run nearly continuously. However, the application may still make progress.
+
+The memory limit is always respected by the Go runtime, so to effectively disable this behavior,
+set the limit very high. math.MaxInt64 is the canonical value for disabling the limit, but values
+much greater than the available memory on the underlying system work just as well.
+
+See https://go.dev/doc/gc-guide for a detailed guide explaining the soft memory limit in more detail,
+as well as a variety of common use-cases and scenarios.
+
+SetMemoryLimit returns the previously set memory limit. A negative input does not adjust the limit,
+and allows for retrieval of the currently set memory limit.
+
+Parameters:
+
+- mem-limit - int
+`,
+	})
+}
+
+func rcSetSoftMemoryLimit(ctx context.Context, in Params) (out Params, err error) {
+	memLimit, err := in.GetInt64("mem-limit")
+	if err != nil {
+		return nil, err
+	}
+	oldMemLimit, err := debug.SetMemoryLimit(memLimit)
+	if err != nil {
+		return nil, err
+	}
+	out = Params{
+		"existing-mem-limit": oldMemLimit,
+	}
+	return out, nil
+}
+
+func init() {
+	Add(Call{
+		Path:  "debug/set-gc-percent",
+		Fn:    rcSetGCPercent,
+		Title: "Call runtime/debug.SetGCPercent for setting the garbage collection target percentage.",
+		Help: `
+SetGCPercent sets the garbage collection target percentage: a collection is triggered
+when the ratio of freshly allocated data to live data remaining after the previous collection
+reaches this percentage. SetGCPercent returns the previous setting. The initial setting is the
+value of the GOGC environment variable at startup, or 100 if the variable is not set.
+
+This setting may be effectively reduced in order to maintain a memory limit.
+A negative percentage effectively disables garbage collection, unless the memory limit is reached.
+
+See https://pkg.go.dev/runtime/debug#SetMemoryLimit for more details.
+
+Parameters:
+
+- gc-percent - int
+`,
+	})
+}
+
+func rcSetGCPercent(ctx context.Context, in Params) (out Params, err error) {
+	gcPercent, err := in.GetInt64("gc-percent")
+	if err != nil {
+		return nil, err
+	}
+	oldGCPercent := debug.SetGCPercent(int(gcPercent))
+	out = Params{
+		"existing-gc-percent": oldGCPercent,
+	}
+	return out, nil
+}
+
 func init() {
 	Add(Call{
 		Path:          "core/command",
@@ -384,7 +472,7 @@ Returns:
 	"result": ""
 }
 
-OR 
+OR
 {
 	"error": true,
 	"result": ""
diff --git a/lib/debug/common.go b/lib/debug/common.go
new file mode 100644
index 0000000000000..ab0a28da92f46
--- /dev/null
+++ b/lib/debug/common.go
@@ -0,0 +1,12 @@
+// Package debug contains functions for dealing with runtime/debug functions across go versions
+package debug
+
+import (
+	"runtime/debug"
+)
+
+// SetGCPercent calls the runtime/debug.SetGCPercent function to set the garbage
+// collection percentage.
+func SetGCPercent(percent int) int {
+	return debug.SetGCPercent(percent)
+}
diff --git a/lib/debug/go1.19.go b/lib/debug/go1.19.go
new file mode 100644
index 0000000000000..1103f20259e8e
--- /dev/null
+++ b/lib/debug/go1.19.go
@@ -0,0 +1,14 @@
+//go:build go1.19
+// +build go1.19
+
+package debug
+
+import (
+	"runtime/debug"
+)
+
+// SetMemoryLimit calls the runtime/debug.SetMemoryLimit function to set the
+// soft-memory limit.
+func SetMemoryLimit(limit int64) (int64, error) {
+	return debug.SetMemoryLimit(limit), nil
+}
diff --git a/lib/debug/go1.19_compat.go b/lib/debug/go1.19_compat.go
new file mode 100644
index 0000000000000..e4ccbc6955b11
--- /dev/null
+++ b/lib/debug/go1.19_compat.go
@@ -0,0 +1,14 @@
+//go:build !go1.19
+// +build !go1.19
+
+package debug
+
+import (
+	"fmt"
+	"runtime"
+)
+
+// SetMemoryLimit is a no-op on Go version < 1.19.
+func SetMemoryLimit(limit int64) (int64, error) {
+	return limit, fmt.Errorf("not implemented on Go version below 1.19: %s", runtime.Version())
+}

From 0e427216dbaefeabb972346e937dc9de542d58a3 Mon Sep 17 00:00:00 2001
From: techknowlogick 
Date: Fri, 11 Nov 2022 05:19:27 +0000
Subject: [PATCH 380/560] s3: Add additional Wasabi locations

---
 backend/s3/s3.go | 38 +++++++++++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index cb8b8cd9ea286..9de4a7e6ae28d 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -1127,15 +1127,39 @@ func init() {
 				Provider: "LyveCloud",
 			}, {
 				Value:    "s3.wasabisys.com",
-				Help:     "Wasabi US East endpoint",
+				Help:     "Wasabi US East 1 (N. Virginia)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.us-east-2.wasabisys.com",
+				Help:     "Wasabi US East 2 (N. Virginia)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.us-central-1.wasabisys.com",
+				Help:     "Wasabi US Central 1 (Texas)",
 				Provider: "Wasabi",
 			}, {
 				Value:    "s3.us-west-1.wasabisys.com",
-				Help:     "Wasabi US West endpoint",
+				Help:     "Wasabi US West 1 (Oregon)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.ca-central-1.wasabisys.com",
+				Help:     "Wasabi CA Central 1 (Toronto)",
 				Provider: "Wasabi",
 			}, {
 				Value:    "s3.eu-central-1.wasabisys.com",
-				Help:     "Wasabi EU Central endpoint",
+				Help:     "Wasabi EU Central 1 (Amsterdam)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.eu-central-2.wasabisys.com",
+				Help:     "Wasabi EU Central 2 (Frankfurt)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.eu-west-1.wasabisys.com",
+				Help:     "Wasabi EU West 1 (London)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.eu-west-2.wasabisys.com",
+				Help:     "Wasabi EU West 2 (Paris)",
 				Provider: "Wasabi",
 			}, {
 				Value:    "s3.ap-northeast-1.wasabisys.com",
@@ -1145,6 +1169,14 @@ func init() {
 				Value:    "s3.ap-northeast-2.wasabisys.com",
 				Help:     "Wasabi AP Northeast 2 (Osaka) endpoint",
 				Provider: "Wasabi",
+			}, {
+				Value:    "s3.ap-southeast-1.wasabisys.com",
+				Help:     "Wasabi AP Southeast 1 (Singapore)",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.ap-southeast-2.wasabisys.com",
+				Help:     "Wasabi AP Southeast 2 (Sydney)",
+				Provider: "Wasabi",
 			}, {
 				Value:    "s3.ir-thr-at1.arvanstorage.com",
 				Help:     "ArvanCloud Tehran Iran (Asiatech) endpoint",

From 1392793334c14d1cb3f4b95eb191a0685e886143 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Tue, 8 Nov 2022 18:52:39 +0100
Subject: [PATCH 381/560] sftp: auto-detect shell type for fish

Fish is different from POSIX-based Unix shells such as bash,
and a bracketed variable references like we use for the
auto-detection echo command is not supported. The command
will return with zero exit code but produce no output on
stdout. There is a message on stderr, but we don't log it
due to the zero exit code:

fish: Variables cannot be bracketed. In fish, please use {$ShellId}.

Fixes #6552
---
 backend/sftp/sftp.go | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go
index b6afcd3558d3c..1097452674a5a 100644
--- a/backend/sftp/sftp.go
+++ b/backend/sftp/sftp.go
@@ -975,20 +975,24 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
 			fs.Debugf(f, "Running shell type detection remote command: %s", shellCmd)
 			err = session.Run(shellCmd)
 			_ = session.Close()
+			f.shellType = defaultShellType
 			if err != nil {
-				f.shellType = defaultShellType
 				fs.Debugf(f, "Remote command failed: %v (stdout=%v) (stderr=%v)", err, bytes.TrimSpace(stdout.Bytes()), bytes.TrimSpace(stderr.Bytes()))
 			} else {
 				outBytes := stdout.Bytes()
 				fs.Debugf(f, "Remote command result: %s", outBytes)
 				outString := string(bytes.TrimSpace(stdout.Bytes()))
-				if strings.HasPrefix(outString, "Microsoft.PowerShell") { // If PowerShell: "Microsoft.PowerShell%ComSpec%"
-					f.shellType = "powershell"
-				} else if !strings.HasSuffix(outString, "%ComSpec%") { // If Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe"
-					f.shellType = "cmd"
-				} else { // If Unix: "%ComSpec%"
-					f.shellType = "unix"
-				}
+				if outString != "" {
+					if strings.HasPrefix(outString, "Microsoft.PowerShell") { // PowerShell: "Microsoft.PowerShell%ComSpec%"
+						f.shellType = "powershell"
+					} else if !strings.HasSuffix(outString, "%ComSpec%") { // Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe"
+						// Additional positive test, to avoid misdetection on unpredicted Unix shell variants
+						s := strings.ToLower(outString)
+						if strings.Contains(s, ".exe") || strings.Contains(s, ".com") {
+							f.shellType = "cmd"
+						}
+					} // POSIX-based Unix shell: "%ComSpec%"
+				} // fish Unix shell: ""
 			}
 		}
 		// Save permanently in config to avoid the extra work next time

From 87fa9f8e463c93e9d366d81caa2fae75c5d0bb9d Mon Sep 17 00:00:00 2001
From: rkettelerij 
Date: Mon, 14 Nov 2022 10:51:49 +0100
Subject: [PATCH 382/560] azureblob: Add support for custom upload headers

---
 backend/azureblob/azureblob.go | 20 ++++++++++++++++++++
 docs/content/azureblob.md      | 12 ++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 8c49a50dd9727..6c46419f51f76 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -1685,6 +1685,26 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
+	// Apply upload options (also allows one to overwrite content-type)
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			httpHeaders.CacheControl = value
+		case "content-disposition":
+			httpHeaders.ContentDisposition = value
+		case "content-encoding":
+			httpHeaders.ContentEncoding = value
+		case "content-language":
+			httpHeaders.ContentLanguage = value
+		case "content-type":
+			httpHeaders.ContentType = value
+		}
+	}
+
 	uploadParts := maxUploadParts
 	if uploadParts < 1 {
 		uploadParts = 1
diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md
index 5a1e2663fca24..6f0200b3be812 100644
--- a/docs/content/azureblob.md
+++ b/docs/content/azureblob.md
@@ -514,6 +514,18 @@ Properties:
 
 {{< rem autogenerated options stop >}}
 
+### Custom upload headers
+
+You can set custom upload headers with the `--header-upload` flag. 
+
+- Cache-Control
+- Content-Disposition
+- Content-Encoding
+- Content-Language
+- Content-Type
+
+Eg `--header-upload "Content-Type: text/potato"`
+
 ## Limitations
 
 MD5 sums are only uploaded with chunked files if the source has an MD5

From 8daacc2b99a474ccad2464f4f213f9d6bd4189db Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 14 Nov 2022 15:13:38 +0000
Subject: [PATCH 383/560] Add techknowlogick to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 69845ef29b775..989785e9d926f 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -660,3 +660,4 @@ put them back in again.` >}}
   * Arnie97 
   * Roel Arents <2691308+roelarents@users.noreply.github.com>
   * Aaron Gokaslan 
+  * techknowlogick 

From 2f77651f649ab825297ef54d9c4e9b7e13d7fda9 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 14 Nov 2022 15:13:38 +0000
Subject: [PATCH 384/560] Add rkettelerij to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 989785e9d926f..84607df6414d5 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -661,3 +661,4 @@ put them back in again.` >}}
   * Roel Arents <2691308+roelarents@users.noreply.github.com>
   * Aaron Gokaslan 
   * techknowlogick 
+  * rkettelerij 

From 0ac5795f8cb167455c566b7ba155005e70cb1dd3 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 10 Nov 2022 17:28:12 +0000
Subject: [PATCH 385/560] fs: make all duration flags take y, M, w, d etc
 suffixes

Fixes #6556
---
 fs/config/flags/flags.go | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/fs/config/flags/flags.go b/fs/config/flags/flags.go
index 31f59e457fdbc..53a4c2f160bef 100644
--- a/fs/config/flags/flags.go
+++ b/fs/config/flags/flags.go
@@ -146,18 +146,22 @@ func Float64VarP(flags *pflag.FlagSet, p *float64, name, shorthand string, value
 
 // DurationP defines a flag which can be set by an environment variable
 //
-// It is a thin wrapper around pflag.DurationP
+// It wraps the duration in an fs.Duration for extra suffixes and
+// passes it to pflag.VarP
 func DurationP(name, shorthand string, value time.Duration, usage string) (out *time.Duration) {
-	out = pflag.DurationP(name, shorthand, value, usage)
+	out = new(time.Duration)
+	*out = value
+	pflag.VarP((*fs.Duration)(out), name, shorthand, usage)
 	setValueFromEnv(pflag.CommandLine, name)
 	return out
 }
 
 // DurationVarP defines a flag which can be set by an environment variable
 //
-// It is a thin wrapper around pflag.DurationVarP
+// It wraps the duration in an fs.Duration for extra suffixes and
+// passes it to pflag.VarP
 func DurationVarP(flags *pflag.FlagSet, p *time.Duration, name, shorthand string, value time.Duration, usage string) {
-	flags.DurationVarP(p, name, shorthand, value, usage)
+	flags.VarP((*fs.Duration)(p), name, shorthand, usage)
 	setValueFromEnv(flags, name)
 }
 

From efd3c6449b0794c4c33f658dccf087406e25477b Mon Sep 17 00:00:00 2001
From: Kamui 
Date: Mon, 14 Nov 2022 09:26:44 -0600
Subject: [PATCH 386/560] rcserver: avoid generating default credentials with
 htpasswd - fixes #4839

---
 fs/rc/rcserver/rcserver.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.go
index 0d246a2f60cdf..125b36903a6a5 100644
--- a/fs/rc/rcserver/rcserver.go
+++ b/fs/rc/rcserver/rcserver.go
@@ -97,11 +97,11 @@ func newServer(ctx context.Context, opt *rc.Options, mux *http.ServeMux) *Server
 		if opt.NoAuth {
 			fs.Logf(nil, "It is recommended to use web gui with auth.")
 		} else {
-			if opt.HTTPOptions.BasicUser == "" {
+			if opt.HTTPOptions.BasicUser == "" && opt.HTTPOptions.HtPasswd == "" {
 				opt.HTTPOptions.BasicUser = "gui"
 				fs.Infof(nil, "No username specified. Using default username: %s \n", rcflags.Opt.HTTPOptions.BasicUser)
 			}
-			if opt.HTTPOptions.BasicPass == "" {
+			if opt.HTTPOptions.BasicPass == "" && opt.HTTPOptions.HtPasswd == "" {
 				randomPass, err := random.Password(128)
 				if err != nil {
 					log.Fatalf("Failed to make password: %v", err)

From bb1fc5b86d94f0cfc9a6658b91cee6e3edeae248 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 14 Nov 2022 17:05:39 +0000
Subject: [PATCH 387/560] Add Kamui to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 84607df6414d5..5fc98a020b03d 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -662,3 +662,4 @@ put them back in again.` >}}
   * Aaron Gokaslan 
   * techknowlogick 
   * rkettelerij 
+  * Kamui 

From 652d3cdee4b505b34556269cf1a8b878a8d9f3c2 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 10 Nov 2022 18:00:32 +0000
Subject: [PATCH 388/560] vfs: windows: fix slow opening of exe files by not
 truncating files when not necessary

Before this change we truncated files in the backing store regardless
of whether we needed to or not.

After, we check to see if the file is the right size and don't
truncate if it is.

Apparently Windows Defender likes to check executables each time they
are modified, and truncating a file to its existing size is enough to
trigger the Windows Defender scan. This was causing a big slowdown for
operations which opened and closed the file a lot, such as looking at
properties on an executable.

See: https://forum.rclone.org/t/for-mount-sftp-why-right-click-on-exe-file-is-so-slow-until-it-freezes/33830
---
 vfs/vfscache/item.go | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go
index b7f40252d9a5b..262f191040cdb 100644
--- a/vfs/vfscache/item.go
+++ b/vfs/vfscache/item.go
@@ -272,11 +272,25 @@ func (item *Item) _truncate(size int64) (err error) {
 		}
 	}
 
-	fs.Debugf(item.name, "vfs cache: truncate to size=%d", size)
+	// Check to see what the current size is, and don't truncate
+	// if it is already the correct size.
+	//
+	// Apparently Windows Defender likes to check executables each
+	// time they are modified, and truncating a file to its
+	// existing size is enough to trigger the Windows Defender
+	// scan. This was causing a big slowdown for operations which
+	// opened and closed the file a lot, such as looking at
+	// properties on an executable.
+	fi, err := fd.Stat()
+	if err == nil && fi.Size() == size {
+		fs.Debugf(item.name, "vfs cache: truncate to size=%d (not needed as size correct)", size)
+	} else {
+		fs.Debugf(item.name, "vfs cache: truncate to size=%d", size)
 
-	err = fd.Truncate(size)
-	if err != nil {
-		return fmt.Errorf("vfs cache: truncate: %w", err)
+		err = fd.Truncate(size)
+		if err != nil {
+			return fmt.Errorf("vfs cache: truncate: %w", err)
+		}
 	}
 
 	item.info.Size = size
@@ -460,7 +474,9 @@ func (item *Item) _createFile(osPath string) (err error) {
 		return errors.New("vfs cache item: internal error: didn't Close file")
 	}
 	item.modified = false
+	// t0 := time.Now()
 	fd, err := file.OpenFile(osPath, os.O_RDWR, 0600)
+	// fs.Debugf(item.name, "OpenFile took %v", time.Since(t0))
 	if err != nil {
 		return fmt.Errorf("vfs cache item: open failed: %w", err)
 	}

From a61d219bcd3ffa3ee0411e7d3759c0ef5e22e495 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 14 Nov 2022 17:59:33 +0000
Subject: [PATCH 389/560] local: fix -L/--copy-links with filters missing
 directories

In this commit

8d1fff9a8237c64f local: obey file filters in listing to fix errors on excluded files

We introduced the concept of local backend filters.

Unfortunately the filters were being applied before we had resolved
the symlink to point to a directory. This meant that symlinks pointing
to directories were filtered out when they shouldn't have been.

This was fixed by moving the filter check until after the symlink had
been resolved.

See: https://forum.rclone.org/t/copy-links-not-following-symlinks-on-1-60-0/34073/7
---
 backend/local/local.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/backend/local/local.go b/backend/local/local.go
index 8d16ad4a86c4c..67784f8e6d798 100644
--- a/backend/local/local.go
+++ b/backend/local/local.go
@@ -520,11 +520,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			name := fi.Name()
 			mode := fi.Mode()
 			newRemote := f.cleanRemote(dir, name)
-			// Don't include non directory if not included
-			// we leave directory filtering to the layer above
-			if useFilter && !fi.IsDir() && !filter.IncludeRemote(newRemote) {
-				continue
-			}
 			// Follow symlinks if required
 			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
 				localPath := filepath.Join(fsDirPath, name)
@@ -541,6 +536,11 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				}
 				mode = fi.Mode()
 			}
+			// Don't include non directory if not included
+			// we leave directory filtering to the layer above
+			if useFilter && !fi.IsDir() && !filter.IncludeRemote(newRemote) {
+				continue
+			}
 			if fi.IsDir() {
 				// Ignore directories which are symlinks.  These are junction points under windows which
 				// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.

From 2c78f56d48fa876a8a6367462cb14df896a36b62 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 15 Nov 2022 09:51:30 +0000
Subject: [PATCH 390/560] webdav: fix Move/Copy/DirMove when using
 -server-side-across-configs

Before this change, when using -server-side-across-configs rclone
would direct Move/Copy/DirMove to the destination server.

However this should be directed to the source server. This is a little
unclear in the RFC, but the name of the parameter "Destination:" seems
clear and this is how dCache and Rucio have implemented it.

See: https://forum.rclone.org/t/webdav-copy-request-implemented-incorrectly/34072/
---
 backend/webdav/webdav.go | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go
index 8a815cef8187a..251202948b10d 100644
--- a/backend/webdav/webdav.go
+++ b/backend/webdav/webdav.go
@@ -991,6 +991,7 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho
 		}
 		return nil, fs.ErrorCantMove
 	}
+	srcFs := srcObj.fs
 	dstPath := f.filePath(remote)
 	err := f.mkParentDir(ctx, dstPath)
 	if err != nil {
@@ -1013,9 +1014,10 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho
 	if f.useOCMtime {
 		opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
 	}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
+	// Direct the MOVE/COPY to the source server
+	err = srcFs.pacer.Call(func() (bool, error) {
+		resp, err = srcFs.srv.Call(ctx, &opts)
+		return srcFs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, fmt.Errorf("Copy call failed: %w", err)
@@ -1109,9 +1111,10 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 			"Overwrite":   "F",
 		},
 	}
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
+	// Direct the MOVE/COPY to the source server
+	err = srcFs.pacer.Call(func() (bool, error) {
+		resp, err = srcFs.srv.Call(ctx, &opts)
+		return srcFs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return fmt.Errorf("DirMove MOVE call failed: %w", err)

From 84219b95abeac4c5a9e4d1a74b0118862a6bb500 Mon Sep 17 00:00:00 2001
From: asdffdsazqqq <90116442+asdffdsazqqq@users.noreply.github.com>
Date: Tue, 15 Nov 2022 12:58:43 -0500
Subject: [PATCH 391/560] docs: faq: how to use a proxy server that requires a
 username and password - fixes #6565

---
 docs/content/faq.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/content/faq.md b/docs/content/faq.md
index a0a99c9302361..b42403af51ee2 100644
--- a/docs/content/faq.md
+++ b/docs/content/faq.md
@@ -108,6 +108,14 @@ possibilities.  So, on Linux, you may end up with code similar to
     export HTTP_PROXY=$http_proxy
     export HTTPS_PROXY=$http_proxy
 
+
+Note: If the proxy server requires a username and password, then use
+
+    export http_proxy=http://username:password@proxyserver:12345
+    export https_proxy=$http_proxy
+    export HTTP_PROXY=$http_proxy
+    export HTTPS_PROXY=$http_proxy
+
 The `NO_PROXY` allows you to disable the proxy for specific hosts.
 Hosts must be comma separated, and can contain domains or parts.
 For instance "foo.com" also matches "bar.foo.com".

From b4a3d1b9ed1a0cb74146a2dd298c3815d9b93c5b Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 15 Nov 2022 18:01:21 +0000
Subject: [PATCH 392/560] Add asdffdsazqqq to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 5fc98a020b03d..1d106b42ec766 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -663,3 +663,4 @@ put them back in again.` >}}
   * techknowlogick 
   * rkettelerij 
   * Kamui 
+  * asdffdsazqqq <90116442+asdffdsazqqq@users.noreply.github.com>

From 591fc3609aea47441fbd4bc01235291efea9bb4e Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 17 Oct 2022 16:46:59 +0100
Subject: [PATCH 393/560] vfs: fix deadlock caused by cache cleaner and upload
 finishing

Before this patch a deadlock could occur if the cache cleaner was
running when an object upload finished.

This fixes the problem by delaying marking the object as clean until
we have notified the VFS layer. This means that the cache cleaner
won't consider the object until **after** the VFS layer has been
notified, thus avoiding the deadlock.

See: https://forum.rclone.org/t/rclone-mount-deadlock-when-dir-cache-time-strikes/33486/
---
 vfs/vfscache/item.go | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go
index 262f191040cdb..e1575eca3b37b 100644
--- a/vfs/vfscache/item.go
+++ b/vfs/vfscache/item.go
@@ -606,20 +606,25 @@ func (item *Item) _store(ctx context.Context, storeFn StoreFn) (err error) {
 		item._updateFingerprint()
 	}
 
-	item.info.Dirty = false
-	err = item._save()
-	if err != nil {
-		fs.Errorf(item.name, "vfs cache: failed to write metadata file: %v", err)
-	}
+	// Write the object back to the VFS layer before we mark it as
+	// clean, otherwise it will become eligible for removal which
+	// can cause a deadlock
 	if storeFn != nil && item.o != nil {
 		fs.Debugf(item.name, "vfs cache: writeback object to VFS layer")
-		// Write the object back to the VFS layer as last
-		// thing we do with mutex unlocked
+		// Write the object back to the VFS layer last with mutex unlocked
 		o := item.o
 		item.mu.Unlock()
 		storeFn(o)
 		item.mu.Lock()
 	}
+
+	// Show item is clean and is elegible for cache removal
+	item.info.Dirty = false
+	err = item._save()
+	if err != nil {
+		fs.Errorf(item.name, "vfs cache: failed to write metadata file: %v", err)
+	}
+
 	return nil
 }
 

From 705e8f2fe0ab14ac71ed00bf53d3f1b8fc3b3db7 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 15 Nov 2022 09:32:36 +0000
Subject: [PATCH 394/560] smb: fix Failed to sync: context canceled at the end
 of syncs

Before this change we were putting connections into the connection
pool which had a local context in.

This meant that when the operation had finished the context was
cancelled and the connection became unusable.

See: https://forum.rclone.org/t/failed-to-sync-context-canceled/34017/
---
 backend/smb/connpool.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/backend/smb/connpool.go b/backend/smb/connpool.go
index 6f6fe82325f30..d4bed787c5ead 100644
--- a/backend/smb/connpool.go
+++ b/backend/smb/connpool.go
@@ -103,6 +103,10 @@ func (f *Fs) getSessions() int32 {
 
 // Open a new connection to the SMB server.
 func (f *Fs) newConnection(ctx context.Context, share string) (c *conn, err error) {
+	// As we are pooling these connections we need to decouple
+	// them from the current context
+	ctx = context.Background()
+
 	c, err = f.dial(ctx, "tcp", f.opt.Host+":"+f.opt.Port)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't connect SMB: %w", err)

From 03b07c280cc4adb3b069da4ca9c2d650ac2e44c1 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 17 Nov 2022 16:32:25 +0000
Subject: [PATCH 395/560] Changelog updates from Version v1.60.1

---
 docs/content/changelog.md | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/docs/content/changelog.md b/docs/content/changelog.md
index f68c8f5a0e492..2c844f25621f2 100644
--- a/docs/content/changelog.md
+++ b/docs/content/changelog.md
@@ -5,6 +5,40 @@ description: "Rclone Changelog"
 
 # Changelog
 
+## v1.60.1 - 2022-11-17
+
+[See commits](https://github.com/rclone/rclone/compare/v1.60.0...v1.60.1)
+
+* Bug Fixes
+    * lib/cache: Fix alias backend shutting down too soon (Nick Craig-Wood)
+    * wasm: Fix walltime link error by adding up-to-date wasm_exec.js (João Henrique Franco)
+    * docs
+        * Update faq.md with bisync (Samuel Johnson)
+        * Corrected download links in windows install docs (coultonluke)
+        * Add direct download link for windows arm64 (albertony)
+        * Remove link to rclone slack as it is no longer supported (Nick Craig-Wood)
+        * Faq: how to use a proxy server that requires a username and password (asdffdsazqqq)
+        * Oracle-object-storage: doc fix (Manoj Ghosh)
+        * Fix typo `remove` in rclone_serve_restic command (Joda Stößer)
+        * Fix character that was incorrectly interpreted as markdown (Clément Notin)
+* VFS
+    * Fix deadlock caused by cache cleaner and upload finishing (Nick Craig-Wood)
+* Local
+    * Clean absolute paths (albertony)
+    * Fix -L/--copy-links with filters missing directories (Nick Craig-Wood)
+* Mailru
+    * Note that an app password is now needed (Nick Craig-Wood)
+    * Allow timestamps to be before the epoch 1970-01-01 (Nick Craig-Wood)
+* S3
+    * Add provider quirk `--s3-might-gzip` to fix corrupted on transfer: sizes differ (Nick Craig-Wood)
+    * Allow Storj to server side copy since it seems to work now (Nick Craig-Wood)
+    * Fix for unchecked err value in s3 listv2 (Aaron Gokaslan)
+    * Add additional Wasabi locations (techknowlogick)
+* Smb
+    * Fix `Failed to sync: context canceled` at the end of syncs (Nick Craig-Wood)
+* WebDAV
+    * Fix Move/Copy/DirMove when using -server-side-across-configs (Nick Craig-Wood)
+
 ## v1.60.0 - 2022-10-21
 
 [See commits](https://github.com/rclone/rclone/compare/v1.59.0...v1.60.0)

From 959cd938bc945c59320a71496d614e71fe4731ae Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 17 Nov 2022 17:56:24 +0000
Subject: [PATCH 396/560] docs: Add minimum versions to all the backend pages
 and some of the other pages

---
 bin/backend-versions.sh             | 17 +++++++++++++++++
 docs/content/alias.md               |  1 +
 docs/content/amazonclouddrive.md    |  1 +
 docs/content/azureblob.md           |  1 +
 docs/content/b2.md                  |  1 +
 docs/content/bisync.md              |  1 +
 docs/content/box.md                 |  1 +
 docs/content/cache.md               |  1 +
 docs/content/chunker.md             |  1 +
 docs/content/combine.md             |  1 +
 docs/content/compress.md            |  1 +
 docs/content/crypt.md               |  1 +
 docs/content/docker.md              |  1 +
 docs/content/drive.md               |  1 +
 docs/content/dropbox.md             |  1 +
 docs/content/fichier.md             |  1 +
 docs/content/filefabric.md          |  1 +
 docs/content/filtering.md           |  1 +
 docs/content/ftp.md                 |  1 +
 docs/content/googlecloudstorage.md  |  1 +
 docs/content/googlephotos.md        |  1 +
 docs/content/gui.md                 |  1 +
 docs/content/hasher.md              |  1 +
 docs/content/hdfs.md                |  1 +
 docs/content/hidrive.md             |  1 +
 docs/content/http.md                |  1 +
 docs/content/internetarchive.md     |  1 +
 docs/content/jottacloud.md          |  1 +
 docs/content/koofr.md               |  1 +
 docs/content/local.md               |  1 +
 docs/content/mailru.md              |  1 +
 docs/content/mega.md                |  1 +
 docs/content/memory.md              |  1 +
 docs/content/netstorage.md          |  1 +
 docs/content/onedrive.md            |  1 +
 docs/content/opendrive.md           |  1 +
 docs/content/oracleobjectstorage.md |  1 +
 docs/content/pcloud.md              |  1 +
 docs/content/premiumizeme.md        |  1 +
 docs/content/putio.md               |  1 +
 docs/content/qingstor.md            |  1 +
 docs/content/rc.md                  |  1 +
 docs/content/s3.md                  |  1 +
 docs/content/seafile.md             |  1 +
 docs/content/sftp.md                |  1 +
 docs/content/sharefile.md           |  1 +
 docs/content/sia.md                 |  1 +
 docs/content/smb.md                 |  1 +
 docs/content/storj.md               |  1 +
 docs/content/sugarsync.md           |  1 +
 docs/content/swift.md               |  1 +
 docs/content/tardigrade.md          |  1 +
 docs/content/union.md               |  1 +
 docs/content/uptobox.md             |  1 +
 docs/content/webdav.md              |  1 +
 docs/content/yandex.md              |  1 +
 docs/content/zoho.md                |  1 +
 docs/layouts/_default/baseof.html   |  1 +
 58 files changed, 74 insertions(+)
 create mode 100755 bin/backend-versions.sh

diff --git a/bin/backend-versions.sh b/bin/backend-versions.sh
new file mode 100755
index 0000000000000..3fb21cc81338a
--- /dev/null
+++ b/bin/backend-versions.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+# This adds the version each backend was released to its docs page
+set -e
+for backend in $( find backend -maxdepth 1 -type d ); do
+    backend=$(basename $backend)
+    if [[ "$backend" == "backend" || "$backend" == "vfs" || "$backend" == "all" || "$backend" == "azurefile" ]]; then
+        continue
+    fi
+    
+    commit=$(git log --oneline -- $backend | tail -1 | cut -d' ' -f1)
+    if [ "$commit" == "" ]; then
+        commit=$(git log --oneline -- backend/$backend | tail -1 | cut -d' ' -f1)
+    fi
+    version=$(git tag --contains $commit | grep ^v | sort -n | head -1)
+    echo $backend $version
+    sed -i~ "4i versionIntroduced: \"$version\"" docs/content/${backend}.md
+done
diff --git a/docs/content/alias.md b/docs/content/alias.md
index eeb7464759f60..a0960c5285169 100644
--- a/docs/content/alias.md
+++ b/docs/content/alias.md
@@ -1,6 +1,7 @@
 ---
 title: "Alias"
 description: "Remote Aliases"
+versionIntroduced: "v1.40"
 ---
 
 # {{< icon "fa fa-link" >}} Alias
diff --git a/docs/content/amazonclouddrive.md b/docs/content/amazonclouddrive.md
index 74ff53df6c61a..2fd16db74828a 100644
--- a/docs/content/amazonclouddrive.md
+++ b/docs/content/amazonclouddrive.md
@@ -1,6 +1,7 @@
 ---
 title: "Amazon Drive"
 description: "Rclone docs for Amazon Drive"
+versionIntroduced: "v1.20"
 ---
 
 # {{< icon "fab fa-amazon" >}} Amazon Drive
diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md
index 6f0200b3be812..537dc1f55fbaf 100644
--- a/docs/content/azureblob.md
+++ b/docs/content/azureblob.md
@@ -1,6 +1,7 @@
 ---
 title: "Microsoft Azure Blob Storage"
 description: "Rclone docs for Microsoft Azure Blob Storage"
+versionIntroduced: "v1.38"
 ---
 
 # {{< icon "fab fa-windows" >}} Microsoft Azure Blob Storage
diff --git a/docs/content/b2.md b/docs/content/b2.md
index bbc8a17c648c0..0bddeaa014532 100644
--- a/docs/content/b2.md
+++ b/docs/content/b2.md
@@ -1,6 +1,7 @@
 ---
 title: "B2"
 description: "Backblaze B2"
+versionIntroduced: "v1.26"
 ---
 
 # {{< icon "fa fa-fire" >}} Backblaze B2
diff --git a/docs/content/bisync.md b/docs/content/bisync.md
index 78bf06a0b267a..77b46bb82b9ce 100644
--- a/docs/content/bisync.md
+++ b/docs/content/bisync.md
@@ -1,6 +1,7 @@
 ---
 title: "Bisync"
 description: "Bidirectional cloud sync solution in rclone"
+versionIntroduced: "v1.58.0"
 ---
 
 ## Getting started {#getting-started}
diff --git a/docs/content/box.md b/docs/content/box.md
index 9da20764a03de..906d5d0ad7580 100644
--- a/docs/content/box.md
+++ b/docs/content/box.md
@@ -1,6 +1,7 @@
 ---
 title: "Box"
 description: "Rclone docs for Box"
+versionIntroduced: "v1.38"
 ---
 
 # {{< icon "fa fa-archive" >}} Box
diff --git a/docs/content/cache.md b/docs/content/cache.md
index 592f9eef4f2f0..bd28b037b0bf0 100644
--- a/docs/content/cache.md
+++ b/docs/content/cache.md
@@ -1,6 +1,7 @@
 ---
 title: "Cache"
 description: "Rclone docs for cache remote"
+versionIntroduced: "v1.39"
 ---
 
 # {{< icon "fa fa-archive" >}} Cache (DEPRECATED)
diff --git a/docs/content/chunker.md b/docs/content/chunker.md
index 4ba8491db71ed..934f102fff21f 100644
--- a/docs/content/chunker.md
+++ b/docs/content/chunker.md
@@ -1,6 +1,7 @@
 ---
 title: "Chunker"
 description: "Split-chunking overlay remote"
+versionIntroduced: "v1.50.0"
 ---
 
 # {{< icon "fa fa-cut" >}}Chunker (BETA)
diff --git a/docs/content/combine.md b/docs/content/combine.md
index f5d3c2da3d043..58137a20a7db1 100644
--- a/docs/content/combine.md
+++ b/docs/content/combine.md
@@ -1,6 +1,7 @@
 ---
 title: "Combine"
 description: "Combine several remotes into one"
+versionIntroduced: "v1.59.0"
 ---
 
 # {{< icon "fa fa-folder-plus" >}} Combine
diff --git a/docs/content/compress.md b/docs/content/compress.md
index f9f3afbc043a1..e6bf533b7ca93 100644
--- a/docs/content/compress.md
+++ b/docs/content/compress.md
@@ -1,6 +1,7 @@
 ---
 title: "Compress"
 description: "Compression Remote"
+versionIntroduced: "v1.54.0"
 ---
 
 # {{< icon "fas fa-compress" >}}Compress (Experimental)
diff --git a/docs/content/crypt.md b/docs/content/crypt.md
index c3af9de33f2cb..3cde0ae024689 100644
--- a/docs/content/crypt.md
+++ b/docs/content/crypt.md
@@ -1,6 +1,7 @@
 ---
 title: "Crypt"
 description: "Encryption overlay remote"
+versionIntroduced: "v1.33"
 ---
 
 # {{< icon "fa fa-lock" >}}Crypt
diff --git a/docs/content/docker.md b/docs/content/docker.md
index 62e6e0b51b575..c8a1746d772a5 100644
--- a/docs/content/docker.md
+++ b/docs/content/docker.md
@@ -1,6 +1,7 @@
 ---
 title: "Docker Volume Plugin"
 description: "Docker Volume Plugin"
+versionIntroduced: "v1.56.0"
 ---
 
 # Docker Volume Plugin
diff --git a/docs/content/drive.md b/docs/content/drive.md
index 0277eba31be45..922961ef442a3 100644
--- a/docs/content/drive.md
+++ b/docs/content/drive.md
@@ -1,6 +1,7 @@
 ---
 title: "Google drive"
 description: "Rclone docs for Google drive"
+versionIntroduced: "v0.91"
 ---
 
 # {{< icon "fab fa-google" >}} Google Drive
diff --git a/docs/content/dropbox.md b/docs/content/dropbox.md
index a8f4fe83f6ea2..e6c69654c10cf 100644
--- a/docs/content/dropbox.md
+++ b/docs/content/dropbox.md
@@ -1,6 +1,7 @@
 ---
 title: "Dropbox"
 description: "Rclone docs for Dropbox"
+versionIntroduced: "v1.02"
 ---
 
 # {{< icon "fab fa-dropbox" >}} Dropbox
diff --git a/docs/content/fichier.md b/docs/content/fichier.md
index 0e2259b693e4b..a71926f7620b9 100644
--- a/docs/content/fichier.md
+++ b/docs/content/fichier.md
@@ -1,6 +1,7 @@
 ---
 title: "1Fichier"
 description: "Rclone docs for 1Fichier"
+versionIntroduced: "v1.49.0"
 ---
 
 # {{< icon "fa fa-archive" >}} 1Fichier
diff --git a/docs/content/filefabric.md b/docs/content/filefabric.md
index d66b48162e164..31469925a431f 100644
--- a/docs/content/filefabric.md
+++ b/docs/content/filefabric.md
@@ -1,6 +1,7 @@
 ---
 title: "Enterprise File Fabric"
 description: "Rclone docs for the Enterprise File Fabric backend"
+versionIntroduced: "v1.54.0"
 ---
 
 # {{< icon "fa fa-cloud" >}} Enterprise File Fabric
diff --git a/docs/content/filtering.md b/docs/content/filtering.md
index cf526961cf991..e315dfffc8f22 100644
--- a/docs/content/filtering.md
+++ b/docs/content/filtering.md
@@ -1,6 +1,7 @@
 ---
 title: "Rclone Filtering"
 description: "Rclone filtering, includes and excludes"
+versionIntroduced: "v1.22"
 ---
 
 # Filtering, includes and excludes
diff --git a/docs/content/ftp.md b/docs/content/ftp.md
index 0ccd0f915446b..563de41322ea4 100644
--- a/docs/content/ftp.md
+++ b/docs/content/ftp.md
@@ -1,6 +1,7 @@
 ---
 title: "FTP"
 description: "Rclone docs for FTP backend"
+versionIntroduced: "v1.37"
 ---
 
 # {{< icon "fa fa-file" >}} FTP
diff --git a/docs/content/googlecloudstorage.md b/docs/content/googlecloudstorage.md
index b98f890352bc1..2a254848f90c9 100644
--- a/docs/content/googlecloudstorage.md
+++ b/docs/content/googlecloudstorage.md
@@ -1,6 +1,7 @@
 ---
 title: "Google Cloud Storage"
 description: "Rclone docs for Google Cloud Storage"
+versionIntroduced: "v1.02"
 ---
 
 # {{< icon "fab fa-google" >}} Google Cloud Storage
diff --git a/docs/content/googlephotos.md b/docs/content/googlephotos.md
index da46f015687f0..bef6f3cf39c9c 100644
--- a/docs/content/googlephotos.md
+++ b/docs/content/googlephotos.md
@@ -1,6 +1,7 @@
 ---
 title: "Google Photos"
 description: "Rclone docs for Google Photos"
+versionIntroduced: "v1.49.0"
 ---
 
 # {{< icon "fa fa-images" >}} Google Photos
diff --git a/docs/content/gui.md b/docs/content/gui.md
index 32236bc117637..35f80a90ae388 100644
--- a/docs/content/gui.md
+++ b/docs/content/gui.md
@@ -1,6 +1,7 @@
 ---
 title: "GUI"
 description: "Web based Graphical User Interface"
+versionIntroduced: "v1.49.0"
 ---
 
 # GUI (Experimental)
diff --git a/docs/content/hasher.md b/docs/content/hasher.md
index c92392af7ff90..aa492d3a771cb 100644
--- a/docs/content/hasher.md
+++ b/docs/content/hasher.md
@@ -1,6 +1,7 @@
 ---
 title: "Hasher"
 description: "Better checksums for other remotes"
+versionIntroduced: "v1.57.0"
 ---
 
 # {{< icon "fa fa-check-double" >}} Hasher (EXPERIMENTAL)
diff --git a/docs/content/hdfs.md b/docs/content/hdfs.md
index d5c1d69e5c176..feee7f561813f 100644
--- a/docs/content/hdfs.md
+++ b/docs/content/hdfs.md
@@ -1,6 +1,7 @@
 ---
 title: "HDFS Remote"
 description: "Remote for Hadoop Distributed Filesystem"
+versionIntroduced: "v1.54.0"
 ---
 
 # {{< icon "fa fa-globe" >}} HDFS
diff --git a/docs/content/hidrive.md b/docs/content/hidrive.md
index cd763cec3e851..0a75686ca62f2 100644
--- a/docs/content/hidrive.md
+++ b/docs/content/hidrive.md
@@ -1,6 +1,7 @@
 ---
 title: "HiDrive"
 description: "Rclone docs for HiDrive"
+versionIntroduced: "v1.59.0"
 ---
 
 # {{< icon "fa fa-cloud" >}} HiDrive
diff --git a/docs/content/http.md b/docs/content/http.md
index d98052c598d81..93c5aee66bd03 100644
--- a/docs/content/http.md
+++ b/docs/content/http.md
@@ -1,6 +1,7 @@
 ---
 title: "HTTP Remote"
 description: "Read only remote for HTTP servers"
+versionIntroduced: "v1.37"
 ---
 
 # {{< icon "fa fa-globe" >}} HTTP
diff --git a/docs/content/internetarchive.md b/docs/content/internetarchive.md
index ae3393d80f2c0..f0a90278c5333 100644
--- a/docs/content/internetarchive.md
+++ b/docs/content/internetarchive.md
@@ -1,6 +1,7 @@
 ---
 title: "Internet Archive"
 description: "Rclone docs for Internet Archive"
+versionIntroduced: "v1.59.0"
 ---
 
 # {{< icon "fa fa-archive" >}} Internet Archive
diff --git a/docs/content/jottacloud.md b/docs/content/jottacloud.md
index 4a61d2e78bd49..24bcda7e9e5ef 100644
--- a/docs/content/jottacloud.md
+++ b/docs/content/jottacloud.md
@@ -1,6 +1,7 @@
 ---
 title: "Jottacloud"
 description: "Rclone docs for Jottacloud"
+versionIntroduced: "v1.43"
 ---
 
 # {{< icon "fa fa-cloud" >}} Jottacloud
diff --git a/docs/content/koofr.md b/docs/content/koofr.md
index 2025d64a318e4..69295385bb6a8 100644
--- a/docs/content/koofr.md
+++ b/docs/content/koofr.md
@@ -1,6 +1,7 @@
 ---
 title: "Koofr"
 description: "Rclone docs for Koofr"
+versionIntroduced: "v1.47.0"
 ---
 
 # {{< icon "fa fa-suitcase" >}} Koofr
diff --git a/docs/content/local.md b/docs/content/local.md
index 8b7920c91a7a3..70fbf9315f468 100644
--- a/docs/content/local.md
+++ b/docs/content/local.md
@@ -1,6 +1,7 @@
 ---
 title: "Local Filesystem"
 description: "Rclone docs for the local filesystem"
+versionIntroduced: "v0.91"
 ---
 
 # {{< icon "fas fa-hdd" >}} Local Filesystem
diff --git a/docs/content/mailru.md b/docs/content/mailru.md
index 5814283c8b291..758cb4b9e5d5f 100644
--- a/docs/content/mailru.md
+++ b/docs/content/mailru.md
@@ -1,6 +1,7 @@
 ---
 title: "Mailru"
 description: "Mail.ru Cloud"
+versionIntroduced: "v1.50.0"
 ---
 
 # {{< icon "fas fa-at" >}} Mail.ru Cloud
diff --git a/docs/content/mega.md b/docs/content/mega.md
index 7cc95adb62001..94563b4700d4f 100644
--- a/docs/content/mega.md
+++ b/docs/content/mega.md
@@ -1,6 +1,7 @@
 ---
 title: "Mega"
 description: "Rclone docs for Mega"
+versionIntroduced: "v1.41"
 ---
 
 # {{< icon "fa fa-archive" >}} Mega
diff --git a/docs/content/memory.md b/docs/content/memory.md
index 2b07fab134ed4..760ed9c1c7344 100644
--- a/docs/content/memory.md
+++ b/docs/content/memory.md
@@ -1,6 +1,7 @@
 ---
 title: "Memory"
 description: "Rclone docs for Memory backend"
+versionIntroduced: "v1.51.0"
 ---
 
 # {{< icon "fas fa-memory" >}} Memory
diff --git a/docs/content/netstorage.md b/docs/content/netstorage.md
index eab479c1ce462..5e16f58902b7f 100644
--- a/docs/content/netstorage.md
+++ b/docs/content/netstorage.md
@@ -1,6 +1,7 @@
 ---
 title: "Akamai Netstorage"
 description: "Rclone docs for Akamai NetStorage"
+versionIntroduced: "v1.58.0"
 ---
 
 # {{< icon "fas fa-database" >}} Akamai NetStorage
diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md
index fcff1b76e3f76..43750597b157b 100644
--- a/docs/content/onedrive.md
+++ b/docs/content/onedrive.md
@@ -1,6 +1,7 @@
 ---
 title: "Microsoft OneDrive"
 description: "Rclone docs for Microsoft OneDrive"
+versionIntroduced: "v1.24"
 ---
 
 # {{< icon "fab fa-windows" >}} Microsoft OneDrive
diff --git a/docs/content/opendrive.md b/docs/content/opendrive.md
index 772bc051fa434..0758470a6631a 100644
--- a/docs/content/opendrive.md
+++ b/docs/content/opendrive.md
@@ -1,6 +1,7 @@
 ---
 title: "OpenDrive"
 description: "Rclone docs for OpenDrive"
+versionIntroduced: "v1.42"
 ---
 
 # {{< icon "fa fa-file" >}} OpenDrive
diff --git a/docs/content/oracleobjectstorage.md b/docs/content/oracleobjectstorage.md
index d25dfc303d5ad..f25dcca5db8f1 100644
--- a/docs/content/oracleobjectstorage.md
+++ b/docs/content/oracleobjectstorage.md
@@ -1,6 +1,7 @@
 ---
 title: "Oracle Object Storage"
 description: "Rclone docs for Oracle Object Storage"
+versionIntroduced: "v1.60.0"
 ---
 
 # {{< icon "fa fa-cloud" >}} Oracle Object Storage
diff --git a/docs/content/pcloud.md b/docs/content/pcloud.md
index 6e0c4458ab088..cd15bb6424b81 100644
--- a/docs/content/pcloud.md
+++ b/docs/content/pcloud.md
@@ -1,6 +1,7 @@
 ---
 title: "pCloud"
 description: "Rclone docs for pCloud"
+versionIntroduced: "v1.39"
 ---
 
 # {{< icon "fa fa-cloud" >}} pCloud
diff --git a/docs/content/premiumizeme.md b/docs/content/premiumizeme.md
index e8039764d1eef..7b53b77b5f2db 100644
--- a/docs/content/premiumizeme.md
+++ b/docs/content/premiumizeme.md
@@ -1,6 +1,7 @@
 ---
 title: "premiumize.me"
 description: "Rclone docs for premiumize.me"
+versionIntroduced: "v1.49.0"
 ---
 
 # {{< icon "fa fa-user" >}} premiumize.me
diff --git a/docs/content/putio.md b/docs/content/putio.md
index d98da528e57ee..79fb346a9d8dd 100644
--- a/docs/content/putio.md
+++ b/docs/content/putio.md
@@ -1,6 +1,7 @@
 ---
 title: "put.io"
 description: "Rclone docs for put.io"
+versionIntroduced: "v1.49.0"
 ---
 
 # {{< icon "fas fa-parking" >}} put.io
diff --git a/docs/content/qingstor.md b/docs/content/qingstor.md
index 971e59bc6f94d..68772f4dda19a 100644
--- a/docs/content/qingstor.md
+++ b/docs/content/qingstor.md
@@ -1,6 +1,7 @@
 ---
 title: "QingStor"
 description: "Rclone docs for QingStor Object Storage"
+versionIntroduced: "v1.38"
 ---
 
 # {{< icon "fas fa-hdd" >}} QingStor
diff --git a/docs/content/rc.md b/docs/content/rc.md
index 4b966635e578c..3143b956d8b06 100644
--- a/docs/content/rc.md
+++ b/docs/content/rc.md
@@ -1,6 +1,7 @@
 ---
 title: "Remote Control / API"
 description: "Remote controlling rclone with its API"
+versionIntroduced: "v1.40"
 ---
 
 # Remote controlling rclone with its API
diff --git a/docs/content/s3.md b/docs/content/s3.md
index 4ffa23922e4b9..a5d8bf0e8cefe 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -1,6 +1,7 @@
 ---
 title: "Amazon S3"
 description: "Rclone docs for Amazon S3"
+versionIntroduced: "v0.91"
 ---
 
 # {{< icon "fab fa-amazon" >}} Amazon S3 Storage Providers
diff --git a/docs/content/seafile.md b/docs/content/seafile.md
index 132a405a13e2b..ab45184ece1d1 100644
--- a/docs/content/seafile.md
+++ b/docs/content/seafile.md
@@ -1,6 +1,7 @@
 ---
 title: "Seafile"
 description: "Seafile"
+versionIntroduced: "v1.52.0"
 ---
 
 # {{< icon "fa fa-server" >}}Seafile
diff --git a/docs/content/sftp.md b/docs/content/sftp.md
index dfaf179b81582..bbbbcf7637fc5 100644
--- a/docs/content/sftp.md
+++ b/docs/content/sftp.md
@@ -1,6 +1,7 @@
 ---
 title: "SFTP"
 description: "SFTP"
+versionIntroduced: "v1.36"
 ---
 
 # {{< icon "fa fa-server" >}} SFTP
diff --git a/docs/content/sharefile.md b/docs/content/sharefile.md
index 19cdc939700a3..d85ec4b19b7b0 100644
--- a/docs/content/sharefile.md
+++ b/docs/content/sharefile.md
@@ -1,6 +1,7 @@
 ---
 title: "Citrix ShareFile"
 description: "Rclone docs for Citrix ShareFile"
+versionIntroduced: "v1.50.0"
 ---
 
 # {{< icon "fas fa-share-square" >}} Citrix ShareFile
diff --git a/docs/content/sia.md b/docs/content/sia.md
index 9cdb24b25bec2..9bc93c5c9f5be 100644
--- a/docs/content/sia.md
+++ b/docs/content/sia.md
@@ -1,6 +1,7 @@
 ---
 title: "Sia"
 description: "Remote for Sia Decentralized Cloud"
+versionIntroduced: "v1.57.0"
 date: "2019-10-02"
 ---
 
diff --git a/docs/content/smb.md b/docs/content/smb.md
index 62c1717bb9274..8f9769411a9f3 100644
--- a/docs/content/smb.md
+++ b/docs/content/smb.md
@@ -1,6 +1,7 @@
 ---
 title: "SMB / CIFS"
 description: "Rclone docs for SMB backend"
+versionIntroduced: "v1.60.0"
 ---
 
 # {{< icon "fa fa-server" >}} SMB
diff --git a/docs/content/storj.md b/docs/content/storj.md
index 6a8e416239e22..dfafc5ba3879a 100644
--- a/docs/content/storj.md
+++ b/docs/content/storj.md
@@ -1,6 +1,7 @@
 ---
 title: "Storj"
 description: "Rclone docs for Storj"
+versionIntroduced: "v1.52.0"
 ---
 
 # {{< icon "fas fa-dove" >}} Storj
diff --git a/docs/content/sugarsync.md b/docs/content/sugarsync.md
index 58c163b325931..798e251dae458 100644
--- a/docs/content/sugarsync.md
+++ b/docs/content/sugarsync.md
@@ -1,6 +1,7 @@
 ---
 title: "SugarSync"
 description: "Rclone docs for SugarSync"
+versionIntroduced: "v1.51.0"
 ---
 
 # {{< icon "fas fa-dove" >}} SugarSync
diff --git a/docs/content/swift.md b/docs/content/swift.md
index c58706efc067b..e768e35ffc37a 100644
--- a/docs/content/swift.md
+++ b/docs/content/swift.md
@@ -1,6 +1,7 @@
 ---
 title: "Swift"
 description: "Swift"
+versionIntroduced: "v0.91"
 ---
 
 # {{< icon "fa fa-space-shuttle" >}}Swift
diff --git a/docs/content/tardigrade.md b/docs/content/tardigrade.md
index 3c84ecca72a77..b7f8819f8d6e8 100644
--- a/docs/content/tardigrade.md
+++ b/docs/content/tardigrade.md
@@ -1,6 +1,7 @@
 ---
 title: "Tardigrade"
 description: "Rclone docs for Tardigrade"
+versionIntroduced: "v1.52.0"
 ---
 
 # {{< icon "fas fa-dove" >}} Tardigrade
diff --git a/docs/content/union.md b/docs/content/union.md
index d7f37fa7194c8..bce1c44aa0b4c 100644
--- a/docs/content/union.md
+++ b/docs/content/union.md
@@ -1,6 +1,7 @@
 ---
 title: "Union"
 description: "Remote Unification"
+versionIntroduced: "v1.44"
 ---
 
 # {{< icon "fa fa-link" >}} Union
diff --git a/docs/content/uptobox.md b/docs/content/uptobox.md
index 83f63916bc7c3..3e2e12e1980fe 100644
--- a/docs/content/uptobox.md
+++ b/docs/content/uptobox.md
@@ -1,6 +1,7 @@
 ---
 title: "Uptobox"
 description: "Rclone docs for Uptobox"
+versionIntroduced: "v1.56.0"
 ---
 
 # {{< icon "fa fa-archive" >}} Uptobox
diff --git a/docs/content/webdav.md b/docs/content/webdav.md
index 9780941ebe4b0..4ded05d1b053a 100644
--- a/docs/content/webdav.md
+++ b/docs/content/webdav.md
@@ -1,6 +1,7 @@
 ---
 title: "WebDAV"
 description: "Rclone docs for WebDAV"
+versionIntroduced: "v1.39"
 ---
 
 # {{< icon "fa fa-globe" >}} WebDAV
diff --git a/docs/content/yandex.md b/docs/content/yandex.md
index 37176a40af752..f6c32e1a907a6 100644
--- a/docs/content/yandex.md
+++ b/docs/content/yandex.md
@@ -1,6 +1,7 @@
 ---
 title: "Yandex"
 description: "Yandex Disk"
+versionIntroduced: "v1.26"
 ---
 
 # {{< icon "fa fa-space-shuttle" >}}Yandex Disk
diff --git a/docs/content/zoho.md b/docs/content/zoho.md
index 187eb5ff1c64d..26e5ccb699db1 100644
--- a/docs/content/zoho.md
+++ b/docs/content/zoho.md
@@ -1,6 +1,7 @@
 ---
 title: "Zoho"
 description: "Zoho WorkDrive"
+versionIntroduced: "v1.54.0"
 ---
 
 # {{< icon "fas fa-folder" >}}Zoho Workdrive
diff --git a/docs/layouts/_default/baseof.html b/docs/layouts/_default/baseof.html
index d3e327ebeab4d..6598c2f8c79dd 100644
--- a/docs/layouts/_default/baseof.html
+++ b/docs/layouts/_default/baseof.html
@@ -26,6 +26,7 @@
     
                  
       
                  
         
                  
+          {{ if .Params.versionIntroduced }}{{ .Params.versionIntroduced }}{{ end }}
           {{ block "main" . }}
           {{ end }}
         
                  

From 7db1c506f261698a536c343de39b628fc82e13bb Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Sat, 19 Nov 2022 00:53:21 +0100
Subject: [PATCH 397/560] smb: fix issue where spurious dot directory is
 created

---
 backend/smb/smb.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/backend/smb/smb.go b/backend/smb/smb.go
index 3e750cc1fb9df..688221ce44c3e 100644
--- a/backend/smb/smb.go
+++ b/backend/smb/smb.go
@@ -478,11 +478,15 @@ func (f *Fs) makeEntryRelative(share, _path, relative string, stat os.FileInfo)
 }
 
 func (f *Fs) ensureDirectory(ctx context.Context, share, _path string) error {
+	dir := path.Dir(_path)
+	if dir == "." {
+		return nil
+	}
 	cn, err := f.getConnection(ctx, share)
 	if err != nil {
 		return err
 	}
-	err = cn.smbShare.MkdirAll(f.toSambaPath(path.Dir(_path)), 0o755)
+	err = cn.smbShare.MkdirAll(f.toSambaPath(dir), 0o755)
 	f.putConnection(&cn)
 	return err
 }

From dd71f5d968e36fb2be65bf73ec9cc518cb66ee17 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Sat, 6 Aug 2022 16:30:45 +0100
Subject: [PATCH 398/560] fs: move operations.NewOverrideRemote to
 fs.NewOverrideRemote

---
 backend/combine/combine.go                |  2 +-
 backend/googlephotos/googlephotos_test.go |  5 +-
 fs/operations/operations.go               | 70 +----------------------
 fs/override.go                            | 68 ++++++++++++++++++++++
 fs/override_test.go                       |  4 ++
 5 files changed, 76 insertions(+), 73 deletions(-)
 create mode 100644 fs/override.go
 create mode 100644 fs/override_test.go

diff --git a/backend/combine/combine.go b/backend/combine/combine.go
index 34644a9681326..63fb4635ecee5 100644
--- a/backend/combine/combine.go
+++ b/backend/combine/combine.go
@@ -631,7 +631,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
 	if err != nil {
 		return nil, err
 	}
-	uSrc := operations.NewOverrideRemote(src, uRemote)
+	uSrc := fs.NewOverrideRemote(src, uRemote)
 	var o fs.Object
 	if stream {
 		o, err = u.f.Features().PutStream(ctx, in, uSrc, options...)
diff --git a/backend/googlephotos/googlephotos_test.go b/backend/googlephotos/googlephotos_test.go
index 3cc3a2706d018..cd1e0943f8411 100644
--- a/backend/googlephotos/googlephotos_test.go
+++ b/backend/googlephotos/googlephotos_test.go
@@ -12,7 +12,6 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
@@ -56,7 +55,7 @@ func TestIntegration(t *testing.T) {
 			require.NoError(t, err)
 			in, err := srcObj.Open(ctx)
 			require.NoError(t, err)
-			dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
+			dstObj, err := f.Put(ctx, in, fs.NewOverrideRemote(srcObj, remote))
 			require.NoError(t, err)
 			assert.Equal(t, remote, dstObj.Remote())
 			_ = in.Close()
@@ -221,7 +220,7 @@ func TestIntegration(t *testing.T) {
 		require.NoError(t, err)
 		in, err := srcObj.Open(ctx)
 		require.NoError(t, err)
-		dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
+		dstObj, err := f.Put(ctx, in, fs.NewOverrideRemote(srcObj, remote))
 		require.NoError(t, err)
 		assert.Equal(t, remote, dstObj.Remote())
 		_ = in.Close()
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 4b4afd041b134..3510663327163 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -297,74 +297,6 @@ func removeFailedCopy(ctx context.Context, dst fs.Object) bool {
 	return true
 }
 
-// OverrideRemote is a wrapper to override the Remote for an
-// ObjectInfo
-type OverrideRemote struct {
-	fs.ObjectInfo
-	remote string
-}
-
-// NewOverrideRemote returns an OverrideRemoteObject which will
-// return the remote specified
-func NewOverrideRemote(oi fs.ObjectInfo, remote string) *OverrideRemote {
-	return &OverrideRemote{
-		ObjectInfo: oi,
-		remote:     remote,
-	}
-}
-
-// Remote returns the overridden remote name
-func (o *OverrideRemote) Remote() string {
-	return o.remote
-}
-
-// MimeType returns the mime type of the underlying object or "" if it
-// can't be worked out
-func (o *OverrideRemote) MimeType(ctx context.Context) string {
-	if do, ok := o.ObjectInfo.(fs.MimeTyper); ok {
-		return do.MimeType(ctx)
-	}
-	return ""
-}
-
-// ID returns the ID of the Object if known, or "" if not
-func (o *OverrideRemote) ID() string {
-	if do, ok := o.ObjectInfo.(fs.IDer); ok {
-		return do.ID()
-	}
-	return ""
-}
-
-// UnWrap returns the Object that this Object is wrapping or nil if it
-// isn't wrapping anything
-func (o *OverrideRemote) UnWrap() fs.Object {
-	if o, ok := o.ObjectInfo.(fs.Object); ok {
-		return o
-	}
-	return nil
-}
-
-// GetTier returns storage tier or class of the Object
-func (o *OverrideRemote) GetTier() string {
-	if do, ok := o.ObjectInfo.(fs.GetTierer); ok {
-		return do.GetTier()
-	}
-	return ""
-}
-
-// Metadata returns metadata for an object
-//
-// It should return nil if there is no Metadata
-func (o *OverrideRemote) Metadata(ctx context.Context) (fs.Metadata, error) {
-	if do, ok := o.ObjectInfo.(fs.Metadataer); ok {
-		return do.Metadata(ctx)
-	}
-	return nil, nil
-}
-
-// Check all optional interfaces satisfied
-var _ fs.FullObjectInfo = (*OverrideRemote)(nil)
-
 // CommonHash returns a single hash.Type and a HashOption with that
 // type which is in common between the two fs.Fs.
 func CommonHash(ctx context.Context, fa, fb fs.Info) (hash.Type, *fs.HashesOption) {
@@ -491,7 +423,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 						var wrappedSrc fs.ObjectInfo = src
 						// We try to pass the original object if possible
 						if src.Remote() != remote {
-							wrappedSrc = NewOverrideRemote(src, remote)
+							wrappedSrc = fs.NewOverrideRemote(src, remote)
 						}
 						options := []fs.OpenOption{hashOption}
 						for _, option := range ci.UploadHeaders {
diff --git a/fs/override.go b/fs/override.go
new file mode 100644
index 0000000000000..014acb8ae4ed1
--- /dev/null
+++ b/fs/override.go
@@ -0,0 +1,68 @@
+package fs
+
+import "context"
+
+// OverrideRemote is a wrapper to override the Remote for an
+// ObjectInfo
+type OverrideRemote struct {
+	ObjectInfo
+	remote string
+}
+
+// NewOverrideRemote returns an OverrideRemoteObject which will
+// return the remote specified
+func NewOverrideRemote(oi ObjectInfo, remote string) *OverrideRemote {
+	return &OverrideRemote{
+		ObjectInfo: oi,
+		remote:     remote,
+	}
+}
+
+// Remote returns the overridden remote name
+func (o *OverrideRemote) Remote() string {
+	return o.remote
+}
+
+// MimeType returns the mime type of the underlying object or "" if it
+// can't be worked out
+func (o *OverrideRemote) MimeType(ctx context.Context) string {
+	if do, ok := o.ObjectInfo.(MimeTyper); ok {
+		return do.MimeType(ctx)
+	}
+	return ""
+}
+
+// ID returns the ID of the Object if known, or "" if not
+func (o *OverrideRemote) ID() string {
+	if do, ok := o.ObjectInfo.(IDer); ok {
+		return do.ID()
+	}
+	return ""
+}
+
+// UnWrap returns the Object that this Object is wrapping or nil if it
+// isn't wrapping anything
+func (o *OverrideRemote) UnWrap() Object {
+	if o, ok := o.ObjectInfo.(Object); ok {
+		return o
+	}
+	return nil
+}
+
+// GetTier returns storage tier or class of the Object
+func (o *OverrideRemote) GetTier() string {
+	if do, ok := o.ObjectInfo.(GetTierer); ok {
+		return do.GetTier()
+	}
+	return ""
+}
+
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *OverrideRemote) Metadata(ctx context.Context) (Metadata, error) {
+	if do, ok := o.ObjectInfo.(Metadataer); ok {
+		return do.Metadata(ctx)
+	}
+	return nil, nil
+}
diff --git a/fs/override_test.go b/fs/override_test.go
new file mode 100644
index 0000000000000..f60dc7bdbfe20
--- /dev/null
+++ b/fs/override_test.go
@@ -0,0 +1,4 @@
+package fs
+
+// Check all optional interfaces satisfied
+var _ FullObjectInfo = (*OverrideRemote)(nil)

From 430bf0d5eb65d948142f3bee41e20e2776372fcb Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Sat, 6 Aug 2022 16:32:58 +0100
Subject: [PATCH 399/560] crypt: fix compress wrapping crypt giving upload
 errors

Before this fix a chain compress -> crypt -> s3 was giving errors

    BadDigest: The Content-MD5 you specified did not match what we received.

This was because the crypt backend was encrypting the underlying local
object to calculate the hash rather than the contents of the metadata
stream.

It did this because the crypt backend incorrectly identified the
object as a local object.

This fixes the problem by making sure the crypt backend does not
unwrap anything but fs.OverrideRemote objects.

See: https://forum.rclone.org/t/not-encrypting-or-compressing-before-upload/32261/10
---
 backend/crypt/crypt.go               |  5 +++--
 backend/crypt/crypt_internal_test.go | 15 +--------------
 2 files changed, 4 insertions(+), 16 deletions(-)

diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go
index 85972298e9011..1aece2753262d 100644
--- a/backend/crypt/crypt.go
+++ b/backend/crypt/crypt.go
@@ -1047,10 +1047,11 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 	// Get the underlying object if there is one
 	if srcObj, ok = o.ObjectInfo.(fs.Object); ok {
 		// Prefer direct interface assertion
-	} else if do, ok := o.ObjectInfo.(fs.ObjectUnWrapper); ok {
-		// Otherwise likely is an operations.OverrideRemote
+	} else if do, ok := o.ObjectInfo.(*fs.OverrideRemote); ok {
+		// Unwrap if it is an operations.OverrideRemote
 		srcObj = do.UnWrap()
 	} else {
+		// Otherwise don't unwrap any further
 		return "", nil
 	}
 	// if this is wrapping a local object then we work out the hash
diff --git a/backend/crypt/crypt_internal_test.go b/backend/crypt/crypt_internal_test.go
index 82587763061d7..07dee6ac525b7 100644
--- a/backend/crypt/crypt_internal_test.go
+++ b/backend/crypt/crypt_internal_test.go
@@ -17,19 +17,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-type testWrapper struct {
-	fs.ObjectInfo
-}
-
-// UnWrap returns the Object that this Object is wrapping or nil if it
-// isn't wrapping anything
-func (o testWrapper) UnWrap() fs.Object {
-	if o, ok := o.ObjectInfo.(fs.Object); ok {
-		return o
-	}
-	return nil
-}
-
 // Create a temporary local fs to upload things from
 
 func makeTempLocalFs(t *testing.T) (localFs fs.Fs, cleanup func()) {
@@ -83,7 +70,7 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	var oi fs.ObjectInfo = obj
 	if wrap {
 		// wrap the object in an fs.ObjectUnwrapper if required
-		oi = testWrapper{oi}
+		oi = fs.NewOverrideRemote(oi, "new_remote")
 	}
 
 	// wrap the object in a crypt for upload using the nonce we

From 83551bb02e2bb92dcc2296f30aed454043c3494b Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 14 Nov 2022 18:16:36 +0000
Subject: [PATCH 400/560] cmount: update cgofuse for FUSE-T support for
 mounting volumes on Mac

See: https://forum.rclone.org/t/fr-fuse-t-support-for-mounting-volumes-on-mac/33110/
---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 5f073eff3cdd6..8d7c46ff46b3f 100644
--- a/go.mod
+++ b/go.mod
@@ -55,7 +55,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.0
 	github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf
-	github.com/winfsp/cgofuse v1.5.1-0.20220421173602-ce7e5a65cac7
+	github.com/winfsp/cgofuse v1.5.1-0.20221106144041-c23bd0f99a49
 	github.com/xanzy/ssh-agent v0.3.1
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0
diff --git a/go.sum b/go.sum
index 141df3fac81f7..6b7e2d6dff843 100644
--- a/go.sum
+++ b/go.sum
@@ -635,6 +635,8 @@ github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/winfsp/cgofuse v1.5.1-0.20220421173602-ce7e5a65cac7 h1:Lvi511+8ZWFr9YWbvXkxudGSjcrjM1NVPDIPeWusvDk=
 github.com/winfsp/cgofuse v1.5.1-0.20220421173602-ce7e5a65cac7/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I=
+github.com/winfsp/cgofuse v1.5.1-0.20221106144041-c23bd0f99a49 h1:+kD4nvjSbuESNeVUdkrppns7qRJNKSrWBWjrg9o3TV8=
+github.com/winfsp/cgofuse v1.5.1-0.20221106144041-c23bd0f99a49/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I=
 github.com/xanzy/ssh-agent v0.3.1 h1:AmzO1SSWxw73zxFZPRwaMN1MohDw8UyHnmuxyceTEGo=
 github.com/xanzy/ssh-agent v0.3.1/go.mod h1:QIE4lCeL7nkC25x+yA3LBIYfwCc1TFziCtG7cBAac6w=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=

From e3d593d40c4a5da19db6e708489c6dc98c49bcbe Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 24 Nov 2022 11:05:54 +0000
Subject: [PATCH 401/560] build: update dependencies

---
 go.mod |  75 +++++-----
 go.sum | 435 +++++++++++++++++++++++++++++++++++++++++++++++----------
 2 files changed, 396 insertions(+), 114 deletions(-)

diff --git a/go.mod b/go.mod
index 8d7c46ff46b3f..3f7055de4366c 100644
--- a/go.mod
+++ b/go.mod
@@ -16,15 +16,15 @@ require (
 	github.com/anacrolix/dms v1.5.0
 	github.com/artyom/mtab v1.0.0
 	github.com/atotto/clipboard v0.1.4
-	github.com/aws/aws-sdk-go v1.44.70
+	github.com/aws/aws-sdk-go v1.44.145
 	github.com/buengese/sgzip v0.1.1
 	github.com/colinmarc/hdfs/v2 v2.3.0
 	github.com/coreos/go-semver v0.3.0
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00
-	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.4
+	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
 	github.com/gabriel-vasile/mimetype v1.4.1
-	github.com/gdamore/tcell/v2 v2.5.2
+	github.com/gdamore/tcell/v2 v2.5.3
 	github.com/go-chi/chi/v5 v5.0.7
 	github.com/google/uuid v1.3.0
 	github.com/hanwen/go-fuse/v2 v2.1.0
@@ -32,49 +32,50 @@ require (
 	github.com/iguanesolutions/go-systemd/v5 v5.1.0
 	github.com/jcmturner/gokrb5/v8 v8.4.3
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004
-	github.com/klauspost/compress v1.15.9
+	github.com/klauspost/compress v1.15.12
 	github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348
 	github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a
-	github.com/mattn/go-colorable v0.1.12
-	github.com/mattn/go-runewidth v0.0.13
+	github.com/mattn/go-colorable v0.1.13
+	github.com/mattn/go-runewidth v0.0.14
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1
 	github.com/ncw/swift/v2 v2.0.1
-	github.com/oracle/oci-go-sdk/v65 v65.1.0
+	github.com/oracle/oci-go-sdk/v65 v65.26.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/sftp v1.13.5
 	github.com/pmezard/go-difflib v1.0.0
-	github.com/prometheus/client_golang v1.12.2
+	github.com/prometheus/client_golang v1.14.0
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8
 	github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6
 	github.com/rfjakob/eme v1.1.2
-	github.com/shirou/gopsutil/v3 v3.22.7
+	github.com/shirou/gopsutil/v3 v3.22.10
 	github.com/sirupsen/logrus v1.9.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
-	github.com/spf13/cobra v1.5.0
+	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.8.0
+	github.com/stretchr/testify v1.8.1
 	github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf
-	github.com/winfsp/cgofuse v1.5.1-0.20221106144041-c23bd0f99a49
-	github.com/xanzy/ssh-agent v0.3.1
+	github.com/winfsp/cgofuse v1.5.1-0.20221118130120-84c0898ad2e0
+	github.com/xanzy/ssh-agent v0.3.3
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0
 	go.etcd.io/bbolt v1.3.6
 	goftp.io/server v0.4.1
-	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa
-	golang.org/x/net v0.1.0
-	golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
-	golang.org/x/sys v0.1.0
+	golang.org/x/crypto v0.3.0
+	golang.org/x/net v0.2.0
+	golang.org/x/oauth2 v0.2.0
+	golang.org/x/sync v0.1.0
+	golang.org/x/sys v0.2.0
 	golang.org/x/text v0.4.0
-	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9
-	google.golang.org/api v0.91.0
+	golang.org/x/time v0.2.0
+	google.golang.org/api v0.103.0
 	gopkg.in/yaml.v2 v2.4.0
 	storj.io/uplink v1.9.0
 )
 
 require (
-	cloud.google.com/go/compute v1.7.0 // indirect
+	cloud.google.com/go/compute v1.12.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.1 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
@@ -92,12 +93,12 @@ require (
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.7.0 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/gofork v1.7.6 // indirect
@@ -108,14 +109,14 @@ require (
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/onsi/gomega v1.13.0 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sony/gobreaker v0.5.0 // indirect
@@ -125,23 +126,23 @@ require (
 	github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	github.com/zeebo/errs v1.3.0 // indirect
-	go.opencensus.io v0.23.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f // indirect
-	google.golang.org/grpc v1.47.0 // indirect
-	google.golang.org/protobuf v1.28.0 // indirect
+	google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect
+	google.golang.org/grpc v1.50.1 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	storj.io/common v0.0.0-20220414110316-a5cb7172d6bf // indirect
 	storj.io/drpc v0.0.30 // indirect
 )
 
 require (
-	github.com/Microsoft/go-winio v0.5.1 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20220730123233-d6ffb7692adf
+	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4
 	github.com/golang-jwt/jwt/v4 v4.1.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/pkg/xattr v0.4.7
-	golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105
-	golang.org/x/term v0.1.0
+	github.com/pkg/xattr v0.4.9
+	golang.org/x/mobile v0.0.0-20221110043201-43a038452099
+	golang.org/x/term v0.2.0
 )
diff --git a/go.sum b/go.sum
index 6b7e2d6dff843..516d8cba7c88d 100644
--- a/go.sum
+++ b/go.sum
@@ -31,34 +31,252 @@ cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW
 cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
 cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
 cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
-cloud.google.com/go v0.102.0 h1:DAq3r8y4mDgyB/ZPJ9v/5VJNqjgJAxTn6ZYLlUywOu8=
 cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc=
+cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU=
+cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA=
+cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y=
+cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM=
+cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4=
+cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o=
+cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw=
+cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY=
+cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI=
+cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4=
+cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk=
+cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc=
+cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno=
+cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4=
+cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0=
+cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ=
+cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk=
+cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0=
+cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o=
+cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s=
+cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0=
+cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ=
+cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY=
+cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw=
+cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI=
+cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
+cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
+cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8=
+cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8=
+cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc=
+cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE=
+cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA=
+cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY=
+cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s=
+cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI=
+cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM=
+cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI=
+cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0=
+cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg=
+cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk=
+cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U=
+cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM=
+cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY=
+cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI=
+cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4=
 cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
 cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
 cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
 cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
 cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
-cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk=
 cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U=
+cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU=
+cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU=
+cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0=
+cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU=
+cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU=
+cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
+cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48=
+cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
+cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY=
+cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg=
+cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I=
+cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4=
+cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0=
+cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs=
+cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc=
+cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE=
+cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM=
+cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ=
+cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo=
+cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE=
+cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38=
+cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I=
+cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ=
+cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA=
+cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s=
+cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo=
+cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo=
+cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ=
+cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g=
+cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c=
+cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4=
+cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0=
+cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8=
+cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek=
+cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM=
+cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU=
+cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU=
+cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k=
+cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y=
+cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg=
+cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk=
+cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w=
+cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI=
+cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc=
+cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w=
+cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk=
+cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg=
+cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY=
+cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM=
+cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA=
+cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w=
+cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60=
+cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o=
+cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A=
+cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0=
+cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0=
+cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA=
+cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc=
+cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM=
 cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
+cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc=
+cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc=
+cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc=
+cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM=
+cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs=
+cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg=
+cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
+cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
+cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
+cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8=
+cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08=
+cloud.google.com/go/longrunning v0.1.1 h1:y50CXG4j0+qvEukslYFBCrzaXX0qpFbBzc3PchSu/LE=
+cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
+cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE=
+cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4=
+cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w=
+cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE=
+cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM=
+cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA=
+cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY=
+cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s=
+cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8=
+cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk=
+cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA=
+cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o=
+cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM=
+cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8=
+cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ=
+cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU=
+cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY=
+cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34=
+cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA=
+cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4=
+cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA=
+cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE=
+cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs=
+cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg=
+cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo=
+cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E=
+cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU=
+cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70=
+cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0=
+cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA=
+cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg=
+cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0=
+cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
 cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
+cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o=
+cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk=
+cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo=
+cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE=
+cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg=
+cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4=
+cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg=
+cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c=
+cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs=
+cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y=
+cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A=
+cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA=
+cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA=
+cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU=
+cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4=
+cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY=
+cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc=
+cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do=
+cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s=
+cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI=
+cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk=
+cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA=
+cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4=
+cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4=
+cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0=
+cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU=
+cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q=
+cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU=
+cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc=
+cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk=
+cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU=
+cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs=
+cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg=
+cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4=
+cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco=
+cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E=
+cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4=
+cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM=
+cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ=
+cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y=
+cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc=
+cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
+cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w=
+cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw=
+cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g=
+cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM=
+cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8=
+cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ=
+cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28=
+cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs=
+cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk=
+cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU=
+cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4=
+cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M=
+cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0=
+cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo=
+cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo=
+cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY=
+cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE=
+cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w=
+cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE=
+cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg=
+cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc=
+cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo=
+cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0=
+cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M=
+cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M=
 dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
@@ -89,12 +307,11 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
 github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5 h1:w/vNc+SQRYKGWBHeDrzvvNttHwZEbSAP0kmTdORl4OI=
 github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc=
-github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
-github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
-github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
+github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
+github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/ProtonMail/go-crypto v0.0.0-20220730123233-d6ffb7692adf h1:aFFtnGZ6/2Qlvx80yxA2fFSYDQWTFjtKozQKB36A3/A=
-github.com/ProtonMail/go-crypto v0.0.0-20220730123233-d6ffb7692adf/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
+github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I=
+github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
 github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
 github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A=
 github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
@@ -124,8 +341,8 @@ github.com/artyom/mtab v1.0.0 h1:r7OSVo5Jeqi8+LotZ0rT2kzfPIBp9KCpEJP8RQqGmSE=
 github.com/artyom/mtab v1.0.0/go.mod h1:EHpkp5OmPfS1yZX+/DFTztlJ9di5UzdDLX1/XzWPXw8=
 github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
-github.com/aws/aws-sdk-go v1.44.70 h1:wrwAbqJqf+ncEK1F/bXTYpgO6zXIgQXi/2ppBgmYI9g=
-github.com/aws/aws-sdk-go v1.44.70/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.44.145 h1:KMVRrIyjBsNz3xGPuHIRnhIuKlb5h3Ii5e5jbi3cgnc=
+github.com/aws/aws-sdk-go v1.44.145/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -176,8 +393,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00 h1:xJBhC00smQpSZw3Kr0ErMUBXhUSjYoLRm2szxdbRBL0=
 github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00/go.mod h1:nNICngOdmNImBb/vuL+dSc0aIg3ryNATpjxThNoPw4g=
-github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.4 h1:/G00pLNOafr+cP0uCcvctt+Q5JxoLW6WLyv28Ze2y5I=
-github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.4/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M=
+github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXIiWJETBpRgERYTGlmMd7HU=
+github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M=
 github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
@@ -202,8 +419,8 @@ github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkF
 github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
 github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
 github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
-github.com/gdamore/tcell/v2 v2.5.2 h1:tKzG29kO9p2V++3oBY2W9zUjYu7IK1MENFeY/BzJSVY=
-github.com/gdamore/tcell/v2 v2.5.2/go.mod h1:wSkrPaXoiIWZqW/g7Px4xc79di6FTcpB8tvaKJ6uGBo=
+github.com/gdamore/tcell/v2 v2.5.3 h1:b9XQrT6QGbgI7JvZOJXFNczOQeIYbo8BfeSMzt2sAV0=
+github.com/gdamore/tcell/v2 v2.5.3/go.mod h1:wSkrPaXoiIWZqW/g7Px4xc79di6FTcpB8tvaKJ6uGBo=
 github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w=
 github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -219,9 +436,11 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
+github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -287,8 +506,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
@@ -320,8 +540,9 @@ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
-github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw=
 github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
+github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs=
+github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
 github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU=
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
@@ -331,8 +552,11 @@ github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pf
 github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
 github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
 github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
-github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk=
 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
+github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
+github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY=
+github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
+github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
 github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg=
@@ -366,8 +590,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/iguanesolutions/go-systemd/v5 v5.1.0 h1:UWprhbpxjLM0vvwu4MxaBR+/KzSxgvnKpM9Q3MBhTAc=
 github.com/iguanesolutions/go-systemd/v5 v5.1.0/go.mod h1:XprNDEZ9zdPzEg1WrmpV1BnGorgP0WP40AGurMxeQOY=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
+github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
 github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
 github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
@@ -405,8 +629,8 @@ github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC
 github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
-github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM=
+github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348 h1:Lrn8srO9JDBCf2iPjqy62stl49UDwoOxZ9/NGVi+fnk=
@@ -439,14 +663,15 @@ github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1j
 github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
 github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8=
 github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI=
-github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
-github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
 github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
-github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
+github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
@@ -486,8 +711,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
 github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
 github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
-github.com/oracle/oci-go-sdk/v65 v65.1.0 h1:CtEPYXdFvv6H+zfYPfTT7DT/V/a5UsULkrj/AnzOtBc=
-github.com/oracle/oci-go-sdk/v65 v65.1.0/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE=
+github.com/oracle/oci-go-sdk/v65 v65.26.1 h1:Ms20RSRj+CuvQmw5ET1TkmzxLBI+bmLjZ6NYANA3gkk=
+github.com/oracle/oci-go-sdk/v65 v65.26.1/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pborman/getopt v1.1.0/go.mod h1:FxXoW1Re00sQG/+KIkuSqRL/LwQgSkv7uyac+STFsbk=
@@ -501,8 +726,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
 github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
-github.com/pkg/xattr v0.4.7 h1:XoA3KzmFvyPlH4RwX5eMcgtzcaGBaSvgt3IoFQfbrmQ=
-github.com/pkg/xattr v0.4.7/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
+github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
@@ -512,26 +737,30 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
-github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
+github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
+github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
+github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
 github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6 h1:J832KfU2Z44Ck3XR5bvw2UxShP0QnjueruNQ6dTYH+g=
@@ -552,8 +781,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/shirou/gopsutil/v3 v3.22.7 h1:flKnuCMfUUrO+oAvwAd6GKZgnPzr098VA/UJ14nhJd4=
-github.com/shirou/gopsutil/v3 v3.22.7/go.mod h1:s648gW4IywYzUfE/KjXxUsqrqx/T2xO5VqOXxONeRfI=
+github.com/shirou/gopsutil/v3 v3.22.10 h1:4KMHdfBRYXGF9skjDWiL4RA2N+E8dRdodU/bOZpPoVg=
+github.com/shirou/gopsutil/v3 v3.22.10/go.mod h1:QNza6r4YQoydyCfo6rH0blGfKahgibh4dQmV5xdFkQk=
 github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
 github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
 github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
@@ -598,15 +827,16 @@ github.com/spacemonkeygo/monkit/v3 v3.0.17 h1:rqIuLhRUr2UtS3WNVbPY/BwvjlwKVvSOVY
 github.com/spacemonkeygo/monkit/v3 v3.0.17/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4=
 github.com/spacemonkeygo/monotime v0.0.0-20180824235756-e3f48a95f98a/go.mod h1:ul4bvvnCOPZgq8w0nTkSmWVg/hauVpFS97Am1YM1XXo=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
-github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
+github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
+github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -616,8 +846,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf h1:Y43S3e9P1NPs/QF4R5/SdlXj2d31540hP4Gk8VKNvDg=
 github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf/go.mod h1:c+cGNU1qi9bO7ZF4IRMYk+KaZTNiQ/gQrSbyMmGFq1Q=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
@@ -633,12 +864,10 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI
 github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k=
 github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc=
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
-github.com/winfsp/cgofuse v1.5.1-0.20220421173602-ce7e5a65cac7 h1:Lvi511+8ZWFr9YWbvXkxudGSjcrjM1NVPDIPeWusvDk=
-github.com/winfsp/cgofuse v1.5.1-0.20220421173602-ce7e5a65cac7/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I=
-github.com/winfsp/cgofuse v1.5.1-0.20221106144041-c23bd0f99a49 h1:+kD4nvjSbuESNeVUdkrppns7qRJNKSrWBWjrg9o3TV8=
-github.com/winfsp/cgofuse v1.5.1-0.20221106144041-c23bd0f99a49/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I=
-github.com/xanzy/ssh-agent v0.3.1 h1:AmzO1SSWxw73zxFZPRwaMN1MohDw8UyHnmuxyceTEGo=
-github.com/xanzy/ssh-agent v0.3.1/go.mod h1:QIE4lCeL7nkC25x+yA3LBIYfwCc1TFziCtG7cBAac6w=
+github.com/winfsp/cgofuse v1.5.1-0.20221118130120-84c0898ad2e0 h1:j3un8DqYvvAOqKI5OPz+/RRVhDFipbPKI4t2Uk5RBJw=
+github.com/winfsp/cgofuse v1.5.1-0.20221118130120-84c0898ad2e0/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I=
+github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
+github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=
 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -646,7 +875,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yunify/qingstor-sdk-go/v3 v3.2.0 h1:9sB2WZMgjwSUNZhrgvaNGazVltoFUUfuS9f0uCWtTr8=
 github.com/yunify/qingstor-sdk-go/v3 v3.2.0/go.mod h1:KciFNuMu6F4WLk9nGwwK69sCGKLCdd9f97ac/wfumS4=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
@@ -668,8 +897,9 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 goftp.io/server v0.4.1 h1:x7KG4HIxSMdK/rpYhExMinRN/aO/T9icvaG/B5e/XfY=
@@ -692,12 +922,13 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A=
+golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -726,8 +957,8 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI
 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105 h1:3vUV5x5+3LfQbgk7paCM6INOaJG9xXQbn79xoNkwfIk=
-golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105/go.mod h1:pe2sM7Uk+2Su1y7u/6Z8KJ24D7lepUjFZbhFOrmDfuQ=
+golang.org/x/mobile v0.0.0-20221110043201-43a038452099 h1:aIu0lKmfdgtn2uTj7JI2oN4TUrQvgB+wzTPO23bCKt8=
+golang.org/x/mobile v0.0.0-20221110043201-43a038452099/go.mod h1:aAjjkJNdrh3PMckS4B10TGS2nag27cbKR1y2BpUxsiY=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
@@ -737,6 +968,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -785,7 +1017,6 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
@@ -794,12 +1025,16 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220524220425-1d687d428aca/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM=
-golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 h1:N9Vc/rorQUDes6B9CNdIxAn5jODGj2wzfrei2x4wNj4=
-golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
-golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
+golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -823,8 +1058,12 @@ golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j
 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
 golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
 golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
-golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c h1:q3gFqPqH7NVofKo3c3yETAP//pPI+G5mvB7qqj1Y5kY=
-golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
+golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
+golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
+golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
+golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
+golang.org/x/oauth2 v0.2.0 h1:GtQkldQ9m7yvzCL1V+LrYow3Khe0eJH0w7RbX/VbaIU=
+golang.org/x/oauth2 v0.2.0/go.mod h1:Cwn6afJ8jrQwYMxQDTpISoXmXW9I6qF6vDeuuoX3Ibs=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -838,8 +1077,11 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -907,10 +1149,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -927,21 +1167,21 @@ golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220804214406-8e32c043e418 h1:9vYwv7OjYaky/tlAeD7C4oC9EsPTlaFl1H2jS++V+ME=
-golang.org/x/sys v0.0.0-20220804214406-8e32c043e418/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc=
-golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -950,16 +1190,16 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 h1:ftMN5LMiBFjbzleLqtoBZk7KdJwhuybIU+FckUHgoyQ=
-golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE=
+golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1018,15 +1258,16 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.8-0.20211022200916-316ba0b74098/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
@@ -1066,11 +1307,22 @@ google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/S
 google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
 google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
 google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
+google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
 google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
 google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
 google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
-google.golang.org/api v0.91.0 h1:731+JzuwaJoZXRQGmPoBiV+SrsAfUaIkdMCWTcQNPyA=
-google.golang.org/api v0.91.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
+google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g=
+google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
+google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
+google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI=
+google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
+google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
+google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
+google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08=
+google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70=
+google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo=
+google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ=
+google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1158,13 +1410,37 @@ google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX
 google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
 google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
 google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
 google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
 google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
 google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
 google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f h1:hJ/Y5SqPXbarffmAsApliUlcvMU+wScNGfyop4bZm8o=
+google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
+google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
+google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE=
+google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc=
+google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
+google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
+google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
+google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
+google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
+google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo=
+google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo=
+google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo=
+google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo=
+google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo=
+google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw=
+google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI=
+google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI=
+google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U=
+google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM=
+google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM=
+google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s=
+google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s=
+google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo=
+google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
@@ -1198,8 +1474,12 @@ google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8=
 google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
+google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
+google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY=
+google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -1214,8 +1494,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

From 6f3682c12f1c2abda5438a9c35baa6b191f5504f Mon Sep 17 00:00:00 2001
From: Nathaniel Wesley Filardo 
Date: Mon, 14 Nov 2022 04:11:44 +0000
Subject: [PATCH 402/560] azureblob: make newServicePrincipalTokenRefresher
 take parsed principal structure

---
 backend/azureblob/azureblob.go      | 14 +++++++-------
 backend/azureblob/azureblob_test.go | 13 +++++++++++--
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 6c46419f51f76..e52a48133ed9b 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -467,12 +467,8 @@ type servicePrincipalCredentials struct {
 const azureActiveDirectoryEndpoint = "https://login.microsoftonline.com/"
 const azureStorageEndpoint = "https://storage.azure.com/"
 
-// newServicePrincipalTokenRefresher takes the client ID and secret, and returns a refresh-able access token.
-func newServicePrincipalTokenRefresher(ctx context.Context, credentialsData []byte) (azblob.TokenRefresher, error) {
-	var spCredentials servicePrincipalCredentials
-	if err := json.Unmarshal(credentialsData, &spCredentials); err != nil {
-		return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err)
-	}
+// newServicePrincipalTokenRefresher takes a servicePrincipalCredentials structure and returns a refresh-able access token.
+func newServicePrincipalTokenRefresher(ctx context.Context, spCredentials servicePrincipalCredentials) (azblob.TokenRefresher, error) {
 	oauthConfig, err := adal.NewOAuthConfig(azureActiveDirectoryEndpoint, spCredentials.Tenant)
 	if err != nil {
 		return nil, fmt.Errorf("error creating oauth config: %w", err)
@@ -729,8 +725,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
 		}
+		var spCredentials servicePrincipalCredentials
+		if err := json.Unmarshal(loadedCreds, &spCredentials); err != nil {
+			return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err)
+		}
 		// Create a token refresher from service principal credentials.
-		tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, loadedCreds)
+		tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, spCredentials)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create a service principal token: %w", err)
 		}
diff --git a/backend/azureblob/azureblob_test.go b/backend/azureblob/azureblob_test.go
index f9ebd7b3d34fa..f12bb177a1c4d 100644
--- a/backend/azureblob/azureblob_test.go
+++ b/backend/azureblob/azureblob_test.go
@@ -7,6 +7,7 @@ package azureblob
 
 import (
 	"context"
+	"encoding/json"
 	"testing"
 
 	"github.com/rclone/rclone/fs"
@@ -42,7 +43,11 @@ func TestServicePrincipalFileSuccess(t *testing.T) {
     "tenant": "my active directory tenant ID"
 }
 `
-	tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
+	var spCredentials servicePrincipalCredentials
+	jerr := json.Unmarshal([]byte(credentials), &spCredentials)
+	assert.Nil(t, jerr)
+
+	tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, spCredentials)
 	if assert.NoError(t, err) {
 		assert.NotNil(t, tokenRefresher)
 	}
@@ -57,7 +62,11 @@ func TestServicePrincipalFileFailure(t *testing.T) {
     "tenant": "my active directory tenant ID"
 }
 `
-	_, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
+	var spCredentials servicePrincipalCredentials
+	jerr := json.Unmarshal([]byte(credentials), &spCredentials)
+	assert.Nil(t, jerr)
+
+	_, err := newServicePrincipalTokenRefresher(ctx, spCredentials)
 	assert.Error(t, err)
 	assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
 }

From f7cdf318dbd53cf6cb4c2497f9f262d683f55129 Mon Sep 17 00:00:00 2001
From: Nathaniel Wesley Filardo 
Date: Mon, 14 Nov 2022 04:12:52 +0000
Subject: [PATCH 403/560] azureblob: support simple "environment credentials"

As per
https://learn.microsoft.com/en-us/dotnet/api/azure.identity.environmentcredential?view=azure-dotnet

This supports only AZURE_CLIENT_SECRET-based authentication, as with the
existing service principal support.

Co-authored-by: Nick Craig-Wood 
---
 backend/azureblob/azureblob.go | 35 ++++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index e52a48133ed9b..728a7405ed1eb 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -88,6 +88,15 @@ Leave blank normally. Needed only if you want to use a service principal instead
 
 See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
 `,
+		}, {
+			Name: "env_auth",
+			Help: `Read credentials from runtime (environment variables).
+
+Pull credentials from AZURE_TENANT_ID and AZURE_CLIENT_{ID,SECRET} environment vars.
+See EnvironmentCredential in the Azure docs for more info.
+
+Other authentication methods will, if specified, override this flag.`,
+			Default: false,
 		}, {
 			Name: "key",
 			Help: "Storage Account Key.\n\nLeave blank to use SAS URL or Emulator.",
@@ -270,6 +279,7 @@ type Options struct {
 	Account              string               `config:"account"`
 	ServicePrincipalFile string               `config:"service_principal_file"`
 	Key                  string               `config:"key"`
+	EnvAuth              bool                 `config:"env_auth"`
 	UseMSI               bool                 `config:"use_msi"`
 	MSIObjectID          string               `config:"msi_object_id"`
 	MSIClientID          string               `config:"msi_client_id"`
@@ -714,20 +724,29 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		} else {
 			serviceURL = azblob.NewServiceURL(*u, pipeline)
 		}
-	case opt.ServicePrincipalFile != "":
+	case opt.ServicePrincipalFile != "" || opt.EnvAuth:
 		// Create a standard URL.
 		u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
 		if err != nil {
 			return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
 		}
-		// Try loading service principal credentials from file.
-		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
-		if err != nil {
-			return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
-		}
 		var spCredentials servicePrincipalCredentials
-		if err := json.Unmarshal(loadedCreds, &spCredentials); err != nil {
-			return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err)
+		if opt.ServicePrincipalFile != "" {
+			// Try loading service principal credentials from file.
+			loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
+			if err != nil {
+				return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
+			}
+
+			if err := json.Unmarshal(loadedCreds, &spCredentials); err != nil {
+				return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err)
+			}
+		} else {
+			spCredentials = servicePrincipalCredentials{
+				Tenant:   os.Getenv("AZURE_TENANT_ID"),
+				AppID:    os.Getenv("AZURE_CLIENT_ID"),
+				Password: os.Getenv("AZURE_CLIENT_SECRET"),
+			}
 		}
 		// Create a token refresher from service principal credentials.
 		tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, spCredentials)

From 0edf6478e305cd6e1cc032b6fd7142a815488d9d Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 24 Nov 2022 17:04:27 +0000
Subject: [PATCH 404/560] Add Nathaniel Wesley Filardo to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 1d106b42ec766..5209d136256e2 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -664,3 +664,4 @@ put them back in again.` >}}
   * rkettelerij 
   * Kamui 
   * asdffdsazqqq <90116442+asdffdsazqqq@users.noreply.github.com>
+  * Nathaniel Wesley Filardo 

From 46b080c0922fc7e811fa53547b2d8d0b2e71383e Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 24 Nov 2022 10:30:01 +0000
Subject: [PATCH 405/560] vfs: Fix IO Error opening a file with
 O_CREATE|O_RDONLY in --vfs-cache-mode not full

Before this fix, opening a file with `O_CREATE|O_RDONLY` caused an IO error to
be returned when using `--vfs-cache-mode off` or `--vfs-cache-mode writes`.

This was because the file was opened with read intent, but the `O_CREATE`
implies write intent to create the file even though the file is opened
`O_RDONLY`.

This fix sets write intent for the file if `O_CREATE` is passed in which fixes
the problem for all the VFS cache modes.

It also extends the exhaustive open flags testing to `--vfs-cache-mode writes`
as well as `--vfs-cache-mode full` which would have caught this problem.

See: https://forum.rclone.org/t/i-o-error-trashing-file-on-sftp-mount/34317/
---
 vfs/file.go            |  5 +++++
 vfs/read_write_test.go | 22 +++++++++++++---------
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/vfs/file.go b/vfs/file.go
index 4e59aee747804..aa2493a5f1631 100644
--- a/vfs/file.go
+++ b/vfs/file.go
@@ -698,6 +698,11 @@ func (f *File) Open(flags int) (fd Handle, err error) {
 		write = true
 	}
 
+	// If create is set then set write to force openRW
+	if flags&os.O_CREATE != 0 {
+		write = true
+	}
+
 	// Open the correct sort of handle
 	f.mu.RLock()
 	d := f.d
diff --git a/vfs/read_write_test.go b/vfs/read_write_test.go
index ee8051c86faae..d9dac353f37c8 100644
--- a/vfs/read_write_test.go
+++ b/vfs/read_write_test.go
@@ -642,15 +642,19 @@ func testRWFileHandleOpenTest(t *testing.T, vfs *VFS, test *openTest) {
 }
 
 func TestRWFileHandleOpenTests(t *testing.T) {
-	opt := vfscommon.DefaultOpt
-	opt.CacheMode = vfscommon.CacheModeFull
-	opt.WriteBack = writeBackDelay
-	_, vfs, cleanup := newTestVFSOpt(t, &opt)
-	defer cleanup()
-
-	for _, test := range openTests {
-		t.Run(test.what, func(t *testing.T) {
-			testRWFileHandleOpenTest(t, vfs, &test)
+	for _, cacheMode := range []vfscommon.CacheMode{vfscommon.CacheModeWrites, vfscommon.CacheModeFull} {
+		t.Run(cacheMode.String(), func(t *testing.T) {
+			opt := vfscommon.DefaultOpt
+			opt.CacheMode = cacheMode
+			opt.WriteBack = writeBackDelay
+			_, vfs, cleanup := newTestVFSOpt(t, &opt)
+			defer cleanup()
+
+			for _, test := range openTests {
+				t.Run(test.what, func(t *testing.T) {
+					testRWFileHandleOpenTest(t, vfs, &test)
+				})
+			}
 		})
 	}
 }

From a4c65532ea78b4c6b7772ffdb0c147ee841e638b Mon Sep 17 00:00:00 2001
From: eNV25 
Date: Sat, 12 Nov 2022 18:49:50 +0530
Subject: [PATCH 406/560] cmd/ncdu: use tcell directly instead of the termbox
 wrapper

Following up on 36add0af, which switched from termbox
to tcell's termbox wrapper.
---
 cmd/ncdu/ncdu.go | 228 ++++++++++++++++++++++++++---------------------
 1 file changed, 126 insertions(+), 102 deletions(-)

diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go
index 9d89ae2374d73..edf0a570c0834 100644
--- a/cmd/ncdu/ncdu.go
+++ b/cmd/ncdu/ncdu.go
@@ -13,13 +13,14 @@ import (
 	"strings"
 
 	"github.com/atotto/clipboard"
-	"github.com/gdamore/tcell/v2/termbox"
+	"github.com/gdamore/tcell/v2"
 	runewidth "github.com/mattn/go-runewidth"
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/cmd/ncdu/scan"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rivo/uniseg"
 	"github.com/spf13/cobra"
 )
 
@@ -114,6 +115,7 @@ func helpText() (tr []string) {
 
 // UI contains the state of the user interface
 type UI struct {
+	s                  tcell.Screen
 	f                  fs.Fs     // fs being displayed
 	fsName             string    // human name of Fs
 	root               *scan.Dir // root directory
@@ -150,77 +152,91 @@ type dirPos struct {
 	offset int
 }
 
+// graphemeWidth returns the number of cells in rs.
+//
+// The original [runewidth.StringWidth] iterates through graphemes
+// and uses this same logic. To avoid iterating through graphemes
+// repeatedly, we separate that out into its own function.
+func graphemeWidth(rs []rune) (wd int) {
+	// copied/adapted from [runewidth.StringWidth]
+	for _, r := range rs {
+		wd = runewidth.RuneWidth(r)
+		if wd > 0 {
+			break
+		}
+	}
+	return
+}
+
 // Print a string
-func Print(x, y int, fg, bg termbox.Attribute, msg string) {
-	for _, c := range msg {
-		termbox.SetCell(x, y, c, fg, bg)
-		x++
+func (u *UI) Print(x, y int, style tcell.Style, msg string) {
+	g := uniseg.NewGraphemes(msg)
+	for g.Next() {
+		rs := g.Runes()
+		u.s.SetContent(x, y, rs[0], rs[1:], style)
+		x += graphemeWidth(rs)
 	}
 }
 
 // Printf a string
-func Printf(x, y int, fg, bg termbox.Attribute, format string, args ...interface{}) {
+func (u *UI) Printf(x, y int, style tcell.Style, format string, args ...interface{}) {
 	s := fmt.Sprintf(format, args...)
-	Print(x, y, fg, bg, s)
+	u.Print(x, y, style, s)
 }
 
 // Line prints a string to given xmax, with given space
-func Line(x, y, xmax int, fg, bg termbox.Attribute, spacer rune, msg string) {
-	for _, c := range msg {
-		termbox.SetCell(x, y, c, fg, bg)
-		x += runewidth.RuneWidth(c)
+func (u *UI) Line(x, y, xmax int, style tcell.Style, spacer rune, msg string) {
+	g := uniseg.NewGraphemes(msg)
+	for g.Next() {
+		rs := g.Runes()
+		u.s.SetContent(x, y, rs[0], rs[1:], style)
+		x += graphemeWidth(rs)
 		if x >= xmax {
 			return
 		}
 	}
 	for ; x < xmax; x++ {
-		termbox.SetCell(x, y, spacer, fg, bg)
+		u.s.SetContent(x, y, spacer, nil, style)
 	}
 }
 
 // Linef a string
-func Linef(x, y, xmax int, fg, bg termbox.Attribute, spacer rune, format string, args ...interface{}) {
+func (u *UI) Linef(x, y, xmax int, style tcell.Style, spacer rune, format string, args ...interface{}) {
 	s := fmt.Sprintf(format, args...)
-	Line(x, y, xmax, fg, bg, spacer, s)
+	u.Line(x, y, xmax, style, spacer, s)
 }
 
 // LineOptions Print line of selectable options
-func LineOptions(x, y, xmax int, fg, bg termbox.Attribute, options []string, selected int) {
-	defaultBg := bg
-	defaultFg := fg
-
-	// Print left+right whitespace to center the options
-	xoffset := ((xmax - x) - lineOptionLength(options)) / 2
-	for j := x; j < x+xoffset; j++ {
-		termbox.SetCell(j, y, ' ', fg, bg)
-	}
-	for j := xmax - xoffset; j < xmax; j++ {
-		termbox.SetCell(j, y, ' ', fg, bg)
+func (u *UI) LineOptions(x, y, xmax int, style tcell.Style, options []string, selected int) {
+	for x := x; x < xmax; x++ {
+		u.s.SetContent(x, y, ' ', nil, style) // fill
 	}
-	x += xoffset
+	x += ((xmax - x) - lineOptionLength(options)) / 2 // center
 
 	for i, o := range options {
-		termbox.SetCell(x, y, ' ', fg, bg)
+		u.s.SetContent(x, y, ' ', nil, style)
+		x++
 
+		ostyle := style
 		if i == selected {
-			bg = termbox.ColorBlack
-			fg = termbox.ColorWhite
+			ostyle = tcell.StyleDefault
 		}
-		termbox.SetCell(x+1, y, '<', fg, bg)
-		x += 2
 
-		// print option text
-		for _, c := range o {
-			termbox.SetCell(x, y, c, fg, bg)
-			x++
+		u.s.SetContent(x, y, '<', nil, ostyle)
+		x++
+
+		g := uniseg.NewGraphemes(o)
+		for g.Next() {
+			rs := g.Runes()
+			u.s.SetContent(x, y, rs[0], rs[1:], ostyle)
+			x += graphemeWidth(rs)
 		}
 
-		termbox.SetCell(x, y, '>', fg, bg)
-		bg = defaultBg
-		fg = defaultFg
+		u.s.SetContent(x, y, '>', nil, ostyle)
+		x++
 
-		termbox.SetCell(x+1, y, ' ', fg, bg)
-		x += 2
+		u.s.SetContent(x, y, ' ', nil, style)
+		x++
 	}
 }
 
@@ -234,7 +250,7 @@ func lineOptionLength(o []string) int {
 
 // Box the u.boxText onto the screen
 func (u *UI) Box() {
-	w, h := termbox.Size()
+	w, h := u.s.Size()
 
 	// Find dimensions of text
 	boxWidth := 10
@@ -260,31 +276,31 @@ func (u *UI) Box() {
 	ymax := y + len(u.boxText)
 
 	// draw text
-	fg, bg := termbox.ColorRed, termbox.ColorWhite
+	style := tcell.StyleDefault.Background(tcell.ColorRed).Reverse(true)
 	for i, s := range u.boxText {
-		Line(x, y+i, xmax, fg, bg, ' ', s)
-		fg = termbox.ColorBlack
+		u.Line(x, y+i, xmax, style, ' ', s)
+		style = tcell.StyleDefault.Reverse(true)
 	}
 
 	if len(u.boxMenu) != 0 {
+		u.LineOptions(x, ymax, xmax, style, u.boxMenu, u.boxMenuButton)
 		ymax++
-		LineOptions(x, ymax-1, xmax, fg, bg, u.boxMenu, u.boxMenuButton)
 	}
 
 	// draw top border
 	for i := y; i < ymax; i++ {
-		termbox.SetCell(x-1, i, '│', fg, bg)
-		termbox.SetCell(xmax, i, '│', fg, bg)
+		u.s.SetContent(x-1, i, tcell.RuneVLine, nil, style)
+		u.s.SetContent(xmax, i, tcell.RuneVLine, nil, style)
 	}
 	for j := x; j < xmax; j++ {
-		termbox.SetCell(j, y-1, '─', fg, bg)
-		termbox.SetCell(j, ymax, '─', fg, bg)
+		u.s.SetContent(j, y-1, tcell.RuneHLine, nil, style)
+		u.s.SetContent(j, ymax, tcell.RuneHLine, nil, style)
 	}
 
-	termbox.SetCell(x-1, y-1, '┌', fg, bg)
-	termbox.SetCell(xmax, y-1, '┐', fg, bg)
-	termbox.SetCell(x-1, ymax, '└', fg, bg)
-	termbox.SetCell(xmax, ymax, '┘', fg, bg)
+	u.s.SetContent(x-1, y-1, tcell.RuneULCorner, nil, style)
+	u.s.SetContent(xmax, y-1, tcell.RuneURCorner, nil, style)
+	u.s.SetContent(x-1, ymax, tcell.RuneLLCorner, nil, style)
+	u.s.SetContent(xmax, ymax, tcell.RuneLRCorner, nil, style)
 }
 
 func (u *UI) moveBox(to int) {
@@ -336,17 +352,17 @@ func (u *UI) hasEmptyDir() bool {
 // Draw the current screen
 func (u *UI) Draw() error {
 	ctx := context.Background()
-	w, h := termbox.Size()
+	w, h := u.s.Size()
 	u.dirListHeight = h - 3
 
 	// Plot
-	termbox.Clear(termbox.ColorWhite, termbox.ColorBlack)
+	u.s.Clear()
 
 	// Header line
-	Linef(0, 0, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "rclone ncdu %s - use the arrow keys to navigate, press ? for help", fs.Version)
+	u.Linef(0, 0, w, tcell.StyleDefault.Reverse(true), ' ', "rclone ncdu %s - use the arrow keys to navigate, press ? for help", fs.Version)
 
 	// Directory line
-	Linef(0, 1, w, termbox.ColorWhite, termbox.ColorBlack, '-', "-- %s ", u.path)
+	u.Linef(0, 1, w, tcell.StyleDefault, '-', "-- %s ", u.path)
 
 	// graphs
 	const (
@@ -377,20 +393,18 @@ func (u *UI) Draw() error {
 				attrs, err = u.d.AttrI(u.sortPerm[n])
 			}
 			_, isSelected := u.selectedEntries[entry.String()]
-			fg := termbox.ColorWhite
+			style := tcell.StyleDefault
 			if attrs.EntriesHaveErrors {
-				fg = termbox.ColorYellow
+				style = style.Foreground(tcell.ColorYellow)
 			}
 			if err != nil {
-				fg = termbox.ColorRed
+				style = style.Foreground(tcell.ColorRed)
 			}
-			const colorLightYellow = termbox.ColorYellow + 8
 			if isSelected {
-				fg = colorLightYellow
+				style = style.Foreground(tcell.ColorLightYellow)
 			}
-			bg := termbox.ColorBlack
 			if n == dirPos.entry {
-				fg, bg = bg, fg
+				style = style.Reverse(true)
 			}
 			mark := ' '
 			if attrs.IsDir {
@@ -449,31 +463,30 @@ func (u *UI) Draw() error {
 				}
 				extras += "[" + graph[graphBars-bars:2*graphBars-bars] + "] "
 			}
-			Linef(0, y, w, fg, bg, ' ', "%c %s %s%c%s%s", fileFlag, operations.SizeStringField(attrs.Size, u.humanReadable, 12), extras, mark, path.Base(entry.Remote()), message)
+			u.Linef(0, y, w, style, ' ', "%c %s %s%c%s%s",
+				fileFlag, operations.SizeStringField(attrs.Size, u.humanReadable, 12), extras, mark, path.Base(entry.Remote()), message)
 			y++
 		}
 	}
 
 	// Footer
 	if u.d == nil {
-		Line(0, h-1, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "Waiting for root directory...")
+		u.Line(0, h-1, w, tcell.StyleDefault.Reverse(true), ' ', "Waiting for root directory...")
 	} else {
 		message := ""
 		if u.listing {
 			message = " [listing in progress]"
 		}
 		size, count := u.d.Attr()
-		Linef(0, h-1, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "Total usage: %s, Objects: %s%s", operations.SizeString(size, u.humanReadable), operations.CountString(count, u.humanReadable), message)
+		u.Linef(0, h-1, w, tcell.StyleDefault.Reverse(true), ' ', "Total usage: %s, Objects: %s%s",
+			operations.SizeString(size, u.humanReadable), operations.CountString(count, u.humanReadable), message)
 	}
 
 	// Show the box on top if required
 	if u.showBox {
 		u.Box()
 	}
-	err := termbox.Flush()
-	if err != nil {
-		return fmt.Errorf("failed to flush screen: %w", err)
-	}
+	u.s.Show()
 	return nil
 }
 
@@ -886,37 +899,34 @@ func NewUI(f fs.Fs) *UI {
 
 // Show shows the user interface
 func (u *UI) Show() error {
-	err := termbox.Init()
+	var err error
+	u.s, err = tcell.NewScreen()
 	if err != nil {
-		return fmt.Errorf("termbox init: %w", err)
+		return fmt.Errorf("screen new: %w", err)
 	}
-	defer termbox.Close()
+	err = u.s.Init()
+	if err != nil {
+		return fmt.Errorf("screen init: %w", err)
+	}
+	defer u.s.Fini()
 
 	// scan the disk in the background
 	u.listing = true
 	rootChan, errChan, updated := scan.Scan(context.Background(), u.f)
 
 	// Poll the events into a channel
-	events := make(chan termbox.Event)
-	doneWithEvent := make(chan bool)
-	go func() {
-		for {
-			events <- termbox.PollEvent()
-			<-doneWithEvent
-		}
-	}()
+	events := make(chan tcell.Event)
+	go u.s.ChannelEvents(events, nil)
 
 	// Main loop, waiting for events and channels
 outer:
 	for {
-		//Reset()
 		err := u.Draw()
 		if err != nil {
 			return fmt.Errorf("draw failed: %w", err)
 		}
-		var root *scan.Dir
 		select {
-		case root = <-rootChan:
+		case root := <-rootChan:
 			u.root = root
 			u.setCurrentDir(root)
 		case err := <-errChan:
@@ -926,39 +936,50 @@ outer:
 			u.listing = false
 		case <-updated:
 			// redraw
-			// might want to limit updates per second
+			// TODO: might want to limit updates per second
 			u.sortCurrentDir()
 		case ev := <-events:
-			doneWithEvent <- true
-			if ev.Type == termbox.EventKey {
-				switch ev.Key + termbox.Key(ev.Ch) {
-				case termbox.KeyEsc, termbox.KeyCtrlC, 'q':
+			switch ev := ev.(type) {
+			case *tcell.EventResize:
+				if u.root != nil {
+					u.sortCurrentDir() // redraw
+				}
+				u.s.Sync()
+			case *tcell.EventKey:
+				var c rune
+				if k := ev.Key(); k == tcell.KeyRune {
+					c = ev.Rune()
+				} else {
+					c = key(k)
+				}
+				switch c {
+				case key(tcell.KeyEsc), key(tcell.KeyCtrlC), 'q':
 					if u.showBox {
 						u.showBox = false
 					} else {
 						break outer
 					}
-				case termbox.KeyArrowDown, 'j':
+				case key(tcell.KeyDown), 'j':
 					u.move(1)
-				case termbox.KeyArrowUp, 'k':
+				case key(tcell.KeyUp), 'k':
 					u.move(-1)
-				case termbox.KeyPgdn, '-', '_':
+				case key(tcell.KeyPgDn), '-', '_':
 					u.move(u.dirListHeight)
-				case termbox.KeyPgup, '=', '+':
+				case key(tcell.KeyPgUp), '=', '+':
 					u.move(-u.dirListHeight)
-				case termbox.KeyArrowLeft, 'h':
+				case key(tcell.KeyLeft), 'h':
 					if u.showBox {
 						u.moveBox(-1)
 						break
 					}
 					u.up()
-				case termbox.KeyEnter:
+				case key(tcell.KeyEnter):
 					if len(u.boxMenu) > 0 {
 						u.handleBoxOption()
 						break
 					}
 					u.enter()
-				case termbox.KeyArrowRight, 'l':
+				case key(tcell.KeyRight), 'l':
 					if u.showBox {
 						u.moveBox(1)
 						break
@@ -1001,11 +1022,8 @@ outer:
 
 				// Refresh the screen. Not obvious what key to map
 				// this onto, but ^L is a common choice.
-				case termbox.KeyCtrlL:
-					err := termbox.Sync()
-					if err != nil {
-						fs.Errorf(nil, "termbox sync returned error: %v", err)
-					}
+				case key(tcell.KeyCtrlL):
+					u.s.Sync()
 				}
 			}
 		}
@@ -1013,3 +1031,9 @@ outer:
 	}
 	return nil
 }
+
+// key returns a rune representing the key k. It is larger than the maximum Unicode code-point.
+func key(k tcell.Key) rune {
+	// This is the maximum possible Unicode code point. Anything greater fails to compile as a Go quoted rune literal.
+	return '\U0010FFFF' + rune(k)
+}

From 7672cde4f348c6485c7de39795ef7d538737b46f Mon Sep 17 00:00:00 2001
From: eNV25 
Date: Sun, 27 Nov 2022 03:32:45 +0530
Subject: [PATCH 407/560] cmd/ncdu: use negative values for key runes

The previous version used values after the maximum Unicode code-point
to encode a key. This could lead to an overflow since a key is a int16,
a rune is int32 and the maximum Unicode code-point is larger than int16.

A better solution is to simply use negative runes for keys.
---
 cmd/ncdu/ncdu.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go
index edf0a570c0834..f6de22f1af0c5 100644
--- a/cmd/ncdu/ncdu.go
+++ b/cmd/ncdu/ncdu.go
@@ -1032,8 +1032,7 @@ outer:
 	return nil
 }
 
-// key returns a rune representing the key k. It is larger than the maximum Unicode code-point.
+// key returns a rune representing the key k. It is a negative value, to not collide with Unicode code-points.
 func key(k tcell.Key) rune {
-	// This is the maximum possible Unicode code point. Anything greater fails to compile as a Go quoted rune literal.
-	return '\U0010FFFF' + rune(k)
+	return rune(-k)
 }

From 50c9678cea8fd9faa155e0663b88433978c041a5 Mon Sep 17 00:00:00 2001
From: ycdtosa 
Date: Tue, 29 Nov 2022 14:58:46 +0100
Subject: [PATCH 408/560] ftp: update help text of implicit/explicit TLS
 options to refer to FTPS instead of FTP

---
 backend/ftp/ftp.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go
index ede6d088ac981..c68e7dc4a1a32 100644
--- a/backend/ftp/ftp.go
+++ b/backend/ftp/ftp.go
@@ -70,7 +70,7 @@ func init() {
 When using implicit FTP over TLS the client connects using TLS
 right from the start which breaks compatibility with
 non-TLS-aware servers. This is usually served over port 990 rather
-than port 21. Cannot be used in combination with explicit FTP.`,
+than port 21. Cannot be used in combination with explicit FTPS.`,
 			Default: false,
 		}, {
 			Name: "explicit_tls",
@@ -78,7 +78,7 @@ than port 21. Cannot be used in combination with explicit FTP.`,
 
 When using explicit FTP over TLS the client explicitly requests
 security from the server in order to upgrade a plain text connection
-to an encrypted one. Cannot be used in combination with implicit FTP.`,
+to an encrypted one. Cannot be used in combination with implicit FTPS.`,
 			Default: false,
 		}, {
 			Name: "concurrency",

From 097be753abe8f4b0bb52fd722988c5ac9c31281a Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Sat, 19 Nov 2022 12:50:20 +0100
Subject: [PATCH 409/560] docs: minor cleanup of headers in backend docs

---
 docs/content/chunker.md  | 2 +-
 docs/content/compress.md | 2 +-
 docs/content/crypt.md    | 2 +-
 docs/content/seafile.md  | 2 +-
 docs/content/swift.md    | 2 +-
 docs/content/yandex.md   | 2 +-
 docs/content/zoho.md     | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/content/chunker.md b/docs/content/chunker.md
index 934f102fff21f..4403ad5d3d38a 100644
--- a/docs/content/chunker.md
+++ b/docs/content/chunker.md
@@ -4,7 +4,7 @@ description: "Split-chunking overlay remote"
 versionIntroduced: "v1.50.0"
 ---
 
-# {{< icon "fa fa-cut" >}}Chunker (BETA)
+# {{< icon "fa fa-cut" >}} Chunker (EXPERIMENTAL)
 
 The `chunker` overlay transparently splits large files into smaller chunks
 during upload to wrapped remote and transparently assembles them back
diff --git a/docs/content/compress.md b/docs/content/compress.md
index e6bf533b7ca93..7ee03a25a5df6 100644
--- a/docs/content/compress.md
+++ b/docs/content/compress.md
@@ -4,7 +4,7 @@ description: "Compression Remote"
 versionIntroduced: "v1.54.0"
 ---
 
-# {{< icon "fas fa-compress" >}}Compress (Experimental)
+# {{< icon "fas fa-compress" >}} Compress (EXPERIMENTAL)
 
 ## Warning
 
diff --git a/docs/content/crypt.md b/docs/content/crypt.md
index 3cde0ae024689..813aa08307e14 100644
--- a/docs/content/crypt.md
+++ b/docs/content/crypt.md
@@ -4,7 +4,7 @@ description: "Encryption overlay remote"
 versionIntroduced: "v1.33"
 ---
 
-# {{< icon "fa fa-lock" >}}Crypt
+# {{< icon "fa fa-lock" >}} Crypt
 
 Rclone `crypt` remotes encrypt and decrypt other remotes.
 
diff --git a/docs/content/seafile.md b/docs/content/seafile.md
index ab45184ece1d1..55d2a178a1ae2 100644
--- a/docs/content/seafile.md
+++ b/docs/content/seafile.md
@@ -4,7 +4,7 @@ description: "Seafile"
 versionIntroduced: "v1.52.0"
 ---
 
-# {{< icon "fa fa-server" >}}Seafile
+# {{< icon "fa fa-server" >}} Seafile
 
 This is a backend for the [Seafile](https://www.seafile.com/) storage service:
 - It works with both the free community edition or the professional edition.
diff --git a/docs/content/swift.md b/docs/content/swift.md
index e768e35ffc37a..b1334ac03967b 100644
--- a/docs/content/swift.md
+++ b/docs/content/swift.md
@@ -4,7 +4,7 @@ description: "Swift"
 versionIntroduced: "v0.91"
 ---
 
-# {{< icon "fa fa-space-shuttle" >}}Swift
+# {{< icon "fa fa-space-shuttle" >}} Swift
 
 Swift refers to [OpenStack Object Storage](https://docs.openstack.org/swift/latest/).
 Commercial implementations of that being:
diff --git a/docs/content/yandex.md b/docs/content/yandex.md
index f6c32e1a907a6..f5716ca7006d8 100644
--- a/docs/content/yandex.md
+++ b/docs/content/yandex.md
@@ -4,7 +4,7 @@ description: "Yandex Disk"
 versionIntroduced: "v1.26"
 ---
 
-# {{< icon "fa fa-space-shuttle" >}}Yandex Disk
+# {{< icon "fa fa-space-shuttle" >}} Yandex Disk
 
 [Yandex Disk](https://disk.yandex.com) is a cloud storage solution created by [Yandex](https://yandex.com).
 
diff --git a/docs/content/zoho.md b/docs/content/zoho.md
index 26e5ccb699db1..c0143dcbf8408 100644
--- a/docs/content/zoho.md
+++ b/docs/content/zoho.md
@@ -4,7 +4,7 @@ description: "Zoho WorkDrive"
 versionIntroduced: "v1.54.0"
 ---
 
-# {{< icon "fas fa-folder" >}}Zoho Workdrive
+# {{< icon "fas fa-folder" >}} Zoho Workdrive
 
 [Zoho WorkDrive](https://www.zoho.com/workdrive/) is a cloud storage solution created by [Zoho](https://zoho.com).
 

From d05fd2a14f67daa7b30d71e05af12ed2c8ad8138 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Sat, 19 Nov 2022 13:33:15 +0100
Subject: [PATCH 410/560] docs: add badge for experimental/beta/deprecated
 status next to version in backend docs

---
 docs/content/cache.md             |  3 ++-
 docs/content/chunker.md           |  3 ++-
 docs/content/compress.md          |  3 ++-
 docs/content/hasher.md            |  3 ++-
 docs/layouts/_default/baseof.html | 13 +++++++++++++
 5 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/docs/content/cache.md b/docs/content/cache.md
index bd28b037b0bf0..9b787366ef79c 100644
--- a/docs/content/cache.md
+++ b/docs/content/cache.md
@@ -2,9 +2,10 @@
 title: "Cache"
 description: "Rclone docs for cache remote"
 versionIntroduced: "v1.39"
+status: Deprecated
 ---
 
-# {{< icon "fa fa-archive" >}} Cache (DEPRECATED)
+# {{< icon "fa fa-archive" >}} Cache
 
 The `cache` remote wraps another existing remote and stores file structure
 and its data for long running tasks like `rclone mount`.
diff --git a/docs/content/chunker.md b/docs/content/chunker.md
index 4403ad5d3d38a..931e9a5d82bfd 100644
--- a/docs/content/chunker.md
+++ b/docs/content/chunker.md
@@ -2,9 +2,10 @@
 title: "Chunker"
 description: "Split-chunking overlay remote"
 versionIntroduced: "v1.50.0"
+status: Beta
 ---
 
-# {{< icon "fa fa-cut" >}} Chunker (EXPERIMENTAL)
+# {{< icon "fa fa-cut" >}} Chunker
 
 The `chunker` overlay transparently splits large files into smaller chunks
 during upload to wrapped remote and transparently assembles them back
diff --git a/docs/content/compress.md b/docs/content/compress.md
index 7ee03a25a5df6..27343236c355a 100644
--- a/docs/content/compress.md
+++ b/docs/content/compress.md
@@ -2,9 +2,10 @@
 title: "Compress"
 description: "Compression Remote"
 versionIntroduced: "v1.54.0"
+status: Experimental
 ---
 
-# {{< icon "fas fa-compress" >}} Compress (EXPERIMENTAL)
+# {{< icon "fas fa-compress" >}} Compress
 
 ## Warning
 
diff --git a/docs/content/hasher.md b/docs/content/hasher.md
index aa492d3a771cb..b52fb3bc9553d 100644
--- a/docs/content/hasher.md
+++ b/docs/content/hasher.md
@@ -2,9 +2,10 @@
 title: "Hasher"
 description: "Better checksums for other remotes"
 versionIntroduced: "v1.57.0"
+status: Experimental
 ---
 
-# {{< icon "fa fa-check-double" >}} Hasher (EXPERIMENTAL)
+# {{< icon "fa fa-check-double" >}} Hasher
 
 Hasher is a special overlay backend to create remotes which handle
 checksums for other remotes. It's main functions include:
diff --git a/docs/layouts/_default/baseof.html b/docs/layouts/_default/baseof.html
index 6598c2f8c79dd..e84ba3f294497 100644
--- a/docs/layouts/_default/baseof.html
+++ b/docs/layouts/_default/baseof.html
@@ -26,7 +26,20 @@
     
                  
       
                  
         
                  
+          {{ $statusMap := dict
+            "deprecated" "This feature is currently deprecated."
+            "experimental" "This feature is currently experimental and any use is at your own risk."
+            "beta" "This feature is currently in beta must be used with care."
+          }}
+
           {{ if .Params.versionIntroduced }}{{ .Params.versionIntroduced }}{{ end }}
+
+          {{ with .Params.status | lower }}
+            {{ $statusCode := . }}
+            {{ $statusMessage := index $statusMap $statusCode }}
+            {{ $statusCode }}
+          {{ end }}
+
           {{ block "main" . }}
           {{ end }}
         
                  

From d74662a751912858d3306e0bc6bb1bd1ad683543 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 25 Nov 2022 23:41:23 +0100
Subject: [PATCH 411/560] docs: add badge showing version introduced and
 experimental/beta/deprecated status to command doc pages

---
 cmd/gendocs/gendocs.go | 28 ++++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/cmd/gendocs/gendocs.go b/cmd/gendocs/gendocs.go
index 4965aa7a38d56..4c5417abccd4a 100644
--- a/cmd/gendocs/gendocs.go
+++ b/cmd/gendocs/gendocs.go
@@ -31,6 +31,7 @@ type frontmatter struct {
 	Slug        string
 	URL         string
 	Source      string
+	Annotations map[string]string
 }
 
 var frontmatterTemplate = template.Must(template.New("frontmatter").Parse(`---
@@ -38,6 +39,9 @@ title: "{{ .Title }}"
 description: "{{ .Description }}"
 slug: {{ .Slug }}
 url: {{ .URL }}
+{{- range $key, $value := .Annotations }}
+{{ $key }}: {{  $value }}
+{{- end }}
 # autogenerated - DO NOT EDIT, instead edit the source code in {{ .Source }} and as part of making a release run "make commanddocs"
 ---
 `))
@@ -75,17 +79,24 @@ rclone.org website.`,
 			return err
 		}
 
-		// Look up name => description for prepender
-		var description = map[string]string{}
-		var addDescription func(root *cobra.Command)
-		addDescription = func(root *cobra.Command) {
+		// Look up name => details for prepender
+		type commandDetails struct {
+			Short       string
+			Annotations map[string]string
+		}
+		var commands = map[string]commandDetails{}
+		var addCommandDetails func(root *cobra.Command)
+		addCommandDetails = func(root *cobra.Command) {
 			name := strings.ReplaceAll(root.CommandPath(), " ", "_") + ".md"
-			description[name] = root.Short
+			commands[name] = commandDetails{
+				Short:       root.Short,
+				Annotations: root.Annotations,
+			}
 			for _, c := range root.Commands() {
-				addDescription(c)
+				addCommandDetails(c)
 			}
 		}
-		addDescription(cmd.Root)
+		addCommandDetails(cmd.Root)
 
 		// markup for the docs files
 		prepender := func(filename string) string {
@@ -94,10 +105,11 @@ rclone.org website.`,
 			data := frontmatter{
 				Date:        now,
 				Title:       strings.ReplaceAll(base, "_", " "),
-				Description: description[name],
+				Description: commands[name].Short,
 				Slug:        base,
 				URL:         "/commands/" + strings.ToLower(base) + "/",
 				Source:      strings.ReplaceAll(strings.ReplaceAll(base, "rclone", "cmd"), "_", "/") + "/",
+				Annotations: commands[name].Annotations,
 			}
 			var buf bytes.Buffer
 			err := frontmatterTemplate.Execute(&buf, data)

From 6d18f607251beffc4566e3e2391368f061f73a3c Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Sat, 26 Nov 2022 23:40:49 +0100
Subject: [PATCH 412/560] docs: add minimum versions to the command pages

---
 cmd/about/about.go                     |  3 ++
 cmd/authorize/authorize.go             |  3 ++
 cmd/backend/backend.go                 |  3 ++
 cmd/bisync/cmd.go                      |  3 ++
 cmd/cachestats/cachestats.go           |  4 +++
 cmd/cat/cat.go                         |  3 ++
 cmd/check/check.go                     |  3 ++
 cmd/checksum/checksum.go               |  3 ++
 cmd/cleanup/cleanup.go                 |  3 ++
 cmd/config/config.go                   | 38 +++++++++++++++++++++++++-
 cmd/copy/copy.go                       |  3 ++
 cmd/copyto/copyto.go                   |  3 ++
 cmd/copyurl/copyurl.go                 |  3 ++
 cmd/cryptcheck/cryptcheck.go           |  3 ++
 cmd/cryptdecode/cryptdecode.go         |  3 ++
 cmd/dedupe/dedupe.go                   |  3 ++
 cmd/delete/delete.go                   |  3 ++
 cmd/deletefile/deletefile.go           |  3 ++
 cmd/genautocomplete/genautocomplete.go |  3 ++
 cmd/gendocs/gendocs.go                 |  3 ++
 cmd/hashsum/hashsum.go                 |  3 ++
 cmd/help.go                            |  3 ++
 cmd/link/link.go                       |  3 ++
 cmd/listremotes/listremotes.go         |  3 ++
 cmd/ls/ls.go                           |  3 ++
 cmd/lsd/lsd.go                         |  3 ++
 cmd/lsf/lsf.go                         |  3 ++
 cmd/lsjson/lsjson.go                   |  3 ++
 cmd/lsl/lsl.go                         |  3 ++
 cmd/md5sum/md5sum.go                   |  3 ++
 cmd/mkdir/mkdir.go                     |  3 ++
 cmd/mountlib/mount.go                  |  3 ++
 cmd/move/move.go                       |  3 ++
 cmd/moveto/moveto.go                   |  3 ++
 cmd/ncdu/ncdu.go                       |  3 ++
 cmd/obscure/obscure.go                 |  3 ++
 cmd/purge/purge.go                     |  3 ++
 cmd/rc/rc.go                           |  3 ++
 cmd/rcat/rcat.go                       |  3 ++
 cmd/rcd/rcd.go                         |  3 ++
 cmd/reveal/reveal.go                   |  3 ++
 cmd/rmdir/rmdir.go                     |  3 ++
 cmd/rmdirs/rmdirs.go                   |  3 ++
 cmd/selfupdate/selfupdate.go           |  3 ++
 cmd/serve/dlna/dlna.go                 |  3 ++
 cmd/serve/docker/docker.go             |  4 ++-
 cmd/serve/ftp/ftp.go                   |  3 ++
 cmd/serve/http/http.go                 |  3 ++
 cmd/serve/restic/restic.go             |  3 ++
 cmd/serve/serve.go                     |  3 ++
 cmd/serve/sftp/sftp.go                 |  3 ++
 cmd/serve/webdav/webdav.go             |  3 ++
 cmd/settier/settier.go                 |  3 ++
 cmd/sha1sum/sha1sum.go                 |  3 ++
 cmd/size/size.go                       |  3 ++
 cmd/sync/sync.go                       |  3 ++
 cmd/test/changenotify/changenotify.go  |  3 ++
 cmd/test/histogram/histogram.go        |  3 ++
 cmd/test/info/info.go                  |  3 ++
 cmd/test/makefiles/makefiles.go        |  6 ++++
 cmd/test/memory/memory.go              |  3 ++
 cmd/test/test.go                       |  3 ++
 cmd/touch/touch.go                     |  3 ++
 cmd/tree/tree.go                       |  3 ++
 cmd/version/version.go                 |  3 ++
 65 files changed, 233 insertions(+), 2 deletions(-)

diff --git a/cmd/about/about.go b/cmd/about/about.go
index 967744b598782..6d3afc91de882 100644
--- a/cmd/about/about.go
+++ b/cmd/about/about.go
@@ -93,6 +93,9 @@ provided by a backend. Where the value is unlimited it is omitted.
 Some backends does not support the ` + "`rclone about`" + ` command at all,
 see complete list in [documentation](https://rclone.org/overview/#optional-features).
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.41",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		f := cmd.NewFsSrc(args)
diff --git a/cmd/authorize/authorize.go b/cmd/authorize/authorize.go
index 87dda815bdc5b..84c0bc0f3c81f 100644
--- a/cmd/authorize/authorize.go
+++ b/cmd/authorize/authorize.go
@@ -30,6 +30,9 @@ rclone config.
 
 Use the --auth-no-open-browser to prevent rclone to open auth
 link in default browser automatically.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.27",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 3, command, args)
 		return config.Authorize(context.Background(), args, noAutoBrowser)
diff --git a/cmd/backend/backend.go b/cmd/backend/backend.go
index 3199f8a5deba0..67d499923698e 100644
--- a/cmd/backend/backend.go
+++ b/cmd/backend/backend.go
@@ -58,6 +58,9 @@ Pass arguments to the backend by placing them on the end of the line
 Note to run these commands on a running backend then see
 [backend/command](/rc/#backend-command) in the rc docs.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.52",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(2, 1e6, command, args)
 		name, remote := args[0], args[1]
diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.go
index c14edf2e72ace..6f7c8fc151634 100644
--- a/cmd/bisync/cmd.go
+++ b/cmd/bisync/cmd.go
@@ -115,6 +115,9 @@ var commandDefinition = &cobra.Command{
 	Use:   "bisync remote1:path1 remote2:path2",
 	Short: shortHelp,
 	Long:  longHelp,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.58",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(2, 2, command, args)
 		fs1, file1, fs2, file2 := cmd.NewFsSrcDstFiles(args)
diff --git a/cmd/cachestats/cachestats.go b/cmd/cachestats/cachestats.go
index 7638db8f4cb3b..f5cbf009586af 100644
--- a/cmd/cachestats/cachestats.go
+++ b/cmd/cachestats/cachestats.go
@@ -25,6 +25,10 @@ var commandDefinition = &cobra.Command{
 Print cache stats for a remote in JSON format
 `,
 	Hidden: true,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+		"status":            "Deprecated",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fs.Logf(nil, `"rclone cachestats" is deprecated, use "rclone backend stats %s" instead`, args[0])
diff --git a/cmd/cat/cat.go b/cmd/cat/cat.go
index ead3614556592..03ed4e6e39ebc 100644
--- a/cmd/cat/cat.go
+++ b/cmd/cat/cat.go
@@ -57,6 +57,9 @@ the end and |--offset| and |--count| to print a section in the middle.
 Note that if offset is negative it will count from the end, so
 |--offset -1 --count 1| is equivalent to |--tail 1|.
 `, "|", "`"),
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.33",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		usedOffset := offset != 0 || count >= 0
 		usedHead := head > 0
diff --git a/cmd/check/check.go b/cmd/check/check.go
index 3c126e176275b..e4bcc156ad242 100644
--- a/cmd/check/check.go
+++ b/cmd/check/check.go
@@ -155,6 +155,9 @@ to check all the data.
 If you supply the |--checkfile HASH| flag with a valid hash name,
 the |source:path| must point to a text file in the SUM format.
 `, "|", "`") + FlagsHelp,
+	Annotations: map[string]string{
+		"versionIntroduced": "v0.90",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(2, 2, command, args)
 		var (
diff --git a/cmd/checksum/checksum.go b/cmd/checksum/checksum.go
index c25aa143d49f7..4dee6611c3020 100644
--- a/cmd/checksum/checksum.go
+++ b/cmd/checksum/checksum.go
@@ -37,6 +37,9 @@ that don't support hashes or if you really want to check all the data.
 
 Note that hash values in the SUM file are treated as case insensitive.
 `, "|", "`") + check.FlagsHelp,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.56",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(3, 3, command, args)
 		var hashType hash.Type
diff --git a/cmd/cleanup/cleanup.go b/cmd/cleanup/cleanup.go
index e54577466897d..d23f0cb91bd17 100644
--- a/cmd/cleanup/cleanup.go
+++ b/cmd/cleanup/cleanup.go
@@ -20,6 +20,9 @@ var commandDefinition = &cobra.Command{
 Clean up the remote if possible.  Empty the trash or delete old file
 versions. Not supported by all remotes.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.31",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
diff --git a/cmd/config/config.go b/cmd/config/config.go
index 66b2a1c6db702..5fb21a0cf460a 100644
--- a/cmd/config/config.go
+++ b/cmd/config/config.go
@@ -44,6 +44,9 @@ var configCommand = &cobra.Command{
 remotes and manage existing ones. You may also set or remove a
 password to protect your configuration.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(0, 0, command, args)
 		return config.EditConfig(context.Background())
@@ -54,12 +57,18 @@ var configEditCommand = &cobra.Command{
 	Use:   "edit",
 	Short: configCommand.Short,
 	Long:  configCommand.Long,
-	Run:   configCommand.Run,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
+	Run: configCommand.Run,
 }
 
 var configFileCommand = &cobra.Command{
 	Use:   "file",
 	Short: `Show path of configuration file in use.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.38",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 0, command, args)
 		config.ShowConfigLocation()
@@ -69,6 +78,9 @@ var configFileCommand = &cobra.Command{
 var configTouchCommand = &cobra.Command{
 	Use:   "touch",
 	Short: `Ensure configuration file exists.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.56",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 0, command, args)
 		config.SaveConfig()
@@ -78,6 +90,9 @@ var configTouchCommand = &cobra.Command{
 var configPathsCommand = &cobra.Command{
 	Use:   "paths",
 	Short: `Show paths used for configuration, cache, temp etc.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.57",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 0, command, args)
 		fmt.Printf("Config file: %s\n", config.GetConfigPath())
@@ -89,6 +104,9 @@ var configPathsCommand = &cobra.Command{
 var configShowCommand = &cobra.Command{
 	Use:   "show []",
 	Short: `Print (decrypted) config file, or the config for a single remote.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.38",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1, command, args)
 		if len(args) == 0 {
@@ -103,6 +121,9 @@ var configShowCommand = &cobra.Command{
 var configDumpCommand = &cobra.Command{
 	Use:   "dump",
 	Short: `Dump the config file as JSON.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(0, 0, command, args)
 		return config.Dump()
@@ -112,6 +133,9 @@ var configDumpCommand = &cobra.Command{
 var configProvidersCommand = &cobra.Command{
 	Use:   "providers",
 	Short: `List in JSON format all the providers and options.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(0, 0, command, args)
 		return config.JSONListProviders()
@@ -226,6 +250,9 @@ using remote authorization you would do this:
 
     rclone config create mydrive drive config_is_local=false
 `, "|", "`") + configPasswordHelp,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(2, 256, command, args)
 		in, err := argsToMap(args[2:])
@@ -289,6 +316,9 @@ require this add an extra parameter thus:
 
     rclone config update myremote env_auth=true config_refresh_token=false
 `, "|", "`") + configPasswordHelp,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 256, command, args)
 		in, err := argsToMap(args[1:])
@@ -304,6 +334,9 @@ require this add an extra parameter thus:
 var configDeleteCommand = &cobra.Command{
 	Use:   "delete name",
 	Short: "Delete an existing remote.",
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		config.DeleteRemote(args[0])
@@ -326,6 +359,9 @@ For example, to set password of a remote of name myremote you would do:
 This command is obsolete now that "config update" and "config create"
 both support obscuring passwords directly.
 `, "|", "`"),
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 256, command, args)
 		in, err := argsToMap(args[1:])
diff --git a/cmd/copy/copy.go b/cmd/copy/copy.go
index c6e42a70ea9bd..6a365b25213e0 100644
--- a/cmd/copy/copy.go
+++ b/cmd/copy/copy.go
@@ -83,6 +83,9 @@ recently very efficiently like this:
 
 **Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything.
 `, "|", "`"),
+	Annotations: map[string]string{
+		"versionIntroduced": "v0.90",
+	},
 	Run: func(command *cobra.Command, args []string) {
 
 		cmd.CheckArgs(2, 2, command, args)
diff --git a/cmd/copyto/copyto.go b/cmd/copyto/copyto.go
index 85b54fa609a62..337a80b3aaa42 100644
--- a/cmd/copyto/copyto.go
+++ b/cmd/copyto/copyto.go
@@ -46,6 +46,9 @@ the destination.
 
 **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.35",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
diff --git a/cmd/copyurl/copyurl.go b/cmd/copyurl/copyurl.go
index a3c079b11c8ef..9b9063861a355 100644
--- a/cmd/copyurl/copyurl.go
+++ b/cmd/copyurl/copyurl.go
@@ -51,6 +51,9 @@ destination if there is one with the same name.
 Setting ` + "`--stdout`" + ` or making the output file name ` + "`-`" + `
 will cause the output to be written to standard output.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.43",
+	},
 	RunE: func(command *cobra.Command, args []string) (err error) {
 		cmd.CheckArgs(1, 2, command, args)
 
diff --git a/cmd/cryptcheck/cryptcheck.go b/cmd/cryptcheck/cryptcheck.go
index aab50ef06f8e5..481c4cecec9ee 100644
--- a/cmd/cryptcheck/cryptcheck.go
+++ b/cmd/cryptcheck/cryptcheck.go
@@ -47,6 +47,9 @@ the files in remote:path.
 
 After it has run it will log the status of the encryptedremote:.
 ` + check.FlagsHelp,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.36",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, fdst := cmd.NewFsSrcDst(args)
diff --git a/cmd/cryptdecode/cryptdecode.go b/cmd/cryptdecode/cryptdecode.go
index 9c6178e9e4c1a..a657bb0c59be7 100644
--- a/cmd/cryptdecode/cryptdecode.go
+++ b/cmd/cryptdecode/cryptdecode.go
@@ -41,6 +41,9 @@ use it like this
 Another way to accomplish this is by using the ` + "`rclone backend encode` (or `decode`)" + ` command.
 See the documentation on the [crypt](/crypt/) overlay for more info.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.38",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 11, command, args)
 		cmd.Run(false, false, command, func() error {
diff --git a/cmd/dedupe/dedupe.go b/cmd/dedupe/dedupe.go
index 94a136ca2f69b..b56d62d9f4095 100644
--- a/cmd/dedupe/dedupe.go
+++ b/cmd/dedupe/dedupe.go
@@ -135,6 +135,9 @@ Or
 
     rclone dedupe rename "drive:Google Photos"
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.27",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 2, command, args)
 		if len(args) > 1 {
diff --git a/cmd/delete/delete.go b/cmd/delete/delete.go
index 047c320af9855..4dcc69ddd3b3c 100644
--- a/cmd/delete/delete.go
+++ b/cmd/delete/delete.go
@@ -53,6 +53,9 @@ delete all files bigger than 100 MiB.
 **Important**: Since this can cause data loss, test first with the
 |--dry-run| or the |--interactive|/|-i| flag.
 `, "|", "`"),
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.27",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
diff --git a/cmd/deletefile/deletefile.go b/cmd/deletefile/deletefile.go
index 9d222771d2f4a..4bf4ad72dfdd9 100644
--- a/cmd/deletefile/deletefile.go
+++ b/cmd/deletefile/deletefile.go
@@ -22,6 +22,9 @@ Remove a single file from remote.  Unlike ` + "`" + `delete` + "`" + ` it cannot
 remove a directory and it doesn't obey include/exclude filters - if the specified file exists,
 it will always be removed.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.42",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fs, fileName := cmd.NewFsFile(args[0])
diff --git a/cmd/genautocomplete/genautocomplete.go b/cmd/genautocomplete/genautocomplete.go
index 9502bbbc9d439..c1408d9f7615a 100644
--- a/cmd/genautocomplete/genautocomplete.go
+++ b/cmd/genautocomplete/genautocomplete.go
@@ -17,4 +17,7 @@ var completionDefinition = &cobra.Command{
 Generates a shell completion script for rclone.
 Run with ` + "`--help`" + ` to list the supported shells.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.33",
+	},
 }
diff --git a/cmd/gendocs/gendocs.go b/cmd/gendocs/gendocs.go
index 4c5417abccd4a..83c54846d44a8 100644
--- a/cmd/gendocs/gendocs.go
+++ b/cmd/gendocs/gendocs.go
@@ -53,6 +53,9 @@ var commandDefinition = &cobra.Command{
 This produces markdown docs for the rclone commands to the directory
 supplied.  These are in a format suitable for hugo to render into the
 rclone.org website.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.33",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 1, command, args)
 		now := time.Now().Format(time.RFC3339)
diff --git a/cmd/hashsum/hashsum.go b/cmd/hashsum/hashsum.go
index 4a1993716eb07..6e39cf7ce82a4 100644
--- a/cmd/hashsum/hashsum.go
+++ b/cmd/hashsum/hashsum.go
@@ -112,6 +112,9 @@ Then
 
 Note that hash names are case insensitive and values are output in lower case.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.41",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(0, 2, command, args)
 		if len(args) == 0 {
diff --git a/cmd/help.go b/cmd/help.go
index ecc34b63ea490..949abe45bb4f8 100644
--- a/cmd/help.go
+++ b/cmd/help.go
@@ -33,6 +33,9 @@ See the home page (https://rclone.org/) for installation, usage,
 documentation, changelog and configuration walkthroughs.
 
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v0.90",
+	},
 	PersistentPostRun: func(cmd *cobra.Command, args []string) {
 		fs.Debugf("rclone", "Version %q finishing with parameters %q", fs.Version, os.Args)
 		atexit.Run()
diff --git a/cmd/link/link.go b/cmd/link/link.go
index a0356cd3369fd..380e5a46b0782 100644
--- a/cmd/link/link.go
+++ b/cmd/link/link.go
@@ -49,6 +49,9 @@ link. Exact capabilities depend on the remote, but the link will
 always by default be created with the least constraints – e.g. no
 expiry, no password protection, accessible without account.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.41",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc, remote := cmd.NewFsFile(args[0])
diff --git a/cmd/listremotes/listremotes.go b/cmd/listremotes/listremotes.go
index bf7b697f235d0..e473fcb212888 100644
--- a/cmd/listremotes/listremotes.go
+++ b/cmd/listremotes/listremotes.go
@@ -30,6 +30,9 @@ rclone listremotes lists all the available remotes from the config file.
 
 When used with the ` + "`--long`" + ` flag it lists the types too.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.34",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 0, command, args)
 		remotes := config.FileSections()
diff --git a/cmd/ls/ls.go b/cmd/ls/ls.go
index 334b12f8e8ecc..cecab52a0ed56 100644
--- a/cmd/ls/ls.go
+++ b/cmd/ls/ls.go
@@ -31,6 +31,9 @@ Eg
         37600 fubuwic
 
 ` + lshelp.Help,
+	Annotations: map[string]string{
+		"versionIntroduced": "v0.90",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
diff --git a/cmd/lsd/lsd.go b/cmd/lsd/lsd.go
index 8d79266fb6985..d6681bdad6204 100644
--- a/cmd/lsd/lsd.go
+++ b/cmd/lsd/lsd.go
@@ -49,6 +49,9 @@ Or
 If you just want the directory names use ` + "`rclone lsf --dirs-only`" + `.
 
 ` + lshelp.Help,
+	Annotations: map[string]string{
+		"versionIntroduced": "v0.90",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		ci := fs.GetConfig(context.Background())
 		cmd.CheckArgs(1, 1, command, args)
diff --git a/cmd/lsf/lsf.go b/cmd/lsf/lsf.go
index 2651e90e7263e..8322e7c25685f 100644
--- a/cmd/lsf/lsf.go
+++ b/cmd/lsf/lsf.go
@@ -142,6 +142,9 @@ those only (without traversing the whole directory structure):
     rclone copy --files-from-raw new_files /path/to/local remote:path
 
 ` + lshelp.Help,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.40",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
diff --git a/cmd/lsjson/lsjson.go b/cmd/lsjson/lsjson.go
index a17ade32749a8..c7da1c8637064 100644
--- a/cmd/lsjson/lsjson.go
+++ b/cmd/lsjson/lsjson.go
@@ -112,6 +112,9 @@ will be shown ("2017-05-31T16:15:57+01:00").
 The whole output can be processed as a JSON blob, or alternatively it
 can be processed line by line as each item is written one to a line.
 ` + lshelp.Help,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.37",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 1, command, args)
 		var fsrc fs.Fs
diff --git a/cmd/lsl/lsl.go b/cmd/lsl/lsl.go
index f8108c5f7b8df..e7a7366f17a08 100644
--- a/cmd/lsl/lsl.go
+++ b/cmd/lsl/lsl.go
@@ -31,6 +31,9 @@ Eg
         37600 2016-06-25 18:55:40.814629136 fubuwic
 
 ` + lshelp.Help,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.02",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
diff --git a/cmd/md5sum/md5sum.go b/cmd/md5sum/md5sum.go
index 91bc66e4468f4..94c0ed3ef9f59 100644
--- a/cmd/md5sum/md5sum.go
+++ b/cmd/md5sum/md5sum.go
@@ -38,6 +38,9 @@ by not passing a remote:path, or by passing a hyphen as remote:path
 when there is data to read (if not, the hyphen will be treated literally,
 as a relative path).
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.02",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(0, 1, command, args)
 		if found, err := hashsum.CreateFromStdinArg(hash.MD5, args, 0); found {
diff --git a/cmd/mkdir/mkdir.go b/cmd/mkdir/mkdir.go
index 3c99180feae52..92922ffd20395 100644
--- a/cmd/mkdir/mkdir.go
+++ b/cmd/mkdir/mkdir.go
@@ -18,6 +18,9 @@ func init() {
 var commandDefinition = &cobra.Command{
 	Use:   "mkdir remote:path",
 	Short: `Make the path if it doesn't already exist.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v0.90",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fdst := cmd.NewFsDir(args)
diff --git a/cmd/mountlib/mount.go b/cmd/mountlib/mount.go
index 907f16efdbe7d..af0eaf5a0e012 100644
--- a/cmd/mountlib/mount.go
+++ b/cmd/mountlib/mount.go
@@ -157,6 +157,9 @@ func NewMountCommand(commandName string, hidden bool, mount MountFn) *cobra.Comm
 		Hidden: hidden,
 		Short:  `Mount the remote as file system on a mountpoint.`,
 		Long:   strings.ReplaceAll(strings.ReplaceAll(mountHelp, "|", "`"), "@", commandName) + vfs.Help,
+		Annotations: map[string]string{
+			"versionIntroduced": "v1.33",
+		},
 		Run: func(command *cobra.Command, args []string) {
 			cmd.CheckArgs(2, 2, command, args)
 
diff --git a/cmd/move/move.go b/cmd/move/move.go
index 65c8adc233bfa..fd1004db11420 100644
--- a/cmd/move/move.go
+++ b/cmd/move/move.go
@@ -60,6 +60,9 @@ can speed transfers up greatly.
 
 **Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
 `, "|", "`"),
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.19",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
diff --git a/cmd/moveto/moveto.go b/cmd/moveto/moveto.go
index 698d7ee186346..f007fb18287d0 100644
--- a/cmd/moveto/moveto.go
+++ b/cmd/moveto/moveto.go
@@ -49,6 +49,9 @@ successful transfer.
 
 **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.35",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
diff --git a/cmd/ncdu/ncdu.go b/cmd/ncdu/ncdu.go
index f6de22f1af0c5..6fdb210a5d702 100644
--- a/cmd/ncdu/ncdu.go
+++ b/cmd/ncdu/ncdu.go
@@ -74,6 +74,9 @@ For a non-interactive listing of the remote, see the
 [tree](/commands/rclone_tree/) command. To just get the total size of
 the remote you can also use the [size](/commands/rclone_size/) command.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.37",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
diff --git a/cmd/obscure/obscure.go b/cmd/obscure/obscure.go
index 08cf8a06a86a8..80a473ae27a57 100644
--- a/cmd/obscure/obscure.go
+++ b/cmd/obscure/obscure.go
@@ -42,6 +42,9 @@ obfuscating the hyphen itself.
 If you want to encrypt the config file then please use config file
 encryption - see [rclone config](/commands/rclone_config/) for more
 info.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.36",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 1, command, args)
 		var password string
diff --git a/cmd/purge/purge.go b/cmd/purge/purge.go
index d41e22cf67ab5..59085f83ee56e 100644
--- a/cmd/purge/purge.go
+++ b/cmd/purge/purge.go
@@ -26,6 +26,9 @@ delete files. To delete empty directories only, use command
 **Important**: Since this can cause data loss, test first with the
 ` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v0.90",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fdst := cmd.NewFsDir(args)
diff --git a/cmd/rc/rc.go b/cmd/rc/rc.go
index 2bb163d036508..b31e340eb0836 100644
--- a/cmd/rc/rc.go
+++ b/cmd/rc/rc.go
@@ -99,6 +99,9 @@ rclone rc server, e.g.:
     rclone rc --loopback operations/about fs=/
 
 Use ` + "`rclone rc`" + ` to see a list of all possible commands.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.40",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1e9, command, args)
 		cmd.Run(false, false, command, func() error {
diff --git a/cmd/rcat/rcat.go b/cmd/rcat/rcat.go
index 4328a8aa81848..c66fe465090ea 100644
--- a/cmd/rcat/rcat.go
+++ b/cmd/rcat/rcat.go
@@ -56,6 +56,9 @@ Note that the upload can also not be retried because the data is
 not kept around until the upload succeeds. If you need to transfer
 a lot of data, you're better off caching locally and then
 ` + "`rclone move`" + ` it to the destination.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.38",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 
diff --git a/cmd/rcd/rcd.go b/cmd/rcd/rcd.go
index 940c300a6fec4..aaa57a065f6b0 100644
--- a/cmd/rcd/rcd.go
+++ b/cmd/rcd/rcd.go
@@ -32,6 +32,9 @@ the browser when rclone is run.
 
 See the [rc documentation](/rc/) for more info on the rc flags.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.45",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1, command, args)
 		if rcflags.Opt.Enabled {
diff --git a/cmd/reveal/reveal.go b/cmd/reveal/reveal.go
index e569a26b44b04..0839941bdccb9 100644
--- a/cmd/reveal/reveal.go
+++ b/cmd/reveal/reveal.go
@@ -16,6 +16,9 @@ func init() {
 var commandDefinition = &cobra.Command{
 	Use:   "reveal password",
 	Short: `Reveal obscured password from rclone.conf`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.43",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		cmd.Run(false, false, command, func() error {
diff --git a/cmd/rmdir/rmdir.go b/cmd/rmdir/rmdir.go
index 778315be83e0a..20fef83141baf 100644
--- a/cmd/rmdir/rmdir.go
+++ b/cmd/rmdir/rmdir.go
@@ -24,6 +24,9 @@ with option ` + "`--rmdirs`" + `) to do that.
 
 To delete a path and any objects in it, use [purge](/commands/rclone_purge/) command.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v0.90",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fdst := cmd.NewFsDir(args)
diff --git a/cmd/rmdirs/rmdirs.go b/cmd/rmdirs/rmdirs.go
index 11a09ca203bc2..62711dea40a25 100644
--- a/cmd/rmdirs/rmdirs.go
+++ b/cmd/rmdirs/rmdirs.go
@@ -38,6 +38,9 @@ used with option ` + "`--rmdirs`" + `).
 To delete a path and any objects in it, use [purge](/commands/rclone_purge/)
 command.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.35",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fdst := cmd.NewFsDir(args)
diff --git a/cmd/selfupdate/selfupdate.go b/cmd/selfupdate/selfupdate.go
index c7f1a3cdacdeb..d045fede0afc6 100644
--- a/cmd/selfupdate/selfupdate.go
+++ b/cmd/selfupdate/selfupdate.go
@@ -64,6 +64,9 @@ var cmdSelfUpdate = &cobra.Command{
 	Aliases: []string{"self-update"},
 	Short:   `Update the rclone binary.`,
 	Long:    strings.ReplaceAll(selfUpdateHelp, "|", "`"),
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.55",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 0, command, args)
 		if Opt.Package == "" {
diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go
index 579d695d8f5a4..2389ce638fda0 100644
--- a/cmd/serve/dlna/dlna.go
+++ b/cmd/serve/dlna/dlna.go
@@ -47,6 +47,9 @@ media transcoding support. This means that some players might show
 files that they are not able to play back correctly.
 
 ` + dlnaflags.Help + vfs.Help,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.46",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		f := cmd.NewFsSrc(args)
diff --git a/cmd/serve/docker/docker.go b/cmd/serve/docker/docker.go
index 5e49718016f0d..d160558651e2a 100644
--- a/cmd/serve/docker/docker.go
+++ b/cmd/serve/docker/docker.go
@@ -48,7 +48,9 @@ var Command = &cobra.Command{
 	Use:   "docker",
 	Short: `Serve any remote on docker's volume plugin API.`,
 	Long:  strings.ReplaceAll(longHelp, "|", "`") + vfs.Help,
-
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.56",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 0, command, args)
 		cmd.Run(false, false, command, func() error {
diff --git a/cmd/serve/ftp/ftp.go b/cmd/serve/ftp/ftp.go
index 2023ba8514e9d..28bd4325f5ed4 100644
--- a/cmd/serve/ftp/ftp.go
+++ b/cmd/serve/ftp/ftp.go
@@ -99,6 +99,9 @@ By default this will serve files without needing a login.
 
 You can set a single username and password with the --user and --pass flags.
 ` + vfs.Help + proxy.Help,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.44",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		var f fs.Fs
 		if proxyflags.Opt.AuthProxy == "" {
diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go
index cdf8afd2860b9..5db094dce0fd1 100644
--- a/cmd/serve/http/http.go
+++ b/cmd/serve/http/http.go
@@ -60,6 +60,9 @@ The server will log errors.  Use ` + "`-v`" + ` to see access logs.
 ` + "`--bwlimit`" + ` will be respected for file transfers.  Use ` + "`--stats`" + ` to
 control the stats printing.
 ` + httplib.Help + data.Help + auth.Help + vfs.Help,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		f := cmd.NewFsSrc(args)
diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go
index 7dfed0e1e1ed6..4aaad2a3a7cf4 100644
--- a/cmd/serve/restic/restic.go
+++ b/cmd/serve/restic/restic.go
@@ -128,6 +128,9 @@ these **must** end with /.  Eg
 The` + "`--private-repos`" + ` flag can be used to limit users to repositories starting
 with a path of ` + "`//`" + `.
 ` + httplib.Help,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.40",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		f := cmd.NewFsSrc(args)
diff --git a/cmd/serve/serve.go b/cmd/serve/serve.go
index f79162c821e56..8f523f52c3f03 100644
--- a/cmd/serve/serve.go
+++ b/cmd/serve/serve.go
@@ -49,6 +49,9 @@ subcommand to specify the protocol, e.g.
 
 Each subcommand has its own options which you can see in their help.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		if len(args) == 0 {
 			return errors.New("serve requires a protocol, e.g. 'rclone serve http remote:'")
diff --git a/cmd/serve/sftp/sftp.go b/cmd/serve/sftp/sftp.go
index 8244435ee2865..2a255224bee92 100644
--- a/cmd/serve/sftp/sftp.go
+++ b/cmd/serve/sftp/sftp.go
@@ -114,6 +114,9 @@ checksumming is possible but less secure and you could use the SFTP server
 provided by OpenSSH in this case.
 
 ` + vfs.Help + proxy.Help,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.48",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		var f fs.Fs
 		if proxyflags.Opt.AuthProxy == "" {
diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go
index bfd0eda31935d..5ab8810192d10 100644
--- a/cmd/serve/webdav/webdav.go
+++ b/cmd/serve/webdav/webdav.go
@@ -61,6 +61,9 @@ supported hash on the backend or you can use a named hash such as
 to see the full list.
 
 ` + httplib.Help + vfs.Help + proxy.Help,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		var f fs.Fs
 		if proxyflags.Opt.AuthProxy == "" {
diff --git a/cmd/settier/settier.go b/cmd/settier/settier.go
index 8a1d234f07e63..d918ebdd0e837 100644
--- a/cmd/settier/settier.go
+++ b/cmd/settier/settier.go
@@ -40,6 +40,9 @@ Or just provide remote directory and all files in directory will be tiered
 
     rclone settier tier remote:path/dir
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.44",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
 		tier := args[0]
diff --git a/cmd/sha1sum/sha1sum.go b/cmd/sha1sum/sha1sum.go
index 053ca0cbbf9e2..483832ec2a740 100644
--- a/cmd/sha1sum/sha1sum.go
+++ b/cmd/sha1sum/sha1sum.go
@@ -41,6 +41,9 @@ as a relative path).
 This command can also hash data received on STDIN, if not passing
 a remote:path.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.27",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(0, 1, command, args)
 		if found, err := hashsum.CreateFromStdinArg(hash.SHA1, args, 0); found {
diff --git a/cmd/size/size.go b/cmd/size/size.go
index c084de5278a84..8be1305292042 100644
--- a/cmd/size/size.go
+++ b/cmd/size/size.go
@@ -44,6 +44,9 @@ Rclone will then show a notice in the log indicating how many such
 files were encountered, and count them in as empty files in the output
 of the size command.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.23",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go
index 33a5b0027e5f2..e9c91f2c93635 100644
--- a/cmd/sync/sync.go
+++ b/cmd/sync/sync.go
@@ -60,6 +60,9 @@ destination that is inside the source directory.
 **Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
 See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v0.90",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
diff --git a/cmd/test/changenotify/changenotify.go b/cmd/test/changenotify/changenotify.go
index 70eab98f32d78..6b88e5f11b9dd 100644
--- a/cmd/test/changenotify/changenotify.go
+++ b/cmd/test/changenotify/changenotify.go
@@ -26,6 +26,9 @@ func init() {
 var commandDefinition = &cobra.Command{
 	Use:   "changenotify remote:",
 	Short: `Log any change notify requests for the remote passed in.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.56",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 1, command, args)
 		f := cmd.NewFsSrc(args)
diff --git a/cmd/test/histogram/histogram.go b/cmd/test/histogram/histogram.go
index a3a2865b8117a..f2044fa697e5d 100644
--- a/cmd/test/histogram/histogram.go
+++ b/cmd/test/histogram/histogram.go
@@ -28,6 +28,9 @@ in filenames in the remote:path specified.
 The data doesn't contain any identifying information but is useful for
 the rclone developers when developing filename compression.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.55",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		f := cmd.NewFsDir(args)
diff --git a/cmd/test/info/info.go b/cmd/test/info/info.go
index 89cc10a01283d..0715547965587 100644
--- a/cmd/test/info/info.go
+++ b/cmd/test/info/info.go
@@ -66,6 +66,9 @@ a bit of go code for each one.
 
 **NB** this can create undeletable files and other hazards - use with care
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.55",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1e6, command, args)
 		if !checkNormalization && !checkControl && !checkLength && !checkStreaming && !all {
diff --git a/cmd/test/makefiles/makefiles.go b/cmd/test/makefiles/makefiles.go
index b778accd0785d..7b0ac57a88920 100644
--- a/cmd/test/makefiles/makefiles.go
+++ b/cmd/test/makefiles/makefiles.go
@@ -74,6 +74,9 @@ func init() {
 var makefilesCmd = &cobra.Command{
 	Use:   "makefiles ",
 	Short: `Make a random file hierarchy in a directory`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.55",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		commonInit()
@@ -105,6 +108,9 @@ var makefilesCmd = &cobra.Command{
 var makefileCmd = &cobra.Command{
 	Use:   "makefile  []+ [flags]",
 	Short: `Make files with random contents of the size given`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.59",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1e6, command, args)
 		commonInit()
diff --git a/cmd/test/memory/memory.go b/cmd/test/memory/memory.go
index 38b7b16838213..4321108810778 100644
--- a/cmd/test/memory/memory.go
+++ b/cmd/test/memory/memory.go
@@ -20,6 +20,9 @@ func init() {
 var commandDefinition = &cobra.Command{
 	Use:   "memory remote:path",
 	Short: `Load all the objects at remote:path into memory and report memory stats.`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.55",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
diff --git a/cmd/test/test.go b/cmd/test/test.go
index 774bf6c3a507b..8888f85c66fc7 100644
--- a/cmd/test/test.go
+++ b/cmd/test/test.go
@@ -25,4 +25,7 @@ Each subcommand has its own options which you can see in their help.
 **NB** Be careful running these commands, they may do strange things
 so reading their documentation first is recommended.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.55",
+	},
 }
diff --git a/cmd/touch/touch.go b/cmd/touch/touch.go
index b0bea31b2c193..7ec4a098060a9 100644
--- a/cmd/touch/touch.go
+++ b/cmd/touch/touch.go
@@ -64,6 +64,9 @@ time instead of the current time. Times may be specified as one of:
 Note that value of ` + "`--timestamp`" + ` is in UTC. If you want local time
 then add the ` + "`--localtime`" + ` flag.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.39",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		f, remote := newFsDst(args)
diff --git a/cmd/tree/tree.go b/cmd/tree/tree.go
index 5705dcec84f41..ae776b0670ace 100644
--- a/cmd/tree/tree.go
+++ b/cmd/tree/tree.go
@@ -95,6 +95,9 @@ short options as they conflict with rclone's short options.
 For a more interactive navigation of the remote see the
 [ncdu](/commands/rclone_ncdu/) command.
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.38",
+	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
diff --git a/cmd/version/version.go b/cmd/version/version.go
index 32416f89a7d8e..5d35cb98ce956 100644
--- a/cmd/version/version.go
+++ b/cmd/version/version.go
@@ -67,6 +67,9 @@ Or
       upgrade: https://beta.rclone.org/v1.42-005-g56e1e820
 
 `,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.33",
+	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 0, command, args)
 		if check {

From 313493d51b390d7f73f0780d15bf31698f2a919a Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Mon, 28 Nov 2022 17:00:51 +0100
Subject: [PATCH 413/560] docs: remove minimum versions from command pages of
 pre v1 commands

---
 cmd/check/check.go | 3 ---
 cmd/copy/copy.go   | 3 ---
 cmd/help.go        | 3 ---
 cmd/ls/ls.go       | 3 ---
 cmd/lsd/lsd.go     | 3 ---
 cmd/mkdir/mkdir.go | 3 ---
 cmd/purge/purge.go | 3 ---
 cmd/rmdir/rmdir.go | 3 ---
 cmd/sync/sync.go   | 3 ---
 9 files changed, 27 deletions(-)

diff --git a/cmd/check/check.go b/cmd/check/check.go
index e4bcc156ad242..3c126e176275b 100644
--- a/cmd/check/check.go
+++ b/cmd/check/check.go
@@ -155,9 +155,6 @@ to check all the data.
 If you supply the |--checkfile HASH| flag with a valid hash name,
 the |source:path| must point to a text file in the SUM format.
 `, "|", "`") + FlagsHelp,
-	Annotations: map[string]string{
-		"versionIntroduced": "v0.90",
-	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(2, 2, command, args)
 		var (
diff --git a/cmd/copy/copy.go b/cmd/copy/copy.go
index 6a365b25213e0..c6e42a70ea9bd 100644
--- a/cmd/copy/copy.go
+++ b/cmd/copy/copy.go
@@ -83,9 +83,6 @@ recently very efficiently like this:
 
 **Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything.
 `, "|", "`"),
-	Annotations: map[string]string{
-		"versionIntroduced": "v0.90",
-	},
 	Run: func(command *cobra.Command, args []string) {
 
 		cmd.CheckArgs(2, 2, command, args)
diff --git a/cmd/help.go b/cmd/help.go
index 949abe45bb4f8..ecc34b63ea490 100644
--- a/cmd/help.go
+++ b/cmd/help.go
@@ -33,9 +33,6 @@ See the home page (https://rclone.org/) for installation, usage,
 documentation, changelog and configuration walkthroughs.
 
 `,
-	Annotations: map[string]string{
-		"versionIntroduced": "v0.90",
-	},
 	PersistentPostRun: func(cmd *cobra.Command, args []string) {
 		fs.Debugf("rclone", "Version %q finishing with parameters %q", fs.Version, os.Args)
 		atexit.Run()
diff --git a/cmd/ls/ls.go b/cmd/ls/ls.go
index cecab52a0ed56..334b12f8e8ecc 100644
--- a/cmd/ls/ls.go
+++ b/cmd/ls/ls.go
@@ -31,9 +31,6 @@ Eg
         37600 fubuwic
 
 ` + lshelp.Help,
-	Annotations: map[string]string{
-		"versionIntroduced": "v0.90",
-	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
diff --git a/cmd/lsd/lsd.go b/cmd/lsd/lsd.go
index d6681bdad6204..8d79266fb6985 100644
--- a/cmd/lsd/lsd.go
+++ b/cmd/lsd/lsd.go
@@ -49,9 +49,6 @@ Or
 If you just want the directory names use ` + "`rclone lsf --dirs-only`" + `.
 
 ` + lshelp.Help,
-	Annotations: map[string]string{
-		"versionIntroduced": "v0.90",
-	},
 	Run: func(command *cobra.Command, args []string) {
 		ci := fs.GetConfig(context.Background())
 		cmd.CheckArgs(1, 1, command, args)
diff --git a/cmd/mkdir/mkdir.go b/cmd/mkdir/mkdir.go
index 92922ffd20395..3c99180feae52 100644
--- a/cmd/mkdir/mkdir.go
+++ b/cmd/mkdir/mkdir.go
@@ -18,9 +18,6 @@ func init() {
 var commandDefinition = &cobra.Command{
 	Use:   "mkdir remote:path",
 	Short: `Make the path if it doesn't already exist.`,
-	Annotations: map[string]string{
-		"versionIntroduced": "v0.90",
-	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fdst := cmd.NewFsDir(args)
diff --git a/cmd/purge/purge.go b/cmd/purge/purge.go
index 59085f83ee56e..d41e22cf67ab5 100644
--- a/cmd/purge/purge.go
+++ b/cmd/purge/purge.go
@@ -26,9 +26,6 @@ delete files. To delete empty directories only, use command
 **Important**: Since this can cause data loss, test first with the
 ` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
 `,
-	Annotations: map[string]string{
-		"versionIntroduced": "v0.90",
-	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fdst := cmd.NewFsDir(args)
diff --git a/cmd/rmdir/rmdir.go b/cmd/rmdir/rmdir.go
index 20fef83141baf..778315be83e0a 100644
--- a/cmd/rmdir/rmdir.go
+++ b/cmd/rmdir/rmdir.go
@@ -24,9 +24,6 @@ with option ` + "`--rmdirs`" + `) to do that.
 
 To delete a path and any objects in it, use [purge](/commands/rclone_purge/) command.
 `,
-	Annotations: map[string]string{
-		"versionIntroduced": "v0.90",
-	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fdst := cmd.NewFsDir(args)
diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go
index e9c91f2c93635..33a5b0027e5f2 100644
--- a/cmd/sync/sync.go
+++ b/cmd/sync/sync.go
@@ -60,9 +60,6 @@ destination that is inside the source directory.
 **Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
 See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
 `,
-	Annotations: map[string]string{
-		"versionIntroduced": "v0.90",
-	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)

From 1628ca0d46b9ec313c425735ab85d11ec2695317 Mon Sep 17 00:00:00 2001
From: Anthony Pessy 
Date: Mon, 5 Dec 2022 17:19:04 +0100
Subject: [PATCH 414/560] ftp: Improve performance to speed up --files-from and
 NewObject

This commit uses the MLST command (where available) to get the status
for single files rather than listing the parent directory and looking
for the file. This makes actions such as using `--files-from` much quicker.

* use getEntry to lookup remote files when supported
*  findItem now expects the full path directly

It makes the expected argument similar to the getInfo method, the
difference now is that one is returning a FileInfo whereas
the other is returning an ftp Entry.

Fixes #6225

Co-authored-by: Nick Craig-Wood 
---
 backend/ftp/ftp.go | 64 ++++++++++++++++++++++++----------------------
 1 file changed, 34 insertions(+), 30 deletions(-)

diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go
index c68e7dc4a1a32..01d686c8e204f 100644
--- a/backend/ftp/ftp.go
+++ b/backend/ftp/ftp.go
@@ -657,8 +657,7 @@ func (f *Fs) dirFromStandardPath(dir string) string {
 // findItem finds a directory entry for the name in its parent directory
 func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	fullPath := path.Join(f.root, remote)
-	if fullPath == "" || fullPath == "." || fullPath == "/" {
+	if remote == "" || remote == "." || remote == "/" {
 		// if root, assume exists and synthesize an entry
 		return &ftp.Entry{
 			Name: "",
@@ -666,13 +665,32 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
 			Time: time.Now(),
 		}, nil
 	}
-	dir := path.Dir(fullPath)
-	base := path.Base(fullPath)
 
 	c, err := f.getFtpConnection(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("findItem: %w", err)
 	}
+
+	// returns TRUE if MLST is supported which is required to call GetEntry
+	if c.IsTimePreciseInList() {
+		entry, err := c.GetEntry(f.opt.Enc.FromStandardPath(remote))
+		f.putFtpConnection(&c, err)
+		if err != nil {
+			err = translateErrorFile(err)
+			if err == fs.ErrorObjectNotFound {
+				return nil, nil
+			}
+			return nil, err
+		}
+		if entry != nil {
+			f.entryToStandard(entry)
+		}
+		return entry, nil
+	}
+
+	dir := path.Dir(remote)
+	base := path.Base(remote)
+
 	files, err := c.List(f.dirFromStandardPath(dir))
 	f.putFtpConnection(&c, err)
 	if err != nil {
@@ -691,7 +709,7 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	entry, err := f.findItem(ctx, remote)
+	entry, err := f.findItem(ctx, path.Join(f.root, remote))
 	if err != nil {
 		return nil, err
 	}
@@ -713,7 +731,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err err
 
 // dirExists checks the directory pointed to by remote exists or not
 func (f *Fs) dirExists(ctx context.Context, remote string) (exists bool, err error) {
-	entry, err := f.findItem(ctx, remote)
+	entry, err := f.findItem(ctx, path.Join(f.root, remote))
 	if err != nil {
 		return false, fmt.Errorf("dirExists: %w", err)
 	}
@@ -857,32 +875,18 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 // getInfo reads the FileInfo for a path
 func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err error) {
 	// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
-	dir := path.Dir(remote)
-	base := path.Base(remote)
-
-	c, err := f.getFtpConnection(ctx)
+	file, err := f.findItem(ctx, remote)
 	if err != nil {
-		return nil, fmt.Errorf("getInfo: %w", err)
-	}
-	files, err := c.List(f.dirFromStandardPath(dir))
-	f.putFtpConnection(&c, err)
-	if err != nil {
-		return nil, translateErrorFile(err)
-	}
-
-	for i := range files {
-		file := files[i]
-		f.entryToStandard(file)
-		if file.Name == base {
-			info := &FileInfo{
-				Name:    remote,
-				Size:    file.Size,
-				ModTime: file.Time,
-				precise: f.fLstTime,
-				IsDir:   file.Type == ftp.EntryTypeFolder,
-			}
-			return info, nil
+		return nil, err
+	} else if file != nil {
+		info := &FileInfo{
+			Name:    remote,
+			Size:    file.Size,
+			ModTime: file.Time,
+			precise: f.fLstTime,
+			IsDir:   file.Type == ftp.EntryTypeFolder,
 		}
+		return info, nil
 	}
 	return nil, fs.ErrorObjectNotFound
 }

From a9bd0c8de6e74460ab990269e1f96834889304d2 Mon Sep 17 00:00:00 2001
From: Erik Agterdenbos 
Date: Tue, 6 Dec 2022 00:07:08 +0100
Subject: [PATCH 415/560] s3: reduce memory consumption for s3 objects

Copying the storageClass string instead of using a pointer to the original string.
This prevents the Go garbage collector from keeping large amounts of
XMLNode structs and references in memory, created by xmlutil.XMLToStruct()
from the aws-sdk-go.
---
 backend/s3/s3.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 9de4a7e6ae28d..5fad59ae54b92 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -3052,7 +3052,8 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje
 		}
 		o.setMD5FromEtag(aws.StringValue(info.ETag))
 		o.bytes = aws.Int64Value(info.Size)
-		o.storageClass = info.StorageClass
+		storageClass := *info.StorageClass // To prevent reference to large XML structures
+		o.storageClass = &storageClass
 		o.versionID = versionID
 	} else if !o.fs.opt.NoHeadObject {
 		err := o.readMetaData(ctx) // reads info and meta, returning an error

From c2dfc3e5b37572eeaef8d2f55fb6bfe3c6a8211e Mon Sep 17 00:00:00 2001
From: Kevin Verstaen <48050031+kverstae@users.noreply.github.com>
Date: Tue, 6 Dec 2022 13:07:06 +0100
Subject: [PATCH 416/560] fs: Add global flag '--color' to control terminal
 colors

* fs: add TerminalColorMode type
* fs: add new config(flags) for TerminalColorMode
* lib/terminal: use TerminalColorMode to determine how to handle colors
* Add documentation for '--terminal-color-mode'
* tree: remove obsolete --color replaced by global --color

This changes the default behaviour of tree. It now displays colors by
default instead of only displaying them when the flag -C/--color was
active. Old behaviour (no color) can be achieved by setting --color to
'never'.

Fixes: #6604
---
 cmd/tree/tree.go                     |  2 +-
 docs/content/commands/rclone_tree.md |  1 -
 docs/content/docs.md                 | 10 ++++
 docs/content/flags.md                |  1 +
 fs/config.go                         |  1 +
 fs/config/configflags/configflags.go |  1 +
 fs/terminalcolormode.go              | 57 +++++++++++++++++++++
 fs/terminalcolormode_test.go         | 74 ++++++++++++++++++++++++++++
 lib/terminal/terminal.go             | 14 +++++-
 9 files changed, 157 insertions(+), 4 deletions(-)
 create mode 100644 fs/terminalcolormode.go
 create mode 100644 fs/terminalcolormode_test.go

diff --git a/cmd/tree/tree.go b/cmd/tree/tree.go
index ae776b0670ace..ca2999aacc633 100644
--- a/cmd/tree/tree.go
+++ b/cmd/tree/tree.go
@@ -61,7 +61,6 @@ func init() {
 	flags.StringVarP(cmdFlags, &sort, "sort", "", "", "Select sort: name,version,size,mtime,ctime")
 	// Graphics
 	flags.BoolVarP(cmdFlags, &opts.NoIndent, "noindent", "", false, "Don't print indentation lines")
-	flags.BoolVarP(cmdFlags, &opts.Colorize, "color", "C", false, "Turn colorization on always")
 }
 
 var commandDefinition = &cobra.Command{
@@ -116,6 +115,7 @@ For a more interactive navigation of the remote see the
 		opts.SizeSort = sort == "size"
 		ci := fs.GetConfig(context.Background())
 		opts.UnitSize = ci.HumanReadable
+		opts.Colorize = ci.TerminalColorMode != fs.TerminalColorModeNever
 		if opts.DeepLevel == 0 {
 			opts.DeepLevel = ci.MaxDepth
 		}
diff --git a/docs/content/commands/rclone_tree.md b/docs/content/commands/rclone_tree.md
index 1995a108f5d9c..3ecc450e515b5 100644
--- a/docs/content/commands/rclone_tree.md
+++ b/docs/content/commands/rclone_tree.md
@@ -48,7 +48,6 @@ rclone tree remote:path [flags]
 
 ```
   -a, --all             All files are listed (list . files too)
-  -C, --color           Turn colorization on always
   -d, --dirs-only       List directories only
       --dirsfirst       List directories before files (-U disables)
       --full-path       Print the full path prefix for each file
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 4b58920a22294..1afa2a9a09026 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -823,6 +823,16 @@ quicker than without the `--checksum` flag.
 When using this flag, rclone won't update mtimes of remote files if
 they are incorrect as it would normally.
 
+### --color WHEN ###
+
+Specifiy when colors (and other ANSI codes) should be added to the output.
+
+`AUTO` (default) only allows ANSI codes when the output is a terminal
+
+`NEVER` never allow ANSI codes
+
+`ALWAYS` always add ANSI codes, regardless of the output format (terminal or file)
+
 ### --compare-dest=DIR ###
 
 When using `sync`, `copy` or `move` DIR is checked in addition to the
diff --git a/docs/content/flags.md b/docs/content/flags.md
index 1cdce0e1dd582..c94a9e90441b2 100644
--- a/docs/content/flags.md
+++ b/docs/content/flags.md
@@ -27,6 +27,7 @@ These flags are available for every command.
   -c, --checksum                             Skip based on checksum (if available) & size, not mod-time & size
       --client-cert string                   Client SSL certificate (PEM) for mutual TLS auth
       --client-key string                    Client SSL private key (PEM) for mutual TLS auth
+      --color                                Define when colors (and other ANSI codes) should be shown AUTO|ALWAYS|NEVER (default AUTO)
       --compare-dest stringArray             Include additional comma separated server-side paths during comparison
       --config string                        Config file (default "$HOME/.config/rclone/rclone.conf")
       --contimeout duration                  Connect timeout (default 1m0s)
diff --git a/fs/config.go b/fs/config.go
index 13d5235589765..d496b4d88f8d6 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -142,6 +142,7 @@ type ConfigInfo struct {
 	DisableHTTPKeepAlives   bool
 	Metadata                bool
 	ServerSideAcrossConfigs bool
+	TerminalColorMode       TerminalColorMode
 }
 
 // NewConfig creates a new config with everything set to the default
diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go
index 42b2b4d05ec75..abd5209d79c17 100644
--- a/fs/config/configflags/configflags.go
+++ b/fs/config/configflags/configflags.go
@@ -142,6 +142,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.")
 	flags.BoolVarP(flagSet, &ci.Metadata, "metadata", "M", ci.Metadata, "If set, preserve metadata when copying objects")
 	flags.BoolVarP(flagSet, &ci.ServerSideAcrossConfigs, "server-side-across-configs", "", ci.ServerSideAcrossConfigs, "Allow server-side operations (e.g. copy) to work across different configs")
+	flags.FVarP(flagSet, &ci.TerminalColorMode, "color", "", "When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS")
 }
 
 // ParseHeaders converts the strings passed in via the header flags into HTTPOptions
diff --git a/fs/terminalcolormode.go b/fs/terminalcolormode.go
new file mode 100644
index 0000000000000..7da58e13fce6b
--- /dev/null
+++ b/fs/terminalcolormode.go
@@ -0,0 +1,57 @@
+package fs
+
+import (
+	"fmt"
+	"strings"
+)
+
+// TerminalColorMode describes how ANSI codes should be handled
+type TerminalColorMode byte
+
+// TerminalColorMode constants
+const (
+	TerminalColorModeAuto TerminalColorMode = iota
+	TerminalColorModeNever
+	TerminalColorModeAlways
+)
+
+var terminalColorModeToString = []string{
+	TerminalColorModeAuto:   "AUTO",
+	TerminalColorModeNever:  "NEVER",
+	TerminalColorModeAlways: "ALWAYS",
+}
+
+// String converts a TerminalColorMode to a string
+func (m TerminalColorMode) String() string {
+	if m >= TerminalColorMode(len(terminalColorModeToString)) {
+		return fmt.Sprintf("TerminalColorMode(%d)", m)
+	}
+	return terminalColorModeToString[m]
+}
+
+// Set a TerminalColorMode
+func (m *TerminalColorMode) Set(s string) error {
+	for n, name := range terminalColorModeToString {
+		if s != "" && name == strings.ToUpper(s) {
+			*m = TerminalColorMode(n)
+			return nil
+		}
+	}
+	return fmt.Errorf("unknown terminal color mode %q", s)
+}
+
+// Type of TerminalColorMode
+func (m TerminalColorMode) Type() string {
+	return "string"
+}
+
+// UnmarshalJSON converts a string/integer in JSON to a TerminalColorMode
+func (m *TerminalColorMode) UnmarshalJSON(in []byte) error {
+	return UnmarshalJSONFlag(in, m, func(i int64) error {
+		if i < 0 || i >= int64(len(terminalColorModeToString)) {
+			return fmt.Errorf("out of range terminal color mode %d", i)
+		}
+		*m = (TerminalColorMode)(i)
+		return nil
+	})
+}
diff --git a/fs/terminalcolormode_test.go b/fs/terminalcolormode_test.go
new file mode 100644
index 0000000000000..6060b4342ff9a
--- /dev/null
+++ b/fs/terminalcolormode_test.go
@@ -0,0 +1,74 @@
+package fs
+
+import (
+	"encoding/json"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTerminalColorModeString(t *testing.T) {
+	for _, test := range []struct {
+		in   TerminalColorMode
+		want string
+	}{
+		{TerminalColorModeAuto, "AUTO"},
+		{TerminalColorModeAlways, "ALWAYS"},
+		{TerminalColorModeNever, "NEVER"},
+		{36, "TerminalColorMode(36)"},
+	} {
+		tcm := test.in
+		assert.Equal(t, test.want, tcm.String(), test.in)
+	}
+}
+
+func TestTerminalColorModeSet(t *testing.T) {
+	for _, test := range []struct {
+		in          string
+		want        TerminalColorMode
+		expectError bool
+	}{
+		{"auto", TerminalColorModeAuto, false},
+		{"ALWAYS", TerminalColorModeAlways, false},
+		{"Never", TerminalColorModeNever, false},
+		{"INVALID", 0, true},
+	} {
+		tcm := TerminalColorMode(0)
+		err := tcm.Set(test.in)
+		if test.expectError {
+			require.Error(t, err, test.in)
+		} else {
+			require.NoError(t, err, test.in)
+		}
+		assert.Equal(t, test.want, tcm, test.in)
+	}
+}
+
+func TestTerminalColorModeUnmarshalJSON(t *testing.T) {
+	for _, test := range []struct {
+		in          string
+		want        TerminalColorMode
+		expectError bool
+	}{
+		{`"auto"`, TerminalColorModeAuto, false},
+		{`"ALWAYS"`, TerminalColorModeAlways, false},
+		{`"Never"`, TerminalColorModeNever, false},
+		{`"Invalid"`, 0, true},
+		{strconv.Itoa(int(TerminalColorModeAuto)), TerminalColorModeAuto, false},
+		{strconv.Itoa(int(TerminalColorModeAlways)), TerminalColorModeAlways, false},
+		{strconv.Itoa(int(TerminalColorModeNever)), TerminalColorModeNever, false},
+		{`99`, 0, true},
+		{`-99`, 0, true},
+	} {
+		var tcm TerminalColorMode
+		err := json.Unmarshal([]byte(test.in), &tcm)
+		if test.expectError {
+			require.Error(t, err, test.in)
+		} else {
+			require.NoError(t, err, test.in)
+		}
+		assert.Equal(t, test.want, tcm, test.in)
+	}
+}
diff --git a/lib/terminal/terminal.go b/lib/terminal/terminal.go
index 4eda1ababaa9c..532cf1fcc33cf 100644
--- a/lib/terminal/terminal.go
+++ b/lib/terminal/terminal.go
@@ -3,12 +3,14 @@
 package terminal
 
 import (
+	"context"
 	"io"
 	"os"
 	"runtime"
 	"sync"
 
 	colorable "github.com/mattn/go-colorable"
+	"github.com/rclone/rclone/fs"
 )
 
 // VT100 codes
@@ -73,13 +75,21 @@ var (
 // Start the terminal - must be called before use
 func Start() {
 	once.Do(func() {
+		ci := fs.GetConfig(context.Background())
+
 		f := os.Stdout
 		if !IsTerminal(int(f.Fd())) {
-			// If stdout not a tty then remove escape codes
-			Out = colorable.NewNonColorable(f)
+			// If stdout is not a tty, remove escape codes EXCEPT if terminal color mode equals "ALWAYS"
+			if ci.TerminalColorMode == fs.TerminalColorModeAlways {
+				Out = colorable.NewColorable(f)
+			} else {
+				Out = colorable.NewNonColorable(f)
+			}
 		} else if runtime.GOOS == "windows" && os.Getenv("TERM") != "" {
 			// If TERM is set just use stdout
 			Out = f
+		} else if ci.TerminalColorMode == fs.TerminalColorModeNever {
+			Out = colorable.NewNonColorable(f)
 		} else {
 			Out = colorable.NewColorable(f)
 		}

From 483e9e1ee3c4b36cd0a57a98d8c4a51b41323eb7 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 6 Dec 2022 12:10:07 +0000
Subject: [PATCH 417/560] Add ycdtosa to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 5209d136256e2..ec570d1b6f2d4 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -665,3 +665,4 @@ put them back in again.` >}}
   * Kamui 
   * asdffdsazqqq <90116442+asdffdsazqqq@users.noreply.github.com>
   * Nathaniel Wesley Filardo 
+  * ycdtosa 

From 4583b61e3d1e0653b30f7205813063465ede5d69 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 6 Dec 2022 12:10:07 +0000
Subject: [PATCH 418/560] Add Erik Agterdenbos to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index ec570d1b6f2d4..ccaff21bbc2ec 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -666,3 +666,4 @@ put them back in again.` >}}
   * asdffdsazqqq <90116442+asdffdsazqqq@users.noreply.github.com>
   * Nathaniel Wesley Filardo 
   * ycdtosa 
+  * Erik Agterdenbos 

From 10aee3926a3e6357ed6dfa2f1e9a61622222f8bb Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 6 Dec 2022 12:11:10 +0000
Subject: [PATCH 419/560] Add Kevin Verstaen to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index ccaff21bbc2ec..eb0253776e67e 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -667,3 +667,4 @@ put them back in again.` >}}
   * Nathaniel Wesley Filardo 
   * ycdtosa 
   * Erik Agterdenbos 
+  * Kevin Verstaen <48050031+kverstae@users.noreply.github.com>

From ab849b3613fe4935379f00f8309958f36ee36d5c Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 5 Dec 2022 16:59:48 +0000
Subject: [PATCH 420/560] s3: fix listing loop when using v2 listing on v1
 server

Before this change, rclone would enter a listing loop if it used v2
listing on a v1 server and the list exceeded 1000 items.

This change detects the problem and gives the user a helpful message.

Fixes #6600
---
 backend/s3/s3.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 5fad59ae54b92..568805a6c6e5b 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -3195,6 +3195,9 @@ func (ls *v2List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versi
 	if err != nil {
 		return nil, nil, err
 	}
+	if aws.BoolValue(resp.IsTruncated) && (resp.ContinuationToken == nil || *resp.ContinuationToken == "") {
+		return nil, nil, errors.New("s3 protocol error: received listing v2 with IsTruncated set and no ContinuationToken. Should you be using `--s3-list-version 1`?")
+	}
 	ls.req.ContinuationToken = resp.NextContinuationToken
 	return resp, nil, nil
 }

From 4f386a1ccd4a773ffa09d9ad3e729c4b36520ccc Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 5 Dec 2022 17:06:59 +0000
Subject: [PATCH 421/560] s3: turn off list v2 support for Alibaba OSS since it
 does not work

See: #6600
---
 backend/s3/s3.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 568805a6c6e5b..0cd82bb12ff5c 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -2730,6 +2730,7 @@ func setQuirks(opt *Options) {
 		mightGzip = false // Never auto gzips objects
 	case "Alibaba":
 		useMultipartEtag = false // Alibaba seems to calculate multipart Etags differently from AWS
+		listObjectsV2 = false    // See #6600
 	case "HuaweiOBS":
 		// Huawei OBS PFS is not support listObjectV2, and if turn on the urlEncodeListing, marker will not work and keep list same page forever.
 		urlEncodeListings = false

From b24c83db211c31a70d1bf8034aba04b686400e00 Mon Sep 17 00:00:00 2001
From: asdffdsazqqq <90116442+asdffdsazqqq@users.noreply.github.com>
Date: Tue, 6 Dec 2022 07:14:25 -0500
Subject: [PATCH 422/560] restic: fix typo in docs 'remove' should be 'remote'

---
 cmd/serve/restic/restic.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go
index 4aaad2a3a7cf4..3662fc58cfe3b 100644
--- a/cmd/serve/restic/restic.go
+++ b/cmd/serve/restic/restic.go
@@ -46,7 +46,7 @@ func init() {
 var Command = &cobra.Command{
 	Use:   "restic remote:path",
 	Short: `Serve the remote for restic's REST API.`,
-	Long: `Run a basic web server to serve a remove over restic's REST backend
+	Long: `Run a basic web server to serve a remote over restic's REST backend
 API over HTTP.  This allows restic to use rclone as a data storage
 mechanism for cloud providers that restic does not support directly.
 

From 0a8b1fe5dee556c375af82d1eff264e4e3b25da5 Mon Sep 17 00:00:00 2001
From: MohammadReza 
Date: Tue, 6 Dec 2022 15:55:23 +0330
Subject: [PATCH 423/560] s3: add Liara LOS to provider list

---
 README.md              |   1 +
 backend/s3/s3.go       |  37 +++++++++++--
 docs/content/_index.md |   1 +
 docs/content/s3.md     | 118 ++++++++++++++++++++++++++++++++++++++---
 4 files changed, 145 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index fc8064ac7260a..9c47e6b770b76 100644
--- a/README.md
+++ b/README.md
@@ -50,6 +50,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
   * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
+  * Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 0cd82bb12ff5c..8b0b19ee3e925 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -66,7 +66,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
+		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
@@ -124,6 +124,9 @@ func init() {
 			}, {
 				Value: "LyveCloud",
 				Help:  "Seagate Lyve Cloud",
+			}, {
+				Value: "Liara",
+				Help:  "Liara Object Storage",
 			}, {
 				Value: "Minio",
 				Help:  "Minio Object Storage",
@@ -437,7 +440,7 @@ func init() {
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive",
+			Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Liara,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -762,6 +765,15 @@ func init() {
 				Value: "s3-eu-south-2.ionoscloud.com",
 				Help:  "Logrono, Spain",
 			}},
+		}, {
+			// Liara endpoints: https://liara.ir/landing/object-storage
+			Name:     "endpoint",
+			Help:     "Endpoint for Liara Object Storage API.",
+			Provider: "Liara",
+			Examples: []fs.OptionExample{{
+				Value: "storage.iran.liara.space",
+				Help:  "The default endpoint\nIran",
+			}},
 		}, {
 			// oss endpoints: https://help.aliyun.com/document_detail/31837.html
 			Name:     "endpoint",
@@ -1092,7 +1104,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu",
+			Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -1177,6 +1189,10 @@ func init() {
 				Value:    "s3.ap-southeast-2.wasabisys.com",
 				Help:     "Wasabi AP Southeast 2 (Sydney)",
 				Provider: "Wasabi",
+			}, {
+				Value:    "storage.iran.liara.space",
+				Help:     "Liara Iran endpoint",
+				Provider: "Liara",
 			}, {
 				Value:    "s3.ir-thr-at1.arvanstorage.com",
 				Help:     "ArvanCloud Tehran Iran (Asiatech) endpoint",
@@ -1560,7 +1576,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
+			Provider: "!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Liara,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1793,6 +1809,15 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 				Value: "STANDARD_IA",
 				Help:  "Infrequent access storage mode",
 			}},
+		}, {
+			// Mapping from here: https://liara.ir/landing/object-storage
+			Name:     "storage_class",
+			Help:     "The storage class to use when storing new objects in Liara",
+			Provider: "Liara",
+			Examples: []fs.OptionExample{{
+				Value: "STANDARD",
+				Help:  "Standard storage class",
+			}},
 		}, {
 			// Mapping from here: https://www.arvancloud.com/en/products/cloud-storage
 			Name:     "storage_class",
@@ -2765,6 +2790,10 @@ func setQuirks(opt *Options) {
 		// listObjectsV2 supported - https://api.ionos.com/docs/s3/#Basic-Operations-get-Bucket-list-type-2
 		virtualHostStyle = false
 		urlEncodeListings = false
+	case "Liara":
+		virtualHostStyle = false
+		urlEncodeListings = false
+		useMultipartEtag = false
 	case "LyveCloud":
 		useMultipartEtag = false // LyveCloud seems to calculate multipart Etags differently from AWS
 	case "Minio":
diff --git a/docs/content/_index.md b/docs/content/_index.md
index 744a42b319589..2940081d887b3 100644
--- a/docs/content/_index.md
+++ b/docs/content/_index.md
@@ -135,6 +135,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="IDrive e2" home="https://www.idrive.com/e2/" config="/s3/#idrive-e2" >}}
 {{< provider name="IONOS Cloud" home="https://cloud.ionos.com/storage/object-storage" config="/s3/#ionos" >}}
 {{< provider name="Koofr" home="https://koofr.eu/" config="/koofr/" >}}
+{{< provider name="Liara Object Storage" home="https://liara.ir/landing/object-storage" config="/s3/#liara-object-storage" >}}
 {{< provider name="Mail.ru Cloud" home="https://cloud.mail.ru/" config="/mailru/" >}}
 {{< provider name="Memset Memstore" home="https://www.memset.com/cloud/storage/" config="/swift/" >}}
 {{< provider name="Mega" home="https://mega.nz/" config="/mega/" >}}
diff --git a/docs/content/s3.md b/docs/content/s3.md
index a5d8bf0e8cefe..f58f67998aafd 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -21,6 +21,7 @@ The S3 backend can be used with a number of different providers:
 {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
 {{< provider name="IDrive e2" home="https://www.idrive.com/e2/" config="/s3/#idrive-e2" >}}
 {{< provider name="IONOS Cloud" home="https://cloud.ionos.com/storage/object-storage" config="/s3/#ionos" >}}
+{{< provider name="Liara Object Storage" home="https://liara.ir/landing/object-storage" config="/s3/#liara-cloud" >}}
 {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}}
 {{< provider name="Qiniu Cloud Object Storage (Kodo)" home="https://www.qiniu.com/en/products/kodo" config="/s3/#qiniu" >}}
 {{< provider name="RackCorp Object Storage" home="https://www.rackcorp.com/" config="/s3/#RackCorp" >}}
@@ -77,7 +78,7 @@ name> remote
 Type of storage to configure.
 Choose a number from below, or type in your own value
 [snip]
-XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS
+XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Liara, Minio, and Tencent COS
    \ "s3"
 [snip]
 Storage> s3
@@ -3488,7 +3489,7 @@ Choose a number from below, or type in your own value
    	\ "alias"
  	2 / Amazon Drive
    	\ "amazon cloud drive"
- 	3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS)
+ 	3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, Liara, ArvanCloud, Minio, IBM COS)
    	\ "s3"
  	4 / Backblaze B2
    	\ "b2"
@@ -4007,7 +4008,7 @@ Choose a number from below, or type in your own value
    \ (alias)
  4 / Amazon Drive
    \ (amazon cloud drive)
- 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi
    \ (s3)
 [snip]
 Storage> s3
@@ -4247,7 +4248,7 @@ Choose `s3` backend
 Type of storage to configure.
 Choose a number from below, or type in your own value.
 [snip]
-XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Liara, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
    \ (s3)
 [snip]
 Storage> s3
@@ -4434,7 +4435,7 @@ name> wasabi
 Type of storage to configure.
 Choose a number from below, or type in your own value
 [snip]
-XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio)
+XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, Liara)
    \ "s3"
 [snip]
 Storage> s3
@@ -4548,7 +4549,7 @@ Type of storage to configure.
 Enter a string value. Press Enter for the default ("").
 Choose a number from below, or type in your own value
 [snip]
- 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS
+ 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS
    \ "s3"
 [snip]
 Storage> s3
@@ -4888,6 +4889,107 @@ d) Delete this remote
 y/e/d> y
 ```
 
+### Liara {#liara-cloud}
+
+Here is an example of making a [Liara Object Storage](https://liara.ir/landing/object-storage)
+configuration.  First run:
+
+    rclone config
+
+This will guide you through an interactive setup process.
+
+```
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+n/s> n
+name> Liara
+Type of storage to configure.
+Choose a number from below, or type in your own value
+[snip]
+XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Liara, Minio)
+   \ "s3"
+[snip]
+Storage> s3
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+AWS Access Key ID - leave blank for anonymous access or runtime credentials.
+access_key_id> YOURACCESSKEY
+AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
+secret_access_key> YOURSECRETACCESSKEY
+Region to connect to.
+Choose a number from below, or type in your own value
+   / The default endpoint
+ 1 | US Region, Northern Virginia, or Pacific Northwest.
+   | Leave location constraint empty.
+   \ "us-east-1"
+[snip]
+region>
+Endpoint for S3 API.
+Leave blank if using Liara to use the default endpoint for the region.
+Specify if using an S3 clone such as Ceph.
+endpoint> storage.iran.liara.space
+Canned ACL used when creating buckets and/or storing objects in S3.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Choose a number from below, or type in your own value
+ 1 / Owner gets FULL_CONTROL. No one else has access rights (default).
+   \ "private"
+[snip]
+acl>
+The server-side encryption algorithm used when storing this object in S3.
+Choose a number from below, or type in your own value
+ 1 / None
+   \ ""
+ 2 / AES256
+   \ "AES256"
+server_side_encryption>
+The storage class to use when storing objects in S3.
+Choose a number from below, or type in your own value
+ 1 / Default
+   \ ""
+ 2 / Standard storage class
+   \ "STANDARD"
+storage_class>
+Remote config
+--------------------
+[Liara]
+env_auth = false
+access_key_id = YOURACCESSKEY
+secret_access_key = YOURSECRETACCESSKEY
+endpoint = storage.iran.liara.space
+location_constraint =
+acl =
+server_side_encryption =
+storage_class =
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+This will leave the config file looking like this.
+
+```
+[Liara]
+type = s3
+provider = Liara
+env_auth = false
+access_key_id = YOURACCESSKEY
+secret_access_key = YOURSECRETACCESSKEY
+region =
+endpoint = storage.iran.liara.space
+location_constraint =
+acl =
+server_side_encryption =
+storage_class =
+```
+
 ### ArvanCloud {#arvan-cloud}
 
 [ArvanCloud](https://www.arvancloud.com/en/products/cloud-storage) ArvanCloud Object Storage goes beyond the limited traditional file storage. 
@@ -4906,7 +5008,7 @@ name> ArvanCloud
 Type of storage to configure.
 Choose a number from below, or type in your own value
 [snip]
-XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio)
+XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Liara, Minio)
    \ "s3"
 [snip]
 Storage> s3
@@ -5029,7 +5131,7 @@ Choose a number from below, or type in your own value
    \ "alias"
  3 / Amazon Drive
    \ "amazon cloud drive"
- 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS
+ 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS
    \ "s3"
 [snip]
 Storage> s3

From 60e4cb6f6f0f46c8adce32c31f8710d747066ed9 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 6 Dec 2022 15:06:51 +0000
Subject: [PATCH 424/560] Add MohammadReza to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index eb0253776e67e..db38dc5d62059 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -668,3 +668,4 @@ put them back in again.` >}}
   * ycdtosa 
   * Erik Agterdenbos 
   * Kevin Verstaen <48050031+kverstae@users.noreply.github.com>
+  * MohammadReza 

From a131da2c3542217f13b07721d8b7ea5801b71ddc Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Wed, 23 Nov 2022 16:46:21 +0000
Subject: [PATCH 425/560] azureblob: Port to new SDK

This commit switches from using the old Azure go modules

    github.com/Azure/azure-pipeline-go/pipeline
    github.com/Azure/azure-storage-blob-go/azblob
    github.com/Azure/go-autorest/autorest/adal

To the new SDK

    github.com/Azure/azure-sdk-for-go/

This stops rclone using deprecated code and enables the full range of
authentication with Azure.

See #6132 and #5284
---
 backend/azureblob/azureblob.go               | 1368 ++++++++++++------
 backend/azureblob/azureblob_internal_test.go |    4 +-
 backend/azureblob/azureblob_test.go          |   54 +-
 backend/azureblob/azureblob_unsupported.go   |    4 +-
 backend/azureblob/imds.go                    |  136 --
 backend/azureblob/imds_test.go               |  118 --
 go.mod                                       |   18 +-
 go.sum                                       |   62 +-
 8 files changed, 956 insertions(+), 808 deletions(-)
 delete mode 100644 backend/azureblob/imds.go
 delete mode 100644 backend/azureblob/imds_test.go

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 728a7405ed1eb..7268dfe231bd6 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -1,30 +1,61 @@
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
+//go:build !plan9 && !solaris && !js && go1.18
+// +build !plan9,!solaris,!js,go1.18
 
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob
 
+/* FIXME
+
+Note these Azure SDK bugs which are affecting the backend
+
+azblob UploadStream produces panic: send on closed channel if input stream has error #19612
+https://github.com/Azure/azure-sdk-for-go/issues/19612
+   - FIXED by re-implementing UploadStream
+
+azblob: when using SharedKey credentials, can't reference some blob names with ? in #19613
+https://github.com/Azure/azure-sdk-for-go/issues/19613
+   - FIXED by url encoding getBlobSVC and getBlockBlobSVC
+
+Azure Blob Storage paths are not URL-escaped #19475
+https://github.com/Azure/azure-sdk-for-go/issues/19475
+   - FIXED by url encoding getBlobSVC and getBlockBlobSVC
+
+Controlling TransferManager #19579
+https://github.com/Azure/azure-sdk-for-go/issues/19579
+   - FIXED by re-implementing UploadStream
+
+azblob: blob.StartCopyFromURL doesn't work with UTF-8 characters in the source blob #19614
+https://github.com/Azure/azure-sdk-for-go/issues/19614
+   - FIXED by url encoding getBlobSVC and getBlockBlobSVC
+
+*/
+
 import (
+	"bytes"
 	"context"
+	"crypto/md5"
 	"encoding/base64"
 	"encoding/hex"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"os"
 	"path"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/Azure/azure-pipeline-go/pipeline"
-	"github.com/Azure/azure-storage-blob-go/azblob"
-	"github.com/Azure/go-autorest/autorest/adal"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
@@ -35,24 +66,23 @@ import (
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/pool"
+	"github.com/rclone/rclone/lib/readers"
+	"golang.org/x/sync/errgroup"
 )
 
 const (
 	minSleep              = 10 * time.Millisecond
 	maxSleep              = 10 * time.Second
-	decayConstant         = 1     // bigger for slower decay, exponential
-	maxListChunkSize      = 5000  // number of items to read at once
-	maxUploadParts        = 50000 // maximum allowed number of parts/blocks in a multi-part upload
+	decayConstant         = 1    // bigger for slower decay, exponential
+	maxListChunkSize      = 5000 // number of items to read at once
 	modTimeKey            = "mtime"
 	timeFormatIn          = time.RFC3339
 	timeFormatOut         = "2006-01-02T15:04:05.000000000Z07:00"
 	storageDefaultBaseURL = "blob.core.windows.net"
 	defaultChunkSize      = 4 * fs.Mebi
-	defaultAccessTier     = azblob.AccessTierNone
-	maxTryTimeout         = time.Hour * 24 * 365 //max time of an azure web request response window (whether or not data is flowing)
+	defaultAccessTier     = blob.AccessTier("") // FIXME AccessTierNone
 	// Default storage account, key and blob endpoint for emulator support,
 	// though it is a base64 key checked in here, it is publicly available secret.
 	emulatorAccount      = "devstoreaccount1"
@@ -251,16 +281,16 @@ This option controls how often unused buffers will be removed from the pool.`,
 		}, {
 			Name:    "public_access",
 			Help:    "Public access level of a container: blob or container.",
-			Default: string(azblob.PublicAccessNone),
+			Default: "",
 			Examples: []fs.OptionExample{
 				{
-					Value: string(azblob.PublicAccessNone),
+					Value: "",
 					Help:  "The container and its blobs can be accessed only with an authorized request.\nIt's a default value.",
 				}, {
-					Value: string(azblob.PublicAccessBlob),
+					Value: string(container.PublicAccessTypeBlob),
 					Help:  "Blob data within this container can be read via anonymous request.",
 				}, {
-					Value: string(azblob.PublicAccessContainer),
+					Value: string(container.PublicAccessTypeContainer),
 					Help:  "Allow full public read access for container and blob data.",
 				},
 			},
@@ -278,8 +308,8 @@ This option controls how often unused buffers will be removed from the pool.`,
 type Options struct {
 	Account              string               `config:"account"`
 	ServicePrincipalFile string               `config:"service_principal_file"`
-	Key                  string               `config:"key"`
 	EnvAuth              bool                 `config:"env_auth"`
+	Key                  string               `config:"key"`
 	UseMSI               bool                 `config:"use_msi"`
 	MSIObjectID          string               `config:"msi_object_id"`
 	MSIClientID          string               `config:"msi_client_id"`
@@ -302,36 +332,35 @@ type Options struct {
 
 // Fs represents a remote azure server
 type Fs struct {
-	name          string                          // name of this remote
-	root          string                          // the path we are working on if any
-	opt           Options                         // parsed config options
-	ci            *fs.ConfigInfo                  // global config
-	features      *fs.Features                    // optional features
-	client        *http.Client                    // http client we are using
-	svcURL        *azblob.ServiceURL              // reference to serviceURL
-	cntURLcacheMu sync.Mutex                      // mutex to protect cntURLcache
-	cntURLcache   map[string]*azblob.ContainerURL // reference to containerURL per container
-	rootContainer string                          // container part of root (if any)
-	rootDirectory string                          // directory part of root (if any)
-	isLimited     bool                            // if limited to one container
-	cache         *bucket.Cache                   // cache for container creation status
-	pacer         *fs.Pacer                       // To pace and retry the API calls
-	imdsPacer     *fs.Pacer                       // Same but for IMDS
-	uploadToken   *pacer.TokenDispenser           // control concurrency
-	pool          *pool.Pool                      // memory pool
-	publicAccess  azblob.PublicAccessType         // Container Public Access Level
+	name          string                       // name of this remote
+	root          string                       // the path we are working on if any
+	opt           Options                      // parsed config options
+	ci            *fs.ConfigInfo               // global config
+	features      *fs.Features                 // optional features
+	cntSVCcacheMu sync.Mutex                   // mutex to protect cntSVCcache
+	cntSVCcache   map[string]*container.Client // reference to containerClient per container
+	svc           *service.Client              // client to access azblob
+	rootContainer string                       // container part of root (if any)
+	rootDirectory string                       // directory part of root (if any)
+	isLimited     bool                         // if limited to one container
+	cache         *bucket.Cache                // cache for container creation status
+	pacer         *fs.Pacer                    // To pace and retry the API calls
+	uploadToken   *pacer.TokenDispenser        // control concurrency
+	pool          *pool.Pool                   // memory pool
+	poolSize      int64                        // size of pages in memory pool
+	publicAccess  container.PublicAccessType   // Container Public Access Level
 }
 
 // Object describes an azure object
 type Object struct {
-	fs         *Fs                   // what this object is part of
-	remote     string                // The remote path
-	modTime    time.Time             // The modified time of the object if known
-	md5        string                // MD5 hash if known
-	size       int64                 // Size of the object
-	mimeType   string                // Content-Type of the object
-	accessTier azblob.AccessTierType // Blob Access Tier
-	meta       map[string]string     // blob metadata
+	fs         *Fs               // what this object is part of
+	remote     string            // The remote path
+	modTime    time.Time         // The modified time of the object if known
+	md5        string            // MD5 hash if known
+	size       int64             // Size of the object
+	mimeType   string            // Content-Type of the object
+	accessTier blob.AccessTier   // Blob Access Tier
+	meta       map[string]string // blob metadata
 }
 
 // ------------------------------------------------------------
@@ -382,17 +411,17 @@ func (o *Object) split() (container, containerPath string) {
 
 // validateAccessTier checks if azureblob supports user supplied tier
 func validateAccessTier(tier string) bool {
-	return strings.EqualFold(tier, string(azblob.AccessTierHot)) ||
-		strings.EqualFold(tier, string(azblob.AccessTierCool)) ||
-		strings.EqualFold(tier, string(azblob.AccessTierArchive))
+	return strings.EqualFold(tier, string(blob.AccessTierHot)) ||
+		strings.EqualFold(tier, string(blob.AccessTierCool)) ||
+		strings.EqualFold(tier, string(blob.AccessTierArchive))
 }
 
 // validatePublicAccess checks if azureblob supports use supplied public access level
 func validatePublicAccess(publicAccess string) bool {
 	switch publicAccess {
-	case string(azblob.PublicAccessNone),
-		string(azblob.PublicAccessBlob),
-		string(azblob.PublicAccessContainer):
+	case "",
+		string(container.PublicAccessTypeBlob),
+		string(container.PublicAccessTypeContainer):
 		// valid cases
 		return true
 	default:
@@ -417,21 +446,19 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 		return false, err
 	}
 	// FIXME interpret special errors - more to do here
-	if storageErr, ok := err.(azblob.StorageError); ok {
-		switch storageErr.ServiceCode() {
+	if storageErr, ok := err.(*azcore.ResponseError); ok {
+		switch storageErr.ErrorCode {
 		case "InvalidBlobOrBlock":
 			// These errors happen sometimes in multipart uploads
 			// because of block concurrency issues
 			return true, err
 		}
-		statusCode := storageErr.Response().StatusCode
+		statusCode := storageErr.StatusCode
 		for _, e := range retryErrorCodes {
 			if statusCode == e {
 				return true, err
 			}
 		}
-	} else if httpErr, ok := err.(httpError); ok {
-		return fserrors.ShouldRetryHTTP(httpErr.Response, retryErrorCodes), err
 	}
 	return fserrors.ShouldRetry(err), err
 }
@@ -452,85 +479,33 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	return
 }
 
-// httpClientFactory creates a Factory object that sends HTTP requests
-// to an rclone's http.Client.
-//
-// copied from azblob.newDefaultHTTPClientFactory
-func httpClientFactory(client *http.Client) pipeline.Factory {
-	return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
-		return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
-			r, err := client.Do(request.WithContext(ctx))
-			if err != nil {
-				err = pipeline.NewError(err, "HTTP request failed")
-			}
-			return pipeline.NewHTTPResponse(r), err
-		}
-	})
-}
-
 type servicePrincipalCredentials struct {
 	AppID    string `json:"appId"`
 	Password string `json:"password"`
 	Tenant   string `json:"tenant"`
 }
 
-const azureActiveDirectoryEndpoint = "https://login.microsoftonline.com/"
-const azureStorageEndpoint = "https://storage.azure.com/"
-
-// newServicePrincipalTokenRefresher takes a servicePrincipalCredentials structure and returns a refresh-able access token.
-func newServicePrincipalTokenRefresher(ctx context.Context, spCredentials servicePrincipalCredentials) (azblob.TokenRefresher, error) {
-	oauthConfig, err := adal.NewOAuthConfig(azureActiveDirectoryEndpoint, spCredentials.Tenant)
-	if err != nil {
-		return nil, fmt.Errorf("error creating oauth config: %w", err)
-	}
-
-	// Create service principal token for Azure Storage.
-	servicePrincipalToken, err := adal.NewServicePrincipalToken(
-		*oauthConfig,
-		spCredentials.AppID,
-		spCredentials.Password,
-		azureStorageEndpoint)
-	if err != nil {
-		return nil, fmt.Errorf("error creating service principal token: %w", err)
-	}
-
-	// Wrap token inside a refresher closure.
-	var tokenRefresher azblob.TokenRefresher = func(credential azblob.TokenCredential) time.Duration {
-		if err := servicePrincipalToken.Refresh(); err != nil {
-			panic(err)
-		}
-		refreshedToken := servicePrincipalToken.Token()
-		credential.SetToken(refreshedToken.AccessToken)
-		exp := refreshedToken.Expires().Sub(time.Now().Add(2 * time.Minute))
-		return exp
-	}
-
-	return tokenRefresher, nil
+// setRoot changes the root of the Fs
+func (f *Fs) setRoot(root string) {
+	f.root = parsePath(root)
+	f.rootContainer, f.rootDirectory = bucket.Split(f.root)
 }
 
-// newPipeline creates a Pipeline using the specified credentials and options.
-//
-// this code was copied from azblob.NewPipeline
-func (f *Fs) newPipeline(c azblob.Credential, o azblob.PipelineOptions) pipeline.Pipeline {
-	// Don't log stuff to syslog/Windows Event log
-	pipeline.SetForceLogEnabled(false)
+// Wrap the http.Transport to satisfy the Transporter interface
+type transporter struct {
+	http.RoundTripper
+}
 
-	// Closest to API goes first; closest to the wire goes last
-	factories := []pipeline.Factory{
-		azblob.NewTelemetryPolicyFactory(o.Telemetry),
-		azblob.NewUniqueRequestIDPolicyFactory(),
-		azblob.NewRetryPolicyFactory(o.Retry),
-		c,
-		pipeline.MethodFactoryMarker(), // indicates at what stage in the pipeline the method factory is invoked
-		azblob.NewRequestLogPolicyFactory(o.RequestLog),
+// Make a new transporter
+func newTransporter(ctx context.Context) transporter {
+	return transporter{
+		RoundTripper: fshttp.NewTransport(ctx),
 	}
-	return pipeline.NewPipeline(factories, pipeline.Options{HTTPSender: httpClientFactory(f.client), Log: o.Log})
 }
 
-// setRoot changes the root of the Fs
-func (f *Fs) setRoot(root string) {
-	f.root = parsePath(root)
-	f.rootContainer, f.rootDirectory = bucket.Split(f.root)
+// Do sends the HTTP request and returns the HTTP response or error.
+func (tr transporter) Do(req *http.Request) (*http.Response, error) {
+	return tr.RoundTripper.RoundTrip(req)
 }
 
 // NewFs constructs an Fs from the path, container:path
@@ -557,12 +532,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt.AccessTier = string(defaultAccessTier)
 	} else if !validateAccessTier(opt.AccessTier) {
 		return nil, fmt.Errorf("supported access tiers are %s, %s and %s",
-			string(azblob.AccessTierHot), string(azblob.AccessTierCool), string(azblob.AccessTierArchive))
+			string(blob.AccessTierHot), string(blob.AccessTierCool), string(blob.AccessTierArchive))
 	}
 
 	if !validatePublicAccess((opt.PublicAccess)) {
 		return nil, fmt.Errorf("supported public access level are %s and %s",
-			string(azblob.PublicAccessBlob), string(azblob.PublicAccessContainer))
+			string(container.PublicAccessTypeBlob), string(container.PublicAccessTypeContainer))
 	}
 
 	ci := fs.GetConfig(ctx)
@@ -571,20 +546,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt:         *opt,
 		ci:          ci,
 		pacer:       fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		imdsPacer:   fs.NewPacer(ctx, pacer.NewAzureIMDS()),
 		uploadToken: pacer.NewTokenDispenser(ci.Transfers),
-		client:      fshttp.NewClient(ctx),
 		cache:       bucket.NewCache(),
-		cntURLcache: make(map[string]*azblob.ContainerURL, 1),
+		cntSVCcache: make(map[string]*container.Client, 1),
 		pool: pool.New(
 			time.Duration(opt.MemoryPoolFlushTime),
 			int(opt.ChunkSize),
 			ci.Transfers,
 			opt.MemoryPoolUseMmap,
 		),
+		poolSize: int64(opt.ChunkSize),
 	}
-	f.publicAccess = azblob.PublicAccessType(opt.PublicAccess)
-	f.imdsPacer.SetRetries(5) // per IMDS documentation
+	f.publicAccess = container.PublicAccessType(opt.PublicAccess)
 	f.setRoot(root)
 	f.features = (&fs.Features{
 		ReadMimeType:      true,
@@ -595,171 +568,237 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		GetTier:           true,
 	}).Fill(ctx, f)
 
+	// var (
+	// 	u          *url.URL
+	// 	serviceURL azblob.ServiceURL
+	// )
+	// switch {
+	// case opt.UseEmulator:
+	// 	var actualEmulatorAccount = emulatorAccount
+	// 	if opt.Account != "" {
+	// 		actualEmulatorAccount = opt.Account
+	// 	}
+	// 	var actualEmulatorKey = emulatorAccountKey
+	// 	if opt.Key != "" {
+	// 		actualEmulatorKey = opt.Key
+	// 	}
+	// 	credential, err := azblob.NewSharedKeyCredential(actualEmulatorAccount, actualEmulatorKey)
+	// 	if err != nil {
+	// 		return nil, fmt.Errorf("failed to parse credentials: %w", err)
+	// 	}
+	// 	var actualEmulatorEndpoint = emulatorBlobEndpoint
+	// 	if opt.Endpoint != "" {
+	// 		actualEmulatorEndpoint = opt.Endpoint
+	// 	}
+	// 	u, err = url.Parse(actualEmulatorEndpoint)
+	// 	if err != nil {
+	// 		return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
+	// 	}
+	// 	pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
+	// 	serviceURL = azblob.NewServiceURL(*u, pipeline)
+	// case opt.UseMSI:
+	// 	var token adal.Token
+	// 	var userMSI = &userMSI{}
+	// 	if len(opt.MSIClientID) > 0 || len(opt.MSIObjectID) > 0 || len(opt.MSIResourceID) > 0 {
+	// 		// Specifying a user-assigned identity. Exactly one of the above IDs must be specified.
+	// 		// Validate and ensure exactly one is set. (To do: better validation.)
+	// 		if len(opt.MSIClientID) > 0 {
+	// 			if len(opt.MSIObjectID) > 0 || len(opt.MSIResourceID) > 0 {
+	// 				return nil, errors.New("more than one user-assigned identity ID is set")
+	// 			}
+	// 			userMSI.Type = msiClientID
+	// 			userMSI.Value = opt.MSIClientID
+	// 		}
+	// 		if len(opt.MSIObjectID) > 0 {
+	// 			if len(opt.MSIClientID) > 0 || len(opt.MSIResourceID) > 0 {
+	// 				return nil, errors.New("more than one user-assigned identity ID is set")
+	// 			}
+	// 			userMSI.Type = msiObjectID
+	// 			userMSI.Value = opt.MSIObjectID
+	// 		}
+	// 		if len(opt.MSIResourceID) > 0 {
+	// 			if len(opt.MSIClientID) > 0 || len(opt.MSIObjectID) > 0 {
+	// 				return nil, errors.New("more than one user-assigned identity ID is set")
+	// 			}
+	// 			userMSI.Type = msiResourceID
+	// 			userMSI.Value = opt.MSIResourceID
+	// 		}
+	// 	} else {
+	// 		userMSI = nil
+	// 	}
+	// 	err = f.imdsPacer.Call(func() (bool, error) {
+	// 		// Retry as specified by the documentation:
+	// 		// https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#retry-guidance
+	// 		token, err = GetMSIToken(ctx, userMSI)
+	// 		return f.shouldRetry(ctx, err)
+	// 	})
+
+	// 	if err != nil {
+	// 		return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
+	// 	}
+
+	// 	u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
+	// 	if err != nil {
+	// 		return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
+	// 	}
+	// 	credential := azblob.NewTokenCredential(token.AccessToken, func(credential azblob.TokenCredential) time.Duration {
+	// 		fs.Debugf(f, "Token refresher called.")
+	// 		var refreshedToken adal.Token
+	// 		err := f.imdsPacer.Call(func() (bool, error) {
+	// 			refreshedToken, err = GetMSIToken(ctx, userMSI)
+	// 			return f.shouldRetry(ctx, err)
+	// 		})
+	// 		if err != nil {
+	// 			// Failed to refresh.
+	// 			return 0
+	// 		}
+	// 		credential.SetToken(refreshedToken.AccessToken)
+	// 		now := time.Now().UTC()
+	// 		// Refresh one minute before expiry.
+	// 		refreshAt := refreshedToken.Expires().UTC().Add(-1 * time.Minute)
+	// 		fs.Debugf(f, "Acquired new token that expires at %v; refreshing in %d s", refreshedToken.Expires(),
+	// 			int(refreshAt.Sub(now).Seconds()))
+	// 		if now.After(refreshAt) {
+	// 			// Acquired a causality violation.
+	// 			return 0
+	// 		}
+	// 		return refreshAt.Sub(now)
+	// 	})
+	// 	pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
+	// 	serviceURL = azblob.NewServiceURL(*u, pipeline)
+	// case opt.Account != "" && opt.Key != "":
+	// 	credential, err := azblob.NewSharedKeyCredential(opt.Account, opt.Key)
+	// 	if err != nil {
+	// 		return nil, fmt.Errorf("failed to parse credentials: %w", err)
+	// 	}
+
+	// 	u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
+	// 	if err != nil {
+	// 		return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
+	// 	}
+	// 	pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
+	// 	serviceURL = azblob.NewServiceURL(*u, pipeline)
+	// case opt.SASURL != "":
+	// 	u, err = url.Parse(opt.SASURL)
+	// 	if err != nil {
+	// 		return nil, fmt.Errorf("failed to parse SAS URL: %w", err)
+	// 	}
+	// 	// use anonymous credentials in case of sas url
+	// 	pipeline := f.newPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
+	// 	// Check if we have container level SAS or account level sas
+	// 	parts := azblob.NewBlobURLParts(*u)
+	// 	if parts.ContainerName != "" {
+	// 		if f.rootContainer != "" && parts.ContainerName != f.rootContainer {
+	// 			return nil, errors.New("container name in SAS URL and container provided in command do not match")
+	// 		}
+	// 		containerURL := azblob.NewContainerURL(*u, pipeline)
+	// 		f.cntSVCcache[parts.ContainerName] = &containerURL
+	// 		f.isLimited = true
+	// 	} else {
+	// 		serviceURL = azblob.NewServiceURL(*u, pipeline)
+	// 	}
+	// case opt.ServicePrincipalFile != "":
+	// 	// Create a standard URL.
+	// 	u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
+	// 	if err != nil {
+	// 		return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
+	// 	}
+	// 	// Try loading service principal credentials from file.
+	// 	loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
+	// 	if err != nil {
+	// 		return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
+	// 	}
+	// 	// Create a token refresher from service principal credentials.
+	// 	tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, loadedCreds)
+	// 	if err != nil {
+	// 		return nil, fmt.Errorf("failed to create a service principal token: %w", err)
+	// 	}
+	// 	options := azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}}
+	// 	pipe := f.newPipeline(azblob.NewTokenCredential("", tokenRefresher), options)
+	// 	serviceURL = azblob.NewServiceURL(*u, pipe)
+	// default:
+	// 	return nil, errors.New("no authentication method configured")
+	// }
+	//f.svcURL = &serviceURL
+
+	u, err := url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
+	if err != nil {
+		return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
+	}
+	serviceURL := u.String()
+	fs.Debugf(f, "Service URL = %q", serviceURL)
+
+	// FIXME Very quick and dirty auth
+
 	var (
-		u          *url.URL
-		serviceURL azblob.ServiceURL
+		cred          azcore.TokenCredential
+		sharedKeyCred *service.SharedKeyCredential
 	)
-	switch {
-	case opt.UseEmulator:
-		var actualEmulatorAccount = emulatorAccount
-		if opt.Account != "" {
-			actualEmulatorAccount = opt.Account
-		}
-		var actualEmulatorKey = emulatorAccountKey
-		if opt.Key != "" {
-			actualEmulatorKey = opt.Key
-		}
-		credential, err := azblob.NewSharedKeyCredential(actualEmulatorAccount, actualEmulatorKey)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse credentials: %w", err)
-		}
-		var actualEmulatorEndpoint = emulatorBlobEndpoint
-		if opt.Endpoint != "" {
-			actualEmulatorEndpoint = opt.Endpoint
-		}
-		u, err = url.Parse(actualEmulatorEndpoint)
-		if err != nil {
-			return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
-		}
-		pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
-		serviceURL = azblob.NewServiceURL(*u, pipeline)
-	case opt.UseMSI:
-		var token adal.Token
-		var userMSI = &userMSI{}
-		if len(opt.MSIClientID) > 0 || len(opt.MSIObjectID) > 0 || len(opt.MSIResourceID) > 0 {
-			// Specifying a user-assigned identity. Exactly one of the above IDs must be specified.
-			// Validate and ensure exactly one is set. (To do: better validation.)
-			if len(opt.MSIClientID) > 0 {
-				if len(opt.MSIObjectID) > 0 || len(opt.MSIResourceID) > 0 {
-					return nil, errors.New("more than one user-assigned identity ID is set")
-				}
-				userMSI.Type = msiClientID
-				userMSI.Value = opt.MSIClientID
-			}
-			if len(opt.MSIObjectID) > 0 {
-				if len(opt.MSIClientID) > 0 || len(opt.MSIResourceID) > 0 {
-					return nil, errors.New("more than one user-assigned identity ID is set")
-				}
-				userMSI.Type = msiObjectID
-				userMSI.Value = opt.MSIObjectID
-			}
-			if len(opt.MSIResourceID) > 0 {
-				if len(opt.MSIClientID) > 0 || len(opt.MSIObjectID) > 0 {
-					return nil, errors.New("more than one user-assigned identity ID is set")
-				}
-				userMSI.Type = msiResourceID
-				userMSI.Value = opt.MSIResourceID
-			}
-		} else {
-			userMSI = nil
-		}
-		err = f.imdsPacer.Call(func() (bool, error) {
-			// Retry as specified by the documentation:
-			// https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#retry-guidance
-			token, err = GetMSIToken(ctx, userMSI)
-			return f.shouldRetry(ctx, err)
-		})
 
+	if opt.EnvAuth {
+		// Read credentials from the environment
+		cred, err = azidentity.NewDefaultAzureCredential(nil)
 		if err != nil {
-			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
+			return nil, fmt.Errorf("create default azure credential failed: %w", err)
 		}
+	} else {
 
-		u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
+		// Use the config file to configure
+		sharedKeyCred, err = service.NewSharedKeyCredential(opt.Account, opt.Key)
 		if err != nil {
-			return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
-		}
-		credential := azblob.NewTokenCredential(token.AccessToken, func(credential azblob.TokenCredential) time.Duration {
-			fs.Debugf(f, "Token refresher called.")
-			var refreshedToken adal.Token
-			err := f.imdsPacer.Call(func() (bool, error) {
-				refreshedToken, err = GetMSIToken(ctx, userMSI)
-				return f.shouldRetry(ctx, err)
-			})
-			if err != nil {
-				// Failed to refresh.
-				return 0
-			}
-			credential.SetToken(refreshedToken.AccessToken)
-			now := time.Now().UTC()
-			// Refresh one minute before expiry.
-			refreshAt := refreshedToken.Expires().UTC().Add(-1 * time.Minute)
-			fs.Debugf(f, "Acquired new token that expires at %v; refreshing in %d s", refreshedToken.Expires(),
-				int(refreshAt.Sub(now).Seconds()))
-			if now.After(refreshAt) {
-				// Acquired a causality violation.
-				return 0
-			}
-			return refreshAt.Sub(now)
-		})
-		pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
-		serviceURL = azblob.NewServiceURL(*u, pipeline)
-	case opt.Account != "" && opt.Key != "":
-		credential, err := azblob.NewSharedKeyCredential(opt.Account, opt.Key)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse credentials: %w", err)
+			return nil, fmt.Errorf("create new shared key credential failed: %w", err)
 		}
+	}
 
-		u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
-		if err != nil {
-			return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
-		}
-		pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
-		serviceURL = azblob.NewServiceURL(*u, pipeline)
-	case opt.SASURL != "":
-		u, err = url.Parse(opt.SASURL)
+	// Specify our own transport
+	clientOpt := service.ClientOptions{
+		ClientOptions: azcore.ClientOptions{
+			Transport: newTransporter(ctx),
+		},
+	}
+	// azClientOpt := azblob.ClientOptions{
+	// 	ClientOptions: azcore.ClientOptions{
+	// 		Transport: clientOpt.ClientOptions.Transport,
+	// 	},
+	// }
+
+	if sharedKeyCred != nil {
+		// create a client for the specified storage account
+		client, err := service.NewClientWithSharedKeyCredential(serviceURL, sharedKeyCred, &clientOpt)
 		if err != nil {
-			return nil, fmt.Errorf("failed to parse SAS URL: %w", err)
+			return nil, fmt.Errorf("create client with shared key failed: %w", err)
 		}
-		// use anonymous credentials in case of sas url
-		pipeline := f.newPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
-		// Check if we have container level SAS or account level sas
-		parts := azblob.NewBlobURLParts(*u)
-		if parts.ContainerName != "" {
-			if f.rootContainer != "" && parts.ContainerName != f.rootContainer {
-				return nil, errors.New("container name in SAS URL and container provided in command do not match")
-			}
-			containerURL := azblob.NewContainerURL(*u, pipeline)
-			f.cntURLcache[parts.ContainerName] = &containerURL
-			f.isLimited = true
-		} else {
-			serviceURL = azblob.NewServiceURL(*u, pipeline)
-		}
-	case opt.ServicePrincipalFile != "" || opt.EnvAuth:
-		// Create a standard URL.
-		u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
+		f.svc = client
+
+		// create a client for the specified storage account
+		//
+		// Annoyingly this is the same type as f.svc just wrapped in a
+		// struct, but there is no way to create one from the other.
+		// azsvc, err := azblob.NewClientWithSharedKeyCredential(serviceURL, sharedKeyCred, &azClientOpt)
+		// if err != nil {
+		// 	return nil, fmt.Errorf("create client failed: %w", err)
+		// }
+		// f.azsvc = azsvc
+	} else {
+		// create a client for the specified storage account
+		// azblob.ClientOptions{}
+		client, err := service.NewClient(serviceURL, cred, &clientOpt)
 		if err != nil {
-			return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
+			return nil, fmt.Errorf("create client failed: %w", err)
 		}
-		var spCredentials servicePrincipalCredentials
-		if opt.ServicePrincipalFile != "" {
-			// Try loading service principal credentials from file.
-			loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
-			if err != nil {
-				return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
-			}
+		f.svc = client
 
-			if err := json.Unmarshal(loadedCreds, &spCredentials); err != nil {
-				return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err)
-			}
-		} else {
-			spCredentials = servicePrincipalCredentials{
-				Tenant:   os.Getenv("AZURE_TENANT_ID"),
-				AppID:    os.Getenv("AZURE_CLIENT_ID"),
-				Password: os.Getenv("AZURE_CLIENT_SECRET"),
-			}
-		}
-		// Create a token refresher from service principal credentials.
-		tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, spCredentials)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create a service principal token: %w", err)
-		}
-		options := azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}}
-		pipe := f.newPipeline(azblob.NewTokenCredential("", tokenRefresher), options)
-		serviceURL = azblob.NewServiceURL(*u, pipe)
-	default:
-		return nil, errors.New("no authentication method configured")
+		// create a client for the specified storage account
+		// azblob.ClientOptions{}
+		//
+		// Annoyingly this is the same type as f.svc just wrapped in a
+		// struct, but there is no way to create one from the other.
+		// azsvc, err := azblob.NewClient(serviceURL, cred, &azClientOpt)
+		// if err != nil {
+		// 	return nil, fmt.Errorf("create client failed: %w", err)
+		// }
+		// f.azsvc = azsvc
 	}
-	f.svcURL = &serviceURL
 
 	if f.rootContainer != "" && f.rootDirectory != "" {
 		// Check to see if the (container,directory) is actually an existing file
@@ -781,24 +820,22 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	return f, nil
 }
 
-// return the container URL for the container passed in
-func (f *Fs) cntURL(container string) (containerURL *azblob.ContainerURL) {
-	f.cntURLcacheMu.Lock()
-	defer f.cntURLcacheMu.Unlock()
+// return the container client for the container passed in
+func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
+	f.cntSVCcacheMu.Lock()
+	defer f.cntSVCcacheMu.Unlock()
 	var ok bool
-	if containerURL, ok = f.cntURLcache[container]; !ok {
-		cntURL := f.svcURL.NewContainerURL(container)
-		containerURL = &cntURL
-		f.cntURLcache[container] = containerURL
+	if containerClient, ok = f.cntSVCcache[containerName]; !ok {
+		containerClient = f.svc.NewContainerClient(containerName)
+		f.cntSVCcache[containerName] = containerClient
 	}
-	return containerURL
-
+	return containerClient
 }
 
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(remote string, info *azblob.BlobItemInternal) (fs.Object, error) {
+func (f *Fs) newObjectWithInfo(remote string, info *container.BlobItem) (fs.Object, error) {
 	o := &Object{
 		fs:     f,
 		remote: remote,
@@ -823,9 +860,20 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
-// getBlobReference creates an empty blob reference with no metadata
-func (f *Fs) getBlobReference(container, containerPath string) azblob.BlobURL {
-	return f.cntURL(container).NewBlobURL(containerPath)
+// getBlobSVC creates a blob client
+func (f *Fs) getBlobSVC(container, containerPath string) *blob.Client {
+	// FIXME the urlEncode here is a workaround for
+	// https://github.com/Azure/azure-sdk-for-go/issues/19613
+	// https://github.com/Azure/azure-sdk-for-go/issues/19475
+	return f.cntSVC(container).NewBlobClient(urlEncode(containerPath))
+}
+
+// getBlockBlobSVC creates a block blob client
+func (f *Fs) getBlockBlobSVC(container, containerPath string) *blockblob.Client {
+	// FIXME the urlEncode here is a workaround for
+	// https://github.com/Azure/azure-sdk-for-go/issues/19613
+	// https://github.com/Azure/azure-sdk-for-go/issues/19475
+	return f.cntSVC(container).NewBlockBlobClient(urlEncode(containerPath))
 }
 
 // updateMetadataWithModTime adds the modTime passed in to o.meta.
@@ -840,7 +888,7 @@ func (o *Object) updateMetadataWithModTime(modTime time.Time) {
 }
 
 // Returns whether file is a directory marker or not
-func isDirectoryMarker(size int64, metadata azblob.Metadata, remote string) bool {
+func isDirectoryMarker(size int64, metadata map[string]string, remote string) bool {
 	// Directory markers are 0 length
 	if size == 0 {
 		// Note that metadata with hdi_isfolder = true seems to be a
@@ -854,8 +902,27 @@ func isDirectoryMarker(size int64, metadata azblob.Metadata, remote string) bool
 	return false
 }
 
+// Returns whether file is a directory marker or not using metadata
+// with pointers to strings as the SDK seems to use both forms rather
+// annoyingly.
+//
+// NB This is a duplicate of isDirectoryMarker
+func isDirectoryMarkerP(size int64, metadata map[string]*string, remote string) bool {
+	// Directory markers are 0 length
+	if size == 0 {
+		// Note that metadata with hdi_isfolder = true seems to be a
+		// defacto standard for marking blobs as directories.
+		endsWithSlash := strings.HasSuffix(remote, "/")
+		if endsWithSlash || remote == "" || (metadata["hdi_isfolder"] != nil && *metadata["hdi_isfolder"] == "true") {
+			return true
+		}
+
+	}
+	return false
+}
+
 // listFn is called from list to handle an object
-type listFn func(remote string, object *azblob.BlobItemInternal, isDirectory bool) error
+type listFn func(remote string, object *container.BlobItem, isDirectory bool) error
 
 // list lists the objects into the function supplied from
 // the container and root supplied
@@ -864,8 +931,8 @@ type listFn func(remote string, object *azblob.BlobItemInternal, isDirectory boo
 //
 // The remote has prefix removed from it and if addContainer is set then
 // it adds the container to the start.
-func (f *Fs) list(ctx context.Context, container, directory, prefix string, addContainer bool, recurse bool, maxResults uint, fn listFn) error {
-	if f.cache.IsDeleted(container) {
+func (f *Fs) list(ctx context.Context, containerName, directory, prefix string, addContainer bool, recurse bool, maxResults int32, fn listFn) error {
+	if f.cache.IsDeleted(containerName) {
 		return fs.ErrorDirNotFound
 	}
 	if prefix != "" {
@@ -879,51 +946,57 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
 		delimiter = "/"
 	}
 
-	options := azblob.ListBlobsSegmentOptions{
-		Details: azblob.BlobListingDetails{
+	pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &container.ListBlobsHierarchyOptions{
+		// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
+		Include: container.ListBlobsInclude{
 			Copy:             false,
 			Metadata:         true,
 			Snapshots:        false,
 			UncommittedBlobs: false,
 			Deleted:          false,
 		},
-		Prefix:     directory,
-		MaxResults: int32(maxResults),
-	}
-	for marker := (azblob.Marker{}); marker.NotDone(); {
-		var response *azblob.ListBlobsHierarchySegmentResponse
+		Prefix:     &directory,
+		MaxResults: &maxResults,
+	})
+	for pager.More() {
+		var response container.ListBlobsHierarchyResponse
 		err := f.pacer.Call(func() (bool, error) {
 			var err error
-			response, err = f.cntURL(container).ListBlobsHierarchySegment(ctx, marker, delimiter, options)
+			response, err = pager.NextPage(ctx)
+			//response, err = f.srv.ListBlobsHierarchySegment(ctx, marker, delimiter, options)
 			return f.shouldRetry(ctx, err)
 		})
 
 		if err != nil {
 			// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
-			if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
+			if storageErr, ok := err.(*azcore.ResponseError); ok && (storageErr.ErrorCode == string(bloberror.ContainerNotFound) || storageErr.StatusCode == http.StatusNotFound) {
 				return fs.ErrorDirNotFound
 			}
 			return err
 		}
 		// Advance marker to next
-		marker = response.NextMarker
+		// marker = response.NextMarker
 		for i := range response.Segment.BlobItems {
-			file := &response.Segment.BlobItems[i]
+			file := response.Segment.BlobItems[i]
 			// Finish if file name no longer has prefix
 			// if prefix != "" && !strings.HasPrefix(file.Name, prefix) {
 			// 	return nil
 			// }
-			remote := f.opt.Enc.ToStandardPath(file.Name)
+			if file.Name == nil {
+				fs.Debugf(f, "Nil name received")
+				continue
+			}
+			remote := f.opt.Enc.ToStandardPath(*file.Name)
 			if !strings.HasPrefix(remote, prefix) {
 				fs.Debugf(f, "Odd name received %q", remote)
 				continue
 			}
 			remote = remote[len(prefix):]
-			if isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote) {
+			if isDirectoryMarkerP(*file.Properties.ContentLength, file.Metadata, remote) {
 				continue // skip directory marker
 			}
 			if addContainer {
-				remote = path.Join(container, remote)
+				remote = path.Join(containerName, remote)
 			}
 			// Send object
 			err = fn(remote, file, false)
@@ -933,7 +1006,11 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
 		}
 		// Send the subdirectories
 		for _, remote := range response.Segment.BlobPrefixes {
-			remote := strings.TrimRight(remote.Name, "/")
+			if remote.Name == nil {
+				fs.Debugf(f, "Nil prefix received")
+				continue
+			}
+			remote := strings.TrimRight(*remote.Name, "/")
 			remote = f.opt.Enc.ToStandardPath(remote)
 			if !strings.HasPrefix(remote, prefix) {
 				fs.Debugf(f, "Odd directory name received %q", remote)
@@ -941,7 +1018,7 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
 			}
 			remote = remote[len(prefix):]
 			if addContainer {
-				remote = path.Join(container, remote)
+				remote = path.Join(containerName, remote)
 			}
 			// Send object
 			err = fn(remote, nil, true)
@@ -954,7 +1031,7 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
 }
 
 // Convert a list item into a DirEntry
-func (f *Fs) itemToDirEntry(remote string, object *azblob.BlobItemInternal, isDirectory bool) (fs.DirEntry, error) {
+func (f *Fs) itemToDirEntry(remote string, object *container.BlobItem, isDirectory bool) (fs.DirEntry, error) {
 	if isDirectory {
 		d := fs.NewDir(remote, time.Time{})
 		return d, nil
@@ -971,9 +1048,9 @@ func (f *Fs) containerOK(container string) bool {
 	if !f.isLimited {
 		return true
 	}
-	f.cntURLcacheMu.Lock()
-	defer f.cntURLcacheMu.Unlock()
-	for limitedContainer := range f.cntURLcache {
+	f.cntSVCcacheMu.Lock()
+	defer f.cntSVCcacheMu.Unlock()
+	for limitedContainer := range f.cntSVCcache {
 		if container == limitedContainer {
 			return true
 		}
@@ -982,11 +1059,11 @@ func (f *Fs) containerOK(container string) bool {
 }
 
 // listDir lists a single directory
-func (f *Fs) listDir(ctx context.Context, container, directory, prefix string, addContainer bool) (entries fs.DirEntries, err error) {
-	if !f.containerOK(container) {
+func (f *Fs) listDir(ctx context.Context, containerName, directory, prefix string, addContainer bool) (entries fs.DirEntries, err error) {
+	if !f.containerOK(containerName) {
 		return nil, fs.ErrorDirNotFound
 	}
-	err = f.list(ctx, container, directory, prefix, addContainer, false, f.opt.ListChunkSize, func(remote string, object *azblob.BlobItemInternal, isDirectory bool) error {
+	err = f.list(ctx, containerName, directory, prefix, addContainer, false, int32(f.opt.ListChunkSize), func(remote string, object *container.BlobItem, isDirectory bool) error {
 		entry, err := f.itemToDirEntry(remote, object, isDirectory)
 		if err != nil {
 			return err
@@ -1000,24 +1077,24 @@ func (f *Fs) listDir(ctx context.Context, container, directory, prefix string, a
 		return nil, err
 	}
 	// container must be present if listing succeeded
-	f.cache.MarkOK(container)
+	f.cache.MarkOK(containerName)
 	return entries, nil
 }
 
 // listContainers returns all the containers to out
 func (f *Fs) listContainers(ctx context.Context) (entries fs.DirEntries, err error) {
 	if f.isLimited {
-		f.cntURLcacheMu.Lock()
-		for container := range f.cntURLcache {
+		f.cntSVCcacheMu.Lock()
+		for container := range f.cntSVCcache {
 			d := fs.NewDir(container, time.Time{})
 			entries = append(entries, d)
 		}
-		f.cntURLcacheMu.Unlock()
+		f.cntSVCcacheMu.Unlock()
 		return entries, nil
 	}
-	err = f.listContainersToFn(func(container *azblob.ContainerItem) error {
-		d := fs.NewDir(f.opt.Enc.ToStandardName(container.Name), container.Properties.LastModified)
-		f.cache.MarkOK(container.Name)
+	err = f.listContainersToFn(func(Name string, LastModified time.Time) error {
+		d := fs.NewDir(f.opt.Enc.ToStandardName(Name), LastModified)
+		f.cache.MarkOK(Name)
 		entries = append(entries, d)
 		return nil
 	})
@@ -1064,10 +1141,10 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Don't implement this unless you have a more efficient way
 // of listing recursively that doing a directory traversal.
 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
-	container, directory := f.split(dir)
+	containerName, directory := f.split(dir)
 	list := walk.NewListRHelper(callback)
-	listR := func(container, directory, prefix string, addContainer bool) error {
-		return f.list(ctx, container, directory, prefix, addContainer, true, f.opt.ListChunkSize, func(remote string, object *azblob.BlobItemInternal, isDirectory bool) error {
+	listR := func(containerName, directory, prefix string, addContainer bool) error {
+		return f.list(ctx, containerName, directory, prefix, addContainer, true, int32(f.opt.ListChunkSize), func(remote string, object *container.BlobItem, isDirectory bool) error {
 			entry, err := f.itemToDirEntry(remote, object, isDirectory)
 			if err != nil {
 				return err
@@ -1075,7 +1152,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 			return list.Add(entry)
 		})
 	}
-	if container == "" {
+	if containerName == "" {
 		entries, err := f.listContainers(ctx)
 		if err != nil {
 			return err
@@ -1094,46 +1171,50 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 			f.cache.MarkOK(container)
 		}
 	} else {
-		if !f.containerOK(container) {
+		if !f.containerOK(containerName) {
 			return fs.ErrorDirNotFound
 		}
-		err = listR(container, directory, f.rootDirectory, f.rootContainer == "")
+		err = listR(containerName, directory, f.rootDirectory, f.rootContainer == "")
 		if err != nil {
 			return err
 		}
 		// container must be present if listing succeeded
-		f.cache.MarkOK(container)
+		f.cache.MarkOK(containerName)
 	}
 	return list.Flush()
 }
 
 // listContainerFn is called from listContainersToFn to handle a container
-type listContainerFn func(*azblob.ContainerItem) error
+type listContainerFn func(Name string, LastModified time.Time) error
 
 // listContainersToFn lists the containers to the function supplied
 func (f *Fs) listContainersToFn(fn listContainerFn) error {
-	params := azblob.ListContainersSegmentOptions{
-		MaxResults: int32(f.opt.ListChunkSize),
-	}
+	max := int32(f.opt.ListChunkSize)
+	pager := f.svc.NewListContainersPager(&service.ListContainersOptions{
+		Include:    service.ListContainersInclude{Metadata: true, Deleted: true},
+		MaxResults: &max,
+	})
 	ctx := context.Background()
-	for marker := (azblob.Marker{}); marker.NotDone(); {
-		var response *azblob.ListContainersSegmentResponse
+	for pager.More() {
+		var response service.ListContainersResponse
 		err := f.pacer.Call(func() (bool, error) {
 			var err error
-			response, err = f.svcURL.ListContainersSegment(ctx, marker, params)
+			response, err = pager.NextPage(ctx)
 			return f.shouldRetry(ctx, err)
 		})
 		if err != nil {
 			return err
 		}
 
-		for i := range response.ContainerItems {
-			err = fn(&response.ContainerItems[i])
+		for _, cnt := range response.ContainerItems {
+			if cnt == nil || cnt.Name == nil || cnt.Properties == nil || cnt.Properties.LastModified == nil {
+				fs.Debugf(f, "nil returned in container info")
+			}
+			err = fn(*cnt.Name, *cnt.Properties.LastModified)
 			if err != nil {
 				return err
 			}
 		}
-		marker = response.NextMarker
 	}
 
 	return nil
@@ -1171,15 +1252,25 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
 		if f.isLimited {
 			return nil
 		}
+		opt := service.CreateContainerOptions{
+			// Specifies whether data in the container may be accessed publicly and the level of access
+			Access: &f.publicAccess,
+
+			// Optional. Specifies a user-defined name-value pair associated with the blob.
+			//Metadata map[string]string
+
+			// Optional. Specifies the encryption scope settings to set on the container.
+			//CpkScopeInfo *CpkScopeInfo
+		}
 		// now try to create the container
 		return f.pacer.Call(func() (bool, error) {
-			_, err := f.cntURL(container).Create(ctx, azblob.Metadata{}, f.publicAccess)
+			_, err := f.svc.CreateContainer(ctx, container, &opt)
 			if err != nil {
-				if storageErr, ok := err.(azblob.StorageError); ok {
-					switch storageErr.ServiceCode() {
-					case azblob.ServiceCodeContainerAlreadyExists:
+				if storageErr, ok := err.(*azcore.ResponseError); ok {
+					switch bloberror.Code(storageErr.ErrorCode) {
+					case bloberror.ContainerAlreadyExists:
 						return false, nil
-					case azblob.ServiceCodeContainerBeingDeleted:
+					case bloberror.ContainerBeingDeleted:
 						// From https://docs.microsoft.com/en-us/rest/api/storageservices/delete-container
 						// When a container is deleted, a container with the same name cannot be created
 						// for at least 30 seconds; the container may not be available for more than 30
@@ -1196,9 +1287,9 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
 }
 
 // isEmpty checks to see if a given (container, directory) is empty and returns an error if not
-func (f *Fs) isEmpty(ctx context.Context, container, directory string) (err error) {
+func (f *Fs) isEmpty(ctx context.Context, containerName, directory string) (err error) {
 	empty := true
-	err = f.list(ctx, container, directory, f.rootDirectory, f.rootContainer == "", true, 1, func(remote string, object *azblob.BlobItemInternal, isDirectory bool) error {
+	err = f.list(ctx, containerName, directory, f.rootDirectory, f.rootContainer == "", true, 1, func(remote string, object *container.BlobItem, isDirectory bool) error {
 		empty = false
 		return nil
 	})
@@ -1213,18 +1304,19 @@ func (f *Fs) isEmpty(ctx context.Context, container, directory string) (err erro
 
 // deleteContainer deletes the container.  It can delete a full
 // container so use isEmpty if you don't want that.
-func (f *Fs) deleteContainer(ctx context.Context, container string) error {
-	return f.cache.Remove(container, func() error {
-		options := azblob.ContainerAccessConditions{}
+func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {
+	return f.cache.Remove(containerName, func() error {
+		getOptions := container.GetPropertiesOptions{}
+		delOptions := container.DeleteOptions{}
 		return f.pacer.Call(func() (bool, error) {
-			_, err := f.cntURL(container).GetProperties(ctx, azblob.LeaseAccessConditions{})
+			_, err := f.cntSVC(containerName).GetProperties(ctx, &getOptions)
 			if err == nil {
-				_, err = f.cntURL(container).Delete(ctx, options)
+				_, err = f.cntSVC(containerName).Delete(ctx, &delOptions)
 			}
 
 			if err != nil {
 				// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
-				if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
+				if storageErr, ok := err.(*azcore.ResponseError); ok && (storageErr.ErrorCode == string(bloberror.ContainerNotFound) || storageErr.StatusCode == http.StatusNotFound) {
 					return false, fs.ErrorDirNotFound
 				}
 
@@ -1291,38 +1383,50 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		fs.Debugf(src, "Can't copy - not same remote type")
 		return nil, fs.ErrorCantCopy
 	}
-	dstBlobURL := f.getBlobReference(dstContainer, dstPath)
-	srcBlobURL := srcObj.getBlobReference()
+	dstBlobSVC := f.getBlobSVC(dstContainer, dstPath)
+	srcBlobSVC := srcObj.getBlobSVC()
+	srcURL := srcBlobSVC.URL()
 
-	source, err := url.Parse(srcBlobURL.String())
-	if err != nil {
-		return nil, err
+	tier := blob.AccessTier(f.opt.AccessTier)
+	options := blob.StartCopyFromURLOptions{
+		Tier: &tier,
 	}
-
-	options := azblob.BlobAccessConditions{}
-	var startCopy *azblob.BlobStartCopyFromURLResponse
-
+	var startCopy blob.StartCopyFromURLResponse
 	err = f.pacer.Call(func() (bool, error) {
-		startCopy, err = dstBlobURL.StartCopyFromURL(ctx, *source, nil, azblob.ModifiedAccessConditions{}, options, azblob.AccessTierType(f.opt.AccessTier), nil)
+		startCopy, err = dstBlobSVC.StartCopyFromURL(ctx, srcURL, &options)
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
 	}
 
-	copyStatus := startCopy.CopyStatus()
-	for copyStatus == azblob.CopyStatusPending {
+	copyStatus := startCopy.CopyStatus
+	getOptions := blob.GetPropertiesOptions{}
+	for copyStatus != nil && string(*copyStatus) == string(container.CopyStatusTypePending) {
 		time.Sleep(1 * time.Second)
-		getMetadata, err := dstBlobURL.GetProperties(ctx, options, azblob.ClientProvidedKeyOptions{})
+		getMetadata, err := dstBlobSVC.GetProperties(ctx, &getOptions)
 		if err != nil {
 			return nil, err
 		}
-		copyStatus = getMetadata.CopyStatus()
+		copyStatus = getMetadata.CopyStatus
 	}
 
 	return f.NewObject(ctx, remote)
 }
 
+func (f *Fs) getMemoryPool(size int64) *pool.Pool {
+	if size == int64(f.opt.ChunkSize) {
+		return f.pool
+	}
+
+	return pool.New(
+		time.Duration(f.opt.MemoryPoolFlushTime),
+		int(size),
+		f.ci.Transfers,
+		f.opt.MemoryPoolUseMmap,
+	)
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -1364,10 +1468,14 @@ func (o *Object) Size() int64 {
 	return o.size
 }
 
-func (o *Object) setMetadata(metadata azblob.Metadata) {
+func (o *Object) setMetadata(metadata map[string]string) {
 	if len(metadata) > 0 {
-		o.meta = metadata
-		if modTime, ok := metadata[modTimeKey]; ok {
+		// Lower case the metadata
+		o.meta = make(map[string]string, len(metadata))
+		for k, v := range metadata {
+			o.meta[strings.ToLower(k)] = v
+		}
+		if modTime, ok := o.meta[modTimeKey]; ok {
 			when, err := time.Parse(timeFormatIn, modTime)
 			if err != nil {
 				fs.Debugf(o, "Couldn't parse %v = %q: %v", modTimeKey, modTime, err)
@@ -1379,6 +1487,22 @@ func (o *Object) setMetadata(metadata azblob.Metadata) {
 	}
 }
 
+// Duplicte of setMetadata but taking pointers to strings
+func (o *Object) setMetadataP(metadata map[string]*string) {
+	if len(metadata) > 0 {
+		// Convert the format of the metadata
+		newMeta := make(map[string]string, len(metadata))
+		for k, v := range metadata {
+			if v != nil {
+				newMeta[k] = *v
+			}
+		}
+		o.setMetadata(newMeta)
+	} else {
+		o.meta = nil
+	}
+}
+
 // decodeMetaDataFromPropertiesResponse sets the metadata from the data passed in
 //
 // Sets
@@ -1388,41 +1512,77 @@ func (o *Object) setMetadata(metadata azblob.Metadata) {
 //	o.size
 //	o.md5
 //	o.meta
-func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetPropertiesResponse) (err error) {
-	metadata := info.NewMetadata()
-	size := info.ContentLength()
+func (o *Object) decodeMetaDataFromPropertiesResponse(info *blob.GetPropertiesResponse) (err error) {
+	metadata := info.Metadata
+	var size int64
+	if info.ContentLength == nil {
+		size = -1
+	} else {
+		size = *info.ContentLength
+	}
 	if isDirectoryMarker(size, metadata, o.remote) {
 		return fs.ErrorNotAFile
 	}
 	// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
 	// this as base64 encoded string.
-	o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5())
-	o.mimeType = info.ContentType()
+	o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5)
+	if info.ContentType == nil {
+		o.mimeType = ""
+	} else {
+		o.mimeType = *info.ContentType
+	}
 	o.size = size
-	o.modTime = info.LastModified()
-	o.accessTier = azblob.AccessTierType(info.AccessTier())
+	if info.LastModified == nil {
+		o.modTime = time.Now()
+	} else {
+		o.modTime = *info.LastModified
+	}
+	if info.AccessTier == nil {
+		o.accessTier = blob.AccessTier("")
+	} else {
+		o.accessTier = blob.AccessTier(*info.AccessTier)
+	}
 	o.setMetadata(metadata)
 
 	return nil
 }
 
-func (o *Object) decodeMetaDataFromDownloadResponse(info *azblob.DownloadResponse) (err error) {
-	metadata := info.NewMetadata()
-	size := info.ContentLength()
+func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamResponse) (err error) {
+	metadata := info.Metadata
+	var size int64
+	if info.ContentLength == nil {
+		size = -1
+	} else {
+		size = *info.ContentLength
+	}
 	if isDirectoryMarker(size, metadata, o.remote) {
 		return fs.ErrorNotAFile
 	}
 	// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
 	// this as base64 encoded string.
-	o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5())
-	o.mimeType = info.ContentType()
+	o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5)
+	if info.ContentType == nil {
+		o.mimeType = ""
+	} else {
+		o.mimeType = *info.ContentType
+	}
 	o.size = size
-	o.modTime = info.LastModified()
-	o.accessTier = o.AccessTier()
+	if info.LastModified == nil {
+		o.modTime = time.Now()
+	} else {
+		o.modTime = *info.LastModified
+	}
+	// FIXME response doesn't appear to have AccessTier in?
+	// if info.AccessTier == nil {
+	// 	o.accessTier = blob.AccessTier("")
+	// } else {
+	// 	o.accessTier = blob.AccessTier(*info.AccessTier)
+	// }
 	o.setMetadata(metadata)
 
 	// If it was a Range request, the size is wrong, so correct it
-	if contentRange := info.ContentRange(); contentRange != "" {
+	if info.ContentRange != nil {
+		contentRange := *info.ContentRange
 		slash := strings.IndexRune(contentRange, '/')
 		if slash >= 0 {
 			i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
@@ -1439,27 +1599,54 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *azblob.DownloadRespons
 	return nil
 }
 
-func (o *Object) decodeMetaDataFromBlob(info *azblob.BlobItemInternal) (err error) {
+func (o *Object) decodeMetaDataFromBlob(info *container.BlobItem) (err error) {
+	if info.Properties == nil {
+		return errors.New("nil Properties in decodeMetaDataFromBlob")
+	}
 	metadata := info.Metadata
-	size := *info.Properties.ContentLength
-	if isDirectoryMarker(size, metadata, o.remote) {
+	var size int64
+	if info.Properties.ContentLength == nil {
+		size = -1
+	} else {
+		size = *info.Properties.ContentLength
+	}
+	if isDirectoryMarkerP(size, metadata, o.remote) {
 		return fs.ErrorNotAFile
 	}
 	// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
 	// this as base64 encoded string.
 	o.md5 = base64.StdEncoding.EncodeToString(info.Properties.ContentMD5)
-	o.mimeType = *info.Properties.ContentType
+	if info.Properties.ContentType == nil {
+		o.mimeType = ""
+	} else {
+		o.mimeType = *info.Properties.ContentType
+	}
 	o.size = size
-	o.modTime = info.Properties.LastModified
-	o.accessTier = info.Properties.AccessTier
-	o.setMetadata(metadata)
+	if info.Properties.LastModified == nil {
+		o.modTime = time.Now()
+	} else {
+		o.modTime = *info.Properties.LastModified
+	}
+	if info.Properties.AccessTier == nil {
+		o.accessTier = blob.AccessTier("")
+	} else {
+		o.accessTier = *info.Properties.AccessTier
+	}
+	o.setMetadataP(metadata)
+
 	return nil
 }
 
-// getBlobReference creates an empty blob reference with no metadata
-func (o *Object) getBlobReference() azblob.BlobURL {
+// getBlobSVC creates a blob client
+func (o *Object) getBlobSVC() *blob.Client {
+	container, directory := o.split()
+	return o.fs.getBlobSVC(container, directory)
+}
+
+// getBlockBlobSVC creates a block blob client
+func (o *Object) getBlockBlobSVC() *blockblob.Client {
 	container, directory := o.split()
-	return o.fs.getBlobReference(container, directory)
+	return o.fs.getBlockBlobSVC(container, directory)
 }
 
 // clearMetaData clears enough metadata so readMetaData will re-read it
@@ -1483,25 +1670,26 @@ func (o *Object) readMetaData() (err error) {
 	if !o.modTime.IsZero() {
 		return nil
 	}
-	blob := o.getBlobReference()
+	blb := o.getBlobSVC()
+	// fs.Debugf(o, "Blob URL = %q", blb.URL())
 
 	// Read metadata (this includes metadata)
-	options := azblob.BlobAccessConditions{}
+	options := blob.GetPropertiesOptions{}
 	ctx := context.Background()
-	var blobProperties *azblob.BlobGetPropertiesResponse
+	var blobProperties blob.GetPropertiesResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
-		blobProperties, err = blob.GetProperties(ctx, options, azblob.ClientProvidedKeyOptions{})
+		blobProperties, err = blb.GetProperties(ctx, &options)
 		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		// On directories - GetProperties does not work and current SDK does not populate service code correctly hence check regular http response as well
-		if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeBlobNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
+		if storageErr, ok := err.(*azcore.ResponseError); ok && (storageErr.ErrorCode == string(bloberror.BlobNotFound) || storageErr.StatusCode == http.StatusNotFound) {
 			return fs.ErrorObjectNotFound
 		}
 		return err
 	}
 
-	return o.decodeMetaDataFromPropertiesResponse(blobProperties)
+	return o.decodeMetaDataFromPropertiesResponse(&blobProperties)
 }
 
 // ModTime returns the modification time of the object
@@ -1523,9 +1711,10 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	// Set modTimeKey in it
 	o.meta[modTimeKey] = modTime.Format(timeFormatOut)
 
-	blob := o.getBlobReference()
+	blb := o.getBlobSVC()
+	opt := blob.SetMetadataOptions{}
 	err := o.fs.pacer.Call(func() (bool, error) {
-		_, err := blob.SetMetadata(ctx, o.meta, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+		_, err := blb.SetMetadata(ctx, o.meta, &opt)
 		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1545,7 +1734,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	// Offset and Count for range download
 	var offset int64
 	var count int64
-	if o.AccessTier() == azblob.AccessTierArchive {
+	if o.AccessTier() == blob.AccessTierArchive {
 		return nil, fmt.Errorf("blob in archive tier, you need to set tier to hot or cool first")
 	}
 	fs.FixRangeOption(options, o.size)
@@ -1564,22 +1753,36 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			}
 		}
 	}
-	blob := o.getBlobReference()
-	ac := azblob.BlobAccessConditions{}
-	var downloadResponse *azblob.DownloadResponse
+	blb := o.getBlobSVC()
+	opt := blob.DownloadStreamOptions{
+		// When set to true and specified together with the Range, the service returns the MD5 hash for the range, as long as the
+		// range is less than or equal to 4 MB in size.
+		//RangeGetContentMD5 *bool
+
+		// Range specifies a range of bytes.  The default value is all bytes.
+		//Range HTTPRange
+		Range: blob.HTTPRange{
+			Offset: offset,
+			Count:  count,
+		},
+
+		// AccessConditions *AccessConditions
+		// CpkInfo          *CpkInfo
+		// CpkScopeInfo     *CpkScopeInfo
+	}
+	var downloadResponse blob.DownloadStreamResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
-		downloadResponse, err = blob.Download(ctx, offset, count, ac, false, azblob.ClientProvidedKeyOptions{})
+		downloadResponse, err = blb.DownloadStream(ctx, &opt)
 		return o.fs.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to open for download: %w", err)
 	}
-	err = o.decodeMetaDataFromDownloadResponse(downloadResponse)
+	err = o.decodeMetaDataFromDownloadResponse(&downloadResponse)
 	if err != nil {
 		return nil, fmt.Errorf("failed to decode metadata for download: %w", err)
 	}
-	in = downloadResponse.Body(azblob.RetryReaderOptions{})
-	return in, nil
+	return downloadResponse.Body, nil
 }
 
 // dontEncode is the characters that do not need percent-encoding
@@ -1603,16 +1806,18 @@ func init() {
 	}
 }
 
-// increment the slice passed in as LSB binary
-func increment(xs []byte) {
-	for i, digit := range xs {
-		newDigit := digit + 1
-		xs[i] = newDigit
-		if newDigit >= digit {
-			// exit if no carry
-			break
+// urlEncode encodes in with % encoding
+func urlEncode(in string) string {
+	var out bytes.Buffer
+	for i := 0; i < len(in); i++ {
+		c := in[i]
+		if noNeedToEncode[c] {
+			_ = out.WriteByte(c)
+		} else {
+			_, _ = out.WriteString(fmt.Sprintf("%%%2X", c))
 		}
 	}
+	return out.String()
 }
 
 // poolWrapper wraps a pool.Pool as an azblob.TransferManager
@@ -1624,7 +1829,7 @@ type poolWrapper struct {
 
 // newPoolWrapper creates an azblob.TransferManager that will use a
 // pool.Pool with maximum concurrency as specified.
-func (f *Fs) newPoolWrapper(concurrency int) azblob.TransferManager {
+func (f *Fs) newPoolWrapper(concurrency int) *poolWrapper {
 	return &poolWrapper{
 		pool:     f.pool,
 		bufToken: make(chan struct{}, concurrency),
@@ -1657,11 +1862,253 @@ func (pw *poolWrapper) Run(f func()) {
 func (pw *poolWrapper) Close() {
 }
 
+// Converts a string into a pointer to a string
+func pString(s string) *string {
+	return &s
+}
+
+// readSeekCloser joins an io.Reader and an io.Seeker and provides a no-op io.Closer
+type readSeekCloser struct {
+	io.Reader
+	io.Seeker
+}
+
+// Close does nothing
+func (rs *readSeekCloser) Close() error {
+	return nil
+}
+
+// increment the slice passed in as LSB binary
+func increment(xs []byte) {
+	for i, digit := range xs {
+		newDigit := digit + 1
+		xs[i] = newDigit
+		if newDigit >= digit {
+			// exit if no carry
+			break
+		}
+	}
+}
+
+var warnStreamUpload sync.Once
+
+// uploadMultipart uploads a file using multipart upload
+//
+// Write a larger blob, using CreateBlockBlob, PutBlock, and PutBlockList.
+func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64, blb *blockblob.Client, httpHeaders *blob.HTTPHeaders) (err error) {
+	// Calculate correct partSize
+	partSize := o.fs.opt.ChunkSize
+	totalParts := -1
+
+	// make concurrency machinery
+	concurrency := o.fs.opt.UploadConcurrency
+	if concurrency < 1 {
+		concurrency = 1
+	}
+	tokens := pacer.NewTokenDispenser(concurrency)
+
+	// Note that the max size of file is 4.75 TB (100 MB X 50,000
+	// blocks) and this is bigger than the max uncommitted block
+	// size (9.52 TB) so we do not need to part commit block lists
+	// or garbage collect uncommitted blocks.
+	//
+	// See: https://docs.microsoft.com/en-gb/rest/api/storageservices/put-block
+
+	// size can be -1 here meaning we don't know the size of the incoming file.  We use ChunkSize
+	// buffers here (default 4MB). With a maximum number of parts (50,000) this will be a file of
+	// 195GB which seems like a not too unreasonable limit.
+	if size == -1 {
+		warnStreamUpload.Do(func() {
+			fs.Logf(o, "Streaming uploads using chunk size %v will have maximum file size of %v",
+				o.fs.opt.ChunkSize, partSize*fs.SizeSuffix(blockblob.MaxBlocks))
+		})
+	} else {
+		partSize = chunksize.Calculator(o, size, blockblob.MaxBlocks, o.fs.opt.ChunkSize)
+		if partSize > fs.SizeSuffix(blockblob.MaxStageBlockBytes) {
+			return fmt.Errorf("can't upload as it is too big %v - takes more than %d chunks of %v", fs.SizeSuffix(size), fs.SizeSuffix(blockblob.MaxBlocks), fs.SizeSuffix(blockblob.MaxStageBlockBytes))
+		}
+		totalParts = int(fs.SizeSuffix(size) / partSize)
+		if fs.SizeSuffix(size)%partSize != 0 {
+			totalParts++
+		}
+	}
+
+	fs.Debugf(o, "Multipart upload session started for %d parts of size %v", totalParts, partSize)
+
+	// unwrap the accounting from the input, we use wrap to put it
+	// back on after the buffering
+	in, wrap := accounting.UnWrap(in)
+
+	// FIXME it would be nice to delete uncommitted blocks
+	// See: https://github.com/rclone/rclone/issues/5583
+	//
+	// However there doesn't seem to be an easy way of doing this other than
+	// by deleting the target.
+	//
+	// This means that a failed upload deletes the target which isn't ideal.
+	//
+	// Uploading a zero length blob and deleting it will remove the
+	// uncommitted blocks I think.
+	//
+	// Could check to see if a file exists already and if it
+	// doesn't then create a 0 length file and delete it to flush
+	// the uncommitted blocks.
+	//
+	// This is what azcopy does
+	// https://github.com/MicrosoftDocs/azure-docs/issues/36347#issuecomment-541457962
+	// defer atexit.OnError(&err, func() {
+	// 	fs.Debugf(o, "Cancelling multipart upload")
+	//      // Code goes here!
+	// })()
+
+	// Upload the chunks
+	var (
+		g, gCtx       = errgroup.WithContext(ctx)
+		remaining     = fs.SizeSuffix(size)                 // remaining size in file for logging only, -1 if size < 0
+		position      = fs.SizeSuffix(0)                    // position in file
+		memPool       = o.fs.getMemoryPool(int64(partSize)) // pool to get memory from
+		finished      = false                               // set when we have read EOF
+		blocks        []string                              // list of blocks for finalize
+		binaryBlockID = make([]byte, 8)                     // block counter as LSB first 8 bytes
+	)
+	for part := 0; !finished; part++ {
+		// Get a block of memory from the pool and a token which limits concurrency
+		tokens.Get()
+		buf := memPool.Get()
+
+		free := func() {
+			memPool.Put(buf) // return the buf
+			tokens.Put()     // return the token
+		}
+
+		// Fail fast, in case an errgroup managed function returns an error
+		// gCtx is cancelled. There is no point in uploading all the other parts.
+		if gCtx.Err() != nil {
+			free()
+			break
+		}
+
+		// Read the chunk
+		n, err := readers.ReadFill(in, buf) // this can never return 0, nil
+		if err == io.EOF {
+			if n == 0 { // end if no data
+				free()
+				break
+			}
+			finished = true
+		} else if err != nil {
+			free()
+			return fmt.Errorf("multipart upload failed to read source: %w", err)
+		}
+		buf = buf[:n]
+
+		// increment the blockID and save the blocks for finalize
+		increment(binaryBlockID)
+		blockID := base64.StdEncoding.EncodeToString(binaryBlockID)
+		blocks = append(blocks, blockID)
+
+		// Transfer the chunk
+		fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %d", part+1, totalParts, position, fs.SizeSuffix(size), len(buf))
+		g.Go(func() (err error) {
+			defer free()
+
+			// Upload the block, with MD5 for check
+			md5sum := md5.Sum(buf)
+			transactionalMD5 := md5sum[:]
+			err = o.fs.pacer.Call(func() (bool, error) {
+				bufferReader := bytes.NewReader(buf)
+				wrappedReader := wrap(bufferReader)
+				rs := readSeekCloser{wrappedReader, bufferReader}
+				options := blockblob.StageBlockOptions{
+					// Specify the transactional md5 for the body, to be validated by the service.
+					TransactionalContentMD5: transactionalMD5,
+				}
+				_, err = blb.StageBlock(ctx, blockID, &rs, &options)
+				return o.fs.shouldRetry(ctx, err)
+			})
+			if err != nil {
+				return fmt.Errorf("multipart upload failed to upload part: %w", err)
+			}
+			return nil
+		})
+
+		// ready for next block
+		if size >= 0 {
+			remaining -= partSize
+		}
+		position += partSize
+	}
+	err = g.Wait()
+	if err != nil {
+		return err
+	}
+
+	tier := blob.AccessTier(o.fs.opt.AccessTier)
+	options := blockblob.CommitBlockListOptions{
+		Metadata:    o.meta,
+		Tier:        &tier,
+		HTTPHeaders: httpHeaders,
+	}
+
+	// Finalise the upload session
+	err = o.fs.pacer.Call(func() (bool, error) {
+		_, err := blb.CommitBlockList(ctx, blocks, &options)
+		return o.fs.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return fmt.Errorf("multipart upload failed to finalize: %w", err)
+	}
+	return nil
+}
+
+// uploadSinglepart uploads a short blob using a single part upload
+func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, blb *blockblob.Client, httpHeaders *blob.HTTPHeaders) (err error) {
+	// fs.Debugf(o, "Single part upload starting of object %d bytes", size)
+	if size > o.fs.poolSize || size < 0 {
+		return fmt.Errorf("internal error: single part upload size too big %d > %d", size, o.fs.opt.ChunkSize)
+	}
+
+	buf := o.fs.pool.Get()
+	defer o.fs.pool.Put(buf)
+
+	n, err := readers.ReadFill(in, buf)
+	if err == nil {
+		// Check to see whether in is exactly len(buf) or bigger
+		var buf2 = []byte{0}
+		n2, err2 := readers.ReadFill(in, buf2)
+		if n2 != 0 || err2 != io.EOF {
+			return fmt.Errorf("single part upload read failed: object longer than expected (expecting %d but got > %d)", size, len(buf))
+		}
+	}
+	if err != nil && err != io.EOF {
+		return fmt.Errorf("single part upload read failed: %w", err)
+	}
+	if int64(n) != size {
+		return fmt.Errorf("single part upload: expecting to read %d bytes but read %d", size, n)
+	}
+
+	b := bytes.NewReader(buf[:n])
+	rs := &readSeekCloser{Reader: b, Seeker: b}
+
+	tier := blob.AccessTier(o.fs.opt.AccessTier)
+	options := blockblob.UploadOptions{
+		Metadata:    o.meta,
+		Tier:        &tier,
+		HTTPHeaders: httpHeaders,
+	}
+
+	// Don't retry, return a retry error instead
+	return o.fs.pacer.CallNoRetry(func() (bool, error) {
+		_, err = blb.Upload(ctx, rs, &options)
+		return o.fs.shouldRetry(ctx, err)
+	})
+}
+
 // Update the object with the contents of the io.Reader, modTime and size
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
-	if o.accessTier == azblob.AccessTierArchive {
+	if o.accessTier == blob.AccessTierArchive {
 		if o.fs.opt.ArchiveTierDelete {
 			fs.Debugf(o, "deleting archive tier blob before updating")
 			err = o.Remove(ctx)
@@ -1687,9 +2134,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	blob := o.getBlobReference()
-	httpHeaders := azblob.BlobHTTPHeaders{}
-	httpHeaders.ContentType = fs.MimeType(ctx, src)
+	// Create the HTTP headers for the upload
+	httpHeaders := blob.HTTPHeaders{
+		BlobContentType: pString(fs.MimeType(ctx, src)),
+	}
 
 	// Compute the Content-MD5 of the file. As we stream all uploads it
 	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
@@ -1697,7 +2145,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if sourceMD5, _ := src.Hash(ctx, hash.MD5); sourceMD5 != "" {
 			sourceMD5bytes, err := hex.DecodeString(sourceMD5)
 			if err == nil {
-				httpHeaders.ContentMD5 = sourceMD5bytes
+				httpHeaders.BlobContentMD5 = sourceMD5bytes
 			} else {
 				fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
 			}
@@ -1712,45 +2160,31 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		case "":
 			// ignore
 		case "cache-control":
-			httpHeaders.CacheControl = value
+			httpHeaders.BlobCacheControl = pString(value)
 		case "content-disposition":
-			httpHeaders.ContentDisposition = value
+			httpHeaders.BlobContentDisposition = pString(value)
 		case "content-encoding":
-			httpHeaders.ContentEncoding = value
+			httpHeaders.BlobContentEncoding = pString(value)
 		case "content-language":
-			httpHeaders.ContentLanguage = value
+			httpHeaders.BlobContentLanguage = pString(value)
 		case "content-type":
-			httpHeaders.ContentType = value
+			httpHeaders.BlobContentType = pString(value)
 		}
 	}
 
-	uploadParts := maxUploadParts
-	if uploadParts < 1 {
-		uploadParts = 1
-	} else if uploadParts > maxUploadParts {
-		uploadParts = maxUploadParts
-	}
-	// calculate size of parts/blocks
-	partSize := chunksize.Calculator(o, src.Size(), uploadParts, o.fs.opt.ChunkSize)
+	blb := o.fs.getBlockBlobSVC(container, containerPath)
+	size := src.Size()
+	multipartUpload := size < 0 || size > o.fs.poolSize
 
-	putBlobOptions := azblob.UploadStreamToBlockBlobOptions{
-		BufferSize:      int(partSize),
-		MaxBuffers:      o.fs.opt.UploadConcurrency,
-		Metadata:        o.meta,
-		BlobHTTPHeaders: httpHeaders,
-		TransferManager: o.fs.newPoolWrapper(o.fs.opt.UploadConcurrency),
+	if multipartUpload {
+		err = o.uploadMultipart(ctx, in, size, blb, &httpHeaders)
+	} else {
+		err = o.uploadSinglepart(ctx, in, size, blb, &httpHeaders)
 	}
-
-	// Don't retry, return a retry error instead
-	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		// Stream contents of the reader object to the given blob URL
-		blockBlobURL := blob.ToBlockBlobURL()
-		_, err = azblob.UploadStreamToBlockBlob(ctx, in, blockBlobURL, putBlobOptions)
-		return o.fs.shouldRetry(ctx, err)
-	})
 	if err != nil {
 		return err
 	}
+
 	// Refresh metadata on object
 	o.clearMetaData()
 	err = o.readMetaData()
@@ -1769,11 +2203,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
-	blob := o.getBlobReference()
-	snapShotOptions := azblob.DeleteSnapshotsOptionNone
-	ac := azblob.BlobAccessConditions{}
+	blb := o.getBlobSVC()
+	//only := blob.DeleteSnapshotsOptionTypeOnly
+	opt := blob.DeleteOptions{
+		//DeleteSnapshots: &only,
+	}
 	return o.fs.pacer.Call(func() (bool, error) {
-		_, err := blob.Delete(ctx, snapShotOptions, ac)
+		_, err := blb.Delete(ctx, &opt)
 		return o.fs.shouldRetry(ctx, err)
 	})
 }
@@ -1784,7 +2220,7 @@ func (o *Object) MimeType(ctx context.Context) string {
 }
 
 // AccessTier of an object, default is of type none
-func (o *Object) AccessTier() azblob.AccessTierType {
+func (o *Object) AccessTier() blob.AccessTier {
 	return o.accessTier
 }
 
@@ -1798,11 +2234,15 @@ func (o *Object) SetTier(tier string) error {
 	if o.GetTier() == tier {
 		return nil
 	}
-	desiredAccessTier := azblob.AccessTierType(tier)
-	blob := o.getBlobReference()
+	desiredAccessTier := blob.AccessTier(tier)
+	blb := o.getBlobSVC()
 	ctx := context.Background()
+	priority := blob.RehydratePriorityStandard
+	opt := blob.SetTierOptions{
+		RehydratePriority: &priority,
+	}
 	err := o.fs.pacer.Call(func() (bool, error) {
-		_, err := blob.SetTier(ctx, desiredAccessTier, azblob.LeaseAccessConditions{}, azblob.RehydratePriorityStandard)
+		_, err := blb.SetTier(ctx, desiredAccessTier, &opt)
 		return o.fs.shouldRetry(ctx, err)
 	})
 
diff --git a/backend/azureblob/azureblob_internal_test.go b/backend/azureblob/azureblob_internal_test.go
index 827653a80847a..daa3b811482c3 100644
--- a/backend/azureblob/azureblob_internal_test.go
+++ b/backend/azureblob/azureblob_internal_test.go
@@ -1,5 +1,5 @@
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
+//go:build !plan9 && !solaris && !js && go1.18
+// +build !plan9,!solaris,!js,go1.18
 
 package azureblob
 
diff --git a/backend/azureblob/azureblob_test.go b/backend/azureblob/azureblob_test.go
index f12bb177a1c4d..fba31da8452a5 100644
--- a/backend/azureblob/azureblob_test.go
+++ b/backend/azureblob/azureblob_test.go
@@ -1,13 +1,11 @@
 // Test AzureBlob filesystem interface
 
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
+//go:build !plan9 && !solaris && !js && go1.18
+// +build !plan9,!solaris,!js,go1.18
 
 package azureblob
 
 import (
-	"context"
-	"encoding/json"
 	"testing"
 
 	"github.com/rclone/rclone/fs"
@@ -18,10 +16,12 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:    "TestAzureBlob:",
-		NilObject:     (*Object)(nil),
-		TiersToTest:   []string{"Hot", "Cool"},
-		ChunkedUpload: fstests.ChunkedUploadConfig{},
+		RemoteName:  "TestAzureBlob:",
+		NilObject:   (*Object)(nil),
+		TiersToTest: []string{"Hot", "Cool"},
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: defaultChunkSize,
+		},
 	})
 }
 
@@ -33,44 +33,6 @@ var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
 )
 
-// TestServicePrincipalFileSuccess checks that, given a proper JSON file, we can create a token.
-func TestServicePrincipalFileSuccess(t *testing.T) {
-	ctx := context.TODO()
-	credentials := `
-{
-    "appId": "my application (client) ID",
-    "password": "my secret",
-    "tenant": "my active directory tenant ID"
-}
-`
-	var spCredentials servicePrincipalCredentials
-	jerr := json.Unmarshal([]byte(credentials), &spCredentials)
-	assert.Nil(t, jerr)
-
-	tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, spCredentials)
-	if assert.NoError(t, err) {
-		assert.NotNil(t, tokenRefresher)
-	}
-}
-
-// TestServicePrincipalFileFailure checks that, given a JSON file with a missing secret, it returns an error.
-func TestServicePrincipalFileFailure(t *testing.T) {
-	ctx := context.TODO()
-	credentials := `
-{
-    "appId": "my application (client) ID",
-    "tenant": "my active directory tenant ID"
-}
-`
-	var spCredentials servicePrincipalCredentials
-	jerr := json.Unmarshal([]byte(credentials), &spCredentials)
-	assert.Nil(t, jerr)
-
-	_, err := newServicePrincipalTokenRefresher(ctx, spCredentials)
-	assert.Error(t, err)
-	assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
-}
-
 func TestValidateAccessTier(t *testing.T) {
 	tests := map[string]struct {
 		accessTier string
diff --git a/backend/azureblob/azureblob_unsupported.go b/backend/azureblob/azureblob_unsupported.go
index 369d6e367bace..08d5c40630aaa 100644
--- a/backend/azureblob/azureblob_unsupported.go
+++ b/backend/azureblob/azureblob_unsupported.go
@@ -1,7 +1,7 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-//go:build plan9 || solaris || js
-// +build plan9 solaris js
+//go:build plan9 || solaris || js || !go1.18
+// +build plan9 solaris js !go1.18
 
 package azureblob
diff --git a/backend/azureblob/imds.go b/backend/azureblob/imds.go
deleted file mode 100644
index 889e224dba707..0000000000000
--- a/backend/azureblob/imds.go
+++ /dev/null
@@ -1,136 +0,0 @@
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
-
-package azureblob
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-
-	"github.com/Azure/go-autorest/autorest/adal"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fshttp"
-)
-
-const (
-	azureResource      = "https://storage.azure.com"
-	imdsAPIVersion     = "2018-02-01"
-	msiEndpointDefault = "http://169.254.169.254/metadata/identity/oauth2/token"
-)
-
-// This custom type is used to add the port the test server has bound to
-// to the request context.
-type testPortKey string
-
-type msiIdentifierType int
-
-const (
-	msiClientID msiIdentifierType = iota
-	msiObjectID
-	msiResourceID
-)
-
-type userMSI struct {
-	Type  msiIdentifierType
-	Value string
-}
-
-type httpError struct {
-	Response *http.Response
-}
-
-func (e httpError) Error() string {
-	return fmt.Sprintf("HTTP error %v (%v)", e.Response.StatusCode, e.Response.Status)
-}
-
-// GetMSIToken attempts to obtain an MSI token from the Azure Instance
-// Metadata Service.
-func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
-	// Attempt to get an MSI token; silently continue if unsuccessful.
-	// This code has been lovingly stolen from azcopy's OAuthTokenManager.
-	result := adal.Token{}
-	req, err := http.NewRequestWithContext(ctx, "GET", msiEndpointDefault, nil)
-	if err != nil {
-		fs.Debugf(nil, "Failed to create request: %v", err)
-		return result, err
-	}
-	params := req.URL.Query()
-	params.Set("resource", azureResource)
-	params.Set("api-version", imdsAPIVersion)
-
-	// Specify user-assigned identity if requested.
-	if identity != nil {
-		switch identity.Type {
-		case msiClientID:
-			params.Set("client_id", identity.Value)
-		case msiObjectID:
-			params.Set("object_id", identity.Value)
-		case msiResourceID:
-			params.Set("mi_res_id", identity.Value)
-		default:
-			// If this happens, the calling function and this one don't agree on
-			// what valid ID types exist.
-			return result, fmt.Errorf("unknown MSI identity type specified")
-		}
-	}
-	req.URL.RawQuery = params.Encode()
-
-	// The Metadata header is required by all calls to IMDS.
-	req.Header.Set("Metadata", "true")
-
-	// If this function is run in a test, query the test server instead of IMDS.
-	testPort, isTest := ctx.Value(testPortKey("testPort")).(int)
-	if isTest {
-		req.URL.Host = fmt.Sprintf("localhost:%d", testPort)
-		req.Host = req.URL.Host
-	}
-
-	// Send request
-	httpClient := fshttp.NewClient(ctx)
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return result, fmt.Errorf("MSI is not enabled on this VM: %w", err)
-	}
-	defer func() { // resp and Body should not be nil
-		_, err = io.Copy(io.Discard, resp.Body)
-		if err != nil {
-			fs.Debugf(nil, "Unable to drain IMDS response: %v", err)
-		}
-		err = resp.Body.Close()
-		if err != nil {
-			fs.Debugf(nil, "Unable to close IMDS response: %v", err)
-		}
-	}()
-	// Check if the status code indicates success
-	// The request returns 200 currently, add 201 and 202 as well for possible extension.
-	switch resp.StatusCode {
-	case 200, 201, 202:
-		break
-	default:
-		body, _ := io.ReadAll(resp.Body)
-		fs.Errorf(nil, "Couldn't obtain OAuth token from IMDS; server returned status code %d and body: %v", resp.StatusCode, string(body))
-		return result, httpError{Response: resp}
-	}
-
-	b, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return result, fmt.Errorf("couldn't read IMDS response: %w", err)
-	}
-	// Remove BOM, if any. azcopy does this so I'm following along.
-	b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf"))
-
-	// This would be a good place to persist the token if a large number of rclone
-	// invocations are being made in a short amount of time. If the token is
-	// persisted, the azureblob code will need to check for expiry before every
-	// storage API call.
-	err = json.Unmarshal(b, &result)
-	if err != nil {
-		return result, fmt.Errorf("couldn't unmarshal IMDS response: %w", err)
-	}
-
-	return result, nil
-}
diff --git a/backend/azureblob/imds_test.go b/backend/azureblob/imds_test.go
deleted file mode 100644
index c449b545c6329..0000000000000
--- a/backend/azureblob/imds_test.go
+++ /dev/null
@@ -1,118 +0,0 @@
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
-
-package azureblob
-
-import (
-	"context"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/Azure/go-autorest/autorest/adal"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func handler(t *testing.T, actual *map[string]string) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		err := r.ParseForm()
-		require.NoError(t, err)
-		parameters := r.URL.Query()
-		(*actual)["path"] = r.URL.Path
-		(*actual)["Metadata"] = r.Header.Get("Metadata")
-		(*actual)["method"] = r.Method
-		for paramName := range parameters {
-			(*actual)[paramName] = parameters.Get(paramName)
-		}
-		// Make response.
-		response := adal.Token{}
-		responseBytes, err := json.Marshal(response)
-		require.NoError(t, err)
-		_, err = w.Write(responseBytes)
-		require.NoError(t, err)
-	}
-}
-
-func TestManagedIdentity(t *testing.T) {
-	// test user-assigned identity specifiers to use
-	testMSIClientID := "d859b29f-5c9c-42f8-a327-ec1bc6408d79"
-	testMSIObjectID := "9ffeb650-3ca0-4278-962b-5a38d520591a"
-	testMSIResourceID := "/subscriptions/fe714c49-b8a4-4d49-9388-96a20daa318f/resourceGroups/somerg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/someidentity"
-	tests := []struct {
-		identity              *userMSI
-		identityParameterName string
-		expectedAbsent        []string
-	}{
-		{&userMSI{msiClientID, testMSIClientID}, "client_id", []string{"object_id", "mi_res_id"}},
-		{&userMSI{msiObjectID, testMSIObjectID}, "object_id", []string{"client_id", "mi_res_id"}},
-		{&userMSI{msiResourceID, testMSIResourceID}, "mi_res_id", []string{"object_id", "client_id"}},
-		{nil, "(default)", []string{"object_id", "client_id", "mi_res_id"}},
-	}
-	alwaysExpected := map[string]string{
-		"path":        "/metadata/identity/oauth2/token",
-		"resource":    "https://storage.azure.com",
-		"Metadata":    "true",
-		"api-version": "2018-02-01",
-		"method":      "GET",
-	}
-	for _, test := range tests {
-		actual := make(map[string]string, 10)
-		testServer := httptest.NewServer(handler(t, &actual))
-		defer testServer.Close()
-		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
-		require.NoError(t, err)
-		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
-		_, err = GetMSIToken(ctx, test.identity)
-		require.NoError(t, err)
-
-		// Validate expected query parameters present
-		expected := make(map[string]string)
-		for k, v := range alwaysExpected {
-			expected[k] = v
-		}
-		if test.identity != nil {
-			expected[test.identityParameterName] = test.identity.Value
-		}
-
-		for key := range expected {
-			value, exists := actual[key]
-			if assert.Truef(t, exists, "test of %s: query parameter %s was not passed",
-				test.identityParameterName, key) {
-				assert.Equalf(t, expected[key], value,
-					"test of %s: parameter %s has incorrect value", test.identityParameterName, key)
-			}
-		}
-
-		// Validate unexpected query parameters absent
-		for _, key := range test.expectedAbsent {
-			_, exists := actual[key]
-			assert.Falsef(t, exists, "query parameter %s was unexpectedly passed")
-		}
-	}
-}
-
-func errorHandler(resultCode int) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		http.Error(w, "Test error generated", resultCode)
-	}
-}
-
-func TestIMDSErrors(t *testing.T) {
-	errorCodes := []int{404, 429, 500}
-	for _, code := range errorCodes {
-		testServer := httptest.NewServer(errorHandler(code))
-		defer testServer.Close()
-		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
-		require.NoError(t, err)
-		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
-		_, err = GetMSIToken(ctx, nil)
-		require.Error(t, err)
-		httpErr, ok := err.(httpError)
-		require.Truef(t, ok, "HTTP error %d did not result in an httpError object", code)
-		assert.Equalf(t, httpErr.Response.StatusCode, code, "desired error %d but didn't get it", code)
-	}
-}
diff --git a/go.mod b/go.mod
index 3f7055de4366c..1f9225d3b0d8e 100644
--- a/go.mod
+++ b/go.mod
@@ -4,9 +4,9 @@ go 1.17
 
 require (
 	bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05
-	github.com/Azure/azure-pipeline-go v0.2.3
-	github.com/Azure/azure-storage-blob-go v0.15.0
-	github.com/Azure/go-autorest/autorest/adal v0.9.21
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1
 	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e
 	github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5
 	github.com/Unknwon/goconfig v1.0.0
@@ -76,10 +76,8 @@ require (
 require (
 	cloud.google.com/go/compute v1.12.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.1 // indirect
-	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
-	github.com/Azure/go-autorest/logger v0.2.1 // indirect
-	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect
 	github.com/anacrolix/log v0.13.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
@@ -106,13 +104,14 @@ require (
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kr/fs v0.1.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
-	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/onsi/gomega v1.13.0 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
+	github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
@@ -131,6 +130,7 @@ require (
 	google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect
 	google.golang.org/grpc v1.50.1 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	storj.io/common v0.0.0-20220414110316-a5cb7172d6bf // indirect
 	storj.io/drpc v0.0.30 // indirect
@@ -139,7 +139,7 @@ require (
 require (
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4
-	github.com/golang-jwt/jwt/v4 v4.1.0 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/pkg/xattr v0.4.9
diff --git a/go.sum b/go.sum
index 516d8cba7c88d..2e41116c1e6e1 100644
--- a/go.sum
+++ b/go.sum
@@ -283,25 +283,22 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr
 dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U=
-github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
-github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk=
-github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58=
-github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
-github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk=
-github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U=
-github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
-github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU=
 github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU=
 github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
+github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
+github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM=
+github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
@@ -390,6 +387,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c=
+github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00 h1:xJBhC00smQpSZw3Kr0ErMUBXhUSjYoLRm2szxdbRBL0=
 github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00/go.mod h1:nNICngOdmNImBb/vuL+dSc0aIg3ryNATpjxThNoPw4g=
@@ -410,7 +409,6 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
 github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@@ -451,9 +449,11 @@ github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14j
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
-github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
+github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
+github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -535,8 +535,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20211108044417-e9b028704de0 h1:rsq1yB2xiFLDYYaYdlGBsSkwVzsCo500wMhxvW5A/bk=
 github.com/google/pprof v0.0.0-20211108044417-e9b028704de0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
@@ -649,8 +649,9 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
@@ -665,8 +666,6 @@ github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz
 github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
-github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
 github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
@@ -685,6 +684,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
+github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -719,10 +720,11 @@ github.com/pborman/getopt v1.1.0/go.mod h1:FxXoW1Re00sQG/+KIkuSqRL/LwQgSkv7uyac+
 github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU=
 github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
+github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
 github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
@@ -919,12 +921,11 @@ golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A=
@@ -989,7 +990,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -1005,6 +1005,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -1016,7 +1017,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
@@ -1102,7 +1102,6 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1519,6 +1518,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=

From f746b2fe85837c86fed49893e6e86a23f3a17dff Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 29 Nov 2022 15:43:22 +0000
Subject: [PATCH 426/560] azureblob: port old authentication methods to new SDK

Co-authored-by: Brad Ackerman 
---
 backend/azureblob/azureblob.go | 412 ++++++++++++++-------------------
 1 file changed, 178 insertions(+), 234 deletions(-)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 7268dfe231bd6..3631474e5e912 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -36,11 +36,14 @@ import (
 	"crypto/md5"
 	"encoding/base64"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -48,11 +51,13 @@ import (
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
@@ -66,6 +71,7 @@ import (
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/readers"
@@ -104,19 +110,12 @@ func init() {
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name: "account",
-			Help: "Storage Account Name.\n\nLeave blank to use SAS URL or Emulator.",
-		}, {
-			Name: "service_principal_file",
-			Help: `Path to file containing credentials for use with a service principal.
+			Help: `Storage Account Name.
 
-Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
-
-    $ az ad sp create-for-rbac --name "" \
-      --role "Storage Blob Data Owner" \
-      --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \
-      > azure-principal.json
+Leave blank to use SAS URL or Emulator.
 
-See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
+If this is blank then if env_auth is set it will be read from the
+environment variable AZURE_CLIENT_ID.
 `,
 		}, {
 			Name: "env_auth",
@@ -129,10 +128,28 @@ Other authentication methods will, if specified, override this flag.`,
 			Default: false,
 		}, {
 			Name: "key",
-			Help: "Storage Account Key.\n\nLeave blank to use SAS URL or Emulator.",
+			Help: `Storage Account Shared Key.
+
+Leave blank to use SAS URL or Emulator.`,
 		}, {
 			Name: "sas_url",
-			Help: "SAS URL for container level access only.\n\nLeave blank if using account/key or Emulator.",
+			Help: `SAS URL for container level access only.
+
+Leave blank if using account/key or Emulator.`,
+		}, {
+			Name: "service_principal_file",
+			Help: `Path to file containing credentials for use with a service principal.
+
+Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
+
+    $ az ad sp create-for-rbac --name "" \
+      --role "Storage Blob Data Owner" \
+      --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \
+      > azure-principal.json
+
+See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
+`,
+			Advanced: true,
 		}, {
 			Name: "use_msi",
 			Help: `Use a managed service identity to authenticate (only works in Azure).
@@ -145,7 +162,8 @@ be used by default. If the resource has no system-assigned but exactly one user-
 the user-assigned identity will be used by default. If the resource has multiple user-assigned
 identities, the identity to use must be explicitly specified using exactly one of the msi_object_id,
 msi_client_id, or msi_mi_res_id parameters.`,
-			Default: false,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     "msi_object_id",
 			Help:     "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_mi_res_id specified.",
@@ -159,9 +177,10 @@ msi_client_id, or msi_mi_res_id parameters.`,
 			Help:     "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.",
 			Advanced: true,
 		}, {
-			Name:    "use_emulator",
-			Help:    "Uses local storage emulator if provided as 'true'.\n\nLeave blank if using real azure storage endpoint.",
-			Default: false,
+			Name:     "use_emulator",
+			Help:     "Uses local storage emulator if provided as 'true'.\n\nLeave blank if using real azure storage endpoint.",
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for the service.\n\nLeave blank normally.",
@@ -307,9 +326,9 @@ This option controls how often unused buffers will be removed from the pool.`,
 // Options defines the configuration for this backend
 type Options struct {
 	Account              string               `config:"account"`
-	ServicePrincipalFile string               `config:"service_principal_file"`
 	EnvAuth              bool                 `config:"env_auth"`
 	Key                  string               `config:"key"`
+	ServicePrincipalFile string               `config:"service_principal_file"`
 	UseMSI               bool                 `config:"use_msi"`
 	MSIObjectID          string               `config:"msi_object_id"`
 	MSIClientID          string               `config:"msi_client_id"`
@@ -485,6 +504,20 @@ type servicePrincipalCredentials struct {
 	Tenant   string `json:"tenant"`
 }
 
+// parseServicePrincipalCredentials unmarshals a service principal credentials JSON file as generated by az cli.
+func parseServicePrincipalCredentials(ctx context.Context, credentialsData []byte) (*servicePrincipalCredentials, error) {
+	var spCredentials servicePrincipalCredentials
+	if err := json.Unmarshal(credentialsData, &spCredentials); err != nil {
+		return nil, fmt.Errorf("error parsing credentials from JSON file: %w", err)
+	}
+	// TODO: support certificate credentials
+	// Validate all fields present
+	if spCredentials.AppID == "" || spCredentials.Password == "" || spCredentials.Tenant == "" {
+		return nil, fmt.Errorf("missing fields in credentials file")
+	}
+	return &spCredentials, nil
+}
+
 // setRoot changes the root of the Fs
 func (f *Fs) setRoot(root string) {
 	f.root = parsePath(root)
@@ -524,9 +557,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.ListChunkSize > maxListChunkSize {
 		return nil, fmt.Errorf("blob list size can't be greater than %v - was %v", maxListChunkSize, opt.ListChunkSize)
 	}
-	if opt.Endpoint == "" {
-		opt.Endpoint = storageDefaultBaseURL
-	}
 
 	if opt.AccessTier == "" {
 		opt.AccessTier = string(defaultAccessTier)
@@ -568,236 +598,150 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		GetTier:           true,
 	}).Fill(ctx, f)
 
-	// var (
-	// 	u          *url.URL
-	// 	serviceURL azblob.ServiceURL
-	// )
-	// switch {
-	// case opt.UseEmulator:
-	// 	var actualEmulatorAccount = emulatorAccount
-	// 	if opt.Account != "" {
-	// 		actualEmulatorAccount = opt.Account
-	// 	}
-	// 	var actualEmulatorKey = emulatorAccountKey
-	// 	if opt.Key != "" {
-	// 		actualEmulatorKey = opt.Key
-	// 	}
-	// 	credential, err := azblob.NewSharedKeyCredential(actualEmulatorAccount, actualEmulatorKey)
-	// 	if err != nil {
-	// 		return nil, fmt.Errorf("failed to parse credentials: %w", err)
-	// 	}
-	// 	var actualEmulatorEndpoint = emulatorBlobEndpoint
-	// 	if opt.Endpoint != "" {
-	// 		actualEmulatorEndpoint = opt.Endpoint
-	// 	}
-	// 	u, err = url.Parse(actualEmulatorEndpoint)
-	// 	if err != nil {
-	// 		return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
-	// 	}
-	// 	pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
-	// 	serviceURL = azblob.NewServiceURL(*u, pipeline)
-	// case opt.UseMSI:
-	// 	var token adal.Token
-	// 	var userMSI = &userMSI{}
-	// 	if len(opt.MSIClientID) > 0 || len(opt.MSIObjectID) > 0 || len(opt.MSIResourceID) > 0 {
-	// 		// Specifying a user-assigned identity. Exactly one of the above IDs must be specified.
-	// 		// Validate and ensure exactly one is set. (To do: better validation.)
-	// 		if len(opt.MSIClientID) > 0 {
-	// 			if len(opt.MSIObjectID) > 0 || len(opt.MSIResourceID) > 0 {
-	// 				return nil, errors.New("more than one user-assigned identity ID is set")
-	// 			}
-	// 			userMSI.Type = msiClientID
-	// 			userMSI.Value = opt.MSIClientID
-	// 		}
-	// 		if len(opt.MSIObjectID) > 0 {
-	// 			if len(opt.MSIClientID) > 0 || len(opt.MSIResourceID) > 0 {
-	// 				return nil, errors.New("more than one user-assigned identity ID is set")
-	// 			}
-	// 			userMSI.Type = msiObjectID
-	// 			userMSI.Value = opt.MSIObjectID
-	// 		}
-	// 		if len(opt.MSIResourceID) > 0 {
-	// 			if len(opt.MSIClientID) > 0 || len(opt.MSIObjectID) > 0 {
-	// 				return nil, errors.New("more than one user-assigned identity ID is set")
-	// 			}
-	// 			userMSI.Type = msiResourceID
-	// 			userMSI.Value = opt.MSIResourceID
-	// 		}
-	// 	} else {
-	// 		userMSI = nil
-	// 	}
-	// 	err = f.imdsPacer.Call(func() (bool, error) {
-	// 		// Retry as specified by the documentation:
-	// 		// https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#retry-guidance
-	// 		token, err = GetMSIToken(ctx, userMSI)
-	// 		return f.shouldRetry(ctx, err)
-	// 	})
-
-	// 	if err != nil {
-	// 		return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
-	// 	}
-
-	// 	u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
-	// 	if err != nil {
-	// 		return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
-	// 	}
-	// 	credential := azblob.NewTokenCredential(token.AccessToken, func(credential azblob.TokenCredential) time.Duration {
-	// 		fs.Debugf(f, "Token refresher called.")
-	// 		var refreshedToken adal.Token
-	// 		err := f.imdsPacer.Call(func() (bool, error) {
-	// 			refreshedToken, err = GetMSIToken(ctx, userMSI)
-	// 			return f.shouldRetry(ctx, err)
-	// 		})
-	// 		if err != nil {
-	// 			// Failed to refresh.
-	// 			return 0
-	// 		}
-	// 		credential.SetToken(refreshedToken.AccessToken)
-	// 		now := time.Now().UTC()
-	// 		// Refresh one minute before expiry.
-	// 		refreshAt := refreshedToken.Expires().UTC().Add(-1 * time.Minute)
-	// 		fs.Debugf(f, "Acquired new token that expires at %v; refreshing in %d s", refreshedToken.Expires(),
-	// 			int(refreshAt.Sub(now).Seconds()))
-	// 		if now.After(refreshAt) {
-	// 			// Acquired a causality violation.
-	// 			return 0
-	// 		}
-	// 		return refreshAt.Sub(now)
-	// 	})
-	// 	pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
-	// 	serviceURL = azblob.NewServiceURL(*u, pipeline)
-	// case opt.Account != "" && opt.Key != "":
-	// 	credential, err := azblob.NewSharedKeyCredential(opt.Account, opt.Key)
-	// 	if err != nil {
-	// 		return nil, fmt.Errorf("failed to parse credentials: %w", err)
-	// 	}
-
-	// 	u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
-	// 	if err != nil {
-	// 		return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
-	// 	}
-	// 	pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
-	// 	serviceURL = azblob.NewServiceURL(*u, pipeline)
-	// case opt.SASURL != "":
-	// 	u, err = url.Parse(opt.SASURL)
-	// 	if err != nil {
-	// 		return nil, fmt.Errorf("failed to parse SAS URL: %w", err)
-	// 	}
-	// 	// use anonymous credentials in case of sas url
-	// 	pipeline := f.newPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
-	// 	// Check if we have container level SAS or account level sas
-	// 	parts := azblob.NewBlobURLParts(*u)
-	// 	if parts.ContainerName != "" {
-	// 		if f.rootContainer != "" && parts.ContainerName != f.rootContainer {
-	// 			return nil, errors.New("container name in SAS URL and container provided in command do not match")
-	// 		}
-	// 		containerURL := azblob.NewContainerURL(*u, pipeline)
-	// 		f.cntSVCcache[parts.ContainerName] = &containerURL
-	// 		f.isLimited = true
-	// 	} else {
-	// 		serviceURL = azblob.NewServiceURL(*u, pipeline)
-	// 	}
-	// case opt.ServicePrincipalFile != "":
-	// 	// Create a standard URL.
-	// 	u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
-	// 	if err != nil {
-	// 		return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
-	// 	}
-	// 	// Try loading service principal credentials from file.
-	// 	loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
-	// 	if err != nil {
-	// 		return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
-	// 	}
-	// 	// Create a token refresher from service principal credentials.
-	// 	tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, loadedCreds)
-	// 	if err != nil {
-	// 		return nil, fmt.Errorf("failed to create a service principal token: %w", err)
-	// 	}
-	// 	options := azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}}
-	// 	pipe := f.newPipeline(azblob.NewTokenCredential("", tokenRefresher), options)
-	// 	serviceURL = azblob.NewServiceURL(*u, pipe)
-	// default:
-	// 	return nil, errors.New("no authentication method configured")
-	// }
-	//f.svcURL = &serviceURL
-
-	u, err := url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint))
-	if err != nil {
-		return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
+	// Client options specifying our own transport
+	policyClientOptions := policy.ClientOptions{
+		Transport: newTransporter(ctx),
+	}
+	clientOpt := service.ClientOptions{
+		ClientOptions: policyClientOptions,
 	}
-	serviceURL := u.String()
-	fs.Debugf(f, "Service URL = %q", serviceURL)
-
-	// FIXME Very quick and dirty auth
 
+	// Here we auth by setting one of cred, sharedKeyCred or f.svc
 	var (
 		cred          azcore.TokenCredential
 		sharedKeyCred *service.SharedKeyCredential
 	)
-
-	if opt.EnvAuth {
+	switch {
+	case opt.EnvAuth:
+		// Read account from environment if needed
+		if opt.Account == "" {
+			opt.Account, _ = os.LookupEnv("AZURE_CLIENT_ID")
+		}
 		// Read credentials from the environment
-		cred, err = azidentity.NewDefaultAzureCredential(nil)
+		options := azidentity.DefaultAzureCredentialOptions{
+			ClientOptions: policyClientOptions,
+		}
+		cred, err = azidentity.NewDefaultAzureCredential(&options)
 		if err != nil {
-			return nil, fmt.Errorf("create default azure credential failed: %w", err)
+			return nil, fmt.Errorf("create azure enviroment credential failed: %w", err)
+		}
+	case opt.UseEmulator:
+		if opt.Account != "" {
+			opt.Account = emulatorAccount
+		}
+		if opt.Key == "" {
+			opt.Key = emulatorAccountKey
+		}
+		if opt.Endpoint != "" {
+			opt.Endpoint = emulatorBlobEndpoint
 		}
-	} else {
-
-		// Use the config file to configure
+		sharedKeyCred, err = service.NewSharedKeyCredential(opt.Account, opt.Key)
+		if err != nil {
+			return nil, fmt.Errorf("create new shared key credential for emulator failed: %w", err)
+		}
+	case opt.UseMSI:
+		// Specifying a user-assigned identity. Exactly one of the above IDs must be specified.
+		// Validate and ensure exactly one is set. (To do: better validation.)
+		var b2i = map[bool]int{false: 0, true: 1}
+		set := b2i[opt.MSIClientID != ""] + b2i[opt.MSIObjectID != ""] + b2i[opt.MSIResourceID != ""]
+		if set > 1 {
+			return nil, errors.New("more than one user-assigned identity ID is set")
+		}
+		var options azidentity.ManagedIdentityCredentialOptions
+		switch {
+		case opt.MSIClientID != "":
+			options.ID = azidentity.ClientID(opt.MSIClientID)
+		case opt.MSIObjectID != "":
+			// FIXME this doesn't appear to be in the new SDK?
+			return nil, fmt.Errorf("MSI object ID is currently unsupported")
+		case opt.MSIResourceID != "":
+			options.ID = azidentity.ResourceID(opt.MSIResourceID)
+		}
+		cred, err = azidentity.NewManagedIdentityCredential(&options)
+		if err != nil {
+			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
+		}
+	case opt.Account != "" && opt.Key != "":
 		sharedKeyCred, err = service.NewSharedKeyCredential(opt.Account, opt.Key)
 		if err != nil {
 			return nil, fmt.Errorf("create new shared key credential failed: %w", err)
 		}
-	}
-
-	// Specify our own transport
-	clientOpt := service.ClientOptions{
-		ClientOptions: azcore.ClientOptions{
-			Transport: newTransporter(ctx),
-		},
-	}
-	// azClientOpt := azblob.ClientOptions{
-	// 	ClientOptions: azcore.ClientOptions{
-	// 		Transport: clientOpt.ClientOptions.Transport,
-	// 	},
-	// }
-
-	if sharedKeyCred != nil {
-		// create a client for the specified storage account
-		client, err := service.NewClientWithSharedKeyCredential(serviceURL, sharedKeyCred, &clientOpt)
+	case opt.SASURL != "":
+		parts, err := sas.ParseURL(opt.SASURL)
 		if err != nil {
-			return nil, fmt.Errorf("create client with shared key failed: %w", err)
+			return nil, fmt.Errorf("failed to parse SAS URL: %w", err)
 		}
-		f.svc = client
-
-		// create a client for the specified storage account
-		//
-		// Annoyingly this is the same type as f.svc just wrapped in a
-		// struct, but there is no way to create one from the other.
-		// azsvc, err := azblob.NewClientWithSharedKeyCredential(serviceURL, sharedKeyCred, &azClientOpt)
-		// if err != nil {
-		// 	return nil, fmt.Errorf("create client failed: %w", err)
-		// }
-		// f.azsvc = azsvc
-	} else {
-		// create a client for the specified storage account
-		// azblob.ClientOptions{}
-		client, err := service.NewClient(serviceURL, cred, &clientOpt)
+		endpoint := opt.SASURL
+		containerName := parts.ContainerName
+		// Check if we have container level SAS or account level SAS
+		if containerName != "" {
+			// Container level SAS
+			if f.rootContainer != "" && containerName != f.rootContainer {
+				return nil, fmt.Errorf("container name in SAS URL (%q) and container provided in command (%q) do not match", containerName, f.rootContainer)
+			}
+			// Rewrite the endpoint string to be without the container
+			parts.ContainerName = ""
+			endpoint = parts.String()
+		}
+		f.svc, err = service.NewClientWithNoCredential(endpoint, &clientOpt)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create SAS URL client: %w", err)
+		}
+		// if using Container level SAS put the container client into the cache
+		if containerName != "" {
+			_ = f.cntSVC(containerName)
+			f.isLimited = true
+		}
+	case opt.ServicePrincipalFile != "":
+		// Try loading service principal credentials from file.
+		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
+		if err != nil {
+			return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
+		}
+		parsedCreds, err := parseServicePrincipalCredentials(ctx, loadedCreds)
 		if err != nil {
-			return nil, fmt.Errorf("create client failed: %w", err)
+			return nil, fmt.Errorf("error parsing service principal credentials file: %w", err)
+		}
+		options := azidentity.ClientSecretCredentialOptions{
+			ClientOptions: policyClientOptions,
+		}
+		cred, err = azidentity.NewClientSecretCredential(parsedCreds.Tenant, parsedCreds.AppID, parsedCreds.Password, &options)
+		if err != nil {
+			return nil, fmt.Errorf("error creating a client secret credential: %w", err)
+		}
+	default:
+		return nil, errors.New("no authentication method configured")
+	}
+
+	// Make the client if not already created
+	if f.svc == nil {
+		// Work out what the endpoint is if it is still unset
+		if opt.Endpoint == "" {
+			if opt.Account == "" {
+				return nil, fmt.Errorf("account must be set: can't make service URL")
+			}
+			u, err := url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, storageDefaultBaseURL))
+			if err != nil {
+				return nil, fmt.Errorf("failed to make azure storage URL from account and endpoint: %w", err)
+			}
+			opt.Endpoint = u.String()
 		}
-		f.svc = client
-
-		// create a client for the specified storage account
-		// azblob.ClientOptions{}
-		//
-		// Annoyingly this is the same type as f.svc just wrapped in a
-		// struct, but there is no way to create one from the other.
-		// azsvc, err := azblob.NewClient(serviceURL, cred, &azClientOpt)
-		// if err != nil {
-		// 	return nil, fmt.Errorf("create client failed: %w", err)
-		// }
-		// f.azsvc = azsvc
+		if sharedKeyCred != nil {
+			// Shared key cred
+			f.svc, err = service.NewClientWithSharedKeyCredential(opt.Endpoint, sharedKeyCred, &clientOpt)
+			if err != nil {
+				return nil, fmt.Errorf("create client with shared key failed: %w", err)
+			}
+		} else if cred != nil {
+			// Azidentity cred
+			f.svc, err = service.NewClient(opt.Endpoint, cred, &clientOpt)
+			if err != nil {
+				return nil, fmt.Errorf("create client failed: %w", err)
+			}
+		}
+	}
+	if f.svc == nil {
+		return nil, fmt.Errorf("internal error: auth failed to make credentials or client")
 	}
 
 	if f.rootContainer != "" && f.rootDirectory != "" {

From 914fbe242cc16ac5d33ede115f750f5b871f0280 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 29 Nov 2022 15:41:46 +0000
Subject: [PATCH 427/560] azureblob: ignore AuthorizationFailure when trying to
 create a create a container

If we get AuthorizationFailure when trying to create a container, then
assume the container has already been created
---
 backend/azureblob/azureblob.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 3631474e5e912..b3a20fc7f410b 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -1222,6 +1222,11 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
 						time.Sleep(6 * time.Second) // default 10 retries will be 60 seconds
 						f.cache.MarkDeleted(container)
 						return true, err
+					case bloberror.AuthorizationFailure:
+						// Assume that the user does not have permission to
+						// create the container and carry on anyway.
+						fs.Debugf(f, "Tried to create container but got %s error - carrying on assuming container exists", storageErr.ErrorCode)
+						return false, nil
 					}
 				}
 			}

From f3c8b7a9483ad582a127fc156fa8245f1a355ace Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 29 Nov 2022 16:00:37 +0000
Subject: [PATCH 428/560] azureblob: add --azureblob-no-check-container to
 assume container exists

Normally rclone will check the container exists before uploading if it
hasn't listed the container yet.

Often rclone will be running with a limited set of permissions which
means rclone can't create the container anyway, so this stops the
check.

This will save a transaction.
---
 backend/azureblob/azureblob.go | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index b3a20fc7f410b..598ecf3940c52 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -314,6 +314,15 @@ This option controls how often unused buffers will be removed from the pool.`,
 				},
 			},
 			Advanced: true,
+		}, {
+			Name: "no_check_container",
+			Help: `If set, don't attempt to check the container exists or create it.
+
+This can be useful when trying to minimise the number of transactions
+rclone does if you know the container exists already.
+`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     "no_head_object",
 			Help:     `If set, do not do HEAD before GET when getting objects.`,
@@ -346,6 +355,7 @@ type Options struct {
 	MemoryPoolUseMmap    bool                 `config:"memory_pool_use_mmap"`
 	Enc                  encoder.MultiEncoder `config:"encoding"`
 	PublicAccess         string               `config:"public_access"`
+	NoCheckContainer     bool                 `config:"no_check_container"`
 	NoHeadObject         bool                 `config:"no_head_object"`
 }
 
@@ -1191,6 +1201,9 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // makeContainer creates the container if it doesn't exist
 func (f *Fs) makeContainer(ctx context.Context, container string) error {
+	if f.opt.NoCheckContainer {
+		return nil
+	}
 	return f.cache.Create(container, func() error {
 		// If this is a SAS URL limited to a container then assume it is already created
 		if f.isLimited {
@@ -1225,7 +1238,7 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error {
 					case bloberror.AuthorizationFailure:
 						// Assume that the user does not have permission to
 						// create the container and carry on anyway.
-						fs.Debugf(f, "Tried to create container but got %s error - carrying on assuming container exists", storageErr.ErrorCode)
+						fs.Debugf(f, "Tried to create container but got %s error - carrying on assuming container exists. Use no_check_container to stop this check..", storageErr.ErrorCode)
 						return false, nil
 					}
 				}

From d7cb17848dac7ab9b004f3a64f580fcde1d69534 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Wed, 30 Nov 2022 11:45:34 +0000
Subject: [PATCH 429/560] azureblob: revamp authentication to include all
 methods and docs

The updates the authentication to include

- Auth from the environment
    1. Environment Variables
    2. Managed Service Identity Credentials
    3. Azure CLI credentials (as used by the az tool)
- Account and Shared Key
- SAS URL
- Service principal with client secret
- Service principal with certificate
- User with username and password
- Managed Service Identity Credentials

And rationalises the auth order.
---
 backend/azureblob/azureblob.go | 267 ++++++++++++++++------
 docs/content/azureblob.md      | 392 +++++++++++++++++++++++++++++----
 2 files changed, 552 insertions(+), 107 deletions(-)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 598ecf3940c52..dae8405a0060b 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -40,7 +40,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -65,6 +64,7 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -110,21 +110,20 @@ func init() {
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name: "account",
-			Help: `Storage Account Name.
+			Help: `Azure Storage Account Name.
 
-Leave blank to use SAS URL or Emulator.
+Set this to the Azure Storage Account Name in use.
 
-If this is blank then if env_auth is set it will be read from the
-environment variable AZURE_CLIENT_ID.
+Leave blank to use SAS URL or Emulator, otherwise it needs to be set.
+
+If this is blank and if env_auth is set it will be read from the
+environment variable ` + "`AZURE_STORAGE_ACCOUNT_NAME`" + ` if possible.
 `,
 		}, {
 			Name: "env_auth",
-			Help: `Read credentials from runtime (environment variables).
-
-Pull credentials from AZURE_TENANT_ID and AZURE_CLIENT_{ID,SECRET} environment vars.
-See EnvironmentCredential in the Azure docs for more info.
+			Help: `Read credentials from runtime (environment variables, CLI or MSI).
 
-Other authentication methods will, if specified, override this flag.`,
+See the [authentication docs](/azureblob#authentication) for full info.`,
 			Default: false,
 		}, {
 			Name: "key",
@@ -136,6 +135,78 @@ Leave blank to use SAS URL or Emulator.`,
 			Help: `SAS URL for container level access only.
 
 Leave blank if using account/key or Emulator.`,
+		}, {
+			Name: "tenant",
+			Help: `ID of the service principal's tenant. Also called its directory ID.
+
+Set this if using
+- Service principal with client secret
+- Service principal with certificate
+- User with username and password
+`,
+		}, {
+			Name: "client_id",
+			Help: `The ID of the client in use.
+
+Set this if using
+- Service principal with client secret
+- Service principal with certificate
+- User with username and password
+`,
+		}, {
+			Name: "client_secret",
+			Help: `One of the service principal's client secrets
+
+Set this if using
+- Service principal with client secret
+`,
+		}, {
+			Name: "client_certificate_path",
+			Help: `Path to a PEM or PKCS12 certificate file including the private key.
+
+Set this if using
+- Service principal with certificate
+`,
+		}, {
+			Name: "client_certificate_password",
+			Help: `Password for the certificate file (optional).
+
+Optionally set this if using
+- Service principal with certificate
+
+And the certificate has a password.
+`,
+			IsPassword: true,
+		}, {
+			Name: "client_send_certificate_chain",
+			Help: `Send the certificate chain when using certificate auth.
+
+Specifies whether an authentication request will include an x5c header
+to support subject name / issuer based authentication. When set to
+true, authentication requests include the x5c header.
+
+Optionally set this if using
+- Service principal with certificate
+`,
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name: "username",
+			Help: `User name (usually an email address)
+
+Set this if using
+- User with username and password
+`,
+			Advanced: true,
+		}, {
+			Name: "password",
+			Help: `The user's password
+
+Set this if using
+- User with username and password
+`,
+			IsPassword: true,
+			Advanced:   true,
 		}, {
 			Name: "service_principal_file",
 			Help: `Path to file containing credentials for use with a service principal.
@@ -148,6 +219,10 @@ Leave blank normally. Needed only if you want to use a service principal instead
       > azure-principal.json
 
 See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
+
+It may be more convenient to put the credentials directly into the
+rclone config file under the ` + "`client_id`, `tenant` and `client_secret`" + `
+keys instead of setting ` + "`service_principal_file`" + `.
 `,
 			Advanced: true,
 		}, {
@@ -334,29 +409,37 @@ rclone does if you know the container exists already.
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account              string               `config:"account"`
-	EnvAuth              bool                 `config:"env_auth"`
-	Key                  string               `config:"key"`
-	ServicePrincipalFile string               `config:"service_principal_file"`
-	UseMSI               bool                 `config:"use_msi"`
-	MSIObjectID          string               `config:"msi_object_id"`
-	MSIClientID          string               `config:"msi_client_id"`
-	MSIResourceID        string               `config:"msi_mi_res_id"`
-	Endpoint             string               `config:"endpoint"`
-	SASURL               string               `config:"sas_url"`
-	ChunkSize            fs.SizeSuffix        `config:"chunk_size"`
-	UploadConcurrency    int                  `config:"upload_concurrency"`
-	ListChunkSize        uint                 `config:"list_chunk"`
-	AccessTier           string               `config:"access_tier"`
-	ArchiveTierDelete    bool                 `config:"archive_tier_delete"`
-	UseEmulator          bool                 `config:"use_emulator"`
-	DisableCheckSum      bool                 `config:"disable_checksum"`
-	MemoryPoolFlushTime  fs.Duration          `config:"memory_pool_flush_time"`
-	MemoryPoolUseMmap    bool                 `config:"memory_pool_use_mmap"`
-	Enc                  encoder.MultiEncoder `config:"encoding"`
-	PublicAccess         string               `config:"public_access"`
-	NoCheckContainer     bool                 `config:"no_check_container"`
-	NoHeadObject         bool                 `config:"no_head_object"`
+	Account                    string               `config:"account"`
+	EnvAuth                    bool                 `config:"env_auth"`
+	Key                        string               `config:"key"`
+	SASURL                     string               `config:"sas_url"`
+	Tenant                     string               `config:"tenant"`
+	ClientID                   string               `config:"client_id"`
+	ClientSecret               string               `config:"client_secret"`
+	ClientCertificatePath      string               `config:"client_certificate_path"`
+	ClientCertificatePassword  string               `config:"client_certificate_password"`
+	ClientSendCertificateChain bool                 `config:"client_send_certificate_chain"`
+	Username                   string               `config:"username"`
+	Password                   string               `config:"password"`
+	ServicePrincipalFile       string               `config:"service_principal_file"`
+	UseMSI                     bool                 `config:"use_msi"`
+	MSIObjectID                string               `config:"msi_object_id"`
+	MSIClientID                string               `config:"msi_client_id"`
+	MSIResourceID              string               `config:"msi_mi_res_id"`
+	Endpoint                   string               `config:"endpoint"`
+	ChunkSize                  fs.SizeSuffix        `config:"chunk_size"`
+	UploadConcurrency          int                  `config:"upload_concurrency"`
+	ListChunkSize              uint                 `config:"list_chunk"`
+	AccessTier                 string               `config:"access_tier"`
+	ArchiveTierDelete          bool                 `config:"archive_tier_delete"`
+	UseEmulator                bool                 `config:"use_emulator"`
+	DisableCheckSum            bool                 `config:"disable_checksum"`
+	MemoryPoolFlushTime        fs.Duration          `config:"memory_pool_flush_time"`
+	MemoryPoolUseMmap          bool                 `config:"memory_pool_use_mmap"`
+	Enc                        encoder.MultiEncoder `config:"encoding"`
+	PublicAccess               string               `config:"public_access"`
+	NoCheckContainer           bool                 `config:"no_check_container"`
+	NoHeadObject               bool                 `config:"no_head_object"`
 }
 
 // Fs represents a remote azure server
@@ -625,7 +708,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	case opt.EnvAuth:
 		// Read account from environment if needed
 		if opt.Account == "" {
-			opt.Account, _ = os.LookupEnv("AZURE_CLIENT_ID")
+			opt.Account, _ = os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME")
 		}
 		// Read credentials from the environment
 		options := azidentity.DefaultAzureCredentialOptions{
@@ -636,41 +719,19 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, fmt.Errorf("create azure enviroment credential failed: %w", err)
 		}
 	case opt.UseEmulator:
-		if opt.Account != "" {
+		if opt.Account == "" {
 			opt.Account = emulatorAccount
 		}
 		if opt.Key == "" {
 			opt.Key = emulatorAccountKey
 		}
-		if opt.Endpoint != "" {
+		if opt.Endpoint == "" {
 			opt.Endpoint = emulatorBlobEndpoint
 		}
 		sharedKeyCred, err = service.NewSharedKeyCredential(opt.Account, opt.Key)
 		if err != nil {
 			return nil, fmt.Errorf("create new shared key credential for emulator failed: %w", err)
 		}
-	case opt.UseMSI:
-		// Specifying a user-assigned identity. Exactly one of the above IDs must be specified.
-		// Validate and ensure exactly one is set. (To do: better validation.)
-		var b2i = map[bool]int{false: 0, true: 1}
-		set := b2i[opt.MSIClientID != ""] + b2i[opt.MSIObjectID != ""] + b2i[opt.MSIResourceID != ""]
-		if set > 1 {
-			return nil, errors.New("more than one user-assigned identity ID is set")
-		}
-		var options azidentity.ManagedIdentityCredentialOptions
-		switch {
-		case opt.MSIClientID != "":
-			options.ID = azidentity.ClientID(opt.MSIClientID)
-		case opt.MSIObjectID != "":
-			// FIXME this doesn't appear to be in the new SDK?
-			return nil, fmt.Errorf("MSI object ID is currently unsupported")
-		case opt.MSIResourceID != "":
-			options.ID = azidentity.ResourceID(opt.MSIResourceID)
-		}
-		cred, err = azidentity.NewManagedIdentityCredential(&options)
-		if err != nil {
-			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
-		}
 	case opt.Account != "" && opt.Key != "":
 		sharedKeyCred, err = service.NewSharedKeyCredential(opt.Account, opt.Key)
 		if err != nil {
@@ -702,9 +763,69 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			_ = f.cntSVC(containerName)
 			f.isLimited = true
 		}
+	case opt.ClientID != "" && opt.Tenant != "" && opt.ClientSecret != "":
+		// Service principal with client secret
+		options := azidentity.ClientSecretCredentialOptions{
+			ClientOptions: policyClientOptions,
+		}
+		cred, err = azidentity.NewClientSecretCredential(opt.Tenant, opt.ClientID, opt.ClientSecret, &options)
+		if err != nil {
+			return nil, fmt.Errorf("error creating a client secret credential: %w", err)
+		}
+	case opt.ClientID != "" && opt.Tenant != "" && opt.ClientCertificatePath != "":
+		// Service principal with certificate
+		//
+		// Read the certificate
+		data, err := os.ReadFile(env.ShellExpand(opt.ClientCertificatePath))
+		if err != nil {
+			return nil, fmt.Errorf("error reading client certificate file: %w", err)
+		}
+		// NewClientCertificateCredential requires at least one *x509.Certificate, and a
+		// crypto.PrivateKey.
+		//
+		// ParseCertificates returns these given certificate data in PEM or PKCS12 format.
+		// It handles common scenarios but has limitations, for example it doesn't load PEM
+		// encrypted private keys.
+		var password []byte
+		if opt.ClientCertificatePassword != "" {
+			pw, err := obscure.Reveal(opt.Password)
+			if err != nil {
+				return nil, fmt.Errorf("certificate password decode failed - did you obscure it?: %w", err)
+			}
+			password = []byte(pw)
+		}
+		certs, key, err := azidentity.ParseCertificates(data, password)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse client certificate file: %w", err)
+		}
+		options := azidentity.ClientCertificateCredentialOptions{
+			ClientOptions:        policyClientOptions,
+			SendCertificateChain: opt.ClientSendCertificateChain,
+		}
+		cred, err = azidentity.NewClientCertificateCredential(
+			opt.Tenant, opt.ClientID, certs, key, &options,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("create azure service principal with client certificate credential failed: %w", err)
+		}
+	case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "":
+		// User with username and password
+		options := azidentity.UsernamePasswordCredentialOptions{
+			ClientOptions: policyClientOptions,
+		}
+		password, err := obscure.Reveal(opt.Password)
+		if err != nil {
+			return nil, fmt.Errorf("user password decode failed - did you obscure it?: %w", err)
+		}
+		cred, err = azidentity.NewUsernamePasswordCredential(
+			opt.Tenant, opt.ClientID, opt.Username, password, &options,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("authenticate user with password failed: %w", err)
+		}
 	case opt.ServicePrincipalFile != "":
-		// Try loading service principal credentials from file.
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
+		// Loading service principal credentials from file.
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
 		}
@@ -719,6 +840,28 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("error creating a client secret credential: %w", err)
 		}
+	case opt.UseMSI:
+		// Specifying a user-assigned identity. Exactly one of the above IDs must be specified.
+		// Validate and ensure exactly one is set. (To do: better validation.)
+		var b2i = map[bool]int{false: 0, true: 1}
+		set := b2i[opt.MSIClientID != ""] + b2i[opt.MSIObjectID != ""] + b2i[opt.MSIResourceID != ""]
+		if set > 1 {
+			return nil, errors.New("more than one user-assigned identity ID is set")
+		}
+		var options azidentity.ManagedIdentityCredentialOptions
+		switch {
+		case opt.MSIClientID != "":
+			options.ID = azidentity.ClientID(opt.MSIClientID)
+		case opt.MSIObjectID != "":
+			// FIXME this doesn't appear to be in the new SDK?
+			return nil, fmt.Errorf("MSI object ID is currently unsupported")
+		case opt.MSIResourceID != "":
+			options.ID = azidentity.ResourceID(opt.MSIResourceID)
+		}
+		cred, err = azidentity.NewManagedIdentityCredential(&options)
+		if err != nil {
+			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
+		}
 	default:
 		return nil, errors.New("no authentication method configured")
 	}
@@ -732,7 +875,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			}
 			u, err := url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, storageDefaultBaseURL))
 			if err != nil {
-				return nil, fmt.Errorf("failed to make azure storage URL from account and endpoint: %w", err)
+				return nil, fmt.Errorf("failed to make azure storage URL from account: %w", err)
 			}
 			opt.Endpoint = u.String()
 		}
@@ -1776,7 +1919,7 @@ func urlEncode(in string) string {
 		if noNeedToEncode[c] {
 			_ = out.WriteByte(c)
 		} else {
-			_, _ = out.WriteString(fmt.Sprintf("%%%2X", c))
+			_, _ = out.WriteString(fmt.Sprintf("%%%02X", c))
 		}
 	}
 	return out.String()
diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md
index 537dc1f55fbaf..1087e2577e0ef 100644
--- a/docs/content/azureblob.md
+++ b/docs/content/azureblob.md
@@ -116,11 +116,80 @@ MD5 hashes are stored with blobs.  However blobs that were uploaded in
 chunks only have an MD5 if the source remote was capable of MD5
 hashes, e.g. the local disk.
 
-### Authenticating with Azure Blob Storage
+### Authentication {#authentication}
 
-Rclone has 3 ways of authenticating with Azure Blob Storage:
+There are a number of ways of supplying credentials for Azure Blob
+Storage. Rclone tries them in the order of the sections below.
 
-#### Account and Key
+#### Env Auth
+
+If the `env_auth` config parameter is `true` then rclone will pull
+credentials from the environment or runtime.
+
+It tries these authentication methods in this order:
+
+1. Environment Variables
+2. Managed Service Identity Credentials
+3. Azure CLI credentials (as used by the az tool)
+
+These are described in the following sections
+
+##### Env Auth: 1. Environment Variables
+
+If `env_auth` is set and environment variables are present rclone
+authenticates a service principal with a secret or certificate, or a
+user with a password, depending on which environment variable are set.
+It reads configuration from these variables, in the following order:
+
+1. Service principal with client secret
+    - `AZURE_TENANT_ID`: ID of the service principal's tenant. Also called its "directory" ID.
+    - `AZURE_CLIENT_ID`: the service principal's client ID
+    - `AZURE_CLIENT_SECRET`: one of the service principal's client secrets
+2. Service principal with certificate
+    - `AZURE_TENANT_ID`: ID of the service principal's tenant. Also called its "directory" ID.
+    - `AZURE_CLIENT_ID`: the service principal's client ID
+    - `AZURE_CLIENT_CERTIFICATE_PATH`: path to a PEM or PKCS12 certificate file including the private key.
+    - `AZURE_CLIENT_CERTIFICATE_PASSWORD`: (optional) password for the certificate file.
+    - `AZURE_CLIENT_SEND_CERTIFICATE_CHAIN`: (optional) Specifies whether an authentication request will include an x5c header to support subject name / issuer based authentication. When set to "true" or "1", authentication requests include the x5c header.
+3. User with username and password
+    - `AZURE_TENANT_ID`: (optional) tenant to authenticate in. Defaults to "organizations".
+    - `AZURE_CLIENT_ID`: client ID of the application the user will authenticate to
+    - `AZURE_USERNAME`: a username (usually an email address)
+    - `AZURE_PASSWORD`: the user's password
+
+##### Env Auth: 2. Managed Service Identity Credentials
+
+When using Managed Service Identity if the VM(SS) on which this
+program is running has a system-assigned identity, it will be used by
+default. If the resource has no system-assigned but exactly one
+user-assigned identity, the user-assigned identity will be used by
+default.
+
+If the resource has multiple user-assigned identities you will need to
+unset `env_auth` and set `use_msi` instead. See the [`use_msi`
+section](#use_msi).
+
+##### Env Auth: 3. Azure CLI credentials (as used by the az tool)
+
+Credentials created with the `az` tool can be picked up using `env_auth`.
+
+For example if you were to login with a service principal like this:
+
+    az login --service-principal -u XXX -p XXX --tenant XXX
+
+Then you could access rclone resources like this:
+
+    rclone lsf :azureblob,env_auth,account=ACCOUNT:CONTAINER
+
+Or
+
+    rclone lsf --azureblob-env-auth --azureblob-acccount=ACCOUNT :azureblob:CONTAINER
+
+Which is analogous to using the `az` tool:
+
+    az storage blob list --container-name CONTAINER --account-name ACCOUNT --auth-mode login
+
+#### Account and Shared Key
 
 This is the most straight forward and least flexible way.  Just fill
 in the `account` and `key` lines and leave the rest blank.
@@ -129,7 +198,7 @@ in the `account` and `key` lines and leave the rest blank.
 
 This can be an account level SAS URL or container level SAS URL.
 
-To use it leave `account`, `key` blank and fill in `sas_url`.
+To use it leave `account` and `key` blank and fill in `sas_url`.
 
 An account level SAS URL or container level SAS URL can be obtained
 from the Azure portal or the Azure Storage Explorer.  To get a
@@ -156,6 +225,60 @@ Container level SAS URLs are useful for temporarily allowing third
 parties access to a single container or putting credentials into an
 untrusted environment such as a CI build server.
 
+#### Service principal with client secret
+
+If these variables are set, rclone will authenticate with a service principal with a client secret.
+
+- `tenant`: ID of the service principal's tenant. Also called its "directory" ID.
+- `client_id`: the service principal's client ID
+- `client_secret`: one of the service principal's client secrets
+
+The credentials can also be placed in a file using the
+`service_principal_file` configuration option.
+
+#### Service principal with certificate
+
+If these variables are set, rclone will authenticate with a service principal with certificate.
+
+- `tenant`: ID of the service principal's tenant. Also called its "directory" ID.
+- `client_id`: the service principal's client ID
+- `client_certificate_path`: path to a PEM or PKCS12 certificate file including the private key.
+- `client_certificate_password`: (optional) password for the certificate file.
+- `client_send_certificate_chain`: (optional) Specifies whether an authentication request will include an x5c header to support subject name / issuer based authentication. When set to "true" or "1", authentication requests include the x5c header.
+
+**NB** `client_certificate_password` must be obscured - see [rclone obscure](/commands/rclone_obscure/).
+
+#### User with username and password
+
+If these variables are set, rclone will authenticate with username and password.
+
+- `tenant`: (optional) tenant to authenticate in. Defaults to "organizations".
+- `client_id`: client ID of the application the user will authenticate to
+- `username`: a username (usually an email address)
+- `password`: the user's password
+
+Microsoft doesn't recommend this kind of authentication, because it's
+less secure than other authentication flows. This method is not
+interactive, so it isn't compatible with any form of multi-factor
+authentication, and the application must already have user or admin
+consent. This credential can only authenticate work and school
+accounts; it can't authenticate Microsoft accounts.
+
+**NB** `password` must be obscured - see [rclone obscure](/commands/rclone_obscure/).
+
+#### Managed Service Identity Credentials {#use_msi}
+
+If `use_msi` is set then managed service identity credentials are
+used. This authentication only works when running in an Azure service.
+`env_auth` needs to be unset to use this.
+
+However if you have multiple user identities to choose from these must
+be explicitly specified using exactly one of the `msi_object_id`,
+`msi_client_id`, or `msi_mi_res_id` parameters.
+
+If none of `msi_object_id`, `msi_client_id`, or `msi_mi_res_id` is
+set, this is is equivalent to using `env_auth`.
+
 {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/azureblob/azureblob.go then run make backenddocs" >}}
 ### Standard options
 
@@ -163,9 +286,15 @@ Here are the Standard options specific to azureblob (Microsoft Azure Blob Storag
 
 #### --azureblob-account
 
-Storage Account Name.
+Azure Storage Account Name.
+
+Set this to the Azure Storage Account Name in use.
+
+Leave blank to use SAS URL or Emulator, otherwise it needs to be set.
+
+If this is blank and if env_auth is set it will be read from the
+environment variable `AZURE_STORAGE_ACCOUNT_NAME` if possible.
 
-Leave blank to use SAS URL.
 
 Properties:
 
@@ -174,32 +303,24 @@ Properties:
 - Type:        string
 - Required:    false
 
-#### --azureblob-service-principal-file
-
-Path to file containing credentials for use with a service principal.
-
-Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
-
-    $ az ad sp create-for-rbac --name "" \
-      --role "Storage Blob Data Owner" \
-      --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \
-      > azure-principal.json
+#### --azureblob-env-auth
 
-See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
+Read credentials from runtime (environment variables, CLI or MSI).
 
+See the [authentication docs](/azureblob#authentication) for full info.
 
 Properties:
 
-- Config:      service_principal_file
-- Env Var:     RCLONE_AZUREBLOB_SERVICE_PRINCIPAL_FILE
-- Type:        string
-- Required:    false
+- Config:      env_auth
+- Env Var:     RCLONE_AZUREBLOB_ENV_AUTH
+- Type:        bool
+- Default:     false
 
 #### --azureblob-key
 
-Storage Account Key.
+Storage Account Shared Key.
 
-Leave blank to use SAS URL.
+Leave blank to use SAS URL or Emulator.
 
 Properties:
 
@@ -221,6 +342,169 @@ Properties:
 - Type:        string
 - Required:    false
 
+#### --azureblob-tenant
+
+ID of the service principal's tenant. Also called its directory ID.
+
+Set this if using
+- Service principal with client secret
+- Service principal with certificate
+- User with username and password
+
+
+Properties:
+
+- Config:      tenant
+- Env Var:     RCLONE_AZUREBLOB_TENANT
+- Type:        string
+- Required:    false
+
+#### --azureblob-client-id
+
+The ID of the client in use.
+
+Set this if using
+- Service principal with client secret
+- Service principal with certificate
+- User with username and password
+
+
+Properties:
+
+- Config:      client_id
+- Env Var:     RCLONE_AZUREBLOB_CLIENT_ID
+- Type:        string
+- Required:    false
+
+#### --azureblob-client-secret
+
+One of the service principal's client secrets
+
+Set this if using
+- Service principal with client secret
+
+
+Properties:
+
+- Config:      client_secret
+- Env Var:     RCLONE_AZUREBLOB_CLIENT_SECRET
+- Type:        string
+- Required:    false
+
+#### --azureblob-client-certificate-path
+
+Path to a PEM or PKCS12 certificate file including the private key.
+
+Set this if using
+- Service principal with certificate
+
+
+Properties:
+
+- Config:      client_certificate_path
+- Env Var:     RCLONE_AZUREBLOB_CLIENT_CERTIFICATE_PATH
+- Type:        string
+- Required:    false
+
+#### --azureblob-client-certificate-password
+
+Password for the certificate file (optional).
+
+Optionally set this if using
+- Service principal with certificate
+
+And the certificate has a password.
+
+
+**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).
+
+Properties:
+
+- Config:      client_certificate_password
+- Env Var:     RCLONE_AZUREBLOB_CLIENT_CERTIFICATE_PASSWORD
+- Type:        string
+- Required:    false
+
+### Advanced options
+
+Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage).
+
+#### --azureblob-client-send-certificate-chain
+
+Send the certificate chain when using certificate auth.
+
+Specifies whether an authentication request will include an x5c header
+to support subject name / issuer based authentication. When set to
+true, authentication requests include the x5c header.
+
+Optionally set this if using
+- Service principal with certificate
+
+
+Properties:
+
+- Config:      client_send_certificate_chain
+- Env Var:     RCLONE_AZUREBLOB_CLIENT_SEND_CERTIFICATE_CHAIN
+- Type:        bool
+- Default:     false
+
+#### --azureblob-username
+
+User name (usually an email address)
+
+Set this if using
+- User with username and password
+
+
+Properties:
+
+- Config:      username
+- Env Var:     RCLONE_AZUREBLOB_USERNAME
+- Type:        string
+- Required:    false
+
+#### --azureblob-password
+
+The user's password
+
+Set this if using
+- User with username and password
+
+
+**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).
+
+Properties:
+
+- Config:      password
+- Env Var:     RCLONE_AZUREBLOB_PASSWORD
+- Type:        string
+- Required:    false
+
+#### --azureblob-service-principal-file
+
+Path to file containing credentials for use with a service principal.
+
+Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
+
+    $ az ad sp create-for-rbac --name "" \
+      --role "Storage Blob Data Owner" \
+      --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \
+      > azure-principal.json
+
+See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details.
+
+It may be more convenient to put the credentials directly into the
+rclone config file under the `client_id`, `tenant` and `client_secret`
+keys instead of setting `service_principal_file`.
+
+
+Properties:
+
+- Config:      service_principal_file
+- Env Var:     RCLONE_AZUREBLOB_SERVICE_PRINCIPAL_FILE
+- Type:        string
+- Required:    false
+
 #### --azureblob-use-msi
 
 Use a managed service identity to authenticate (only works in Azure).
@@ -241,25 +525,6 @@ Properties:
 - Type:        bool
 - Default:     false
 
-#### --azureblob-use-emulator
-
-Uses local storage emulator if provided as 'true'.
-
-Leave blank if using real azure storage endpoint.
-
-Provide `account`, `key`, and `endpoint` if desired to override their _azurite_ defaults.
-
-Properties:
-
-- Config:      use_emulator
-- Env Var:     RCLONE_AZUREBLOB_USE_EMULATOR
-- Type:        bool
-- Default:     false
-
-### Advanced options
-
-Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage).
-
 #### --azureblob-msi-object-id
 
 Object ID of the user-assigned MSI to use, if any.
@@ -299,6 +564,19 @@ Properties:
 - Type:        string
 - Required:    false
 
+#### --azureblob-use-emulator
+
+Uses local storage emulator if provided as 'true'.
+
+Leave blank if using real azure storage endpoint.
+
+Properties:
+
+- Config:      use_emulator
+- Env Var:     RCLONE_AZUREBLOB_USE_EMULATOR
+- Type:        bool
+- Default:     false
+
 #### --azureblob-endpoint
 
 Endpoint for the service.
@@ -502,6 +780,21 @@ Properties:
     - "container"
         - Allow full public read access for container and blob data.
 
+#### --azureblob-no-check-container
+
+If set, don't attempt to check the container exists or create it.
+
+This can be useful when trying to minimise the number of transactions
+rclone does if you know the container exists already.
+
+
+Properties:
+
+- Config:      no_check_container
+- Env Var:     RCLONE_AZUREBLOB_NO_CHECK_CONTAINER
+- Type:        bool
+- Default:     false
+
 #### --azureblob-no-head-object
 
 If set, do not do HEAD before GET when getting objects.
@@ -541,8 +834,17 @@ See [List of backends that do not support rclone about](https://rclone.org/overv
 
 ## Azure Storage Emulator Support
 
-You can run rclone with storage emulator (usually _azurite_).
+You can run rclone with the storage emulator (usually _azurite_).
 
-To do this, just set up a new remote with `rclone config` following instructions described in introduction and set `use_emulator` config as `true`. You do not need to provide default account name neither an account key. But you can override them in the _account_ and _key_ options. (Prior to v1.61 they were hard coded to _azurite_'s `devstoreaccount1`.)
+To do this, just set up a new remote with `rclone config` following
+the instructions in the introduction and set `use_emulator` in the
+advanced settings as `true`. You do not need to provide a default
+account name nor an account key. But you can override them in the
+`account` and `key` options. (Prior to v1.61 they were hard coded to
+_azurite_'s `devstoreaccount1`.)
 
-Also, if you want to access a storage emulator instance running on a different machine, you can override _Endpoint_ parameter in advanced settings, setting it to `http(s)://:/devstoreaccount1` (e.g. `http://10.254.2.5:10000/devstoreaccount1`).
+Also, if you want to access a storage emulator instance running on a
+different machine, you can override the `endpoint` parameter in the
+advanced settings, setting it to
+`http(s)://:/devstoreaccount1`
+(e.g. `http://10.254.2.5:10000/devstoreaccount1`).

From 1dbdc48a77cfd2cf9efb33d1fc97103f5788b639 Mon Sep 17 00:00:00 2001
From: Matthew Vernon 
Date: Wed, 7 Dec 2022 12:11:49 +0000
Subject: [PATCH 430/560] WASM: comply with wasm_exec.js licence terms

The BSD-style license that Go uses requires the license to be included
with the source distribution; so add it as LICENSE.wasmexec (to avoid
confusion with the other licenses in rclone) and note the location of
the license in wasm_exec.js itself.
---
 fs/rc/js/LICENSE.wasmexec | 27 +++++++++++++++++++++++++++
 fs/rc/js/wasm_exec.js     |  2 +-
 2 files changed, 28 insertions(+), 1 deletion(-)
 create mode 100644 fs/rc/js/LICENSE.wasmexec

diff --git a/fs/rc/js/LICENSE.wasmexec b/fs/rc/js/LICENSE.wasmexec
new file mode 100644
index 0000000000000..49ea0f928825a
--- /dev/null
+++ b/fs/rc/js/LICENSE.wasmexec
@@ -0,0 +1,27 @@
+Copyright (c) 2018 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/fs/rc/js/wasm_exec.js b/fs/rc/js/wasm_exec.js
index 52ce4f76dc0c6..fc629d077ec1d 100644
--- a/fs/rc/js/wasm_exec.js
+++ b/fs/rc/js/wasm_exec.js
@@ -1,6 +1,6 @@
 // Copyright 2018 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// license that can be found in the LICENSE.wasmexec file.
 
 "use strict";
 

From 450c3664035007875e2ade2ab5fc37a0c8b889e5 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 8 Dec 2022 12:41:23 +0000
Subject: [PATCH 431/560] s3: fix nil pointer exception when using Versions

This was caused by

a9bd0c8de6e74460 s3: reduce memory consumption for s3 objects

Which assumed that the StorageClass would always be set, but it isn't
set for Versions.
---
 backend/s3/s3.go | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 8b0b19ee3e925..36b6a9e613ec0 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -3057,6 +3057,17 @@ func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s
 	return info, versionID, nil
 }
 
+// stringClonePointer clones the string pointed to by sp into new
+// memory. This is useful to stop us keeping references to small
+// strings carved out of large XML responses.
+func stringClonePointer(sp *string) *string {
+	if sp == nil {
+		return nil
+	}
+	var s = *sp
+	return &s
+}
+
 // Return an Object from a path
 //
 // If it can't be found it returns the error ErrorObjectNotFound.
@@ -3082,8 +3093,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje
 		}
 		o.setMD5FromEtag(aws.StringValue(info.ETag))
 		o.bytes = aws.Int64Value(info.Size)
-		storageClass := *info.StorageClass // To prevent reference to large XML structures
-		o.storageClass = &storageClass
+		o.storageClass = stringClonePointer(info.StorageClass)
 		o.versionID = versionID
 	} else if !o.fs.opt.NoHeadObject {
 		err := o.readMetaData(ctx) // reads info and meta, returning an error

From be783a1856ca8cf23abd80e566a35c097cf2faaa Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 6 Dec 2022 13:58:12 +0000
Subject: [PATCH 432/560] dlna: properly attribute code used from
 https://github.com/anacrolix/dms

Fixes #4101
---
 cmd/serve/dlna/LICENSE.anacrolix | 27 +++++++++++++++++++++++++++
 docs/content/authors.md          |  1 +
 2 files changed, 28 insertions(+)
 create mode 100644 cmd/serve/dlna/LICENSE.anacrolix

diff --git a/cmd/serve/dlna/LICENSE.anacrolix b/cmd/serve/dlna/LICENSE.anacrolix
new file mode 100644
index 0000000000000..836e8f7c0b677
--- /dev/null
+++ b/cmd/serve/dlna/LICENSE.anacrolix
@@ -0,0 +1,27 @@
+This directory contains code derived from https://github.com/anacrolix/dms
+which is under the following license.
+
+Copyright (c) 2012, Matt Joiner .
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the  nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL  BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/docs/content/authors.md b/docs/content/authors.md
index db38dc5d62059..279f465776d60 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -231,6 +231,7 @@ put them back in again.` >}}
   * Jay 
   * andrea rota 
   * nicolov 
+  * Matt Joiner 
   * Dario Guzik 
   * qip 
   * yair@unicorn 

From 8e507075d10c0eddc4d6718ae008ff69023947ff Mon Sep 17 00:00:00 2001
From: Eng Zer Jun 
Date: Thu, 8 Dec 2022 20:43:53 +0800
Subject: [PATCH 433/560] test: replace defer cleanup with `t.Cleanup`

Reference: https://pkg.go.dev/testing#T.Cleanup
Signed-off-by: Eng Zer Jun 
---
 backend/cache/cache_internal_test.go         | 61 ++++++++------------
 backend/cache/cache_upload_test.go           | 24 ++------
 backend/crypt/crypt_internal_test.go         | 31 +++++-----
 backend/http/http_internal_test.go           | 42 ++++++--------
 backend/local/local_internal_test.go         |  6 --
 cmd/selfupdate/selfupdate_test.go            | 32 ++--------
 cmd/touch/touch_test.go                      | 11 ----
 cmdtest/cmdtest_test.go                      | 21 ++-----
 cmdtest/environment_test.go                  |  2 -
 fs/cache/cache_test.go                       | 35 ++++-------
 fs/march/march_test.go                       |  2 -
 fs/operations/check_test.go                  |  3 -
 fs/operations/dedupe_test.go                 | 11 ----
 fs/operations/listdirsorted_test.go          |  1 -
 fs/operations/lsjson_test.go                 |  2 -
 fs/operations/multithread_test.go            |  1 -
 fs/operations/operations_test.go             | 34 +----------
 fs/operations/rc_test.go                     | 18 ------
 fs/rc/webgui/rc_test.go                      | 17 ++----
 fs/sync/rc_test.go                           |  3 -
 fs/sync/sync_test.go                         | 57 ------------------
 fstest/run.go                                |  3 +-
 vfs/dir_handle_test.go                       |  9 +--
 vfs/dir_test.go                              | 51 ++++++----------
 vfs/file_test.go                             | 30 ++++------
 vfs/rc_test.go                               | 24 +++-----
 vfs/read_test.go                             | 21 +++----
 vfs/read_write_test.go                       | 58 +++++++------------
 vfs/vfs_case_test.go                         |  1 -
 vfs/vfs_test.go                              | 36 +++++-------
 vfs/vfscache/cache_test.go                   | 56 +++++++-----------
 vfs/vfscache/downloaders/downloaders_test.go |  1 -
 vfs/vfscache/item_test.go                    | 44 +++++---------
 vfs/write_test.go                            | 24 +++-----
 34 files changed, 217 insertions(+), 555 deletions(-)

diff --git a/backend/cache/cache_internal_test.go b/backend/cache/cache_internal_test.go
index 849354274069e..82ae0bf012f12 100644
--- a/backend/cache/cache_internal_test.go
+++ b/backend/cache/cache_internal_test.go
@@ -101,14 +101,12 @@ func TestMain(m *testing.M) {
 
 func TestInternalListRootAndInnerRemotes(t *testing.T) {
 	id := fmt.Sprintf("tilrair%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 
 	// Instantiate inner fs
 	innerFolder := "inner"
 	runInstance.mkdir(t, rootFs, innerFolder)
-	rootFs2, boltDb2 := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs2, boltDb2)
+	rootFs2, _ := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil)
 
 	runInstance.writeObjectString(t, rootFs2, "one", "content")
 	listRoot, err := runInstance.list(t, rootFs, "")
@@ -225,8 +223,7 @@ func TestInternalVfsCache(t *testing.T) {
 
 func TestInternalObjWrapFsFound(t *testing.T) {
 	id := fmt.Sprintf("tiowff%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -258,8 +255,7 @@ func TestInternalObjWrapFsFound(t *testing.T) {
 
 func TestInternalObjNotFound(t *testing.T) {
 	id := fmt.Sprintf("tionf%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	obj, err := rootFs.NewObject(context.Background(), "404")
 	require.Error(t, err)
@@ -269,8 +265,7 @@ func TestInternalObjNotFound(t *testing.T) {
 func TestInternalCachedWrittenContentMatches(t *testing.T) {
 	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticwcm%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -297,8 +292,7 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 		t.Skip("Skip test on windows/386")
 	}
 	id := fmt.Sprintf("tidwcm%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	// write the object
 	runInstance.writeRemoteString(t, rootFs, "one", "one content")
@@ -316,8 +310,7 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticucm%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 	var err error
 
 	// create some rand test data
@@ -346,8 +339,7 @@ func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tiwwcm%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
 	}
@@ -377,8 +369,7 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 func TestInternalLargeWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tilwcm%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
 	}
@@ -404,8 +395,7 @@ func TestInternalLargeWrittenContentMatches(t *testing.T) {
 
 func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	id := fmt.Sprintf("tiwfcns%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -459,8 +449,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 
 func TestInternalMoveWithNotify(t *testing.T) {
 	id := fmt.Sprintf("timwn%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 	if !runInstance.wrappedIsExternal {
 		t.Skipf("Not external")
 	}
@@ -546,8 +535,7 @@ func TestInternalMoveWithNotify(t *testing.T) {
 
 func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	id := fmt.Sprintf("tincep%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 	if !runInstance.wrappedIsExternal {
 		t.Skipf("Not external")
 	}
@@ -633,8 +621,7 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 
 func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	id := fmt.Sprintf("ticsadcf%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -666,8 +653,7 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 
 func TestInternalCacheWrites(t *testing.T) {
 	id := "ticw"
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"writes": "true"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"writes": "true"})
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -688,8 +674,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 		t.Skip("Skip test on windows/386")
 	}
 	id := fmt.Sprintf("timcsr%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"workers": "1"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"workers": "1"})
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -724,8 +709,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 func TestInternalExpiredEntriesRemoved(t *testing.T) {
 	id := fmt.Sprintf("tieer%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second * 4 // needs to be lower than the defined
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, map[string]string{"info_age": "5s"}, nil)
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
 
@@ -762,9 +746,7 @@ func TestInternalBug2117(t *testing.T) {
 	vfsflags.Opt.DirCacheTime = time.Second * 10
 
 	id := fmt.Sprintf("tib2117%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil,
-		map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
 
 	if runInstance.rootIsCrypt {
 		t.Skipf("skipping crypt")
@@ -865,7 +847,7 @@ func (r *run) encryptRemoteIfNeeded(t *testing.T, remote string) string {
 	return enc
 }
 
-func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, cfg map[string]string, flags map[string]string) (fs.Fs, *cache.Persistent) {
+func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, flags map[string]string) (fs.Fs, *cache.Persistent) {
 	fstest.Initialise()
 	remoteExists := false
 	for _, s := range config.FileSections() {
@@ -958,10 +940,15 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	}
 	err = f.Mkdir(context.Background(), "")
 	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		runInstance.cleanupFs(t, f)
+	})
+
 	return f, boltDb
 }
 
-func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
+func (r *run) cleanupFs(t *testing.T, f fs.Fs) {
 	err := f.Features().Purge(context.Background(), "")
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
diff --git a/backend/cache/cache_upload_test.go b/backend/cache/cache_upload_test.go
index 3356613bda086..b6a2c584274bd 100644
--- a/backend/cache/cache_upload_test.go
+++ b/backend/cache/cache_upload_test.go
@@ -21,10 +21,8 @@ import (
 
 func TestInternalUploadTempDirCreated(t *testing.T) {
 	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
-		nil,
+	runInstance.newCacheFs(t, remoteName, id, false, true,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id)})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
 	require.NoError(t, err)
@@ -63,9 +61,7 @@ func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltD
 func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "0s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
@@ -73,19 +69,15 @@ func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1m"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
 
 func TestInternalUploadMoveExistingFile(t *testing.T) {
 	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "3s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
@@ -119,10 +111,8 @@ func TestInternalUploadMoveExistingFile(t *testing.T) {
 
 func TestInternalUploadTempPathCleaned(t *testing.T) {
 	id := fmt.Sprintf("tiutpc%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
 		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
@@ -162,10 +152,8 @@ func TestInternalUploadTempPathCleaned(t *testing.T) {
 
 func TestInternalUploadQueueMoreFiles(t *testing.T) {
 	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
+	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "test")
 	require.NoError(t, err)
@@ -213,9 +201,7 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 func TestInternalUploadTempFileOperations(t *testing.T) {
 	id := "tiutfo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 
@@ -343,9 +329,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	id := "tiuufo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 
diff --git a/backend/crypt/crypt_internal_test.go b/backend/crypt/crypt_internal_test.go
index 07dee6ac525b7..a6eaecbc5a2e6 100644
--- a/backend/crypt/crypt_internal_test.go
+++ b/backend/crypt/crypt_internal_test.go
@@ -19,26 +19,26 @@ import (
 
 // Create a temporary local fs to upload things from
 
-func makeTempLocalFs(t *testing.T) (localFs fs.Fs, cleanup func()) {
+func makeTempLocalFs(t *testing.T) (localFs fs.Fs) {
 	localFs, err := fs.TemporaryLocalFs(context.Background())
 	require.NoError(t, err)
-	cleanup = func() {
+	t.Cleanup(func() {
 		require.NoError(t, localFs.Rmdir(context.Background(), ""))
-	}
-	return localFs, cleanup
+	})
+	return localFs
 }
 
 // Upload a file to a remote
-func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object, cleanup func()) {
+func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object) {
 	inBuf := bytes.NewBufferString(contents)
 	t1 := time.Date(2012, time.December, 17, 18, 32, 31, 0, time.UTC)
 	upSrc := object.NewStaticObjectInfo(remote, t1, int64(len(contents)), true, nil, nil)
 	obj, err := f.Put(context.Background(), inBuf, upSrc)
 	require.NoError(t, err)
-	cleanup = func() {
+	t.Cleanup(func() {
 		require.NoError(t, obj.Remove(context.Background()))
-	}
-	return obj, cleanup
+	})
+	return obj
 }
 
 // Test the ObjectInfo
@@ -52,11 +52,9 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 		path = "_wrap"
 	}
 
-	localFs, cleanupLocalFs := makeTempLocalFs(t)
-	defer cleanupLocalFs()
+	localFs := makeTempLocalFs(t)
 
-	obj, cleanupObj := uploadFile(t, localFs, path, contents)
-	defer cleanupObj()
+	obj := uploadFile(t, localFs, path, contents)
 
 	// encrypt the data
 	inBuf := bytes.NewBufferString(contents)
@@ -103,16 +101,13 @@ func testComputeHash(t *testing.T, f *Fs) {
 		t.Skipf("%v: does not support hashes", f.Fs)
 	}
 
-	localFs, cleanupLocalFs := makeTempLocalFs(t)
-	defer cleanupLocalFs()
+	localFs := makeTempLocalFs(t)
 
 	// Upload a file to localFs as a test object
-	localObj, cleanupLocalObj := uploadFile(t, localFs, path, contents)
-	defer cleanupLocalObj()
+	localObj := uploadFile(t, localFs, path, contents)
 
 	// Upload the same data to the remote Fs also
-	remoteObj, cleanupRemoteObj := uploadFile(t, f, path, contents)
-	defer cleanupRemoteObj()
+	remoteObj := uploadFile(t, f, path, contents)
 
 	// Calculate the expected Hash of the remote object
 	computedHash, err := f.ComputeHash(ctx, remoteObj.(*Object), localObj, hashType)
diff --git a/backend/http/http_internal_test.go b/backend/http/http_internal_test.go
index d0aba666e9bd4..307bbc559f67a 100644
--- a/backend/http/http_internal_test.go
+++ b/backend/http/http_internal_test.go
@@ -33,8 +33,9 @@ var (
 	lineEndSize = 1
 )
 
-// prepareServer the test server and return a function to tidy it up afterwards
-func prepareServer(t *testing.T) (configmap.Simple, func()) {
+// prepareServer prepares the test server and shuts it down automatically
+// when the test completes.
+func prepareServer(t *testing.T) configmap.Simple {
 	// file server for test/files
 	fileServer := http.FileServer(http.Dir(filesPath))
 
@@ -78,20 +79,21 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
 		"url":     ts.URL,
 		"headers": strings.Join(headers, ","),
 	}
+	t.Cleanup(ts.Close)
 
-	// return a function to tidy up
-	return m, ts.Close
+	return m
 }
 
-// prepare the test server and return a function to tidy it up afterwards
-func prepare(t *testing.T) (fs.Fs, func()) {
-	m, tidy := prepareServer(t)
+// prepare prepares the test server and shuts it down automatically
+// when the test completes.
+func prepare(t *testing.T) fs.Fs {
+	m := prepareServer(t)
 
 	// Instantiate it
 	f, err := NewFs(context.Background(), remoteName, "", m)
 	require.NoError(t, err)
 
-	return f, tidy
+	return f
 }
 
 func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
@@ -134,22 +136,19 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 }
 
 func TestListRoot(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 	testListRoot(t, f, false)
 }
 
 func TestListRootNoSlash(t *testing.T) {
-	f, tidy := prepare(t)
+	f := prepare(t)
 	f.(*Fs).opt.NoSlash = true
-	defer tidy()
 
 	testListRoot(t, f, true)
 }
 
 func TestListSubDir(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 
 	entries, err := f.List(context.Background(), "three")
 	require.NoError(t, err)
@@ -166,8 +165,7 @@ func TestListSubDir(t *testing.T) {
 }
 
 func TestNewObject(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 
 	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
@@ -194,8 +192,7 @@ func TestNewObject(t *testing.T) {
 }
 
 func TestOpen(t *testing.T) {
-	m, tidy := prepareServer(t)
-	defer tidy()
+	m := prepareServer(t)
 
 	for _, head := range []bool{false, true} {
 		if !head {
@@ -257,8 +254,7 @@ func TestOpen(t *testing.T) {
 }
 
 func TestMimeType(t *testing.T) {
-	f, tidy := prepare(t)
-	defer tidy()
+	f := prepare(t)
 
 	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
@@ -269,8 +265,7 @@ func TestMimeType(t *testing.T) {
 }
 
 func TestIsAFileRoot(t *testing.T) {
-	m, tidy := prepareServer(t)
-	defer tidy()
+	m := prepareServer(t)
 
 	f, err := NewFs(context.Background(), remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
@@ -279,8 +274,7 @@ func TestIsAFileRoot(t *testing.T) {
 }
 
 func TestIsAFileSubDir(t *testing.T) {
-	m, tidy := prepareServer(t)
-	defer tidy()
+	m := prepareServer(t)
 
 	f, err := NewFs(context.Background(), remoteName, "three/underthree.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go
index a8c48329a04e5..0b5d124809f2a 100644
--- a/backend/local/local_internal_test.go
+++ b/backend/local/local_internal_test.go
@@ -33,7 +33,6 @@ func TestMain(m *testing.M) {
 // Test copy with source file that's updating
 func TestUpdatingCheck(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	filePath := "sub dir/local test"
 	r.WriteFile(filePath, "content", time.Now())
 
@@ -78,7 +77,6 @@ func TestUpdatingCheck(t *testing.T) {
 func TestSymlink(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	f := r.Flocal.(*Fs)
 	dir := f.root
 
@@ -177,7 +175,6 @@ func TestSymlinkError(t *testing.T) {
 func TestHashOnUpdate(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	const filePath = "file.txt"
 	when := time.Now()
 	r.WriteFile(filePath, "content", when)
@@ -208,7 +205,6 @@ func TestHashOnUpdate(t *testing.T) {
 func TestHashOnDelete(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	const filePath = "file.txt"
 	when := time.Now()
 	r.WriteFile(filePath, "content", when)
@@ -237,7 +233,6 @@ func TestHashOnDelete(t *testing.T) {
 func TestMetadata(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	const filePath = "metafile.txt"
 	when := time.Now()
 	const dayLength = len("2001-01-01")
@@ -372,7 +367,6 @@ func TestMetadata(t *testing.T) {
 func TestFilter(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	when := time.Now()
 	r.WriteFile("included", "included file", when)
 	r.WriteFile("excluded", "excluded file", when)
diff --git a/cmd/selfupdate/selfupdate_test.go b/cmd/selfupdate/selfupdate_test.go
index d4ce2ec2ad279..9c02f458412ac 100644
--- a/cmd/selfupdate/selfupdate_test.go
+++ b/cmd/selfupdate/selfupdate_test.go
@@ -15,8 +15,6 @@ import (
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fstest/testy"
-	"github.com/rclone/rclone/lib/file"
-	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -46,20 +44,6 @@ func TestGetVersion(t *testing.T) {
 	assert.Equal(t, "v1.52.3", resultVer)
 }
 
-func makeTestDir() (testDir string, err error) {
-	const maxAttempts = 5
-	testDirBase := filepath.Join(os.TempDir(), "rclone-test-selfupdate.")
-
-	for attempt := 0; attempt < maxAttempts; attempt++ {
-		testDir = testDirBase + random.String(4)
-		err = file.MkdirAll(testDir, os.ModePerm)
-		if err == nil {
-			break
-		}
-	}
-	return
-}
-
 func TestInstallOnLinux(t *testing.T) {
 	testy.SkipUnreliable(t)
 	if runtime.GOOS != "linux" {
@@ -68,13 +52,8 @@ func TestInstallOnLinux(t *testing.T) {
 
 	// Prepare for test
 	ctx := context.Background()
-	testDir, err := makeTestDir()
-	assert.NoError(t, err)
+	testDir := t.TempDir()
 	path := filepath.Join(testDir, "rclone")
-	defer func() {
-		_ = os.Chmod(path, 0644)
-		_ = os.RemoveAll(testDir)
-	}()
 
 	regexVer := regexp.MustCompile(`v[0-9]\S+`)
 
@@ -87,6 +66,9 @@ func TestInstallOnLinux(t *testing.T) {
 	// Must fail on non-writable file
 	assert.NoError(t, os.WriteFile(path, []byte("test"), 0644))
 	assert.NoError(t, os.Chmod(path, 0000))
+	defer func() {
+		_ = os.Chmod(path, 0644)
+	}()
 	err = (InstallUpdate(ctx, &Options{Beta: true, Output: path}))
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "run self-update as root")
@@ -122,11 +104,7 @@ func TestRenameOnWindows(t *testing.T) {
 	// Prepare for test
 	ctx := context.Background()
 
-	testDir, err := makeTestDir()
-	assert.NoError(t, err)
-	defer func() {
-		_ = os.RemoveAll(testDir)
-	}()
+	testDir := t.TempDir()
 
 	path := filepath.Join(testDir, "rclone.exe")
 	regexVer := regexp.MustCompile(`v[0-9]\S+`)
diff --git a/cmd/touch/touch_test.go b/cmd/touch/touch_test.go
index 4156154383777..47e9045495cbe 100644
--- a/cmd/touch/touch_test.go
+++ b/cmd/touch/touch_test.go
@@ -28,7 +28,6 @@ func TestMain(m *testing.M) {
 
 func TestTouchOneFile(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	err := Touch(context.Background(), r.Fremote, "newFile")
 	require.NoError(t, err)
@@ -38,7 +37,6 @@ func TestTouchOneFile(t *testing.T) {
 
 func TestTouchWithNoCreateFlag(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	notCreateNewFile = true
 	err := Touch(context.Background(), r.Fremote, "newFile")
@@ -50,7 +48,6 @@ func TestTouchWithNoCreateFlag(t *testing.T) {
 
 func TestTouchWithTimestamp(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	timeAsArgument = "060102"
 	srcFileName := "oldFile"
@@ -61,7 +58,6 @@ func TestTouchWithTimestamp(t *testing.T) {
 
 func TestTouchWithLongerTimestamp(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	timeAsArgument = "2006-01-02T15:04:05"
 	srcFileName := "oldFile"
@@ -72,7 +68,6 @@ func TestTouchWithLongerTimestamp(t *testing.T) {
 
 func TestTouchUpdateTimestamp(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	srcFileName := "a"
 	content := "aaa"
@@ -87,7 +82,6 @@ func TestTouchUpdateTimestamp(t *testing.T) {
 
 func TestTouchUpdateTimestampWithCFlag(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	srcFileName := "a"
 	content := "aaa"
@@ -104,7 +98,6 @@ func TestTouchUpdateTimestampWithCFlag(t *testing.T) {
 
 func TestTouchCreateMultipleDirAndFile(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	longPath := "a/b/c.txt"
 	err := Touch(context.Background(), r.Fremote, longPath)
@@ -115,7 +108,6 @@ func TestTouchCreateMultipleDirAndFile(t *testing.T) {
 
 func TestTouchEmptyName(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	err := Touch(context.Background(), r.Fremote, "")
 	require.NoError(t, err)
@@ -124,7 +116,6 @@ func TestTouchEmptyName(t *testing.T) {
 
 func TestTouchEmptyDir(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	err := r.Fremote.Mkdir(context.Background(), "a")
 	require.NoError(t, err)
@@ -135,7 +126,6 @@ func TestTouchEmptyDir(t *testing.T) {
 
 func TestTouchDirWithFiles(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	err := r.Fremote.Mkdir(context.Background(), "a")
 	require.NoError(t, err)
@@ -148,7 +138,6 @@ func TestTouchDirWithFiles(t *testing.T) {
 
 func TestRecursiveTouchDirWithFiles(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	err := r.Fremote.Mkdir(context.Background(), "a/b/c")
 	require.NoError(t, err)
diff --git a/cmdtest/cmdtest_test.go b/cmdtest/cmdtest_test.go
index 62f51f28a4978..c8870258cc36f 100644
--- a/cmdtest/cmdtest_test.go
+++ b/cmdtest/cmdtest_test.go
@@ -111,13 +111,6 @@ func createTestEnvironment(t *testing.T) {
 var testFolder string
 var testConfig string
 
-// removeTestEnvironment removes the test environment created by createTestEnvironment
-func removeTestEnvironment(t *testing.T) {
-	// Remove temporary folder with all contents
-	err := os.RemoveAll(testFolder)
-	require.NoError(t, err)
-}
-
 // createTestFile creates the file testFolder/name
 func createTestFile(name string, t *testing.T) string {
 	err := os.WriteFile(testFolder+"/"+name, []byte("content_of_"+name), 0666)
@@ -146,19 +139,18 @@ func createSimpleTestData(t *testing.T) string {
 	createTestFolder("testdata/folderB", t)
 	createTestFile("testdata/folderB/fileB1.txt", t)
 	createTestFile("testdata/folderB/fileB2.txt", t)
-	return testFolder + "/testdata"
-}
 
-// removeSimpleTestData removes the test data created by createSimpleTestData
-func removeSimpleTestData(t *testing.T) {
-	err := os.RemoveAll(testFolder + "/testdata")
-	require.NoError(t, err)
+	t.Cleanup(func() {
+		err := os.RemoveAll(testFolder + "/testdata")
+		require.NoError(t, err)
+	})
+
+	return testFolder + "/testdata"
 }
 
 // TestCmdTest demonstrates and verifies the test functions for end-to-end testing of rclone
 func TestCmdTest(t *testing.T) {
 	createTestEnvironment(t)
-	defer removeTestEnvironment(t)
 
 	// Test simple call and output from rclone
 	out, err := rclone("version")
@@ -213,7 +205,6 @@ func TestCmdTest(t *testing.T) {
 
 	// Test creation of simple test data
 	createSimpleTestData(t)
-	defer removeSimpleTestData(t)
 
 	// Test access to config file and simple test data
 	out, err = rclone("lsl", "myLocal:"+testFolder)
diff --git a/cmdtest/environment_test.go b/cmdtest/environment_test.go
index b5b27eab56f42..f0688200dc882 100644
--- a/cmdtest/environment_test.go
+++ b/cmdtest/environment_test.go
@@ -17,10 +17,8 @@ import (
 func TestEnvironmentVariables(t *testing.T) {
 
 	createTestEnvironment(t)
-	defer removeTestEnvironment(t)
 
 	testdataPath := createSimpleTestData(t)
-	defer removeSimpleTestData(t)
 
 	// Non backend flags
 	// =================
diff --git a/fs/cache/cache_test.go b/fs/cache/cache_test.go
index c03aa7dfaa098..f4e5eb9c26809 100644
--- a/fs/cache/cache_test.go
+++ b/fs/cache/cache_test.go
@@ -16,7 +16,7 @@ var (
 	errSentinel = errors.New("an error")
 )
 
-func mockNewFs(t *testing.T) (func(), func(ctx context.Context, path string) (fs.Fs, error)) {
+func mockNewFs(t *testing.T) func(ctx context.Context, path string) (fs.Fs, error) {
 	called = 0
 	create := func(ctx context.Context, path string) (f fs.Fs, err error) {
 		assert.Equal(t, 0, called)
@@ -32,15 +32,12 @@ func mockNewFs(t *testing.T) (func(), func(ctx context.Context, path string) (fs
 		t.Fatalf("Unknown path %q", path)
 		panic("unreachable")
 	}
-	cleanup := func() {
-		Clear()
-	}
-	return cleanup, create
+	t.Cleanup(Clear)
+	return create
 }
 
 func TestGet(t *testing.T) {
-	cleanup, create := mockNewFs(t)
-	defer cleanup()
+	create := mockNewFs(t)
 
 	assert.Equal(t, 0, Entries())
 
@@ -56,8 +53,7 @@ func TestGet(t *testing.T) {
 }
 
 func TestGetFile(t *testing.T) {
-	cleanup, create := mockNewFs(t)
-	defer cleanup()
+	create := mockNewFs(t)
 
 	assert.Equal(t, 0, Entries())
 
@@ -82,8 +78,7 @@ func TestGetFile(t *testing.T) {
 }
 
 func TestGetFile2(t *testing.T) {
-	cleanup, create := mockNewFs(t)
-	defer cleanup()
+	create := mockNewFs(t)
 
 	assert.Equal(t, 0, Entries())
 
@@ -108,8 +103,7 @@ func TestGetFile2(t *testing.T) {
 }
 
 func TestGetError(t *testing.T) {
-	cleanup, create := mockNewFs(t)
-	defer cleanup()
+	create := mockNewFs(t)
 
 	assert.Equal(t, 0, Entries())
 
@@ -121,8 +115,7 @@ func TestGetError(t *testing.T) {
 }
 
 func TestPut(t *testing.T) {
-	cleanup, create := mockNewFs(t)
-	defer cleanup()
+	create := mockNewFs(t)
 
 	f := mockfs.NewFs(context.Background(), "mock", "/alien")
 
@@ -151,8 +144,7 @@ func TestPut(t *testing.T) {
 }
 
 func TestPin(t *testing.T) {
-	cleanup, create := mockNewFs(t)
-	defer cleanup()
+	create := mockNewFs(t)
 
 	// Test pinning and unpinning nonexistent
 	f := mockfs.NewFs(context.Background(), "mock", "/alien")
@@ -167,8 +159,7 @@ func TestPin(t *testing.T) {
 }
 
 func TestClearConfig(t *testing.T) {
-	cleanup, create := mockNewFs(t)
-	defer cleanup()
+	create := mockNewFs(t)
 
 	assert.Equal(t, 0, Entries())
 
@@ -183,8 +174,7 @@ func TestClearConfig(t *testing.T) {
 }
 
 func TestClear(t *testing.T) {
-	cleanup, create := mockNewFs(t)
-	defer cleanup()
+	create := mockNewFs(t)
 
 	// Create something
 	_, err := GetFn(context.Background(), "mock:/", create)
@@ -198,8 +188,7 @@ func TestClear(t *testing.T) {
 }
 
 func TestEntries(t *testing.T) {
-	cleanup, create := mockNewFs(t)
-	defer cleanup()
+	create := mockNewFs(t)
 
 	assert.Equal(t, 0, Entries())
 
diff --git a/fs/march/march_test.go b/fs/march/march_test.go
index 31814255ef8df..9906b8ee80003 100644
--- a/fs/march/march_test.go
+++ b/fs/march/march_test.go
@@ -170,7 +170,6 @@ func TestMarch(t *testing.T) {
 	} {
 		t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) {
 			r := fstest.NewRun(t)
-			defer r.Finalise()
 
 			var srcOnly []fstest.Item
 			var dstOnly []fstest.Item
@@ -242,7 +241,6 @@ func TestMarchNoTraverse(t *testing.T) {
 	} {
 		t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) {
 			r := fstest.NewRun(t)
-			defer r.Finalise()
 
 			var srcOnly []fstest.Item
 			var match []fstest.Item
diff --git a/fs/operations/check_test.go b/fs/operations/check_test.go
index be22051fe5fd4..592485ed201b4 100644
--- a/fs/operations/check_test.go
+++ b/fs/operations/check_test.go
@@ -24,7 +24,6 @@ import (
 
 func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operations.CheckOpt) error) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	ctx := context.Background()
 	ci := fs.GetConfig(ctx)
 
@@ -279,7 +278,6 @@ func TestCheckEqualReaders(t *testing.T) {
 
 func TestParseSumFile(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	ctx := context.Background()
 
 	const sumFile = "test.sum"
@@ -342,7 +340,6 @@ func testCheckSum(t *testing.T, download bool) {
 
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	subRemote := r.FremoteName
 	if !strings.HasSuffix(subRemote, ":") {
diff --git a/fs/operations/dedupe_test.go b/fs/operations/dedupe_test.go
index 60c960510d050..15a4e2db9b64a 100644
--- a/fs/operations/dedupe_test.go
+++ b/fs/operations/dedupe_test.go
@@ -45,7 +45,6 @@ func skipIfNoModTime(t *testing.T, f fs.Fs) {
 
 func TestDeduplicateInteractive(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	skipIfCantDedupe(t, r.Fremote)
 	skipIfNoHash(t, r.Fremote)
 
@@ -62,7 +61,6 @@ func TestDeduplicateInteractive(t *testing.T) {
 
 func TestDeduplicateSkip(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	skipIfCantDedupe(t, r.Fremote)
 	haveHash := r.Fremote.Hashes().GetOne() != hash.None
 
@@ -84,7 +82,6 @@ func TestDeduplicateSkip(t *testing.T) {
 
 func TestDeduplicateSizeOnly(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	skipIfCantDedupe(t, r.Fremote)
 	ctx := context.Background()
 	ci := fs.GetConfig(ctx)
@@ -107,7 +104,6 @@ func TestDeduplicateSizeOnly(t *testing.T) {
 
 func TestDeduplicateFirst(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	skipIfCantDedupe(t, r.Fremote)
 
 	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
@@ -136,7 +132,6 @@ func TestDeduplicateFirst(t *testing.T) {
 
 func TestDeduplicateNewest(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	skipIfCantDedupe(t, r.Fremote)
 	skipIfNoModTime(t, r.Fremote)
 
@@ -153,7 +148,6 @@ func TestDeduplicateNewest(t *testing.T) {
 
 func TestDeduplicateNewestByHash(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	skipIfNoHash(t, r.Fremote)
 	skipIfNoModTime(t, r.Fremote)
 	contents := random.String(100)
@@ -172,7 +166,6 @@ func TestDeduplicateNewestByHash(t *testing.T) {
 
 func TestDeduplicateOldest(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	skipIfCantDedupe(t, r.Fremote)
 
 	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
@@ -188,7 +181,6 @@ func TestDeduplicateOldest(t *testing.T) {
 
 func TestDeduplicateLargest(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	skipIfCantDedupe(t, r.Fremote)
 
 	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
@@ -204,7 +196,6 @@ func TestDeduplicateLargest(t *testing.T) {
 
 func TestDeduplicateSmallest(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	skipIfCantDedupe(t, r.Fremote)
 
 	file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
@@ -220,7 +211,6 @@ func TestDeduplicateSmallest(t *testing.T) {
 
 func TestDeduplicateRename(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	skipIfCantDedupe(t, r.Fremote)
 
 	file1 := r.WriteUncheckedObject(context.Background(), "one.txt", "This is one", t1)
@@ -260,7 +250,6 @@ func TestDeduplicateRename(t *testing.T) {
 // doesn't have enough tools to make it easy
 func TestMergeDirs(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	mergeDirs := r.Fremote.Features().MergeDirs
 	if mergeDirs == nil {
diff --git a/fs/operations/listdirsorted_test.go b/fs/operations/listdirsorted_test.go
index 5a4e3eeeade1a..1bcd8860e124b 100644
--- a/fs/operations/listdirsorted_test.go
+++ b/fs/operations/listdirsorted_test.go
@@ -16,7 +16,6 @@ import (
 // which can't be tested there due to import loops.
 func TestListDirSorted(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	ctx := context.Background()
 	fi := filter.GetConfig(ctx)
diff --git a/fs/operations/lsjson_test.go b/fs/operations/lsjson_test.go
index a8943cfbd0602..c9be36173f116 100644
--- a/fs/operations/lsjson_test.go
+++ b/fs/operations/lsjson_test.go
@@ -37,7 +37,6 @@ func compareListJSONItem(t *testing.T, a, b *operations.ListJSONItem, precision
 func TestListJSON(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "file1", "file1", t1)
 	file2 := r.WriteBoth(ctx, "sub/file2", "sub/file2", t2)
 
@@ -245,7 +244,6 @@ func TestListJSON(t *testing.T) {
 func TestStatJSON(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "file1", "file1", t1)
 	file2 := r.WriteBoth(ctx, "sub/file2", "sub/file2", t2)
 
diff --git a/fs/operations/multithread_test.go b/fs/operations/multithread_test.go
index 4c4a71b0a95b3..a036139e5d924 100644
--- a/fs/operations/multithread_test.go
+++ b/fs/operations/multithread_test.go
@@ -106,7 +106,6 @@ func TestMultithreadCalculateChunks(t *testing.T) {
 
 func TestMultithreadCopy(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	ctx := context.Background()
 
 	for _, test := range []struct {
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index b9be649027ba3..d1e2d8fdb6e2d 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -65,7 +65,6 @@ func TestMain(m *testing.M) {
 func TestMkdir(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	err := operations.Mkdir(ctx, r.Fremote, "")
 	require.NoError(t, err)
@@ -78,7 +77,6 @@ func TestMkdir(t *testing.T) {
 func TestLsd(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1)
 
 	r.CheckRemoteItems(t, file1)
@@ -93,7 +91,6 @@ func TestLsd(t *testing.T) {
 func TestLs(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
 	file2 := r.WriteBoth(ctx, "empty space", "-", t2)
 
@@ -111,7 +108,6 @@ func TestLsWithFilesFrom(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
 	file2 := r.WriteBoth(ctx, "empty space", "-", t2)
 
@@ -143,7 +139,6 @@ func TestLsWithFilesFrom(t *testing.T) {
 func TestLsLong(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
 	file2 := r.WriteBoth(ctx, "empty space", "-", t2)
 
@@ -186,7 +181,6 @@ func TestLsLong(t *testing.T) {
 func TestHashSums(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
 	file2 := r.WriteBoth(ctx, "empty space", "-", t2)
 
@@ -392,7 +386,6 @@ func TestCount(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
 	file2 := r.WriteBoth(ctx, "empty space", "-", t2)
 	file3 := r.WriteBoth(ctx, "sub dir/potato3", "hello", t2)
@@ -416,7 +409,6 @@ func TestDelete(t *testing.T) {
 	fi.Opt.MaxSize = 60
 	ctx = filter.ReplaceConfig(ctx, fi)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteObject(ctx, "small", "1234567890", t2)                                                                                           // 10 bytes
 	file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1)                                        // 60 bytes
 	file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
@@ -457,7 +449,6 @@ func TestRetry(t *testing.T) {
 func TestCat(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "file1", "ABCDEFGHIJ", t1)
 	file2 := r.WriteBoth(ctx, "file2", "012345678", t2)
 
@@ -488,7 +479,6 @@ func TestCat(t *testing.T) {
 func TestPurge(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRunIndividual(t) // make new container (azureblob has delayed mkdir after rmdir)
-	defer r.Finalise()
 	r.Mkdir(ctx, r.Fremote)
 
 	// Make some files and dirs
@@ -561,7 +551,6 @@ func TestPurge(t *testing.T) {
 func TestRmdirsNoLeaveRoot(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	r.Mkdir(ctx, r.Fremote)
 
 	// Make some files and dirs we expect to keep
@@ -642,7 +631,6 @@ func TestRmdirsNoLeaveRoot(t *testing.T) {
 func TestRmdirsLeaveRoot(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	r.Mkdir(ctx, r.Fremote)
 
 	r.ForceMkdir(ctx, r.Fremote)
@@ -682,7 +670,6 @@ func TestRmdirsWithFilter(t *testing.T) {
 	require.NoError(t, fi.AddRule("+ /A1/B1/**"))
 	require.NoError(t, fi.AddRule("- *"))
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	r.Mkdir(ctx, r.Fremote)
 
 	r.ForceMkdir(ctx, r.Fremote)
@@ -720,7 +707,6 @@ func TestCopyURL(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	contents := "file contents\n"
 	file1 := r.WriteFile("file1", contents, t1)
@@ -835,7 +821,6 @@ func TestCopyURLToWriter(t *testing.T) {
 func TestMoveFile(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	file1 := r.WriteFile("file1", "file1 contents", t1)
 	r.CheckLocalItems(t, file1)
@@ -866,7 +851,6 @@ func TestMoveFileWithIgnoreExisting(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("file1", "file1 contents", t1)
 	r.CheckLocalItems(t, file1)
 
@@ -891,7 +875,6 @@ func TestMoveFileWithIgnoreExisting(t *testing.T) {
 func TestCaseInsensitiveMoveFile(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	if !r.Fremote.Features().CaseInsensitive {
 		return
 	}
@@ -928,7 +911,6 @@ func TestMoveFileBackupDir(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	if !operations.CanServerSideMove(r.Fremote) {
 		t.Skip("Skipping test as remote does not support server-side move or copy")
 	}
@@ -951,7 +933,6 @@ func TestMoveFileBackupDir(t *testing.T) {
 func TestCopyFile(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	file1 := r.WriteFile("file1", "file1 contents", t1)
 	r.CheckLocalItems(t, file1)
@@ -979,7 +960,6 @@ func TestCopyFileBackupDir(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	if !operations.CanServerSideMove(r.Fremote) {
 		t.Skip("Skipping test as remote does not support server-side move or copy")
 	}
@@ -1004,7 +984,6 @@ func TestCopyFileCompareDest(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
 	fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
@@ -1083,7 +1062,6 @@ func TestCopyFileCopyDest(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	if r.Fremote.Features().Copy == nil {
 		t.Skip("Skipping test as remote does not support server-side copy")
@@ -1457,7 +1435,6 @@ func TestListFormat(t *testing.T) {
 func TestDirMove(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	r.Mkdir(ctx, r.Fremote)
 
@@ -1538,7 +1515,6 @@ func TestDirMove(t *testing.T) {
 
 func TestGetFsInfo(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	f := r.Fremote
 	info := operations.GetFsInfo(f)
@@ -1559,7 +1535,7 @@ func TestGetFsInfo(t *testing.T) {
 func TestRcat(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
-	check := func(withChecksum, ignoreChecksum bool) {
+	check := func(t *testing.T, withChecksum, ignoreChecksum bool) {
 		ci.CheckSum, ci.IgnoreChecksum = withChecksum, ignoreChecksum
 
 		var prefix string
@@ -1573,7 +1549,6 @@ func TestRcat(t *testing.T) {
 		}
 
 		r := fstest.NewRun(t)
-		defer r.Finalise()
 
 		if *fstest.SizeLimit > 0 && int64(ci.StreamingUploadCutoff) > *fstest.SizeLimit {
 			savedCutoff := ci.StreamingUploadCutoff
@@ -1606,14 +1581,13 @@ func TestRcat(t *testing.T) {
 		withChecksum := (i & 1) != 0
 		ignoreChecksum := (i & 2) != 0
 		t.Run(fmt.Sprintf("withChecksum=%v,ignoreChecksum=%v", withChecksum, ignoreChecksum), func(t *testing.T) {
-			check(withChecksum, ignoreChecksum)
+			check(t, withChecksum, ignoreChecksum)
 		})
 	}
 }
 
 func TestRcatMetadata(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	if !r.Fremote.Features().UserMetadata {
 		t.Skip("Skipping as destination doesn't support user metadata")
@@ -1670,7 +1644,6 @@ func TestRcatMetadata(t *testing.T) {
 func TestRcatSize(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	const body = "------------------------------------------------------------"
 	file1 := r.WriteFile("potato1", body, t1)
@@ -1696,7 +1669,6 @@ func TestRcatSize(t *testing.T) {
 
 func TestRcatSizeMetadata(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	if !r.Fremote.Features().UserMetadata {
 		t.Skip("Skipping as destination doesn't support user metadata")
@@ -1750,7 +1722,6 @@ func TestCopyFileMaxTransfer(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	defer accounting.Stats(ctx).ResetCounters()
 
 	const sizeCutoff = 2048
@@ -1817,7 +1788,6 @@ func TestCopyFileMaxTransfer(t *testing.T) {
 func TestTouchDir(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	if r.Fremote.Precision() == fs.ModTimeNotSupported {
 		t.Skip("Skipping test as remote does not support modtime")
diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go
index 16b3267f4357f..b73c63ff9fa01 100644
--- a/fs/operations/rc_test.go
+++ b/fs/operations/rc_test.go
@@ -35,7 +35,6 @@ func rcNewRun(t *testing.T, method string) (*fstest.Run, *rc.Call) {
 // operations/about: Return the space used on the remote
 func TestRcAbout(t *testing.T) {
 	r, call := rcNewRun(t, "operations/about")
-	defer r.Finalise()
 	r.Mkdir(context.Background(), r.Fremote)
 
 	// Will get an error if remote doesn't support About
@@ -58,7 +57,6 @@ func TestRcAbout(t *testing.T) {
 // operations/cleanup: Remove trashed files in the remote or path
 func TestRcCleanup(t *testing.T) {
 	r, call := rcNewRun(t, "operations/cleanup")
-	defer r.Finalise()
 
 	in := rc.Params{
 		"fs": r.LocalName,
@@ -72,7 +70,6 @@ func TestRcCleanup(t *testing.T) {
 // operations/copyfile: Copy a file from source remote to destination remote
 func TestRcCopyfile(t *testing.T) {
 	r, call := rcNewRun(t, "operations/copyfile")
-	defer r.Finalise()
 	file1 := r.WriteFile("file1", "file1 contents", t1)
 	r.Mkdir(context.Background(), r.Fremote)
 	r.CheckLocalItems(t, file1)
@@ -96,7 +93,6 @@ func TestRcCopyfile(t *testing.T) {
 // operations/copyurl: Copy the URL to the object
 func TestRcCopyurl(t *testing.T) {
 	r, call := rcNewRun(t, "operations/copyurl")
-	defer r.Finalise()
 	contents := "file1 contents\n"
 	file1 := r.WriteFile("file1", contents, t1)
 	r.Mkdir(context.Background(), r.Fremote)
@@ -159,7 +155,6 @@ func TestRcCopyurl(t *testing.T) {
 // operations/delete: Remove files in the path
 func TestRcDelete(t *testing.T) {
 	r, call := rcNewRun(t, "operations/delete")
-	defer r.Finalise()
 
 	file1 := r.WriteObject(context.Background(), "small", "1234567890", t2)                                                                                           // 10 bytes
 	file2 := r.WriteObject(context.Background(), "medium", "------------------------------------------------------------", t1)                                        // 60 bytes
@@ -179,7 +174,6 @@ func TestRcDelete(t *testing.T) {
 // operations/deletefile: Remove the single file pointed to
 func TestRcDeletefile(t *testing.T) {
 	r, call := rcNewRun(t, "operations/deletefile")
-	defer r.Finalise()
 
 	file1 := r.WriteObject(context.Background(), "small", "1234567890", t2)                                                    // 10 bytes
 	file2 := r.WriteObject(context.Background(), "medium", "------------------------------------------------------------", t1) // 60 bytes
@@ -199,7 +193,6 @@ func TestRcDeletefile(t *testing.T) {
 // operations/list: List the given remote and path in JSON format.
 func TestRcList(t *testing.T) {
 	r, call := rcNewRun(t, "operations/list")
-	defer r.Finalise()
 
 	file1 := r.WriteObject(context.Background(), "a", "a", t1)
 	file2 := r.WriteObject(context.Background(), "subdir/b", "bb", t2)
@@ -264,7 +257,6 @@ func TestRcList(t *testing.T) {
 // operations/stat: Stat the given remote and path in JSON format.
 func TestRcStat(t *testing.T) {
 	r, call := rcNewRun(t, "operations/stat")
-	defer r.Finalise()
 
 	file1 := r.WriteObject(context.Background(), "subdir/a", "a", t1)
 
@@ -318,7 +310,6 @@ func TestRcStat(t *testing.T) {
 func TestRcMkdir(t *testing.T) {
 	ctx := context.Background()
 	r, call := rcNewRun(t, "operations/mkdir")
-	defer r.Finalise()
 	r.Mkdir(context.Background(), r.Fremote)
 
 	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote))
@@ -337,7 +328,6 @@ func TestRcMkdir(t *testing.T) {
 // operations/movefile: Move a file from source remote to destination remote
 func TestRcMovefile(t *testing.T) {
 	r, call := rcNewRun(t, "operations/movefile")
-	defer r.Finalise()
 	file1 := r.WriteFile("file1", "file1 contents", t1)
 	r.Mkdir(context.Background(), r.Fremote)
 	r.CheckLocalItems(t, file1)
@@ -362,7 +352,6 @@ func TestRcMovefile(t *testing.T) {
 func TestRcPurge(t *testing.T) {
 	ctx := context.Background()
 	r, call := rcNewRun(t, "operations/purge")
-	defer r.Finalise()
 	file1 := r.WriteObject(context.Background(), "subdir/file1", "subdir/file1 contents", t1)
 
 	fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote))
@@ -382,7 +371,6 @@ func TestRcPurge(t *testing.T) {
 func TestRcRmdir(t *testing.T) {
 	ctx := context.Background()
 	r, call := rcNewRun(t, "operations/rmdir")
-	defer r.Finalise()
 	r.Mkdir(context.Background(), r.Fremote)
 	assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir"))
 
@@ -403,7 +391,6 @@ func TestRcRmdir(t *testing.T) {
 func TestRcRmdirs(t *testing.T) {
 	ctx := context.Background()
 	r, call := rcNewRun(t, "operations/rmdirs")
-	defer r.Finalise()
 	r.Mkdir(context.Background(), r.Fremote)
 	assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir"))
 	assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir/subsubdir"))
@@ -439,7 +426,6 @@ func TestRcRmdirs(t *testing.T) {
 // operations/size: Count the number of bytes and files in remote
 func TestRcSize(t *testing.T) {
 	r, call := rcNewRun(t, "operations/size")
-	defer r.Finalise()
 	file1 := r.WriteObject(context.Background(), "small", "1234567890", t2)                                                           // 10 bytes
 	file2 := r.WriteObject(context.Background(), "subdir/medium", "------------------------------------------------------------", t1) // 60 bytes
 	file3 := r.WriteObject(context.Background(), "subdir/subsubdir/large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1)  // 50 bytes
@@ -460,7 +446,6 @@ func TestRcSize(t *testing.T) {
 // operations/publiclink: Create or retrieve a public link to the given file or folder.
 func TestRcPublicLink(t *testing.T) {
 	r, call := rcNewRun(t, "operations/publiclink")
-	defer r.Finalise()
 	in := rc.Params{
 		"fs":     r.FremoteName,
 		"remote": "",
@@ -475,7 +460,6 @@ func TestRcPublicLink(t *testing.T) {
 // operations/fsinfo: Return information about the remote
 func TestRcFsInfo(t *testing.T) {
 	r, call := rcNewRun(t, "operations/fsinfo")
-	defer r.Finalise()
 	in := rc.Params{
 		"fs": r.FremoteName,
 	}
@@ -502,7 +486,6 @@ func TestRcFsInfo(t *testing.T) {
 // operations/uploadfile : Tests if upload file succeeds
 func TestUploadFile(t *testing.T) {
 	r, call := rcNewRun(t, "operations/uploadfile")
-	defer r.Finalise()
 	ctx := context.Background()
 
 	testFileName := "test.txt"
@@ -558,7 +541,6 @@ func TestUploadFile(t *testing.T) {
 // operations/command: Runs a backend command
 func TestRcCommand(t *testing.T) {
 	r, call := rcNewRun(t, "backend/command")
-	defer r.Finalise()
 	in := rc.Params{
 		"fs":      r.FremoteName,
 		"command": "noop",
diff --git a/fs/rc/webgui/rc_test.go b/fs/rc/webgui/rc_test.go
index 4ba6b5cdb0778..ec003d23fb188 100644
--- a/fs/rc/webgui/rc_test.go
+++ b/fs/rc/webgui/rc_test.go
@@ -2,7 +2,6 @@ package webgui
 
 import (
 	"context"
-	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -22,7 +21,7 @@ func init() {
 	rcflags.Opt.WebUI = true
 }
 
-func setCacheDir(t *testing.T) string {
+func setCacheDir(t *testing.T) {
 	cacheDir := t.TempDir()
 	PluginsPath = filepath.Join(cacheDir, "plugins")
 	pluginsConfigPath = filepath.Join(cacheDir, "config")
@@ -30,11 +29,6 @@ func setCacheDir(t *testing.T) string {
 	loadedPlugins = newPlugins(availablePluginsJSONPath)
 	err := loadedPlugins.readFromFile()
 	assert.Nil(t, err)
-	return cacheDir
-}
-
-func cleanCacheDir(t *testing.T, cacheDir string) {
-	_ = os.RemoveAll(cacheDir)
 }
 
 func addPlugin(t *testing.T) {
@@ -88,8 +82,7 @@ func removePlugin(t *testing.T) {
 //}
 
 func TestAddPlugin(t *testing.T) {
-	cacheDir := setCacheDir(t)
-	defer cleanCacheDir(t, cacheDir)
+	setCacheDir(t)
 
 	addPlugin(t)
 	_, ok := loadedPlugins.LoadedPlugins[testPluginKey]
@@ -101,8 +94,7 @@ func TestAddPlugin(t *testing.T) {
 }
 
 func TestListPlugins(t *testing.T) {
-	cacheDir := setCacheDir(t)
-	defer cleanCacheDir(t, cacheDir)
+	setCacheDir(t)
 
 	addPlugin := rc.Calls.Get("pluginsctl/listPlugins")
 	assert.NotNil(t, addPlugin)
@@ -117,8 +109,7 @@ func TestListPlugins(t *testing.T) {
 }
 
 func TestRemovePlugin(t *testing.T) {
-	cacheDir := setCacheDir(t)
-	defer cleanCacheDir(t, cacheDir)
+	setCacheDir(t)
 
 	addPlugin(t)
 	removePluginCall := rc.Calls.Get("pluginsctl/removePlugin")
diff --git a/fs/sync/rc_test.go b/fs/sync/rc_test.go
index 0e0931bb28c3e..ad5145633b0ad 100644
--- a/fs/sync/rc_test.go
+++ b/fs/sync/rc_test.go
@@ -26,7 +26,6 @@ func rcNewRun(t *testing.T, method string) (*fstest.Run, *rc.Call) {
 // sync/copy: copy a directory from source remote to destination remote
 func TestRcCopy(t *testing.T) {
 	r, call := rcNewRun(t, "sync/copy")
-	defer r.Finalise()
 	r.Mkdir(context.Background(), r.Fremote)
 
 	file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1)
@@ -51,7 +50,6 @@ func TestRcCopy(t *testing.T) {
 // sync/move: move a directory from source remote to destination remote
 func TestRcMove(t *testing.T) {
 	r, call := rcNewRun(t, "sync/move")
-	defer r.Finalise()
 	r.Mkdir(context.Background(), r.Fremote)
 
 	file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1)
@@ -76,7 +74,6 @@ func TestRcMove(t *testing.T) {
 // sync/sync: sync a directory from source remote to destination remote
 func TestRcSync(t *testing.T) {
 	r, call := rcNewRun(t, "sync/sync")
-	defer r.Finalise()
 	r.Mkdir(context.Background(), r.Fremote)
 
 	file1 := r.WriteBoth(context.Background(), "file1", "file1 contents", t1)
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index 72e6a65e781f3..f439712484cb8 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -41,7 +41,6 @@ func TestCopyWithDryRun(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
 	r.Mkdir(ctx, r.Fremote)
 
@@ -57,7 +56,6 @@ func TestCopyWithDryRun(t *testing.T) {
 func TestCopy(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
 	r.Mkdir(ctx, r.Fremote)
 
@@ -71,7 +69,6 @@ func TestCopy(t *testing.T) {
 func TestCopyMissingDirectory(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	r.Mkdir(ctx, r.Fremote)
 
 	nonExistingFs, err := fs.NewFs(ctx, "/non-existing")
@@ -88,7 +85,6 @@ func TestCopyNoTraverse(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	ci.NoTraverse = true
 
@@ -106,7 +102,6 @@ func TestCopyCheckFirst(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	ci.CheckFirst = true
 
@@ -124,7 +119,6 @@ func TestSyncNoTraverse(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	ci.NoTraverse = true
 
@@ -143,7 +137,6 @@ func TestCopyWithDepth(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
 	file2 := r.WriteFile("hello world2", "hello world2", t2)
 
@@ -162,7 +155,6 @@ func testCopyWithFilesFrom(t *testing.T, noTraverse bool) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("potato2", "hello world", t1)
 	file2 := r.WriteFile("hello world2", "hello world2", t2)
 
@@ -190,7 +182,6 @@ func TestCopyWithFilesFromAndNoTraverse(t *testing.T) { testCopyWithFilesFrom(t,
 func TestCopyEmptyDirectories(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
 	err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
 	require.NoError(t, err)
@@ -215,7 +206,6 @@ func TestCopyEmptyDirectories(t *testing.T) {
 func TestMoveEmptyDirectories(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
 	err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
 	require.NoError(t, err)
@@ -240,7 +230,6 @@ func TestMoveEmptyDirectories(t *testing.T) {
 func TestSyncEmptyDirectories(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
 	err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
 	require.NoError(t, err)
@@ -265,7 +254,6 @@ func TestSyncEmptyDirectories(t *testing.T) {
 func TestServerSideCopy(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1)
 	r.CheckRemoteItems(t, file1)
 
@@ -285,7 +273,6 @@ func TestServerSideCopy(t *testing.T) {
 func TestCopyAfterDelete(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1)
 	r.CheckLocalItems(t)
 	r.CheckRemoteItems(t, file1)
@@ -304,7 +291,6 @@ func TestCopyAfterDelete(t *testing.T) {
 func TestCopyRedownload(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteObject(ctx, "sub dir/hello world", "hello world", t1)
 	r.CheckRemoteItems(t, file1)
 
@@ -322,7 +308,6 @@ func TestSyncBasedOnCheckSum(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	ci.CheckSum = true
 
 	file1 := r.WriteFile("check sum", "-", t1)
@@ -357,7 +342,6 @@ func TestSyncSizeOnly(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	ci.SizeOnly = true
 
 	file1 := r.WriteFile("sizeonly", "potato", t1)
@@ -392,7 +376,6 @@ func TestSyncIgnoreSize(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	ci.IgnoreSize = true
 
 	file1 := r.WriteFile("ignore-size", "contents", t1)
@@ -424,7 +407,6 @@ func TestSyncIgnoreTimes(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "existing", "potato", t1)
 	r.CheckRemoteItems(t, file1)
 
@@ -454,7 +436,6 @@ func TestSyncIgnoreExisting(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("existing", "potato", t1)
 
 	ci.IgnoreExisting = true
@@ -479,7 +460,6 @@ func TestSyncIgnoreErrors(t *testing.T) {
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
 	ci.IgnoreErrors = true
-	defer r.Finalise()
 	file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)
 	file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2)
 	file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2)
@@ -541,7 +521,6 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("empty space", "-", t2)
 	file2 := r.WriteObject(ctx, "empty space", "-", t1)
 
@@ -571,7 +550,6 @@ func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	if r.Fremote.Hashes().Count() == 0 {
 		t.Logf("Can't check this if no hashes supported")
@@ -597,7 +575,6 @@ func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) {
 func TestSyncDoesntUpdateModtime(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	if fs.GetModifyWindow(ctx, r.Fremote) == fs.ModTimeNotSupported {
 		t.Skip("Can't run this test on fs which doesn't support mod time")
 	}
@@ -622,7 +599,6 @@ func TestSyncDoesntUpdateModtime(t *testing.T) {
 func TestSyncAfterAddingAFile(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "empty space", "-", t2)
 	file2 := r.WriteFile("potato", "------------------------------------------------------------", t3)
 
@@ -639,7 +615,6 @@ func TestSyncAfterAddingAFile(t *testing.T) {
 func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteObject(ctx, "potato", "------------------------------------------------------------", t3)
 	file2 := r.WriteFile("potato", "smaller but same date", t3)
 	r.CheckRemoteItems(t, file1)
@@ -657,7 +632,6 @@ func TestSyncAfterChangingFilesSizeOnly(t *testing.T) {
 func TestSyncAfterChangingContentsOnly(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	var file1 fstest.Item
 	if r.Fremote.Precision() == fs.ModTimeNotSupported {
 		t.Logf("ModTimeNotSupported so forcing file to be a different size")
@@ -681,7 +655,6 @@ func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
 	file2 := r.WriteObject(ctx, "potato", "SMALLER BUT SAME DATE", t2)
 	file3 := r.WriteBoth(ctx, "empty space", "-", t2)
@@ -699,7 +672,6 @@ func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
 // Sync after removing a file and adding a file
 func testSyncAfterRemovingAFileAndAddingAFile(ctx context.Context, t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
 	file2 := r.WriteObject(ctx, "potato", "SMALLER BUT SAME DATE", t2)
 	file3 := r.WriteBoth(ctx, "empty space", "-", t2)
@@ -720,7 +692,6 @@ func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
 // Sync after removing a file and adding a file
 func testSyncAfterRemovingAFileAndAddingAFileSubDir(ctx context.Context, t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)
 	file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2)
 	file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2)
@@ -788,7 +759,6 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDir(t *testing.T) {
 func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)
 	file2 := r.WriteObject(ctx, "b/potato", "SMALLER BUT SAME DATE", t2)
 	file3 := r.WriteBoth(ctx, "c/non empty space", "AhHa!", t2)
@@ -882,7 +852,6 @@ func TestCopyDeleteBefore(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	ci.DeleteMode = fs.DeleteModeBefore
 
@@ -903,7 +872,6 @@ func TestCopyDeleteBefore(t *testing.T) {
 func TestSyncWithExclude(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
 	file2 := r.WriteBoth(ctx, "empty space", "-", t2)
 	file3 := r.WriteFile("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
@@ -932,7 +900,6 @@ func TestSyncWithExclude(t *testing.T) {
 func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1) // 60 bytes
 	file2 := r.WriteBoth(ctx, "empty space", "-", t2)
 	file3 := r.WriteBoth(ctx, "enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
@@ -963,7 +930,6 @@ func TestSyncWithUpdateOlder(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	if fs.GetModifyWindow(ctx, r.Fremote) == fs.ModTimeNotSupported {
 		t.Skip("Can't run this test on fs which doesn't support mod time")
 	}
@@ -1009,7 +975,6 @@ func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) {
 		t.Skip("Skipping test on non local remote")
 	}
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	maxDuration := 250 * time.Millisecond
 	ci.MaxDuration = maxDuration
@@ -1063,7 +1028,6 @@ func TestSyncWithTrackRenames(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	ci.TrackRenames = true
 	defer func() {
@@ -1135,7 +1099,6 @@ func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	ci.TrackRenames = true
 	ci.TrackRenamesStrategy = "modtime"
@@ -1171,7 +1134,6 @@ func TestSyncWithTrackRenamesStrategyLeaf(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	ci.TrackRenames = true
 	ci.TrackRenamesStrategy = "leaf"
@@ -1285,7 +1247,6 @@ func testServerSideMove(ctx context.Context, t *testing.T, r *fstest.Run, withFi
 func TestMoveWithDeleteEmptySrcDirs(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
 	file2 := r.WriteFile("nested/sub dir/file", "nested", t1)
 	r.Mkdir(ctx, r.Fremote)
@@ -1305,7 +1266,6 @@ func TestMoveWithDeleteEmptySrcDirs(t *testing.T) {
 func TestMoveWithoutDeleteEmptySrcDirs(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
 	file2 := r.WriteFile("nested/sub dir/file", "nested", t1)
 	r.Mkdir(ctx, r.Fremote)
@@ -1329,7 +1289,6 @@ func TestMoveWithIgnoreExisting(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	file1 := r.WriteFile("existing", "potato", t1)
 	file2 := r.WriteFile("existing-b", "tomato", t1)
 
@@ -1380,7 +1339,6 @@ func TestMoveWithIgnoreExisting(t *testing.T) {
 func TestServerSideMove(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	testServerSideMove(ctx, t, r, false, false)
 }
 
@@ -1388,7 +1346,6 @@ func TestServerSideMove(t *testing.T) {
 func TestServerSideMoveWithFilter(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	fi, err := filter.NewFilter(nil)
 	require.NoError(t, err)
@@ -1402,7 +1359,6 @@ func TestServerSideMoveWithFilter(t *testing.T) {
 func TestServerSideMoveDeleteEmptySourceDirs(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	testServerSideMove(ctx, t, r, false, true)
 }
 
@@ -1410,7 +1366,6 @@ func TestServerSideMoveDeleteEmptySourceDirs(t *testing.T) {
 func TestServerSideMoveOverlap(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	if r.Fremote.Features().DirMove != nil {
 		t.Skip("Skipping test as remote supports DirMove")
@@ -1441,7 +1396,6 @@ func TestServerSideMoveOverlap(t *testing.T) {
 func TestSyncOverlap(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	subRemoteName := r.FremoteName + "/rclone-sync-test"
 	FremoteSync, err := fs.NewFs(ctx, subRemoteName)
@@ -1463,7 +1417,6 @@ func TestSyncOverlap(t *testing.T) {
 func TestSyncOverlapWithFilter(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	fi, err := filter.NewFilter(nil)
 	require.NoError(t, err)
@@ -1521,7 +1474,6 @@ func TestSyncCompareDest(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	ci.CompareDest = []string{r.FremoteName + "/CompareDest"}
 
@@ -1637,7 +1589,6 @@ func TestSyncMultipleCompareDest(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	precision := fs.GetModifyWindow(ctx, r.Fremote, r.Flocal)
 
 	ci.CompareDest = []string{r.FremoteName + "/pre-dest1", r.FremoteName + "/pre-dest2"}
@@ -1669,7 +1620,6 @@ func TestSyncCopyDest(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	if r.Fremote.Features().Copy == nil {
 		t.Skip("Skipping test as remote does not support server-side copy")
@@ -1771,7 +1721,6 @@ func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeep
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	if !operations.CanServerSideMove(r.Fremote) {
 		t.Skip("Skipping test as remote does not support server-side move")
@@ -1869,7 +1818,6 @@ func testSyncSuffix(t *testing.T, suffix string, suffixKeepExtension bool) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	if !operations.CanServerSideMove(r.Fremote) {
 		t.Skip("Skipping test as remote does not support server-side move")
@@ -1957,7 +1905,6 @@ func TestSyncUTFNorm(t *testing.T) {
 	}
 
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	// Two strings with different unicode normalization (from OS X)
 	Encoding1 := "Testêé"
@@ -1988,7 +1935,6 @@ func TestSyncImmutable(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	ci.Immutable = true
 
@@ -2022,7 +1968,6 @@ func TestSyncIgnoreCase(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	// Only test if filesystems are case sensitive
 	if r.Fremote.Features().CaseInsensitive || r.Flocal.Features().CaseInsensitive {
@@ -2056,7 +2001,6 @@ func TestMaxTransfer(t *testing.T) {
 
 	test := func(t *testing.T, cutoff fs.CutoffMode) {
 		r := fstest.NewRun(t)
-		defer r.Finalise()
 		ci.CutoffMode = cutoff
 
 		if r.Fremote.Name() != "local" {
@@ -2098,7 +2042,6 @@ func testSyncConcurrent(t *testing.T, subtest string) {
 	ci.Transfers = NTRANSFERS
 
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 	stats := accounting.GlobalStats()
 
 	itemsBefore := []fstest.Item{}
diff --git a/fstest/run.go b/fstest/run.go
index eb9b0c3b58f90..d03d0ef83ef07 100644
--- a/fstest/run.go
+++ b/fstest/run.go
@@ -178,6 +178,7 @@ func newRunIndividual(t *testing.T, individual bool) *Run {
 	r.Logf = t.Logf
 	r.Fatalf = t.Fatalf
 	r.Logf("Remote %q, Local %q, Modify Window %q", r.Fremote, r.Flocal, fs.GetModifyWindow(ctx, r.Fremote))
+	t.Cleanup(r.Finalise)
 	return r
 }
 
@@ -186,8 +187,6 @@ func newRunIndividual(t *testing.T, individual bool) *Run {
 //
 // r.Flocal is an empty local Fs
 // r.Fremote is an empty remote Fs
-//
-// Finalise() will tidy them away when done.
 func NewRun(t *testing.T) *Run {
 	return newRunIndividual(t, *Individual)
 }
diff --git a/vfs/dir_handle_test.go b/vfs/dir_handle_test.go
index 780093a054277..cfb215c2f2afc 100644
--- a/vfs/dir_handle_test.go
+++ b/vfs/dir_handle_test.go
@@ -11,8 +11,7 @@ import (
 )
 
 func TestDirHandleMethods(t *testing.T) {
-	_, _, dir, _, cleanup := dirCreate(t)
-	defer cleanup()
+	_, _, dir, _ := dirCreate(t)
 
 	h, err := dir.Open(os.O_RDONLY)
 	require.NoError(t, err)
@@ -38,8 +37,7 @@ func TestDirHandleMethods(t *testing.T) {
 }
 
 func TestDirHandleReaddir(t *testing.T) {
-	r, vfs, cleanup := newTestVFS(t)
-	defer cleanup()
+	r, vfs := newTestVFS(t)
 
 	file1 := r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1)
 	file2 := r.WriteObject(context.Background(), "dir/file2", "file2- contents", t2)
@@ -93,8 +91,7 @@ func TestDirHandleReaddir(t *testing.T) {
 }
 
 func TestDirHandleReaddirnames(t *testing.T) {
-	_, _, dir, _, cleanup := dirCreate(t)
-	defer cleanup()
+	_, _, dir, _ := dirCreate(t)
 
 	fh, err := dir.Open(os.O_RDONLY)
 	require.NoError(t, err)
diff --git a/vfs/dir_test.go b/vfs/dir_test.go
index acdbd3aa5c741..788471eb7cae0 100644
--- a/vfs/dir_test.go
+++ b/vfs/dir_test.go
@@ -15,8 +15,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func dirCreate(t *testing.T) (r *fstest.Run, vfs *VFS, dir *Dir, item fstest.Item, cleanup func()) {
-	r, vfs, cleanup = newTestVFS(t)
+func dirCreate(t *testing.T) (r *fstest.Run, vfs *VFS, dir *Dir, item fstest.Item) {
+	r, vfs = newTestVFS(t)
 
 	file1 := r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1)
 	r.CheckRemoteItems(t, file1)
@@ -25,12 +25,11 @@ func dirCreate(t *testing.T) (r *fstest.Run, vfs *VFS, dir *Dir, item fstest.Ite
 	require.NoError(t, err)
 	require.True(t, node.IsDir())
 
-	return r, vfs, node.(*Dir), file1, cleanup
+	return r, vfs, node.(*Dir), file1
 }
 
 func TestDirMethods(t *testing.T) {
-	_, vfs, dir, _, cleanup := dirCreate(t)
-	defer cleanup()
+	_, vfs, dir, _ := dirCreate(t)
 
 	// String
 	assert.Equal(t, "dir/", dir.String())
@@ -81,8 +80,7 @@ func TestDirMethods(t *testing.T) {
 }
 
 func TestDirForgetAll(t *testing.T) {
-	_, vfs, dir, file1, cleanup := dirCreate(t)
-	defer cleanup()
+	_, vfs, dir, file1 := dirCreate(t)
 
 	// Make sure / and dir are in cache
 	_, err := vfs.Stat(file1.Path)
@@ -109,8 +107,7 @@ func TestDirForgetAll(t *testing.T) {
 }
 
 func TestDirForgetPath(t *testing.T) {
-	_, vfs, dir, file1, cleanup := dirCreate(t)
-	defer cleanup()
+	_, vfs, dir, file1 := dirCreate(t)
 
 	// Make sure / and dir are in cache
 	_, err := vfs.Stat(file1.Path)
@@ -141,8 +138,7 @@ func TestDirForgetPath(t *testing.T) {
 }
 
 func TestDirWalk(t *testing.T) {
-	r, vfs, _, file1, cleanup := dirCreate(t)
-	defer cleanup()
+	r, vfs, _, file1 := dirCreate(t)
 
 	file2 := r.WriteObject(context.Background(), "fil/a/b/c", "super long file", t1)
 	r.CheckRemoteItems(t, file1, file2)
@@ -210,8 +206,7 @@ func TestDirWalk(t *testing.T) {
 }
 
 func TestDirSetModTime(t *testing.T) {
-	_, vfs, dir, _, cleanup := dirCreate(t)
-	defer cleanup()
+	_, vfs, dir, _ := dirCreate(t)
 
 	err := dir.SetModTime(t1)
 	require.NoError(t, err)
@@ -227,8 +222,7 @@ func TestDirSetModTime(t *testing.T) {
 }
 
 func TestDirStat(t *testing.T) {
-	_, _, dir, _, cleanup := dirCreate(t)
-	defer cleanup()
+	_, _, dir, _ := dirCreate(t)
 
 	node, err := dir.Stat("file1")
 	require.NoError(t, err)
@@ -253,8 +247,7 @@ func checkListing(t *testing.T, dir *Dir, want []string) {
 }
 
 func TestDirReadDirAll(t *testing.T) {
-	r, vfs, cleanup := newTestVFS(t)
-	defer cleanup()
+	r, vfs := newTestVFS(t)
 
 	file1 := r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1)
 	file2 := r.WriteObject(context.Background(), "dir/file2", "file2- contents", t2)
@@ -334,8 +327,7 @@ func TestDirReadDirAll(t *testing.T) {
 }
 
 func TestDirOpen(t *testing.T) {
-	_, _, dir, _, cleanup := dirCreate(t)
-	defer cleanup()
+	_, _, dir, _ := dirCreate(t)
 
 	fd, err := dir.Open(os.O_RDONLY)
 	require.NoError(t, err)
@@ -348,8 +340,7 @@ func TestDirOpen(t *testing.T) {
 }
 
 func TestDirCreate(t *testing.T) {
-	_, vfs, dir, _, cleanup := dirCreate(t)
-	defer cleanup()
+	_, vfs, dir, _ := dirCreate(t)
 
 	file, err := dir.Create("potato", os.O_WRONLY|os.O_CREATE)
 	require.NoError(t, err)
@@ -388,8 +379,7 @@ func TestDirCreate(t *testing.T) {
 }
 
 func TestDirMkdir(t *testing.T) {
-	r, vfs, dir, file1, cleanup := dirCreate(t)
-	defer cleanup()
+	r, vfs, dir, file1 := dirCreate(t)
 
 	_, err := dir.Mkdir("file1")
 	assert.Error(t, err)
@@ -410,8 +400,7 @@ func TestDirMkdir(t *testing.T) {
 }
 
 func TestDirMkdirSub(t *testing.T) {
-	r, vfs, dir, file1, cleanup := dirCreate(t)
-	defer cleanup()
+	r, vfs, dir, file1 := dirCreate(t)
 
 	_, err := dir.Mkdir("file1")
 	assert.Error(t, err)
@@ -436,8 +425,7 @@ func TestDirMkdirSub(t *testing.T) {
 }
 
 func TestDirRemove(t *testing.T) {
-	r, vfs, dir, _, cleanup := dirCreate(t)
-	defer cleanup()
+	r, vfs, dir, _ := dirCreate(t)
 
 	// check directory is there
 	node, err := vfs.Stat("dir")
@@ -476,8 +464,7 @@ func TestDirRemove(t *testing.T) {
 }
 
 func TestDirRemoveAll(t *testing.T) {
-	r, vfs, dir, _, cleanup := dirCreate(t)
-	defer cleanup()
+	r, vfs, dir, _ := dirCreate(t)
 
 	// Remove the directory and contents
 	err := dir.RemoveAll()
@@ -498,8 +485,7 @@ func TestDirRemoveAll(t *testing.T) {
 }
 
 func TestDirRemoveName(t *testing.T) {
-	r, vfs, dir, _, cleanup := dirCreate(t)
-	defer cleanup()
+	r, vfs, dir, _ := dirCreate(t)
 
 	err := dir.RemoveName("file1")
 	require.NoError(t, err)
@@ -518,8 +504,7 @@ func TestDirRemoveName(t *testing.T) {
 }
 
 func TestDirRename(t *testing.T) {
-	r, vfs, dir, file1, cleanup := dirCreate(t)
-	defer cleanup()
+	r, vfs, dir, file1 := dirCreate(t)
 
 	features := r.Fremote.Features()
 	if features.DirMove == nil && features.Move == nil && features.Copy == nil {
diff --git a/vfs/file_test.go b/vfs/file_test.go
index a2b7423150dc1..c54be40d1687e 100644
--- a/vfs/file_test.go
+++ b/vfs/file_test.go
@@ -17,11 +17,11 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func fileCreate(t *testing.T, mode vfscommon.CacheMode) (r *fstest.Run, vfs *VFS, fh *File, item fstest.Item, cleanup func()) {
+func fileCreate(t *testing.T, mode vfscommon.CacheMode) (r *fstest.Run, vfs *VFS, fh *File, item fstest.Item) {
 	opt := vfscommon.DefaultOpt
 	opt.CacheMode = mode
 	opt.WriteBack = writeBackDelay
-	r, vfs, cleanup = newTestVFSOpt(t, &opt)
+	r, vfs = newTestVFSOpt(t, &opt)
 
 	file1 := r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1)
 	r.CheckRemoteItems(t, file1)
@@ -30,12 +30,11 @@ func fileCreate(t *testing.T, mode vfscommon.CacheMode) (r *fstest.Run, vfs *VFS
 	require.NoError(t, err)
 	require.True(t, node.Mode().IsRegular())
 
-	return r, vfs, node.(*File), file1, cleanup
+	return r, vfs, node.(*File), file1
 }
 
 func TestFileMethods(t *testing.T) {
-	r, vfs, file, _, cleanup := fileCreate(t, vfscommon.CacheModeOff)
-	defer cleanup()
+	r, vfs, file, _ := fileCreate(t, vfscommon.CacheModeOff)
 
 	// String
 	assert.Equal(t, "dir/file1", file.String())
@@ -92,8 +91,7 @@ func testFileSetModTime(t *testing.T, cacheMode vfscommon.CacheMode, open bool,
 	if !canSetModTimeValue {
 		t.Skip("can't set mod time")
 	}
-	r, vfs, file, file1, cleanup := fileCreate(t, cacheMode)
-	defer cleanup()
+	r, vfs, file, file1 := fileCreate(t, cacheMode)
 	if !canSetModTime(t, r) {
 		t.Skip("can't set mod time")
 	}
@@ -176,8 +174,7 @@ func fileCheckContents(t *testing.T, file *File) {
 }
 
 func TestFileOpenRead(t *testing.T) {
-	_, _, file, _, cleanup := fileCreate(t, vfscommon.CacheModeOff)
-	defer cleanup()
+	_, _, file, _ := fileCreate(t, vfscommon.CacheModeOff)
 
 	fileCheckContents(t, file)
 }
@@ -229,8 +226,7 @@ func TestFileOpenReadUnknownSize(t *testing.T) {
 }
 
 func TestFileOpenWrite(t *testing.T) {
-	_, vfs, file, _, cleanup := fileCreate(t, vfscommon.CacheModeOff)
-	defer cleanup()
+	_, vfs, file, _ := fileCreate(t, vfscommon.CacheModeOff)
 
 	fd, err := file.openWrite(os.O_WRONLY | os.O_TRUNC)
 	require.NoError(t, err)
@@ -249,8 +245,7 @@ func TestFileOpenWrite(t *testing.T) {
 }
 
 func TestFileRemove(t *testing.T) {
-	r, vfs, file, _, cleanup := fileCreate(t, vfscommon.CacheModeOff)
-	defer cleanup()
+	r, vfs, file, _ := fileCreate(t, vfscommon.CacheModeOff)
 
 	err := file.Remove()
 	require.NoError(t, err)
@@ -263,8 +258,7 @@ func TestFileRemove(t *testing.T) {
 }
 
 func TestFileRemoveAll(t *testing.T) {
-	r, vfs, file, _, cleanup := fileCreate(t, vfscommon.CacheModeOff)
-	defer cleanup()
+	r, vfs, file, _ := fileCreate(t, vfscommon.CacheModeOff)
 
 	err := file.RemoveAll()
 	require.NoError(t, err)
@@ -277,8 +271,7 @@ func TestFileRemoveAll(t *testing.T) {
 }
 
 func TestFileOpen(t *testing.T) {
-	_, _, file, _, cleanup := fileCreate(t, vfscommon.CacheModeOff)
-	defer cleanup()
+	_, _, file, _ := fileCreate(t, vfscommon.CacheModeOff)
 
 	fd, err := file.Open(os.O_RDONLY)
 	require.NoError(t, err)
@@ -303,8 +296,7 @@ func TestFileOpen(t *testing.T) {
 }
 
 func testFileRename(t *testing.T, mode vfscommon.CacheMode, inCache bool, forceCache bool) {
-	r, vfs, file, item, cleanup := fileCreate(t, mode)
-	defer cleanup()
+	r, vfs, file, item := fileCreate(t, mode)
 
 	if !operations.CanServerSideMove(r.Fremote) {
 		t.Skip("skip as can't rename files")
diff --git a/vfs/rc_test.go b/vfs/rc_test.go
index 102355debf7cc..01aa3921e3021 100644
--- a/vfs/rc_test.go
+++ b/vfs/rc_test.go
@@ -12,14 +12,14 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func rcNewRun(t *testing.T, method string) (r *fstest.Run, vfs *VFS, cleanup func(), call *rc.Call) {
+func rcNewRun(t *testing.T, method string) (r *fstest.Run, vfs *VFS, call *rc.Call) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping test on non local remote")
 	}
-	r, vfs, cleanup = newTestVFS(t)
+	r, vfs = newTestVFS(t)
 	call = rc.Calls.Get(method)
 	assert.NotNil(t, call)
-	return r, vfs, cleanup, call
+	return r, vfs, call
 }
 
 func TestRcGetVFS(t *testing.T) {
@@ -29,8 +29,7 @@ func TestRcGetVFS(t *testing.T) {
 	assert.Contains(t, err.Error(), "no VFS active")
 	assert.Nil(t, vfs)
 
-	r, vfs2, cleanup := newTestVFS(t)
-	defer cleanup()
+	r, vfs2 := newTestVFS(t)
 
 	vfs, err = getVFS(in)
 	require.NoError(t, err)
@@ -65,8 +64,7 @@ func TestRcGetVFS(t *testing.T) {
 }
 
 func TestRcForget(t *testing.T) {
-	r, vfs, cleanup, call := rcNewRun(t, "vfs/forget")
-	defer cleanup()
+	r, vfs, call := rcNewRun(t, "vfs/forget")
 	_, _ = r, vfs
 	in := rc.Params{"fs": fs.ConfigString(r.Fremote)}
 	out, err := call.Fn(context.Background(), in)
@@ -78,8 +76,7 @@ func TestRcForget(t *testing.T) {
 }
 
 func TestRcRefresh(t *testing.T) {
-	r, vfs, cleanup, call := rcNewRun(t, "vfs/refresh")
-	defer cleanup()
+	r, vfs, call := rcNewRun(t, "vfs/refresh")
 	_, _ = r, vfs
 	in := rc.Params{"fs": fs.ConfigString(r.Fremote)}
 	out, err := call.Fn(context.Background(), in)
@@ -93,8 +90,7 @@ func TestRcRefresh(t *testing.T) {
 }
 
 func TestRcPollInterval(t *testing.T) {
-	r, vfs, cleanup, call := rcNewRun(t, "vfs/poll-interval")
-	defer cleanup()
+	r, vfs, call := rcNewRun(t, "vfs/poll-interval")
 	_ = vfs
 	if r.Fremote.Features().ChangeNotify == nil {
 		t.Skip("ChangeNotify not supported")
@@ -106,8 +102,7 @@ func TestRcPollInterval(t *testing.T) {
 }
 
 func TestRcList(t *testing.T) {
-	r, vfs, cleanup, call := rcNewRun(t, "vfs/list")
-	defer cleanup()
+	r, vfs, call := rcNewRun(t, "vfs/list")
 	_ = vfs
 
 	out, err := call.Fn(context.Background(), nil)
@@ -121,8 +116,7 @@ func TestRcList(t *testing.T) {
 }
 
 func TestRcStats(t *testing.T) {
-	r, vfs, cleanup, call := rcNewRun(t, "vfs/stats")
-	defer cleanup()
+	r, vfs, call := rcNewRun(t, "vfs/stats")
 	out, err := call.Fn(context.Background(), nil)
 	require.NoError(t, err)
 	assert.Equal(t, fs.ConfigString(r.Fremote), out["fs"])
diff --git a/vfs/read_test.go b/vfs/read_test.go
index 0f9d35dbd3218..db8d0f8417808 100644
--- a/vfs/read_test.go
+++ b/vfs/read_test.go
@@ -12,8 +12,8 @@ import (
 )
 
 // Open a file for write
-func readHandleCreate(t *testing.T) (r *fstest.Run, vfs *VFS, fh *ReadFileHandle, cleanup func()) {
-	r, vfs, cleanup = newTestVFS(t)
+func readHandleCreate(t *testing.T) (r *fstest.Run, vfs *VFS, fh *ReadFileHandle) {
+	r, vfs = newTestVFS(t)
 
 	file1 := r.WriteObject(context.Background(), "dir/file1", "0123456789abcdef", t1)
 	r.CheckRemoteItems(t, file1)
@@ -23,7 +23,7 @@ func readHandleCreate(t *testing.T) (r *fstest.Run, vfs *VFS, fh *ReadFileHandle
 	fh, ok := h.(*ReadFileHandle)
 	require.True(t, ok)
 
-	return r, vfs, fh, cleanup
+	return r, vfs, fh
 }
 
 // read data from the string
@@ -37,8 +37,7 @@ func readString(t *testing.T, fh *ReadFileHandle, n int) string {
 }
 
 func TestReadFileHandleMethods(t *testing.T) {
-	_, _, fh, cleanup := readHandleCreate(t)
-	defer cleanup()
+	_, _, fh := readHandleCreate(t)
 
 	// String
 	assert.Equal(t, "dir/file1 (r)", fh.String())
@@ -80,8 +79,7 @@ func TestReadFileHandleMethods(t *testing.T) {
 }
 
 func TestReadFileHandleSeek(t *testing.T) {
-	_, _, fh, cleanup := readHandleCreate(t)
-	defer cleanup()
+	_, _, fh := readHandleCreate(t)
 
 	assert.Equal(t, "0", readString(t, fh, 1))
 
@@ -123,8 +121,7 @@ func TestReadFileHandleSeek(t *testing.T) {
 }
 
 func TestReadFileHandleReadAt(t *testing.T) {
-	_, _, fh, cleanup := readHandleCreate(t)
-	defer cleanup()
+	_, _, fh := readHandleCreate(t)
 
 	// read from start
 	buf := make([]byte, 1)
@@ -179,8 +176,7 @@ func TestReadFileHandleReadAt(t *testing.T) {
 }
 
 func TestReadFileHandleFlush(t *testing.T) {
-	_, _, fh, cleanup := readHandleCreate(t)
-	defer cleanup()
+	_, _, fh := readHandleCreate(t)
 
 	// Check Flush does nothing if read not called
 	err := fh.Flush()
@@ -208,8 +204,7 @@ func TestReadFileHandleFlush(t *testing.T) {
 }
 
 func TestReadFileHandleRelease(t *testing.T) {
-	_, _, fh, cleanup := readHandleCreate(t)
-	defer cleanup()
+	_, _, fh := readHandleCreate(t)
 
 	// Check Release does nothing if file not read from
 	err := fh.Release()
diff --git a/vfs/read_write_test.go b/vfs/read_write_test.go
index d9dac353f37c8..75b930f73f4a9 100644
--- a/vfs/read_write_test.go
+++ b/vfs/read_write_test.go
@@ -29,11 +29,11 @@ var (
 )
 
 // Create a file and open it with the flags passed in
-func rwHandleCreateFlags(t *testing.T, create bool, filename string, flags int) (r *fstest.Run, vfs *VFS, fh *RWFileHandle, cleanup func()) {
+func rwHandleCreateFlags(t *testing.T, create bool, filename string, flags int) (r *fstest.Run, vfs *VFS, fh *RWFileHandle) {
 	opt := vfscommon.DefaultOpt
 	opt.CacheMode = vfscommon.CacheModeFull
 	opt.WriteBack = writeBackDelay
-	r, vfs, cleanup = newTestVFSOpt(t, &opt)
+	r, vfs = newTestVFSOpt(t, &opt)
 
 	if create {
 		file1 := r.WriteObject(context.Background(), filename, "0123456789abcdef", t1)
@@ -45,16 +45,16 @@ func rwHandleCreateFlags(t *testing.T, create bool, filename string, flags int)
 	fh, ok := h.(*RWFileHandle)
 	require.True(t, ok)
 
-	return r, vfs, fh, cleanup
+	return r, vfs, fh
 }
 
 // Open a file for read
-func rwHandleCreateReadOnly(t *testing.T) (r *fstest.Run, vfs *VFS, fh *RWFileHandle, cleanup func()) {
+func rwHandleCreateReadOnly(t *testing.T) (r *fstest.Run, vfs *VFS, fh *RWFileHandle) {
 	return rwHandleCreateFlags(t, true, "dir/file1", os.O_RDONLY)
 }
 
 // Open a file for write
-func rwHandleCreateWriteOnly(t *testing.T) (r *fstest.Run, vfs *VFS, fh *RWFileHandle, cleanup func()) {
+func rwHandleCreateWriteOnly(t *testing.T) (r *fstest.Run, vfs *VFS, fh *RWFileHandle) {
 	return rwHandleCreateFlags(t, false, "file1", os.O_WRONLY|os.O_CREATE)
 }
 
@@ -69,8 +69,7 @@ func rwReadString(t *testing.T, fh *RWFileHandle, n int) string {
 }
 
 func TestRWFileHandleMethodsRead(t *testing.T) {
-	_, _, fh, cleanup := rwHandleCreateReadOnly(t)
-	defer cleanup()
+	_, _, fh := rwHandleCreateReadOnly(t)
 
 	// String
 	assert.Equal(t, "dir/file1 (rw)", fh.String())
@@ -116,8 +115,7 @@ func TestRWFileHandleMethodsRead(t *testing.T) {
 }
 
 func TestRWFileHandleSeek(t *testing.T) {
-	_, _, fh, cleanup := rwHandleCreateReadOnly(t)
-	defer cleanup()
+	_, _, fh := rwHandleCreateReadOnly(t)
 
 	assert.Equal(t, fh.opened, false)
 
@@ -166,8 +164,7 @@ func TestRWFileHandleSeek(t *testing.T) {
 }
 
 func TestRWFileHandleReadAt(t *testing.T) {
-	_, _, fh, cleanup := rwHandleCreateReadOnly(t)
-	defer cleanup()
+	_, _, fh := rwHandleCreateReadOnly(t)
 
 	// read from start
 	buf := make([]byte, 1)
@@ -216,8 +213,7 @@ func TestRWFileHandleReadAt(t *testing.T) {
 }
 
 func TestRWFileHandleFlushRead(t *testing.T) {
-	_, _, fh, cleanup := rwHandleCreateReadOnly(t)
-	defer cleanup()
+	_, _, fh := rwHandleCreateReadOnly(t)
 
 	// Check Flush does nothing if read not called
 	err := fh.Flush()
@@ -245,8 +241,7 @@ func TestRWFileHandleFlushRead(t *testing.T) {
 }
 
 func TestRWFileHandleReleaseRead(t *testing.T) {
-	_, _, fh, cleanup := rwHandleCreateReadOnly(t)
-	defer cleanup()
+	_, _, fh := rwHandleCreateReadOnly(t)
 
 	// Read data
 	buf := make([]byte, 256)
@@ -268,8 +263,7 @@ func TestRWFileHandleReleaseRead(t *testing.T) {
 /// ------------------------------------------------------------
 
 func TestRWFileHandleMethodsWrite(t *testing.T) {
-	r, vfs, fh, cleanup := rwHandleCreateWriteOnly(t)
-	defer cleanup()
+	r, vfs, fh := rwHandleCreateWriteOnly(t)
 
 	// String
 	assert.Equal(t, "file1 (rw)", fh.String())
@@ -344,8 +338,7 @@ func TestRWFileHandleMethodsWrite(t *testing.T) {
 }
 
 func TestRWFileHandleWriteAt(t *testing.T) {
-	r, vfs, fh, cleanup := rwHandleCreateWriteOnly(t)
-	defer cleanup()
+	r, vfs, fh := rwHandleCreateWriteOnly(t)
 
 	offset := func() int64 {
 		n, err := fh.Seek(0, io.SeekCurrent)
@@ -392,8 +385,7 @@ func TestRWFileHandleWriteAt(t *testing.T) {
 }
 
 func TestRWFileHandleWriteNoWrite(t *testing.T) {
-	r, vfs, fh, cleanup := rwHandleCreateWriteOnly(t)
-	defer cleanup()
+	r, vfs, fh := rwHandleCreateWriteOnly(t)
 
 	// Close the file without writing to it
 	err := fh.Close()
@@ -426,8 +418,7 @@ func TestRWFileHandleWriteNoWrite(t *testing.T) {
 }
 
 func TestRWFileHandleFlushWrite(t *testing.T) {
-	_, _, fh, cleanup := rwHandleCreateWriteOnly(t)
-	defer cleanup()
+	_, _, fh := rwHandleCreateWriteOnly(t)
 
 	// Check that the file has been create and is open
 	assert.True(t, fh.opened)
@@ -455,8 +446,7 @@ func TestRWFileHandleFlushWrite(t *testing.T) {
 }
 
 func TestRWFileHandleReleaseWrite(t *testing.T) {
-	_, _, fh, cleanup := rwHandleCreateWriteOnly(t)
-	defer cleanup()
+	_, _, fh := rwHandleCreateWriteOnly(t)
 
 	// Write some data
 	n, err := fh.Write([]byte("hello"))
@@ -485,8 +475,7 @@ func assertSize(t *testing.T, vfs *VFS, fh *RWFileHandle, filepath string, size
 }
 
 func TestRWFileHandleSizeTruncateExisting(t *testing.T) {
-	_, vfs, fh, cleanup := rwHandleCreateFlags(t, true, "dir/file1", os.O_WRONLY|os.O_TRUNC)
-	defer cleanup()
+	_, vfs, fh := rwHandleCreateFlags(t, true, "dir/file1", os.O_WRONLY|os.O_TRUNC)
 
 	// check initial size after opening
 	assertSize(t, vfs, fh, "dir/file1", 0)
@@ -507,8 +496,7 @@ func TestRWFileHandleSizeTruncateExisting(t *testing.T) {
 }
 
 func TestRWFileHandleSizeCreateExisting(t *testing.T) {
-	_, vfs, fh, cleanup := rwHandleCreateFlags(t, true, "dir/file1", os.O_WRONLY|os.O_CREATE)
-	defer cleanup()
+	_, vfs, fh := rwHandleCreateFlags(t, true, "dir/file1", os.O_WRONLY|os.O_CREATE)
 
 	// check initial size after opening
 	assertSize(t, vfs, fh, "dir/file1", 16)
@@ -537,8 +525,7 @@ func TestRWFileHandleSizeCreateExisting(t *testing.T) {
 }
 
 func TestRWFileHandleSizeCreateNew(t *testing.T) {
-	_, vfs, fh, cleanup := rwHandleCreateFlags(t, false, "file1", os.O_WRONLY|os.O_CREATE)
-	defer cleanup()
+	_, vfs, fh := rwHandleCreateFlags(t, false, "file1", os.O_WRONLY|os.O_CREATE)
 
 	// check initial size after opening
 	assertSize(t, vfs, fh, "file1", 0)
@@ -647,8 +634,7 @@ func TestRWFileHandleOpenTests(t *testing.T) {
 			opt := vfscommon.DefaultOpt
 			opt.CacheMode = cacheMode
 			opt.WriteBack = writeBackDelay
-			_, vfs, cleanup := newTestVFSOpt(t, &opt)
-			defer cleanup()
+			_, vfs := newTestVFSOpt(t, &opt)
 
 			for _, test := range openTests {
 				t.Run(test.what, func(t *testing.T) {
@@ -661,8 +647,7 @@ func TestRWFileHandleOpenTests(t *testing.T) {
 
 // tests mod time on open files
 func TestRWFileModTimeWithOpenWriters(t *testing.T) {
-	r, vfs, fh, cleanup := rwHandleCreateWriteOnly(t)
-	defer cleanup()
+	r, vfs, fh := rwHandleCreateWriteOnly(t)
 	if !canSetModTime(t, r) {
 		t.Skip("can't set mod time")
 	}
@@ -700,8 +685,7 @@ func TestRWCacheRename(t *testing.T) {
 	opt := vfscommon.DefaultOpt
 	opt.CacheMode = vfscommon.CacheModeFull
 	opt.WriteBack = writeBackDelay
-	r, vfs, cleanup := newTestVFSOpt(t, &opt)
-	defer cleanup()
+	r, vfs := newTestVFSOpt(t, &opt)
 
 	if !operations.CanServerSideMove(r.Fremote) {
 		t.Skip("skip as can't rename files")
diff --git a/vfs/vfs_case_test.go b/vfs/vfs_case_test.go
index 275c8691cfff2..5ba81a169b7c7 100644
--- a/vfs/vfs_case_test.go
+++ b/vfs/vfs_case_test.go
@@ -13,7 +13,6 @@ import (
 
 func TestCaseSensitivity(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	if r.Fremote.Features().CaseInsensitive {
 		t.Skip("Can't test case sensitivity - this remote is officially not case-sensitive")
diff --git a/vfs/vfs_test.go b/vfs/vfs_test.go
index 05e7b052a704c..e262e7d2aed4d 100644
--- a/vfs/vfs_test.go
+++ b/vfs/vfs_test.go
@@ -46,18 +46,17 @@ func cleanupVFS(t *testing.T, vfs *VFS) {
 }
 
 // Create a new VFS
-func newTestVFSOpt(t *testing.T, opt *vfscommon.Options) (r *fstest.Run, vfs *VFS, cleanup func()) {
+func newTestVFSOpt(t *testing.T, opt *vfscommon.Options) (r *fstest.Run, vfs *VFS) {
 	r = fstest.NewRun(t)
 	vfs = New(r.Fremote, opt)
-	cleanup = func() {
+	t.Cleanup(func() {
 		cleanupVFS(t, vfs)
-		r.Finalise()
-	}
-	return r, vfs, cleanup
+	})
+	return r, vfs
 }
 
 // Create a new VFS with default options
-func newTestVFS(t *testing.T) (r *fstest.Run, vfs *VFS, cleanup func()) {
+func newTestVFS(t *testing.T) (r *fstest.Run, vfs *VFS) {
 	return newTestVFSOpt(t, nil)
 }
 
@@ -136,7 +135,7 @@ func TestVFSNew(t *testing.T) {
 
 	checkActiveCacheEntries(0)
 
-	r, vfs, cleanup := newTestVFS(t)
+	r, vfs := newTestVFS(t)
 
 	// Check making a VFS with nil options
 	var defaultOpt = vfscommon.DefaultOpt
@@ -158,7 +157,7 @@ func TestVFSNew(t *testing.T) {
 
 	checkActiveCacheEntries(1)
 
-	cleanup()
+	cleanupVFS(t, vfs)
 
 	checkActiveCacheEntries(0)
 }
@@ -169,8 +168,7 @@ func TestVFSNewWithOpts(t *testing.T) {
 	opt.DirPerms = 0777
 	opt.FilePerms = 0666
 	opt.Umask = 0002
-	_, vfs, cleanup := newTestVFSOpt(t, &opt)
-	defer cleanup()
+	_, vfs := newTestVFSOpt(t, &opt)
 
 	assert.Equal(t, os.FileMode(0775)|os.ModeDir, vfs.Opt.DirPerms)
 	assert.Equal(t, os.FileMode(0664), vfs.Opt.FilePerms)
@@ -178,8 +176,7 @@ func TestVFSNewWithOpts(t *testing.T) {
 
 // TestRoot checks root directory is present and correct
 func TestVFSRoot(t *testing.T) {
-	_, vfs, cleanup := newTestVFS(t)
-	defer cleanup()
+	_, vfs := newTestVFS(t)
 
 	root, err := vfs.Root()
 	require.NoError(t, err)
@@ -189,8 +186,7 @@ func TestVFSRoot(t *testing.T) {
 }
 
 func TestVFSStat(t *testing.T) {
-	r, vfs, cleanup := newTestVFS(t)
-	defer cleanup()
+	r, vfs := newTestVFS(t)
 
 	file1 := r.WriteObject(context.Background(), "file1", "file1 contents", t1)
 	file2 := r.WriteObject(context.Background(), "dir/file2", "file2 contents", t2)
@@ -225,8 +221,7 @@ func TestVFSStat(t *testing.T) {
 }
 
 func TestVFSStatParent(t *testing.T) {
-	r, vfs, cleanup := newTestVFS(t)
-	defer cleanup()
+	r, vfs := newTestVFS(t)
 
 	file1 := r.WriteObject(context.Background(), "file1", "file1 contents", t1)
 	file2 := r.WriteObject(context.Background(), "dir/file2", "file2 contents", t2)
@@ -258,8 +253,7 @@ func TestVFSStatParent(t *testing.T) {
 }
 
 func TestVFSOpenFile(t *testing.T) {
-	r, vfs, cleanup := newTestVFS(t)
-	defer cleanup()
+	r, vfs := newTestVFS(t)
 
 	file1 := r.WriteObject(context.Background(), "file1", "file1 contents", t1)
 	file2 := r.WriteObject(context.Background(), "dir/file2", "file2 contents", t2)
@@ -293,8 +287,7 @@ func TestVFSOpenFile(t *testing.T) {
 }
 
 func TestVFSRename(t *testing.T) {
-	r, vfs, cleanup := newTestVFS(t)
-	defer cleanup()
+	r, vfs := newTestVFS(t)
 
 	features := r.Fremote.Features()
 	if features.Move == nil && features.Copy == nil {
@@ -322,8 +315,7 @@ func TestVFSRename(t *testing.T) {
 }
 
 func TestVFSStatfs(t *testing.T) {
-	r, vfs, cleanup := newTestVFS(t)
-	defer cleanup()
+	r, vfs := newTestVFS(t)
 
 	// pre-conditions
 	assert.Nil(t, vfs.usage)
diff --git a/vfs/vfscache/cache_test.go b/vfs/vfscache/cache_test.go
index bd08736a090e2..5ed346a13cee8 100644
--- a/vfs/vfscache/cache_test.go
+++ b/vfs/vfscache/cache_test.go
@@ -81,7 +81,7 @@ func addVirtual(remote string, size int64, isDir bool) error {
 	return nil
 }
 
-func newTestCacheOpt(t *testing.T, opt vfscommon.Options) (r *fstest.Run, c *Cache, cleanup func()) {
+func newTestCacheOpt(t *testing.T, opt vfscommon.Options) (r *fstest.Run, c *Cache) {
 	r = fstest.NewRun(t)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -90,18 +90,17 @@ func newTestCacheOpt(t *testing.T, opt vfscommon.Options) (r *fstest.Run, c *Cac
 	c, err := New(ctx, r.Fremote, &opt, addVirtual)
 	require.NoError(t, err)
 
-	cleanup = func() {
+	t.Cleanup(func() {
 		err := c.CleanUp()
 		require.NoError(t, err)
 		assertPathNotExist(t, c.root)
 		cancel()
-		r.Finalise()
-	}
+	})
 
-	return r, c, cleanup
+	return r, c
 }
 
-func newTestCache(t *testing.T) (r *fstest.Run, c *Cache, cleanup func()) {
+func newTestCache(t *testing.T) (r *fstest.Run, c *Cache) {
 	opt := vfscommon.DefaultOpt
 
 	// Disable the cache cleaner as it interferes with these tests
@@ -114,8 +113,7 @@ func newTestCache(t *testing.T) (r *fstest.Run, c *Cache, cleanup func()) {
 }
 
 func TestCacheNew(t *testing.T) {
-	r, c, cleanup := newTestCache(t)
-	defer cleanup()
+	r, c := newTestCache(t)
 
 	assert.Contains(t, c.root, "vfs")
 	assert.Contains(t, c.fcache.Root(), filepath.Base(r.Fremote.Root()))
@@ -191,8 +189,7 @@ func TestCacheNew(t *testing.T) {
 }
 
 func TestCacheOpens(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	assert.Equal(t, []string(nil), itemAsString(c))
 	potato := c.Item("potato")
@@ -240,8 +237,7 @@ func TestCacheOpens(t *testing.T) {
 
 // test the open, createItemDir, purge, close, purge sequence
 func TestCacheOpenMkdir(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	// open
 	potato := c.Item("sub/potato")
@@ -288,8 +284,7 @@ func TestCacheOpenMkdir(t *testing.T) {
 }
 
 func TestCachePurgeOld(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	// Test funcs
 	c.purgeOld(-10 * time.Second)
@@ -342,8 +337,7 @@ func TestCachePurgeOld(t *testing.T) {
 }
 
 func TestCachePurgeOverQuota(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	// Test funcs
 
@@ -428,8 +422,7 @@ func TestCachePurgeOverQuota(t *testing.T) {
 
 // test reset clean files
 func TestCachePurgeClean(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 	contents, obj, potato1 := newFile(t, r, c, "existing")
 	_ = contents
 
@@ -491,8 +484,7 @@ func TestCachePurgeClean(t *testing.T) {
 }
 
 func TestCacheInUse(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	assert.False(t, c.InUse("potato"))
 
@@ -510,8 +502,7 @@ func TestCacheInUse(t *testing.T) {
 }
 
 func TestCacheDirtyItem(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	assert.Nil(t, c.DirtyItem("potato"))
 
@@ -530,8 +521,7 @@ func TestCacheDirtyItem(t *testing.T) {
 }
 
 func TestCacheExistsAndRemove(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	assert.False(t, c.Exists("potato"))
 
@@ -554,8 +544,7 @@ func TestCacheExistsAndRemove(t *testing.T) {
 }
 
 func TestCacheRename(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	// setup
 
@@ -611,8 +600,7 @@ func TestCacheCleaner(t *testing.T) {
 	opt := vfscommon.DefaultOpt
 	opt.CachePollInterval = 10 * time.Millisecond
 	opt.CacheMaxAge = 20 * time.Millisecond
-	_, c, cleanup := newTestCacheOpt(t, opt)
-	defer cleanup()
+	_, c := newTestCacheOpt(t, opt)
 
 	time.Sleep(2 * opt.CachePollInterval)
 
@@ -634,8 +622,7 @@ func TestCacheCleaner(t *testing.T) {
 }
 
 func TestCacheSetModTime(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	t1 := time.Date(2010, 1, 2, 3, 4, 5, 9, time.UTC)
 
@@ -653,8 +640,7 @@ func TestCacheSetModTime(t *testing.T) {
 }
 
 func TestCacheTotaInUse(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	assert.Equal(t, int(0), c.TotalInUse())
 
@@ -681,8 +667,7 @@ func TestCacheTotaInUse(t *testing.T) {
 }
 
 func TestCacheDump(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	out := (*Cache)(nil).Dump()
 	assert.Equal(t, "Cache: \n", out)
@@ -703,8 +688,7 @@ func TestCacheDump(t *testing.T) {
 }
 
 func TestCacheStats(t *testing.T) {
-	_, c, cleanup := newTestCache(t)
-	defer cleanup()
+	_, c := newTestCache(t)
 
 	out := c.Stats()
 	assert.Equal(t, int64(0), out["bytesUsed"])
diff --git a/vfs/vfscache/downloaders/downloaders_test.go b/vfs/vfscache/downloaders/downloaders_test.go
index d079986b501bc..bc31848c9f28c 100644
--- a/vfs/vfscache/downloaders/downloaders_test.go
+++ b/vfs/vfscache/downloaders/downloaders_test.go
@@ -75,7 +75,6 @@ func (item *testItem) WriteAtNoOverwrite(b []byte, off int64) (n int, skipped in
 
 func TestDownloaders(t *testing.T) {
 	r := fstest.NewRun(t)
-	defer r.Finalise()
 
 	var (
 		ctx    = context.Background()
diff --git a/vfs/vfscache/item_test.go b/vfs/vfscache/item_test.go
index 676380f11e4e6..27a40cbbd812f 100644
--- a/vfs/vfscache/item_test.go
+++ b/vfs/vfscache/item_test.go
@@ -23,7 +23,7 @@ import (
 
 var zeroes = string(make([]byte, 100))
 
-func newItemTestCache(t *testing.T) (r *fstest.Run, c *Cache, cleanup func()) {
+func newItemTestCache(t *testing.T) (r *fstest.Run, c *Cache) {
 	opt := vfscommon.DefaultOpt
 
 	// Disable the cache cleaner as it interferes with these tests
@@ -61,8 +61,7 @@ func newFile(t *testing.T, r *fstest.Run, c *Cache, remote string) (contents str
 }
 
 func TestItemExists(t *testing.T) {
-	_, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	_, c := newItemTestCache(t)
 	item, _ := c.get("potato")
 
 	assert.False(t, item.Exists())
@@ -75,8 +74,7 @@ func TestItemExists(t *testing.T) {
 }
 
 func TestItemGetSize(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 	item, _ := c.get("potato")
 	require.NoError(t, item.Open(nil))
 
@@ -97,8 +95,7 @@ func TestItemGetSize(t *testing.T) {
 }
 
 func TestItemDirty(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 	item, _ := c.get("potato")
 	require.NoError(t, item.Open(nil))
 
@@ -122,8 +119,7 @@ func TestItemDirty(t *testing.T) {
 }
 
 func TestItemSync(t *testing.T) {
-	_, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	_, c := newItemTestCache(t)
 	item, _ := c.get("potato")
 
 	require.Error(t, item.Sync())
@@ -136,8 +132,7 @@ func TestItemSync(t *testing.T) {
 }
 
 func TestItemTruncateNew(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 	item, _ := c.get("potato")
 
 	require.Error(t, item.Truncate(0))
@@ -164,8 +159,7 @@ func TestItemTruncateNew(t *testing.T) {
 }
 
 func TestItemTruncateExisting(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 
 	contents, obj, item := newFile(t, r, c, "existing")
 
@@ -184,8 +178,7 @@ func TestItemTruncateExisting(t *testing.T) {
 }
 
 func TestItemReadAt(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 
 	contents, obj, item := newFile(t, r, c, "existing")
 	buf := make([]byte, 10)
@@ -219,8 +212,7 @@ func TestItemReadAt(t *testing.T) {
 }
 
 func TestItemWriteAtNew(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 	item, _ := c.get("potato")
 	buf := make([]byte, 10)
 
@@ -251,8 +243,7 @@ func TestItemWriteAtNew(t *testing.T) {
 }
 
 func TestItemWriteAtExisting(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 
 	contents, obj, item := newFile(t, r, c, "existing")
 
@@ -276,8 +267,7 @@ func TestItemWriteAtExisting(t *testing.T) {
 }
 
 func TestItemLoadMeta(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 
 	contents, obj, item := newFile(t, r, c, "existing")
 	_ = contents
@@ -305,8 +295,7 @@ func TestItemLoadMeta(t *testing.T) {
 }
 
 func TestItemReload(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 
 	contents, obj, item := newFile(t, r, c, "existing")
 	_ = contents
@@ -350,8 +339,7 @@ func TestItemReload(t *testing.T) {
 }
 
 func TestItemReloadRemoteGone(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 
 	contents, obj, item := newFile(t, r, c, "existing")
 	_ = contents
@@ -394,8 +382,7 @@ func TestItemReloadRemoteGone(t *testing.T) {
 }
 
 func TestItemReloadCacheStale(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 
 	contents, obj, item := newFile(t, r, c, "existing")
 
@@ -450,8 +437,7 @@ func TestItemReloadCacheStale(t *testing.T) {
 }
 
 func TestItemReadWrite(t *testing.T) {
-	r, c, cleanup := newItemTestCache(t)
-	defer cleanup()
+	r, c := newItemTestCache(t)
 	const (
 		size     = 50*1024*1024 + 123
 		fileName = "large"
diff --git a/vfs/write_test.go b/vfs/write_test.go
index 99f5d63b25d74..b059cdad16eb2 100644
--- a/vfs/write_test.go
+++ b/vfs/write_test.go
@@ -17,20 +17,19 @@ import (
 )
 
 // Open a file for write
-func writeHandleCreate(t *testing.T) (r *fstest.Run, vfs *VFS, fh *WriteFileHandle, cleanup func()) {
-	r, vfs, cleanup = newTestVFS(t)
+func writeHandleCreate(t *testing.T) (r *fstest.Run, vfs *VFS, fh *WriteFileHandle) {
+	r, vfs = newTestVFS(t)
 
 	h, err := vfs.OpenFile("file1", os.O_WRONLY|os.O_CREATE, 0777)
 	require.NoError(t, err)
 	fh, ok := h.(*WriteFileHandle)
 	require.True(t, ok)
 
-	return r, vfs, fh, cleanup
+	return r, vfs, fh
 }
 
 func TestWriteFileHandleMethods(t *testing.T) {
-	r, vfs, fh, cleanup := writeHandleCreate(t)
-	defer cleanup()
+	r, vfs, fh := writeHandleCreate(t)
 
 	// String
 	assert.Equal(t, "file1 (w)", fh.String())
@@ -132,8 +131,7 @@ func TestWriteFileHandleMethods(t *testing.T) {
 }
 
 func TestWriteFileHandleWriteAt(t *testing.T) {
-	r, vfs, fh, cleanup := writeHandleCreate(t)
-	defer cleanup()
+	r, vfs, fh := writeHandleCreate(t)
 
 	// Preconditions
 	assert.Equal(t, int64(0), fh.offset)
@@ -177,8 +175,7 @@ func TestWriteFileHandleWriteAt(t *testing.T) {
 }
 
 func TestWriteFileHandleFlush(t *testing.T) {
-	_, vfs, fh, cleanup := writeHandleCreate(t)
-	defer cleanup()
+	_, vfs, fh := writeHandleCreate(t)
 
 	// Check Flush already creates file for unwritten handles, without closing it
 	err := fh.Flush()
@@ -210,8 +207,7 @@ func TestWriteFileHandleFlush(t *testing.T) {
 }
 
 func TestWriteFileHandleRelease(t *testing.T) {
-	_, _, fh, cleanup := writeHandleCreate(t)
-	defer cleanup()
+	_, _, fh := writeHandleCreate(t)
 
 	// Check Release closes file
 	err := fh.Release()
@@ -258,8 +254,7 @@ func canSetModTime(t *testing.T, r *fstest.Run) bool {
 
 // tests mod time on open files
 func TestWriteFileModTimeWithOpenWriters(t *testing.T) {
-	r, vfs, fh, cleanup := writeHandleCreate(t)
-	defer cleanup()
+	r, vfs, fh := writeHandleCreate(t)
 
 	if !canSetModTime(t, r) {
 		t.Skip("can't set mod time")
@@ -286,8 +281,7 @@ func TestWriteFileModTimeWithOpenWriters(t *testing.T) {
 }
 
 func testFileReadAt(t *testing.T, n int) {
-	_, vfs, fh, cleanup := writeHandleCreate(t)
-	defer cleanup()
+	_, vfs, fh := writeHandleCreate(t)
 
 	contents := []byte(random.String(n))
 	if n != 0 {

From beea4d5119555c0424cf3bc8b3e67d625813b014 Mon Sep 17 00:00:00 2001
From: Ole Frost <82263101+olefrost@users.noreply.github.com>
Date: Wed, 7 Dec 2022 20:44:28 +0100
Subject: [PATCH 434/560] lib/oauthutil: Improved usability of config flows
 needing web browser

The config question "Use auto config?" confused many users and lead to
recurring forum posts from users that were unaware that they were using
a remote or headless machine.

This commit makes the question and possible options more descriptive
and precise.

This commit also adds references to the guide on remote setup in the
documentation of backends using oauth as primary authentication.
---
 cmd/config/config.go               |  2 +-
 docs/content/amazonclouddrive.md   |  7 ++++---
 docs/content/box.md                | 14 ++++++++------
 docs/content/drive.md              | 13 +++++++++----
 docs/content/dropbox.md            | 10 ++++++++++
 docs/content/googlecloudstorage.md | 13 +++++++++----
 docs/content/googlephotos.md       | 13 +++++++++----
 docs/content/hidrive.md            |  5 ++++-
 docs/content/onedrive.md           |  7 ++++---
 docs/content/pcloud.md             |  7 ++++---
 docs/content/premiumizeme.md       |  7 ++++---
 docs/content/putio.md              | 13 +++++++++----
 docs/content/remote_setup.md       | 22 ++++++++++++----------
 docs/content/sharefile.md          |  7 ++++---
 docs/content/yandex.md             |  7 ++++---
 docs/content/zoho.md               |  7 ++++---
 lib/oauthutil/oauthutil.go         |  2 +-
 17 files changed, 100 insertions(+), 56 deletions(-)

diff --git a/cmd/config/config.go b/cmd/config/config.go
index 5fb21a0cf460a..e3b916aad904c 100644
--- a/cmd/config/config.go
+++ b/cmd/config/config.go
@@ -176,7 +176,7 @@ This will look something like (some irrelevant detail removed):
     "State": "*oauth-islocal,teamdrive,,",
     "Option": {
         "Name": "config_is_local",
-        "Help": "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n",
+        "Help": "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n",
         "Default": true,
         "Examples": [
             {
diff --git a/docs/content/amazonclouddrive.md b/docs/content/amazonclouddrive.md
index 2fd16db74828a..b0068f94f524d 100644
--- a/docs/content/amazonclouddrive.md
+++ b/docs/content/amazonclouddrive.md
@@ -79,9 +79,10 @@ Token server url - leave blank to use Amazon's.
 token_url> Optional token URL
 Remote config
 Make sure your Redirect URL is set to "http://127.0.0.1:53682/" in your custom config.
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
diff --git a/docs/content/box.md b/docs/content/box.md
index 906d5d0ad7580..69b8676a66e0b 100644
--- a/docs/content/box.md
+++ b/docs/content/box.md
@@ -57,9 +57,10 @@ Choose a number from below, or type in your own value
    \ "enterprise"
 box_sub_type>
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
@@ -176,9 +177,10 @@ Already have a token - refresh?
 y) Yes
 n) No
 y/n> y
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
diff --git a/docs/content/drive.md b/docs/content/drive.md
index 922961ef442a3..80d20f42fb3f5 100644
--- a/docs/content/drive.md
+++ b/docs/content/drive.md
@@ -62,9 +62,10 @@ scope> 1
 Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login.
 service_account_file>
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine or Y didn't work
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
@@ -91,8 +92,12 @@ d) Delete this remote
 y/e/d> y
 ```
 
+See the [remote setup docs](/remote_setup/) for how to set it up on a
+machine with no Internet browser available.
+
 Note that rclone runs a webserver on your local machine to collect the
-token as returned from Google if you use auto config mode. This only
+token as returned from Google if using web browser to automatically 
+authenticate. This only
 runs from the moment it opens your browser to the moment you get back
 the verification code.  This is on `http://127.0.0.1:53682/` and it
 may require you to unblock it temporarily if you are running a host
diff --git a/docs/content/dropbox.md b/docs/content/dropbox.md
index e6c69654c10cf..cb9af2e1c6dd2 100644
--- a/docs/content/dropbox.md
+++ b/docs/content/dropbox.md
@@ -56,6 +56,16 @@ d) Delete this remote
 y/e/d> y
 ```
 
+See the [remote setup docs](/remote_setup/) for how to set it up on a
+machine with no Internet browser available.
+
+Note that rclone runs a webserver on your local machine to collect the
+token as returned from Dropbox. This only
+runs from the moment it opens your browser to the moment you get back
+the verification code.  This is on `http://127.0.0.1:53682/` and it
+may require you to unblock it temporarily if you are running a host
+firewall, or use manual mode.
+
 You can then use it like this,
 
 List directories in top level of your dropbox
diff --git a/docs/content/googlecloudstorage.md b/docs/content/googlecloudstorage.md
index 2a254848f90c9..1932696f2f235 100644
--- a/docs/content/googlecloudstorage.md
+++ b/docs/content/googlecloudstorage.md
@@ -117,9 +117,10 @@ Choose a number from below, or type in your own value
    \ "DURABLE_REDUCED_AVAILABILITY"
 storage_class> 5
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine or Y didn't work
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
@@ -143,8 +144,12 @@ d) Delete this remote
 y/e/d> y
 ```
 
+See the [remote setup docs](/remote_setup/) for how to set it up on a
+machine with no Internet browser available.
+
 Note that rclone runs a webserver on your local machine to collect the
-token as returned from Google if you use auto config mode. This only
+token as returned from Google if using web browser to automatically 
+authenticate. This only
 runs from the moment it opens your browser to the moment you get back
 the verification code.  This is on `http://127.0.0.1:53682/` and this
 it may require you to unblock it temporarily if you are running a host
diff --git a/docs/content/googlephotos.md b/docs/content/googlephotos.md
index bef6f3cf39c9c..0076458ae8b16 100644
--- a/docs/content/googlephotos.md
+++ b/docs/content/googlephotos.md
@@ -62,9 +62,10 @@ y) Yes
 n) No
 y/n> n
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
@@ -88,8 +89,12 @@ d) Delete this remote
 y/e/d> y
 ```
 
+See the [remote setup docs](/remote_setup/) for how to set it up on a
+machine with no Internet browser available.
+
 Note that rclone runs a webserver on your local machine to collect the
-token as returned from Google if you use auto config mode. This only
+token as returned from Google if using web browser to automatically 
+authenticate. This only
 runs from the moment it opens your browser to the moment you get back
 the verification code.  This is on `http://127.0.0.1:53682/` and this
 may require you to unblock it temporarily if you are running a host
diff --git a/docs/content/hidrive.md b/docs/content/hidrive.md
index 0a75686ca62f2..750e0890aadd9 100644
--- a/docs/content/hidrive.md
+++ b/docs/content/hidrive.md
@@ -45,7 +45,10 @@ Leave blank normally.
 scope_access>
 Edit advanced config?
 y/n> n
-Use auto config?
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y/n> y
 If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx
 Log in and authorize rclone for access
diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md
index 43750597b157b..4b46fac8cf3b5 100644
--- a/docs/content/onedrive.md
+++ b/docs/content/onedrive.md
@@ -53,9 +53,10 @@ y) Yes
 n) No
 y/n> n
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
diff --git a/docs/content/pcloud.md b/docs/content/pcloud.md
index cd15bb6424b81..bbf06a1415a27 100644
--- a/docs/content/pcloud.md
+++ b/docs/content/pcloud.md
@@ -40,9 +40,10 @@ client_id>
 Pcloud App Client Secret - leave blank normally.
 client_secret> 
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
diff --git a/docs/content/premiumizeme.md b/docs/content/premiumizeme.md
index 7b53b77b5f2db..ae7f44372ead1 100644
--- a/docs/content/premiumizeme.md
+++ b/docs/content/premiumizeme.md
@@ -39,9 +39,10 @@ Storage> premiumizeme
 ** See help for premiumizeme backend at: https://rclone.org/premiumizeme/ **
 
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
diff --git a/docs/content/putio.md b/docs/content/putio.md
index 79fb346a9d8dd..ae67f1eb61278 100644
--- a/docs/content/putio.md
+++ b/docs/content/putio.md
@@ -41,9 +41,10 @@ Storage> putio
 ** See help for putio backend at: https://rclone.org/putio/ **
 
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
@@ -76,8 +77,12 @@ q) Quit config
 e/n/d/r/c/s/q> q
 ```
 
+See the [remote setup docs](/remote_setup/) for how to set it up on a
+machine with no Internet browser available.
+
 Note that rclone runs a webserver on your local machine to collect the
-token as returned from Google if you use auto config mode. This only
+token as returned from put.io  if using web browser to automatically 
+authenticate. This only
 runs from the moment it opens your browser to the moment you get back
 the verification code.  This is on `http://127.0.0.1:53682/` and this
 it may require you to unblock it temporarily if you are running a host
diff --git a/docs/content/remote_setup.md b/docs/content/remote_setup.md
index b6d81c1b057e1..057e285dafc70 100644
--- a/docs/content/remote_setup.md
+++ b/docs/content/remote_setup.md
@@ -15,15 +15,16 @@ two ways of doing it, described below.
 
 ## Configuring using rclone authorize ##
 
-On the headless box run `rclone` config but answer `N` to the `Use
-auto config?` question.
+On the headless box run `rclone` config but answer `N` to the `Use web browser 
+to automatically authenticate?` question.
 
 ```
 ...
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes (default)
 n) No
 y/n> n
@@ -99,15 +100,16 @@ Linux and MacOS users can utilize SSH Tunnel to redirect the headless box port 5
 ```
 ssh -L localhost:53682:localhost:53682 username@remote_server
 ```
-Then on the headless box run `rclone` config and answer `Y` to the `Use
-auto config?` question.
+Then on the headless box run `rclone` config and answer `Y` to the `Use web 
+browser to automatically authenticate?` question.
 
 ```
 ...
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes (default)
 n) No
 y/n> y
diff --git a/docs/content/sharefile.md b/docs/content/sharefile.md
index d85ec4b19b7b0..c02315872f1b8 100644
--- a/docs/content/sharefile.md
+++ b/docs/content/sharefile.md
@@ -57,9 +57,10 @@ y) Yes
 n) No
 y/n> n
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
diff --git a/docs/content/yandex.md b/docs/content/yandex.md
index f5716ca7006d8..0093d623593f7 100644
--- a/docs/content/yandex.md
+++ b/docs/content/yandex.md
@@ -34,9 +34,10 @@ client_id>
 Yandex Client Secret - leave blank normally.
 client_secret>
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
diff --git a/docs/content/zoho.md b/docs/content/zoho.md
index c0143dcbf8408..e24c5be173e5b 100644
--- a/docs/content/zoho.md
+++ b/docs/content/zoho.md
@@ -45,9 +45,10 @@ y) Yes
 n) No (default)
 y/n> n
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes (default)
 n) No
 y/n> 
diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go
index a7326e8866242..cd9cb74d3c28d 100644
--- a/lib/oauthutil/oauthutil.go
+++ b/lib/oauthutil/oauthutil.go
@@ -480,7 +480,7 @@ func ConfigOAuth(ctx context.Context, name string, m configmap.Mapper, ri *fs.Re
 		if in.Result == "false" {
 			return fs.ConfigGoto(newState("*oauth-done"))
 		}
-		return fs.ConfigConfirm(newState("*oauth-islocal"), true, "config_is_local", "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n")
+		return fs.ConfigConfirm(newState("*oauth-islocal"), true, "config_is_local", "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n")
 	case "*oauth-islocal":
 		if in.Result == "true" {
 			return fs.ConfigGoto(newState("*oauth-do"))

From b98d7f6634a5cb1b260c66917a546e1352f7e1dd Mon Sep 17 00:00:00 2001
From: Kaloyan Raev 
Date: Thu, 8 Dec 2022 17:06:13 +0200
Subject: [PATCH 435/560] storj: implement server side Copy

---
 backend/storj/fs.go      | 41 ++++++++++++++++++++++++++++++++++++++++
 docs/content/overview.md |  2 +-
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/backend/storj/fs.go b/backend/storj/fs.go
index b39e0b25d8e52..9a0a5ebe27356 100644
--- a/backend/storj/fs.go
+++ b/backend/storj/fs.go
@@ -160,6 +160,7 @@ var (
 	_ fs.ListRer     = &Fs{}
 	_ fs.PutStreamer = &Fs{}
 	_ fs.Mover       = &Fs{}
+	_ fs.Copier      = &Fs{}
 )
 
 // NewFs creates a filesystem backed by Storj.
@@ -720,3 +721,43 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	// Read the new object
 	return f.NewObject(ctx, remote)
 }
+
+// Copy src to this remote using server-side copy operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+
+	// Copy parameters
+	srcBucket, srcKey := bucket.Split(srcObj.absolute)
+	dstBucket, dstKey := f.absolute(remote)
+	options := uplink.CopyObjectOptions{}
+
+	// Do the copy
+	newObject, err := f.project.CopyObject(ctx, srcBucket, srcKey, dstBucket, dstKey, &options)
+	if err != nil {
+		// Make sure destination bucket exists
+		_, err := f.project.EnsureBucket(ctx, dstBucket)
+		if err != nil {
+			return nil, fmt.Errorf("copy object failed to create destination bucket: %w", err)
+		}
+		// And try again
+		newObject, err = f.project.CopyObject(ctx, srcBucket, srcKey, dstBucket, dstKey, &options)
+		if err != nil {
+			return nil, fmt.Errorf("copy object failed: %w", err)
+		}
+	}
+
+	// Return the new object
+	return newObjectFromUplink(f, remote, newObject), nil
+}
diff --git a/docs/content/overview.md b/docs/content/overview.md
index 790dbf7fd9235..eca35d8d49802 100644
--- a/docs/content/overview.md
+++ b/docs/content/overview.md
@@ -504,7 +504,7 @@ upon backend-specific capabilities.
 | Sia                          | No    | No   | No   | No      | No      | No    | Yes          | No           | No    | Yes      |
 | SMB                          | No    | No   | Yes  | Yes     | No      | No    | Yes          | No           | No    | Yes      |
 | SugarSync                    | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes          | Yes          | No    | Yes      |
-| Storj                        | Yes † | No   | Yes  | No      | No      | Yes   | Yes          | No           | No    | No       |
+| Storj                        | Yes † | Yes  | Yes  | No      | No      | Yes   | Yes          | No           | No    | No       |
 | Uptobox                      | No    | Yes  | Yes  | Yes     | No      | No    | No           | No           | No    | No       |
 | WebDAV                       | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes ‡        | No           | Yes   | Yes      |
 | Yandex Disk                  | Yes   | Yes  | Yes  | Yes     | Yes     | No    | Yes          | Yes          | Yes   | Yes      |

From c5a2c9b04632d87af4774ab7c6c06d4d80146a87 Mon Sep 17 00:00:00 2001
From: vanplus <60313789+vanplus@users.noreply.github.com>
Date: Mon, 12 Dec 2022 20:04:28 +0800
Subject: [PATCH 436/560] onedrive: document workaround for shared with me
 files

---
 docs/content/onedrive.md | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md
index 4b46fac8cf3b5..236f77d7d43fe 100644
--- a/docs/content/onedrive.md
+++ b/docs/content/onedrive.md
@@ -714,3 +714,28 @@ public links to be made for the organisation/sharepoint library. To fix the
 permissions as an admin, take a look at the docs:
 [1](https://docs.microsoft.com/en-us/sharepoint/turn-external-sharing-on-or-off),
 [2](https://support.microsoft.com/en-us/office/set-up-and-manage-access-requests-94b26e0b-2822-49d4-929a-8455698654b3).
+
+### Can not access `Shared` with me files
+
+Shared with me files is not supported by rclone [currently](https://github.com/rclone/rclone/issues/4062), but there is a workaround:
+
+1. Visit [https://onedrive.live.com](https://onedrive.live.com/)
+2. Right click a item in `Shared`, then click `Add shortcut to My files` in the context
+
                  
+   Screenshot (Shared with me)
+
+   ![make_shortcut](https://user-images.githubusercontent.com/60313789/206118040-7e762b3b-aa61-41a1-8649-cc18889f3572.png)    
+
                   
+     
+3. The shortcut will appear in `My files`, you can access it with rclone, it behaves like a normal folder/file.
+
                  
+   Screenshot (My Files)
+
+   ![in_my_files](https://i.imgur.com/0S8H3li.png)
+
                  
+
+
                  
+   Screenshot (rclone mount)
+
+   ![rclone_mount](https://i.imgur.com/2Iq66sW.png)
+
                  

From 397f428c48a164b3cf63e5727e3cf52fb775f85d Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 12 Dec 2022 12:48:48 +0000
Subject: [PATCH 437/560] Add vanplus to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 279f465776d60..995c45f2d8b79 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -670,3 +670,4 @@ put them back in again.` >}}
   * Erik Agterdenbos 
   * Kevin Verstaen <48050031+kverstae@users.noreply.github.com>
   * MohammadReza 
+  * vanplus <60313789+vanplus@users.noreply.github.com>

From 53ff5bb2051c79245f7b5205384fdf9e302d11ae Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 12 Dec 2022 12:47:02 +0000
Subject: [PATCH 438/560] build: Update golang.org/x/net/http2 to fix
 GO-2022-1144

An attacker can cause excessive memory growth in a Go server accepting
HTTP/2 requests. HTTP/2 server connections contain a cache of HTTP
header keys sent by the client. While the total number of entries in
this cache is capped, an attacker sending very large keys can cause
the server to allocate approximately 64 MiB per open connection.
---
 go.mod | 8 ++++----
 go.sum | 8 ++++++++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/go.mod b/go.mod
index 1f9225d3b0d8e..873b24fb978e5 100644
--- a/go.mod
+++ b/go.mod
@@ -62,11 +62,11 @@ require (
 	go.etcd.io/bbolt v1.3.6
 	goftp.io/server v0.4.1
 	golang.org/x/crypto v0.3.0
-	golang.org/x/net v0.2.0
+	golang.org/x/net v0.4.0
 	golang.org/x/oauth2 v0.2.0
 	golang.org/x/sync v0.1.0
-	golang.org/x/sys v0.2.0
-	golang.org/x/text v0.4.0
+	golang.org/x/sys v0.3.0
+	golang.org/x/text v0.5.0
 	golang.org/x/time v0.2.0
 	google.golang.org/api v0.103.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -144,5 +144,5 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/pkg/xattr v0.4.9
 	golang.org/x/mobile v0.0.0-20221110043201-43a038452099
-	golang.org/x/term v0.2.0
+	golang.org/x/term v0.3.0
 )
diff --git a/go.sum b/go.sum
index 2e41116c1e6e1..38a637db50897 100644
--- a/go.sum
+++ b/go.sum
@@ -1035,6 +1035,8 @@ golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfS
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
+golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1175,12 +1177,16 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
+golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1193,6 +1199,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
+golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

From 4120dffcc1b83ed8e1890d8372242e2b1ff2ccc8 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 16 Sep 2022 16:11:54 +0200
Subject: [PATCH 439/560] fspath: add unit tests for remote names with space

---
 fs/fspath/path_test.go | 46 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/fs/fspath/path_test.go b/fs/fspath/path_test.go
index e1ac7866d6f4b..402692136463a 100644
--- a/fs/fspath/path_test.go
+++ b/fs/fspath/path_test.go
@@ -37,6 +37,10 @@ func TestCheckConfigName(t *testing.T) {
 		{".", nil},
 		{"..", nil},
 		{".r.e.m.o.t.e.", nil},
+		{"rem ote", nil},
+		{"remote ", nil},
+		{" remote", nil},
+		{" remote ", nil},
 	} {
 		got := CheckConfigName(test.in)
 		assert.Equal(t, test.want, got, test.in)
@@ -54,6 +58,10 @@ func TestCheckRemoteName(t *testing.T) {
 		{".:", nil},
 		{"..:", nil},
 		{".r.e.m.o.t.e.:", nil},
+		{"rem ote:", nil},
+		{"remote :", nil},
+		{" remote:", nil},
+		{" remote :", nil},
 		{"", errInvalidCharacters},
 		{"rem:ote", errInvalidCharacters},
 		{"rem:ote:", errInvalidCharacters},
@@ -209,6 +217,34 @@ func TestParse(t *testing.T) {
 				Name:         "rem.ote",
 				Path:         "/path/to/file",
 			},
+		}, {
+			in: "rem ote:/path/to/file",
+			wantParsed: Parsed{
+				ConfigString: "rem ote",
+				Name:         "rem ote",
+				Path:         "/path/to/file",
+			},
+		}, {
+			in: "remote :/path/to/file",
+			wantParsed: Parsed{
+				ConfigString: "remote ",
+				Name:         "remote ",
+				Path:         "/path/to/file",
+			},
+		}, {
+			in: " remote:/path/to/file",
+			wantParsed: Parsed{
+				ConfigString: " remote",
+				Name:         " remote",
+				Path:         "/path/to/file",
+			},
+		}, {
+			in: " remote :/path/to/file",
+			wantParsed: Parsed{
+				ConfigString: " remote ",
+				Name:         " remote ",
+				Path:         "/path/to/file",
+			},
 		}, {
 			in:      "rem#ote:/path/to/file",
 			wantErr: errInvalidCharacters,
@@ -444,6 +480,11 @@ func TestSplitFs(t *testing.T) {
 		{"remote:potato/sausage", "remote:", "potato/sausage", nil},
 		{"rem.ote:potato/sausage", "rem.ote:", "potato/sausage", nil},
 
+		{"rem ote:", "rem ote:", "", nil},
+		{"remote :", "remote :", "", nil},
+		{" remote:", " remote:", "", nil},
+		{" remote :", " remote :", "", nil},
+
 		{".:", ".:", "", nil},
 		{"..:", "..:", "", nil},
 		{".:potato/sausage", ".:", "potato/sausage", nil},
@@ -496,6 +537,11 @@ func TestSplit(t *testing.T) {
 		{"remote:potato/sausage", "remote:potato/", "sausage", nil},
 		{"rem.ote:potato/sausage", "rem.ote:potato/", "sausage", nil},
 
+		{"rem ote:", "rem ote:", "", nil},
+		{"remote :", "remote :", "", nil},
+		{" remote:", " remote:", "", nil},
+		{" remote :", " remote :", "", nil},
+
 		{".:", ".:", "", nil},
 		{"..:", "..:", "", nil},
 		{".:potato/sausage", ".:potato/", "sausage", nil},

From f73be767a4128441bf667049ebb1dccffdc2e90f Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 16 Sep 2022 16:27:01 +0200
Subject: [PATCH 440/560] fspath: add unit tests for remote names with leading
 dash

---
 fs/fspath/path_test.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fs/fspath/path_test.go b/fs/fspath/path_test.go
index 402692136463a..c72423c0ca1e3 100644
--- a/fs/fspath/path_test.go
+++ b/fs/fspath/path_test.go
@@ -58,6 +58,7 @@ func TestCheckRemoteName(t *testing.T) {
 		{".:", nil},
 		{"..:", nil},
 		{".r.e.m.o.t.e.:", nil},
+		{"-r-emote-:", nil},
 		{"rem ote:", nil},
 		{"remote :", nil},
 		{" remote:", nil},

From 6b4a2c1c4e9cf3a6cc3b785dcdc180a17eb9be0a Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Sat, 17 Sep 2022 09:49:55 +0200
Subject: [PATCH 441/560] fspath: remove superfluous underscore covered by
 existing word character class in remote name regex

---
 fs/fspath/path.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fs/fspath/path.go b/fs/fspath/path.go
index 285d843fff760..7f0868b8ff825 100644
--- a/fs/fspath/path.go
+++ b/fs/fspath/path.go
@@ -13,7 +13,7 @@ import (
 )
 
 const (
-	configNameRe = `[\w_. -]+`
+	configNameRe = `[\w. -]+`
 	remoteNameRe = `^(:?` + configNameRe + `)`
 )
 

From 04a663829b52558c45925aced85de92d92c1ad64 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 16 Sep 2022 16:16:57 +0200
Subject: [PATCH 442/560] fspath: remove duplicate start-of-line anchor in
 remote name regex

---
 fs/fspath/path.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fs/fspath/path.go b/fs/fspath/path.go
index 7f0868b8ff825..7943894c5f8c9 100644
--- a/fs/fspath/path.go
+++ b/fs/fspath/path.go
@@ -14,7 +14,7 @@ import (
 
 const (
 	configNameRe = `[\w. -]+`
-	remoteNameRe = `^(:?` + configNameRe + `)`
+	remoteNameRe = `(:?` + configNameRe + `)`
 )
 
 var (

From c571200812133e79d7d0e833cfa46a8fa7a2b9d9 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 16 Sep 2022 16:18:26 +0200
Subject: [PATCH 443/560] fspath: remove unused capture group in remote name
 regex

---
 fs/fspath/path.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fs/fspath/path.go b/fs/fspath/path.go
index 7943894c5f8c9..810189a07d806 100644
--- a/fs/fspath/path.go
+++ b/fs/fspath/path.go
@@ -14,7 +14,7 @@ import (
 
 const (
 	configNameRe = `[\w. -]+`
-	remoteNameRe = `(:?` + configNameRe + `)`
+	remoteNameRe = `:?` + configNameRe
 )
 
 var (
@@ -35,7 +35,7 @@ var (
 	configNameMatcher = regexp.MustCompile(`^` + configNameRe + `$`)
 
 	// remoteNameMatcher is a pattern to match an rclone remote name at the start of a config
-	remoteNameMatcher = regexp.MustCompile(`^` + remoteNameRe + `(:$|,)`)
+	remoteNameMatcher = regexp.MustCompile(`^` + remoteNameRe + `(?::$|,)`)
 )
 
 // CheckConfigName returns an error if configName is invalid

From 3937233e1ea865052be73f9eeac5aa815ae82c79 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 16 Sep 2022 16:20:19 +0200
Subject: [PATCH 444/560] fspath: refactor away unnecessary constant for remote
 name regex

---
 fs/fspath/path.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/fs/fspath/path.go b/fs/fspath/path.go
index 810189a07d806..26575f7259165 100644
--- a/fs/fspath/path.go
+++ b/fs/fspath/path.go
@@ -14,7 +14,6 @@ import (
 
 const (
 	configNameRe = `[\w. -]+`
-	remoteNameRe = `:?` + configNameRe
 )
 
 var (
@@ -35,7 +34,7 @@ var (
 	configNameMatcher = regexp.MustCompile(`^` + configNameRe + `$`)
 
 	// remoteNameMatcher is a pattern to match an rclone remote name at the start of a config
-	remoteNameMatcher = regexp.MustCompile(`^` + remoteNameRe + `(?::$|,)`)
+	remoteNameMatcher = regexp.MustCompile(`^:?` + configNameRe + `(?::$|,)`)
 )
 
 // CheckConfigName returns an error if configName is invalid

From 683178a1f4d205a56c8a96593cf77ba6ef149474 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 16 Sep 2022 16:57:49 +0200
Subject: [PATCH 445/560] fspath: change remote name regex to not match when
 leading/trailing space

---
 fs/fspath/path.go      |  2 +-
 fs/fspath/path_test.go | 48 ++++++++++++++++--------------------------
 2 files changed, 19 insertions(+), 31 deletions(-)

diff --git a/fs/fspath/path.go b/fs/fspath/path.go
index 26575f7259165..6b8617feb1900 100644
--- a/fs/fspath/path.go
+++ b/fs/fspath/path.go
@@ -13,7 +13,7 @@ import (
 )
 
 const (
-	configNameRe = `[\w. -]+`
+	configNameRe = `[\w.-]+(?: +[\w.-]+)*`
 )
 
 var (
diff --git a/fs/fspath/path_test.go b/fs/fspath/path_test.go
index c72423c0ca1e3..9d80ec4a8c899 100644
--- a/fs/fspath/path_test.go
+++ b/fs/fspath/path_test.go
@@ -38,9 +38,9 @@ func TestCheckConfigName(t *testing.T) {
 		{"..", nil},
 		{".r.e.m.o.t.e.", nil},
 		{"rem ote", nil},
-		{"remote ", nil},
-		{" remote", nil},
-		{" remote ", nil},
+		{"remote ", errInvalidCharacters},
+		{" remote", errInvalidCharacters},
+		{" remote ", errInvalidCharacters},
 	} {
 		got := CheckConfigName(test.in)
 		assert.Equal(t, test.want, got, test.in)
@@ -60,9 +60,9 @@ func TestCheckRemoteName(t *testing.T) {
 		{".r.e.m.o.t.e.:", nil},
 		{"-r-emote-:", nil},
 		{"rem ote:", nil},
-		{"remote :", nil},
-		{" remote:", nil},
-		{" remote :", nil},
+		{"remote :", errInvalidCharacters},
+		{" remote:", errInvalidCharacters},
+		{" remote :", errInvalidCharacters},
 		{"", errInvalidCharacters},
 		{"rem:ote", errInvalidCharacters},
 		{"rem:ote:", errInvalidCharacters},
@@ -226,26 +226,14 @@ func TestParse(t *testing.T) {
 				Path:         "/path/to/file",
 			},
 		}, {
-			in: "remote :/path/to/file",
-			wantParsed: Parsed{
-				ConfigString: "remote ",
-				Name:         "remote ",
-				Path:         "/path/to/file",
-			},
+			in:      "remote :/path/to/file",
+			wantErr: errInvalidCharacters,
 		}, {
-			in: " remote:/path/to/file",
-			wantParsed: Parsed{
-				ConfigString: " remote",
-				Name:         " remote",
-				Path:         "/path/to/file",
-			},
+			in:      " remote:/path/to/file",
+			wantErr: errInvalidCharacters,
 		}, {
-			in: " remote :/path/to/file",
-			wantParsed: Parsed{
-				ConfigString: " remote ",
-				Name:         " remote ",
-				Path:         "/path/to/file",
-			},
+			in:      " remote :/path/to/file",
+			wantErr: errInvalidCharacters,
 		}, {
 			in:      "rem#ote:/path/to/file",
 			wantErr: errInvalidCharacters,
@@ -482,9 +470,9 @@ func TestSplitFs(t *testing.T) {
 		{"rem.ote:potato/sausage", "rem.ote:", "potato/sausage", nil},
 
 		{"rem ote:", "rem ote:", "", nil},
-		{"remote :", "remote :", "", nil},
-		{" remote:", " remote:", "", nil},
-		{" remote :", " remote :", "", nil},
+		{"remote :", "", "", errInvalidCharacters},
+		{" remote:", "", "", errInvalidCharacters},
+		{" remote :", "", "", errInvalidCharacters},
 
 		{".:", ".:", "", nil},
 		{"..:", "..:", "", nil},
@@ -539,9 +527,9 @@ func TestSplit(t *testing.T) {
 		{"rem.ote:potato/sausage", "rem.ote:potato/", "sausage", nil},
 
 		{"rem ote:", "rem ote:", "", nil},
-		{"remote :", "remote :", "", nil},
-		{" remote:", " remote:", "", nil},
-		{" remote :", " remote :", "", nil},
+		{"remote :", "", "", errInvalidCharacters},
+		{" remote:", "", "", errInvalidCharacters},
+		{" remote :", "", "", errInvalidCharacters},
 
 		{".:", ".:", "", nil},
 		{"..:", "..:", "", nil},

From f650a543ef9fe0453e07f60e149d4ed9b92c0b47 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Fri, 16 Sep 2022 16:59:08 +0200
Subject: [PATCH 446/560] docs: remote names may not start or end with space

---
 docs/content/docs.md | 1 +
 fs/fspath/path.go    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/content/docs.md b/docs/content/docs.md
index 1afa2a9a09026..d984970b54ac7 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -340,6 +340,7 @@ Will get their own names
 Remote names are case sensitive, and must adhere to the following rules:
  - May only contain `0`-`9`, `A`-`Z`, `a`-`z`, `_`, `-`, `.` and space.
  - May not start with `-` or space.
+ - May not end with space.
 
 Quoting and the shell
 ---------------------
diff --git a/fs/fspath/path.go b/fs/fspath/path.go
index 6b8617feb1900..a29b0e36323c9 100644
--- a/fs/fspath/path.go
+++ b/fs/fspath/path.go
@@ -17,7 +17,7 @@ const (
 )
 
 var (
-	errInvalidCharacters = errors.New("config name contains invalid characters - may only contain `0-9`, `A-Z`, `a-z`, `_`, `-`, `.` and space")
+	errInvalidCharacters = errors.New("config name contains invalid characters - may only contain `0-9`, `A-Z`, `a-z`, `_`, `-`, `.` and space, while not start or end with space")
 	errCantBeEmpty       = errors.New("can't use empty string as a path")
 	errCantStartWithDash = errors.New("config name starts with `-`")
 	errBadConfigParam    = errors.New("config parameters may only contain `0-9`, `A-Z`, `a-z` and `_`")

From 8e6a469f984c70115bfc388f61f5bbfdf99a8ed1 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Sat, 10 Dec 2022 15:53:42 +0100
Subject: [PATCH 447/560] fspath: allow unicode numbers and letters in remote
 names

Previously it was limited to plain ASCII (0-9, A-Z, a-z).

Implemented by adding \p{L}\p{N} alongside the \w in the regex,
even though these overlap it means we can be sure it is 100%
backwards compatible.

Fixes #6618
---
 backend/drive/drive.go |  2 +-
 docs/content/docs.md   | 10 +++++++++-
 fs/fspath/path.go      |  4 ++--
 fs/fspath/path_test.go |  6 ++++++
 4 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/backend/drive/drive.go b/backend/drive/drive.go
index cca4245c3636f..93fab7f8d3646 100644
--- a/backend/drive/drive.go
+++ b/backend/drive/drive.go
@@ -3431,7 +3431,7 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		if err != nil {
 			return nil, err
 		}
-		re := regexp.MustCompile(`[^\w_. -]+`)
+		re := regexp.MustCompile(`[^\w\p{L}\p{N}. -]+`)
 		if _, ok := opt["config"]; ok {
 			lines := []string{}
 			upstreams := []string{}
diff --git a/docs/content/docs.md b/docs/content/docs.md
index d984970b54ac7..eab4b5b29c3d7 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -338,10 +338,18 @@ Will get their own names
 ### Valid remote names
 
 Remote names are case sensitive, and must adhere to the following rules:
- - May only contain `0`-`9`, `A`-`Z`, `a`-`z`, `_`, `-`, `.` and space.
+ - May contain number, letter, `_`, `-`, `.` and space.
  - May not start with `-` or space.
  - May not end with space.
 
+Starting with rclone version 1.61, any Unicode numbers and letters are allowed,
+while in older versions it was limited to plain ASCII (0-9, A-Z, a-z). If you use
+the same rclone configuration from different shells, which may be configured with
+different character encoding, you must be cautious to use characters that are
+possible to write in all of them. This is mostly a problem on Windows, where
+the console traditionally uses a non-Unicode character set - defined
+by the so-called "code page".
+
 Quoting and the shell
 ---------------------
 
diff --git a/fs/fspath/path.go b/fs/fspath/path.go
index a29b0e36323c9..a9dd858d12aca 100644
--- a/fs/fspath/path.go
+++ b/fs/fspath/path.go
@@ -13,11 +13,11 @@ import (
 )
 
 const (
-	configNameRe = `[\w.-]+(?: +[\w.-]+)*`
+	configNameRe = `[\w\p{L}\p{N}.-]+(?: +[\w\p{L}\p{N}.-]+)*`
 )
 
 var (
-	errInvalidCharacters = errors.New("config name contains invalid characters - may only contain `0-9`, `A-Z`, `a-z`, `_`, `-`, `.` and space, while not start or end with space")
+	errInvalidCharacters = errors.New("config name contains invalid characters - may only contain numbers, letters, `_`, `-`, `.` and space, while not start or end with space")
 	errCantBeEmpty       = errors.New("can't use empty string as a path")
 	errCantStartWithDash = errors.New("config name starts with `-`")
 	errBadConfigParam    = errors.New("config parameters may only contain `0-9`, `A-Z`, `a-z` and `_`")
diff --git a/fs/fspath/path_test.go b/fs/fspath/path_test.go
index 9d80ec4a8c899..6c4ac754d461c 100644
--- a/fs/fspath/path_test.go
+++ b/fs/fspath/path_test.go
@@ -23,6 +23,7 @@ func TestCheckConfigName(t *testing.T) {
 		want error
 	}{
 		{"remote", nil},
+		{"REMOTE", nil},
 		{"", errInvalidCharacters},
 		{":remote:", errInvalidCharacters},
 		{"remote:", errInvalidCharacters},
@@ -38,6 +39,8 @@ func TestCheckConfigName(t *testing.T) {
 		{"..", nil},
 		{".r.e.m.o.t.e.", nil},
 		{"rem ote", nil},
+		{"blåbær", nil},
+		{"chữ Quốc ngữ", nil},
 		{"remote ", errInvalidCharacters},
 		{" remote", errInvalidCharacters},
 		{" remote ", errInvalidCharacters},
@@ -53,6 +56,7 @@ func TestCheckRemoteName(t *testing.T) {
 		want error
 	}{
 		{":remote:", nil},
+		{":REMOTE:", nil},
 		{":s3:", nil},
 		{"remote:", nil},
 		{".:", nil},
@@ -60,6 +64,8 @@ func TestCheckRemoteName(t *testing.T) {
 		{".r.e.m.o.t.e.:", nil},
 		{"-r-emote-:", nil},
 		{"rem ote:", nil},
+		{"blåbær:", nil},
+		{"chữ Quốc ngữ:", nil},
 		{"remote :", errInvalidCharacters},
 		{" remote:", errInvalidCharacters},
 		{" remote :", errInvalidCharacters},

From 8b9f3bbe294855bc8c99ee43568aea61871cc882 Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Mon, 12 Dec 2022 20:03:39 +0100
Subject: [PATCH 448/560] fspath: improved detection of illegal remote names
 starting with dash (related to #4261)

---
 fs/fspath/path.go      | 9 ++-------
 fs/fspath/path_test.go | 4 ++--
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/fs/fspath/path.go b/fs/fspath/path.go
index a9dd858d12aca..2c9c42b00f512 100644
--- a/fs/fspath/path.go
+++ b/fs/fspath/path.go
@@ -13,13 +13,12 @@ import (
 )
 
 const (
-	configNameRe = `[\w\p{L}\p{N}.-]+(?: +[\w\p{L}\p{N}.-]+)*`
+	configNameRe = `[\w\p{L}\p{N}.]+(?:[ -]+[\w\p{L}\p{N}.-]+)*` // don't allow it to start with `-` as it complicates usage (#4261)
 )
 
 var (
-	errInvalidCharacters = errors.New("config name contains invalid characters - may only contain numbers, letters, `_`, `-`, `.` and space, while not start or end with space")
+	errInvalidCharacters = errors.New("config name contains invalid characters - may only contain numbers, letters, `_`, `-`, `.` and space, while not start with `-` or space, and not end with space")
 	errCantBeEmpty       = errors.New("can't use empty string as a path")
-	errCantStartWithDash = errors.New("config name starts with `-`")
 	errBadConfigParam    = errors.New("config parameters may only contain `0-9`, `A-Z`, `a-z` and `_`")
 	errEmptyConfigParam  = errors.New("config parameters can't be empty")
 	errConfigNameEmpty   = errors.New("config name can't be empty")
@@ -42,10 +41,6 @@ func CheckConfigName(configName string) error {
 	if !configNameMatcher.MatchString(configName) {
 		return errInvalidCharacters
 	}
-	// Reject configName, if it starts with -, complicates usage. (#4261)
-	if strings.HasPrefix(configName, "-") {
-		return errCantStartWithDash
-	}
 	return nil
 }
 
diff --git a/fs/fspath/path_test.go b/fs/fspath/path_test.go
index 6c4ac754d461c..ad58ef3993d93 100644
--- a/fs/fspath/path_test.go
+++ b/fs/fspath/path_test.go
@@ -32,7 +32,7 @@ func TestCheckConfigName(t *testing.T) {
 		{"rem\\ote", errInvalidCharacters},
 		{"[remote", errInvalidCharacters},
 		{"*", errInvalidCharacters},
-		{"-remote", errCantStartWithDash},
+		{"-remote", errInvalidCharacters},
 		{"r-emote-", nil},
 		{"_rem_ote_", nil},
 		{".", nil},
@@ -62,7 +62,7 @@ func TestCheckRemoteName(t *testing.T) {
 		{".:", nil},
 		{"..:", nil},
 		{".r.e.m.o.t.e.:", nil},
-		{"-r-emote-:", nil},
+		{"-r-emote-:", errInvalidCharacters},
 		{"rem ote:", nil},
 		{"blåbær:", nil},
 		{"chữ Quốc ngữ:", nil},

From 5a59b49b6b67c8d955ee3f4e69fc6aa98944e21f Mon Sep 17 00:00:00 2001
From: albertony <12441419+albertony@users.noreply.github.com>
Date: Mon, 12 Dec 2022 20:05:12 +0100
Subject: [PATCH 449/560] drive: handle shared drives with leading/trailing
 space in name (related to #6618)

---
 backend/drive/drive.go |  4 +--
 fs/fspath/path.go      | 21 +++++++++++++++-
 fs/fspath/path_test.go | 56 ++++++++++++++++++++++--------------------
 3 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/backend/drive/drive.go b/backend/drive/drive.go
index 93fab7f8d3646..7bd0c41e53076 100644
--- a/backend/drive/drive.go
+++ b/backend/drive/drive.go
@@ -18,7 +18,6 @@ import (
 	"net/http"
 	"os"
 	"path"
-	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -3431,13 +3430,12 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		if err != nil {
 			return nil, err
 		}
-		re := regexp.MustCompile(`[^\w\p{L}\p{N}. -]+`)
 		if _, ok := opt["config"]; ok {
 			lines := []string{}
 			upstreams := []string{}
 			names := make(map[string]struct{}, len(drives))
 			for i, drive := range drives {
-				name := re.ReplaceAllString(drive.Name, "_")
+				name := fspath.MakeConfigName(drive.Name)
 				for {
 					if _, found := names[name]; !found {
 						break
diff --git a/fs/fspath/path.go b/fs/fspath/path.go
index 2c9c42b00f512..5a6775207f610 100644
--- a/fs/fspath/path.go
+++ b/fs/fspath/path.go
@@ -13,7 +13,8 @@ import (
 )
 
 const (
-	configNameRe = `[\w\p{L}\p{N}.]+(?:[ -]+[\w\p{L}\p{N}.-]+)*` // don't allow it to start with `-` as it complicates usage (#4261)
+	configNameRe              = `[\w\p{L}\p{N}.]+(?:[ -]+[\w\p{L}\p{N}.-]+)*` // May contain Unicode numbers and letters, as well as `_`, `-`, `.` and space, but not start with `-` (it complicates usage, see #4261) or space, and not end with space
+	illegalPartOfConfigNameRe = `^[ -]+|[^\w\p{L}\p{N}. -]+|[ ]+$`
 )
 
 var (
@@ -32,6 +33,9 @@ var (
 	// configNameMatcher is a pattern to match an rclone config name
 	configNameMatcher = regexp.MustCompile(`^` + configNameRe + `$`)
 
+	// illegalPartOfConfigNameMatcher is a pattern to match a sequence of characters not allowed in an rclone config name
+	illegalPartOfConfigNameMatcher = regexp.MustCompile(illegalPartOfConfigNameRe)
+
 	// remoteNameMatcher is a pattern to match an rclone remote name at the start of a config
 	remoteNameMatcher = regexp.MustCompile(`^:?` + configNameRe + `(?::$|,)`)
 )
@@ -44,6 +48,21 @@ func CheckConfigName(configName string) error {
 	return nil
 }
 
+// MakeConfigName makes an input into something legal to be used as a config name.
+// Returns a string where any sequences of illegal characters are replaced with
+// a single underscore. If the input is already valid as a config name, it is
+// returned unchanged. If the input is an empty string, a single underscore is
+// returned.
+func MakeConfigName(name string) string {
+	if name == "" {
+		return "_"
+	}
+	if configNameMatcher.MatchString(name) {
+		return name
+	}
+	return illegalPartOfConfigNameMatcher.ReplaceAllString(name, "_")
+}
+
 // checkRemoteName returns an error if remoteName is invalid
 func checkRemoteName(remoteName string) error {
 	if remoteName == ":" || remoteName == "::" {
diff --git a/fs/fspath/path_test.go b/fs/fspath/path_test.go
index ad58ef3993d93..d1828c1026e64 100644
--- a/fs/fspath/path_test.go
+++ b/fs/fspath/path_test.go
@@ -19,34 +19,38 @@ var (
 
 func TestCheckConfigName(t *testing.T) {
 	for _, test := range []struct {
-		in   string
-		want error
+		in      string
+		problem error
+		fixed   string
 	}{
-		{"remote", nil},
-		{"REMOTE", nil},
-		{"", errInvalidCharacters},
-		{":remote:", errInvalidCharacters},
-		{"remote:", errInvalidCharacters},
-		{"rem:ote", errInvalidCharacters},
-		{"rem/ote", errInvalidCharacters},
-		{"rem\\ote", errInvalidCharacters},
-		{"[remote", errInvalidCharacters},
-		{"*", errInvalidCharacters},
-		{"-remote", errInvalidCharacters},
-		{"r-emote-", nil},
-		{"_rem_ote_", nil},
-		{".", nil},
-		{"..", nil},
-		{".r.e.m.o.t.e.", nil},
-		{"rem ote", nil},
-		{"blåbær", nil},
-		{"chữ Quốc ngữ", nil},
-		{"remote ", errInvalidCharacters},
-		{" remote", errInvalidCharacters},
-		{" remote ", errInvalidCharacters},
+		{"remote", nil, "remote"},
+		{"REMOTE", nil, "REMOTE"},
+		{"", errInvalidCharacters, "_"},
+		{":remote:", errInvalidCharacters, "_remote_"},
+		{"remote:", errInvalidCharacters, "remote_"},
+		{"rem:ote", errInvalidCharacters, "rem_ote"},
+		{"rem/ote", errInvalidCharacters, "rem_ote"},
+		{"rem\\ote", errInvalidCharacters, "rem_ote"},
+		{"[remote", errInvalidCharacters, "_remote"},
+		{"*", errInvalidCharacters, "_"},
+		{"-remote", errInvalidCharacters, "_remote"},
+		{"r-emote-", nil, "r-emote-"},
+		{"---rem:::ote???", errInvalidCharacters, "_rem_ote_"},
+		{"_rem_ote_", nil, "_rem_ote_"},
+		{".", nil, "."},
+		{"..", nil, ".."},
+		{".r.e.m.o.t.e.", nil, ".r.e.m.o.t.e."},
+		{"rem ote", nil, "rem ote"},
+		{"blåbær", nil, "blåbær"},
+		{"chữ Quốc ngữ", nil, "chữ Quốc ngữ"},
+		{"remote ", errInvalidCharacters, "remote_"},
+		{" remote", errInvalidCharacters, "_remote"},
+		{" remote ", errInvalidCharacters, "_remote_"},
 	} {
-		got := CheckConfigName(test.in)
-		assert.Equal(t, test.want, got, test.in)
+		problem := CheckConfigName(test.in)
+		assert.Equal(t, test.problem, problem, test.in)
+		fixed := MakeConfigName(test.in)
+		assert.Equal(t, test.fixed, fixed, test.in)
 	}
 }
 

From 6c407dbe15f933f30df0f94daad0799e14e09c71 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 13 Dec 2022 12:19:00 +0000
Subject: [PATCH 450/560] s3: fix detection of listing routines which don't
 support v2 properly

In this commit

ab849b3613fe4935 s3: fix listing loop when using v2 listing on v1 server

The ContinuationToken was tested for existence, but it is the
NextContinuationToken that we are interested in.

See: #6600
---
 backend/s3/s3.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 36b6a9e613ec0..9a255689e664a 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -3235,8 +3235,8 @@ func (ls *v2List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versi
 	if err != nil {
 		return nil, nil, err
 	}
-	if aws.BoolValue(resp.IsTruncated) && (resp.ContinuationToken == nil || *resp.ContinuationToken == "") {
-		return nil, nil, errors.New("s3 protocol error: received listing v2 with IsTruncated set and no ContinuationToken. Should you be using `--s3-list-version 1`?")
+	if aws.BoolValue(resp.IsTruncated) && (resp.NextContinuationToken == nil || *resp.NextContinuationToken == "") {
+		return nil, nil, errors.New("s3 protocol error: received listing v2 with IsTruncated set and no NextContinuationToken. Should you be using `--s3-list-version 1`?")
 	}
 	ls.req.ContinuationToken = resp.NextContinuationToken
 	return resp, nil, nil

From c446651be80a42ab13333528df710a275ecdd76a Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 13 Dec 2022 12:21:12 +0000
Subject: [PATCH 451/560] Revert "s3: turn off list v2 support for Alibaba OSS
 since it does not work"

This reverts commit 4f386a1ccd4a773ffa09d9ad3e729c4b36520ccc.

It turns out that Alibaba OSS does support list v2 and the detection
code was wrong.

This means that users of the gov version of Alibaba will have to add
`list_version 1` to their config files.

See #6600
---
 backend/s3/s3.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 9a255689e664a..286ebb2832446 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -2755,7 +2755,6 @@ func setQuirks(opt *Options) {
 		mightGzip = false // Never auto gzips objects
 	case "Alibaba":
 		useMultipartEtag = false // Alibaba seems to calculate multipart Etags differently from AWS
-		listObjectsV2 = false    // See #6600
 	case "HuaweiOBS":
 		// Huawei OBS PFS is not support listObjectV2, and if turn on the urlEncodeListing, marker will not work and keep list same page forever.
 		urlEncodeListings = false

From 43bf177ff71d164c9877546c33b171379cda0bc9 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 13 Dec 2022 11:50:51 +0000
Subject: [PATCH 452/560] s3: fix excess memory usage when using versions

Before this change, we were taking the version ID straight from the
XML blob returned by the SDK and thus pinning the XML into memory
which bulked up the average memory per object from about 400 bytes to
4k.

Copying the string fixes the excess memory usage.
---
 backend/s3/s3.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 286ebb2832446..c943dc800acb4 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -3093,7 +3093,7 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje
 		o.setMD5FromEtag(aws.StringValue(info.ETag))
 		o.bytes = aws.Int64Value(info.Size)
 		o.storageClass = stringClonePointer(info.StorageClass)
-		o.versionID = versionID
+		o.versionID = stringClonePointer(versionID)
 	} else if !o.fs.opt.NoHeadObject {
 		err := o.readMetaData(ctx) // reads info and meta, returning an error
 		if err != nil {

From 3d291da0f69378768eb51e0f7f41d655106e0009 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 13 Dec 2022 10:45:40 +0000
Subject: [PATCH 453/560] azureblob: fix directory marker detection after SDK
 upgrade

When the SDK was upgraded it started delivering metadata where the
keys were not in lower case as per the old SDK.

Rclone normalises the case of the keys for storage in the Object, but
the directory marker check was being done with the unnormalised keys
as it needs to be done before the Object is created.

This fixes the directory marker check to do a case insensitive compare
of the metadata keys.
---
 backend/azureblob/azureblob.go | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index dae8405a0060b..80050a991ff5f 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -988,13 +988,18 @@ func (o *Object) updateMetadataWithModTime(modTime time.Time) {
 func isDirectoryMarker(size int64, metadata map[string]string, remote string) bool {
 	// Directory markers are 0 length
 	if size == 0 {
-		// Note that metadata with hdi_isfolder = true seems to be a
-		// defacto standard for marking blobs as directories.
 		endsWithSlash := strings.HasSuffix(remote, "/")
-		if endsWithSlash || remote == "" || metadata["hdi_isfolder"] == "true" {
+		if endsWithSlash || remote == "" {
 			return true
 		}
-
+		// Note that metadata with hdi_isfolder = true seems to be a
+		// defacto standard for marking blobs as directories.
+		// Note also that the metadata hasn't been normalised to lower case yet
+		for k, v := range metadata {
+			if strings.EqualFold(k, "hdi_isfolder") && v == "true" {
+				return true
+			}
+		}
 	}
 	return false
 }
@@ -1007,13 +1012,18 @@ func isDirectoryMarker(size int64, metadata map[string]string, remote string) bo
 func isDirectoryMarkerP(size int64, metadata map[string]*string, remote string) bool {
 	// Directory markers are 0 length
 	if size == 0 {
-		// Note that metadata with hdi_isfolder = true seems to be a
-		// defacto standard for marking blobs as directories.
 		endsWithSlash := strings.HasSuffix(remote, "/")
-		if endsWithSlash || remote == "" || (metadata["hdi_isfolder"] != nil && *metadata["hdi_isfolder"] == "true") {
+		if endsWithSlash || remote == "" {
 			return true
 		}
-
+		// Note that metadata with hdi_isfolder = true seems to be a
+		// defacto standard for marking blobs as directories.
+		// Note also that the metadata hasn't been normalised to lower case yet
+		for k, pv := range metadata {
+			if strings.EqualFold(k, "hdi_isfolder") && pv != nil && *pv == "true" {
+				return true
+			}
+		}
 	}
 	return false
 }

From 9a81885b51ad00ce3327eabfe07b4e820fff4d76 Mon Sep 17 00:00:00 2001
From: Jack <16779171+jkpe@users.noreply.github.com>
Date: Wed, 14 Dec 2022 14:00:04 +0000
Subject: [PATCH 454/560] s3: add DigitalOcean Spaces regions sfo3, fra1, syd1

---
 backend/s3/s3.go | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index c943dc800acb4..8d4a4c5f0c1d6 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -66,7 +66,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
+		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
@@ -105,7 +105,7 @@ func init() {
 				Help:  "Arvan Cloud Object Storage (AOS)",
 			}, {
 				Value: "DigitalOcean",
-				Help:  "Digital Ocean Spaces",
+				Help:  "DigitalOcean Spaces",
 			}, {
 				Value: "Dreamhost",
 				Help:  "Dreamhost DreamObjects",
@@ -1109,17 +1109,29 @@ func init() {
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
 				Provider: "Dreamhost",
+			}, {
+				Value:    "syd1.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces Sydney 1",
+				Provider: "DigitalOcean",
+			}, {
+				Value:    "sfo3.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces San Francisco 3",
+				Provider: "DigitalOcean",
+			}, {
+				Value:    "fra1.digitaloceanspaces.com",
+				Help:     "DigitalOcean Spaces Frankfurt 1",
+				Provider: "DigitalOcean",
 			}, {
 				Value:    "nyc3.digitaloceanspaces.com",
-				Help:     "Digital Ocean Spaces New York 3",
+				Help:     "DigitalOcean Spaces New York 3",
 				Provider: "DigitalOcean",
 			}, {
 				Value:    "ams3.digitaloceanspaces.com",
-				Help:     "Digital Ocean Spaces Amsterdam 3",
+				Help:     "DigitalOcean Spaces Amsterdam 3",
 				Provider: "DigitalOcean",
 			}, {
 				Value:    "sgp1.digitaloceanspaces.com",
-				Help:     "Digital Ocean Spaces Singapore 1",
+				Help:     "DigitalOcean Spaces Singapore 1",
 				Provider: "DigitalOcean",
 			}, {
 				Value:    "localhost:8333",

From c6ad15e3b8c171a16372450ef2d87dc9e397ddd8 Mon Sep 17 00:00:00 2001
From: Jack <16779171+jkpe@users.noreply.github.com>
Date: Wed, 14 Dec 2022 14:12:58 +0000
Subject: [PATCH 455/560] s3: make DigitalOcean name canonical

---
 docs/content/s3.md | 36 +++++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/docs/content/s3.md b/docs/content/s3.md
index f58f67998aafd..34e34230fdb07 100644
--- a/docs/content/s3.md
+++ b/docs/content/s3.md
@@ -88,7 +88,7 @@ Choose a number from below, or type in your own value
    \ "AWS"
  2 / Ceph Object Storage
    \ "Ceph"
- 3 / Digital Ocean Spaces
+ 3 / DigitalOcean Spaces
    \ "DigitalOcean"
  4 / Dreamhost DreamObjects
    \ "Dreamhost"
@@ -643,7 +643,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t
 {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}}
 ### Standard options
 
-Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
+Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
 
 #### --s3-provider
 
@@ -669,7 +669,7 @@ Properties:
     - "ArvanCloud"
         - Arvan Cloud Object Storage (AOS)
     - "DigitalOcean"
-        - Digital Ocean Spaces
+        - DigitalOcean Spaces
     - "Dreamhost"
         - Dreamhost DreamObjects
     - "HuaweiOBS"
@@ -1618,12 +1618,18 @@ Properties:
 - Examples:
     - "objects-us-east-1.dream.io"
         - Dream Objects endpoint
+    - "syd1.digitaloceanspaces.com"
+        - DigitalOcean Spaces Sydney 1
+    - "sfo3.digitaloceanspaces.com"
+        - DigitalOcean Spaces San Francisco 3
+    - "fra1.digitaloceanspaces.com"
+        - DigitalOcean Spaces Frankfurt 1
     - "nyc3.digitaloceanspaces.com"
-        - Digital Ocean Spaces New York 3
+        - DigitalOcean Spaces New York 3
     - "ams3.digitaloceanspaces.com"
-        - Digital Ocean Spaces Amsterdam 3
+        - DigitalOcean Spaces Amsterdam 3
     - "sgp1.digitaloceanspaces.com"
-        - Digital Ocean Spaces Singapore 1
+        - DigitalOcean Spaces Singapore 1
     - "localhost:8333"
         - SeaweedFS S3 localhost
     - "s3.us-east-1.lyvecloud.seagate.com"
@@ -3192,7 +3198,7 @@ Option Storage.
 Type of storage to configure.
 Choose a number from below, or type in your own value.
 ...
-XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
    \ (s3)
 ...
 Storage> s3
@@ -3361,7 +3367,7 @@ Option Storage.
 Type of storage to configure.
 Choose a number from below, or type in your own value.
 [snip]
- 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
    \ (s3)
 [snip]
 Storage> 5
@@ -3656,7 +3662,7 @@ Option Storage.
 Type of storage to configure.
 Choose a number from below, or type in your own value.
 [snip]
-XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
    \ (s3)
 [snip]
 Storage> s3
@@ -3762,7 +3768,7 @@ Option Storage.
 Type of storage to configure.
 Choose a number from below, or type in your own value.
 [snip]
-XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
    \ (s3)
 [snip]
 Storage> s3
@@ -4008,7 +4014,7 @@ Choose a number from below, or type in your own value
    \ (alias)
  4 / Amazon Drive
    \ (amazon cloud drive)
- 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi
    \ (s3)
 [snip]
 Storage> s3
@@ -4248,7 +4254,7 @@ Choose `s3` backend
 Type of storage to configure.
 Choose a number from below, or type in your own value.
 [snip]
-XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Liara, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
    \ (s3)
 [snip]
 Storage> s3
@@ -4549,7 +4555,7 @@ Type of storage to configure.
 Enter a string value. Press Enter for the default ("").
 Choose a number from below, or type in your own value
 [snip]
- 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS
+ 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS
    \ "s3"
 [snip]
 Storage> s3
@@ -4659,7 +4665,7 @@ Option Storage.
 Type of storage to configure.
 Choose a number from below, or type in your own value.
  ...
- 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
    \ (s3)
  ...
 Storage> s3
@@ -5131,7 +5137,7 @@ Choose a number from below, or type in your own value
    \ "alias"
  3 / Amazon Drive
    \ "amazon cloud drive"
- 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS
+ 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS
    \ "s3"
 [snip]
 Storage> s3

From a171497a8b636b38668c798039abb42017b6f2bf Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 15 Dec 2022 11:34:03 +0000
Subject: [PATCH 456/560] Add Jack to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index 995c45f2d8b79..aac3fb694ca6d 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -671,3 +671,4 @@ put them back in again.` >}}
   * Kevin Verstaen <48050031+kverstae@users.noreply.github.com>
   * MohammadReza 
   * vanplus <60313789+vanplus@users.noreply.github.com>
+  * Jack <16779171+jkpe@users.noreply.github.com>

From ec3cee89d31ca21c7377cac384cf13f5c9c1a494 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 15 Dec 2022 11:33:31 +0000
Subject: [PATCH 457/560] fstest: switch to port forwarding now Owncloud
 disallows wildcards

A recent security fix in the Owncloud container now causes it to
disallow wildcards in the OWNCLOUD_TRUSTED_DOMAINS setting.

This patch works around the problem by using port forwarding from the
host so we can keep the domain name constant.
---
 fstest/testserver/init.d/TestWebdavOwncloud | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/fstest/testserver/init.d/TestWebdavOwncloud b/fstest/testserver/init.d/TestWebdavOwncloud
index c12cb22596c59..41c40d9472bb3 100755
--- a/fstest/testserver/init.d/TestWebdavOwncloud
+++ b/fstest/testserver/init.d/TestWebdavOwncloud
@@ -5,6 +5,7 @@ set -e
 NAME=owncloud
 USER=rclone
 PASS=HarperGrayerFewest5
+PORT=38081
 
 . $(dirname "$0")/docker.bash
 
@@ -17,15 +18,16 @@ start() {
            -e "OWNCLOUD_ADMIN_PASSWORD=$PASS" \
            -e "OWNCLOUD_MYSQL_UTF8MB4=true" \
            -e "OWNCLOUD_REDIS_ENABLED=false" \
-           -e "OWNCLOUD_TRUSTED_DOMAINS=*.*.*.*" \
+           -e "OWNCLOUD_TRUSTED_DOMAINS=127.0.0.1" \
+           -p 127.0.0.1:${PORT}:8080 \
            owncloud/server
     
     echo type=webdav
-    echo url=http://$(docker_ip):8080/remote.php/webdav/
+    echo url=http://127.0.0.1:${PORT}/remote.php/webdav/
     echo user=$USER
     echo pass=$(rclone obscure $PASS)
     echo vendor=owncloud
-    echo _connect=$(docker_ip):8080
+    echo _connect=127.0.0.1:${PORT}
 }
 
 . $(dirname "$0")/run.bash

From 43506f80863a94b422fb6a54b260d37b671a76f5 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 15 Dec 2022 12:37:19 +0000
Subject: [PATCH 458/560] test memory: read metadata if -M flag is specified

---
 cmd/test/memory/memory.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/cmd/test/memory/memory.go b/cmd/test/memory/memory.go
index 4321108810778..058a5e8a5b736 100644
--- a/cmd/test/memory/memory.go
+++ b/cmd/test/memory/memory.go
@@ -29,6 +29,7 @@ var commandDefinition = &cobra.Command{
 		cmd.Run(false, false, command, func() error {
 			ctx := context.Background()
 			ci := fs.GetConfig(context.Background())
+			metadata := ci.Metadata && fsrc.Features().ReadMetadata
 			objects, _, _, err := operations.Count(ctx, fsrc)
 			if err != nil {
 				return err
@@ -39,6 +40,13 @@ var commandDefinition = &cobra.Command{
 			runtime.ReadMemStats(&before)
 			var mu sync.Mutex
 			err = operations.ListFn(ctx, fsrc, func(o fs.Object) {
+				// Read the metadata so it gets cached in the object
+				if metadata {
+					_, err := fs.GetMetadata(ctx, o)
+					if err != nil {
+						fs.Errorf(o, "Failed to read metadata: %v", err)
+					}
+				}
 				mu.Lock()
 				objs = append(objs, o)
 				mu.Unlock()

From dfd8ad2fffadf36b71560b119bfcac983f747864 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 12 Dec 2022 11:45:33 +0000
Subject: [PATCH 459/560] Add compiletest target to compile all the tests only

---
 Makefile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Makefile b/Makefile
index f921a27432a3e..7a9f4a660914a 100644
--- a/Makefile
+++ b/Makefile
@@ -81,6 +81,9 @@ quicktest:
 racequicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./...
 
+compiletest:
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -run XXX ./...
+
 # Do source code quality checks
 check:	rclone
 	@echo "-- START CODE QUALITY REPORT -------------------------------"

From 6d622672271fa638612aa82fec81e4889f9c09fb Mon Sep 17 00:00:00 2001
From: Tom Mombourquette 
Date: Tue, 8 Nov 2022 07:49:19 -0400
Subject: [PATCH 460/560] serve http: support unix sockets and multiple
 listners

- add support for unix sockets (which skip the auth).
- add support for multiple listeners
- collapse unnecessary internal structure of lib/http so it can all be
  imported together
- moves files in sub directories of lib/http into the main lib/http
  directory and reworks the code that uses them.

See: https://forum.rclone.org/t/wip-rc-rcd-over-unix-socket/33619
Fixes: #6605
---
 cmd/serve/http/http.go        | 103 ++++---
 cmd/serve/http/http_test.go   |  38 +--
 lib/http/auth.go              |  70 +++++
 lib/http/auth/auth.go         |  84 ------
 lib/http/auth/basic.go        | 120 --------
 lib/http/context.go           |  68 +++++
 lib/http/http.go              | 441 -----------------------------
 lib/http/http_test.go         | 515 ----------------------------------
 lib/http/middleware.go        | 171 +++++++++++
 lib/http/middleware_test.go   | 231 +++++++++++++++
 lib/http/serve/dir_test.go    |   6 +-
 lib/http/serve/serve_test.go  |  12 +-
 lib/http/server.go            | 430 ++++++++++++++++++++++++++++
 lib/http/server_test.go       | 444 +++++++++++++++++++++++++++++
 lib/http/template.go          |  95 +++++++
 lib/http/templates/index.html | 389 +++++++++++++++++++++++++
 lib/http/testdata/.htpasswd   |   3 +
 lib/http/testdata/local.crt   |  32 +++
 lib/http/testdata/local.key   |  52 ++++
 19 files changed, 2079 insertions(+), 1225 deletions(-)
 create mode 100644 lib/http/auth.go
 delete mode 100644 lib/http/auth/auth.go
 delete mode 100644 lib/http/auth/basic.go
 create mode 100644 lib/http/context.go
 delete mode 100644 lib/http/http.go
 delete mode 100644 lib/http/http_test.go
 create mode 100644 lib/http/middleware.go
 create mode 100644 lib/http/middleware_test.go
 create mode 100644 lib/http/server.go
 create mode 100644 lib/http/server_test.go
 create mode 100644 lib/http/template.go
 create mode 100644 lib/http/templates/index.html
 create mode 100644 lib/http/testdata/.htpasswd
 create mode 100644 lib/http/testdata/local.crt
 create mode 100644 lib/http/testdata/local.key

diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go
index 5db094dce0fd1..594ac551c8bb3 100644
--- a/cmd/serve/http/http.go
+++ b/cmd/serve/http/http.go
@@ -2,7 +2,8 @@
 package http
 
 import (
-	"html/template"
+	"context"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
@@ -10,16 +11,15 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
-	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/rclone/rclone/cmd"
-	"github.com/rclone/rclone/cmd/serve/http/data"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
-	httplib "github.com/rclone/rclone/lib/http"
-	"github.com/rclone/rclone/lib/http/auth"
+	"github.com/rclone/rclone/lib/atexit"
+	libhttp "github.com/rclone/rclone/lib/http"
 	"github.com/rclone/rclone/lib/http/serve"
 	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfsflags"
@@ -28,20 +28,27 @@ import (
 
 // Options required for http server
 type Options struct {
-	data.Options
+	Auth     libhttp.AuthConfig
+	HTTP     libhttp.HTTPConfig
+	Template libhttp.TemplateConfig
 }
 
 // DefaultOpt is the default values used for Options
-var DefaultOpt = Options{}
+var DefaultOpt = Options{
+	Auth:     libhttp.DefaultAuthCfg(),
+	HTTP:     libhttp.DefaultHTTPCfg(),
+	Template: libhttp.DefaultTemplateCfg(),
+}
 
 // Opt is options set by command line flags
 var Opt = DefaultOpt
 
 func init() {
-	data.AddFlags(Command.Flags(), "", &Opt.Options)
-	httplib.AddFlags(Command.Flags())
-	auth.AddFlags(Command.Flags())
-	vfsflags.AddFlags(Command.Flags())
+	flagSet := Command.Flags()
+	libhttp.AddAuthFlagsPrefix(flagSet, "", &Opt.Auth)
+	libhttp.AddHTTPFlagsPrefix(flagSet, "", &Opt.HTTP)
+	libhttp.AddTemplateFlagsPrefix(flagSet, "", &Opt.Template)
+	vfsflags.AddFlags(flagSet)
 }
 
 // Command definition for cobra
@@ -59,60 +66,78 @@ The server will log errors.  Use ` + "`-v`" + ` to see access logs.
 
 ` + "`--bwlimit`" + ` will be respected for file transfers.  Use ` + "`--stats`" + ` to
 control the stats printing.
-` + httplib.Help + data.Help + auth.Help + vfs.Help,
+` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp + vfs.Help,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.39",
 	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		f := cmd.NewFsSrc(args)
+
 		cmd.Run(false, true, command, func() error {
-			s := newServer(f, Opt.Template)
-			router, err := httplib.Router()
+			ctx := context.Background()
+
+			s, err := run(ctx, f, Opt)
 			if err != nil {
-				return err
+				log.Fatal(err)
 			}
-			s.Bind(router)
-			httplib.Wait()
+
+			var finaliseOnce sync.Once
+			finalise := func() {
+				finaliseOnce.Do(func() {
+					if err := s.server.Shutdown(); err != nil {
+						log.Printf("error shutting down server: %v", err)
+					}
+				})
+			}
+			fnHandle := atexit.Register(finalise)
+			defer atexit.Unregister(fnHandle)
+
+			s.server.Wait()
 			return nil
 		})
 	},
 }
 
 // server contains everything to run the server
-type server struct {
-	f            fs.Fs
-	vfs          *vfs.VFS
-	HTMLTemplate *template.Template // HTML template for web interface
+type serveCmd struct {
+	f      fs.Fs
+	vfs    *vfs.VFS
+	server libhttp.Server
 }
 
-func newServer(f fs.Fs, templatePath string) *server {
-	htmlTemplate, templateErr := data.GetTemplate(templatePath)
-	if templateErr != nil {
-		log.Fatalf(templateErr.Error())
-	}
-	s := &server{
-		f:            f,
-		vfs:          vfs.New(f, &vfsflags.Opt),
-		HTMLTemplate: htmlTemplate,
+func run(ctx context.Context, f fs.Fs, opt Options) (*serveCmd, error) {
+	var err error
+
+	s := &serveCmd{
+		f:   f,
+		vfs: vfs.New(f, &vfsflags.Opt),
 	}
-	return s
-}
 
-func (s *server) Bind(router chi.Router) {
-	if m := auth.Auth(auth.Opt); m != nil {
-		router.Use(m)
+	s.server, err = libhttp.NewServer(ctx,
+		libhttp.WithConfig(opt.HTTP),
+		libhttp.WithAuth(opt.Auth),
+		libhttp.WithTemplate(opt.Template),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init server: %w", err)
 	}
+
+	router := s.server.Router()
 	router.Use(
 		middleware.SetHeader("Accept-Ranges", "bytes"),
 		middleware.SetHeader("Server", "rclone/"+fs.Version),
 	)
 	router.Get("/*", s.handler)
 	router.Head("/*", s.handler)
+
+	s.server.Serve()
+
+	return s, nil
 }
 
 // handler reads incoming requests and dispatches them
-func (s *server) handler(w http.ResponseWriter, r *http.Request) {
+func (s *serveCmd) handler(w http.ResponseWriter, r *http.Request) {
 	isDir := strings.HasSuffix(r.URL.Path, "/")
 	remote := strings.Trim(r.URL.Path, "/")
 	if isDir {
@@ -123,7 +148,7 @@ func (s *server) handler(w http.ResponseWriter, r *http.Request) {
 }
 
 // serveDir serves a directory index at dirRemote
-func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
+func (s *serveCmd) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
 	// List the directory
 	node, err := s.vfs.Stat(dirRemote)
 	if err == vfs.ENOENT {
@@ -145,7 +170,7 @@ func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stri
 	}
 
 	// Make the entries for display
-	directory := serve.NewDirectory(dirRemote, s.HTMLTemplate)
+	directory := serve.NewDirectory(dirRemote, s.server.HTMLTemplate())
 	for _, node := range dirEntries {
 		if vfsflags.Opt.NoModTime {
 			directory.AddHTMLEntry(node.Path(), node.IsDir(), node.Size(), time.Time{})
@@ -165,7 +190,7 @@ func (s *server) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stri
 }
 
 // serveFile serves a file object at remote
-func (s *server) serveFile(w http.ResponseWriter, r *http.Request, remote string) {
+func (s *serveCmd) serveFile(w http.ResponseWriter, r *http.Request, remote string) {
 	node, err := s.vfs.Stat(remote)
 	if err == vfs.ENOENT {
 		fs.Infof(remote, "%s: File not found", r.RemoteAddr)
diff --git a/cmd/serve/http/http_test.go b/cmd/serve/http/http_test.go
index 0c77e4b61aeea..9f264adce9897 100644
--- a/cmd/serve/http/http_test.go
+++ b/cmd/serve/http/http_test.go
@@ -14,14 +14,14 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/filter"
-	httplib "github.com/rclone/rclone/lib/http"
+	libhttp "github.com/rclone/rclone/lib/http"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 var (
 	updateGolden = flag.Bool("updategolden", false, "update golden files for regression test")
-	httpServer   *server
+	sc           *serveCmd
 	testURL      string
 )
 
@@ -30,16 +30,25 @@ const (
 	testTemplate    = "testdata/golden/testindex.html"
 )
 
-func startServer(t *testing.T, f fs.Fs) {
-	opt := httplib.DefaultOpt
-	opt.ListenAddr = testBindAddress
-	httpServer = newServer(f, testTemplate)
-	router, err := httplib.Router()
-	if err != nil {
-		t.Fatal(err.Error())
+func start(t *testing.T, f fs.Fs) {
+	ctx := context.Background()
+
+	opts := Options{
+		HTTP: libhttp.DefaultHTTPCfg(),
+		Template: libhttp.TemplateConfig{
+			Path: testTemplate,
+		},
 	}
-	httpServer.Bind(router)
-	testURL = httplib.URL()
+	opts.HTTP.ListenAddr = []string{testBindAddress}
+
+	s, err := run(ctx, f, opts)
+	require.NoError(t, err, "failed to start server")
+	sc = s
+
+	urls := s.server.URLs()
+	require.Len(t, urls, 1, "expected one URL")
+
+	testURL = urls[0]
 
 	// try to connect to the test server
 	pause := time.Millisecond
@@ -54,7 +63,6 @@ func startServer(t *testing.T, f fs.Fs) {
 		pause *= 2
 	}
 	t.Fatal("couldn't connect to server")
-
 }
 
 var (
@@ -84,7 +92,7 @@ func TestInit(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, obj.SetModTime(context.Background(), expectedTime))
 
-	startServer(t, f)
+	start(t, f)
 }
 
 // check body against the file, or re-write body if -updategolden is
@@ -229,7 +237,3 @@ func TestGET(t *testing.T) {
 		checkGolden(t, test.Golden, body)
 	}
 }
-
-func TestFinalise(t *testing.T) {
-	_ = httplib.Shutdown()
-}
diff --git a/lib/http/auth.go b/lib/http/auth.go
new file mode 100644
index 0000000000000..b7784c4b44aa4
--- /dev/null
+++ b/lib/http/auth.go
@@ -0,0 +1,70 @@
+package http
+
+import (
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/spf13/pflag"
+)
+
+// Help contains text describing the http authentication to add to the command
+// help.
+var AuthHelp = `
+#### Authentication
+
+By default this will serve files without needing a login.
+
+You can either use an htpasswd file which can take lots of users, or
+set a single username and password with the ` + "`--user` and `--pass`" + ` flags.
+
+Use ` + "`--htpasswd /path/to/htpasswd`" + ` to provide an htpasswd file.  This is
+in standard apache format and supports MD5, SHA1 and BCrypt for basic
+authentication.  Bcrypt is recommended.
+
+To create an htpasswd file:
+
+    touch htpasswd
+    htpasswd -B htpasswd user
+    htpasswd -B htpasswd anotherUser
+
+The password file can be updated while rclone is running.
+
+Use ` + "`--realm`" + ` to set the authentication realm.
+
+Use ` + "`--salt`" + ` to change the password hashing salt from the default.
+`
+
+// CustomAuthFn if used will be used to authenticate user, pass. If an error
+// is returned then the user is not authenticated.
+//
+// If a non nil value is returned then it is added to the context under the key
+type CustomAuthFn func(user, pass string) (value interface{}, err error)
+
+// AuthConfig contains options for the http authentication
+type AuthConfig struct {
+	HtPasswd     string       // htpasswd file - if not provided no authentication is done
+	Realm        string       // realm for authentication
+	BasicUser    string       // single username for basic auth if not using Htpasswd
+	BasicPass    string       // password for BasicUser
+	Salt         string       // password hashing salt
+	CustomAuthFn CustomAuthFn `json:"-"` // custom Auth (not set by command line flags)
+}
+
+// AddFlagsPrefix adds flags to the flag set for AuthConfig
+func (cfg *AuthConfig) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) {
+	flags.StringVarP(flagSet, &cfg.HtPasswd, prefix+"htpasswd", "", cfg.HtPasswd, "A htpasswd file - if not provided no authentication is done")
+	flags.StringVarP(flagSet, &cfg.Realm, prefix+"realm", "", cfg.Realm, "Realm for authentication")
+	flags.StringVarP(flagSet, &cfg.BasicUser, prefix+"user", "", cfg.BasicUser, "User name for authentication")
+	flags.StringVarP(flagSet, &cfg.BasicPass, prefix+"pass", "", cfg.BasicPass, "Password for authentication")
+	flags.StringVarP(flagSet, &cfg.Salt, prefix+"salt", "", cfg.Salt, "Password hashing salt")
+}
+
+// AddAuthFlagsPrefix adds flags to the flag set for AuthConfig
+func AddAuthFlagsPrefix(flagSet *pflag.FlagSet, prefix string, cfg *AuthConfig) {
+	cfg.AddFlagsPrefix(flagSet, prefix)
+}
+
+// DefaultAuthCfg returns a new config which can be customized by command line flags
+func DefaultAuthCfg() AuthConfig {
+	return AuthConfig{
+		Salt: "dlPL2MqE",
+	}
+}
diff --git a/lib/http/auth/auth.go b/lib/http/auth/auth.go
deleted file mode 100644
index db50c8a303148..0000000000000
--- a/lib/http/auth/auth.go
+++ /dev/null
@@ -1,84 +0,0 @@
-// Package auth provides authentication for http.
-package auth
-
-import (
-	"github.com/rclone/rclone/fs/config/flags"
-	"github.com/rclone/rclone/lib/http"
-	"github.com/spf13/pflag"
-)
-
-// Help contains text describing the http authentication to add to the command
-// help.
-var Help = `
-#### Authentication
-
-By default this will serve files without needing a login.
-
-You can either use an htpasswd file which can take lots of users, or
-set a single username and password with the ` + "`--user` and `--pass`" + ` flags.
-
-Use ` + "`--htpasswd /path/to/htpasswd`" + ` to provide an htpasswd file.  This is
-in standard apache format and supports MD5, SHA1 and BCrypt for basic
-authentication.  Bcrypt is recommended.
-
-To create an htpasswd file:
-
-    touch htpasswd
-    htpasswd -B htpasswd user
-    htpasswd -B htpasswd anotherUser
-
-The password file can be updated while rclone is running.
-
-Use ` + "`--realm`" + ` to set the authentication realm.
-
-Use ` + "`--salt`" + ` to change the password hashing salt from the default.
-`
-
-// CustomAuthFn if used will be used to authenticate user, pass. If an error
-// is returned then the user is not authenticated.
-//
-// If a non nil value is returned then it is added to the context under the key
-type CustomAuthFn func(user, pass string) (value interface{}, err error)
-
-// Options contains options for the http authentication
-type Options struct {
-	HtPasswd  string       // htpasswd file - if not provided no authentication is done
-	Realm     string       // realm for authentication
-	BasicUser string       // single username for basic auth if not using Htpasswd
-	BasicPass string       // password for BasicUser
-	Salt      string       // password hashing salt
-	Auth      CustomAuthFn `json:"-"` // custom Auth (not set by command line flags)
-}
-
-// Auth instantiates middleware that authenticates users based on the configuration
-func Auth(opt Options) http.Middleware {
-	if opt.Auth != nil {
-		return CustomAuth(opt.Auth, opt.Realm)
-	} else if opt.HtPasswd != "" {
-		return HtPasswdAuth(opt.HtPasswd, opt.Realm)
-	} else if opt.BasicUser != "" {
-		return SingleAuth(opt.BasicUser, opt.BasicPass, opt.Realm, opt.Salt)
-	}
-	return nil
-}
-
-// Options set by command line flags
-var (
-	Opt = Options{
-		Salt: "dlPL2MqE",
-	}
-)
-
-// AddFlagsPrefix adds flags for http/auth
-func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *Options) {
-	flags.StringVarP(flagSet, &Opt.HtPasswd, prefix+"htpasswd", "", Opt.HtPasswd, "A htpasswd file - if not provided no authentication is done")
-	flags.StringVarP(flagSet, &Opt.Realm, prefix+"realm", "", Opt.Realm, "Realm for authentication")
-	flags.StringVarP(flagSet, &Opt.BasicUser, prefix+"user", "", Opt.BasicUser, "User name for authentication")
-	flags.StringVarP(flagSet, &Opt.BasicPass, prefix+"pass", "", Opt.BasicPass, "Password for authentication")
-	flags.StringVarP(flagSet, &Opt.Salt, prefix+"salt", "", Opt.Salt, "Password hashing salt")
-}
-
-// AddFlags adds flags for the http/auth
-func AddFlags(flagSet *pflag.FlagSet) {
-	AddFlagsPrefix(flagSet, "", &Opt)
-}
diff --git a/lib/http/auth/basic.go b/lib/http/auth/basic.go
deleted file mode 100644
index 69907c009371e..0000000000000
--- a/lib/http/auth/basic.go
+++ /dev/null
@@ -1,120 +0,0 @@
-package auth
-
-import (
-	"context"
-	"encoding/base64"
-	"net/http"
-	"strings"
-
-	auth "github.com/abbot/go-http-auth"
-	"github.com/rclone/rclone/fs"
-	httplib "github.com/rclone/rclone/lib/http"
-)
-
-// parseAuthorization parses the Authorization header into user, pass
-// it returns a boolean as to whether the parse was successful
-func parseAuthorization(r *http.Request) (user, pass string, ok bool) {
-	authHeader := r.Header.Get("Authorization")
-	if authHeader != "" {
-		s := strings.SplitN(authHeader, " ", 2)
-		if len(s) == 2 && s[0] == "Basic" {
-			b, err := base64.StdEncoding.DecodeString(s[1])
-			if err == nil {
-				parts := strings.SplitN(string(b), ":", 2)
-				user = parts[0]
-				if len(parts) > 1 {
-					pass = parts[1]
-					ok = true
-				}
-			}
-		}
-	}
-	return
-}
-
-type contextUserType struct{}
-
-// ContextUserKey is a simple context key for storing the username of the request
-var ContextUserKey = &contextUserType{}
-
-type contextAuthType struct{}
-
-// ContextAuthKey is a simple context key for storing info returned by CustomAuthFn
-var ContextAuthKey = &contextAuthType{}
-
-// LoggedBasicAuth extends BasicAuth to include access logging
-type LoggedBasicAuth struct {
-	auth.BasicAuth
-}
-
-// CheckAuth extends BasicAuth.CheckAuth to emit a log entry for unauthorised requests
-func (a *LoggedBasicAuth) CheckAuth(r *http.Request) string {
-	username := a.BasicAuth.CheckAuth(r)
-	if username == "" {
-		user, _, _ := parseAuthorization(r)
-		fs.Infof(r.URL.Path, "%s: Unauthorized request from %s", r.RemoteAddr, user)
-	}
-	return username
-}
-
-// NewLoggedBasicAuthenticator instantiates a new instance of LoggedBasicAuthenticator
-func NewLoggedBasicAuthenticator(realm string, secrets auth.SecretProvider) *LoggedBasicAuth {
-	return &LoggedBasicAuth{BasicAuth: auth.BasicAuth{Realm: realm, Secrets: secrets}}
-}
-
-// Helper to generate required interface for middleware
-func basicAuth(authenticator *LoggedBasicAuth) httplib.Middleware {
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			if username := authenticator.CheckAuth(r); username == "" {
-				authenticator.RequireAuth(w, r)
-			} else {
-				r = r.WithContext(context.WithValue(r.Context(), ContextUserKey, username))
-				next.ServeHTTP(w, r)
-			}
-		})
-	}
-}
-
-// HtPasswdAuth instantiates middleware that authenticates against the passed htpasswd file
-func HtPasswdAuth(path, realm string) httplib.Middleware {
-	fs.Infof(nil, "Using %q as htpasswd storage", path)
-	secretProvider := auth.HtpasswdFileProvider(path)
-	authenticator := NewLoggedBasicAuthenticator(realm, secretProvider)
-	return basicAuth(authenticator)
-}
-
-// SingleAuth instantiates middleware that authenticates for a single user
-func SingleAuth(user, pass, realm, salt string) httplib.Middleware {
-	fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", user)
-	pass = string(auth.MD5Crypt([]byte(pass), []byte(salt), []byte("$1$")))
-	secretProvider := func(u, r string) string {
-		if user == u {
-			return pass
-		}
-		return ""
-	}
-	authenticator := NewLoggedBasicAuthenticator(realm, secretProvider)
-	return basicAuth(authenticator)
-}
-
-// CustomAuth instantiates middleware that authenticates using a custom function
-func CustomAuth(fn CustomAuthFn, realm string) httplib.Middleware {
-	return func(next http.Handler) http.Handler {
-		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			user, pass, ok := parseAuthorization(r)
-			if ok {
-				value, err := fn(user, pass)
-				if err != nil {
-					fs.Infof(r.URL.Path, "%s: Auth failed from %s: %v", r.RemoteAddr, user, err)
-					auth.NewBasicAuthenticator(realm, func(user, realm string) string { return "" }).RequireAuth(w, r) //Reuse BasicAuth error reporting
-					return
-				}
-				if value != nil {
-					r = r.WithContext(context.WithValue(r.Context(), ContextAuthKey, value))
-				}
-				next.ServeHTTP(w, r)
-			}
-		})
-	}
-}
diff --git a/lib/http/context.go b/lib/http/context.go
new file mode 100644
index 0000000000000..34bff8fa63e26
--- /dev/null
+++ b/lib/http/context.go
@@ -0,0 +1,68 @@
+package http
+
+import (
+	"context"
+	"net"
+	"net/http"
+)
+
+type ctxKey int
+
+const (
+	ctxKeyAuth ctxKey = iota
+	ctxKeyPublicURL
+	ctxKeyUnixSock
+	ctxKeyUser
+)
+
+// NewBaseContext initializes the context for all requests, adding info for use in middleware and handlers
+func NewBaseContext(ctx context.Context, url string) func(l net.Listener) context.Context {
+	return func(l net.Listener) context.Context {
+		if l.Addr().Network() == "unix" {
+			ctx = context.WithValue(ctx, ctxKeyUnixSock, true)
+			return ctx
+		}
+
+		ctx = context.WithValue(ctx, ctxKeyPublicURL, url)
+		return ctx
+	}
+}
+
+// IsAuthenticated checks if this request was authenticated via a middleware
+func IsAuthenticated(r *http.Request) bool {
+	if v := r.Context().Value(ctxKeyAuth); v != nil {
+		return true
+	}
+	if v := r.Context().Value(ctxKeyUser); v != nil {
+		return true
+	}
+	return false
+}
+
+// IsUnixSocket checks if the request was received on a unix socket, used to skip auth & CORS
+func IsUnixSocket(r *http.Request) bool {
+	v, _ := r.Context().Value(ctxKeyUnixSock).(bool)
+	return v
+}
+
+// PublicURL returns the URL defined in NewBaseContext, used for logging & CORS
+func PublicURL(r *http.Request) string {
+	v, _ := r.Context().Value(ctxKeyPublicURL).(string)
+	return v
+}
+
+// CtxGetAuth is a wrapper over the private Auth context key
+func CtxGetAuth(ctx context.Context) interface{} {
+	return ctx.Value(ctxKeyAuth)
+}
+
+// CtxGetUser is a wrapper over the private User context key
+func CtxGetUser(ctx context.Context) (string, bool) {
+	v, ok := ctx.Value(ctxKeyUser).(string)
+	return v, ok
+}
+
+// CtxSetUser is a test helper that injects a User value into context
+func CtxSetUser(ctx context.Context, value string) context.Context {
+	return context.WithValue(ctx, ctxKeyUser, value)
+}
diff --git a/lib/http/http.go b/lib/http/http.go
deleted file mode 100644
index 0138b0fec8b79..0000000000000
--- a/lib/http/http.go
+++ /dev/null
@@ -1,441 +0,0 @@
-// Package http provides a registration interface for http services
-package http
-
-import (
-	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"errors"
-	"fmt"
-	"log"
-	"net"
-	"net/http"
-	"os"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/go-chi/chi/v5"
-	"github.com/rclone/rclone/fs/config/flags"
-	"github.com/spf13/pflag"
-)
-
-// Help contains text describing the http server to add to the command
-// help.
-var Help = `
-### Server options
-
-Use ` + "`--addr`" + ` to specify which IP address and port the server should
-listen on, eg ` + "`--addr 1.2.3.4:8000` or `--addr :8080`" + ` to listen to all
-IPs.  By default it only listens on localhost.  You can use port
-:0 to let the OS choose an available port.
-
-If you set ` + "`--addr`" + ` to listen on a public or LAN accessible IP address
-then using Authentication is advised - see the next section for info.
-
-` + "`--server-read-timeout` and `--server-write-timeout`" + ` can be used to
-control the timeouts on the server.  Note that this is the total time
-for a transfer.
-
-` + "`--max-header-bytes`" + ` controls the maximum number of bytes the server will
-accept in the HTTP header.
-
-` + "`--baseurl`" + ` controls the URL prefix that rclone serves from.  By default
-rclone will serve from the root.  If you used ` + "`--baseurl \"/rclone\"`" + ` then
-rclone would serve from a URL starting with "/rclone/".  This is
-useful if you wish to proxy rclone serve.  Rclone automatically
-inserts leading and trailing "/" on ` + "`--baseurl`" + `, so ` + "`--baseurl \"rclone\"`" + `,
-` + "`--baseurl \"/rclone\"` and `--baseurl \"/rclone/\"`" + ` are all treated
-identically.
-
-#### SSL/TLS
-
-By default this will serve over http.  If you want you can serve over
-https.  You will need to supply the ` + "`--cert` and `--key`" + ` flags.
-If you wish to do client side certificate validation then you will need to
-supply ` + "`--client-ca`" + ` also.
-
-` + "`--cert`" + ` should be a either a PEM encoded certificate or a concatenation
-of that with the CA certificate.  ` + "`--key`" + ` should be the PEM encoded
-private key and ` + "`--client-ca`" + ` should be the PEM encoded client
-certificate authority certificate.
-
---min-tls-version is minimum TLS version that is acceptable. Valid
-  values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
-  "tls1.0").
-`
-
-// Middleware function signature required by chi.Router.Use()
-type Middleware func(http.Handler) http.Handler
-
-// Options contains options for the http Server
-type Options struct {
-	ListenAddr         string        // Port to listen on
-	BaseURL            string        // prefix to strip from URLs
-	ServerReadTimeout  time.Duration // Timeout for server reading data
-	ServerWriteTimeout time.Duration // Timeout for server writing data
-	MaxHeaderBytes     int           // Maximum size of request header
-	SslCert            string        // Path to SSL PEM key (concatenation of certificate and CA certificate)
-	SslKey             string        // Path to SSL PEM Private key
-	SslCertBody        []byte        // SSL PEM key (concatenation of certificate and CA certificate) body, ignores SslCert
-	SslKeyBody         []byte        // SSL PEM Private key body, ignores SslKey
-	ClientCA           string        // Client certificate authority to verify clients with
-	MinTLSVersion      string        // MinTLSVersion contains the minimum TLS version that is acceptable.
-}
-
-// DefaultOpt is the default values used for Options
-var DefaultOpt = Options{
-	ListenAddr:         "127.0.0.1:8080",
-	ServerReadTimeout:  1 * time.Hour,
-	ServerWriteTimeout: 1 * time.Hour,
-	MaxHeaderBytes:     4096,
-	MinTLSVersion:      "tls1.0",
-}
-
-// Server interface of http server
-type Server interface {
-	Router() chi.Router
-	Route(pattern string, fn func(r chi.Router)) chi.Router
-	Mount(pattern string, h http.Handler)
-	Shutdown() error
-}
-
-type server struct {
-	addrs        []net.Addr
-	tlsAddrs     []net.Addr
-	listeners    []net.Listener
-	tlsListeners []net.Listener
-	httpServer   *http.Server
-	baseRouter   chi.Router
-	closing      *sync.WaitGroup
-	useSSL       bool
-}
-
-var (
-	defaultServer        *server
-	defaultServerOptions = DefaultOpt
-	defaultServerMutex   sync.Mutex
-)
-
-func useSSL(opt Options) bool {
-	return opt.SslKey != "" || len(opt.SslKeyBody) > 0
-}
-
-// NewServer instantiates a new http server using provided listeners and options
-// This function is provided if the default http server does not meet a services requirements and should not generally be used
-// A http server can listen using multiple listeners. For example, a listener for port 80, and a listener for port 443.
-// tlsListeners are ignored if opt.SslKey is not provided
-func NewServer(listeners, tlsListeners []net.Listener, opt Options) (Server, error) {
-	// Validate input
-	if len(listeners) == 0 && len(tlsListeners) == 0 {
-		return nil, errors.New("can't create server without listeners")
-	}
-
-	// Prepare TLS config
-	var tlsConfig *tls.Config
-
-	useSSL := useSSL(opt)
-	if (len(opt.SslCertBody) > 0) != (len(opt.SslKeyBody) > 0) {
-		err := errors.New("need both SslCertBody and SslKeyBody to use SSL")
-		log.Fatalf(err.Error())
-		return nil, err
-	}
-	if (opt.SslCert != "") != (opt.SslKey != "") {
-		err := errors.New("need both -cert and -key to use SSL")
-		log.Fatalf(err.Error())
-		return nil, err
-	}
-
-	if useSSL {
-		var cert tls.Certificate
-		var err error
-		if len(opt.SslCertBody) > 0 {
-			cert, err = tls.X509KeyPair(opt.SslCertBody, opt.SslKeyBody)
-		} else {
-			cert, err = tls.LoadX509KeyPair(opt.SslCert, opt.SslKey)
-		}
-		if err != nil {
-			log.Fatal(err)
-		}
-		var minTLSVersion uint16
-		switch opt.MinTLSVersion {
-		case "tls1.0":
-			minTLSVersion = tls.VersionTLS10
-		case "tls1.1":
-			minTLSVersion = tls.VersionTLS11
-		case "tls1.2":
-			minTLSVersion = tls.VersionTLS12
-		case "tls1.3":
-			minTLSVersion = tls.VersionTLS13
-		default:
-			err = errors.New("Invalid value for --min-tls-version")
-			log.Fatalf(err.Error())
-			return nil, err
-		}
-		tlsConfig = &tls.Config{
-			MinVersion:   minTLSVersion,
-			Certificates: []tls.Certificate{cert},
-		}
-	} else if len(listeners) == 0 && len(tlsListeners) != 0 {
-		return nil, errors.New("no SslKey or non-tlsListeners")
-	}
-
-	if opt.ClientCA != "" {
-		if !useSSL {
-			err := errors.New("can't use --client-ca without --cert and --key")
-			log.Fatalf(err.Error())
-			return nil, err
-		}
-		certpool := x509.NewCertPool()
-		pem, err := os.ReadFile(opt.ClientCA)
-		if err != nil {
-			log.Fatalf("Failed to read client certificate authority: %v", err)
-			return nil, err
-		}
-		if !certpool.AppendCertsFromPEM(pem) {
-			err := errors.New("can't parse client certificate authority")
-			log.Fatalf(err.Error())
-			return nil, err
-		}
-		tlsConfig.ClientCAs = certpool
-		tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
-	}
-
-	// Ignore passing "/" for BaseURL
-	opt.BaseURL = strings.Trim(opt.BaseURL, "/")
-	if opt.BaseURL != "" {
-		opt.BaseURL = "/" + opt.BaseURL
-	}
-
-	// Build base router
-	var router chi.Router = chi.NewRouter()
-	router.MethodNotAllowed(func(w http.ResponseWriter, _ *http.Request) {
-		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
-	})
-	router.NotFound(func(w http.ResponseWriter, _ *http.Request) {
-		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
-	})
-
-	handler := router.(http.Handler)
-	if opt.BaseURL != "" {
-		handler = http.StripPrefix(opt.BaseURL, handler)
-	}
-
-	// Serve on listeners
-	httpServer := &http.Server{
-		Handler:           handler,
-		ReadTimeout:       opt.ServerReadTimeout,
-		WriteTimeout:      opt.ServerWriteTimeout,
-		MaxHeaderBytes:    opt.MaxHeaderBytes,
-		ReadHeaderTimeout: 10 * time.Second, // time to send the headers
-		IdleTimeout:       60 * time.Second, // time to keep idle connections open
-		TLSConfig:         tlsConfig,
-	}
-
-	addrs, tlsAddrs := make([]net.Addr, len(listeners)), make([]net.Addr, len(tlsListeners))
-
-	wg := &sync.WaitGroup{}
-
-	for i, l := range listeners {
-		addrs[i] = l.Addr()
-	}
-
-	if useSSL {
-		for i, l := range tlsListeners {
-			tlsAddrs[i] = l.Addr()
-		}
-	}
-
-	return &server{addrs, tlsAddrs, listeners, tlsListeners, httpServer, router, wg, useSSL}, nil
-}
-
-func (s *server) Serve() {
-	serve := func(l net.Listener, tls bool) {
-		defer s.closing.Done()
-		var err error
-		if tls {
-			err = s.httpServer.ServeTLS(l, "", "")
-		} else {
-			err = s.httpServer.Serve(l)
-		}
-		if err != http.ErrServerClosed && err != nil {
-			log.Fatalf(err.Error())
-		}
-	}
-
-	s.closing.Add(len(s.listeners))
-	for _, l := range s.listeners {
-		go serve(l, false)
-	}
-
-	if s.useSSL {
-		s.closing.Add(len(s.tlsListeners))
-		for _, l := range s.tlsListeners {
-			go serve(l, true)
-		}
-	}
-}
-
-// Wait blocks while the server is serving requests
-func (s *server) Wait() {
-	s.closing.Wait()
-}
-
-// Router returns the server base router
-func (s *server) Router() chi.Router {
-	return s.baseRouter
-}
-
-// Route mounts a sub-Router along a `pattern` string.
-func (s *server) Route(pattern string, fn func(r chi.Router)) chi.Router {
-	return s.baseRouter.Route(pattern, fn)
-}
-
-// Mount attaches another http.Handler along ./pattern/*
-func (s *server) Mount(pattern string, h http.Handler) {
-	s.baseRouter.Mount(pattern, h)
-}
-
-// Shutdown gracefully shuts down the server
-func (s *server) Shutdown() error {
-	if err := s.httpServer.Shutdown(context.Background()); err != nil {
-		return err
-	}
-	s.closing.Wait()
-	return nil
-}
-
-//---- Default HTTP server convenience functions ----
-
-// Router returns the server base router
-func Router() (chi.Router, error) {
-	if err := start(); err != nil {
-		return nil, err
-	}
-	return defaultServer.baseRouter, nil
-}
-
-// Route mounts a sub-Router along a `pattern` string.
-func Route(pattern string, fn func(r chi.Router)) (chi.Router, error) {
-	if err := start(); err != nil {
-		return nil, err
-	}
-	return defaultServer.Route(pattern, fn), nil
-}
-
-// Mount attaches another http.Handler along ./pattern/*
-func Mount(pattern string, h http.Handler) error {
-	if err := start(); err != nil {
-		return err
-	}
-	defaultServer.Mount(pattern, h)
-	return nil
-}
-
-// Restart or start the default http server using the default options and no handlers
-func Restart() error {
-	if e := Shutdown(); e != nil {
-		return e
-	}
-
-	return start()
-}
-
-// Wait blocks while the default http server is serving requests
-func Wait() {
-	defaultServer.Wait()
-}
-
-// Start the default server
-func start() error {
-	defaultServerMutex.Lock()
-	defer defaultServerMutex.Unlock()
-
-	if defaultServer != nil {
-		// Server already started, do nothing
-		return nil
-	}
-
-	var err error
-	var l net.Listener
-	l, err = net.Listen("tcp", defaultServerOptions.ListenAddr)
-	if err != nil {
-		return err
-	}
-
-	var s Server
-	if useSSL(defaultServerOptions) {
-		s, err = NewServer([]net.Listener{}, []net.Listener{l}, defaultServerOptions)
-	} else {
-		s, err = NewServer([]net.Listener{l}, []net.Listener{}, defaultServerOptions)
-	}
-	if err != nil {
-		return err
-	}
-	defaultServer = s.(*server)
-	defaultServer.Serve()
-	return nil
-}
-
-// Shutdown gracefully shuts down the default http server
-func Shutdown() error {
-	defaultServerMutex.Lock()
-	defer defaultServerMutex.Unlock()
-	if defaultServer != nil {
-		s := defaultServer
-		defaultServer = nil
-		return s.Shutdown()
-	}
-	return nil
-}
-
-// GetOptions thread safe getter for the default server options
-func GetOptions() Options {
-	defaultServerMutex.Lock()
-	defer defaultServerMutex.Unlock()
-	return defaultServerOptions
-}
-
-// SetOptions thread safe setter for the default server options
-func SetOptions(opt Options) {
-	defaultServerMutex.Lock()
-	defer defaultServerMutex.Unlock()
-	defaultServerOptions = opt
-}
-
-//---- Utility functions ----
-
-// URL of default http server
-func URL() string {
-	if defaultServer == nil {
-		panic("Server not running")
-	}
-	for _, a := range defaultServer.addrs {
-		return fmt.Sprintf("http://%s%s/", a.String(), defaultServerOptions.BaseURL)
-	}
-	for _, a := range defaultServer.tlsAddrs {
-		return fmt.Sprintf("https://%s%s/", a.String(), defaultServerOptions.BaseURL)
-	}
-	panic("Server is running with no listener")
-}
-
-//---- Command line flags ----
-
-// AddFlagsPrefix adds flags for the httplib
-func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *Options) {
-	flags.StringVarP(flagSet, &Opt.ListenAddr, prefix+"addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to")
-	flags.DurationVarP(flagSet, &Opt.ServerReadTimeout, prefix+"server-read-timeout", "", Opt.ServerReadTimeout, "Timeout for server reading data")
-	flags.DurationVarP(flagSet, &Opt.ServerWriteTimeout, prefix+"server-write-timeout", "", Opt.ServerWriteTimeout, "Timeout for server writing data")
-	flags.IntVarP(flagSet, &Opt.MaxHeaderBytes, prefix+"max-header-bytes", "", Opt.MaxHeaderBytes, "Maximum size of request header")
-	flags.StringVarP(flagSet, &Opt.SslCert, prefix+"cert", "", Opt.SslCert, "SSL PEM key (concatenation of certificate and CA certificate)")
-	flags.StringVarP(flagSet, &Opt.SslKey, prefix+"key", "", Opt.SslKey, "SSL PEM Private key")
-	flags.StringVarP(flagSet, &Opt.ClientCA, prefix+"client-ca", "", Opt.ClientCA, "Client certificate authority to verify clients with")
-	flags.StringVarP(flagSet, &Opt.BaseURL, prefix+"baseurl", "", Opt.BaseURL, "Prefix for URLs - leave blank for root")
-	flags.StringVarP(flagSet, &Opt.MinTLSVersion, prefix+"min-tls-version", "", Opt.MinTLSVersion, "Minimum TLS version that is acceptable")
-
-}
-
-// AddFlags adds flags for the httplib
-func AddFlags(flagSet *pflag.FlagSet) {
-	AddFlagsPrefix(flagSet, "", &defaultServerOptions)
-}
diff --git a/lib/http/http_test.go b/lib/http/http_test.go
deleted file mode 100644
index fd9850059d769..0000000000000
--- a/lib/http/http_test.go
+++ /dev/null
@@ -1,515 +0,0 @@
-package http
-
-import (
-	"crypto/tls"
-	"net"
-	"net/http"
-	"reflect"
-	"strings"
-	"testing"
-	"time"
-
-	"golang.org/x/net/nettest"
-
-	"github.com/go-chi/chi/v5"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestGetOptions(t *testing.T) {
-	tests := []struct {
-		name string
-		want Options
-	}{
-		{name: "basic", want: defaultServerOptions},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := GetOptions(); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("GetOptions() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestMount(t *testing.T) {
-	type args struct {
-		pattern string
-		h       http.Handler
-	}
-	tests := []struct {
-		name    string
-		args    args
-		wantErr bool
-	}{
-		{name: "basic", args: args{
-			pattern: "/",
-			h:       http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {}),
-		}, wantErr: false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if tt.wantErr {
-				require.Error(t, Mount(tt.args.pattern, tt.args.h))
-			} else {
-				require.NoError(t, Mount(tt.args.pattern, tt.args.h))
-			}
-			assert.NotNil(t, defaultServer)
-			assert.True(t, defaultServer.baseRouter.Match(chi.NewRouteContext(), "GET", tt.args.pattern), "Failed to match route after registering")
-		})
-		if err := Shutdown(); err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func TestNewServer(t *testing.T) {
-	type args struct {
-		listeners    []net.Listener
-		tlsListeners []net.Listener
-		opt          Options
-	}
-	listener, err := nettest.NewLocalListener("tcp")
-	if err != nil {
-		t.Fatal(err)
-	}
-	tests := []struct {
-		name    string
-		args    args
-		wantErr bool
-	}{
-		{name: "default http", args: args{
-			listeners:    []net.Listener{listener},
-			tlsListeners: []net.Listener{},
-			opt:          defaultServerOptions,
-		}, wantErr: false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := NewServer(tt.args.listeners, tt.args.tlsListeners, tt.args.opt)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("NewServer() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			s, ok := got.(*server)
-			require.True(t, ok, "NewServer returned unexpected type")
-			if len(tt.args.listeners) > 0 {
-				assert.Equal(t, listener.Addr(), s.addrs[0])
-			} else {
-				assert.Empty(t, s.addrs)
-			}
-			if len(tt.args.tlsListeners) > 0 {
-				assert.Equal(t, listener.Addr(), s.tlsAddrs[0])
-			} else {
-				assert.Empty(t, s.tlsAddrs)
-			}
-			if tt.args.opt.BaseURL != "" {
-				assert.NotSame(t, s.baseRouter, s.httpServer.Handler, "should have wrapped baseRouter")
-			} else {
-				assert.Same(t, s.baseRouter, s.httpServer.Handler, "should be baseRouter")
-			}
-			if useSSL(tt.args.opt) {
-				assert.NotNil(t, s.httpServer.TLSConfig, "missing SSL config")
-			} else {
-				assert.Nil(t, s.httpServer.TLSConfig, "unexpectedly has SSL config")
-			}
-		})
-	}
-}
-
-func TestRestart(t *testing.T) {
-	tests := []struct {
-		name    string
-		started bool
-		wantErr bool
-	}{
-		{name: "started", started: true, wantErr: false},
-		{name: "stopped", started: false, wantErr: false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if tt.started {
-				require.NoError(t, Restart()) // Call it twice basically
-			} else {
-				require.NoError(t, Shutdown())
-			}
-			current := defaultServer
-			if err := Restart(); (err != nil) != tt.wantErr {
-				t.Errorf("Restart() error = %v, wantErr %v", err, tt.wantErr)
-			}
-			assert.NotNil(t, defaultServer, "failed to start default server")
-			assert.NotSame(t, current, defaultServer, "same server instance as before restart")
-		})
-	}
-}
-
-func TestRoute(t *testing.T) {
-	type args struct {
-		pattern string
-		fn      func(r chi.Router)
-	}
-	tests := []struct {
-		name string
-		args args
-		test func(t *testing.T, r chi.Router)
-	}{
-		{
-			name: "basic",
-			args: args{
-				pattern: "/basic",
-				fn:      func(r chi.Router) {},
-			},
-			test: func(t *testing.T, r chi.Router) {
-				require.Len(t, r.Routes(), 1)
-				assert.Equal(t, r.Routes()[0].Pattern, "/basic/*")
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			require.NoError(t, Restart())
-			_, err := Route(tt.args.pattern, tt.args.fn)
-			require.NoError(t, err)
-			tt.test(t, defaultServer.baseRouter)
-		})
-
-		if err := Shutdown(); err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func TestSetOptions(t *testing.T) {
-	type args struct {
-		opt Options
-	}
-	tests := []struct {
-		name string
-		args args
-	}{
-		{
-			name: "basic",
-			args: args{opt: Options{
-				ListenAddr:         "127.0.0.1:9999",
-				BaseURL:            "/basic",
-				ServerReadTimeout:  1,
-				ServerWriteTimeout: 1,
-				MaxHeaderBytes:     1,
-			}},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			SetOptions(tt.args.opt)
-			require.Equal(t, tt.args.opt, defaultServerOptions)
-			require.NoError(t, Restart())
-			if useSSL(tt.args.opt) {
-				assert.Equal(t, tt.args.opt.ListenAddr, defaultServer.tlsAddrs[0].String())
-			} else {
-				assert.Equal(t, tt.args.opt.ListenAddr, defaultServer.addrs[0].String())
-			}
-			assert.Equal(t, tt.args.opt.ServerReadTimeout, defaultServer.httpServer.ReadTimeout)
-			assert.Equal(t, tt.args.opt.ServerWriteTimeout, defaultServer.httpServer.WriteTimeout)
-			assert.Equal(t, tt.args.opt.MaxHeaderBytes, defaultServer.httpServer.MaxHeaderBytes)
-			if tt.args.opt.BaseURL != "" && tt.args.opt.BaseURL != "/" {
-				assert.NotSame(t, defaultServer.httpServer.Handler, defaultServer.baseRouter, "BaseURL ignored")
-			}
-		})
-		SetOptions(DefaultOpt)
-	}
-}
-
-func TestShutdown(t *testing.T) {
-	tests := []struct {
-		name    string
-		started bool
-		wantErr bool
-	}{
-		{name: "started", started: true, wantErr: false},
-		{name: "stopped", started: false, wantErr: false},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if tt.started {
-				require.NoError(t, Restart())
-			} else {
-				require.NoError(t, Shutdown()) // Call it twice basically
-			}
-			if err := Shutdown(); (err != nil) != tt.wantErr {
-				t.Errorf("Shutdown() error = %v, wantErr %v", err, tt.wantErr)
-			}
-			assert.Nil(t, defaultServer, "default server not deleted")
-		})
-	}
-}
-
-func TestURL(t *testing.T) {
-	tests := []struct {
-		name string
-		want string
-	}{
-		{name: "basic", want: "http://127.0.0.1:8080/"},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			require.NoError(t, Restart())
-			if got := URL(); got != tt.want {
-				t.Errorf("URL() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_server_Mount(t *testing.T) {
-	type args struct {
-		pattern string
-		h       http.Handler
-	}
-	tests := []struct {
-		name string
-		args args
-		opt  Options
-	}{
-		{name: "basic", args: args{
-			pattern: "/",
-			h:       http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {}),
-		}, opt: defaultServerOptions},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			listener, err := nettest.NewLocalListener("tcp")
-			require.NoError(t, err)
-			s, err2 := NewServer([]net.Listener{listener}, []net.Listener{}, tt.opt)
-			require.NoError(t, err2)
-			s.Mount(tt.args.pattern, tt.args.h)
-			srv, ok := s.(*server)
-			require.True(t, ok)
-			assert.NotNil(t, srv)
-			assert.True(t, srv.baseRouter.Match(chi.NewRouteContext(), "GET", tt.args.pattern), "Failed to Match() route after registering")
-		})
-	}
-}
-
-func Test_server_Route(t *testing.T) {
-	type args struct {
-		pattern string
-		fn      func(r chi.Router)
-	}
-	tests := []struct {
-		name string
-		args args
-		opt  Options
-		test func(t *testing.T, r chi.Router)
-	}{
-		{
-			name: "basic",
-			args: args{
-				pattern: "/basic",
-				fn: func(r chi.Router) {
-
-				},
-			},
-			test: func(t *testing.T, r chi.Router) {
-				require.Len(t, r.Routes(), 1)
-				assert.Equal(t, r.Routes()[0].Pattern, "/basic/*")
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			listener, err := nettest.NewLocalListener("tcp")
-			require.NoError(t, err)
-			s, err2 := NewServer([]net.Listener{listener}, []net.Listener{}, tt.opt)
-			require.NoError(t, err2)
-			s.Route(tt.args.pattern, tt.args.fn)
-			srv, ok := s.(*server)
-			require.True(t, ok)
-			assert.NotNil(t, srv)
-			tt.test(t, srv.baseRouter)
-		})
-	}
-}
-
-func Test_server_Shutdown(t *testing.T) {
-	tests := []struct {
-		name    string
-		opt     Options
-		wantErr bool
-	}{
-		{
-			name:    "basic",
-			opt:     defaultServerOptions,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			listener, err := nettest.NewLocalListener("tcp")
-			require.NoError(t, err)
-			s, err2 := NewServer([]net.Listener{listener}, []net.Listener{}, tt.opt)
-			require.NoError(t, err2)
-			srv, ok := s.(*server)
-			require.True(t, ok)
-			if err := s.Shutdown(); (err != nil) != tt.wantErr {
-				t.Errorf("Shutdown() error = %v, wantErr %v", err, tt.wantErr)
-			}
-			assert.EqualError(t, srv.httpServer.Serve(listener), http.ErrServerClosed.Error())
-		})
-	}
-}
-
-func Test_start(t *testing.T) {
-	http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
-	sslServerOptions := defaultServerOptions
-	sslServerOptions.SslCertBody = []byte(`-----BEGIN CERTIFICATE-----
-MIIBhTCCASugAwIBAgIQIRi6zePL6mKjOipn+dNuaTAKBggqhkjOPQQDAjASMRAw
-DgYDVQQKEwdBY21lIENvMB4XDTE3MTAyMDE5NDMwNloXDTE4MTAyMDE5NDMwNlow
-EjEQMA4GA1UEChMHQWNtZSBDbzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABD0d
-7VNhbWvZLWPuj/RtHFjvtJBEwOkhbN/BnnE8rnZR8+sbwnc/KhCk3FhnpHZnQz7B
-5aETbbIgmuvewdjvSBSjYzBhMA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggr
-BgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MCkGA1UdEQQiMCCCDmxvY2FsaG9zdDo1
-NDUzgg4xMjcuMC4wLjE6NTQ1MzAKBggqhkjOPQQDAgNIADBFAiEA2zpJEPQyz6/l
-Wf86aX6PepsntZv2GYlA5UpabfT2EZICICpJ5h/iI+i341gBmLiAFQOyTDT+/wQc
-6MF9+Yw1Yy0t
------END CERTIFICATE-----`)
-	sslServerOptions.SslKeyBody = []byte(`-----BEGIN EC PRIVATE KEY-----
-MHcCAQEEIIrYSSNQFaA2Hwf1duRSxKtLYX5CB04fSeQ6tF1aY/PuoAoGCCqGSM49
-AwEHoUQDQgAEPR3tU2Fta9ktY+6P9G0cWO+0kETA6SFs38GecTyudlHz6xvCdz8q
-EKTcWGekdmdDPsHloRNtsiCa697B2O9IFA==
------END EC PRIVATE KEY-----`)
-	tests := []struct {
-		name    string
-		opt     Options
-		ssl     bool
-		wantErr bool
-	}{
-		{
-			name:    "basic",
-			opt:     defaultServerOptions,
-			wantErr: false,
-		},
-		{
-			name:    "ssl",
-			opt:     sslServerOptions,
-			ssl:     true,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			defer func() {
-				err := Shutdown()
-				if err != nil {
-					t.Fatal("couldn't shutdown server")
-				}
-			}()
-			SetOptions(tt.opt)
-			if err := start(); (err != nil) != tt.wantErr {
-				t.Errorf("start() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			s := defaultServer
-			router := s.Router()
-			router.Head("/", func(writer http.ResponseWriter, request *http.Request) {
-				writer.WriteHeader(201)
-			})
-			testURL := URL()
-			if tt.ssl {
-				assert.True(t, useSSL(tt.opt))
-				assert.Equal(t, tt.opt.ListenAddr, s.tlsAddrs[0].String())
-				assert.True(t, strings.HasPrefix(testURL, "https://"))
-			} else {
-				assert.True(t, strings.HasPrefix(testURL, "http://"))
-				assert.Equal(t, tt.opt.ListenAddr, s.addrs[0].String())
-			}
-
-			// try to connect to the test server
-			pause := time.Millisecond
-			for i := 0; i < 10; i++ {
-				resp, err := http.Head(testURL)
-				if err == nil {
-					_ = resp.Body.Close()
-					return
-				}
-				// t.Logf("couldn't connect, sleeping for %v: %v", pause, err)
-				time.Sleep(pause)
-				pause *= 2
-			}
-			t.Fatal("couldn't connect to server")
-
-			/* accessing s.httpServer.* can't be done synchronously and is a race condition
-			assert.Equal(t, tt.opt.ServerReadTimeout, defaultServer.httpServer.ReadTimeout)
-			assert.Equal(t, tt.opt.ServerWriteTimeout, defaultServer.httpServer.WriteTimeout)
-			assert.Equal(t, tt.opt.MaxHeaderBytes, defaultServer.httpServer.MaxHeaderBytes)
-			if tt.opt.BaseURL != "" && tt.opt.BaseURL != "/" {
-				assert.NotSame(t, s.baseRouter, s.httpServer.Handler, "should have wrapped baseRouter")
-			} else {
-				assert.Same(t, s.baseRouter, s.httpServer.Handler, "should be baseRouter")
-			}
-			if useSSL(tt.opt) {
-				require.NotNil(t, s.httpServer.TLSConfig, "missing SSL config")
-				assert.NotEmpty(t, s.httpServer.TLSConfig.Certificates, "missing SSL config")
-			} else if s.httpServer.TLSConfig != nil {
-				assert.Empty(t, s.httpServer.TLSConfig.Certificates, "unexpectedly has SSL config")
-			}
-			*/
-		})
-	}
-}
-
-func Test_useSSL(t *testing.T) {
-	type args struct {
-		opt Options
-	}
-	tests := []struct {
-		name string
-		args args
-		want bool
-	}{
-		{
-			name: "basic",
-			args: args{opt: Options{
-				SslCert:  "",
-				SslKey:   "",
-				ClientCA: "",
-			}},
-			want: false,
-		},
-		{
-			name: "basic",
-			args: args{opt: Options{
-				SslCert:  "",
-				SslKey:   "test",
-				ClientCA: "",
-			}},
-			want: true,
-		},
-		{
-			name: "body",
-			args: args{opt: Options{
-				SslCert:    "",
-				SslKey:     "",
-				SslKeyBody: []byte(`test`),
-				ClientCA:   "",
-			}},
-			want: true,
-		},
-		{
-			name: "basic",
-			args: args{opt: Options{
-				SslCert:       "",
-				SslKey:        "test",
-				ClientCA:      "",
-				MinTLSVersion: "tls1.2",
-			}},
-			want: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := useSSL(tt.args.opt); got != tt.want {
-				t.Errorf("useSSL() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
diff --git a/lib/http/middleware.go b/lib/http/middleware.go
new file mode 100644
index 0000000000000..d19d75259b4a8
--- /dev/null
+++ b/lib/http/middleware.go
@@ -0,0 +1,171 @@
+package http
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+
+	goauth "github.com/abbot/go-http-auth"
+	"github.com/rclone/rclone/fs"
+)
+
+// parseAuthorization parses the Authorization header into user, pass
+// it returns a boolean as to whether the parse was successful
+func parseAuthorization(r *http.Request) (user, pass string, ok bool) {
+	authHeader := r.Header.Get("Authorization")
+	if authHeader != "" {
+		s := strings.SplitN(authHeader, " ", 2)
+		if len(s) == 2 && s[0] == "Basic" {
+			b, err := base64.StdEncoding.DecodeString(s[1])
+			if err == nil {
+				parts := strings.SplitN(string(b), ":", 2)
+				user = parts[0]
+				if len(parts) > 1 {
+					pass = parts[1]
+					ok = true
+				}
+			}
+		}
+	}
+	return
+}
+
+type LoggedBasicAuth struct {
+	goauth.BasicAuth
+}
+
+// CheckAuth extends BasicAuth.CheckAuth to emit a log entry for unauthorised requests
+func (a *LoggedBasicAuth) CheckAuth(r *http.Request) string {
+	username := a.BasicAuth.CheckAuth(r)
+	if username == "" {
+		user, _, _ := parseAuthorization(r)
+		fs.Infof(r.URL.Path, "%s: Unauthorized request from %s", r.RemoteAddr, user)
+	}
+	return username
+}
+
+// NewLoggedBasicAuthenticator instantiates a new instance of LoggedBasicAuthenticator
+func NewLoggedBasicAuthenticator(realm string, secrets goauth.SecretProvider) *LoggedBasicAuth {
+	return &LoggedBasicAuth{BasicAuth: goauth.BasicAuth{Realm: realm, Secrets: secrets}}
+}
+
+// Helper to generate required interface for middleware
+func basicAuth(authenticator *LoggedBasicAuth) func(next http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// skip auth for unix socket
+			if IsUnixSocket(r) {
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			username := authenticator.CheckAuth(r)
+			if username == "" {
+				authenticator.RequireAuth(w, r)
+				return
+			}
+			ctx := context.WithValue(r.Context(), ctxKeyUser, username)
+			next.ServeHTTP(w, r.WithContext(ctx))
+		})
+	}
+}
+
+// MiddlewareAuthHtpasswd instantiates middleware that authenticates against the passed htpasswd file
+func MiddlewareAuthHtpasswd(path, realm string) Middleware {
+	fs.Infof(nil, "Using %q as htpasswd storage", path)
+	secretProvider := goauth.HtpasswdFileProvider(path)
+	authenticator := NewLoggedBasicAuthenticator(realm, secretProvider)
+	return basicAuth(authenticator)
+}
+
+// MiddlewareAuthBasic instantiates middleware that authenticates for a single user
+func MiddlewareAuthBasic(user, pass, realm, salt string) Middleware {
+	fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", user)
+	pass = string(goauth.MD5Crypt([]byte(pass), []byte(salt), []byte("$1$")))
+	secretProvider := func(u, r string) string {
+		if user == u {
+			return pass
+		}
+		return ""
+	}
+	authenticator := NewLoggedBasicAuthenticator(realm, secretProvider)
+	return basicAuth(authenticator)
+}
+
+// MiddlewareAuthCustom instantiates middleware that authenticates using a custom function
+func MiddlewareAuthCustom(fn CustomAuthFn, realm string) Middleware {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// skip auth for unix socket
+			if IsUnixSocket(r) {
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			user, pass, ok := parseAuthorization(r)
+			if !ok {
+				code := http.StatusUnauthorized
+				w.Header().Set("Content-Type", "text/plain")
+				w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Basic realm=%q, charset="UTF-8"`, realm))
+				http.Error(w, http.StatusText(code), code)
+				return
+			}
+
+			value, err := fn(user, pass)
+			if err != nil {
+				fs.Infof(r.URL.Path, "%s: Auth failed from %s: %v", r.RemoteAddr, user, err)
+				goauth.NewBasicAuthenticator(realm, func(user, realm string) string { return "" }).RequireAuth(w, r) //Reuse BasicAuth error reporting
+				return
+			}
+
+			if value != nil {
+				r = r.WithContext(context.WithValue(r.Context(), ctxKeyAuth, value))
+			}
+
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+var onlyOnceWarningAllowOrigin sync.Once
+
+// MiddlewareCORS instantiates middleware that handles basic CORS protections for rcd
+func MiddlewareCORS(allowOrigin string) Middleware {
+	onlyOnceWarningAllowOrigin.Do(func() {
+		if allowOrigin == "*" {
+			fs.Logf(nil, "Warning: Allow origin set to *. This can cause serious security problems.")
+		}
+	})
+
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// skip cors for unix sockets
+			if IsUnixSocket(r) {
+				next.ServeHTTP(w, r)
+				return
+			}
+
+			if allowOrigin != "" {
+				w.Header().Add("Access-Control-Allow-Origin", allowOrigin)
+			} else {
+				w.Header().Add("Access-Control-Allow-Origin", PublicURL(r))
+			}
+
+			// echo back access control headers client needs
+			w.Header().Add("Access-Control-Request-Method", "POST, OPTIONS, GET, HEAD")
+			w.Header().Add("Access-Control-Allow-Headers", "authorization, Content-Type")
+
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+// MiddlewareStripPrefix instantiates middleware that removes the BaseURL from the path
+func MiddlewareStripPrefix(prefix string) Middleware {
+	return func(next http.Handler) http.Handler {
+		return http.StripPrefix(prefix, next)
+	}
+}
diff --git a/lib/http/middleware_test.go b/lib/http/middleware_test.go
new file mode 100644
index 0000000000000..4839b87e31510
--- /dev/null
+++ b/lib/http/middleware_test.go
@@ -0,0 +1,231 @@
+package http
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMiddlewareAuth(t *testing.T) {
+	servers := []struct {
+		name string
+		http HTTPConfig
+		auth AuthConfig
+		user string
+		pass string
+	}{
+		{
+			name: "Basic",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				Realm:     "test",
+				BasicUser: "test",
+				BasicPass: "test",
+			},
+			user: "test",
+			pass: "test",
+		},
+		{
+			name: "Htpasswd/MD5",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				Realm:    "test",
+				HtPasswd: "./testdata/.htpasswd",
+			},
+			user: "md5",
+			pass: "md5",
+		},
+		{
+			name: "Htpasswd/SHA",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				Realm:    "test",
+				HtPasswd: "./testdata/.htpasswd",
+			},
+			user: "sha",
+			pass: "sha",
+		},
+		{
+			name: "Htpasswd/Bcrypt",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				Realm:    "test",
+				HtPasswd: "./testdata/.htpasswd",
+			},
+			user: "bcrypt",
+			pass: "bcrypt",
+		},
+		{
+			name: "Custom",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			auth: AuthConfig{
+				Realm: "test",
+				CustomAuthFn: func(user, pass string) (value interface{}, err error) {
+					if user == "custom" && pass == "custom" {
+						return true, nil
+					}
+					return nil, errors.New("invalid credentials")
+				},
+			},
+			user: "custom",
+			pass: "custom",
+		},
+	}
+
+	for _, ss := range servers {
+		t.Run(ss.name, func(t *testing.T) {
+			s, err := NewServer(context.Background(), WithConfig(ss.http), WithAuth(ss.auth))
+			require.NoError(t, err)
+			defer func() {
+				require.NoError(t, s.Shutdown())
+			}()
+
+			expected := []byte("secret-page")
+			s.Router().Mount("/", testEchoHandler(expected))
+			s.Serve()
+
+			url := testGetServerURL(t, s)
+
+			t.Run("NoCreds", func(t *testing.T) {
+				client := &http.Client{}
+				req, err := http.NewRequest("GET", url, nil)
+				require.NoError(t, err)
+
+				resp, err := client.Do(req)
+				require.NoError(t, err)
+				defer func() {
+					_ = resp.Body.Close()
+				}()
+
+				require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "using no creds should return unauthorized")
+
+				wwwAuthHeader := resp.Header.Get("WWW-Authenticate")
+				require.NotEmpty(t, wwwAuthHeader, "resp should contain WWW-Authtentication header")
+				require.Contains(t, wwwAuthHeader, fmt.Sprintf("realm=%q", ss.auth.Realm), "WWW-Authtentication header should contain relam")
+			})
+
+			t.Run("BadCreds", func(t *testing.T) {
+				client := &http.Client{}
+				req, err := http.NewRequest("GET", url, nil)
+				require.NoError(t, err)
+
+				req.SetBasicAuth(ss.user+"BAD", ss.pass+"BAD")
+
+				resp, err := client.Do(req)
+				require.NoError(t, err)
+				defer func() {
+					_ = resp.Body.Close()
+				}()
+
+				require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "using bad creds should return unauthorized")
+
+				wwwAuthHeader := resp.Header.Get("WWW-Authenticate")
+				require.NotEmpty(t, wwwAuthHeader, "resp should contain WWW-Authtentication header")
+				require.Contains(t, wwwAuthHeader, fmt.Sprintf("realm=%q", ss.auth.Realm), "WWW-Authtentication header should contain relam")
+			})
+
+			t.Run("GoodCreds", func(t *testing.T) {
+				client := &http.Client{}
+				req, err := http.NewRequest("GET", url, nil)
+				require.NoError(t, err)
+
+				req.SetBasicAuth(ss.user, ss.pass)
+
+				resp, err := client.Do(req)
+				require.NoError(t, err)
+				defer func() {
+					_ = resp.Body.Close()
+				}()
+
+				require.Equal(t, http.StatusOK, resp.StatusCode, "using good creds should return ok")
+
+				testExpectRespBody(t, resp, expected)
+			})
+		})
+	}
+}
+
+var _testCORSHeaderKeys = []string{
+	"Access-Control-Allow-Origin",
+	"Access-Control-Request-Method",
+	"Access-Control-Allow-Headers",
+}
+
+func TestMiddlewareCORS(t *testing.T) {
+	servers := []struct {
+		name   string
+		http   HTTPConfig
+		origin string
+	}{
+		{
+			name: "EmptyOrigin",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			origin: "",
+		},
+		{
+			name: "CustomOrigin",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+			},
+			origin: "http://test.rclone.org",
+		},
+	}
+
+	for _, ss := range servers {
+		t.Run(ss.name, func(t *testing.T) {
+			s, err := NewServer(context.Background(), WithConfig(ss.http))
+			require.NoError(t, err)
+			defer func() {
+				require.NoError(t, s.Shutdown())
+			}()
+
+			s.Router().Use(MiddlewareCORS(ss.origin))
+
+			expected := []byte("data")
+			s.Router().Mount("/", testEchoHandler(expected))
+			s.Serve()
+
+			url := testGetServerURL(t, s)
+
+			client := &http.Client{}
+			req, err := http.NewRequest("GET", url, nil)
+			require.NoError(t, err)
+
+			resp, err := client.Do(req)
+			require.NoError(t, err)
+			defer func() {
+				_ = resp.Body.Close()
+			}()
+
+			require.Equal(t, http.StatusOK, resp.StatusCode, "should return ok")
+
+			testExpectRespBody(t, resp, expected)
+
+			for _, key := range _testCORSHeaderKeys {
+				require.Contains(t, resp.Header, key, "CORS headers should be sent")
+			}
+
+			expectedOrigin := url
+			if ss.origin != "" {
+				expectedOrigin = ss.origin
+			}
+			require.Equal(t, expectedOrigin, resp.Header.Get("Access-Control-Allow-Origin"), "allow origin should match")
+		})
+	}
+}
diff --git a/lib/http/serve/dir_test.go b/lib/http/serve/dir_test.go
index be128ae33adc8..bdcf3f9486986 100644
--- a/lib/http/serve/dir_test.go
+++ b/lib/http/serve/dir_test.go
@@ -3,7 +3,7 @@ package serve
 import (
 	"errors"
 	"html/template"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -94,7 +94,7 @@ func TestError(t *testing.T) {
 	Error("potato", w, "sausage", err)
 	resp := w.Result()
 	assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "sausage.\n", string(body))
 }
 
@@ -108,7 +108,7 @@ func TestServe(t *testing.T) {
 	d.Serve(w, r)
 	resp := w.Result()
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, `
 
 
diff --git a/lib/http/serve/serve_test.go b/lib/http/serve/serve_test.go
index b6cc70976d32b..934fabe658169 100644
--- a/lib/http/serve/serve_test.go
+++ b/lib/http/serve/serve_test.go
@@ -1,7 +1,7 @@
 package serve
 
 import (
-	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -17,7 +17,7 @@ func TestObjectBadMethod(t *testing.T) {
 	Object(w, r, o)
 	resp := w.Result()
 	assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "Method Not Allowed\n", string(body))
 }
 
@@ -30,7 +30,7 @@ func TestObjectHEAD(t *testing.T) {
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	assert.Equal(t, "5", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "", string(body))
 }
 
@@ -43,7 +43,7 @@ func TestObjectGET(t *testing.T) {
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	assert.Equal(t, "5", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "hello", string(body))
 }
 
@@ -58,7 +58,7 @@ func TestObjectRange(t *testing.T) {
 	assert.Equal(t, "3", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
 	assert.Equal(t, "bytes 3-5/10", resp.Header.Get("Content-Range"))
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "345", string(body))
 }
 
@@ -71,6 +71,6 @@ func TestObjectBadRange(t *testing.T) {
 	resp := w.Result()
 	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
 	assert.Equal(t, "10", resp.Header.Get("Content-Length"))
-	body, _ := io.ReadAll(resp.Body)
+	body, _ := ioutil.ReadAll(resp.Body)
 	assert.Equal(t, "Bad Request\n", string(body))
 }
diff --git a/lib/http/server.go b/lib/http/server.go
new file mode 100644
index 0000000000000..d4e5e827b0565
--- /dev/null
+++ b/lib/http/server.go
@@ -0,0 +1,430 @@
+// Package http provides a registration interface for http services
+package http
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"html/template"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/spf13/pflag"
+)
+
+// Help contains text describing the http server to add to the command
+// help.
+var Help = `
+### Server options
+
+Use ` + "`--addr`" + ` to specify which IP address and port the server should
+listen on, eg ` + "`--addr 1.2.3.4:8000` or `--addr :8080`" + ` to listen to all
+IPs.  By default it only listens on localhost.  You can use port
+:0 to let the OS choose an available port.
+
+If you set ` + "`--addr`" + ` to listen on a public or LAN accessible IP address
+then using Authentication is advised - see the next section for info.
+
+You can use a unix socket by setting the url to ` + "`unix:///path/to/socket`" + `
+or just by using an absolute path name. Note that unix sockets bypass the
+authentication - this is expected to be done with file system permissions.
+
+` + "`--addr`" + ` may be repeated to listen on multiple IPs/ports/sockets.
+
+` + "`--server-read-timeout` and `--server-write-timeout`" + ` can be used to
+control the timeouts on the server.  Note that this is the total time
+for a transfer.
+
+` + "`--max-header-bytes`" + ` controls the maximum number of bytes the server will
+accept in the HTTP header.
+
+` + "`--baseurl`" + ` controls the URL prefix that rclone serves from.  By default
+rclone will serve from the root.  If you used ` + "`--baseurl \"/rclone\"`" + ` then
+rclone would serve from a URL starting with "/rclone/".  This is
+useful if you wish to proxy rclone serve.  Rclone automatically
+inserts leading and trailing "/" on ` + "`--baseurl`" + `, so ` + "`--baseurl \"rclone\"`" + `,
+` + "`--baseurl \"/rclone\"` and `--baseurl \"/rclone/\"`" + ` are all treated
+identically.
+
+#### TLS (SSL)
+
+By default this will serve over http.  If you want you can serve over
+https.  You will need to supply the ` + "`--cert` and `--key`" + ` flags.
+If you wish to do client side certificate validation then you will need to
+supply ` + "`--client-ca`" + ` also.
+
+` + "`--cert`" + ` should be a either a PEM encoded certificate or a concatenation
+of that with the CA certificate.  ` + "`--key`" + ` should be the PEM encoded
+private key and ` + "`--client-ca`" + ` should be the PEM encoded client
+certificate authority certificate.
+
+--min-tls-version is minimum TLS version that is acceptable. Valid
+  values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
+  "tls1.0").
+`
+
+// Middleware function signature required by chi.Router.Use()
+type Middleware func(http.Handler) http.Handler
+
+// HTTPConfig contains options for the http Server
+type HTTPConfig struct {
+	ListenAddr         []string      // Port to listen on
+	BaseURL            string        // prefix to strip from URLs
+	ServerReadTimeout  time.Duration // Timeout for server reading data
+	ServerWriteTimeout time.Duration // Timeout for server writing data
+	MaxHeaderBytes     int           // Maximum size of request header
+	TLSCert            string        // Path to TLS PEM key (concatenation of certificate and CA certificate)
+	TLSKey             string        // Path to TLS PEM Private key
+	TLSCertBody        []byte        // TLS PEM key (concatenation of certificate and CA certificate) body, ignores TLSCert
+	TLSKeyBody         []byte        // TLS PEM Private key body, ignores TLSKey
+	ClientCA           string        // Client certificate authority to verify clients with
+	MinTLSVersion      string        // MinTLSVersion contains the minimum TLS version that is acceptable.
+	Template           string
+}
+
+// AddFlagsPrefix adds flags for the httplib
+func (cfg *HTTPConfig) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) {
+	flags.StringArrayVarP(flagSet, &cfg.ListenAddr, prefix+"addr", "", cfg.ListenAddr, "IPaddress:Port or :Port to bind server to")
+	flags.DurationVarP(flagSet, &cfg.ServerReadTimeout, prefix+"server-read-timeout", "", cfg.ServerReadTimeout, "Timeout for server reading data")
+	flags.DurationVarP(flagSet, &cfg.ServerWriteTimeout, prefix+"server-write-timeout", "", cfg.ServerWriteTimeout, "Timeout for server writing data")
+	flags.IntVarP(flagSet, &cfg.MaxHeaderBytes, prefix+"max-header-bytes", "", cfg.MaxHeaderBytes, "Maximum size of request header")
+	flags.StringVarP(flagSet, &cfg.TLSCert, prefix+"cert", "", cfg.TLSCert, "TLS PEM key (concatenation of certificate and CA certificate)")
+	flags.StringVarP(flagSet, &cfg.TLSKey, prefix+"key", "", cfg.TLSKey, "TLS PEM Private key")
+	flags.StringVarP(flagSet, &cfg.ClientCA, prefix+"client-ca", "", cfg.ClientCA, "Client certificate authority to verify clients with")
+	flags.StringVarP(flagSet, &cfg.BaseURL, prefix+"baseurl", "", cfg.BaseURL, "Prefix for URLs - leave blank for root")
+	flags.StringVarP(flagSet, &cfg.MinTLSVersion, prefix+"min-tls-version", "", cfg.MinTLSVersion, "Minimum TLS version that is acceptable")
+}
+
+// AddHTTPFlagsPrefix adds flags for the httplib
+func AddHTTPFlagsPrefix(flagSet *pflag.FlagSet, prefix string, cfg *HTTPConfig) {
+	cfg.AddFlagsPrefix(flagSet, prefix)
+}
+
+// DefaultHTTPCfg is the default values used for Config
+func DefaultHTTPCfg() HTTPConfig {
+	return HTTPConfig{
+		ListenAddr:         []string{"127.0.0.1:8080"},
+		ServerReadTimeout:  1 * time.Hour,
+		ServerWriteTimeout: 1 * time.Hour,
+		MaxHeaderBytes:     4096,
+		MinTLSVersion:      "tls1.0",
+	}
+}
+
+// Server interface of http server
+type Server interface {
+	Router() chi.Router
+	Serve()
+	Shutdown() error
+	HTMLTemplate() *template.Template
+	URLs() []string
+	Wait()
+}
+
+type instance struct {
+	url        string
+	listener   net.Listener
+	httpServer *http.Server
+}
+
+func (s instance) serve(wg *sync.WaitGroup) {
+	defer wg.Done()
+	err := s.httpServer.Serve(s.listener)
+	if err != http.ErrServerClosed && err != nil {
+		log.Printf("%s: unexpected error: %s", s.listener.Addr(), err.Error())
+	}
+}
+
+type server struct {
+	wg           sync.WaitGroup
+	mux          chi.Router
+	tlsConfig    *tls.Config
+	instances    []instance
+	auth         AuthConfig
+	cfg          HTTPConfig
+	template     *TemplateConfig
+	htmlTemplate *template.Template
+}
+
+// Option allows customizing the server
+type Option func(*server)
+
+// WithAuth option initializes the appropriate auth middleware
+func WithAuth(cfg AuthConfig) Option {
+	return func(s *server) {
+		s.auth = cfg
+	}
+}
+
+// WithConfig option applies the HTTPConfig to the server, overriding defaults
+func WithConfig(cfg HTTPConfig) Option {
+	return func(s *server) {
+		s.cfg = cfg
+	}
+}
+
+// WithTemplate option allows the parsing of a template
+func WithTemplate(cfg TemplateConfig) Option {
+	return func(s *server) {
+		s.template = &cfg
+	}
+}
+
+// NewServer instantiates a new http server using provided listeners and options
+// This function is provided if the default http server does not meet a services requirements and should not generally be used
+// A http server can listen using multiple listeners. For example, a listener for port 80, and a listener for port 443.
+// tlsListeners are ignored if opt.TLSKey is not provided
+func NewServer(ctx context.Context, options ...Option) (*server, error) {
+	s := &server{
+		mux: chi.NewRouter(),
+		cfg: DefaultHTTPCfg(),
+	}
+
+	for _, opt := range options {
+		opt(s)
+	}
+
+	// Build base router
+	s.mux.MethodNotAllowed(func(w http.ResponseWriter, _ *http.Request) {
+		http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
+	})
+	s.mux.NotFound(func(w http.ResponseWriter, _ *http.Request) {
+		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+	})
+
+	// Ignore passing "/" for BaseURL
+	s.cfg.BaseURL = strings.Trim(s.cfg.BaseURL, "/")
+	if s.cfg.BaseURL != "" {
+		s.cfg.BaseURL = "/" + s.cfg.BaseURL
+		s.mux.Use(MiddlewareStripPrefix(s.cfg.BaseURL))
+	}
+
+	s.initAuth()
+
+	err := s.initTemplate()
+	if err != nil {
+		return nil, err
+	}
+
+	err = s.initTLS()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, addr := range s.cfg.ListenAddr {
+		var url string
+		var network = "tcp"
+		var tlsCfg *tls.Config
+
+		if strings.HasPrefix(addr, "unix://") || filepath.IsAbs(addr) {
+			network = "unix"
+			addr = strings.TrimPrefix(addr, "unix://")
+			url = addr
+
+		} else if strings.HasPrefix(addr, "tls://") || (len(s.cfg.ListenAddr) == 1 && s.tlsConfig != nil) {
+			tlsCfg = s.tlsConfig
+			addr = strings.TrimPrefix(addr, "tls://")
+		}
+
+		var listener net.Listener
+		if tlsCfg == nil {
+			listener, err = net.Listen(network, addr)
+		} else {
+			listener, err = tls.Listen(network, addr, tlsCfg)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		if network == "tcp" {
+			var secure string
+			if tlsCfg != nil {
+				secure = "s"
+			}
+			url = fmt.Sprintf("http%s://%s%s/", secure, listener.Addr().String(), s.cfg.BaseURL)
+		}
+
+		ii := instance{
+			url:      url,
+			listener: listener,
+			httpServer: &http.Server{
+				Handler:           s.mux,
+				ReadTimeout:       s.cfg.ServerReadTimeout,
+				WriteTimeout:      s.cfg.ServerWriteTimeout,
+				MaxHeaderBytes:    s.cfg.MaxHeaderBytes,
+				ReadHeaderTimeout: 10 * time.Second, // time to send the headers
+				IdleTimeout:       60 * time.Second, // time to keep idle connections open
+				TLSConfig:         tlsCfg,
+				BaseContext:       NewBaseContext(ctx, url),
+			},
+		}
+
+		s.instances = append(s.instances, ii)
+	}
+
+	return s, nil
+}
+
+func (s *server) initAuth() {
+	if s.auth.CustomAuthFn != nil {
+		s.mux.Use(MiddlewareAuthCustom(s.auth.CustomAuthFn, s.auth.Realm))
+		return
+	}
+
+	if s.auth.HtPasswd != "" {
+		s.mux.Use(MiddlewareAuthHtpasswd(s.auth.HtPasswd, s.auth.Realm))
+		return
+	}
+
+	if s.auth.BasicUser != "" {
+		s.mux.Use(MiddlewareAuthBasic(s.auth.BasicUser, s.auth.BasicPass, s.auth.Realm, s.auth.Salt))
+		return
+	}
+}
+
+func (s *server) initTemplate() error {
+	if s.template == nil {
+		return nil
+	}
+
+	var err error
+	s.htmlTemplate, err = GetTemplate(s.template.Path)
+	if err != nil {
+		err = fmt.Errorf("failed to get template: %w", err)
+	}
+
+	return err
+}
+
+var (
+	// hard coded errors, allowing for easier testing
+	ErrInvalidMinTLSVersion = errors.New("invalid value for --min-tls-version")
+	ErrTLSBodyMismatch      = errors.New("need both TLSCertBody and TLSKeyBody to use TLS")
+	ErrTLSFileMismatch      = errors.New("need both --cert and --key to use TLS")
+	ErrTLSParseCA           = errors.New("unable to parse client certificate authority")
+)
+
+func (s *server) initTLS() error {
+	if s.cfg.TLSCert == "" && s.cfg.TLSKey == "" && len(s.cfg.TLSCertBody) == 0 && len(s.cfg.TLSKeyBody) == 0 {
+		return nil
+	}
+
+	if (len(s.cfg.TLSCertBody) > 0) != (len(s.cfg.TLSKeyBody) > 0) {
+		return ErrTLSBodyMismatch
+	}
+
+	if (s.cfg.TLSCert != "") != (s.cfg.TLSKey != "") {
+		return ErrTLSFileMismatch
+	}
+
+	var cert tls.Certificate
+	var err error
+	if len(s.cfg.TLSCertBody) > 0 {
+		cert, err = tls.X509KeyPair(s.cfg.TLSCertBody, s.cfg.TLSKeyBody)
+	} else {
+		cert, err = tls.LoadX509KeyPair(s.cfg.TLSCert, s.cfg.TLSKey)
+	}
+	if err != nil {
+		return err
+	}
+
+	var minTLSVersion uint16
+	switch s.cfg.MinTLSVersion {
+	case "tls1.0":
+		minTLSVersion = tls.VersionTLS10
+	case "tls1.1":
+		minTLSVersion = tls.VersionTLS11
+	case "tls1.2":
+		minTLSVersion = tls.VersionTLS12
+	case "tls1.3":
+		minTLSVersion = tls.VersionTLS13
+	default:
+		return fmt.Errorf("%w: %s", ErrInvalidMinTLSVersion, s.cfg.MinTLSVersion)
+	}
+
+	s.tlsConfig = &tls.Config{
+		MinVersion:   minTLSVersion,
+		Certificates: []tls.Certificate{cert},
+	}
+
+	if s.cfg.ClientCA != "" {
+		// if !useTLS {
+		// 	err := errors.New("can't use --client-ca without --cert and --key")
+		// 	log.Fatalf(err.Error())
+		// }
+		certpool := x509.NewCertPool()
+		pem, err := os.ReadFile(s.cfg.ClientCA)
+		if err != nil {
+			return err
+		}
+
+		if !certpool.AppendCertsFromPEM(pem) {
+			return ErrTLSParseCA
+		}
+
+		s.tlsConfig.ClientCAs = certpool
+		s.tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert
+	}
+
+	return nil
+}
+
+// Serve starts the HTTP server on each listener
+func (s *server) Serve() {
+	s.wg.Add(len(s.instances))
+	for _, ii := range s.instances {
+		// TODO: decide how/when to log listening url
+		// log.Printf("listening on %s", ii.url)
+		go ii.serve(&s.wg)
+	}
+}
+
+// Wait blocks while the server is serving requests
+func (s *server) Wait() {
+	s.wg.Wait()
+}
+
+// Router returns the server base router
+func (s *server) Router() chi.Router {
+	return s.mux
+}
+
+// Shutdown gracefully shuts down the server
+func (s *server) Shutdown() error {
+	ctx := context.Background()
+	for _, ii := range s.instances {
+		if err := ii.httpServer.Shutdown(ctx); err != nil {
+			log.Printf("error shutting down server: %s", err)
+			continue
+		}
+	}
+	s.wg.Wait()
+	return nil
+}
+
+// HTMLTemplate returns the parsed template, if WithTemplate option was passed.
+func (s *server) HTMLTemplate() *template.Template {
+	return s.htmlTemplate
+}
+
+// URLs returns all configured URLS
+func (s *server) URLs() []string {
+	var out []string
+	for _, ii := range s.instances {
+		if ii.listener.Addr().Network() == "unix" {
+			continue
+		}
+		out = append(out, ii.url)
+	}
+	return out
+}
diff --git a/lib/http/server_test.go b/lib/http/server_test.go
new file mode 100644
index 0000000000000..884214f43802c
--- /dev/null
+++ b/lib/http/server_test.go
@@ -0,0 +1,444 @@
+package http
+
+import (
+	"context"
+	"crypto/tls"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func testEchoHandler(data []byte) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write(data)
+	})
+}
+
+func testExpectRespBody(t *testing.T, resp *http.Response, expected []byte) {
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, expected, body)
+}
+
+func testGetServerURL(t *testing.T, s Server) string {
+	urls := s.URLs()
+	require.GreaterOrEqual(t, len(urls), 1, "server should return at least one url")
+	return urls[0]
+}
+
+func testNewHTTPClientUnix(path string) *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial("unix", path)
+			},
+		},
+	}
+}
+
+func testReadTestdataFile(t *testing.T, path string) []byte {
+	data, err := os.ReadFile(filepath.Join("./testdata", path))
+	require.NoError(t, err, "")
+	return data
+}
+
+func TestNewServerUnix(t *testing.T) {
+	ctx := context.Background()
+
+	tempDir := t.TempDir()
+	path := filepath.Join(tempDir, "rclone.sock")
+
+	cfg := DefaultHTTPCfg()
+	cfg.ListenAddr = []string{path}
+
+	auth := AuthConfig{
+		BasicUser: "test",
+		BasicPass: "test",
+	}
+
+	s, err := NewServer(ctx, WithConfig(cfg), WithAuth(auth))
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, s.Shutdown())
+		_, err := os.Stat(path)
+		require.ErrorIs(t, err, os.ErrNotExist, "shutdown should remove socket")
+	}()
+
+	require.Empty(t, s.URLs(), "unix socket should not appear in URLs")
+
+	s.Router().Use(MiddlewareCORS(""))
+
+	expected := []byte("hello world")
+	s.Router().Mount("/", testEchoHandler(expected))
+	s.Serve()
+
+	client := testNewHTTPClientUnix(path)
+	req, err := http.NewRequest("GET", "http://unix", nil)
+	require.NoError(t, err)
+
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+
+	testExpectRespBody(t, resp, expected)
+
+	require.Equal(t, http.StatusOK, resp.StatusCode, "unix sockets should ignore auth")
+
+	for _, key := range _testCORSHeaderKeys {
+		require.NotContains(t, resp.Header, key, "unix sockets should not be sent CORS headers")
+	}
+}
+
+func TestNewServerHTTP(t *testing.T) {
+	ctx := context.Background()
+
+	cfg := DefaultHTTPCfg()
+	cfg.ListenAddr = []string{"127.0.0.1:0"}
+
+	auth := AuthConfig{
+		BasicUser: "test",
+		BasicPass: "test",
+	}
+
+	s, err := NewServer(ctx, WithConfig(cfg), WithAuth(auth))
+	require.NoError(t, err)
+	defer func() {
+		require.NoError(t, s.Shutdown())
+	}()
+
+	url := testGetServerURL(t, s)
+	require.True(t, strings.HasPrefix(url, "http://"), "url should have http scheme")
+
+	expected := []byte("hello world")
+	s.Router().Mount("/", testEchoHandler(expected))
+	s.Serve()
+
+	t.Run("StatusUnauthorized", func(t *testing.T) {
+		client := &http.Client{}
+		req, err := http.NewRequest("GET", url, nil)
+		require.NoError(t, err)
+
+		resp, err := client.Do(req)
+		require.NoError(t, err)
+		defer func() {
+			_ = resp.Body.Close()
+		}()
+
+		require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "no basic auth creds should return unauthorized")
+	})
+
+	t.Run("StatusOK", func(t *testing.T) {
+		client := &http.Client{}
+		req, err := http.NewRequest("GET", url, nil)
+		require.NoError(t, err)
+
+		req.SetBasicAuth(auth.BasicUser, auth.BasicPass)
+
+		resp, err := client.Do(req)
+		require.NoError(t, err)
+		defer func() {
+			_ = resp.Body.Close()
+		}()
+
+		require.Equal(t, http.StatusOK, resp.StatusCode, "using basic auth creds should return ok")
+
+		testExpectRespBody(t, resp, expected)
+	})
+}
+func TestNewServerBaseURL(t *testing.T) {
+	servers := []struct {
+		name   string
+		http   HTTPConfig
+		suffix string
+	}{
+		{
+			name: "Empty",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+				BaseURL:    "",
+			},
+			suffix: "/",
+		},
+		{
+			name: "Single/NoTrailingSlash",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+				BaseURL:    "/rclone",
+			},
+			suffix: "/rclone/",
+		},
+		{
+			name: "Single/TrailingSlash",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+				BaseURL:    "/rclone/",
+			},
+			suffix: "/rclone/",
+		},
+		{
+			name: "Multi/NoTrailingSlash",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+				BaseURL:    "/rclone/test/base/url",
+			},
+			suffix: "/rclone/test/base/url/",
+		},
+		{
+			name: "Multi/TrailingSlash",
+			http: HTTPConfig{
+				ListenAddr: []string{"127.0.0.1:0"},
+				BaseURL:    "/rclone/test/base/url/",
+			},
+			suffix: "/rclone/test/base/url/",
+		},
+	}
+
+	for _, ss := range servers {
+		t.Run(ss.name, func(t *testing.T) {
+			s, err := NewServer(context.Background(), WithConfig(ss.http))
+			require.NoError(t, err)
+			defer func() {
+				require.NoError(t, s.Shutdown())
+			}()
+
+			expected := []byte("data")
+			s.Router().Get("/", testEchoHandler(expected).ServeHTTP)
+			s.Serve()
+
+			url := testGetServerURL(t, s)
+			require.True(t, strings.HasPrefix(url, "http://"), "url should have http scheme")
+			require.True(t, strings.HasSuffix(url, ss.suffix), "url should have the expected suffix")
+
+			client := &http.Client{}
+			req, err := http.NewRequest("GET", url, nil)
+			require.NoError(t, err)
+
+			resp, err := client.Do(req)
+			require.NoError(t, err)
+			defer func() {
+				_ = resp.Body.Close()
+			}()
+
+			t.Log(url, resp.Request.URL)
+
+			require.Equal(t, http.StatusOK, resp.StatusCode, "should return ok")
+
+			testExpectRespBody(t, resp, expected)
+		})
+	}
+}
+
+func TestNewServerTLS(t *testing.T) {
+	certBytes := testReadTestdataFile(t, "local.crt")
+	keyBytes := testReadTestdataFile(t, "local.key")
+
+	// TODO: generate a proper cert with SAN
+	// TODO: generate CA, test mTLS
+	// clientCert, err := tls.X509KeyPair(certBytes, keyBytes)
+	// require.NoError(t, err, "should be testing with a valid self signed certificate")
+
+	servers := []struct {
+		name    string
+		wantErr bool
+		err     error
+		http    HTTPConfig
+	}{
+		{
+			name: "FromFile/Valid",
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCert:       "./testdata/local.crt",
+				TLSKey:        "./testdata/local.key",
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromFile/NoCert",
+			wantErr: true,
+			err:     ErrTLSFileMismatch,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCert:       "",
+				TLSKey:        "./testdata/local.key",
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromFile/InvalidCert",
+			wantErr: true,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCert:       "./testdata/local.crt.invalid",
+				TLSKey:        "./testdata/local.key",
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromFile/NoKey",
+			wantErr: true,
+			err:     ErrTLSFileMismatch,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCert:       "./testdata/local.crt",
+				TLSKey:        "",
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromFile/InvalidKey",
+			wantErr: true,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCert:       "./testdata/local.crt",
+				TLSKey:        "./testdata/local.key.invalid",
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name: "FromBody/Valid",
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromBody/NoCert",
+			wantErr: true,
+			err:     ErrTLSBodyMismatch,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   nil,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromBody/InvalidCert",
+			wantErr: true,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   []byte("JUNK DATA"),
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromBody/NoKey",
+			wantErr: true,
+			err:     ErrTLSBodyMismatch,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    nil,
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name:    "FromBody/InvalidKey",
+			wantErr: true,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    []byte("JUNK DATA"),
+				MinTLSVersion: "tls1.0",
+			},
+		},
+		{
+			name: "MinTLSVersion/Valid/1.1",
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.1",
+			},
+		},
+		{
+			name: "MinTLSVersion/Valid/1.2",
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.2",
+			},
+		},
+		{
+			name: "MinTLSVersion/Valid/1.3",
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls1.3",
+			},
+		},
+		{
+			name:    "MinTLSVersion/Invalid",
+			wantErr: true,
+			err:     ErrInvalidMinTLSVersion,
+			http: HTTPConfig{
+				ListenAddr:    []string{"127.0.0.1:0"},
+				TLSCertBody:   certBytes,
+				TLSKeyBody:    keyBytes,
+				MinTLSVersion: "tls0.9",
+			},
+		},
+	}
+
+	for _, ss := range servers {
+		t.Run(ss.name, func(t *testing.T) {
+			s, err := NewServer(context.Background(), WithConfig(ss.http))
+			if ss.wantErr == true {
+				if ss.err != nil {
+					require.ErrorIs(t, err, ss.err, "new server should return the expected error")
+				} else {
+					require.Error(t, err, "new server should return error for invalid TLS config")
+				}
+				return
+			}
+
+			require.NoError(t, err)
+			defer func() {
+				require.NoError(t, s.Shutdown())
+			}()
+
+			expected := []byte("secret-page")
+			s.Router().Mount("/", testEchoHandler(expected))
+			s.Serve()
+
+			url := testGetServerURL(t, s)
+			require.True(t, strings.HasPrefix(url, "https://"), "url should have https scheme")
+
+			client := &http.Client{
+				Transport: &http.Transport{
+					DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+						dest := strings.TrimPrefix(url, "https://")
+						dest = strings.TrimSuffix(dest, "/")
+						return net.Dial("tcp", dest)
+					},
+					TLSClientConfig: &tls.Config{
+						// Certificates: []tls.Certificate{clientCert},
+						InsecureSkipVerify: true,
+					},
+				},
+			}
+			req, err := http.NewRequest("GET", "https://dev.rclone.org", nil)
+			require.NoError(t, err)
+
+			resp, err := client.Do(req)
+			require.NoError(t, err)
+			defer func() {
+				_ = resp.Body.Close()
+			}()
+
+			require.Equal(t, http.StatusOK, resp.StatusCode, "should return ok")
+
+			testExpectRespBody(t, resp, expected)
+		})
+	}
+}
diff --git a/lib/http/template.go b/lib/http/template.go
new file mode 100644
index 0000000000000..bed80dc308b0b
--- /dev/null
+++ b/lib/http/template.go
@@ -0,0 +1,95 @@
+package http
+
+import (
+	"embed"
+	"html/template"
+	"os"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"github.com/rclone/rclone/fs/config/flags"
+)
+
+// TemplateHelp describes how to use a custom template
+var TemplateHelp = `
+#### Template
+
+` + "`--template`" + ` allows a user to specify a custom markup template for HTTP
+and WebDAV serve functions.  The server exports the following markup
+to be used within the template to server pages:
+
+| Parameter   | Description |
+| :---------- | :---------- |
+| .Name       | The full path of a file/directory. |
+| .Title      | Directory listing of .Name |
+| .Sort       | The current sort used.  This is changeable via ?sort= parameter |
+|             | Sort Options: namedirfirst,name,size,time (default namedirfirst) |
+| .Order      | The current ordering used.  This is changeable via ?order= parameter |
+|             | Order Options: asc,desc (default asc) |
+| .Query      | Currently unused. |
+| .Breadcrumb | Allows for creating a relative navigation |
+|-- .Link     | The relative to the root link of the Text. |
+|-- .Text     | The Name of the directory. |
+| .Entries    | Information about a specific file/directory. |
+|-- .URL      | The 'url' of an entry.  |
+|-- .Leaf     | Currently same as 'URL' but intended to be 'just' the name. |
+|-- .IsDir    | Boolean for if an entry is a directory or not. |
+|-- .Size     | Size in Bytes of the entry. |
+|-- .ModTime  | The UTC timestamp of an entry. |
+`
+
+// TemplateConfig for the templating functionality
+type TemplateConfig struct {
+	Path string
+}
+
+// AddFlagsPrefix for the templating functionality
+func (cfg *TemplateConfig) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) {
+	flags.StringVarP(flagSet, &cfg.Path, prefix+"template", "", cfg.Path, "User-specified template")
+}
+
+// AddTemplateFlagsPrefix for the templating functionality
+func AddTemplateFlagsPrefix(flagSet *pflag.FlagSet, prefix string, cfg *TemplateConfig) {
+	cfg.AddFlagsPrefix(flagSet, prefix)
+}
+
+// DefaultTemplateCfg returns a new config which can be customized by command line flags
+func DefaultTemplateCfg() TemplateConfig {
+	return TemplateConfig{}
+}
+
+// AfterEpoch returns the time since the epoch for the given time
+func AfterEpoch(t time.Time) bool {
+	return t.After(time.Time{})
+}
+
+// Assets holds the embedded filesystem for the default template
+//
+//go:embed templates
+var Assets embed.FS
+
+// GetTemplate returns the HTML template for serving directories via HTTP/WebDAV
+func GetTemplate(tmpl string) (*template.Template, error) {
+	var readFile = os.ReadFile
+	if tmpl == "" {
+		tmpl = "templates/index.html"
+		readFile = Assets.ReadFile
+	}
+
+	data, err := readFile(tmpl)
+	if err != nil {
+		return nil, err
+	}
+
+	funcMap := template.FuncMap{
+		"afterEpoch": AfterEpoch,
+	}
+
+	tpl, err := template.New("index").Funcs(funcMap).Parse(string(data))
+	if err != nil {
+		return nil, err
+	}
+
+	return tpl, nil
+}
diff --git a/lib/http/templates/index.html b/lib/http/templates/index.html
new file mode 100644
index 0000000000000..348050c0278e4
--- /dev/null
+++ b/lib/http/templates/index.html
@@ -0,0 +1,389 @@
+
+
+
+
+	
+		{{html .Name}}
+		
+		
+		
+
+	
+	
+		
+			
+				
+				
+					
+					
+				
+				
+					
+						
+							
+							
+						
+						
+					
+				
+				
+				
+					
+					
+				
+				
+					
+						
+							
+							
+						
+						
+					
+				
+				
+				
+					
+				
+				
+				
+					
+				
+			
+		
+		
                  
+			
                  
+				{{range $i, $crumb := .Breadcrumb}}{{html $crumb.Text}}{{if ne $i 0}}/{{end}}{{end}}
+			
                  
+		
                  
+		
                  
+			
                  
+				
                  
+					
+				
                  
+			
                  
+			
                  
+				
+					
+					
+						
+						
+						
+						
+						
+					
+					
+					
+					
+						
+						
+						
+						
+						
+					
+					{{- range .Entries}}
+					
+						
+						
+						{{- if .IsDir}}
+						
+						{{- else}}
+						
+						{{- end}}
+						{{- if .ModTime | afterEpoch }}
+						
+						{{- else}}
+						
+						{{- end}}
+						
+					
+					{{- end}}
+					
+				
                  
+							
+							
+							Name
+						
+							Size
+						
+							Modified
+						
                  
+							
+								Go up
+							
+						——
                  
+						
+							{{- if .IsDir}}
+							
+							{{- else}}
+							
+							{{- end}}
+							{{html .Leaf}}
+						—{{.Size}}—
                  
+			
                  
+		
                  
+		
+	
+
diff --git a/lib/http/testdata/.htpasswd b/lib/http/testdata/.htpasswd
new file mode 100644
index 0000000000000..5d0c17779af81
--- /dev/null
+++ b/lib/http/testdata/.htpasswd
@@ -0,0 +1,3 @@
+sha:{SHA}2PRZAyDhNDqRW2OUFwZQqPNdaSY=
+md5:$apr1$s7fogein$IK9ItbnGM14ct0bY4Uyik1
+bcrypt:$2y$10$K/b3mVXUA6X857TOTYIL9.Lbaeg9oBjMQwUX5NefpVUCcYP0Z5KY2
\ No newline at end of file
diff --git a/lib/http/testdata/local.crt b/lib/http/testdata/local.crt
new file mode 100644
index 0000000000000..c2bcf28fea348
--- /dev/null
+++ b/lib/http/testdata/local.crt
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFeTCCA2GgAwIBAgIUewDoUKLIPBZlSAdOaBHPtfFDiUYwDQYJKoZIhvcNAQEL
+BQAwTDELMAkGA1UEBhMCVVMxDzANBgNVBAoMBnJjbG9uZTETMBEGA1UECwwKcmNs
+b25lLWRldjEXMBUGA1UEAwwOZGV2LnJjbG9uZS5vcmcwHhcNMjIxMDI3MDM1NjE2
+WhcNMzIxMDI0MDM1NjE2WjBMMQswCQYDVQQGEwJVUzEPMA0GA1UECgwGcmNsb25l
+MRMwEQYDVQQLDApyY2xvbmUtZGV2MRcwFQYDVQQDDA5kZXYucmNsb25lLm9yZzCC
+AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAJVMPbsmBVBi2DG17cPrJQxM
+hdUxkd8pcRtrKkLzqIQS18IKJ20BTzrVlJJETO1uMeXETSVVQ4znWLvhZo5Pgq72
+FlvEMEl6QXNNihEB+Bx9f6iArS0Tgo919Kg3JEYu9g3HfsPxt3C1xNJOlSrpPo4w
+YAX8MU+uy3IqPwgURVhPauR/2E64b9RSIsmNCJJXWnREeJVOpeYSUXx6S0fNM/wo
+BvbPSuB1BErYug9AfuvTavJO7VYnMuEEig/1DXYnPJpTTaZWzyhQIGDU52T5iOZk
+sdxv/Yxbq1kKDHmyhG9ALpCHqQMmR3bOEhLijU5UHfFR03eKnzfcQ7y5BMJBDIVs
+fbaBxlJZFHKq9ixUT/7crc1iUXb+gPg/FxMRuR96/fTPrluWlDYaZV3QMa3rxuvn
+PJM5boqGIH9fNxnsmho0V4ZrXgxg5VsnlwHZIlMuJVJGkUQFTl6XhIzkQRmjQM4C
+WO9KPSusvCM3gCM7j9Tyhi3xl7XRVlN9xb8vbnbyqcZe0lgJU/y3nrbWqORvUKb0
+FvarmtIH3f9yqu+UB2s8DIg5zPjVmTnIMSJYJi5Cjh1YT2YVJ4+JJ0KlxAwOVTU4
+zdup3fUA3Ne2o9ehJEJwuFPbMDCEHeIuTzOKleKCDbHRIvFLleJP4J9/qA1P2C2/
+ZAoS4M8vgd2O84eVm7pzAgMBAAGjUzBRMB0GA1UdDgQWBBTDsVFbaf4C4yDDOGml
+BMyKSgsT5DAfBgNVHSMEGDAWgBTDsVFbaf4C4yDDOGmlBMyKSgsT5DAPBgNVHRMB
+Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQAx2wDldW5wZfBGdIv9UCTL1hdB
+a858v+mWCAjXHaFCyMAe2UOBkSF4M8oIqgMXP3ZwysAhB54qKcpKgxKv6s5oyA/k
+diheH8Zl7GYnUS+A1v9ZUx5AwJbVNKCSXYfXQxTsoC1jV0W3HqeJIoxix3XJDZIL
+TCiA8+BMk72vtJBPNlV8wmuN2+aUEktZF1PqjbqU/6ajRvDIa4ogXIVoSunpFXt9
+h5Af2zOE051zqjqshr9Egz6Hl2EItjJgqZtwLD8qSZFzcXYgDvtG16YceGt+ljeC
+yLYE8Qvm9lfQka9Sxucu4+2kzCwjg5ubFaaSkiX6b/ue9KvPHm36JAV9NDwyyA/q
+BNNxK+0PniNPLBdxIkFZvVraLieDNCXV3cqH7781IP2PRvbzNAR1PFfo19U7bmHV
+PsOBj9kIQBhOLcxtxWvK93ptC6vLJYRsPha7kClVI5kht9oB8Mkfi2mAu2Pi7Pka
+ZYHl14XnJPUWEbxX26I4CAU09yGjnQhRwPfGGNPCaMMGsYl+2nn4j/rzbQ8uz7Kg
+l1TQS4WBpX4T6pxWM/mWERbBTLxniE8DNYPgHpgZJSXD16uG/ksyjdet7B86KA0v
+kymH793pOqW8rtrKbziSZyShJy5AYsGy0Xu4ymW03F8S5FUyjWIkRmyUbGTB1q02
+nniyTJh2BffUcH2iCg==
+-----END CERTIFICATE-----
diff --git a/lib/http/testdata/local.key b/lib/http/testdata/local.key
new file mode 100644
index 0000000000000..e855ac9fe613a
--- /dev/null
+++ b/lib/http/testdata/local.key
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCVTD27JgVQYtgx
+te3D6yUMTIXVMZHfKXEbaypC86iEEtfCCidtAU861ZSSREztbjHlxE0lVUOM51i7
+4WaOT4Ku9hZbxDBJekFzTYoRAfgcfX+ogK0tE4KPdfSoNyRGLvYNx37D8bdwtcTS
+TpUq6T6OMGAF/DFPrstyKj8IFEVYT2rkf9hOuG/UUiLJjQiSV1p0RHiVTqXmElF8
+ektHzTP8KAb2z0rgdQRK2LoPQH7r02ryTu1WJzLhBIoP9Q12JzyaU02mVs8oUCBg
+1Odk+YjmZLHcb/2MW6tZCgx5soRvQC6Qh6kDJkd2zhIS4o1OVB3xUdN3ip833EO8
+uQTCQQyFbH22gcZSWRRyqvYsVE/+3K3NYlF2/oD4PxcTEbkfev30z65blpQ2GmVd
+0DGt68br5zyTOW6KhiB/XzcZ7JoaNFeGa14MYOVbJ5cB2SJTLiVSRpFEBU5el4SM
+5EEZo0DOAljvSj0rrLwjN4AjO4/U8oYt8Ze10VZTfcW/L2528qnGXtJYCVP8t562
+1qjkb1Cm9Bb2q5rSB93/cqrvlAdrPAyIOcz41Zk5yDEiWCYuQo4dWE9mFSePiSdC
+pcQMDlU1OM3bqd31ANzXtqPXoSRCcLhT2zAwhB3iLk8zipXigg2x0SLxS5XiT+Cf
+f6gNT9gtv2QKEuDPL4HdjvOHlZu6cwIDAQABAoICAD7EzqFb01kgLZf8zqmTt8BL
+fesLy7IA6OpnrF14tq1MhMSyYzALoGVyfWPfbl5WeYkJ9otPJTbc3ywikG0dlap8
+kRrkyY5i5ZiWDYmoA8nqo5zS+LweW0J4i7Obd1dAkDdr2+qCuiabbVQkMMfZR3Ed
+eomZpZvEOAnYJCb/6sW9ognOjEFQfsfL/o8xidyI+GEwlmfjqJEpu3OzsOnPpt8J
+byAeN/NVj0fuhY87BQGeIfUc5ODXWydKsscRtqapyWtywY9BKRhgU7SSXnTQCtQe
+mr68oON9ePVW6bbSrKZfXBRszMyjr+ENs4CYGmPHrs0SI2+7asRgCWSTfyIymhHs
+DjF6BHAlQaOLnVcffuGhc6c0bnzIFvg1JenWVj9uQSkADW8TMkNgMLdcx8JoMd30
+9HiHZyQy7hMuzWjZSgS1wRp3KMGBo+WuMd4NF662nQPfylG2iJvedLtbvY7mBGkX
+1e6L4b1fkFAZWyfBj3JX1SqgcNDnqzDiqqUDwpY8b1G9nAegvWBeFbEgnORa6pVF
+8xWNtWlMsXYjTvPLbvDSZc6l0wOpFw7roGZgQpbaTjmbeqrZMsr/XDtH7ksLCBOh
+V/s2oMU0NUBu/mokvpLupBzVMbA7/TKGr36OrHw57P8dhsdD3bqx+UbUMgjsq6xs
+UAMcjWJFoc3ulgsVHF2BAoIBAQDEo5i63JkQL3+8rXQ25StbEgqVGXcGS2oZLu+G
+GYklJ0NyGrjoYWmZ/QQf9NbAhQKDSe/X2cWtTLMF9Y2yVsxQ2x1df3lonk9dHsoX
+O+Yp8C9zKt24dRWUsyklgkxAmb/nq3dRPtm/80ztNVQvYy5SQ7G5s/suXt9Wyd2y
+nl08ZWclaDkHNS33/8hYLHP3RkjLgcOu45UDb/3vXk6ljnlrnqe7nlRX5UrnVXs3
+K41xJpCSHmUe7SUoW1ExjzWhtQBkWWVWWV8gE8CNpU0nheTkA5KyXQ8QfQ13jxJj
+zPy/3OPGMIElbvEBcXxvcRncpuUIVO4tRy+oZuz2bmLwbBiBAoIBAQDCXhS5wMgJ
+fWNzsd9prFrENVz7CR2NK3LSdVpm4ZtR5f4mLyzYj/RcSIYC4U7uO0WXaoLTnhKs
+KbKAj5KC2HZUcqhLscajXPl6OjBOcEAlujh9VeTjEnx6XU81IwmAjrYr98xAz3HJ
+IOaD8MvfZ13JWDB9q5bq3A8EKTEAPh66mv/w4Ic8rnueFZgxEFpD4Z2bSB7xmEHj
+tE7VuUbdkLQ4jDrxwZuj7vC0+50pqDrSe0VY/KaGOnNtF1fVs3N+xlI9yUBhr9Yy
+ANqUlx8Ql0i2aIuTSR9h/Wk9uu3NmsON9FkLoYK4bg9jiMcA6eKyRK/9Txn/CygG
+zNy3+s2RhHjzAoIBAQCCZ5Pz6DPB3h4yPD2j4hr8jFxkQL0EeaLlDJFgNzMSZpV9
+6GbUBTYJHxhLMQ3yIsNl2fSrCwrjQMhAnXXY3WMmBAnXZaBYVxR+xtpyyhB7o4N0
+NutPVqZ3NNGGxIBZHx17P+UjBjFV8L4FWaZ4vqeLesU0SD29pMEsRzc1K3zdfsoG
+rrWTKBtSKljs0J4fUIcaHvZs1xSNcQnQYpR5iqDPVCocbIW2vKMOA0xxa/qjHVYm
+8O1SsyY/Oz//Q9/nW6fk5LwlpaNGHJNH3GXsXglLhWsVyk0hPC1gKouhj+HWQ2Dy
+oFwlPQurT12ccj8aa7vb6KcDdAARCCEB1HbcxnMBAoIBADnY6FA0eRSh9eRsDvMT
+cdwtiaPJHbtzL/RFKweto51nVxGkPrOhfHeuufvHdMdgaqDa+V7kD+ifbFno4REC
+PY16pm4I1fau6C0hflkJ/X19A+0BkGKokNWWScmlyOEzGDLTyD2Nv+69VP31v6eY
+ywfusFfmpr71iZ6SZ9wLoPemw/+7w2QjBfWRtb78f/DuCAs8FsGOsCWF92SShO3S
+cGDYE376QUk0Bv3GWQsZ34/fUk9eumz+nnXcWa7nfrs/aSCscfXg8F3ndSZ+J6e3
+btOjH89RFv8B/b16keX8ZrEsBQh6JD6huwDDp361HVwzJzG7xh/rARmtBQ/YnC/v
+/lMCggEAZ+EA8vRD7KSDLaxUl/DG0VQYtyJWsf83/NcLAizvYirI9batLjUF8ogy
+pNHmA1STVJko+P+M2V5v56lJWnrj6HPj91NPIqdgJ/hIIZNI1zXm/DlLOegrFH0W
+7fIY3+ZxzCIMuN4TvZ2H7n6NDBvTr6Vno17nFRGzy1suceIk7tx66KiFGmVt0sT/
+yyCQdyk+uM0KG3j19+QDrbtOTTmK3cOHTEOs7D2RmHSbn51jOckzL2mrhXvcGfTS
+cqYW21Gm9U2P7VjgQ9vHuGzUFIrskltAGuX38XdPiBu9vZp1hwPbYY9MLxrbW8UM
+ySc0NWyPv8/wyfen4dBRJ9OthwwcXw==
+-----END PRIVATE KEY-----

From 2a2fcf101212f0f84098743b6e948661b8502d40 Mon Sep 17 00:00:00 2001
From: Tom Mombourquette 
Date: Tue, 8 Nov 2022 08:03:26 -0400
Subject: [PATCH 461/560] lib/http: rationalise names in test servers to be
 more consistent

---
 cmd/serve/http/http.go      |  4 ++--
 cmd/serve/http/http_test.go |  2 +-
 lib/http/auth.go            |  3 +--
 lib/http/middleware.go      |  1 +
 lib/http/middleware_test.go | 18 +++++++-------
 lib/http/server.go          | 33 +++++++++++++------------
 lib/http/server_test.go     | 48 ++++++++++++++++++-------------------
 7 files changed, 56 insertions(+), 53 deletions(-)

diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go
index 594ac551c8bb3..d88714dddb7e3 100644
--- a/cmd/serve/http/http.go
+++ b/cmd/serve/http/http.go
@@ -29,14 +29,14 @@ import (
 // Options required for http server
 type Options struct {
 	Auth     libhttp.AuthConfig
-	HTTP     libhttp.HTTPConfig
+	HTTP     libhttp.Config
 	Template libhttp.TemplateConfig
 }
 
 // DefaultOpt is the default values used for Options
 var DefaultOpt = Options{
 	Auth:     libhttp.DefaultAuthCfg(),
-	HTTP:     libhttp.DefaultHTTPCfg(),
+	HTTP:     libhttp.DefaultCfg(),
 	Template: libhttp.DefaultTemplateCfg(),
 }
 
diff --git a/cmd/serve/http/http_test.go b/cmd/serve/http/http_test.go
index 9f264adce9897..f285c126148db 100644
--- a/cmd/serve/http/http_test.go
+++ b/cmd/serve/http/http_test.go
@@ -34,7 +34,7 @@ func start(t *testing.T, f fs.Fs) {
 	ctx := context.Background()
 
 	opts := Options{
-		HTTP: libhttp.DefaultHTTPCfg(),
+		HTTP: libhttp.DefaultCfg(),
 		Template: libhttp.TemplateConfig{
 			Path: testTemplate,
 		},
diff --git a/lib/http/auth.go b/lib/http/auth.go
index b7784c4b44aa4..50b024f707790 100644
--- a/lib/http/auth.go
+++ b/lib/http/auth.go
@@ -5,8 +5,7 @@ import (
 	"github.com/spf13/pflag"
 )
 
-// Help contains text describing the http authentication to add to the command
-// help.
+// AuthHelp contains text describing the http authentication to add to the command help.
 var AuthHelp = `
 #### Authentication
 
diff --git a/lib/http/middleware.go b/lib/http/middleware.go
index d19d75259b4a8..cef2485d68c81 100644
--- a/lib/http/middleware.go
+++ b/lib/http/middleware.go
@@ -33,6 +33,7 @@ func parseAuthorization(r *http.Request) (user, pass string, ok bool) {
 	return
 }
 
+// LoggedBasicAuth simply wraps the goauth.BasicAuth struct
 type LoggedBasicAuth struct {
 	goauth.BasicAuth
 }
diff --git a/lib/http/middleware_test.go b/lib/http/middleware_test.go
index 4839b87e31510..8932ca4fd1b95 100644
--- a/lib/http/middleware_test.go
+++ b/lib/http/middleware_test.go
@@ -13,14 +13,14 @@ import (
 func TestMiddlewareAuth(t *testing.T) {
 	servers := []struct {
 		name string
-		http HTTPConfig
+		http Config
 		auth AuthConfig
 		user string
 		pass string
 	}{
 		{
 			name: "Basic",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 			},
 			auth: AuthConfig{
@@ -33,7 +33,7 @@ func TestMiddlewareAuth(t *testing.T) {
 		},
 		{
 			name: "Htpasswd/MD5",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 			},
 			auth: AuthConfig{
@@ -45,7 +45,7 @@ func TestMiddlewareAuth(t *testing.T) {
 		},
 		{
 			name: "Htpasswd/SHA",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 			},
 			auth: AuthConfig{
@@ -57,7 +57,7 @@ func TestMiddlewareAuth(t *testing.T) {
 		},
 		{
 			name: "Htpasswd/Bcrypt",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 			},
 			auth: AuthConfig{
@@ -69,7 +69,7 @@ func TestMiddlewareAuth(t *testing.T) {
 		},
 		{
 			name: "Custom",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 			},
 			auth: AuthConfig{
@@ -168,19 +168,19 @@ var _testCORSHeaderKeys = []string{
 func TestMiddlewareCORS(t *testing.T) {
 	servers := []struct {
 		name   string
-		http   HTTPConfig
+		http   Config
 		origin string
 	}{
 		{
 			name: "EmptyOrigin",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 			},
 			origin: "",
 		},
 		{
 			name: "CustomOrigin",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 			},
 			origin: "http://test.rclone.org",
diff --git a/lib/http/server.go b/lib/http/server.go
index d4e5e827b0565..4dbc31c8ef17f 100644
--- a/lib/http/server.go
+++ b/lib/http/server.go
@@ -76,8 +76,8 @@ certificate authority certificate.
 // Middleware function signature required by chi.Router.Use()
 type Middleware func(http.Handler) http.Handler
 
-// HTTPConfig contains options for the http Server
-type HTTPConfig struct {
+// Config contains options for the http Server
+type Config struct {
 	ListenAddr         []string      // Port to listen on
 	BaseURL            string        // prefix to strip from URLs
 	ServerReadTimeout  time.Duration // Timeout for server reading data
@@ -93,7 +93,7 @@ type HTTPConfig struct {
 }
 
 // AddFlagsPrefix adds flags for the httplib
-func (cfg *HTTPConfig) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) {
+func (cfg *Config) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) {
 	flags.StringArrayVarP(flagSet, &cfg.ListenAddr, prefix+"addr", "", cfg.ListenAddr, "IPaddress:Port or :Port to bind server to")
 	flags.DurationVarP(flagSet, &cfg.ServerReadTimeout, prefix+"server-read-timeout", "", cfg.ServerReadTimeout, "Timeout for server reading data")
 	flags.DurationVarP(flagSet, &cfg.ServerWriteTimeout, prefix+"server-write-timeout", "", cfg.ServerWriteTimeout, "Timeout for server writing data")
@@ -106,13 +106,13 @@ func (cfg *HTTPConfig) AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string) {
 }
 
 // AddHTTPFlagsPrefix adds flags for the httplib
-func AddHTTPFlagsPrefix(flagSet *pflag.FlagSet, prefix string, cfg *HTTPConfig) {
+func AddHTTPFlagsPrefix(flagSet *pflag.FlagSet, prefix string, cfg *Config) {
 	cfg.AddFlagsPrefix(flagSet, prefix)
 }
 
-// DefaultHTTPCfg is the default values used for Config
-func DefaultHTTPCfg() HTTPConfig {
-	return HTTPConfig{
+// DefaultCfg is the default values used for Config
+func DefaultCfg() Config {
+	return Config{
 		ListenAddr:         []string{"127.0.0.1:8080"},
 		ServerReadTimeout:  1 * time.Hour,
 		ServerWriteTimeout: 1 * time.Hour,
@@ -151,7 +151,7 @@ type server struct {
 	tlsConfig    *tls.Config
 	instances    []instance
 	auth         AuthConfig
-	cfg          HTTPConfig
+	cfg          Config
 	template     *TemplateConfig
 	htmlTemplate *template.Template
 }
@@ -166,8 +166,8 @@ func WithAuth(cfg AuthConfig) Option {
 	}
 }
 
-// WithConfig option applies the HTTPConfig to the server, overriding defaults
-func WithConfig(cfg HTTPConfig) Option {
+// WithConfig option applies the Config to the server, overriding defaults
+func WithConfig(cfg Config) Option {
 	return func(s *server) {
 		s.cfg = cfg
 	}
@@ -187,7 +187,7 @@ func WithTemplate(cfg TemplateConfig) Option {
 func NewServer(ctx context.Context, options ...Option) (*server, error) {
 	s := &server{
 		mux: chi.NewRouter(),
-		cfg: DefaultHTTPCfg(),
+		cfg: DefaultCfg(),
 	}
 
 	for _, opt := range options {
@@ -307,11 +307,14 @@ func (s *server) initTemplate() error {
 }
 
 var (
-	// hard coded errors, allowing for easier testing
+	// ErrInvalidMinTLSVersion - hard coded errors, allowing for easier testing
 	ErrInvalidMinTLSVersion = errors.New("invalid value for --min-tls-version")
-	ErrTLSBodyMismatch      = errors.New("need both TLSCertBody and TLSKeyBody to use TLS")
-	ErrTLSFileMismatch      = errors.New("need both --cert and --key to use TLS")
-	ErrTLSParseCA           = errors.New("unable to parse client certificate authority")
+	// ErrTLSBodyMismatch - hard coded errors, allowing for easier testing
+	ErrTLSBodyMismatch = errors.New("need both TLSCertBody and TLSKeyBody to use TLS")
+	// ErrTLSFileMismatch - hard coded errors, allowing for easier testing
+	ErrTLSFileMismatch = errors.New("need both --cert and --key to use TLS")
+	// ErrTLSParseCA - hard coded errors, allowing for easier testing
+	ErrTLSParseCA = errors.New("unable to parse client certificate authority")
 )
 
 func (s *server) initTLS() error {
diff --git a/lib/http/server_test.go b/lib/http/server_test.go
index 884214f43802c..42c630ea404bd 100644
--- a/lib/http/server_test.go
+++ b/lib/http/server_test.go
@@ -54,7 +54,7 @@ func TestNewServerUnix(t *testing.T) {
 	tempDir := t.TempDir()
 	path := filepath.Join(tempDir, "rclone.sock")
 
-	cfg := DefaultHTTPCfg()
+	cfg := DefaultCfg()
 	cfg.ListenAddr = []string{path}
 
 	auth := AuthConfig{
@@ -97,7 +97,7 @@ func TestNewServerUnix(t *testing.T) {
 func TestNewServerHTTP(t *testing.T) {
 	ctx := context.Background()
 
-	cfg := DefaultHTTPCfg()
+	cfg := DefaultCfg()
 	cfg.ListenAddr = []string{"127.0.0.1:0"}
 
 	auth := AuthConfig{
@@ -153,12 +153,12 @@ func TestNewServerHTTP(t *testing.T) {
 func TestNewServerBaseURL(t *testing.T) {
 	servers := []struct {
 		name   string
-		http   HTTPConfig
+		cfg    Config
 		suffix string
 	}{
 		{
 			name: "Empty",
-			http: HTTPConfig{
+			cfg: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 				BaseURL:    "",
 			},
@@ -166,7 +166,7 @@ func TestNewServerBaseURL(t *testing.T) {
 		},
 		{
 			name: "Single/NoTrailingSlash",
-			http: HTTPConfig{
+			cfg: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 				BaseURL:    "/rclone",
 			},
@@ -174,7 +174,7 @@ func TestNewServerBaseURL(t *testing.T) {
 		},
 		{
 			name: "Single/TrailingSlash",
-			http: HTTPConfig{
+			cfg: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 				BaseURL:    "/rclone/",
 			},
@@ -182,7 +182,7 @@ func TestNewServerBaseURL(t *testing.T) {
 		},
 		{
 			name: "Multi/NoTrailingSlash",
-			http: HTTPConfig{
+			cfg: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 				BaseURL:    "/rclone/test/base/url",
 			},
@@ -190,7 +190,7 @@ func TestNewServerBaseURL(t *testing.T) {
 		},
 		{
 			name: "Multi/TrailingSlash",
-			http: HTTPConfig{
+			cfg: Config{
 				ListenAddr: []string{"127.0.0.1:0"},
 				BaseURL:    "/rclone/test/base/url/",
 			},
@@ -200,7 +200,7 @@ func TestNewServerBaseURL(t *testing.T) {
 
 	for _, ss := range servers {
 		t.Run(ss.name, func(t *testing.T) {
-			s, err := NewServer(context.Background(), WithConfig(ss.http))
+			s, err := NewServer(context.Background(), WithConfig(ss.cfg))
 			require.NoError(t, err)
 			defer func() {
 				require.NoError(t, s.Shutdown())
@@ -246,11 +246,11 @@ func TestNewServerTLS(t *testing.T) {
 		name    string
 		wantErr bool
 		err     error
-		http    HTTPConfig
+		http    Config
 	}{
 		{
 			name: "FromFile/Valid",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCert:       "./testdata/local.crt",
 				TLSKey:        "./testdata/local.key",
@@ -261,7 +261,7 @@ func TestNewServerTLS(t *testing.T) {
 			name:    "FromFile/NoCert",
 			wantErr: true,
 			err:     ErrTLSFileMismatch,
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCert:       "",
 				TLSKey:        "./testdata/local.key",
@@ -271,7 +271,7 @@ func TestNewServerTLS(t *testing.T) {
 		{
 			name:    "FromFile/InvalidCert",
 			wantErr: true,
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCert:       "./testdata/local.crt.invalid",
 				TLSKey:        "./testdata/local.key",
@@ -282,7 +282,7 @@ func TestNewServerTLS(t *testing.T) {
 			name:    "FromFile/NoKey",
 			wantErr: true,
 			err:     ErrTLSFileMismatch,
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCert:       "./testdata/local.crt",
 				TLSKey:        "",
@@ -292,7 +292,7 @@ func TestNewServerTLS(t *testing.T) {
 		{
 			name:    "FromFile/InvalidKey",
 			wantErr: true,
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCert:       "./testdata/local.crt",
 				TLSKey:        "./testdata/local.key.invalid",
@@ -301,7 +301,7 @@ func TestNewServerTLS(t *testing.T) {
 		},
 		{
 			name: "FromBody/Valid",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCertBody:   certBytes,
 				TLSKeyBody:    keyBytes,
@@ -312,7 +312,7 @@ func TestNewServerTLS(t *testing.T) {
 			name:    "FromBody/NoCert",
 			wantErr: true,
 			err:     ErrTLSBodyMismatch,
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCertBody:   nil,
 				TLSKeyBody:    keyBytes,
@@ -322,7 +322,7 @@ func TestNewServerTLS(t *testing.T) {
 		{
 			name:    "FromBody/InvalidCert",
 			wantErr: true,
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCertBody:   []byte("JUNK DATA"),
 				TLSKeyBody:    keyBytes,
@@ -333,7 +333,7 @@ func TestNewServerTLS(t *testing.T) {
 			name:    "FromBody/NoKey",
 			wantErr: true,
 			err:     ErrTLSBodyMismatch,
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCertBody:   certBytes,
 				TLSKeyBody:    nil,
@@ -343,7 +343,7 @@ func TestNewServerTLS(t *testing.T) {
 		{
 			name:    "FromBody/InvalidKey",
 			wantErr: true,
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCertBody:   certBytes,
 				TLSKeyBody:    []byte("JUNK DATA"),
@@ -352,7 +352,7 @@ func TestNewServerTLS(t *testing.T) {
 		},
 		{
 			name: "MinTLSVersion/Valid/1.1",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCertBody:   certBytes,
 				TLSKeyBody:    keyBytes,
@@ -361,7 +361,7 @@ func TestNewServerTLS(t *testing.T) {
 		},
 		{
 			name: "MinTLSVersion/Valid/1.2",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCertBody:   certBytes,
 				TLSKeyBody:    keyBytes,
@@ -370,7 +370,7 @@ func TestNewServerTLS(t *testing.T) {
 		},
 		{
 			name: "MinTLSVersion/Valid/1.3",
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCertBody:   certBytes,
 				TLSKeyBody:    keyBytes,
@@ -381,7 +381,7 @@ func TestNewServerTLS(t *testing.T) {
 			name:    "MinTLSVersion/Invalid",
 			wantErr: true,
 			err:     ErrInvalidMinTLSVersion,
-			http: HTTPConfig{
+			http: Config{
 				ListenAddr:    []string{"127.0.0.1:0"},
 				TLSCertBody:   certBytes,
 				TLSKeyBody:    keyBytes,

From ec7cc2b3c3b1049b61dbb64c6152c2e1e0dc44f4 Mon Sep 17 00:00:00 2001
From: Tom Mombourquette 
Date: Wed, 23 Nov 2022 05:44:53 -0400
Subject: [PATCH 462/560] lib/http: Simplify server.go to export an http server
 rather than an interface

This also makes the implementation public.
---
 cmd/serve/http/http.go       |  2 +-
 lib/http/serve/dir_test.go   |  6 ++---
 lib/http/serve/serve_test.go | 12 ++++-----
 lib/http/server.go           | 48 +++++++++++++++++-------------------
 lib/http/server_test.go      |  2 +-
 5 files changed, 33 insertions(+), 37 deletions(-)

diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go
index d88714dddb7e3..78f6870f0ee4d 100644
--- a/cmd/serve/http/http.go
+++ b/cmd/serve/http/http.go
@@ -103,7 +103,7 @@ control the stats printing.
 type serveCmd struct {
 	f      fs.Fs
 	vfs    *vfs.VFS
-	server libhttp.Server
+	server *libhttp.Server
 }
 
 func run(ctx context.Context, f fs.Fs, opt Options) (*serveCmd, error) {
diff --git a/lib/http/serve/dir_test.go b/lib/http/serve/dir_test.go
index bdcf3f9486986..be128ae33adc8 100644
--- a/lib/http/serve/dir_test.go
+++ b/lib/http/serve/dir_test.go
@@ -3,7 +3,7 @@ package serve
 import (
 	"errors"
 	"html/template"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -94,7 +94,7 @@ func TestError(t *testing.T) {
 	Error("potato", w, "sausage", err)
 	resp := w.Result()
 	assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "sausage.\n", string(body))
 }
 
@@ -108,7 +108,7 @@ func TestServe(t *testing.T) {
 	d.Serve(w, r)
 	resp := w.Result()
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, `
 
 
diff --git a/lib/http/serve/serve_test.go b/lib/http/serve/serve_test.go
index 934fabe658169..b6cc70976d32b 100644
--- a/lib/http/serve/serve_test.go
+++ b/lib/http/serve/serve_test.go
@@ -1,7 +1,7 @@
 package serve
 
 import (
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -17,7 +17,7 @@ func TestObjectBadMethod(t *testing.T) {
 	Object(w, r, o)
 	resp := w.Result()
 	assert.Equal(t, http.StatusMethodNotAllowed, resp.StatusCode)
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "Method Not Allowed\n", string(body))
 }
 
@@ -30,7 +30,7 @@ func TestObjectHEAD(t *testing.T) {
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	assert.Equal(t, "5", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "", string(body))
 }
 
@@ -43,7 +43,7 @@ func TestObjectGET(t *testing.T) {
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 	assert.Equal(t, "5", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "hello", string(body))
 }
 
@@ -58,7 +58,7 @@ func TestObjectRange(t *testing.T) {
 	assert.Equal(t, "3", resp.Header.Get("Content-Length"))
 	assert.Equal(t, "bytes", resp.Header.Get("Accept-Ranges"))
 	assert.Equal(t, "bytes 3-5/10", resp.Header.Get("Content-Range"))
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "345", string(body))
 }
 
@@ -71,6 +71,6 @@ func TestObjectBadRange(t *testing.T) {
 	resp := w.Result()
 	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
 	assert.Equal(t, "10", resp.Header.Get("Content-Length"))
-	body, _ := ioutil.ReadAll(resp.Body)
+	body, _ := io.ReadAll(resp.Body)
 	assert.Equal(t, "Bad Request\n", string(body))
 }
diff --git a/lib/http/server.go b/lib/http/server.go
index 4dbc31c8ef17f..9d69c65856ce1 100644
--- a/lib/http/server.go
+++ b/lib/http/server.go
@@ -121,16 +121,6 @@ func DefaultCfg() Config {
 	}
 }
 
-// Server interface of http server
-type Server interface {
-	Router() chi.Router
-	Serve()
-	Shutdown() error
-	HTMLTemplate() *template.Template
-	URLs() []string
-	Wait()
-}
-
 type instance struct {
 	url        string
 	listener   net.Listener
@@ -145,7 +135,8 @@ func (s instance) serve(wg *sync.WaitGroup) {
 	}
 }
 
-type server struct {
+// Server contains info about the running http server
+type Server struct {
 	wg           sync.WaitGroup
 	mux          chi.Router
 	tlsConfig    *tls.Config
@@ -157,25 +148,25 @@ type server struct {
 }
 
 // Option allows customizing the server
-type Option func(*server)
+type Option func(*Server)
 
 // WithAuth option initializes the appropriate auth middleware
 func WithAuth(cfg AuthConfig) Option {
-	return func(s *server) {
+	return func(s *Server) {
 		s.auth = cfg
 	}
 }
 
 // WithConfig option applies the Config to the server, overriding defaults
 func WithConfig(cfg Config) Option {
-	return func(s *server) {
+	return func(s *Server) {
 		s.cfg = cfg
 	}
 }
 
 // WithTemplate option allows the parsing of a template
 func WithTemplate(cfg TemplateConfig) Option {
-	return func(s *server) {
+	return func(s *Server) {
 		s.template = &cfg
 	}
 }
@@ -184,12 +175,17 @@ func WithTemplate(cfg TemplateConfig) Option {
 // This function is provided if the default http server does not meet a services requirements and should not generally be used
 // A http server can listen using multiple listeners. For example, a listener for port 80, and a listener for port 443.
 // tlsListeners are ignored if opt.TLSKey is not provided
-func NewServer(ctx context.Context, options ...Option) (*server, error) {
-	s := &server{
+func NewServer(ctx context.Context, options ...Option) (*Server, error) {
+	s := &Server{
 		mux: chi.NewRouter(),
 		cfg: DefaultCfg(),
 	}
 
+	// Make sure default logger is logging where everything else is
+	// middleware.DefaultLogger = middleware.RequestLogger(&middleware.DefaultLogFormatter{Logger: log.Default(), NoColor: true})
+	// Log requests
+	// s.mux.Use(middleware.Logger)
+
 	for _, opt := range options {
 		opt(s)
 	}
@@ -275,7 +271,7 @@ func NewServer(ctx context.Context, options ...Option) (*server, error) {
 	return s, nil
 }
 
-func (s *server) initAuth() {
+func (s *Server) initAuth() {
 	if s.auth.CustomAuthFn != nil {
 		s.mux.Use(MiddlewareAuthCustom(s.auth.CustomAuthFn, s.auth.Realm))
 		return
@@ -292,7 +288,7 @@ func (s *server) initAuth() {
 	}
 }
 
-func (s *server) initTemplate() error {
+func (s *Server) initTemplate() error {
 	if s.template == nil {
 		return nil
 	}
@@ -317,7 +313,7 @@ var (
 	ErrTLSParseCA = errors.New("unable to parse client certificate authority")
 )
 
-func (s *server) initTLS() error {
+func (s *Server) initTLS() error {
 	if s.cfg.TLSCert == "" && s.cfg.TLSKey == "" && len(s.cfg.TLSCertBody) == 0 && len(s.cfg.TLSKeyBody) == 0 {
 		return nil
 	}
@@ -383,7 +379,7 @@ func (s *server) initTLS() error {
 }
 
 // Serve starts the HTTP server on each listener
-func (s *server) Serve() {
+func (s *Server) Serve() {
 	s.wg.Add(len(s.instances))
 	for _, ii := range s.instances {
 		// TODO: decide how/when to log listening url
@@ -393,17 +389,17 @@ func (s *server) Serve() {
 }
 
 // Wait blocks while the server is serving requests
-func (s *server) Wait() {
+func (s *Server) Wait() {
 	s.wg.Wait()
 }
 
 // Router returns the server base router
-func (s *server) Router() chi.Router {
+func (s *Server) Router() chi.Router {
 	return s.mux
 }
 
 // Shutdown gracefully shuts down the server
-func (s *server) Shutdown() error {
+func (s *Server) Shutdown() error {
 	ctx := context.Background()
 	for _, ii := range s.instances {
 		if err := ii.httpServer.Shutdown(ctx); err != nil {
@@ -416,12 +412,12 @@ func (s *server) Shutdown() error {
 }
 
 // HTMLTemplate returns the parsed template, if WithTemplate option was passed.
-func (s *server) HTMLTemplate() *template.Template {
+func (s *Server) HTMLTemplate() *template.Template {
 	return s.htmlTemplate
 }
 
 // URLs returns all configured URLS
-func (s *server) URLs() []string {
+func (s *Server) URLs() []string {
 	var out []string
 	for _, ii := range s.instances {
 		if ii.listener.Addr().Network() == "unix" {
diff --git a/lib/http/server_test.go b/lib/http/server_test.go
index 42c630ea404bd..506a78d51668f 100644
--- a/lib/http/server_test.go
+++ b/lib/http/server_test.go
@@ -26,7 +26,7 @@ func testExpectRespBody(t *testing.T, resp *http.Response, expected []byte) {
 	require.Equal(t, expected, body)
 }
 
-func testGetServerURL(t *testing.T, s Server) string {
+func testGetServerURL(t *testing.T, s *Server) string {
 	urls := s.URLs()
 	require.GreaterOrEqual(t, len(urls), 1, "server should return at least one url")
 	return urls[0]

From 3167292c2f5b52523b5dc76c54c11dbb846b9481 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Sat, 10 Dec 2022 10:51:36 +0000
Subject: [PATCH 463/560] lib/http: remove unused Template from Config

---
 lib/http/server.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/http/server.go b/lib/http/server.go
index 9d69c65856ce1..253bdc60ecbf5 100644
--- a/lib/http/server.go
+++ b/lib/http/server.go
@@ -89,7 +89,6 @@ type Config struct {
 	TLSKeyBody         []byte        // TLS PEM Private key body, ignores TLSKey
 	ClientCA           string        // Client certificate authority to verify clients with
 	MinTLSVersion      string        // MinTLSVersion contains the minimum TLS version that is acceptable.
-	Template           string
 }
 
 // AddFlagsPrefix adds flags for the httplib

From a9ce86f9a33cf54044f060b77aed594caae97ce1 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Sat, 10 Dec 2022 10:51:57 +0000
Subject: [PATCH 464/560] lib/http: add UsingAuth method

---
 lib/http/server.go | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib/http/server.go b/lib/http/server.go
index 253bdc60ecbf5..5e76e9615492c 100644
--- a/lib/http/server.go
+++ b/lib/http/server.go
@@ -144,6 +144,7 @@ type Server struct {
 	cfg          Config
 	template     *TemplateConfig
 	htmlTemplate *template.Template
+	usingAuth    bool // set if we are using auth middleware
 }
 
 // Option allows customizing the server
@@ -272,19 +273,23 @@ func NewServer(ctx context.Context, options ...Option) (*Server, error) {
 
 func (s *Server) initAuth() {
 	if s.auth.CustomAuthFn != nil {
+		s.usingAuth = true
 		s.mux.Use(MiddlewareAuthCustom(s.auth.CustomAuthFn, s.auth.Realm))
 		return
 	}
 
 	if s.auth.HtPasswd != "" {
+		s.usingAuth = true
 		s.mux.Use(MiddlewareAuthHtpasswd(s.auth.HtPasswd, s.auth.Realm))
 		return
 	}
 
 	if s.auth.BasicUser != "" {
+		s.usingAuth = true
 		s.mux.Use(MiddlewareAuthBasic(s.auth.BasicUser, s.auth.BasicPass, s.auth.Realm, s.auth.Salt))
 		return
 	}
+	s.usingAuth = false
 }
 
 func (s *Server) initTemplate() error {
@@ -426,3 +431,8 @@ func (s *Server) URLs() []string {
 	}
 	return out
 }
+
+// UsingAuth returns true if authentication is required
+func (s *Server) UsingAuth() bool {
+	return s.usingAuth
+}

From 08a1ca434b1f8727041106cf98578dd9c137f41f Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Sun, 11 Dec 2022 14:47:47 +0000
Subject: [PATCH 465/560] rcd: refactor rclone rc server to use lib/http

---
 cmd/rcd/rcd.go                  |   3 +-
 fs/rc/config_test.go            |   2 -
 fs/rc/rc.go                     |  12 ++-
 fs/rc/rcflags/rcflags.go        |   5 +-
 fs/rc/rcserver/rcserver.go      | 142 ++++++++++++++++++++------------
 fs/rc/rcserver/rcserver_test.go |  94 +++++++++++++++------
 6 files changed, 174 insertions(+), 84 deletions(-)

diff --git a/cmd/rcd/rcd.go b/cmd/rcd/rcd.go
index aaa57a065f6b0..818580f3ab43e 100644
--- a/cmd/rcd/rcd.go
+++ b/cmd/rcd/rcd.go
@@ -11,6 +11,7 @@ import (
 	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/rclone/rclone/fs/rc/rcserver"
 	"github.com/rclone/rclone/lib/atexit"
+	libhttp "github.com/rclone/rclone/lib/http"
 	"github.com/spf13/cobra"
 )
 
@@ -31,7 +32,7 @@ for GET requests on the URL passed in.  It will also open the URL in
 the browser when rclone is run.
 
 See the [rc documentation](/rc/) for more info on the rc flags.
-`,
+` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.45",
 	},
diff --git a/fs/rc/config_test.go b/fs/rc/config_test.go
index 01b25983cd8ca..f3c75d1643ff5 100644
--- a/fs/rc/config_test.go
+++ b/fs/rc/config_test.go
@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/rclone/rclone/cmd/serve/httplib"
 	"github.com/rclone/rclone/fs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -79,7 +78,6 @@ func TestOptionsGetMarshal(t *testing.T) {
 	ci := fs.GetConfig(ctx)
 
 	// Add some real options
-	AddOption("http", &httplib.DefaultOpt)
 	AddOption("main", ci)
 	AddOption("rc", &DefaultOpt)
 
diff --git a/fs/rc/rc.go b/fs/rc/rc.go
index 57d9f18dac091..c45f61cf9928e 100644
--- a/fs/rc/rc.go
+++ b/fs/rc/rc.go
@@ -13,12 +13,14 @@ import (
 	_ "net/http/pprof" // install the pprof http handlers
 	"time"
 
-	"github.com/rclone/rclone/cmd/serve/httplib"
+	libhttp "github.com/rclone/rclone/lib/http"
 )
 
 // Options contains options for the remote control server
 type Options struct {
-	HTTPOptions              httplib.Options
+	HTTP                     libhttp.Config
+	Auth                     libhttp.AuthConfig
+	Template                 libhttp.TemplateConfig
 	Enabled                  bool   // set to enable the server
 	Serve                    bool   // set to serve files from remotes
 	Files                    string // set to enable serving files locally
@@ -36,14 +38,16 @@ type Options struct {
 
 // DefaultOpt is the default values used for Options
 var DefaultOpt = Options{
-	HTTPOptions:       httplib.DefaultOpt,
+	HTTP:              libhttp.DefaultCfg(),
+	Auth:              libhttp.DefaultAuthCfg(),
+	Template:          libhttp.DefaultTemplateCfg(),
 	Enabled:           false,
 	JobExpireDuration: 60 * time.Second,
 	JobExpireInterval: 10 * time.Second,
 }
 
 func init() {
-	DefaultOpt.HTTPOptions.ListenAddr = "localhost:5572"
+	DefaultOpt.HTTP.ListenAddr = []string{"localhost:5572"}
 }
 
 // WriteJSON writes JSON in out to w
diff --git a/fs/rc/rcflags/rcflags.go b/fs/rc/rcflags/rcflags.go
index a7d85dfd86e4c..db2412e06dbc3 100644
--- a/fs/rc/rcflags/rcflags.go
+++ b/fs/rc/rcflags/rcflags.go
@@ -2,7 +2,6 @@
 package rcflags
 
 import (
-	"github.com/rclone/rclone/cmd/serve/httplib/httpflags"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/spf13/pflag"
@@ -29,5 +28,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &Opt.EnableMetrics, "rc-enable-metrics", "", false, "Enable prometheus metrics on /metrics")
 	flags.DurationVarP(flagSet, &Opt.JobExpireDuration, "rc-job-expire-duration", "", Opt.JobExpireDuration, "Expire finished async jobs older than this value")
 	flags.DurationVarP(flagSet, &Opt.JobExpireInterval, "rc-job-expire-interval", "", Opt.JobExpireInterval, "Interval to check for expired async jobs")
-	httpflags.AddFlagsPrefix(flagSet, "rc-", &Opt.HTTPOptions)
+	Opt.HTTP.AddFlagsPrefix(flagSet, "rc-")
+	Opt.Auth.AddFlagsPrefix(flagSet, "rc-")
+	Opt.Template.AddFlagsPrefix(flagSet, "rc-")
 }
diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.go
index 125b36903a6a5..f353c3ea16080 100644
--- a/fs/rc/rcserver/rcserver.go
+++ b/fs/rc/rcserver/rcserver.go
@@ -18,9 +18,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/go-chi/chi/v5/middleware"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/rclone/rclone/cmd/serve/httplib"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/cache"
@@ -31,6 +31,7 @@ import (
 	"github.com/rclone/rclone/fs/rc/jobs"
 	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/rclone/rclone/fs/rc/webgui"
+	libhttp "github.com/rclone/rclone/lib/http"
 	"github.com/rclone/rclone/lib/http/serve"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/skratchdot/open-golang/open"
@@ -59,7 +60,10 @@ func Start(ctx context.Context, opt *rc.Options) (*Server, error) {
 	jobs.SetOpt(opt) // set the defaults for jobs
 	if opt.Enabled {
 		// Serve on the DefaultServeMux so can have global registrations appear
-		s := newServer(ctx, opt, http.DefaultServeMux)
+		s, err := newServer(ctx, opt, http.DefaultServeMux)
+		if err != nil {
+			return nil, err
+		}
 		return s, s.Serve()
 	}
 	return nil, nil
@@ -67,14 +71,14 @@ func Start(ctx context.Context, opt *rc.Options) (*Server, error) {
 
 // Server contains everything to run the rc server
 type Server struct {
-	*httplib.Server
 	ctx            context.Context // for global config
+	server         *libhttp.Server
 	files          http.Handler
 	pluginsHandler http.Handler
 	opt            *rc.Options
 }
 
-func newServer(ctx context.Context, opt *rc.Options, mux *http.ServeMux) *Server {
+func newServer(ctx context.Context, opt *rc.Options, mux *http.ServeMux) (*Server, error) {
 	fileHandler := http.Handler(nil)
 	pluginsHandler := http.Handler(nil)
 	// Add some more mime types which are often missing
@@ -97,16 +101,16 @@ func newServer(ctx context.Context, opt *rc.Options, mux *http.ServeMux) *Server
 		if opt.NoAuth {
 			fs.Logf(nil, "It is recommended to use web gui with auth.")
 		} else {
-			if opt.HTTPOptions.BasicUser == "" && opt.HTTPOptions.HtPasswd == "" {
-				opt.HTTPOptions.BasicUser = "gui"
-				fs.Infof(nil, "No username specified. Using default username: %s \n", rcflags.Opt.HTTPOptions.BasicUser)
+			if opt.Auth.BasicUser == "" && opt.Auth.HtPasswd == "" {
+				opt.Auth.BasicUser = "gui"
+				fs.Infof(nil, "No username specified. Using default username: %s \n", rcflags.Opt.Auth.BasicUser)
 			}
-			if opt.HTTPOptions.BasicPass == "" && opt.HTTPOptions.HtPasswd == "" {
+			if opt.Auth.BasicPass == "" && opt.Auth.HtPasswd == "" {
 				randomPass, err := random.Password(128)
 				if err != nil {
 					log.Fatalf("Failed to make password: %v", err)
 				}
-				opt.HTTPOptions.BasicPass = randomPass
+				opt.Auth.BasicPass = randomPass
 				fs.Infof(nil, "No password specified. Using random password: %s \n", randomPass)
 			}
 		}
@@ -119,53 +123,76 @@ func newServer(ctx context.Context, opt *rc.Options, mux *http.ServeMux) *Server
 	}
 
 	s := &Server{
-		Server:         httplib.NewServer(mux, &opt.HTTPOptions),
 		ctx:            ctx,
 		opt:            opt,
 		files:          fileHandler,
 		pluginsHandler: pluginsHandler,
 	}
-	mux.HandleFunc("/", s.handler)
 
-	return s
+	var err error
+	s.server, err = libhttp.NewServer(ctx,
+		libhttp.WithConfig(opt.HTTP),
+		libhttp.WithAuth(opt.Auth),
+		libhttp.WithTemplate(opt.Template),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init server: %w", err)
+	}
+
+	router := s.server.Router()
+	router.Use(
+		middleware.SetHeader("Accept-Ranges", "bytes"),
+		middleware.SetHeader("Server", "rclone/"+fs.Version),
+	)
+
+	// Add the debug handler which is installed in the default mux
+	router.Handle("/debug/*", mux)
+
+	// FIXME split these up into individual functions
+	router.Get("/*", s.handler)
+	router.Head("/*", s.handler)
+	router.Post("/*", s.handler)
+	router.Options("/*", s.handler)
+
+	return s, nil
 }
 
 // Serve runs the http server in the background.
 //
 // Use s.Close() and s.Wait() to shutdown server
 func (s *Server) Serve() error {
-	err := s.Server.Serve()
-	if err != nil {
-		return err
-	}
-	fs.Logf(nil, "Serving remote control on %s", s.URL())
-	// Open the files in the browser if set
-	if s.files != nil {
-		openURL, err := url.Parse(s.URL())
-		if err != nil {
-			return fmt.Errorf("invalid serving URL: %w", err)
-		}
-		// Add username, password into the URL if they are set
-		user, pass := s.opt.HTTPOptions.BasicUser, s.opt.HTTPOptions.BasicPass
-		if user != "" && pass != "" {
-			openURL.User = url.UserPassword(user, pass)
-
-			// Base64 encode username and password to be sent through url
-			loginToken := user + ":" + pass
-			parameters := url.Values{}
-			encodedToken := base64.URLEncoding.EncodeToString([]byte(loginToken))
-			fs.Debugf(nil, "login_token %q", encodedToken)
-			parameters.Add("login_token", encodedToken)
-			openURL.RawQuery = parameters.Encode()
-			openURL.RawPath = "/#/login"
-		}
-		// Don't open browser if serving in testing environment or required not to do so.
-		if flag.Lookup("test.v") == nil && !s.opt.WebGUINoOpenBrowser {
-			if err := open.Start(openURL.String()); err != nil {
-				fs.Errorf(nil, "Failed to open Web GUI in browser: %v. Manually access it at: %s", err, openURL.String())
+	s.server.Serve()
+
+	for _, URL := range s.server.URLs() {
+		fs.Logf(nil, "Serving remote control on %s", URL)
+		// Open the files in the browser if set
+		if s.files != nil {
+			openURL, err := url.Parse(URL)
+			if err != nil {
+				return fmt.Errorf("invalid serving URL: %w", err)
+			}
+			// Add username, password into the URL if they are set
+			user, pass := s.opt.Auth.BasicUser, s.opt.Auth.BasicPass
+			if user != "" && pass != "" {
+				openURL.User = url.UserPassword(user, pass)
+
+				// Base64 encode username and password to be sent through url
+				loginToken := user + ":" + pass
+				parameters := url.Values{}
+				encodedToken := base64.URLEncoding.EncodeToString([]byte(loginToken))
+				fs.Debugf(nil, "login_token %q", encodedToken)
+				parameters.Add("login_token", encodedToken)
+				openURL.RawQuery = parameters.Encode()
+				openURL.RawPath = "/#/login"
+			}
+			// Don't open browser if serving in testing environment or required not to do so.
+			if flag.Lookup("test.v") == nil && !s.opt.WebGUINoOpenBrowser {
+				if err := open.Start(openURL.String()); err != nil {
+					fs.Errorf(nil, "Failed to open Web GUI in browser: %v. Manually access it at: %s", err, openURL.String())
+				}
+			} else {
+				fs.Logf(nil, "Web GUI is not automatically opening browser. Navigate to %s to use.", openURL.String())
 			}
-		} else {
-			fs.Logf(nil, "Web GUI is not automatically opening browser. Navigate to %s to use.", openURL.String())
 		}
 	}
 	return nil
@@ -185,11 +212,7 @@ func writeError(path string, in rc.Params, w http.ResponseWriter, err error, sta
 
 // handler reads incoming requests and dispatches them
 func (s *Server) handler(w http.ResponseWriter, r *http.Request) {
-	urlPath, ok := s.Path(w, r)
-	if !ok {
-		return
-	}
-	path := strings.TrimLeft(urlPath, "/")
+	path := strings.TrimLeft(r.URL.Path, "/")
 
 	allowOrigin := rcflags.Opt.AccessControlAllowOrigin
 	if allowOrigin != "" {
@@ -200,7 +223,12 @@ func (s *Server) handler(w http.ResponseWriter, r *http.Request) {
 		})
 		w.Header().Add("Access-Control-Allow-Origin", allowOrigin)
 	} else {
-		w.Header().Add("Access-Control-Allow-Origin", s.URL())
+		urls := s.server.URLs()
+		if len(urls) == 1 {
+			w.Header().Add("Access-Control-Allow-Origin", urls[0])
+		} else {
+			fs.Errorf(nil, "Warning, need exactly 1 URL for Access-Control-Allow-Origin, got %d %q", len(urls), urls)
+		}
 	}
 
 	// echo back access control headers client needs
@@ -260,7 +288,7 @@ func (s *Server) handlePost(w http.ResponseWriter, r *http.Request, path string)
 	}
 
 	// Check to see if it requires authorisation
-	if !s.opt.NoAuth && call.AuthRequired && !s.UsingAuth() {
+	if !s.opt.NoAuth && call.AuthRequired && !s.server.UsingAuth() {
 		writeError(path, in, w, fmt.Errorf("authentication must be set up on the rc server to use %q or the --rc-no-auth flag must be in use", path), http.StatusForbidden)
 		return
 	}
@@ -305,7 +333,7 @@ func (s *Server) handleOptions(w http.ResponseWriter, r *http.Request, path stri
 func (s *Server) serveRoot(w http.ResponseWriter, r *http.Request) {
 	remotes := config.FileSections()
 	sort.Strings(remotes)
-	directory := serve.NewDirectory("", s.HTMLTemplate)
+	directory := serve.NewDirectory("", s.server.HTMLTemplate())
 	directory.Name = "List of all rclone remotes."
 	q := url.Values{}
 	for _, remote := range remotes {
@@ -333,7 +361,7 @@ func (s *Server) serveRemote(w http.ResponseWriter, r *http.Request, path string
 			return
 		}
 		// Make the entries for display
-		directory := serve.NewDirectory(path, s.HTMLTemplate)
+		directory := serve.NewDirectory(path, s.server.HTMLTemplate())
 		for _, entry := range entries {
 			_, isDir := entry.(fs.Directory)
 			//directory.AddHTMLEntry(entry.Remote(), isDir, entry.Size(), entry.ModTime(r.Context()))
@@ -401,3 +429,13 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)
 	}
 	http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
 }
+
+// Wait blocks while the server is serving requests
+func (s *Server) Wait() {
+	s.server.Wait()
+}
+
+// Shutdown gracefully shuts down the server
+func (s *Server) Shutdown() error {
+	return s.server.Shutdown()
+}
diff --git a/fs/rc/rcserver/rcserver_test.go b/fs/rc/rcserver/rcserver_test.go
index 62c5e25374680..5a6c9a12931be 100644
--- a/fs/rc/rcserver/rcserver_test.go
+++ b/fs/rc/rcserver/rcserver_test.go
@@ -13,14 +13,13 @@ import (
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/rc"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -49,24 +48,24 @@ func TestMain(m *testing.M) {
 // We'll do the majority of the testing with the httptest framework
 func TestRcServer(t *testing.T) {
 	opt := rc.DefaultOpt
-	opt.HTTPOptions.ListenAddr = testBindAddress
-	opt.HTTPOptions.Template = testTemplate
+	opt.HTTP.ListenAddr = []string{testBindAddress}
+	opt.Template.Path = testTemplate
 	opt.Enabled = true
 	opt.Serve = true
 	opt.Files = testFs
 	mux := http.NewServeMux()
-	rcServer := newServer(context.Background(), &opt, mux)
+	rcServer, err := newServer(context.Background(), &opt, mux)
+	require.NoError(t, err)
 	assert.NoError(t, rcServer.Serve())
 	defer func() {
-		rcServer.Close()
+		assert.NoError(t, rcServer.Shutdown())
 		rcServer.Wait()
 	}()
-	testURL := rcServer.Server.URL()
+	testURL := rcServer.server.URLs()[0]
 
 	// Do the simplest possible test to check the server is alive
 	// Do it a few times to wait for the server to start
 	var resp *http.Response
-	var err error
 	for i := 0; i < 10; i++ {
 		resp, err = http.Get(testURL + "file.txt")
 		if err == nil {
@@ -89,6 +88,8 @@ func TestRcServer(t *testing.T) {
 type testRun struct {
 	Name        string
 	URL         string
+	User        string
+	Pass        string
 	Status      int
 	Method      string
 	Range       string
@@ -103,9 +104,11 @@ type testRun struct {
 func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
 	ctx := context.Background()
 	configfile.Install()
-	mux := http.NewServeMux()
-	opt.HTTPOptions.Template = testTemplate
-	rcServer := newServer(ctx, opt, mux)
+	opt.Template.Path = testTemplate
+	rcServer, err := newServer(ctx, opt, http.DefaultServeMux)
+	require.NoError(t, err)
+	testURL := rcServer.server.URLs()[0]
+	mux := rcServer.server.Router()
 	for _, test := range tests {
 		t.Run(test.Name, func(t *testing.T) {
 			method := test.Method
@@ -125,9 +128,12 @@ func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
 			if test.ContentType != "" {
 				req.Header.Add("Content-Type", test.ContentType)
 			}
+			if test.User != "" && test.Pass != "" {
+				req.SetBasicAuth(test.User, test.Pass)
+			}
 
 			w := httptest.NewRecorder()
-			rcServer.handler(w, req)
+			mux.ServeHTTP(w, req)
 			resp := w.Result()
 
 			assert.Equal(t, test.Status, resp.StatusCode)
@@ -141,6 +147,9 @@ func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
 			}
 
 			for k, v := range test.Headers {
+				if v == "testURL" {
+					v = testURL
+				}
 				assert.Equal(t, v, resp.Header.Get(k), k)
 			}
 		})
@@ -151,6 +160,7 @@ func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
 func newTestOpt() rc.Options {
 	opt := rc.DefaultOpt
 	opt.Enabled = true
+	opt.HTTP.ListenAddr = []string{testBindAddress}
 	return opt
 }
 
@@ -550,7 +560,7 @@ func TestMethods(t *testing.T) {
 		Status:   http.StatusOK,
 		Expected: "",
 		Headers: map[string]string{
-			"Access-Control-Allow-Origin":   "http://localhost:5572/",
+			"Access-Control-Allow-Origin":   "testURL",
 			"Access-Control-Request-Method": "POST, OPTIONS, GET, HEAD",
 			"Access-Control-Allow-Headers":  "authorization, Content-Type",
 		},
@@ -559,12 +569,7 @@ func TestMethods(t *testing.T) {
 		URL:    "",
 		Method: "POTATO",
 		Status: http.StatusMethodNotAllowed,
-		Expected: `{
-	"error": "method \"POTATO\" not allowed",
-	"input": null,
-	"path": "",
-	"status": 405
-}
+		Expected: `Method Not Allowed
 `,
 	}}
 	opt := newTestOpt()
@@ -732,20 +737,40 @@ func TestNoAuth(t *testing.T) {
 
 func TestWithUserPass(t *testing.T) {
 	tests := []testRun{{
-		Name:        "auth",
+		Name:        "authMissing",
+		URL:         "rc/noopauth",
+		Method:      "POST",
+		Body:        `{}`,
+		ContentType: "application/javascript",
+		Status:      http.StatusUnauthorized,
+		Expected:    "401 Unauthorized\n",
+	}, {
+		Name:        "authWrong",
+		URL:         "rc/noopauth",
+		Method:      "POST",
+		Body:        `{}`,
+		ContentType: "application/javascript",
+		Status:      http.StatusUnauthorized,
+		Expected:    "401 Unauthorized\n",
+		User:        "user1",
+		Pass:        "pass2",
+	}, {
+		Name:        "authOK",
 		URL:         "rc/noopauth",
 		Method:      "POST",
 		Body:        `{}`,
 		ContentType: "application/javascript",
 		Status:      http.StatusOK,
 		Expected:    "{}\n",
+		User:        "user",
+		Pass:        "pass",
 	}}
 	opt := newTestOpt()
 	opt.Serve = false
 	opt.Files = ""
 	opt.NoAuth = false
-	opt.HTTPOptions.BasicUser = "user"
-	opt.HTTPOptions.BasicPass = "pass"
+	opt.Auth.BasicUser = "user"
+	opt.Auth.BasicPass = "pass"
 	testServer(t, tests, &opt)
 }
 
@@ -780,3 +805,26 @@ func TestRCAsync(t *testing.T) {
 	opt.Files = ""
 	testServer(t, tests, &opt)
 }
+
+// Check the debug handlers are attached
+func TestRCDebug(t *testing.T) {
+	tests := []testRun{{
+		Name:        "index",
+		URL:         "debug/pprof/",
+		Method:      "GET",
+		ContentType: "text/html",
+		Status:      http.StatusOK,
+		Contains:    regexp.MustCompile(`Types of profiles available`),
+	}, {
+		Name:        "goroutines",
+		URL:         "debug/pprof/goroutine?debug=1",
+		Method:      "GET",
+		ContentType: "text/html",
+		Status:      http.StatusOK,
+		Contains:    regexp.MustCompile(`goroutine profile`),
+	}}
+	opt := newTestOpt()
+	opt.Serve = true
+	opt.Files = ""
+	testServer(t, tests, &opt)
+}

From 4444d2d102cc187f8cc4907f3e06611576c10a36 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Sun, 11 Dec 2022 18:53:55 +0000
Subject: [PATCH 466/560] serve webdav: refactor to use lib/http

---
 cmd/serve/webdav/webdav.go      | 149 ++++++++++++++++++++++----------
 cmd/serve/webdav/webdav_test.go |  43 ++++-----
 2 files changed, 123 insertions(+), 69 deletions(-)

diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go
index 5ab8810192d10..00af1f033c1cc 100644
--- a/cmd/serve/webdav/webdav.go
+++ b/cmd/serve/webdav/webdav.go
@@ -10,14 +10,15 @@ import (
 	"strings"
 	"time"
 
+	chi "github.com/go-chi/chi/v5"
+	"github.com/go-chi/chi/v5/middleware"
 	"github.com/rclone/rclone/cmd"
-	"github.com/rclone/rclone/cmd/serve/httplib"
-	"github.com/rclone/rclone/cmd/serve/httplib/httpflags"
 	"github.com/rclone/rclone/cmd/serve/proxy"
 	"github.com/rclone/rclone/cmd/serve/proxy/proxyflags"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/hash"
+	libhttp "github.com/rclone/rclone/lib/http"
 	"github.com/rclone/rclone/lib/http/serve"
 	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfsflags"
@@ -25,19 +26,37 @@ import (
 	"golang.org/x/net/webdav"
 )
 
-var (
-	hashName      string
-	hashType      = hash.None
-	disableGETDir = false
-)
+// Options required for http server
+type Options struct {
+	Auth          libhttp.AuthConfig
+	HTTP          libhttp.Config
+	Template      libhttp.TemplateConfig
+	HashName      string
+	HashType      hash.Type
+	DisableGETDir bool
+}
+
+// DefaultOpt is the default values used for Options
+var DefaultOpt = Options{
+	Auth:          libhttp.DefaultAuthCfg(),
+	HTTP:          libhttp.DefaultCfg(),
+	Template:      libhttp.DefaultTemplateCfg(),
+	HashType:      hash.None,
+	DisableGETDir: false,
+}
+
+// Opt is options set by command line flags
+var Opt = DefaultOpt
 
 func init() {
 	flagSet := Command.Flags()
-	httpflags.AddFlags(flagSet)
+	libhttp.AddAuthFlagsPrefix(flagSet, "", &Opt.Auth)
+	libhttp.AddHTTPFlagsPrefix(flagSet, "", &Opt.HTTP)
+	libhttp.AddTemplateFlagsPrefix(flagSet, "", &Opt.Template)
 	vfsflags.AddFlags(flagSet)
 	proxyflags.AddFlags(flagSet)
-	flags.StringVarP(flagSet, &hashName, "etag-hash", "", "", "Which hash to use for the ETag, or auto or blank for off")
-	flags.BoolVarP(flagSet, &disableGETDir, "disable-dir-list", "", false, "Disable HTML directory list on GET request for a directory")
+	flags.StringVarP(flagSet, &Opt.HashName, "etag-hash", "", "", "Which hash to use for the ETag, or auto or blank for off")
+	flags.BoolVarP(flagSet, &Opt.DisableGETDir, "disable-dir-list", "", false, "Disable HTML directory list on GET request for a directory")
 }
 
 // Command definition for cobra
@@ -60,7 +79,7 @@ supported hash on the backend or you can use a named hash such as
 "MD5" or "SHA-1". Use the [hashsum](/commands/rclone_hashsum/) command
 to see the full list.
 
-` + httplib.Help + vfs.Help + proxy.Help,
+` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp + vfs.Help + proxy.Help,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.39",
 	},
@@ -72,21 +91,24 @@ to see the full list.
 		} else {
 			cmd.CheckArgs(0, 0, command, args)
 		}
-		hashType = hash.None
-		if hashName == "auto" {
-			hashType = f.Hashes().GetOne()
-		} else if hashName != "" {
-			err := hashType.Set(hashName)
+		Opt.HashType = hash.None
+		if Opt.HashName == "auto" {
+			Opt.HashType = f.Hashes().GetOne()
+		} else if Opt.HashName != "" {
+			err := Opt.HashType.Set(Opt.HashName)
 			if err != nil {
 				return err
 			}
 		}
-		if hashType != hash.None {
-			fs.Debugf(f, "Using hash %v for ETag", hashType)
+		if Opt.HashType != hash.None {
+			fs.Debugf(f, "Using hash %v for ETag", Opt.HashType)
 		}
 		cmd.Run(false, false, command, func() error {
-			s := newWebDAV(context.Background(), f, &httpflags.Opt)
-			err := s.serve()
+			s, err := newWebDAV(context.Background(), f, &Opt)
+			if err != nil {
+				return err
+			}
+			err = s.serve()
 			if err != nil {
 				return err
 			}
@@ -110,7 +132,8 @@ to see the full list.
 // might apply". In particular, whether or not renaming a file or directory
 // overwriting another existing file or directory is an error is OS-dependent.
 type WebDAV struct {
-	*httplib.Server
+	*libhttp.Server
+	opt           Options
 	f             fs.Fs
 	_vfs          *vfs.VFS // don't use directly, use getVFS
 	webdavhandler *webdav.Handler
@@ -122,29 +145,63 @@ type WebDAV struct {
 var _ webdav.FileSystem = (*WebDAV)(nil)
 
 // Make a new WebDAV to serve the remote
-func newWebDAV(ctx context.Context, f fs.Fs, opt *httplib.Options) *WebDAV {
-	w := &WebDAV{
+func newWebDAV(ctx context.Context, f fs.Fs, opt *Options) (w *WebDAV, err error) {
+	w = &WebDAV{
 		f:   f,
 		ctx: ctx,
+		opt: *opt,
 	}
 	if proxyflags.Opt.AuthProxy != "" {
 		w.proxy = proxy.New(ctx, &proxyflags.Opt)
 		// override auth
-		copyOpt := *opt
-		copyOpt.Auth = w.auth
-		opt = ©Opt
+		w.opt.Auth.CustomAuthFn = w.auth
 	} else {
 		w._vfs = vfs.New(f, &vfsflags.Opt)
 	}
-	w.Server = httplib.NewServer(http.HandlerFunc(w.handler), opt)
+
+	w.Server, err = libhttp.NewServer(ctx,
+		libhttp.WithConfig(w.opt.HTTP),
+		libhttp.WithAuth(w.opt.Auth),
+		libhttp.WithTemplate(w.opt.Template),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init server: %w", err)
+	}
+
 	webdavHandler := &webdav.Handler{
-		Prefix:     w.Server.Opt.BaseURL,
+		Prefix:     w.opt.HTTP.BaseURL,
 		FileSystem: w,
 		LockSystem: webdav.NewMemLS(),
 		Logger:     w.logRequest, // FIXME
 	}
 	w.webdavhandler = webdavHandler
-	return w
+
+	router := w.Server.Router()
+	router.Use(
+		middleware.SetHeader("Accept-Ranges", "bytes"),
+		middleware.SetHeader("Server", "rclone/"+fs.Version),
+	)
+
+	router.Handle("/*", w)
+
+	// Webdav only methods not defined in chi
+	methods := []string{
+		"COPY",      // Copies the resource.
+		"LOCK",      // Locks the resource.
+		"MKCOL",     // Creates the collection specified.
+		"MOVE",      // Moves the resource.
+		"PROPFIND",  // Performs a property find on the server.
+		"PROPPATCH", // Sets or removes properties on the server.
+		"UNLOCK",    // Unlocks the resource.
+	}
+	for _, method := range methods {
+		chi.RegisterMethod(method)
+		router.Method(method, "/*", w)
+	}
+
+	w.Server.Serve()
+
+	return w, nil
 }
 
 // Gets the VFS in use for this request
@@ -152,7 +209,7 @@ func (w *WebDAV) getVFS(ctx context.Context) (VFS *vfs.VFS, err error) {
 	if w._vfs != nil {
 		return w._vfs, nil
 	}
-	value := ctx.Value(httplib.ContextAuthKey)
+	value := libhttp.CtxGetAuth(ctx)
 	if value == nil {
 		return nil, errors.New("no VFS found in context")
 	}
@@ -172,14 +229,11 @@ func (w *WebDAV) auth(user, pass string) (value interface{}, err error) {
 	return VFS, err
 }
 
-func (w *WebDAV) handler(rw http.ResponseWriter, r *http.Request) {
-	urlPath, ok := w.Path(rw, r)
-	if !ok {
-		return
-	}
+func (w *WebDAV) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
+	urlPath := r.URL.Path
 	isDir := strings.HasSuffix(urlPath, "/")
 	remote := strings.Trim(urlPath, "/")
-	if !disableGETDir && (r.Method == "GET" || r.Method == "HEAD") && isDir {
+	if !w.opt.DisableGETDir && (r.Method == "GET" || r.Method == "HEAD") && isDir {
 		w.serveDir(rw, r, remote)
 		return
 	}
@@ -217,7 +271,7 @@ func (w *WebDAV) serveDir(rw http.ResponseWriter, r *http.Request, dirRemote str
 	}
 
 	// Make the entries for display
-	directory := serve.NewDirectory(dirRemote, w.HTMLTemplate)
+	directory := serve.NewDirectory(dirRemote, w.Server.HTMLTemplate())
 	for _, node := range dirEntries {
 		if vfsflags.Opt.NoModTime {
 			directory.AddHTMLEntry(node.Path(), node.IsDir(), node.Size(), time.Time{})
@@ -237,11 +291,8 @@ func (w *WebDAV) serveDir(rw http.ResponseWriter, r *http.Request, dirRemote str
 //
 // Use s.Close() and s.Wait() to shutdown server
 func (w *WebDAV) serve() error {
-	err := w.Serve()
-	if err != nil {
-		return err
-	}
-	fs.Logf(w.f, "WebDav Server started on %s", w.URL())
+	w.Serve()
+	fs.Logf(w.f, "WebDav Server started on %s", w.URLs())
 	return nil
 }
 
@@ -276,7 +327,7 @@ func (w *WebDAV) OpenFile(ctx context.Context, name string, flags int, perm os.F
 	if err != nil {
 		return nil, err
 	}
-	return Handle{f}, nil
+	return Handle{Handle: f, w: w}, nil
 }
 
 // RemoveAll removes a file or a directory and its contents
@@ -318,12 +369,13 @@ func (w *WebDAV) Stat(ctx context.Context, name string) (fi os.FileInfo, err err
 	if err != nil {
 		return nil, err
 	}
-	return FileInfo{fi}, nil
+	return FileInfo{FileInfo: fi, w: w}, nil
 }
 
 // Handle represents an open file
 type Handle struct {
 	vfs.Handle
+	w *WebDAV
 }
 
 // Readdir reads directory entries from the handle
@@ -334,7 +386,7 @@ func (h Handle) Readdir(count int) (fis []os.FileInfo, err error) {
 	}
 	// Wrap each FileInfo
 	for i := range fis {
-		fis[i] = FileInfo{fis[i]}
+		fis[i] = FileInfo{FileInfo: fis[i], w: h.w}
 	}
 	return fis, nil
 }
@@ -345,19 +397,20 @@ func (h Handle) Stat() (fi os.FileInfo, err error) {
 	if err != nil {
 		return nil, err
 	}
-	return FileInfo{fi}, nil
+	return FileInfo{FileInfo: fi, w: h.w}, nil
 }
 
 // FileInfo represents info about a file satisfying os.FileInfo and
 // also some additional interfaces for webdav for ETag and ContentType
 type FileInfo struct {
 	os.FileInfo
+	w *WebDAV
 }
 
 // ETag returns an ETag for the FileInfo
 func (fi FileInfo) ETag(ctx context.Context) (etag string, err error) {
 	// defer log.Trace(fi, "")("etag=%q, err=%v", &etag, &err)
-	if hashType == hash.None {
+	if fi.w.opt.HashType == hash.None {
 		return "", webdav.ErrNotImplemented
 	}
 	node, ok := (fi.FileInfo).(vfs.Node)
@@ -370,7 +423,7 @@ func (fi FileInfo) ETag(ctx context.Context) (etag string, err error) {
 	if !ok {
 		return "", webdav.ErrNotImplemented
 	}
-	hash, err := o.Hash(ctx, hashType)
+	hash, err := o.Hash(ctx, fi.w.opt.HashType)
 	if err != nil || hash == "" {
 		return "", webdav.ErrNotImplemented
 	}
diff --git a/cmd/serve/webdav/webdav_test.go b/cmd/serve/webdav/webdav_test.go
index bb95b07931775..fa06260757c78 100644
--- a/cmd/serve/webdav/webdav_test.go
+++ b/cmd/serve/webdav/webdav_test.go
@@ -19,7 +19,6 @@ import (
 	"time"
 
 	_ "github.com/rclone/rclone/backend/local"
-	"github.com/rclone/rclone/cmd/serve/httplib"
 	"github.com/rclone/rclone/cmd/serve/servetest"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
@@ -40,9 +39,9 @@ const (
 
 // check interfaces
 var (
-	_ os.FileInfo         = FileInfo{nil}
-	_ webdav.ETager       = FileInfo{nil}
-	_ webdav.ContentTyper = FileInfo{nil}
+	_ os.FileInfo         = FileInfo{nil, nil}
+	_ webdav.ETager       = FileInfo{nil, nil}
+	_ webdav.ContentTyper = FileInfo{nil, nil}
 )
 
 // TestWebDav runs the webdav server then runs the unit tests for the
@@ -50,28 +49,29 @@ var (
 func TestWebDav(t *testing.T) {
 	// Configure and start the server
 	start := func(f fs.Fs) (configmap.Simple, func()) {
-		opt := httplib.DefaultOpt
-		opt.ListenAddr = testBindAddress
-		opt.BasicUser = testUser
-		opt.BasicPass = testPass
-		opt.Template = testTemplate
-		hashType = hash.MD5
+		opt := DefaultOpt
+		opt.HTTP.ListenAddr = []string{testBindAddress}
+		opt.Auth.BasicUser = testUser
+		opt.Auth.BasicPass = testPass
+		opt.Template.Path = testTemplate
+		opt.HashType = hash.MD5
 
 		// Start the server
-		w := newWebDAV(context.Background(), f, &opt)
-		assert.NoError(t, w.serve())
+		w, err := newWebDAV(context.Background(), f, &opt)
+		require.NoError(t, err)
+		require.NoError(t, w.serve())
 
 		// Config for the backend we'll use to connect to the server
 		config := configmap.Simple{
 			"type":   "webdav",
 			"vendor": "other",
-			"url":    w.Server.URL(),
+			"url":    w.Server.URLs()[0],
 			"user":   testUser,
 			"pass":   obscure.MustObscure(testPass),
 		}
 
 		return config, func() {
-			w.Close()
+			assert.NoError(t, w.Shutdown())
 			w.Wait()
 		}
 	}
@@ -98,18 +98,19 @@ func TestHTTPFunction(t *testing.T) {
 	f, err := fs.NewFs(context.Background(), "../http/testdata/files")
 	assert.NoError(t, err)
 
-	opt := httplib.DefaultOpt
-	opt.ListenAddr = testBindAddress
-	opt.Template = testTemplate
+	opt := DefaultOpt
+	opt.HTTP.ListenAddr = []string{testBindAddress}
+	opt.Template.Path = testTemplate
 
 	// Start the server
-	w := newWebDAV(context.Background(), f, &opt)
-	assert.NoError(t, w.serve())
+	w, err := newWebDAV(context.Background(), f, &opt)
+	assert.NoError(t, err)
+	require.NoError(t, w.serve())
 	defer func() {
-		w.Close()
+		assert.NoError(t, w.Shutdown())
 		w.Wait()
 	}()
-	testURL := w.Server.URL()
+	testURL := w.Server.URLs()[0]
 	pause := time.Millisecond
 	i := 0
 	for ; i < 10; i++ {

From 52443c2444e3d3c471b76fbce3f31483d8c11be9 Mon Sep 17 00:00:00 2001
From: Nolan Woods 
Date: Sun, 2 May 2021 00:56:24 -0700
Subject: [PATCH 467/560] restic: refactor to use lib/http

Co-authored-by: Nick Craig-Wood 
---
 cmd/serve/restic/cache.go                    |  18 +-
 cmd/serve/restic/cache_test.go               |   4 +-
 cmd/serve/restic/restic.go                   | 290 ++++++++++++-------
 cmd/serve/restic/restic_appendonly_test.go   |  20 +-
 cmd/serve/restic/restic_privaterepos_test.go |  59 ++--
 cmd/serve/restic/restic_test.go              |  42 ++-
 cmd/serve/restic/restic_utils_test.go        |   8 +-
 7 files changed, 265 insertions(+), 176 deletions(-)

diff --git a/cmd/serve/restic/cache.go b/cmd/serve/restic/cache.go
index f7f376c8236a5..4d22151290804 100644
--- a/cmd/serve/restic/cache.go
+++ b/cmd/serve/restic/cache.go
@@ -9,20 +9,22 @@ import (
 
 // cache implements a simple object cache
 type cache struct {
-	mu    sync.RWMutex         // protects the cache
-	items map[string]fs.Object // cache of objects
+	mu           sync.RWMutex         // protects the cache
+	items        map[string]fs.Object // cache of objects
+	cacheObjects bool                 // whether we are actually caching
 }
 
 // create a new cache
-func newCache() *cache {
+func newCache(cacheObjects bool) *cache {
 	return &cache{
-		items: map[string]fs.Object{},
+		items:        map[string]fs.Object{},
+		cacheObjects: cacheObjects,
 	}
 }
 
 // find the object at remote or return nil
 func (c *cache) find(remote string) fs.Object {
-	if !cacheObjects {
+	if !c.cacheObjects {
 		return nil
 	}
 	c.mu.RLock()
@@ -33,7 +35,7 @@ func (c *cache) find(remote string) fs.Object {
 
 // add the object to the cache
 func (c *cache) add(remote string, o fs.Object) {
-	if !cacheObjects {
+	if !c.cacheObjects {
 		return
 	}
 	c.mu.Lock()
@@ -43,7 +45,7 @@ func (c *cache) add(remote string, o fs.Object) {
 
 // remove the object from the cache
 func (c *cache) remove(remote string) {
-	if !cacheObjects {
+	if !c.cacheObjects {
 		return
 	}
 	c.mu.Lock()
@@ -53,7 +55,7 @@ func (c *cache) remove(remote string) {
 
 // remove all the items with prefix from the cache
 func (c *cache) removePrefix(prefix string) {
-	if !cacheObjects {
+	if !c.cacheObjects {
 		return
 	}
 
diff --git a/cmd/serve/restic/cache_test.go b/cmd/serve/restic/cache_test.go
index 05687ad84ca30..ffabfa7738c73 100644
--- a/cmd/serve/restic/cache_test.go
+++ b/cmd/serve/restic/cache_test.go
@@ -21,7 +21,7 @@ func (c *cache) String() string {
 }
 
 func TestCacheCRUD(t *testing.T) {
-	c := newCache()
+	c := newCache(true)
 	assert.Equal(t, "", c.String())
 	assert.Nil(t, c.find("potato"))
 	o := mockobject.New("potato")
@@ -35,7 +35,7 @@ func TestCacheCRUD(t *testing.T) {
 }
 
 func TestCacheRemovePrefix(t *testing.T) {
-	c := newCache()
+	c := newCache(true)
 	for _, remote := range []string{
 		"a",
 		"b",
diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go
index 3662fc58cfe3b..0d4db85bd3855 100644
--- a/cmd/serve/restic/restic.go
+++ b/cmd/serve/restic/restic.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"os"
 	"path"
@@ -12,34 +13,48 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/chi/v5/middleware"
 	"github.com/rclone/rclone/cmd"
-	"github.com/rclone/rclone/cmd/serve/httplib"
-	"github.com/rclone/rclone/cmd/serve/httplib/httpflags"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
+	libhttp "github.com/rclone/rclone/lib/http"
 	"github.com/rclone/rclone/lib/http/serve"
 	"github.com/rclone/rclone/lib/terminal"
 	"github.com/spf13/cobra"
 	"golang.org/x/net/http2"
 )
 
-var (
-	stdio        bool
-	appendOnly   bool
-	privateRepos bool
-	cacheObjects bool
-)
+// Options required for http server
+type Options struct {
+	Auth         libhttp.AuthConfig
+	HTTP         libhttp.Config
+	Stdio        bool
+	AppendOnly   bool
+	PrivateRepos bool
+	CacheObjects bool
+}
+
+// DefaultOpt is the default values used for Options
+var DefaultOpt = Options{
+	Auth: libhttp.DefaultAuthCfg(),
+	HTTP: libhttp.DefaultCfg(),
+}
+
+// Opt is options set by command line flags
+var Opt = DefaultOpt
 
 func init() {
-	httpflags.AddFlags(Command.Flags())
 	flagSet := Command.Flags()
-	flags.BoolVarP(flagSet, &stdio, "stdio", "", false, "Run an HTTP2 server on stdin/stdout")
-	flags.BoolVarP(flagSet, &appendOnly, "append-only", "", false, "Disallow deletion of repository data")
-	flags.BoolVarP(flagSet, &privateRepos, "private-repos", "", false, "Users can only access their private repo")
-	flags.BoolVarP(flagSet, &cacheObjects, "cache-objects", "", true, "Cache listed objects")
+	libhttp.AddAuthFlagsPrefix(flagSet, "", &Opt.Auth)
+	libhttp.AddHTTPFlagsPrefix(flagSet, "", &Opt.HTTP)
+	flags.BoolVarP(flagSet, &Opt.Stdio, "stdio", "", false, "Run an HTTP2 server on stdin/stdout")
+	flags.BoolVarP(flagSet, &Opt.AppendOnly, "append-only", "", false, "Disallow deletion of repository data")
+	flags.BoolVarP(flagSet, &Opt.PrivateRepos, "private-repos", "", false, "Users can only access their private repo")
+	flags.BoolVarP(flagSet, &Opt.CacheObjects, "cache-objects", "", true, "Cache listed objects")
 }
 
 // Command definition for cobra
@@ -127,16 +142,21 @@ these **must** end with /.  Eg
 
 The` + "`--private-repos`" + ` flag can be used to limit users to repositories starting
 with a path of ` + "`//`" + `.
-` + httplib.Help,
+` + libhttp.Help + libhttp.AuthHelp,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.40",
 	},
 	Run: func(command *cobra.Command, args []string) {
+		ctx := context.Background()
 		cmd.CheckArgs(1, 1, command, args)
 		f := cmd.NewFsSrc(args)
 		cmd.Run(false, true, command, func() error {
-			s := NewServer(f, &httpflags.Opt)
-			if stdio {
+			s, err := newServer(ctx, f, &Opt)
+			if err != nil {
+				return err
+			}
+			fs.Logf(s.f, "Serving restic REST API on %s", s.URLs())
+			if s.opt.Stdio {
 				if terminal.IsTerminal(int(os.Stdout.Fd())) {
 					return errors.New("refusing to run HTTP2 server directly on a terminal, please let restic start rclone")
 				}
@@ -148,16 +168,11 @@ with a path of ` + "`//`" + `.
 
 				httpSrv := &http2.Server{}
 				opts := &http2.ServeConnOpts{
-					Handler: s,
+					Handler: s.Server.Router(),
 				}
 				httpSrv.ServeConn(conn, opts)
 				return nil
 			}
-			err := s.Serve()
-			if err != nil {
-				return err
-			}
-			s.Wait()
 			return nil
 		})
 	},
@@ -167,101 +182,130 @@ const (
 	resticAPIV2 = "application/vnd.x.restic.rest.v2"
 )
 
-// Server contains everything to run the Server
-type Server struct {
-	*httplib.Server
-	f     fs.Fs
-	cache *cache
-}
+type contextRemoteType struct{}
 
-// NewServer returns an HTTP server that speaks the rest protocol
-func NewServer(f fs.Fs, opt *httplib.Options) *Server {
-	mux := http.NewServeMux()
-	s := &Server{
-		Server: httplib.NewServer(mux, opt),
-		f:      f,
-		cache:  newCache(),
-	}
-	mux.HandleFunc(s.Opt.BaseURL+"/", s.ServeHTTP)
-	return s
-}
+// ContextRemoteKey is a simple context key for storing the username of the request
+var ContextRemoteKey = &contextRemoteType{}
 
-// Serve runs the http server in the background.
-//
-// Use s.Close() and s.Wait() to shutdown server
-func (s *Server) Serve() error {
-	err := s.Server.Serve()
-	if err != nil {
-		return err
-	}
-	fs.Logf(s.f, "Serving restic REST API on %s", s.URL())
-	return nil
+// WithRemote makes a remote from a URL path.  This implements the backend layout
+// required by restic.
+func WithRemote(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var urlpath string
+		rctx := chi.RouteContext(r.Context())
+		if rctx != nil && rctx.RoutePath != "" {
+			urlpath = rctx.RoutePath
+		} else {
+			urlpath = r.URL.Path
+		}
+		urlpath = strings.Trim(urlpath, "/")
+		parts := matchData.FindStringSubmatch(urlpath)
+		// if no data directory, layout is flat
+		if parts != nil {
+			// otherwise map
+			// data/2159dd48 to
+			// data/21/2159dd48
+			fileName := parts[1]
+			prefix := urlpath[:len(urlpath)-len(fileName)]
+			urlpath = prefix + fileName[:2] + "/" + fileName
+		}
+		ctx := context.WithValue(r.Context(), ContextRemoteKey, urlpath)
+		next.ServeHTTP(w, r.WithContext(ctx))
+	})
 }
 
-var matchData = regexp.MustCompile("(?:^|/)data/([^/]{2,})$")
-
-// Makes a remote from a URL path.  This implements the backend layout
-// required by restic.
-func makeRemote(path string) string {
-	path = strings.Trim(path, "/")
-	parts := matchData.FindStringSubmatch(path)
-	// if no data directory, layout is flat
-	if parts == nil {
-		return path
-	}
-	// otherwise map
-	// data/2159dd48 to
-	// data/21/2159dd48
-	fileName := parts[1]
-	prefix := path[:len(path)-len(fileName)]
-	return prefix + fileName[:2] + "/" + fileName
+// Middleware to ensure authenticated user is accessing their own private folder
+func checkPrivate(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		user := chi.URLParam(r, "userID")
+		userID, ok := libhttp.CtxGetUser(r.Context())
+		if ok && user != "" && user == userID {
+			next.ServeHTTP(w, r)
+		} else {
+			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
+		}
+	})
 }
 
-// ServeHTTP reads incoming requests and dispatches them
-func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	w.Header().Set("Accept-Ranges", "bytes")
-	w.Header().Set("Server", "rclone/"+fs.Version)
+// server contains everything to run the server
+type server struct {
+	*libhttp.Server
+	f     fs.Fs
+	cache *cache
+	opt   Options
+}
 
-	path, ok := s.Path(w, r)
-	if !ok {
-		return
+func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *server, err error) {
+	s = &server{
+		f:     f,
+		cache: newCache(opt.CacheObjects),
+		opt:   *opt,
 	}
-	remote := makeRemote(path)
-	fs.Debugf(s.f, "%s %s", r.Method, path)
-
-	v := r.Context().Value(httplib.ContextUserKey)
-	if privateRepos && (v == nil || !strings.HasPrefix(path, "/"+v.(string)+"/")) {
-		http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
-		return
+	s.Server, err = libhttp.NewServer(ctx,
+		libhttp.WithConfig(opt.HTTP),
+		libhttp.WithAuth(opt.Auth),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init server: %w", err)
 	}
+	router := s.Router()
+	s.Bind(router)
+	s.Server.Serve()
+	return s, nil
+}
 
-	// Dispatch on path then method
-	if strings.HasSuffix(path, "/") {
-		switch r.Method {
-		case "GET":
-			s.listObjects(w, r, remote)
-		case "POST":
-			s.createRepo(w, r, remote)
-		default:
-			http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
+// bind helper for main Bind method
+func (s *server) bind(router chi.Router) {
+	router.MethodFunc("GET", "/*", func(w http.ResponseWriter, r *http.Request) {
+		urlpath := chi.URLParam(r, "*")
+		if urlpath == "" || strings.HasSuffix(urlpath, "/") {
+			s.listObjects(w, r)
+		} else {
+			s.serveObject(w, r)
 		}
-	} else {
-		switch r.Method {
-		case "GET", "HEAD":
-			s.serveObject(w, r, remote)
-		case "POST":
-			s.postObject(w, r, remote)
-		case "DELETE":
-			s.deleteObject(w, r, remote)
-		default:
-			http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
+	})
+	router.MethodFunc("POST", "/*", func(w http.ResponseWriter, r *http.Request) {
+		urlpath := chi.URLParam(r, "*")
+		if urlpath == "" || strings.HasSuffix(urlpath, "/") {
+			s.createRepo(w, r)
+		} else {
+			s.postObject(w, r)
 		}
+	})
+	router.MethodFunc("HEAD", "/*", s.serveObject)
+	router.MethodFunc("DELETE", "/*", s.deleteObject)
+}
+
+// Bind restic server routes to passed router
+func (s *server) Bind(router chi.Router) {
+	// FIXME
+	// if m := authX.Auth(authX.Opt); m != nil {
+	// 	router.Use(m)
+	// }
+	router.Use(
+		middleware.SetHeader("Accept-Ranges", "bytes"),
+		middleware.SetHeader("Server", "rclone/"+fs.Version),
+		WithRemote,
+	)
+
+	if s.opt.PrivateRepos {
+		router.Route("/{userID}", func(r chi.Router) {
+			r.Use(checkPrivate)
+			s.bind(r)
+		})
+		router.NotFound(func(w http.ResponseWriter, _ *http.Request) {
+			http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
+		})
+	} else {
+		s.bind(router)
 	}
 }
 
+var matchData = regexp.MustCompile("(?:^|/)data/([^/]{2,})$")
+
 // newObject returns an object with the remote given either from the
 // cache or directly
-func (s *Server) newObject(ctx context.Context, remote string) (fs.Object, error) {
+func (s *server) newObject(ctx context.Context, remote string) (fs.Object, error) {
 	o := s.cache.find(remote)
 	if o != nil {
 		return o, nil
@@ -275,7 +319,12 @@ func (s *Server) newObject(ctx context.Context, remote string) (fs.Object, error
 }
 
 // get the remote
-func (s *Server) serveObject(w http.ResponseWriter, r *http.Request, remote string) {
+func (s *server) serveObject(w http.ResponseWriter, r *http.Request) {
+	remote, ok := r.Context().Value(ContextRemoteKey).(string)
+	if !ok {
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
 	o, err := s.newObject(r.Context(), remote)
 	if err != nil {
 		fs.Debugf(remote, "%s request error: %v", r.Method, err)
@@ -286,8 +335,13 @@ func (s *Server) serveObject(w http.ResponseWriter, r *http.Request, remote stri
 }
 
 // postObject posts an object to the repository
-func (s *Server) postObject(w http.ResponseWriter, r *http.Request, remote string) {
-	if appendOnly {
+func (s *server) postObject(w http.ResponseWriter, r *http.Request) {
+	remote, ok := r.Context().Value(ContextRemoteKey).(string)
+	if !ok {
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
+	if s.opt.AppendOnly {
 		// make sure the file does not exist yet
 		_, err := s.newObject(r.Context(), remote)
 		if err == nil {
@@ -312,8 +366,13 @@ func (s *Server) postObject(w http.ResponseWriter, r *http.Request, remote strin
 }
 
 // delete the remote
-func (s *Server) deleteObject(w http.ResponseWriter, r *http.Request, remote string) {
-	if appendOnly {
+func (s *server) deleteObject(w http.ResponseWriter, r *http.Request) {
+	remote, ok := r.Context().Value(ContextRemoteKey).(string)
+	if !ok {
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
+	if s.opt.AppendOnly {
 		parts := strings.Split(r.URL.Path, "/")
 
 		// if path doesn't end in "/locks/:name", disallow the operation
@@ -362,14 +421,18 @@ func (ls *listItems) add(o fs.Object) {
 }
 
 // listObjects lists all Objects of a given type in an arbitrary order.
-func (s *Server) listObjects(w http.ResponseWriter, r *http.Request, remote string) {
-	fs.Debugf(remote, "list request")
-
+func (s *server) listObjects(w http.ResponseWriter, r *http.Request) {
+	remote, ok := r.Context().Value(ContextRemoteKey).(string)
+	if !ok {
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
 	if r.Header.Get("Accept") != resticAPIV2 {
-		fs.Errorf(remote, "Restic v2 API required")
-		http.Error(w, "Restic v2 API required", http.StatusBadRequest)
+		fs.Errorf(remote, "Restic v2 API required for List Objects")
+		http.Error(w, "Restic v2 API required for List Objects", http.StatusBadRequest)
 		return
 	}
+	fs.Debugf(remote, "list request")
 
 	// make sure an empty list is returned, and not a 'nil' value
 	ls := listItems{}
@@ -408,7 +471,12 @@ func (s *Server) listObjects(w http.ResponseWriter, r *http.Request, remote stri
 // createRepo creates repository directories.
 //
 // We don't bother creating the data dirs as rclone will create them on the fly
-func (s *Server) createRepo(w http.ResponseWriter, r *http.Request, remote string) {
+func (s *server) createRepo(w http.ResponseWriter, r *http.Request) {
+	remote, ok := r.Context().Value(ContextRemoteKey).(string)
+	if !ok {
+		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+		return
+	}
 	fs.Infof(remote, "Creating repository")
 
 	if r.URL.Query().Get("create") != "true" {
diff --git a/cmd/serve/restic/restic_appendonly_test.go b/cmd/serve/restic/restic_appendonly_test.go
index b3562db9edcd5..2499481a1a054 100644
--- a/cmd/serve/restic/restic_appendonly_test.go
+++ b/cmd/serve/restic/restic_appendonly_test.go
@@ -1,6 +1,7 @@
 package restic
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/hex"
 	"io"
@@ -9,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/cmd"
-	"github.com/rclone/rclone/cmd/serve/httplib/httpflags"
 	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/stretchr/testify/require"
 )
@@ -62,6 +62,7 @@ func createOverwriteDeleteSeq(t testing.TB, path string) []TestRequest {
 
 // TestResticHandler runs tests on the restic handler code, especially in append-only mode.
 func TestResticHandler(t *testing.T) {
+	ctx := context.Background()
 	configfile.Install()
 	buf := make([]byte, 32)
 	_, err := io.ReadFull(rand.Reader, buf)
@@ -110,19 +111,18 @@ func TestResticHandler(t *testing.T) {
 	// setup rclone with a local backend in a temporary directory
 	tempdir := t.TempDir()
 
-	// globally set append-only mode
-	prev := appendOnly
-	appendOnly = true
-	defer func() {
-		appendOnly = prev // reset when done
-	}()
+	// set append-only mode
+	opt := newOpt()
+	opt.AppendOnly = true
 
 	// make a new file system in the temp dir
 	f := cmd.NewFsSrc([]string{tempdir})
-	srv := NewServer(f, &httpflags.Opt)
+	s, err := newServer(ctx, f, &opt)
+	require.NoError(t, err)
+	router := s.Server.Router()
 
 	// create the repo
-	checkRequest(t, srv.ServeHTTP,
+	checkRequest(t, router.ServeHTTP,
 		newRequest(t, "POST", "/?create=true", nil),
 		[]wantFunc{wantCode(http.StatusOK)})
 
@@ -130,7 +130,7 @@ func TestResticHandler(t *testing.T) {
 		t.Run("", func(t *testing.T) {
 			for i, seq := range test.seq {
 				t.Logf("request %v: %v %v", i, seq.req.Method, seq.req.URL.Path)
-				checkRequest(t, srv.ServeHTTP, seq.req, seq.want)
+				checkRequest(t, router.ServeHTTP, seq.req, seq.want)
 			}
 		})
 	}
diff --git a/cmd/serve/restic/restic_privaterepos_test.go b/cmd/serve/restic/restic_privaterepos_test.go
index 4faae363cc2f3..1b89e8681fdf2 100644
--- a/cmd/serve/restic/restic_privaterepos_test.go
+++ b/cmd/serve/restic/restic_privaterepos_test.go
@@ -8,23 +8,21 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/rclone/rclone/cmd/serve/httplib"
-
 	"github.com/rclone/rclone/cmd"
-	"github.com/rclone/rclone/cmd/serve/httplib/httpflags"
 	"github.com/stretchr/testify/require"
 )
 
 // newAuthenticatedRequest returns a new HTTP request with the given params.
-func newAuthenticatedRequest(t testing.TB, method, path string, body io.Reader) *http.Request {
+func newAuthenticatedRequest(t testing.TB, method, path string, body io.Reader, user, pass string) *http.Request {
 	req := newRequest(t, method, path, body)
-	req = req.WithContext(context.WithValue(req.Context(), httplib.ContextUserKey, "test"))
+	req.SetBasicAuth(user, pass)
 	req.Header.Add("Accept", resticAPIV2)
 	return req
 }
 
 // TestResticPrivateRepositories runs tests on the restic handler code for private repositories
 func TestResticPrivateRepositories(t *testing.T) {
+	ctx := context.Background()
 	buf := make([]byte, 32)
 	_, err := io.ReadFull(rand.Reader, buf)
 	require.NoError(t, err)
@@ -32,42 +30,49 @@ func TestResticPrivateRepositories(t *testing.T) {
 	// setup rclone with a local backend in a temporary directory
 	tempdir := t.TempDir()
 
-	// globally set private-repos mode & test user
-	prev := privateRepos
-	prevUser := httpflags.Opt.BasicUser
-	prevPassword := httpflags.Opt.BasicPass
-	privateRepos = true
-	httpflags.Opt.BasicUser = "test"
-	httpflags.Opt.BasicPass = "password"
-	// reset when done
-	defer func() {
-		privateRepos = prev
-		httpflags.Opt.BasicUser = prevUser
-		httpflags.Opt.BasicPass = prevPassword
-	}()
+	opt := newOpt()
+
+	// set private-repos mode & test user
+	opt.PrivateRepos = true
+	opt.Auth.BasicUser = "test"
+	opt.Auth.BasicPass = "password"
 
 	// make a new file system in the temp dir
 	f := cmd.NewFsSrc([]string{tempdir})
-	srv := NewServer(f, &httpflags.Opt)
+	s, err := newServer(ctx, f, &opt)
+	require.NoError(t, err)
+	router := s.Server.Router()
 
 	// Requesting /test/ should allow access
 	reqs := []*http.Request{
-		newAuthenticatedRequest(t, "POST", "/test/?create=true", nil),
-		newAuthenticatedRequest(t, "POST", "/test/config", strings.NewReader("foobar test config")),
-		newAuthenticatedRequest(t, "GET", "/test/config", nil),
+		newAuthenticatedRequest(t, "POST", "/test/?create=true", nil, opt.Auth.BasicUser, opt.Auth.BasicPass),
+		newAuthenticatedRequest(t, "POST", "/test/config", strings.NewReader("foobar test config"), opt.Auth.BasicUser, opt.Auth.BasicPass),
+		newAuthenticatedRequest(t, "GET", "/test/config", nil, opt.Auth.BasicUser, opt.Auth.BasicPass),
+	}
+	for _, req := range reqs {
+		checkRequest(t, router.ServeHTTP, req, []wantFunc{wantCode(http.StatusOK)})
+	}
+
+	// Requesting with bad credentials should raise unauthorised errors
+	reqs = []*http.Request{
+		newRequest(t, "GET", "/test/config", nil),
+		newAuthenticatedRequest(t, "GET", "/test/config", nil, opt.Auth.BasicUser, ""),
+		newAuthenticatedRequest(t, "GET", "/test/config", nil, "", opt.Auth.BasicPass),
+		newAuthenticatedRequest(t, "GET", "/test/config", nil, opt.Auth.BasicUser+"x", opt.Auth.BasicPass),
+		newAuthenticatedRequest(t, "GET", "/test/config", nil, opt.Auth.BasicUser, opt.Auth.BasicPass+"x"),
 	}
 	for _, req := range reqs {
-		checkRequest(t, srv.ServeHTTP, req, []wantFunc{wantCode(http.StatusOK)})
+		checkRequest(t, router.ServeHTTP, req, []wantFunc{wantCode(http.StatusUnauthorized)})
 	}
 
 	// Requesting everything else should raise forbidden errors
 	reqs = []*http.Request{
-		newAuthenticatedRequest(t, "GET", "/", nil),
-		newAuthenticatedRequest(t, "POST", "/other_user", nil),
-		newAuthenticatedRequest(t, "GET", "/other_user/config", nil),
+		newAuthenticatedRequest(t, "GET", "/", nil, opt.Auth.BasicUser, opt.Auth.BasicPass),
+		newAuthenticatedRequest(t, "POST", "/other_user", nil, opt.Auth.BasicUser, opt.Auth.BasicPass),
+		newAuthenticatedRequest(t, "GET", "/other_user/config", nil, opt.Auth.BasicUser, opt.Auth.BasicPass),
 	}
 	for _, req := range reqs {
-		checkRequest(t, srv.ServeHTTP, req, []wantFunc{wantCode(http.StatusForbidden)})
+		checkRequest(t, router.ServeHTTP, req, []wantFunc{wantCode(http.StatusForbidden)})
 	}
 
 }
diff --git a/cmd/serve/restic/restic_test.go b/cmd/serve/restic/restic_test.go
index d9f343ca55a4f..3db040bddcaf4 100644
--- a/cmd/serve/restic/restic_test.go
+++ b/cmd/serve/restic/restic_test.go
@@ -5,14 +5,16 @@ package restic
 
 import (
 	"context"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"testing"
 
 	_ "github.com/rclone/rclone/backend/all"
-	"github.com/rclone/rclone/cmd/serve/httplib"
 	"github.com/rclone/rclone/fstest"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -20,16 +22,24 @@ const (
 	resticSource    = "../../../../../restic/restic"
 )
 
+func newOpt() Options {
+	opt := DefaultOpt
+	opt.HTTP.ListenAddr = []string{testBindAddress}
+	return opt
+}
+
 // TestRestic runs the restic server then runs the unit tests for the
 // restic remote against it.
-func TestRestic(t *testing.T) {
+//
+// Requires the restic source code in the location indicated by resticSource.
+func TestResticIntegration(t *testing.T) {
+	ctx := context.Background()
 	_, err := os.Stat(resticSource)
 	if err != nil {
 		t.Skipf("Skipping test as restic source not found: %v", err)
 	}
 
-	opt := httplib.DefaultOpt
-	opt.ListenAddr = testBindAddress
+	opt := newOpt()
 
 	fstest.Initialise()
 
@@ -41,16 +51,16 @@ func TestRestic(t *testing.T) {
 	assert.NoError(t, err)
 
 	// Start the server
-	w := NewServer(fremote, &opt)
-	assert.NoError(t, w.Serve())
+	s, err := newServer(ctx, fremote, &opt)
+	require.NoError(t, err)
+	testURL := s.Server.URLs()[0]
 	defer func() {
-		w.Close()
-		w.Wait()
+		_ = s.Shutdown()
 	}()
 
 	// Change directory to run the tests
 	err = os.Chdir(resticSource)
-	assert.NoError(t, err, "failed to cd to restic source code")
+	require.NoError(t, err, "failed to cd to restic source code")
 
 	// Run the restic tests
 	runTests := func(path string) {
@@ -60,7 +70,7 @@ func TestRestic(t *testing.T) {
 		}
 		cmd := exec.Command("go", args...)
 		cmd.Env = append(os.Environ(),
-			"RESTIC_TEST_REST_REPOSITORY=rest:"+w.Server.URL()+path,
+			"RESTIC_TEST_REST_REPOSITORY=rest:"+testURL+path,
 			"GO111MODULE=on",
 		)
 		out, err := cmd.CombinedOutput()
@@ -81,7 +91,6 @@ func TestMakeRemote(t *testing.T) {
 	for _, test := range []struct {
 		in, want string
 	}{
-		{"", ""},
 		{"/", ""},
 		{"/data", "data"},
 		{"/data/", "data"},
@@ -94,7 +103,14 @@ func TestMakeRemote(t *testing.T) {
 		{"/keys/12", "keys/12"},
 		{"/keys/123", "keys/123"},
 	} {
-		got := makeRemote(test.in)
-		assert.Equal(t, test.want, got, test.in)
+		r := httptest.NewRequest("GET", test.in, nil)
+		w := httptest.NewRecorder()
+		next := http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) {
+			remote, ok := request.Context().Value(ContextRemoteKey).(string)
+			assert.True(t, ok, "Failed to get remote from context")
+			assert.Equal(t, test.want, remote, test.in)
+		})
+		got := WithRemote(next)
+		got.ServeHTTP(w, r)
 	}
 }
diff --git a/cmd/serve/restic/restic_utils_test.go b/cmd/serve/restic/restic_utils_test.go
index 0721f7c0f3adb..587c7e1abc45b 100644
--- a/cmd/serve/restic/restic_utils_test.go
+++ b/cmd/serve/restic/restic_utils_test.go
@@ -7,7 +7,6 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 // declare a few helper functions
@@ -15,11 +14,10 @@ import (
 // wantFunc tests the HTTP response in res and marks the test as errored if something is incorrect.
 type wantFunc func(t testing.TB, res *httptest.ResponseRecorder)
 
-// newRequest returns a new HTTP request with the given params. On error, the
-// test is marked as failed.
+// newRequest returns a new HTTP request with the given params
 func newRequest(t testing.TB, method, path string, body io.Reader) *http.Request {
-	req, err := http.NewRequest(method, path, body)
-	require.NoError(t, err)
+	req := httptest.NewRequest(method, path, body)
+	req.Header.Add("Accept", resticAPIV2)
 	return req
 }
 

From a85c0b0cc222699cfc7dc6577ad1f0d9ec7754c7 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Mon, 12 Dec 2022 11:47:15 +0000
Subject: [PATCH 468/560] cmd/serve/httplib: remove as it is now replaced by
 lib/http

---
 cmd/serve/httplib/httpflags/httpflags.go |  39 --
 cmd/serve/httplib/httplib.go             | 438 -----------------------
 2 files changed, 477 deletions(-)
 delete mode 100644 cmd/serve/httplib/httpflags/httpflags.go
 delete mode 100644 cmd/serve/httplib/httplib.go

diff --git a/cmd/serve/httplib/httpflags/httpflags.go b/cmd/serve/httplib/httpflags/httpflags.go
deleted file mode 100644
index d06ab8076d071..0000000000000
--- a/cmd/serve/httplib/httpflags/httpflags.go
+++ /dev/null
@@ -1,39 +0,0 @@
-// Package httpflags provides utility functionality to HTTP.
-package httpflags
-
-import (
-	"github.com/rclone/rclone/cmd/serve/httplib"
-	"github.com/rclone/rclone/fs/config/flags"
-	"github.com/rclone/rclone/fs/rc"
-	"github.com/spf13/pflag"
-)
-
-// Options set by command line flags
-var (
-	Opt = httplib.DefaultOpt
-)
-
-// AddFlagsPrefix adds flags for the httplib
-func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *httplib.Options) {
-	rc.AddOption(prefix+"http", &Opt)
-	flags.StringVarP(flagSet, &Opt.ListenAddr, prefix+"addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to")
-	flags.DurationVarP(flagSet, &Opt.ServerReadTimeout, prefix+"server-read-timeout", "", Opt.ServerReadTimeout, "Timeout for server reading data")
-	flags.DurationVarP(flagSet, &Opt.ServerWriteTimeout, prefix+"server-write-timeout", "", Opt.ServerWriteTimeout, "Timeout for server writing data")
-	flags.IntVarP(flagSet, &Opt.MaxHeaderBytes, prefix+"max-header-bytes", "", Opt.MaxHeaderBytes, "Maximum size of request header")
-	flags.StringVarP(flagSet, &Opt.SslCert, prefix+"cert", "", Opt.SslCert, "SSL PEM key (concatenation of certificate and CA certificate)")
-	flags.StringVarP(flagSet, &Opt.SslKey, prefix+"key", "", Opt.SslKey, "SSL PEM Private key")
-	flags.StringVarP(flagSet, &Opt.ClientCA, prefix+"client-ca", "", Opt.ClientCA, "Client certificate authority to verify clients with")
-	flags.StringVarP(flagSet, &Opt.HtPasswd, prefix+"htpasswd", "", Opt.HtPasswd, "htpasswd file - if not provided no authentication is done")
-	flags.StringVarP(flagSet, &Opt.Realm, prefix+"realm", "", Opt.Realm, "Realm for authentication")
-	flags.StringVarP(flagSet, &Opt.BasicUser, prefix+"user", "", Opt.BasicUser, "User name for authentication")
-	flags.StringVarP(flagSet, &Opt.BasicPass, prefix+"pass", "", Opt.BasicPass, "Password for authentication")
-	flags.StringVarP(flagSet, &Opt.BaseURL, prefix+"baseurl", "", Opt.BaseURL, "Prefix for URLs - leave blank for root")
-	flags.StringVarP(flagSet, &Opt.Template, prefix+"template", "", Opt.Template, "User-specified template")
-	flags.StringVarP(flagSet, &Opt.MinTLSVersion, prefix+"min-tls-version", "", Opt.MinTLSVersion, "Minimum TLS version that is acceptable")
-
-}
-
-// AddFlags adds flags for the httplib
-func AddFlags(flagSet *pflag.FlagSet) {
-	AddFlagsPrefix(flagSet, "", &Opt)
-}
diff --git a/cmd/serve/httplib/httplib.go b/cmd/serve/httplib/httplib.go
deleted file mode 100644
index d8a625ea0a892..0000000000000
--- a/cmd/serve/httplib/httplib.go
+++ /dev/null
@@ -1,438 +0,0 @@
-// Package httplib provides common functionality for http servers
-//
-// Deprecated: httplib has been replaced with lib/http
-package httplib
-
-import (
-	"context"
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/base64"
-	"fmt"
-	"html/template"
-	"log"
-	"net"
-	"net/http"
-	"os"
-	"strings"
-	"time"
-
-	auth "github.com/abbot/go-http-auth"
-	"github.com/rclone/rclone/cmd/serve/http/data"
-	"github.com/rclone/rclone/fs"
-)
-
-// Globals
-var ()
-
-// Help contains text describing the http server to add to the command
-// help.
-var Help = `
-### Server options
-
-Use ` + "`--addr`" + ` to specify which IP address and port the server should
-listen on, e.g. ` + "`--addr 1.2.3.4:8000` or `--addr :8080`" + ` to
-listen to all IPs.  By default it only listens on localhost.  You can use port
-:0 to let the OS choose an available port.
-
-If you set ` + "`--addr`" + ` to listen on a public or LAN accessible IP address
-then using Authentication is advised - see the next section for info.
-
-` + "`--server-read-timeout` and `--server-write-timeout`" + ` can be used to
-control the timeouts on the server.  Note that this is the total time
-for a transfer.
-
-` + "`--max-header-bytes`" + ` controls the maximum number of bytes the server will
-accept in the HTTP header.
-
-` + "`--baseurl`" + ` controls the URL prefix that rclone serves from.  By default
-rclone will serve from the root.  If you used ` + "`--baseurl \"/rclone\"`" + ` then
-rclone would serve from a URL starting with "/rclone/".  This is
-useful if you wish to proxy rclone serve.  Rclone automatically
-inserts leading and trailing "/" on ` + "`--baseurl`" + `, so ` + "`--baseurl \"rclone\"`" + `,
-` + "`--baseurl \"/rclone\"` and `--baseurl \"/rclone/\"`" + ` are all treated
-identically.
-
-` + "`--template`" + ` allows a user to specify a custom markup template for HTTP
-and WebDAV serve functions.  The server exports the following markup
-to be used within the template to server pages:
-
-| Parameter   | Description |
-| :---------- | :---------- |
-| .Name       | The full path of a file/directory. |
-| .Title      | Directory listing of .Name |
-| .Sort       | The current sort used.  This is changeable via ?sort= parameter |
-|             | Sort Options: namedirfirst,name,size,time (default namedirfirst) |
-| .Order      | The current ordering used.  This is changeable via ?order= parameter |
-|             | Order Options: asc,desc (default asc) |
-| .Query      | Currently unused. |
-| .Breadcrumb | Allows for creating a relative navigation |
-|-- .Link     | The relative to the root link of the Text. |
-|-- .Text     | The Name of the directory. |
-| .Entries    | Information about a specific file/directory. |
-|-- .URL      | The 'url' of an entry.  |
-|-- .Leaf     | Currently same as 'URL' but intended to be 'just' the name. |
-|-- .IsDir    | Boolean for if an entry is a directory or not. |
-|-- .Size     | Size in Bytes of the entry. |
-|-- .ModTime  | The UTC timestamp of an entry. |
-
-#### Authentication
-
-By default this will serve files without needing a login.
-
-You can either use an htpasswd file which can take lots of users, or
-set a single username and password with the ` + "`--user` and `--pass`" + ` flags.
-
-Use ` + "`--htpasswd /path/to/htpasswd`" + ` to provide an htpasswd file.  This is
-in standard apache format and supports MD5, SHA1 and BCrypt for basic
-authentication.  Bcrypt is recommended.
-
-To create an htpasswd file:
-
-    touch htpasswd
-    htpasswd -B htpasswd user
-    htpasswd -B htpasswd anotherUser
-
-The password file can be updated while rclone is running.
-
-Use ` + "`--realm`" + ` to set the authentication realm.
-
-#### SSL/TLS
-
-By default this will serve over HTTP.  If you want you can serve over
-HTTPS.  You will need to supply the ` + "`--cert` and `--key`" + ` flags.
-If you wish to do client side certificate validation then you will need to
-supply ` + "`--client-ca`" + ` also.
-
-` + "`--cert`" + ` should be either a PEM encoded certificate or a concatenation
-of that with the CA certificate.  ` + "`--key`" + ` should be the PEM encoded
-private key and ` + "`--client-ca`" + ` should be the PEM encoded client
-certificate authority certificate.
-
---min-tls-version is minimum TLS version that is acceptable. Valid
-  values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default
-  "tls1.0").
-`
-
-// Options contains options for the http Server
-type Options struct {
-	ListenAddr         string        // Port to listen on
-	BaseURL            string        // prefix to strip from URLs
-	ServerReadTimeout  time.Duration // Timeout for server reading data
-	ServerWriteTimeout time.Duration // Timeout for server writing data
-	MaxHeaderBytes     int           // Maximum size of request header
-	SslCert            string        // SSL PEM key (concatenation of certificate and CA certificate)
-	SslKey             string        // SSL PEM Private key
-	ClientCA           string        // Client certificate authority to verify clients with
-	HtPasswd           string        // htpasswd file - if not provided no authentication is done
-	Realm              string        // realm for authentication
-	BasicUser          string        // single username for basic auth if not using Htpasswd
-	BasicPass          string        // password for BasicUser
-	Auth               AuthFn        `json:"-"` // custom Auth (not set by command line flags)
-	Template           string        // User specified template
-	MinTLSVersion      string        // MinTLSVersion contains the minimum TLS version that is acceptable
-}
-
-// AuthFn if used will be used to authenticate user, pass. If an error
-// is returned then the user is not authenticated.
-//
-// If a non nil value is returned then it is added to the context under the key
-type AuthFn func(user, pass string) (value interface{}, err error)
-
-// DefaultOpt is the default values used for Options
-var DefaultOpt = Options{
-	ListenAddr:         "localhost:8080",
-	Realm:              "rclone",
-	ServerReadTimeout:  1 * time.Hour,
-	ServerWriteTimeout: 1 * time.Hour,
-	MaxHeaderBytes:     4096,
-	MinTLSVersion:      "tls1.0",
-}
-
-// Server contains info about the running http server
-type Server struct {
-	Opt             Options
-	handler         http.Handler // original handler
-	listener        net.Listener
-	waitChan        chan struct{} // for waiting on the listener to close
-	httpServer      *http.Server
-	basicPassHashed string
-	useSSL          bool               // if server is configured for SSL/TLS
-	usingAuth       bool               // set if authentication is configured
-	HTMLTemplate    *template.Template // HTML template for web interface
-}
-
-type contextUserType struct{}
-
-// ContextUserKey is a simple context key for storing the username of the request
-var ContextUserKey = &contextUserType{}
-
-type contextAuthType struct{}
-
-// ContextAuthKey is a simple context key for storing info returned by AuthFn
-var ContextAuthKey = &contextAuthType{}
-
-// singleUserProvider provides the encrypted password for a single user
-func (s *Server) singleUserProvider(user, realm string) string {
-	if user == s.Opt.BasicUser {
-		return s.basicPassHashed
-	}
-	return ""
-}
-
-// parseAuthorization parses the Authorization header into user, pass
-// it returns a boolean as to whether the parse was successful
-func parseAuthorization(r *http.Request) (user, pass string, ok bool) {
-	authHeader := r.Header.Get("Authorization")
-	if authHeader != "" {
-		s := strings.SplitN(authHeader, " ", 2)
-		if len(s) == 2 && s[0] == "Basic" {
-			b, err := base64.StdEncoding.DecodeString(s[1])
-			if err == nil {
-				parts := strings.SplitN(string(b), ":", 2)
-				user = parts[0]
-				if len(parts) > 1 {
-					pass = parts[1]
-					ok = true
-				}
-			}
-		}
-	}
-	return
-}
-
-// NewServer creates an http server.  The opt can be nil in which case
-// the default options will be used.
-func NewServer(handler http.Handler, opt *Options) *Server {
-	s := &Server{
-		handler: handler,
-	}
-
-	// Make a copy of the options
-	if opt != nil {
-		s.Opt = *opt
-	} else {
-		s.Opt = DefaultOpt
-	}
-
-	// Use htpasswd if required on everything
-	if s.Opt.HtPasswd != "" || s.Opt.BasicUser != "" || s.Opt.Auth != nil {
-		var authenticator *auth.BasicAuth
-		if s.Opt.Auth == nil {
-			var secretProvider auth.SecretProvider
-			if s.Opt.HtPasswd != "" {
-				fs.Infof(nil, "Using %q as htpasswd storage", s.Opt.HtPasswd)
-				secretProvider = auth.HtpasswdFileProvider(s.Opt.HtPasswd)
-			} else {
-				fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", s.Opt.BasicUser)
-				s.basicPassHashed = string(auth.MD5Crypt([]byte(s.Opt.BasicPass), []byte("dlPL2MqE"), []byte("$1$")))
-				secretProvider = s.singleUserProvider
-			}
-			authenticator = auth.NewBasicAuthenticator(s.Opt.Realm, secretProvider)
-		}
-		oldHandler := handler
-		handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			// No auth wanted for OPTIONS method
-			if r.Method == "OPTIONS" {
-				oldHandler.ServeHTTP(w, r)
-				return
-			}
-			unauthorized := func() {
-				w.Header().Set("Content-Type", "text/plain")
-				w.Header().Set("WWW-Authenticate", `Basic realm="`+s.Opt.Realm+`"`)
-				http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
-			}
-			user, pass, authValid := parseAuthorization(r)
-			if !authValid {
-				unauthorized()
-				return
-			}
-			if s.Opt.Auth == nil {
-				if username := authenticator.CheckAuth(r); username == "" {
-					fs.Infof(r.URL.Path, "%s: Unauthorized request from %s", r.RemoteAddr, user)
-					unauthorized()
-					return
-				}
-			} else {
-				// Custom Auth
-				value, err := s.Opt.Auth(user, pass)
-				if err != nil {
-					fs.Infof(r.URL.Path, "%s: Auth failed from %s: %v", r.RemoteAddr, user, err)
-					unauthorized()
-					return
-				}
-				if value != nil {
-					r = r.WithContext(context.WithValue(r.Context(), ContextAuthKey, value))
-				}
-			}
-			r = r.WithContext(context.WithValue(r.Context(), ContextUserKey, user))
-			oldHandler.ServeHTTP(w, r)
-		})
-		s.usingAuth = true
-	}
-
-	s.useSSL = s.Opt.SslKey != ""
-	if (s.Opt.SslCert != "") != s.useSSL {
-		log.Fatalf("Need both -cert and -key to use SSL")
-	}
-
-	// If a Base URL is set then serve from there
-	s.Opt.BaseURL = strings.Trim(s.Opt.BaseURL, "/")
-	if s.Opt.BaseURL != "" {
-		s.Opt.BaseURL = "/" + s.Opt.BaseURL
-	}
-
-	var minTLSVersion uint16
-	switch opt.MinTLSVersion {
-	case "tls1.0":
-		minTLSVersion = tls.VersionTLS10
-	case "tls1.1":
-		minTLSVersion = tls.VersionTLS11
-	case "tls1.2":
-		minTLSVersion = tls.VersionTLS12
-	case "tls1.3":
-		minTLSVersion = tls.VersionTLS13
-	default:
-		log.Fatalf("Invalid value for --min-tls-version")
-	}
-
-	// FIXME make a transport?
-	s.httpServer = &http.Server{
-		Addr:              s.Opt.ListenAddr,
-		Handler:           handler,
-		ReadTimeout:       s.Opt.ServerReadTimeout,
-		WriteTimeout:      s.Opt.ServerWriteTimeout,
-		MaxHeaderBytes:    s.Opt.MaxHeaderBytes,
-		ReadHeaderTimeout: 10 * time.Second, // time to send the headers
-		IdleTimeout:       60 * time.Second, // time to keep idle connections open
-		TLSConfig: &tls.Config{
-			MinVersion: minTLSVersion,
-		},
-	}
-
-	if s.Opt.ClientCA != "" {
-		if !s.useSSL {
-			log.Fatalf("Can't use --client-ca without --cert and --key")
-		}
-		certpool := x509.NewCertPool()
-		pem, err := os.ReadFile(s.Opt.ClientCA)
-		if err != nil {
-			log.Fatalf("Failed to read client certificate authority: %v", err)
-		}
-		if !certpool.AppendCertsFromPEM(pem) {
-			log.Fatalf("Can't parse client certificate authority")
-		}
-		s.httpServer.TLSConfig.ClientCAs = certpool
-		s.httpServer.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert
-	}
-
-	htmlTemplate, templateErr := data.GetTemplate(s.Opt.Template)
-	if templateErr != nil {
-		log.Fatalf(templateErr.Error())
-	}
-	s.HTMLTemplate = htmlTemplate
-
-	return s
-}
-
-// Serve runs the server - returns an error only if
-// the listener was not started; does not block, so
-// use s.Wait() to block on the listener indefinitely.
-func (s *Server) Serve() error {
-	ln, err := net.Listen("tcp", s.httpServer.Addr)
-	if err != nil {
-		return fmt.Errorf("start server failed: %w", err)
-	}
-	s.listener = ln
-	s.waitChan = make(chan struct{})
-	go func() {
-		var err error
-		if s.useSSL {
-			// hacky hack to get this to work with old Go versions, which
-			// don't have ServeTLS on http.Server; see PR #2194.
-			type tlsServer interface {
-				ServeTLS(ln net.Listener, cert, key string) error
-			}
-			srvIface := interface{}(s.httpServer)
-			if tlsSrv, ok := srvIface.(tlsServer); ok {
-				// yay -- we get easy TLS support with HTTP/2
-				err = tlsSrv.ServeTLS(s.listener, s.Opt.SslCert, s.Opt.SslKey)
-			} else {
-				// oh well -- we can still do TLS but might not have HTTP/2
-				tlsConfig := new(tls.Config)
-				tlsConfig.Certificates = make([]tls.Certificate, 1)
-				tlsConfig.Certificates[0], err = tls.LoadX509KeyPair(s.Opt.SslCert, s.Opt.SslKey)
-				if err != nil {
-					log.Printf("Error loading key pair: %v", err)
-				}
-				tlsLn := tls.NewListener(s.listener, tlsConfig)
-				err = s.httpServer.Serve(tlsLn)
-			}
-		} else {
-			err = s.httpServer.Serve(s.listener)
-		}
-		if err != nil {
-			log.Printf("Error on serving HTTP server: %v", err)
-		}
-	}()
-	return nil
-}
-
-// Wait blocks while the listener is open.
-func (s *Server) Wait() {
-	<-s.waitChan
-}
-
-// Close shuts the running server down
-func (s *Server) Close() {
-	err := s.httpServer.Close()
-	if err != nil {
-		log.Printf("Error on closing HTTP server: %v", err)
-		return
-	}
-	close(s.waitChan)
-}
-
-// URL returns the serving address of this server
-func (s *Server) URL() string {
-	proto := "http"
-	if s.useSSL {
-		proto = "https"
-	}
-	addr := s.Opt.ListenAddr
-	// prefer actual listener address if using ":port" or "addr:0"
-	useActualAddress := addr == "" || addr[0] == ':' || addr[len(addr)-1] == ':' || strings.HasSuffix(addr, ":0")
-	if s.listener != nil && useActualAddress {
-		// use actual listener address; required if using 0-port
-		// (i.e. port assigned by operating system)
-		addr = s.listener.Addr().String()
-	}
-	return fmt.Sprintf("%s://%s%s/", proto, addr, s.Opt.BaseURL)
-}
-
-// UsingAuth returns true if authentication is required
-func (s *Server) UsingAuth() bool {
-	return s.usingAuth
-}
-
-// Path returns the current path with the Prefix stripped
-//
-// If it returns false, then the path was invalid and the handler
-// should exit as the error response has already been sent
-func (s *Server) Path(w http.ResponseWriter, r *http.Request) (Path string, ok bool) {
-	Path = r.URL.Path
-	if s.Opt.BaseURL == "" {
-		return Path, true
-	}
-	if !strings.HasPrefix(Path, s.Opt.BaseURL+"/") {
-		// Send a redirect if the BaseURL was requested without a /
-		if Path == s.Opt.BaseURL {
-			http.Redirect(w, r, s.Opt.BaseURL+"/", http.StatusPermanentRedirect)
-			return Path, false
-		}
-		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
-		return Path, false
-	}
-	Path = Path[len(s.Opt.BaseURL):]
-	return Path, true
-}

From 5db4493557cd45fa799d951f53183e3a3a9d6aba Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Wed, 14 Dec 2022 15:37:18 +0000
Subject: [PATCH 469/560] lib/http: fix race condition

---
 lib/http/context.go | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/lib/http/context.go b/lib/http/context.go
index 34bff8fa63e26..239572c756522 100644
--- a/lib/http/context.go
+++ b/lib/http/context.go
@@ -19,12 +19,9 @@ const (
 func NewBaseContext(ctx context.Context, url string) func(l net.Listener) context.Context {
 	return func(l net.Listener) context.Context {
 		if l.Addr().Network() == "unix" {
-			ctx = context.WithValue(ctx, ctxKeyUnixSock, true)
-			return ctx
+			return context.WithValue(ctx, ctxKeyUnixSock, true)
 		}
-
-		ctx = context.WithValue(ctx, ctxKeyPublicURL, url)
-		return ctx
+		return context.WithValue(ctx, ctxKeyPublicURL, url)
 	}
 }
 

From 7edb4c0162a67768c47e3dd7bbae3ae9819039e8 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Wed, 14 Dec 2022 14:37:07 +0000
Subject: [PATCH 470/560] sftp: fix NewObject with leading /

This was breaking the use of operations/stat with remote with an
initial /

See: https://forum.rclone.org/t/rclone-rc-api-operations-stat-is-not-working-for-sftp-remotes/34560
---
 backend/sftp/sftp.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go
index 1097452674a5a..c998180c7c6e0 100644
--- a/backend/sftp/sftp.go
+++ b/backend/sftp/sftp.go
@@ -1775,11 +1775,14 @@ func (o *Object) setMetadata(info os.FileInfo) {
 
 // statRemote stats the file or directory at the remote given
 func (f *Fs) stat(ctx context.Context, remote string) (info os.FileInfo, err error) {
+	absPath := remote
+	if !strings.HasPrefix(remote, "/") {
+		absPath = path.Join(f.absRoot, remote)
+	}
 	c, err := f.getSftpConnection(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("stat: %w", err)
 	}
-	absPath := path.Join(f.absRoot, remote)
 	info, err = c.sftpClient.Stat(absPath)
 	f.putSftpConnection(&c, err)
 	return info, err

From 67fc227684616fd210e81b26766c58e780e5e199 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Fri, 2 Dec 2022 13:44:08 +0000
Subject: [PATCH 471/560] config: add config/setpath for setting config path
 via rc/librclone

---
 fs/config/rc.go         | 24 ++++++++++++++++++++++++
 fs/config/rc_test.go    | 18 ++++++++++++++++++
 librclone/ctest/ctest.c |  3 +++
 3 files changed, 45 insertions(+)

diff --git a/fs/config/rc.go b/fs/config/rc.go
index 78f6d4866d632..f5902628d1338 100644
--- a/fs/config/rc.go
+++ b/fs/config/rc.go
@@ -217,3 +217,27 @@ func rcDelete(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	DeleteRemote(name)
 	return nil, nil
 }
+
+func init() {
+	rc.Add(rc.Call{
+		Path:         "config/setpath",
+		Fn:           rcSetPath,
+		Title:        "Set the path of the config file",
+		AuthRequired: true,
+		Help: `
+Parameters:
+
+- path - path to the config file to use
+`,
+	})
+}
+
+// Set the config file path
+func rcSetPath(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+	path, err := in.GetString("path")
+	if err != nil {
+		return nil, err
+	}
+	err = SetConfigPath(path)
+	return nil, err
+}
diff --git a/fs/config/rc_test.go b/fs/config/rc_test.go
index 0136f869a03f3..fc9f3756337bb 100644
--- a/fs/config/rc_test.go
+++ b/fs/config/rc_test.go
@@ -153,3 +153,21 @@ func TestRcProviders(t *testing.T) {
 	}
 	assert.True(t, foundLocal, "didn't find local provider")
 }
+
+func TestRcSetPath(t *testing.T) {
+	oldPath := config.GetConfigPath()
+	newPath := oldPath + ".newPath"
+	call := rc.Calls.Get("config/setpath")
+	assert.NotNil(t, call)
+	in := rc.Params{
+		"path": newPath,
+	}
+	_, err := call.Fn(context.Background(), in)
+	require.NoError(t, err)
+	assert.Equal(t, newPath, config.GetConfigPath())
+
+	in["path"] = oldPath
+	_, err = call.Fn(context.Background(), in)
+	require.NoError(t, err)
+	assert.Equal(t, oldPath, config.GetConfigPath())
+}
diff --git a/librclone/ctest/ctest.c b/librclone/ctest/ctest.c
index a3ee72b261896..64b1bb1e84366 100644
--- a/librclone/ctest/ctest.c
+++ b/librclone/ctest/ctest.c
@@ -113,6 +113,9 @@ int main(int argc, char** argv) {
     /* testCopyFile(); */
     /* testListRemotes(); */
 
+    /* testRPC("config/setpath", "{\"path\":\"/tmp/rclone.conf\"}"); */
+    /* testRPC("config/listremotes", "{}"); */
+
     RcloneFinalize();
     return EXIT_SUCCESS;
 }

From 6f8112ff6710ed1242243eaeb4bbc75e1e55aac9 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 15 Dec 2022 15:58:25 +0000
Subject: [PATCH 472/560] Add Abdullah Saglam to contributors

---
 docs/content/authors.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/content/authors.md b/docs/content/authors.md
index aac3fb694ca6d..b96d53e0a1cdd 100644
--- a/docs/content/authors.md
+++ b/docs/content/authors.md
@@ -672,3 +672,4 @@ put them back in again.` >}}
   * MohammadReza 
   * vanplus <60313789+vanplus@users.noreply.github.com>
   * Jack <16779171+jkpe@users.noreply.github.com>
+  * Abdullah Saglam 

From 7be9855a706d1e09504f17949a90c54cd56fb2a5 Mon Sep 17 00:00:00 2001
From: Abdullah Saglam 
Date: Thu, 15 Dec 2022 15:10:53 +0000
Subject: [PATCH 473/560] azureblob: implement --use-server-modtime

This patch implements --use-server-modtime for the Azureblob backend.

It does this by not reading the time from the metadata if the global
flag is set.
---
 backend/azureblob/azureblob.go | 4 +++-
 docs/content/azureblob.md      | 8 +++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 80050a991ff5f..af6ff9dddba83 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -1590,7 +1590,9 @@ func (o *Object) setMetadata(metadata map[string]string) {
 		for k, v := range metadata {
 			o.meta[strings.ToLower(k)] = v
 		}
-		if modTime, ok := o.meta[modTimeKey]; ok {
+		// Set o.modTime from metadata if it exists and
+		// UseServerModTime isn't in use.
+		if modTime, ok := o.meta[modTimeKey]; !o.fs.ci.UseServerModTime && ok {
 			when, err := time.Parse(timeFormatIn, modTime)
 			if err != nil {
 				fs.Debugf(o, "Couldn't parse %v = %q: %v", modTimeKey, modTime, err)
diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md
index 1087e2577e0ef..e1ea77086aa02 100644
--- a/docs/content/azureblob.md
+++ b/docs/content/azureblob.md
@@ -80,7 +80,13 @@ docs](/docs/#fast-list) for more details.
 The modified time is stored as metadata on the object with the `mtime`
 key.  It is stored using RFC3339 Format time with nanosecond
 precision.  The metadata is supplied during directory listings so
-there is no overhead to using it.
+there is no performance overhead to using it.
+
+If you wish to use the Azure standard `LastModified` time stored on
+the object as the modified time, then use the `--use-server-modtime`
+flag. Note that rclone can't set `LastModified`, so using the
+`--update` flag when syncing is recommended if using
+`--use-server-modtime`.
 
 ### Performance
 

From 4a31961c4fd09fc1d6083f3e0b10aa40c2a5af86 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Sat, 8 Oct 2022 17:56:04 +0100
Subject: [PATCH 474/560] filter: factor rules into its own file

---
 fs/filter/filter.go | 188 ++------------------------------
 fs/filter/rules.go  | 255 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 265 insertions(+), 178 deletions(-)
 create mode 100644 fs/filter/rules.go

diff --git a/fs/filter/filter.go b/fs/filter/filter.go
index 997070bd79397..047eb4bbbbf04 100644
--- a/fs/filter/filter.go
+++ b/fs/filter/filter.go
@@ -2,14 +2,11 @@
 package filter
 
 import (
-	"bufio"
 	"context"
 	"errors"
 	"fmt"
 	"log"
-	"os"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 
@@ -22,73 +19,11 @@ import (
 // This is accessed through GetConfig and AddConfig
 var globalConfig = mustNewFilter(nil)
 
-// rule is one filter rule
-type rule struct {
-	Include bool
-	Regexp  *regexp.Regexp
-}
-
-// Match returns true if rule matches path
-func (r *rule) Match(path string) bool {
-	return r.Regexp.MatchString(path)
-}
-
-// String the rule
-func (r *rule) String() string {
-	c := "-"
-	if r.Include {
-		c = "+"
-	}
-	return fmt.Sprintf("%s %s", c, r.Regexp.String())
-}
-
-// rules is a slice of rules
-type rules struct {
-	rules    []rule
-	existing map[string]struct{}
-}
-
-// add adds a rule if it doesn't exist already
-func (rs *rules) add(Include bool, re *regexp.Regexp) {
-	if rs.existing == nil {
-		rs.existing = make(map[string]struct{})
-	}
-	newRule := rule{
-		Include: Include,
-		Regexp:  re,
-	}
-	newRuleString := newRule.String()
-	if _, ok := rs.existing[newRuleString]; ok {
-		return // rule already exists
-	}
-	rs.rules = append(rs.rules, newRule)
-	rs.existing[newRuleString] = struct{}{}
-}
-
-// clear clears all the rules
-func (rs *rules) clear() {
-	rs.rules = nil
-	rs.existing = nil
-}
-
-// len returns the number of rules
-func (rs *rules) len() int {
-	return len(rs.rules)
-}
-
-// FilesMap describes the map of files to transfer
-type FilesMap map[string]struct{}
-
 // Opt configures the filter
 type Opt struct {
 	DeleteExcluded bool
-	FilterRule     []string
-	FilterFrom     []string
-	ExcludeRule    []string
-	ExcludeFrom    []string
+	RulesOpt       // embedded so we don't change the JSON API
 	ExcludeFile    []string
-	IncludeRule    []string
-	IncludeFrom    []string
 	FilesFrom      []string
 	FilesFromRaw   []string
 	MinAge         fs.Duration
@@ -106,6 +41,9 @@ var DefaultOpt = Opt{
 	MaxSize: fs.SizeSuffix(-1),
 }
 
+// FilesMap describes the map of files to transfer
+type FilesMap map[string]struct{}
+
 // Filter describes any filtering in operation
 type Filter struct {
 	Opt         Opt
@@ -142,57 +80,9 @@ func NewFilter(opt *Opt) (f *Filter, err error) {
 		fs.Debugf(nil, "--max-age %v to %v", f.Opt.MaxAge, f.ModTimeFrom)
 	}
 
-	addImplicitExclude := false
-	foundExcludeRule := false
-
-	for _, rule := range f.Opt.IncludeRule {
-		err = f.Add(true, rule)
-		if err != nil {
-			return nil, err
-		}
-		addImplicitExclude = true
-	}
-	for _, rule := range f.Opt.IncludeFrom {
-		err := forEachLine(rule, false, func(line string) error {
-			return f.Add(true, line)
-		})
-		if err != nil {
-			return nil, err
-		}
-		addImplicitExclude = true
-	}
-	for _, rule := range f.Opt.ExcludeRule {
-		err = f.Add(false, rule)
-		if err != nil {
-			return nil, err
-		}
-		foundExcludeRule = true
-	}
-	for _, rule := range f.Opt.ExcludeFrom {
-		err := forEachLine(rule, false, func(line string) error {
-			return f.Add(false, line)
-		})
-		if err != nil {
-			return nil, err
-		}
-		foundExcludeRule = true
-	}
-
-	if addImplicitExclude && foundExcludeRule {
-		fs.Errorf(nil, "Using --filter is recommended instead of both --include and --exclude as the order they are parsed in is indeterminate")
-	}
-
-	for _, rule := range f.Opt.FilterRule {
-		err = f.AddRule(rule)
-		if err != nil {
-			return nil, err
-		}
-	}
-	for _, rule := range f.Opt.FilterFrom {
-		err := forEachLine(rule, false, f.AddRule)
-		if err != nil {
-			return nil, err
-		}
+	err = parseRules(&f.Opt.RulesOpt, f.Add, f.Clear)
+	if err != nil {
+		return nil, err
 	}
 
 	inActive := f.InActive()
@@ -225,12 +115,6 @@ func NewFilter(opt *Opt) (f *Filter, err error) {
 		}
 	}
 
-	if addImplicitExclude {
-		err = f.Add(false, "/**")
-		if err != nil {
-			return nil, err
-		}
-	}
 	if fs.GetConfig(context.Background()).Dump&fs.DumpFilters != 0 {
 		fmt.Println("--- start filters ---")
 		fmt.Println(f.DumpFilters())
@@ -309,16 +193,7 @@ func (f *Filter) Add(Include bool, glob string) error {
 //
 // Line comments may be introduced with '#' or ';'
 func (f *Filter) AddRule(rule string) error {
-	switch {
-	case rule == "!":
-		f.Clear()
-		return nil
-	case strings.HasPrefix(rule, "- "):
-		return f.Add(false, rule[2:])
-	case strings.HasPrefix(rule, "+ "):
-		return f.Add(true, rule[2:])
-	}
-	return fmt.Errorf("malformed rule %q", rule)
+	return addRule(rule, f.Add, f.Clear)
 }
 
 // initAddFile creates f.files and f.dirs
@@ -380,12 +255,7 @@ func (f *Filter) IncludeRemote(remote string) bool {
 		_, include := f.files[remote]
 		return include
 	}
-	for _, rule := range f.fileRules.rules {
-		if rule.Match(remote) {
-			return rule.Include
-		}
-	}
-	return true
+	return f.fileRules.include(remote)
 }
 
 // ListContainsExcludeFile checks if exclude file is present in the list.
@@ -428,13 +298,7 @@ func (f *Filter) IncludeDirectory(ctx context.Context, fs fs.Fs) func(string) (b
 			return include, nil
 		}
 		remote += "/"
-		for _, rule := range f.dirRules.rules {
-			if rule.Match(remote) {
-				return rule.Include, nil
-			}
-		}
-
-		return true, nil
+		return f.dirRules.include(remote), nil
 	}
 }
 
@@ -490,41 +354,9 @@ func (f *Filter) IncludeObject(ctx context.Context, o fs.Object) bool {
 	} else {
 		modTime = time.Unix(0, 0)
 	}
-
 	return f.Include(o.Remote(), o.Size(), modTime)
 }
 
-// forEachLine calls fn on every line in the file pointed to by path
-//
-// It ignores empty lines and lines starting with '#' or ';' if raw is false
-func forEachLine(path string, raw bool, fn func(string) error) (err error) {
-	var scanner *bufio.Scanner
-	if path == "-" {
-		scanner = bufio.NewScanner(os.Stdin)
-	} else {
-		in, err := os.Open(path)
-		if err != nil {
-			return err
-		}
-		scanner = bufio.NewScanner(in)
-		defer fs.CheckClose(in, &err)
-	}
-	for scanner.Scan() {
-		line := scanner.Text()
-		if !raw {
-			line = strings.TrimSpace(line)
-			if len(line) == 0 || line[0] == '#' || line[0] == ';' {
-				continue
-			}
-		}
-		err := fn(line)
-		if err != nil {
-			return err
-		}
-	}
-	return scanner.Err()
-}
-
 // DumpFilters dumps the filters in textual form, 1 per line
 func (f *Filter) DumpFilters() string {
 	rules := []string{}
diff --git a/fs/filter/rules.go b/fs/filter/rules.go
new file mode 100644
index 0000000000000..683cc7037d619
--- /dev/null
+++ b/fs/filter/rules.go
@@ -0,0 +1,255 @@
+package filter
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// RulesOpt is configuration for a rule set
+type RulesOpt struct {
+	FilterRule  []string
+	FilterFrom  []string
+	ExcludeRule []string
+	ExcludeFrom []string
+	IncludeRule []string
+	IncludeFrom []string
+}
+
+// rule is one filter rule
+type rule struct {
+	Include bool
+	Regexp  *regexp.Regexp
+}
+
+// Match returns true if rule matches path
+func (r *rule) Match(path string) bool {
+	return r.Regexp.MatchString(path)
+}
+
+// String the rule
+func (r *rule) String() string {
+	c := "-"
+	if r.Include {
+		c = "+"
+	}
+	return fmt.Sprintf("%s %s", c, r.Regexp.String())
+}
+
+// rules is a slice of rules
+type rules struct {
+	rules    []rule
+	existing map[string]struct{}
+}
+
+type addFn func(Include bool, glob string) error
+
+// add adds a rule if it doesn't exist already
+func (rs *rules) add(Include bool, re *regexp.Regexp) {
+	if rs.existing == nil {
+		rs.existing = make(map[string]struct{})
+	}
+	newRule := rule{
+		Include: Include,
+		Regexp:  re,
+	}
+	newRuleString := newRule.String()
+	if _, ok := rs.existing[newRuleString]; ok {
+		return // rule already exists
+	}
+	rs.rules = append(rs.rules, newRule)
+	rs.existing[newRuleString] = struct{}{}
+}
+
+// Add adds a filter rule with include or exclude status indicated
+func (rs *rules) Add(Include bool, glob string) error {
+	re, err := GlobToRegexp(glob, false /* f.Opt.IgnoreCase */)
+	if err != nil {
+		return err
+	}
+	rs.add(Include, re)
+	return nil
+}
+
+type clearFn func()
+
+// clear clears all the rules
+func (rs *rules) clear() {
+	rs.rules = nil
+	rs.existing = nil
+}
+
+// len returns the number of rules
+func (rs *rules) len() int {
+	return len(rs.rules)
+}
+
+// include returns whether this remote passes the filter rules.
+func (rs *rules) include(remote string) bool {
+	for _, rule := range rs.rules {
+		if rule.Match(remote) {
+			return rule.Include
+		}
+	}
+	return true
+}
+
+// include returns whether this collection of strings remote passes
+// the filter rules.
+//
+// the first rule is evaluated on all the remotes and if it matches
+// then the result is returned. If not the next rule is tested and so
+// on.
+func (rs *rules) includeMany(remotes []string) bool {
+	for _, rule := range rs.rules {
+		for _, remote := range remotes {
+			if rule.Match(remote) {
+				return rule.Include
+			}
+		}
+	}
+	return true
+}
+
+// forEachLine calls fn on every line in the file pointed to by path
+//
+// It ignores empty lines and lines starting with '#' or ';' if raw is false
+func forEachLine(path string, raw bool, fn func(string) error) (err error) {
+	var scanner *bufio.Scanner
+	if path == "-" {
+		scanner = bufio.NewScanner(os.Stdin)
+	} else {
+		in, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		scanner = bufio.NewScanner(in)
+		defer fs.CheckClose(in, &err)
+	}
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !raw {
+			line = strings.TrimSpace(line)
+			if len(line) == 0 || line[0] == '#' || line[0] == ';' {
+				continue
+			}
+		}
+		err := fn(line)
+		if err != nil {
+			return err
+		}
+	}
+	return scanner.Err()
+}
+
+// AddRule adds a filter rule with include/exclude indicated by the prefix
+//
+// These are
+//
+//	# Comment
+//	+ glob
+//	- glob
+//	!
+//
+// '+' includes the glob, '-' excludes it and '!' resets the filter list
+//
+// Line comments may be introduced with '#' or ';'
+func addRule(rule string, add addFn, clear clearFn) error {
+	switch {
+	case rule == "!":
+		clear()
+		return nil
+	case strings.HasPrefix(rule, "- "):
+		return add(false, rule[2:])
+	case strings.HasPrefix(rule, "+ "):
+		return add(true, rule[2:])
+	}
+	return fmt.Errorf("malformed rule %q", rule)
+}
+
+// AddRule adds a filter rule with include/exclude indicated by the prefix
+//
+// These are
+//
+//	# Comment
+//	+ glob
+//	- glob
+//	!
+//
+// '+' includes the glob, '-' excludes it and '!' resets the filter list
+//
+// Line comments may be introduced with '#' or ';'
+func (rs *rules) AddRule(rule string) error {
+	return addRule(rule, rs.Add, rs.clear)
+}
+
+// Parse the rules passed in and add them to the function
+func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) {
+	addImplicitExclude := false
+	foundExcludeRule := false
+
+	for _, rule := range opt.IncludeRule {
+		err = add(true, rule)
+		if err != nil {
+			return err
+		}
+		addImplicitExclude = true
+	}
+	for _, rule := range opt.IncludeFrom {
+		err := forEachLine(rule, false, func(line string) error {
+			return add(true, line)
+		})
+		if err != nil {
+			return err
+		}
+		addImplicitExclude = true
+	}
+	for _, rule := range opt.ExcludeRule {
+		err = add(false, rule)
+		if err != nil {
+			return err
+		}
+		foundExcludeRule = true
+	}
+	for _, rule := range opt.ExcludeFrom {
+		err := forEachLine(rule, false, func(line string) error {
+			return add(false, line)
+		})
+		if err != nil {
+			return err
+		}
+		foundExcludeRule = true
+	}
+
+	if addImplicitExclude && foundExcludeRule {
+		fs.Errorf(nil, "Using --filter is recommended instead of both --include and --exclude as the order they are parsed in is indeterminate")
+	}
+
+	for _, rule := range opt.FilterRule {
+		err = addRule(rule, add, clear)
+		if err != nil {
+			return err
+		}
+	}
+	for _, rule := range opt.FilterFrom {
+		err := forEachLine(rule, false, func(rule string) error {
+			return addRule(rule, add, clear)
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	if addImplicitExclude {
+		err = add(false, "/**")
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

From 3a6f1f5cd7823272b95dcce67d7bd5d8ae5fe93f Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Thu, 4 Aug 2022 18:19:05 +0100
Subject: [PATCH 475/560] filter: add metadata filters
 --metadata-include/exclude/filter and friends

Fixes #6353
---
 docs/content/docs.md                 |  6 +++
 docs/content/filtering.md            | 41 ++++++++++++++++++++-
 docs/content/internetarchive.md      | 16 ++++++++
 fs/filter/filter.go                  | 43 +++++++++++++++++++++-
 fs/filter/filter_test.go             | 55 +++++++++++++++++++++++++++-
 fs/filter/filterflags/filterflags.go | 23 +++++++++---
 fs/operations/operations.go          |  2 +-
 7 files changed, 173 insertions(+), 13 deletions(-)

diff --git a/docs/content/docs.md b/docs/content/docs.md
index eab4b5b29c3d7..535ed2486bfc1 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -2355,6 +2355,12 @@ For the filtering options
   * `--min-age`
   * `--max-age`
   * `--dump filters`
+  * `--metadata-include`
+  * `--metadata-include-from`
+  * `--metadata-exclude`
+  * `--metadata-exclude-from`
+  * `--metadata-filter`
+  * `--metadata-filter-from`
 
 See the [filtering section](/filtering/).
 
diff --git a/docs/content/filtering.md b/docs/content/filtering.md
index e315dfffc8f22..bfff3eab94e74 100644
--- a/docs/content/filtering.md
+++ b/docs/content/filtering.md
@@ -32,7 +32,7 @@ you expect. Instead use a `--filter...` flag.
 
 ## Patterns for matching path/file names
 
-### Pattern syntax
+### Pattern syntax {#patterns}
 
 Here is a formal definition of the pattern syntax,
 [examples](#examples) are below.
@@ -194,7 +194,7 @@ them into regular expressions.
 | Rooted Regexp | `/{{.*\.jpe?g}}` | `/file.jpeg`  | `/file.png`    |
 |             |                  | `/file.jpg`   | `/dir/file.jpg` |
 
-## How filter rules are applied to files
+## How filter rules are applied to files {#how-filter-rules-work}
 
 Rclone path/file name filters are made up of one or more of the following flags:
 
@@ -757,6 +757,43 @@ E.g. for the following directory structure:
 The command `rclone ls --exclude-if-present .ignore dir1` does
 not list `dir3`, `file3` or `.ignore`.
 
+## Metadata filters {#metadata}
+
+The metadata filters work in a very similar way to the normal file
+name filters, except they match [metadata](/docs/#metadata) on the
+object.
+
+The metadata should be specified as `key=value` patterns. This may be
+wildcarded using the normal [filter patterns](#patterns) or [regular
+expressions](#regexp).
+
+For example if you wished to list only local files with a mode of
+`100664` you could do that with:
+
+    rclone lsf -M --files-only --metadata-include "mode=100664" .
+
+Or if you wished to show files with an `atime`, `mtime` or `btime` at a given date:
+
+    rclone lsf -M --files-only --metadata-include "[abm]time=2022-12-16*" .
+
+Like file filtering, metadata filtering only applies to files not to
+directories.
+
+The filters can be applied using these flags.
+
+- `--metadata-include`      - Include metadatas matching pattern
+- `--metadata-include-from` - Read metadata include patterns from file (use - to read from stdin)
+- `--metadata-exclude`      - Exclude metadatas matching pattern
+- `--metadata-exclude-from` - Read metadata exclude patterns from file (use - to read from stdin)
+- `--metadata-filter`       - Add a metadata filtering rule
+- `--metadata-filter-from`  - Read metadata filtering patterns from a file (use - to read from stdin)
+
+Each flag can be repeated. See the section on [how filter rules are
+applied](#how-filter-rules-work) for more details - these flags work
+in an identical way to the file name filtering flags, but instead of
+file name patterns have metadata patterns.
+
+
 ## Common pitfalls
 
 The most frequent filter support issues on
diff --git a/docs/content/internetarchive.md b/docs/content/internetarchive.md
index f0a90278c5333..e02690849cda3 100644
--- a/docs/content/internetarchive.md
+++ b/docs/content/internetarchive.md
@@ -66,6 +66,22 @@ It can be triggered when you did a server-side copy.
 
 Reading metadata will also provide custom (non-standard nor reserved) ones.
 
+## Filtering auto generated files
+
+The Internet Archive automatically creates metadata files after
+upload. These can cause problems when doing an `rclone sync` as rclone
+will try, and fail, to delete them. These metadata files are not
+changeable, as they are created by the Internet Archive automatically.
+
+These auto-created files can be excluded from the sync using [metadata
+filtering](/filtering/#metadata).
+
+    rclone sync ... --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata"
+
+Which excludes from the sync any files which have the
+`source=metadata` or `format=Metadata` flags which are added to
+Internet Archive auto-created files.
+
 ## Configuration
 
 Here is an example of making an internetarchive configuration.
diff --git a/fs/filter/filter.go b/fs/filter/filter.go
index 047eb4bbbbf04..156dbd4a7b309 100644
--- a/fs/filter/filter.go
+++ b/fs/filter/filter.go
@@ -26,6 +26,7 @@ type Opt struct {
 	ExcludeFile    []string
 	FilesFrom      []string
 	FilesFromRaw   []string
+	MetaRules      RulesOpt
 	MinAge         fs.Duration
 	MaxAge         fs.Duration
 	MinSize        fs.SizeSuffix
@@ -51,6 +52,7 @@ type Filter struct {
 	ModTimeTo   time.Time
 	fileRules   rules
 	dirRules    rules
+	metaRules   rules
 	files       FilesMap // files if filesFrom
 	dirs        FilesMap // dirs from filesFrom
 }
@@ -85,6 +87,11 @@ func NewFilter(opt *Opt) (f *Filter, err error) {
 		return nil, err
 	}
 
+	err = parseRules(&f.Opt.MetaRules, f.metaRules.Add, f.metaRules.clear)
+	if err != nil {
+		return nil, err
+	}
+
 	inActive := f.InActive()
 
 	for _, rule := range f.Opt.FilesFrom {
@@ -234,6 +241,7 @@ func (f *Filter) Files() FilesMap {
 func (f *Filter) Clear() {
 	f.fileRules.clear()
 	f.dirRules.clear()
+	f.metaRules.clear()
 }
 
 // InActive returns false if any filters are active
@@ -245,6 +253,7 @@ func (f *Filter) InActive() bool {
 		f.Opt.MaxSize < 0 &&
 		f.fileRules.len() == 0 &&
 		f.dirRules.len() == 0 &&
+		f.metaRules.len() == 0 &&
 		len(f.Opt.ExcludeFile) == 0)
 }
 
@@ -322,7 +331,7 @@ func (f *Filter) DirContainsExcludeFile(ctx context.Context, fremote fs.Fs, remo
 
 // Include returns whether this object should be included into the
 // sync or not
-func (f *Filter) Include(remote string, size int64, modTime time.Time) bool {
+func (f *Filter) Include(remote string, size int64, modTime time.Time, metadata fs.Metadata) bool {
 	// filesFrom takes precedence
 	if f.files != nil {
 		_, include := f.files[remote]
@@ -340,6 +349,20 @@ func (f *Filter) Include(remote string, size int64, modTime time.Time) bool {
 	if f.Opt.MaxSize >= 0 && size > int64(f.Opt.MaxSize) {
 		return false
 	}
+	if f.metaRules.len() > 0 {
+		metadatas := make([]string, 0, len(metadata)+1)
+		for key, value := range metadata {
+			metadatas = append(metadatas, fmt.Sprintf("%s=%s", key, value))
+		}
+		if len(metadata) == 0 {
+			// If there is no metadata, add a null one
+			// otherwise the default action isn't taken
+			metadatas = append(metadatas, "\x00=\x00")
+		}
+		if !f.metaRules.includeMany(metadatas) {
+			return false
+		}
+	}
 	return f.IncludeRemote(remote)
 }
 
@@ -354,7 +377,17 @@ func (f *Filter) IncludeObject(ctx context.Context, o fs.Object) bool {
 	} else {
 		modTime = time.Unix(0, 0)
 	}
-	return f.Include(o.Remote(), o.Size(), modTime)
+	var metadata fs.Metadata
+	if f.metaRules.len() > 0 {
+		var err error
+		metadata, err = fs.GetMetadata(ctx, o)
+		if err != nil {
+			fs.Errorf(o, "Failed to read metadata: %v", err)
+			metadata = nil
+		}
+
+	}
+	return f.Include(o.Remote(), o.Size(), modTime, metadata)
 }
 
 // DumpFilters dumps the filters in textual form, 1 per line
@@ -374,6 +407,12 @@ func (f *Filter) DumpFilters() string {
 	for _, dirRule := range f.dirRules.rules {
 		rules = append(rules, dirRule.String())
 	}
+	if f.metaRules.len() > 0 {
+		rules = append(rules, "--- Metadata filter rules ---")
+		for _, metaRule := range f.metaRules.rules {
+			rules = append(rules, metaRule.String())
+		}
+	}
 	return strings.Join(rules, "\n")
 }
 
diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go
index 6feab8ecc6322..ca24ecf27c97c 100644
--- a/fs/filter/filter_test.go
+++ b/fs/filter/filter_test.go
@@ -23,6 +23,7 @@ func TestNewFilterDefault(t *testing.T) {
 	assert.Equal(t, fs.SizeSuffix(-1), f.Opt.MaxSize)
 	assert.Len(t, f.fileRules.rules, 0)
 	assert.Len(t, f.dirRules.rules, 0)
+	assert.Len(t, f.metaRules.rules, 0)
 	assert.Nil(t, f.files)
 	assert.True(t, f.InActive())
 }
@@ -207,7 +208,7 @@ type includeTest struct {
 
 func testInclude(t *testing.T, f *Filter, tests []includeTest) {
 	for _, test := range tests {
-		got := f.Include(test.in, test.size, time.Unix(test.modTime, 0))
+		got := f.Include(test.in, test.size, time.Unix(test.modTime, 0), nil)
 		assert.Equal(t, test.want, got, fmt.Sprintf("in=%q, size=%v, modTime=%v", test.in, test.size, time.Unix(test.modTime, 0)))
 	}
 }
@@ -527,6 +528,56 @@ func TestNewFilterMatchesRegexp(t *testing.T) {
 	assert.False(t, f.InActive())
 }
 
+type includeTestMetadata struct {
+	in       string
+	metadata fs.Metadata
+	want     bool
+}
+
+func testIncludeMetadata(t *testing.T, f *Filter, tests []includeTestMetadata) {
+	for _, test := range tests {
+		got := f.Include(test.in, 0, time.Time{}, test.metadata)
+		assert.Equal(t, test.want, got, fmt.Sprintf("in=%q, metadata=%+v", test.in, test.metadata))
+	}
+}
+
+func TestNewFilterMetadataInclude(t *testing.T) {
+	f, err := NewFilter(nil)
+	require.NoError(t, err)
+	add := func(s string) {
+		err := f.metaRules.AddRule(s)
+		require.NoError(t, err)
+	}
+	add(`+ t*=t*`)
+	add(`- *`)
+	testIncludeMetadata(t, f, []includeTestMetadata{
+		{"nil", nil, false},
+		{"empty", fs.Metadata{}, false},
+		{"ok1", fs.Metadata{"thing": "thang"}, true},
+		{"ok2", fs.Metadata{"thing1": "thang1"}, true},
+		{"missing", fs.Metadata{"Thing1": "Thang1"}, false},
+	})
+	assert.False(t, f.InActive())
+}
+
+func TestNewFilterMetadataExclude(t *testing.T) {
+	f, err := NewFilter(nil)
+	require.NoError(t, err)
+	add := func(s string) {
+		err := f.metaRules.AddRule(s)
+		require.NoError(t, err)
+	}
+	add(`- thing=thang`)
+	add(`+ *`)
+	testIncludeMetadata(t, f, []includeTestMetadata{
+		{"nil", nil, true},
+		{"empty", fs.Metadata{}, true},
+		{"ok1", fs.Metadata{"thing": "thang"}, false},
+		{"missing1", fs.Metadata{"thing1": "thang1"}, true},
+	})
+	assert.False(t, f.InActive())
+}
+
 func TestFilterAddDirRuleOrFileRule(t *testing.T) {
 	for _, test := range []struct {
 		included bool
@@ -713,7 +764,7 @@ func TestFilterMatchesFromDocs(t *testing.T) {
 		require.NoError(t, err)
 		err = f.Add(false, "*")
 		require.NoError(t, err)
-		included := f.Include(test.file, 0, time.Unix(0, 0))
+		included := f.Include(test.file, 0, time.Unix(0, 0), nil)
 		if included != test.included {
 			t.Errorf("%q match %q: want %v got %v", test.glob, test.file, test.included, included)
 		}
diff --git a/fs/filter/filterflags/filterflags.go b/fs/filter/filterflags/filterflags.go
index 3963e0c04d476..5bb17b8a3c06e 100644
--- a/fs/filter/filterflags/filterflags.go
+++ b/fs/filter/filterflags/filterflags.go
@@ -3,6 +3,7 @@ package filterflags
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/filter"
@@ -26,17 +27,27 @@ func Reload(ctx context.Context) (err error) {
 	return nil
 }
 
+// AddRuleFlags add a set of rules flags with prefix
+func AddRuleFlags(flagSet *pflag.FlagSet, Opt *filter.RulesOpt, what, prefix string) {
+	shortFilter := ""
+	if prefix == "" {
+		shortFilter = "f"
+	}
+	flags.StringArrayVarP(flagSet, &Opt.FilterRule, prefix+"filter", shortFilter, nil, fmt.Sprintf("Add a %s filtering rule", what))
+	flags.StringArrayVarP(flagSet, &Opt.FilterFrom, prefix+"filter-from", "", nil, fmt.Sprintf("Read %s filtering patterns from a file (use - to read from stdin)", what))
+	flags.StringArrayVarP(flagSet, &Opt.ExcludeRule, prefix+"exclude", "", nil, fmt.Sprintf("Exclude %ss matching pattern", what))
+	flags.StringArrayVarP(flagSet, &Opt.ExcludeFrom, prefix+"exclude-from", "", nil, fmt.Sprintf("Read %s exclude patterns from file (use - to read from stdin)", what))
+	flags.StringArrayVarP(flagSet, &Opt.IncludeRule, prefix+"include", "", nil, fmt.Sprintf("Include %ss matching pattern", what))
+	flags.StringArrayVarP(flagSet, &Opt.IncludeFrom, prefix+"include-from", "", nil, fmt.Sprintf("Read %s include patterns from file (use - to read from stdin)", what))
+}
+
 // AddFlags adds the non filing system specific flags to the command
 func AddFlags(flagSet *pflag.FlagSet) {
 	rc.AddOptionReload("filter", &Opt, Reload)
 	flags.BoolVarP(flagSet, &Opt.DeleteExcluded, "delete-excluded", "", false, "Delete files on dest excluded from sync")
-	flags.StringArrayVarP(flagSet, &Opt.FilterRule, "filter", "f", nil, "Add a file-filtering rule")
-	flags.StringArrayVarP(flagSet, &Opt.FilterFrom, "filter-from", "", nil, "Read filtering patterns from a file (use - to read from stdin)")
-	flags.StringArrayVarP(flagSet, &Opt.ExcludeRule, "exclude", "", nil, "Exclude files matching pattern")
-	flags.StringArrayVarP(flagSet, &Opt.ExcludeFrom, "exclude-from", "", nil, "Read exclude patterns from file (use - to read from stdin)")
+	AddRuleFlags(flagSet, &Opt.RulesOpt, "file", "")
+	AddRuleFlags(flagSet, &Opt.MetaRules, "metadata", "metadata-")
 	flags.StringArrayVarP(flagSet, &Opt.ExcludeFile, "exclude-if-present", "", nil, "Exclude directories if filename is present")
-	flags.StringArrayVarP(flagSet, &Opt.IncludeRule, "include", "", nil, "Include files matching pattern")
-	flags.StringArrayVarP(flagSet, &Opt.IncludeFrom, "include-from", "", nil, "Read include patterns from file (use - to read from stdin)")
 	flags.StringArrayVarP(flagSet, &Opt.FilesFrom, "files-from", "", nil, "Read list of source-file names from file (use - to read from stdin)")
 	flags.StringArrayVarP(flagSet, &Opt.FilesFromRaw, "files-from-raw", "", nil, "Read list of source-file names from file without any processing of lines (use - to read from stdin)")
 	flags.FVarP(flagSet, &Opt.MinAge, "min-age", "", "Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y")
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 3510663327163..fe4a27eb4c39b 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -1469,7 +1469,7 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
 		dir := toDelete[i]
 		// If a filter matches the directory then that
 		// directory is a candidate for deletion
-		if !fi.Include(dir+"/", 0, time.Now()) {
+		if !fi.IncludeRemote(dir + "/") {
 			continue
 		}
 		err = TryRmdir(ctx, f, dir)

From 614d79121a728d25c7fd45ca93fa21a63e1c1afd Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Fri, 16 Dec 2022 14:32:39 +0000
Subject: [PATCH 476/560] serve dlna: fix panic: Logger uninitialized.

Before this change we forgot to initialize the logger for the dlna
server. This meant when it needed to log something, it paniced
instead.

See: https://forum.rclone.org/t/rclone-serve-dlna-after-few-hours-of-idle-running-panic-logger-uninitialized-names/34835
---
 cmd/serve/dlna/dlna.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go
index 2389ce638fda0..19db02f16ae88 100644
--- a/cmd/serve/dlna/dlna.go
+++ b/cmd/serve/dlna/dlna.go
@@ -17,6 +17,7 @@ import (
 	"github.com/anacrolix/dms/soap"
 	"github.com/anacrolix/dms/ssdp"
 	"github.com/anacrolix/dms/upnp"
+	"github.com/anacrolix/log"
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/cmd/serve/dlna/data"
 	"github.com/rclone/rclone/cmd/serve/dlna/dlnaflags"
@@ -404,6 +405,7 @@ func (s *server) ssdpInterface(intf net.Interface) {
 		Server:         serverField,
 		UUID:           s.RootDeviceUUID,
 		NotifyInterval: s.AnnounceInterval,
+		Logger:         log.Default,
 	}
 
 	// An interface with these flags should be valid for SSDP.

From 01877e5a0fc0ccfe36316435bed836845ec78a93 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Fri, 16 Dec 2022 11:44:58 +0000
Subject: [PATCH 477/560] s3: ignore versionIDs from uploads unless using
 --s3-versions or --s3-versions-at

Before this change, when a new object was created s3 returns its
versionID (on a versioned bucket) and rclone recorded it in the
object.

This means that when rclone came to delete the object it would delete
it with the versionID.

However it is common to forbid actions with versionIDs on buckets so
as to preserve the historical record and these operations would fail
whereas they succeeded in pre-v1.60.0 versions.

This patch fixes the problem by not recording versions of objects
supplied by the S3 API on upload unless `--s3-versions` or
`--s3-version-at` is used. This makes rclone behave as it did before
v1.60.0 when version support was introduced.

See: https://forum.rclone.org/t/s3-and-intermittent-403-errors-with-file-renames-and-drag-and-drop-operations-in-windows-explorer/34773
---
 backend/s3/s3.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 8d4a4c5f0c1d6..c6977b8c90e57 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -5507,7 +5507,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return err
 	}
-	o.versionID = versionID
+	// Only record versionID if we are using --s3-versions or --s3-version-at
+	if o.fs.opt.Versions || o.fs.opt.VersionAt.IsSet() {
+		o.versionID = versionID
+	} else {
+		o.versionID = nil
+	}
 
 	// User requested we don't HEAD the object after uploading it
 	// so make up the object as best we can assuming it got

From a35490bf70d2746d4a182a40d8f0854bd741c97c Mon Sep 17 00:00:00 2001
From: Ole Frost <82263101+olefrost@users.noreply.github.com>
Date: Mon, 19 Dec 2022 10:28:25 +0100
Subject: [PATCH 478/560] docs: Added note on Box API rate limits

---
 docs/content/box.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/content/box.md b/docs/content/box.md
index 69b8676a66e0b..61634f8c9cbcc 100644
--- a/docs/content/box.md
+++ b/docs/content/box.md
@@ -464,6 +464,8 @@ Reverse Solidus).
 
 Box only supports filenames up to 255 characters in length.
 
+Box has [API rate limits](https://developer.box.com/guides/api-calls/permissions-and-errors/rate-limits/) that sometimes reduce the speed of rclone.
+
 `rclone about` is not supported by the Box backend. Backends without
 this capability cannot determine free space for an rclone mount or
 use policy `mfs` (most free space) as a member of an rclone union

From 2001cc08315f7f488c4a3f9950a57fbb81f26d96 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood 
Date: Tue, 20 Dec 2022 17:16:14 +0000
Subject: [PATCH 479/560] Version v1.61.0

---
 MANUAL.html                                   | 2580 +++++++-----
 MANUAL.md                                     | 2765 +++++++++----
 MANUAL.txt                                    | 2883 ++++++++++----
 docs/content/changelog.md                     |   95 +
 docs/content/commands/rclone_about.md         |    1 +
 docs/content/commands/rclone_authorize.md     |    1 +
 docs/content/commands/rclone_backend.md       |    1 +
 docs/content/commands/rclone_bisync.md        |    1 +
 docs/content/commands/rclone_cat.md           |    1 +
 docs/content/commands/rclone_checksum.md      |    1 +
 docs/content/commands/rclone_cleanup.md       |    1 +
 docs/content/commands/rclone_config.md        |    1 +
 docs/content/commands/rclone_config_create.md |    3 +-
 docs/content/commands/rclone_config_delete.md |    1 +
 docs/content/commands/rclone_config_dump.md   |    1 +
 docs/content/commands/rclone_config_file.md   |    1 +
 .../commands/rclone_config_password.md        |    1 +
 docs/content/commands/rclone_config_paths.md  |    1 +
 .../commands/rclone_config_providers.md       |    1 +
 docs/content/commands/rclone_config_show.md   |    1 +
 docs/content/commands/rclone_config_touch.md  |    1 +
 docs/content/commands/rclone_config_update.md |    3 +-
 docs/content/commands/rclone_copyto.md        |    1 +
 docs/content/commands/rclone_copyurl.md       |    1 +
 docs/content/commands/rclone_cryptcheck.md    |    1 +
 docs/content/commands/rclone_cryptdecode.md   |    1 +
 docs/content/commands/rclone_dedupe.md        |    1 +
 docs/content/commands/rclone_delete.md        |    1 +
 docs/content/commands/rclone_deletefile.md    |    1 +
 .../commands/rclone_genautocomplete.md        |    1 +
 docs/content/commands/rclone_gendocs.md       |    1 +
 docs/content/commands/rclone_hashsum.md       |    1 +
 docs/content/commands/rclone_link.md          |    1 +
 docs/content/commands/rclone_listremotes.md   |    1 +
 docs/content/commands/rclone_lsf.md           |    1 +
 docs/content/commands/rclone_lsjson.md        |    2 +
 docs/content/commands/rclone_lsl.md           |    1 +
 docs/content/commands/rclone_md5sum.md        |    1 +
 docs/content/commands/rclone_mount.md         |   21 +-
 docs/content/commands/rclone_move.md          |    1 +
 docs/content/commands/rclone_moveto.md        |    1 +
 docs/content/commands/rclone_ncdu.md          |    6 +-
 docs/content/commands/rclone_obscure.md       |    1 +
 docs/content/commands/rclone_rc.md            |    1 +
 docs/content/commands/rclone_rcat.md          |    1 +
 docs/content/commands/rclone_rcd.md           |   96 +
 docs/content/commands/rclone_rmdirs.md        |    1 +
 docs/content/commands/rclone_selfupdate.md    |    1 +
 docs/content/commands/rclone_serve.md         |    1 +
 docs/content/commands/rclone_serve_dlna.md    |   17 +-
 docs/content/commands/rclone_serve_docker.md  |   21 +-
 docs/content/commands/rclone_serve_ftp.md     |   15 +-
 docs/content/commands/rclone_serve_http.md    |   33 +-
 docs/content/commands/rclone_serve_restic.md  |   80 +-
 docs/content/commands/rclone_serve_sftp.md    |   15 +-
 docs/content/commands/rclone_serve_webdav.md  |   74 +-
 docs/content/commands/rclone_settier.md       |    1 +
 docs/content/commands/rclone_sha1sum.md       |    1 +
 docs/content/commands/rclone_size.md          |    1 +
 docs/content/commands/rclone_test.md          |    1 +
 .../commands/rclone_test_changenotify.md      |    3 +-
 .../content/commands/rclone_test_histogram.md |    1 +
 docs/content/commands/rclone_test_info.md     |    3 +-
 docs/content/commands/rclone_test_makefile.md |    1 +
 .../content/commands/rclone_test_makefiles.md |    2 +
 docs/content/commands/rclone_test_memory.md   |    1 +
 docs/content/commands/rclone_touch.md         |    1 +
 docs/content/commands/rclone_tree.md          |    1 +
 docs/content/commands/rclone_version.md       |    1 +
 docs/content/flags.md                         | 1117 +++---
 docs/content/ftp.md                           |    4 +-
 docs/content/local.md                         |    2 +-
 docs/content/mailru.md                        |    5 +
 docs/content/rc.md                            |   58 +-
 docs/content/s3.md                            |  106 +-
 docs/content/sftp.md                          |   61 +
 go.mod                                        |    4 +-
 go.sum                                        |    4 -
 rclone.1                                      | 3489 ++++++++++++-----
 79 files changed, 9379 insertions(+), 4237 deletions(-)

diff --git a/MANUAL.html b/MANUAL.html
index 4b87ec4b17fcf..f37861a9649b1 100644
--- a/MANUAL.html
+++ b/MANUAL.html
@@ -19,7 +19,7 @@
 
                  
 
                  rclone(1) User Manual
                  
 
                  Nick Craig-Wood
                  
-
                  Oct 21, 2022
                  
+
                  Dec 20, 2022
                  
 
                  
 
                  Rclone syncs your files to cloud storage
                  
 
                  rclone logo
                  
@@ -105,6 +105,7 @@ 
                  Supported providers
                  
 
                • IDrive e2
                  • 
 
                • IONOS Cloud
                  • 
 
                • Koofr
                  • 
+
                • Liara Object Storage
                  • 
 
                • Mail.ru Cloud
                  • 
 
                • Memset Memstore
                  • 
 
                • Mega
                  • 
@@ -227,9 +228,9 @@ 
                  Windows installation
                  
 
                  Precompiled binary
                  
 
                  Fetch the correct binary for your processor type by clicking on these links. If not sure, use the first link.
                  
 
 
                  Open this file in the Explorer and extract rclone.exe. Rclone is a portable executable so you can place it wherever is convenient.
                  
 
                  Open a CMD window (or powershell) and run the binary. Note that rclone does not launch a GUI by default, it runs in the CMD Window.
                  
@@ -1189,7 +1190,7 @@ 
                  Synopsis
                  
     "State": "*oauth-islocal,teamdrive,,",
     "Option": {
         "Name": "config_is_local",
-        "Help": "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n",
+        "Help": "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n",
         "Default": true,
         "Examples": [
             {
@@ -1386,7 +1387,7 @@ 
                  Synopsis
                  
     "State": "*oauth-islocal,teamdrive,,",
     "Option": {
         "Name": "config_is_local",
-        "Help": "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n",
+        "Help": "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n",
         "Default": true,
         "Examples": [
             {
@@ -1858,6 +1859,7 @@ 
                  Options
                  
       --hash                    Include hashes in the output (may take longer)
       --hash-type stringArray   Show only this hash type (may be repeated)
   -h, --help                    help for lsjson
+  -M, --metadata                Add metadata to the listing
       --no-mimetype             Don't read the mime type (can speed things up)
       --no-modtime              Don't read the modification time (can speed things up)
       --original                Show the ID of the underlying Object
@@ -2106,14 +2108,14 @@ 
                  Options
                  
       --allow-other                            Allow access to other users (not supported on Windows)
       --allow-root                             Allow access to root user (not supported on Windows)
       --async-read                             Use asynchronous reads (not supported on Windows) (default true)
-      --attr-timeout duration                  Time for which file/directory attributes are cached (default 1s)
+      --attr-timeout Duration                  Time for which file/directory attributes are cached (default 1s)
       --daemon                                 Run mount in background and exit parent process (as background output is suppressed, use --log-file with --log-format=pid,... to monitor) (not supported on Windows)
-      --daemon-timeout duration                Time limit for rclone to respond to kernel (not supported on Windows)
-      --daemon-wait duration                   Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s)
+      --daemon-timeout Duration                Time limit for rclone to respond to kernel (not supported on Windows) (default 0s)
+      --daemon-wait Duration                   Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s)
       --debug-fuse                             Debug the FUSE internals - needs -v
       --default-permissions                    Makes kernel enforce access control based on the file mode (not supported on Windows)
       --devname string                         Set the device name - default is remote:path
-      --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
+      --dir-cache-time Duration                Time to cache directory entries for (default 5m0s)
       --dir-perms FileMode                     Directory permissions (default 0777)
       --file-perms FileMode                    File permissions (default 0666)
       --fuse-flag stringArray                  Flags or arguments to be passed direct to libfuse/WinFsp (repeat if required)
@@ -2127,24 +2129,24 @@ 
                  Options
                  
       --noappledouble                          Ignore Apple Double (._) and .DS_Store files (supported on OSX only) (default true)
       --noapplexattr                           Ignore all "com.apple.*" extended attributes (supported on OSX only)
   -o, --option stringArray                     Option for libfuse/WinFsp (repeat if required)
-      --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
+      --poll-interval Duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --read-only                              Only allow read-only access
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
-      --vfs-cache-max-age duration             Max age of objects in the cache (default 1h0m0s)
+      --vfs-cache-max-age Duration             Max age of objects in the cache (default 1h0m0s)
       --vfs-cache-max-size SizeSuffix          Max total size of objects in the cache (default off)
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
-      --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
+      --vfs-cache-poll-interval Duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
       --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
       --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
-      --vfs-read-wait duration                 Time to wait for in-sequence read before seeking (default 20ms)
+      --vfs-read-wait Duration                 Time to wait for in-sequence read before seeking (default 20ms)
       --vfs-used-is-size rclone size           Use the rclone size algorithm for Used size
-      --vfs-write-back duration                Time to writeback files after last use when using cache (default 5s)
-      --vfs-write-wait duration                Time to wait for in-sequence write before giving error (default 1s)
+      --vfs-write-back Duration                Time to writeback files after last use when using cache (default 5s)
+      --vfs-write-wait Duration                Time to wait for in-sequence write before giving error (default 1s)
       --volname string                         Set the volume name (supported on Windows and OSX only)
       --write-back-cache                       Makes kernel buffer writes before sending them to rclone (without this, writethrough caching is used) (not supported on Windows)
      96. 
 
      See the global flags page for global options not listed here.
      
@@ -2186,11 +2188,12 @@ 
      Synopsis
      
 
       ↑,↓ or k,j to Move
  →,l to enter
  ←,h to return
- c toggle counts
  g toggle graph
+ c toggle counts
  a toggle average size in directory
+ m toggle modified time
  u toggle human-readable format
- n,s,C,A sort by name,size,count,average size
+ n,s,C,A,M sort by name,size,count,asize,mtime
  d delete file/directory
  v select file/directory
  V enter visual select mode
@@ -2302,6 +2305,109 @@ 
      Synopsis
      
 
      This is useful if you are controlling rclone via the rc API.
      
 
      If you pass in a path to a directory, rclone will serve that directory for GET requests on the URL passed in. It will also open the URL in the browser when rclone is run.
      
 
      See the rc documentation for more info on the rc flags.
      
+
      Server options
      
+
      Use --addr to specify which IP address and port the server should listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.
      
+
      If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.
      
+
      You can use a unix socket by setting the url to unix:///path/to/socket or just by using an absolute path name. Note that unix sockets bypass the authentication - this is expected to be done with file system permissions.
      
+
      --addr may be repeated to listen on multiple IPs/ports/sockets.
      
+
      --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.
      
+
      --max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.
      
+
      --baseurl controls the URL prefix that rclone serves from. By default rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically.
      
+
      TLS (SSL)
      
+
      By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.
      
+
      --cert should be a either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.
      
+
      --min-tls-version is minimum TLS version that is acceptable. Valid values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").
      
+
      Template
      
+
      --template allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages:
      
+
++++
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
      



      ParameterDescription
      .NameThe full path of a file/directory.
      .TitleDirectory listing of .Name
      .SortThe current sort used. This is changeable via ?sort= parameter
      Sort Options: namedirfirst,name,size,time (default namedirfirst)
      .OrderThe current ordering used. This is changeable via ?order= parameter
      Order Options: asc,desc (default asc)
      .QueryCurrently unused.
      .BreadcrumbAllows for creating a relative navigation
      -- .LinkThe relative to the root link of the Text.
      -- .TextThe Name of the directory.
      .EntriesInformation about a specific file/directory.
      -- .URLThe 'url' of an entry.
      -- .LeafCurrently same as 'URL' but intended to be 'just' the name.
      -- .IsDirBoolean for if an entry is a directory or not.
      -- .SizeSize in Bytes of the entry.
      -- .ModTimeThe UTC timestamp of an entry.
      
+
      Authentication
      
+
      By default this will serve files without needing a login.
      
+
      You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.
      
+
      Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.
      
+
      To create an htpasswd file:
      
+
      touch htpasswd
+htpasswd -B htpasswd user
+htpasswd -B htpasswd anotherUser
      
+
      The password file can be updated while rclone is running.
      
+
      Use --realm to set the authentication realm.
      
+
      Use --salt to change the password hashing salt from the default.
      
 
      rclone rcd <path to files to serve>* [flags]
      
 
      Options
      
 
        -h, --help   help for rcd
      
@@ -2377,7 +2483,7 @@ 
      rclone serve dlna
      
 
      Synopsis
      
 
      Run a DLNA media server for media stored in an rclone remote. Many devices, such as the Xbox and PlayStation, can automatically discover this server in the LAN and play audio/video from it. VLC is also supported. Service discovery uses UDP multicast packets (SSDP) and will thus only work on LANs.
      
 
      Rclone will list all files present in the remote, without filtering based on media formats or file extensions. Additionally, there is no media transcoding support. This means that some players might show files that they are not able to play back correctly.
      
-
      Server options
      
+
      Server options
      
 
      Use --addr to specify which IP address and port the server should listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs.
      
 
      Use --name to choose the friendly server name, which is by default "rclone (hostname)".
      
 
      Use --log-trace in conjunction with -vv to enable additional debug logging of all UPNP traffic.
      
@@ -2499,8 +2605,8 @@ 
      Alternate report of used bytes
      
 
      rclone serve dlna remote:path [flags]
      
 
      Options
      
 
            --addr string                            The ip:port or :port to bind the DLNA http server to (default ":7879")
-      --announce-interval duration             The interval between SSDP announcements (default 12m0s)
-      --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
+      --announce-interval Duration             The interval between SSDP announcements (default 12m0s)
+      --dir-cache-time Duration                Time to cache directory entries for (default 5m0s)
       --dir-perms FileMode                     Directory permissions (default 0777)
       --file-perms FileMode                    File permissions (default 0666)
       --gid uint32                             Override the gid field set by the filesystem (not supported on Windows) (default 1000)
@@ -2511,24 +2617,24 @@ 
      Options
      
       --no-checksum                            Don't compare checksums on up/download
       --no-modtime                             Don't read/write the modification time (can speed things up)
       --no-seek                                Don't allow seeking in files
-      --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
+      --poll-interval Duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --read-only                              Only allow read-only access
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
-      --vfs-cache-max-age duration             Max age of objects in the cache (default 1h0m0s)
+      --vfs-cache-max-age Duration             Max age of objects in the cache (default 1h0m0s)
       --vfs-cache-max-size SizeSuffix          Max total size of objects in the cache (default off)
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
-      --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
+      --vfs-cache-poll-interval Duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
       --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
       --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
-      --vfs-read-wait duration                 Time to wait for in-sequence read before seeking (default 20ms)
+      --vfs-read-wait Duration                 Time to wait for in-sequence read before seeking (default 20ms)
       --vfs-used-is-size rclone size           Use the rclone size algorithm for Used size
-      --vfs-write-back duration                Time to writeback files after last use when using cache (default 5s)
-      --vfs-write-wait duration                Time to wait for in-sequence write before giving error (default 1s)
      
+      --vfs-write-back Duration                Time to writeback files after last use when using cache (default 5s)
+      --vfs-write-wait Duration                Time to wait for in-sequence write before giving error (default 1s)
      
 
      See the global flags page for global options not listed here.
      
 
      SEE ALSO
      
 
        
@@ -2665,15 +2771,15 @@ 
        Options
        
       --allow-other                            Allow access to other users (not supported on Windows)
       --allow-root                             Allow access to root user (not supported on Windows)
       --async-read                             Use asynchronous reads (not supported on Windows) (default true)
-      --attr-timeout duration                  Time for which file/directory attributes are cached (default 1s)
+      --attr-timeout Duration                  Time for which file/directory attributes are cached (default 1s)
       --base-dir string                        Base directory for volumes (default "/var/lib/docker-volumes/rclone")
       --daemon                                 Run mount in background and exit parent process (as background output is suppressed, use --log-file with --log-format=pid,... to monitor) (not supported on Windows)
-      --daemon-timeout duration                Time limit for rclone to respond to kernel (not supported on Windows)
-      --daemon-wait duration                   Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s)
+      --daemon-timeout Duration                Time limit for rclone to respond to kernel (not supported on Windows) (default 0s)
+      --daemon-wait Duration                   Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s)
       --debug-fuse                             Debug the FUSE internals - needs -v
       --default-permissions                    Makes kernel enforce access control based on the file mode (not supported on Windows)
       --devname string                         Set the device name - default is remote:path
-      --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
+      --dir-cache-time Duration                Time to cache directory entries for (default 5m0s)
       --dir-perms FileMode                     Directory permissions (default 0777)
       --file-perms FileMode                    File permissions (default 0666)
       --forget-state                           Skip restoring previous state
@@ -2689,26 +2795,26 @@ 
        Options
        
       --noappledouble                          Ignore Apple Double (._) and .DS_Store files (supported on OSX only) (default true)
       --noapplexattr                           Ignore all "com.apple.*" extended attributes (supported on OSX only)
   -o, --option stringArray                     Option for libfuse/WinFsp (repeat if required)
-      --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
+      --poll-interval Duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --read-only                              Only allow read-only access
       --socket-addr string                     Address <host:port> or absolute path (default: /run/docker/plugins/rclone.sock)
       --socket-gid int                         GID for unix socket (default: current process GID) (default 1000)
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
-      --vfs-cache-max-age duration             Max age of objects in the cache (default 1h0m0s)
+      --vfs-cache-max-age Duration             Max age of objects in the cache (default 1h0m0s)
       --vfs-cache-max-size SizeSuffix          Max total size of objects in the cache (default off)
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
-      --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
+      --vfs-cache-poll-interval Duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
       --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
       --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
-      --vfs-read-wait duration                 Time to wait for in-sequence read before seeking (default 20ms)
+      --vfs-read-wait Duration                 Time to wait for in-sequence read before seeking (default 20ms)
       --vfs-used-is-size rclone size           Use the rclone size algorithm for Used size
-      --vfs-write-back duration                Time to writeback files after last use when using cache (default 5s)
-      --vfs-write-wait duration                Time to wait for in-sequence write before giving error (default 1s)
+      --vfs-write-back Duration                Time to writeback files after last use when using cache (default 5s)
+      --vfs-write-wait Duration                Time to wait for in-sequence write before giving error (default 1s)
       --volname string                         Set the volume name (supported on Windows and OSX only)
       --write-back-cache                       Makes kernel buffer writes before sending them to rclone (without this, writethrough caching is used) (not supported on Windows)

      See the global flags page for global options not listed here.

      @@ -2720,10 +2826,10 @@

      rclone serve ftp

      Serve remote:path over FTP.

      Synopsis

      Run a basic FTP server to serve a remote over FTP protocol. This can be viewed with a FTP client or you can make a remote of type FTP to read and write it.

      -

      Server options

      +

      Server options

      Use --addr to specify which IP address and port the server should listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.

      If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.

      -

      Authentication

      +

      Authentication

      By default this will serve files without needing a login.

      You can set a single username and password with the --user and --pass flags.

      VFS - Virtual File System

      @@ -2876,7 +2982,7 @@

      Options

            --addr string                            IPaddress:Port or :Port to bind server to (default "localhost:2121")
       --auth-proxy string                      A program to use to create the backend from the auth
       --cert string                            TLS PEM key (concatenation of certificate and CA certificate)
-      --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
+      --dir-cache-time Duration                Time to cache directory entries for (default 5m0s)
       --dir-perms FileMode                     Directory permissions (default 0777)
       --file-perms FileMode                    File permissions (default 0666)
       --gid uint32                             Override the gid field set by the filesystem (not supported on Windows) (default 1000)
@@ -2887,26 +2993,26 @@ 
      Options
      
       --no-seek                                Don't allow seeking in files
       --pass string                            Password for authentication (empty value allow every password)
       --passive-port string                    Passive port range to use (default "30000-32000")
-      --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
+      --poll-interval Duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --public-ip string                       Public IP address to advertise for passive connections
       --read-only                              Only allow read-only access
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
       --user string                            User name for authentication (default "anonymous")
-      --vfs-cache-max-age duration             Max age of objects in the cache (default 1h0m0s)
+      --vfs-cache-max-age Duration             Max age of objects in the cache (default 1h0m0s)
       --vfs-cache-max-size SizeSuffix          Max total size of objects in the cache (default off)
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
-      --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
+      --vfs-cache-poll-interval Duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
       --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
       --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
-      --vfs-read-wait duration                 Time to wait for in-sequence read before seeking (default 20ms)
+      --vfs-read-wait Duration                 Time to wait for in-sequence read before seeking (default 20ms)
       --vfs-used-is-size rclone size           Use the rclone size algorithm for Used size
-      --vfs-write-back duration                Time to writeback files after last use when using cache (default 5s)
-      --vfs-write-wait duration                Time to wait for in-sequence write before giving error (default 1s)
      + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s)

      See the global flags page for global options not listed here.

      SEE ALSO

        @@ -2919,17 +3025,19 @@

        Synopsis

        You can use the filter flags (e.g. --include, --exclude) to control what is served.

        The server will log errors. Use -v to see access logs.

        --bwlimit will be respected for file transfers. Use --stats to control the stats printing.

        -

        Server options

        +

        Server options

        Use --addr to specify which IP address and port the server should listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.

        If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.

        +

        You can use a unix socket by setting the url to unix:///path/to/socket or just by using an absolute path name. Note that unix sockets bypass the authentication - this is expected to be done with file system permissions.

        +

        --addr may be repeated to listen on multiple IPs/ports/sockets.

        --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.

        --max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.

        --baseurl controls the URL prefix that rclone serves from. By default rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically.

        -

        SSL/TLS

        +

        TLS (SSL)

        By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.

        --cert should be a either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.

        --min-tls-version is minimum TLS version that is acceptable. Valid values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").

        -

        Template

        +

        Template

        --template allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages:

        @@ -3009,7 +3117,7 @@

        Template

        -

        Authentication

        +

        Authentication

        By default this will serve files without needing a login.

        You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.

        Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.

        @@ -3137,47 +3245,47 @@

        Alternate report of used bytes

        WARNING. Contrary to rclone size, this flag ignores filters so that the result is accurate. However, this is very inefficient and may cost lots of API calls resulting in extra charges. Use it as a last resort and only with caching.

        rclone serve http remote:path [flags]

        Options

        -
              --addr string                            IPaddress:Port or :Port to bind server to (default "127.0.0.1:8080")
+
              --addr stringArray                       IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080])
       --baseurl string                         Prefix for URLs - leave blank for root
-      --cert string                            SSL PEM key (concatenation of certificate and CA certificate)
+      --cert string                            TLS PEM key (concatenation of certificate and CA certificate)
       --client-ca string                       Client certificate authority to verify clients with
-      --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
+      --dir-cache-time Duration                Time to cache directory entries for (default 5m0s)
       --dir-perms FileMode                     Directory permissions (default 0777)
       --file-perms FileMode                    File permissions (default 0666)
       --gid uint32                             Override the gid field set by the filesystem (not supported on Windows) (default 1000)
   -h, --help                                   help for http
       --htpasswd string                        A htpasswd file - if not provided no authentication is done
-      --key string                             SSL PEM Private key
+      --key string                             TLS PEM Private key
       --max-header-bytes int                   Maximum size of request header (default 4096)
       --min-tls-version string                 Minimum TLS version that is acceptable (default "tls1.0")
       --no-checksum                            Don't compare checksums on up/download
       --no-modtime                             Don't read/write the modification time (can speed things up)
       --no-seek                                Don't allow seeking in files
       --pass string                            Password for authentication
-      --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
+      --poll-interval Duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --read-only                              Only allow read-only access
       --realm string                           Realm for authentication
       --salt string                            Password hashing salt (default "dlPL2MqE")
-      --server-read-timeout duration           Timeout for server reading data (default 1h0m0s)
-      --server-write-timeout duration          Timeout for server writing data (default 1h0m0s)
+      --server-read-timeout Duration           Timeout for server reading data (default 1h0m0s)
+      --server-write-timeout Duration          Timeout for server writing data (default 1h0m0s)
       --template string                        User-specified template
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
       --user string                            User name for authentication
-      --vfs-cache-max-age duration             Max age of objects in the cache (default 1h0m0s)
+      --vfs-cache-max-age Duration             Max age of objects in the cache (default 1h0m0s)
       --vfs-cache-max-size SizeSuffix          Max total size of objects in the cache (default off)
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
-      --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
+      --vfs-cache-poll-interval Duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
       --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
       --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
-      --vfs-read-wait duration                 Time to wait for in-sequence read before seeking (default 20ms)
+      --vfs-read-wait Duration                 Time to wait for in-sequence read before seeking (default 20ms)
       --vfs-used-is-size rclone size           Use the rclone size algorithm for Used size
-      --vfs-write-back duration                Time to writeback files after last use when using cache (default 5s)
-      --vfs-write-wait duration                Time to wait for in-sequence write before giving error (default 1s)
        
+      --vfs-write-back Duration                Time to writeback files after last use when using cache (default 5s)
+      --vfs-write-wait Duration                Time to wait for in-sequence write before giving error (default 1s)

        See the global flags page for global options not listed here.

        SEE ALSO

          @@ -3186,7 +3294,7 @@

          SEE ALSO

          rclone serve restic

          Serve the remote for restic's REST API.

          Synopsis

          -

          Run a basic web server to serve a remove over restic's REST backend API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly.

          +

          Run a basic web server to serve a remote over restic's REST backend API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly.

          Restic is a command-line program for doing backups.

          The server will log errors. Use -v to see access logs.

          --bwlimit will be respected for file transfers. Use --stats to control the stats printing.

          @@ -3226,92 +3334,19 @@

          Multiple repositories

          # backup user2 stuff

          Private repositories

          The--private-repos flag can be used to limit users to repositories starting with a path of /<username>/.

          -

          Server options

          -

          Use --addr to specify which IP address and port the server should listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.

          +

          Server options

          +

          Use --addr to specify which IP address and port the server should listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.

          If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.

          +

          You can use a unix socket by setting the url to unix:///path/to/socket or just by using an absolute path name. Note that unix sockets bypass the authentication - this is expected to be done with file system permissions.

          +

          --addr may be repeated to listen on multiple IPs/ports/sockets.

          --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.

          --max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.

          --baseurl controls the URL prefix that rclone serves from. By default rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically.

          -

          --template allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages:

          - ---- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
          ParameterDescription
          .NameThe full path of a file/directory.
          .TitleDirectory listing of .Name
          .SortThe current sort used. This is changeable via ?sort= parameter
          Sort Options: namedirfirst,name,size,time (default namedirfirst)
          .OrderThe current ordering used. This is changeable via ?order= parameter
          Order Options: asc,desc (default asc)
          .QueryCurrently unused.
          .BreadcrumbAllows for creating a relative navigation
          -- .LinkThe relative to the root link of the Text.
          -- .TextThe Name of the directory.
          .EntriesInformation about a specific file/directory.
          -- .URLThe 'url' of an entry.
          -- .LeafCurrently same as 'URL' but intended to be 'just' the name.
          -- .IsDirBoolean for if an entry is a directory or not.
          -- .SizeSize in Bytes of the entry.
          -- .ModTimeThe UTC timestamp of an entry.
          -

          Authentication

          +

          TLS (SSL)

          +

          By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.

          +

          --cert should be a either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.

          +

          --min-tls-version is minimum TLS version that is acceptable. Valid values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").

          +

          Authentication

          By default this will serve files without needing a login.

          You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.

          Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.

          @@ -3321,30 +3356,27 @@

          Authentication

          htpasswd -B htpasswd anotherUser

          The password file can be updated while rclone is running.

          Use --realm to set the authentication realm.

          -

          SSL/TLS

          -

          By default this will serve over HTTP. If you want you can serve over HTTPS. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.

          -

          --cert should be either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.

          -

          --min-tls-version is minimum TLS version that is acceptable. Valid values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").

          +

          Use --salt to change the password hashing salt from the default.

          rclone serve restic remote:path [flags]

          Options

          -
                --addr string                     IPaddress:Port or :Port to bind server to (default "localhost:8080")
+
                --addr stringArray                IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080])
       --append-only                     Disallow deletion of repository data
       --baseurl string                  Prefix for URLs - leave blank for root
       --cache-objects                   Cache listed objects (default true)
-      --cert string                     SSL PEM key (concatenation of certificate and CA certificate)
+      --cert string                     TLS PEM key (concatenation of certificate and CA certificate)
       --client-ca string                Client certificate authority to verify clients with
   -h, --help                            help for restic
-      --htpasswd string                 htpasswd file - if not provided no authentication is done
-      --key string                      SSL PEM Private key
+      --htpasswd string                 A htpasswd file - if not provided no authentication is done
+      --key string                      TLS PEM Private key
       --max-header-bytes int            Maximum size of request header (default 4096)
       --min-tls-version string          Minimum TLS version that is acceptable (default "tls1.0")
       --pass string                     Password for authentication
       --private-repos                   Users can only access their private repo
-      --realm string                    Realm for authentication (default "rclone")
-      --server-read-timeout duration    Timeout for server reading data (default 1h0m0s)
-      --server-write-timeout duration   Timeout for server writing data (default 1h0m0s)
+      --realm string                    Realm for authentication
+      --salt string                     Password hashing salt (default "dlPL2MqE")
+      --server-read-timeout Duration    Timeout for server reading data (default 1h0m0s)
+      --server-write-timeout Duration   Timeout for server writing data (default 1h0m0s)
       --stdio                           Run an HTTP2 server on stdin/stdout
-      --template string                 User-specified template
       --user string                     User name for authentication
          
 
          See the global flags page for global options not listed here.
          
 
          SEE ALSO
          
@@ -3518,7 +3550,7 @@ 
          Options
          
 
                --addr string                            IPaddress:Port or :Port to bind server to (default "localhost:2022")
       --auth-proxy string                      A program to use to create the backend from the auth
       --authorized-keys string                 Authorized keys file (default "~/.ssh/authorized_keys")
-      --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
+      --dir-cache-time Duration                Time to cache directory entries for (default 5m0s)
       --dir-perms FileMode                     Directory permissions (default 0777)
       --file-perms FileMode                    File permissions (default 0666)
       --gid uint32                             Override the gid field set by the filesystem (not supported on Windows) (default 1000)
@@ -3529,26 +3561,26 @@ 
          Options
          
       --no-modtime                             Don't read/write the modification time (can speed things up)
       --no-seek                                Don't allow seeking in files
       --pass string                            Password for authentication
-      --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
+      --poll-interval Duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --read-only                              Only allow read-only access
       --stdio                                  Run an sftp server on stdin/stdout
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
       --user string                            User name for authentication
-      --vfs-cache-max-age duration             Max age of objects in the cache (default 1h0m0s)
+      --vfs-cache-max-age Duration             Max age of objects in the cache (default 1h0m0s)
       --vfs-cache-max-size SizeSuffix          Max total size of objects in the cache (default off)
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
-      --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
+      --vfs-cache-poll-interval Duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
       --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
       --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
-      --vfs-read-wait duration                 Time to wait for in-sequence read before seeking (default 20ms)
+      --vfs-read-wait Duration                 Time to wait for in-sequence read before seeking (default 20ms)
       --vfs-used-is-size rclone size           Use the rclone size algorithm for Used size
-      --vfs-write-back duration                Time to writeback files after last use when using cache (default 5s)
-      --vfs-write-wait duration                Time to wait for in-sequence write before giving error (default 1s)
          
+      --vfs-write-back Duration                Time to writeback files after last use when using cache (default 5s)
+      --vfs-write-wait Duration                Time to wait for in-sequence write before giving error (default 1s)

          See the global flags page for global options not listed here.

          SEE ALSO

            @@ -3562,12 +3594,19 @@

            WebDAV options

            --etag-hash

            This controls the ETag header. Without this flag the ETag will be based on the ModTime and Size of the object.

            If this flag is set to "auto" then rclone will choose the first supported hash on the backend or you can use a named hash such as "MD5" or "SHA-1". Use the hashsum command to see the full list.

            -

            Server options

            -

            Use --addr to specify which IP address and port the server should listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.

            +

            Server options

            +

            Use --addr to specify which IP address and port the server should listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.

            If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info.

            +

            You can use a unix socket by setting the url to unix:///path/to/socket or just by using an absolute path name. Note that unix sockets bypass the authentication - this is expected to be done with file system permissions.

            +

            --addr may be repeated to listen on multiple IPs/ports/sockets.

            --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer.

            --max-header-bytes controls the maximum number of bytes the server will accept in the HTTP header.

            --baseurl controls the URL prefix that rclone serves from. By default rclone will serve from the root. If you used --baseurl "/rclone" then rclone would serve from a URL starting with "/rclone/". This is useful if you wish to proxy rclone serve. Rclone automatically inserts leading and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically.

            +

            TLS (SSL)

            +

            By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.

            +

            --cert should be a either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.

            +

            --min-tls-version is minimum TLS version that is acceptable. Valid values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").

            +

            Template

            --template allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages:

            @@ -3647,7 +3686,7 @@

            Server options

            -

            Authentication

            +

            Authentication

            By default this will serve files without needing a login.

            You can either use an htpasswd file which can take lots of users, or set a single username and password with the --user and --pass flags.

            Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in standard apache format and supports MD5, SHA1 and BCrypt for basic authentication. Bcrypt is recommended.

            @@ -3657,10 +3696,7 @@

            Authentication

            htpasswd -B htpasswd anotherUser

            The password file can be updated while rclone is running.

            Use --realm to set the authentication realm.

            -

            SSL/TLS

            -

            By default this will serve over HTTP. If you want you can serve over HTTPS. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.

            -

            --cert should be either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.

            -

            --min-tls-version is minimum TLS version that is acceptable. Valid values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0").

            +

            Use --salt to change the password hashing salt from the default.

            VFS - Virtual File System

            This command uses the VFS layer. This adapts the cloud storage objects that rclone uses into something which looks much more like a disk filing system.

            Cloud storage objects have lots of properties which aren't like disk files - you can't extend them or write to the middle of them, so the VFS layer has to deal with that. Because there is no one right way of doing this there are various options explained below.

            @@ -3808,49 +3844,50 @@

            Auth Proxy

            This can be used to build general purpose proxies to any kind of backend that rclone supports.

            rclone serve webdav remote:path [flags]

            Options

            -
                  --addr string                            IPaddress:Port or :Port to bind server to (default "localhost:8080")
+
                  --addr stringArray                       IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080])
       --auth-proxy string                      A program to use to create the backend from the auth
       --baseurl string                         Prefix for URLs - leave blank for root
-      --cert string                            SSL PEM key (concatenation of certificate and CA certificate)
+      --cert string                            TLS PEM key (concatenation of certificate and CA certificate)
       --client-ca string                       Client certificate authority to verify clients with
-      --dir-cache-time duration                Time to cache directory entries for (default 5m0s)
+      --dir-cache-time Duration                Time to cache directory entries for (default 5m0s)
       --dir-perms FileMode                     Directory permissions (default 0777)
       --disable-dir-list                       Disable HTML directory list on GET request for a directory
       --etag-hash string                       Which hash to use for the ETag, or auto or blank for off
       --file-perms FileMode                    File permissions (default 0666)
       --gid uint32                             Override the gid field set by the filesystem (not supported on Windows) (default 1000)
   -h, --help                                   help for webdav
-      --htpasswd string                        htpasswd file - if not provided no authentication is done
-      --key string                             SSL PEM Private key
+      --htpasswd string                        A htpasswd file - if not provided no authentication is done
+      --key string                             TLS PEM Private key
       --max-header-bytes int                   Maximum size of request header (default 4096)
       --min-tls-version string                 Minimum TLS version that is acceptable (default "tls1.0")
       --no-checksum                            Don't compare checksums on up/download
       --no-modtime                             Don't read/write the modification time (can speed things up)
       --no-seek                                Don't allow seeking in files
       --pass string                            Password for authentication
-      --poll-interval duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
+      --poll-interval Duration                 Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s)
       --read-only                              Only allow read-only access
-      --realm string                           Realm for authentication (default "rclone")
-      --server-read-timeout duration           Timeout for server reading data (default 1h0m0s)
-      --server-write-timeout duration          Timeout for server writing data (default 1h0m0s)
+      --realm string                           Realm for authentication
+      --salt string                            Password hashing salt (default "dlPL2MqE")
+      --server-read-timeout Duration           Timeout for server reading data (default 1h0m0s)
+      --server-write-timeout Duration          Timeout for server writing data (default 1h0m0s)
       --template string                        User-specified template
       --uid uint32                             Override the uid field set by the filesystem (not supported on Windows) (default 1000)
       --umask int                              Override the permission bits set by the filesystem (not supported on Windows) (default 2)
       --user string                            User name for authentication
-      --vfs-cache-max-age duration             Max age of objects in the cache (default 1h0m0s)
+      --vfs-cache-max-age Duration             Max age of objects in the cache (default 1h0m0s)
       --vfs-cache-max-size SizeSuffix          Max total size of objects in the cache (default off)
       --vfs-cache-mode CacheMode               Cache mode off|minimal|writes|full (default off)
-      --vfs-cache-poll-interval duration       Interval to poll the cache for stale objects (default 1m0s)
+      --vfs-cache-poll-interval Duration       Interval to poll the cache for stale objects (default 1m0s)
       --vfs-case-insensitive                   If a file name not found, find a case insensitive match
       --vfs-disk-space-total-size SizeSuffix   Specify the total space of disk (default off)
       --vfs-fast-fingerprint                   Use fast (less accurate) fingerprints for change detection
       --vfs-read-ahead SizeSuffix              Extra read ahead over --buffer-size when using cache-mode full
       --vfs-read-chunk-size SizeSuffix         Read the source objects in chunks (default 128Mi)
       --vfs-read-chunk-size-limit SizeSuffix   If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off)
-      --vfs-read-wait duration                 Time to wait for in-sequence read before seeking (default 20ms)
+      --vfs-read-wait Duration                 Time to wait for in-sequence read before seeking (default 20ms)
       --vfs-used-is-size rclone size           Use the rclone size algorithm for Used size
-      --vfs-write-back duration                Time to writeback files after last use when using cache (default 5s)
-      --vfs-write-wait duration                Time to wait for in-sequence write before giving error (default 1s)
            
+      --vfs-write-back Duration                Time to writeback files after last use when using cache (default 5s)
+      --vfs-write-wait Duration                Time to wait for in-sequence write before giving error (default 1s)

            See the global flags page for global options not listed here.

            SEE ALSO

              @@ -3901,7 +3938,7 @@

              rclone test changenotify

              rclone test changenotify remote: [flags]

              Options

                -h, --help                     help for changenotify
-      --poll-interval duration   Time to wait between polling for changes (default 10s)
              + --poll-interval Duration Time to wait between polling for changes (default 10s)

              See the global flags page for global options not listed here.

              SEE ALSO

                @@ -3933,7 +3970,7 @@

                Options

                --check-normalization Check UTF-8 Normalization --check-streaming Check uploads with indeterminate file size -h, --help help for info - --upload-wait duration Wait after writing a file + --upload-wait Duration Wait after writing a file (default 0s) --write-json string Write results to file

                See the global flags page for global options not listed here.

                SEE ALSO

                @@ -3965,6 +4002,7 @@

                Options

                --files int Number of files to create (default 1000) --files-per-directory int Average number of files per directory (default 10) -h, --help help for makefiles + --max-depth int Maximum depth of directory hierarchy (default 10) --max-file-size SizeSuffix Maximum size of files to create (default 100) --max-name-length int Maximum size of file names (default 12) --min-file-size SizeSuffix Minimum size of file to create @@ -4034,7 +4072,6 @@

                Synopsis

                rclone tree remote:path [flags]

                Options

                  -a, --all             All files are listed (list . files too)
-  -C, --color           Turn colorization on always
   -d, --dirs-only       List directories only
       --dirsfirst       List directories before files (-U disables)
       --full-path       Print the full path prefix for each file
@@ -4133,7 +4170,8 @@ 
                Connection strings, config and lo
 
                Will get their own names
                
 
                DEBUG : :s3: detected overridden config - adding "{YTu53}" suffix to name
                
 
                Valid remote names
                
-
                Remote names are case sensitive, and must adhere to the following rules: - May only contain 0-9, A-Z, a-z, _, -, . and space. - May not start with - or space.
                
+
                Remote names are case sensitive, and must adhere to the following rules: - May contain number, letter, _, -, . and space. - May not start with - or space. - May not end with space.
                
+
                Starting with rclone version 1.61, any Unicode numbers and letters are allowed, while in older versions it was limited to plain ASCII (0-9, A-Z, a-z). If you use the same rclone configuration from different shells, which may be configured with different character encoding, you must be cautious to use characters that are possible to write in all of them. This is mostly a problem on Windows, where the console traditionally uses a non-Unicode character set - defined by the so-called "code page".
                
 
                Quoting and the shell
                
 
                When you are typing commands to your computer you are using something called the command line shell. This interprets various characters in an OS specific way.
                
 
                Here are some gotchas which may help users unfamiliar with the shell rules
                
@@ -4377,6 +4415,11 @@ 
                -c, --checksum
                
 
                This is very useful when transferring between remotes which store the same hash type on the object, e.g. Drive and Swift. For details of which remotes support which hash type see the table in the overview section.
                
 
                Eg rclone --checksum sync s3:/bucket swift:/bucket would run much quicker than without the --checksum flag.
                
 
                When using this flag, rclone won't update mtimes of remote files if they are incorrect as it would normally.
                
+
                --color WHEN
                
+
                Specifiy when colors (and other ANSI codes) should be added to the output.
                
+
                AUTO (default) only allows ANSI codes when the output is a terminal
                
+
                NEVER never allow ANSI codes
                
+
                ALWAYS always add ANSI codes, regardless of the output format (terminal or file)
                
 
                --compare-dest=DIR
                
 
                When using sync, copy or move DIR is checked in addition to the destination for files. If a file identical to the source is found that file is NOT copied from source. This is useful to copy just files that have changed since the last backup.
                
 
                You must use the same remote as the destination of the sync. The compare directory must not overlap the destination directory.
                
@@ -4933,6 +4976,12 @@ 
                Filtering
                
 
              • --min-age
                • 
 
              • --max-age
                • 
 
              • --dump filters
                • 
+
              • --metadata-include
                • 
+
              • --metadata-include-from
                • 
+
              • --metadata-exclude
                • 
+
              • --metadata-exclude-from
                • 
+
              • --metadata-filter
                • 
+
              • --metadata-filter-from
                •

              See the filtering section.

              Remote control

              @@ -5035,12 +5084,13 @@

              Configuring rclone on a

              Some of the configurations (those involving oauth2) require an Internet connected web browser.

              If you are trying to set rclone up on a remote or headless box with no browser available on it (e.g. a NAS or a server in a datacenter) then you will need to use an alternative means of configuration. There are two ways of doing it, described below.

              Configuring using rclone authorize

              -

              On the headless box run rclone config but answer N to the Use auto config? question.

              +

              On the headless box run rclone config but answer N to the Use web browser to automatically authenticate? question.

              ...
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes (default)
 n) No
 y/n> n
@@ -5090,12 +5140,13 @@ 
              Configuring by copying the confi
 
              Configuring using SSH Tunnel
              
 
              Linux and MacOS users can utilize SSH Tunnel to redirect the headless box port 53682 to local machine by using the following command:
              
 
              ssh -L localhost:53682:localhost:53682 username@remote_server
              
-
              Then on the headless box run rclone config and answer Y to the Use auto config? question.
              
+
              Then on the headless box run rclone config and answer Y to the Use web  browser to automatically authenticate? question.
              
 
              ...
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes (default)
 n) No
 y/n> y
              
@@ -5109,7 +5160,7 @@ 
              Filtering, includes and excludes
              
 
              E.g. rclone copy "remote:dir*.jpg" /path/to/dir does not have a filter effect. rclone copy remote:dir /path/to/dir --include "*.jpg" does.
              
 
              Important Avoid mixing any two of --include..., --exclude... or --filter... flags in an rclone command. The results may not be what you expect. Instead use a --filter... flag.
              
 
              Patterns for matching path/file names
              
-
              Pattern syntax
              
+
              Pattern syntax
              
 
              Here is a formal definition of the pattern syntax, examples are below.
              
 
              Rclone matching rules follow a glob style:
              
 
              *         matches any sequence of non-separator (/) characters
@@ -5313,7 +5364,7 @@ 
              Filter pattern examples
              
 
 
 
-
              How filter rules are applied to files
              
+
              How filter rules are applied to files
              
 
              Rclone path/file name filters are made up of one or more of the following flags:
              
 
                
 
              • --include
                • 
@@ -5539,6 +5590,24 @@ 
                Exclude directory based on a file
              
 
              The command rclone ls --exclude-if-present .ignore dir1 does not list dir3, file3 or .ignore.
              
+
              Metadata filters
              
+
              The metadata filters work in a very similar way to the normal file name filters, except they match metadata on the object.
              
+
              The metadata should be specified as key=value patterns. This may be wildcarded using the normal filter patterns or regular expressions.
              
+
              For example if you wished to list only local files with a mode of 100664 you could do that with:
              
+
              rclone lsf -M --files-only --metadata-include "mode=100664" .
              
+
              Or if you wished to show files with an atime, mtime or btime at a given date:
              
+
              rclone lsf -M --files-only --metadata-include "[abm]time=2022-12-16*" .
              
+
              Like file filtering, metadata filtering only applies to files not to directories.
              
+
              The filters can be applied using these flags.
              
+
                
+
              • --metadata-include - Include metadatas matching pattern
                • 
+
              • --metadata-include-from - Read metadata include patterns from file (use - to read from stdin)
                • 
+
              • --metadata-exclude - Exclude metadatas matching pattern
                • 
+
              • --metadata-exclude-from - Read metadata exclude patterns from file (use - to read from stdin)
                • 
+
              • --metadata-filter - Add a metadata filtering rule
                • 
+
              • --metadata-filter-from - Read metadata filtering patterns from a file (use - to read from stdin)
                • 
+
              
+
              Each flag can be repeated. See the section on how filter rules are applied for more details - these flags work in an identical way to the file name filtering flags, but instead of file name patterns have metadata patterns.
              
 
              Common pitfalls
              
 
              The most frequent filter support issues on the rclone forum are:
              
 
                
@@ -5933,6 +6002,12 @@ 
                config/providers: Shows how providers are configured i
 
                Returns a JSON object: - providers - array of objects
                
 
                See the config providers command for more information on the above.
                
 
                Authentication is required for this call.
                
+
                config/setpath: Set the path of the config file
                
+
                Parameters:
                
+
                  
+
                • path - path to the config file to use
                  • 
+
                
+
                Authentication is required for this call.
                
 
                config/update: update the config for a remote.
                
 
                This takes the following parameters:
                
 
                  
@@ -6016,7 +6091,7 @@ 
                  core/command: Run a rclone terminal command over rc.
                  
     "result": "<Raw command line output>"
 }
 
-OR 
+OR
 {
     "error": true,
     "result": "<Raw command line output>"
@@ -6154,6 +6229,14 @@ 
                  debug/set-block-profile-rate: Set runtime.
 
                    
 
                  • rate - int
                    • 
 
                  
+
                  debug/set-gc-percent: Call runtime/debug.SetGCPercent for setting the garbage collection target percentage.
                  
+
                  SetGCPercent sets the garbage collection target percentage: a collection is triggered when the ratio of freshly allocated data to live data remaining after the previous collection reaches this percentage. SetGCPercent returns the previous setting. The initial setting is the value of the GOGC environment variable at startup, or 100 if the variable is not set.
                  
+
                  This setting may be effectively reduced in order to maintain a memory limit. A negative percentage effectively disables garbage collection, unless the memory limit is reached.
                  
+
                  See https://pkg.go.dev/runtime/debug#SetMemoryLimit for more details.
                  
+
                  Parameters:
                  
+
                    
+
                  • gc-percent - int
                    • 
+
                  
 
                  debug/set-mutex-profile-fraction: Set runtime.SetMutexProfileFraction for mutex profiling.
                  
 
                  SetMutexProfileFraction controls the fraction of mutex contention events that are reported in the mutex profile. On average 1/rate events are reported. The previous rate is returned.
                  
 
                  To turn off profiling entirely, pass rate 0. To just read the current rate, pass rate < 0. (For n>1 the details of sampling may change.)
                  
@@ -6167,6 +6250,18 @@ 
                  debug/set-mutex-profile-fraction: Set
 
                    
 
                  • previousRate - int
                    • 
 
                  
+
                  debug/set-soft-memory-limit: Call runtime/debug.SetMemoryLimit for setting a soft memory limit for the runtime.
                  
+
                  SetMemoryLimit provides the runtime with a soft memory limit.
                  
+
                  The runtime undertakes several processes to try to respect this memory limit, including adjustments to the frequency of garbage collections and returning memory to the underlying system more aggressively. This limit will be respected even if GOGC=off (or, if SetGCPercent(-1) is executed).
                  
+
                  The input limit is provided as bytes, and includes all memory mapped, managed, and not released by the Go runtime. Notably, it does not account for space used by the Go binary and memory external to Go, such as memory managed by the underlying system on behalf of the process, or memory managed by non-Go code inside the same process. Examples of excluded memory sources include: OS kernel memory held on behalf of the process, memory allocated by C code, and memory mapped by syscall.Mmap (because it is not managed by the Go runtime).
                  
+
                  A zero limit or a limit that's lower than the amount of memory used by the Go runtime may cause the garbage collector to run nearly continuously. However, the application may still make progress.
                  
+
                  The memory limit is always respected by the Go runtime, so to effectively disable this behavior, set the limit very high. math.MaxInt64 is the canonical value for disabling the limit, but values much greater than the available memory on the underlying system work just as well.
                  
+
                  See https://go.dev/doc/gc-guide for a detailed guide explaining the soft memory limit in more detail, as well as a variety of common use-cases and scenarios.
                  
+
                  SetMemoryLimit returns the previously set memory limit. A negative input does not adjust the limit, and allows for retrieval of the currently set memory limit.
                  
+
                  Parameters:
                  
+
                    
+
                  • mem-limit - int
                    • 
+
                  
 
                  fscache/clear: Clear the Fs cache.
                  
 
                  This clears the fs cache. This is where remotes created from backends are cached for a short while to make repeated rc calls more efficient.
                  
 
                  If you change the parameters of a backend then you may want to call this to clear an existing remote out of the cache before re-creating it.
                  
@@ -8081,13 +8176,13 @@ 
                  Optional Features
                  
 
 
 Oracle Object Storage
-Yes
+No
 Yes
 No
 No
 Yes
 Yes
-No
+Yes
 No
 No
 No
@@ -8212,7 +8307,7 @@ 
                  Optional Features
                  
 
 Storj
 Yes †
-No
+Yes
 Yes
 No
 No
@@ -8336,9 +8431,10 @@ 
                  Non Backend Flags
                  
   -c, --checksum                             Skip based on checksum (if available) & size, not mod-time & size
       --client-cert string                   Client SSL certificate (PEM) for mutual TLS auth
       --client-key string                    Client SSL private key (PEM) for mutual TLS auth
+      --color string                         When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS (default "AUTO")
       --compare-dest stringArray             Include additional comma separated server-side paths during comparison
       --config string                        Config file (default "$HOME/.config/rclone/rclone.conf")
-      --contimeout duration                  Connect timeout (default 1m0s)
+      --contimeout Duration                  Connect timeout (default 1m0s)
       --copy-dest stringArray                Implies --compare-dest but also copies files from paths into destination
       --cpuprofile string                    Write cpu profile to file
       --cutoff-mode string                   Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS (default "HARD")
@@ -8356,16 +8452,16 @@ 
                  Non Backend Flags
                  
       --dump-headers                         Dump HTTP headers - may contain sensitive info
       --error-on-no-transfer                 Sets exit code 9 if no files are transferred, useful in scripts
       --exclude stringArray                  Exclude files matching pattern
-      --exclude-from stringArray             Read exclude patterns from file (use - to read from stdin)
+      --exclude-from stringArray             Read file exclude patterns from file (use - to read from stdin)
       --exclude-if-present stringArray       Exclude directories if filename is present
-      --expect-continue-timeout duration     Timeout when using expect / 100-continue in HTTP (default 1s)
+      --expect-continue-timeout Duration     Timeout when using expect / 100-continue in HTTP (default 1s)
       --fast-list                            Use recursive list if available; uses more memory but fewer transactions
       --files-from stringArray               Read list of source-file names from file (use - to read from stdin)
       --files-from-raw stringArray           Read list of source-file names from file without any processing of lines (use - to read from stdin)
-  -f, --filter stringArray                   Add a file-filtering rule
-      --filter-from stringArray              Read filtering patterns from a file (use - to read from stdin)
-      --fs-cache-expire-duration duration    Cache remotes for this long (0 to disable caching) (default 5m0s)
-      --fs-cache-expire-interval duration    Interval to check for expired remotes (default 1m0s)
+  -f, --filter stringArray                   Add a file filtering rule
+      --filter-from stringArray              Read file filtering patterns from a file (use - to read from stdin)
+      --fs-cache-expire-duration Duration    Cache remotes for this long (0 to disable caching) (default 5m0s)
+      --fs-cache-expire-interval Duration    Interval to check for expired remotes (default 1m0s)
       --header stringArray                   Set HTTP header for all transactions
       --header-download stringArray          Set HTTP header for download transactions
       --header-upload stringArray            Set HTTP header for upload transactions
@@ -8379,9 +8475,9 @@ 
                  Non Backend Flags
                  
   -I, --ignore-times                         Don't skip files that match size and time - transfer all files
       --immutable                            Do not modify files, fail if existing files have been modified
       --include stringArray                  Include files matching pattern
-      --include-from stringArray             Read include patterns from file (use - to read from stdin)
+      --include-from stringArray             Read file include patterns from file (use - to read from stdin)
   -i, --interactive                          Enable interactive mode
-      --kv-lock-time duration                Maximum time to keep key-value database locked by process (default 1s)
+      --kv-lock-time Duration                Maximum time to keep key-value database locked by process (default 1s)
       --log-file string                      Log everything to this file
       --log-format string                    Comma separated list of log format options (default "date,time")
       --log-level string                     Log level DEBUG|INFO|NOTICE|ERROR (default "NOTICE")
@@ -8391,16 +8487,22 @@ 
                  Non Backend Flags
                  
       --max-backlog int                      Maximum number of objects in sync or check backlog (default 10000)
       --max-delete int                       When synchronizing, limit the number of deletes (default -1)
       --max-depth int                        If set limits the recursion depth to this (default -1)
-      --max-duration duration                Maximum duration rclone will transfer data for
+      --max-duration Duration                Maximum duration rclone will transfer data for (default 0s)
       --max-size SizeSuffix                  Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P (default off)
       --max-stats-groups int                 Maximum number of stats groups to keep in memory, on max oldest is discarded (default 1000)
       --max-transfer SizeSuffix              Maximum size of data to transfer (default off)
       --memprofile string                    Write memory profile to file
   -M, --metadata                             If set, preserve metadata when copying objects
+      --metadata-exclude stringArray         Exclude metadatas matching pattern
+      --metadata-exclude-from stringArray    Read metadata exclude patterns from file (use - to read from stdin)
+      --metadata-filter stringArray          Add a metadata filtering rule
+      --metadata-filter-from stringArray     Read metadata filtering patterns from a file (use - to read from stdin)
+      --metadata-include stringArray         Include metadatas matching pattern
+      --metadata-include-from stringArray    Read metadata include patterns from file (use - to read from stdin)
       --metadata-set stringArray             Add metadata key=value when uploading
       --min-age Duration                     Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off)
       --min-size SizeSuffix                  Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off)
-      --modify-window duration               Max time diff to be considered the same (default 1ns)
+      --modify-window Duration               Max time diff to be considered the same (default 1ns)
       --multi-thread-cutoff SizeSuffix       Use multi-thread downloads for files above this size (default 250Mi)
       --multi-thread-streams int             Max number of streams to use for multi-thread downloads (default 4)
       --no-check-certificate                 Do not verify the server SSL certificate (insecure)
@@ -8416,25 +8518,26 @@ 
                  Non Backend Flags
                  
       --progress-terminal-title              Show progress on the terminal title (requires -P/--progress)
   -q, --quiet                                Print as little stuff as possible
       --rc                                   Enable the remote control server
-      --rc-addr string                       IPaddress:Port or :Port to bind server to (default "localhost:5572")
+      --rc-addr stringArray                  IPaddress:Port or :Port to bind server to (default [localhost:5572])
       --rc-allow-origin string               Set the allowed origin for CORS
       --rc-baseurl string                    Prefix for URLs - leave blank for root
-      --rc-cert string                       SSL PEM key (concatenation of certificate and CA certificate)
+      --rc-cert string                       TLS PEM key (concatenation of certificate and CA certificate)
       --rc-client-ca string                  Client certificate authority to verify clients with
       --rc-enable-metrics                    Enable prometheus metrics on /metrics
       --rc-files string                      Path to local files to serve on the HTTP server
-      --rc-htpasswd string                   htpasswd file - if not provided no authentication is done
-      --rc-job-expire-duration duration      Expire finished async jobs older than this value (default 1m0s)
-      --rc-job-expire-interval duration      Interval to check for expired async jobs (default 10s)
-      --rc-key string                        SSL PEM Private key
+      --rc-htpasswd string                   A htpasswd file - if not provided no authentication is done
+      --rc-job-expire-duration Duration      Expire finished async jobs older than this value (default 1m0s)
+      --rc-job-expire-interval Duration      Interval to check for expired async jobs (default 10s)
+      --rc-key string                        TLS PEM Private key
       --rc-max-header-bytes int              Maximum size of request header (default 4096)
       --rc-min-tls-version string            Minimum TLS version that is acceptable (default "tls1.0")
       --rc-no-auth                           Don't require auth for certain methods
       --rc-pass string                       Password for authentication
-      --rc-realm string                      Realm for authentication (default "rclone")
+      --rc-realm string                      Realm for authentication
+      --rc-salt string                       Password hashing salt (default "dlPL2MqE")
       --rc-serve                             Enable the serving of remote objects
-      --rc-server-read-timeout duration      Timeout for server reading data (default 1h0m0s)
-      --rc-server-write-timeout duration     Timeout for server writing data (default 1h0m0s)
+      --rc-server-read-timeout Duration      Timeout for server reading data (default 1h0m0s)
+      --rc-server-write-timeout Duration     Timeout for server writing data (default 1h0m0s)
       --rc-template string                   User-specified template
       --rc-user string                       User name for authentication
       --rc-web-fetch-url string              URL to fetch the releases for webgui (default "https://api.github.com/repos/rclone/rclone-webui-react/releases/latest")
@@ -8444,10 +8547,10 @@ 
                  Non Backend Flags
                  
       --rc-web-gui-update                    Check and update to latest version of web gui
       --refresh-times                        Refresh the modtime of remote files
       --retries int                          Retry operations this many times if they fail (default 3)
-      --retries-sleep duration               Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)
+      --retries-sleep Duration               Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) (default 0s)
       --server-side-across-configs           Allow server-side operations (e.g. copy) to work across different configs
       --size-only                            Skip based on size only, not mod-time or checksum
-      --stats duration                       Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s)
+      --stats Duration                       Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s)
       --stats-file-name-length int           Max file name length in stats (0 for no limit) (default 45)
       --stats-log-level string               Log level to show --stats output DEBUG|INFO|NOTICE|ERROR (default "INFO")
       --stats-one-line                       Make the stats fit on one line
@@ -8460,7 +8563,7 @@ 
                  Non Backend Flags
                  
       --syslog                               Use Syslog for logging
       --syslog-facility string               Facility for syslog, e.g. KERN,USER,... (default "DAEMON")
       --temp-dir string                      Directory rclone will use for temporary files (default "/tmp")
-      --timeout duration                     IO idle timeout (default 5m0s)
+      --timeout Duration                     IO idle timeout (default 5m0s)
       --tpslimit float                       Limit HTTP transactions per second to this
       --tpslimit-burst int                   Max burst of transactions for --tpslimit (default 1)
       --track-renames                        When synchronizing, track file renames and do a server-side move if possible
@@ -8471,533 +8574,547 @@ 
                  Non Backend Flags
                  
       --use-json-log                         Use json log format
       --use-mmap                             Use mmap allocator (see docs)
       --use-server-modtime                   Use server modified time instead of object metadata
-      --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.60.0")
+      --user-agent string                    Set the user-agent to a specified string (default "rclone/v1.61.0")
   -v, --verbose count                        Print lots more stuff (repeat for more)

              Backend Flags

              These flags are available for every command. They control the backends and may be set in the config file.

              -
                    --acd-auth-url string                          Auth server URL
-      --acd-client-id string                         OAuth Client Id
-      --acd-client-secret string                     OAuth Client Secret
-      --acd-encoding MultiEncoder                    The encoding for the backend (default Slash,InvalidUtf8,Dot)
-      --acd-templink-threshold SizeSuffix            Files >= this size will be downloaded via their tempLink (default 9Gi)
-      --acd-token string                             OAuth Access Token as a JSON blob
-      --acd-token-url string                         Token server url
-      --acd-upload-wait-per-gb Duration              Additional time per GiB to wait after a failed complete upload to see if it appears (default 3m0s)
-      --alias-remote string                          Remote or path to alias
-      --azureblob-access-tier string                 Access tier of blob: hot, cool or archive
-      --azureblob-account string                     Storage Account Name
-      --azureblob-archive-tier-delete                Delete archive tier blobs before overwriting
-      --azureblob-chunk-size SizeSuffix              Upload chunk size (default 4Mi)
-      --azureblob-disable-checksum                   Don't store MD5 checksum with object metadata
-      --azureblob-encoding MultiEncoder              The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8)
-      --azureblob-endpoint string                    Endpoint for the service
-      --azureblob-key string                         Storage Account Key
-      --azureblob-list-chunk int                     Size of blob list (default 5000)
-      --azureblob-memory-pool-flush-time Duration    How often internal memory buffer pools will be flushed (default 1m0s)
-      --azureblob-memory-pool-use-mmap               Whether to use mmap buffers in internal memory pool
-      --azureblob-msi-client-id string               Object ID of the user-assigned MSI to use, if any
-      --azureblob-msi-mi-res-id string               Azure resource ID of the user-assigned MSI to use, if any
-      --azureblob-msi-object-id string               Object ID of the user-assigned MSI to use, if any
-      --azureblob-no-head-object                     If set, do not do HEAD before GET when getting objects
-      --azureblob-public-access string               Public access level of a container: blob or container
-      --azureblob-sas-url string                     SAS URL for container level access only
-      --azureblob-service-principal-file string      Path to file containing credentials for use with a service principal
-      --azureblob-upload-concurrency int             Concurrency for multipart uploads (default 16)
-      --azureblob-upload-cutoff string               Cutoff for switching to chunked upload (<= 256 MiB) (deprecated)
-      --azureblob-use-emulator                       Uses local storage emulator if provided as 'true'
-      --azureblob-use-msi                            Use a managed service identity to authenticate (only works in Azure)
-      --b2-account string                            Account ID or Application Key ID
-      --b2-chunk-size SizeSuffix                     Upload chunk size (default 96Mi)
-      --b2-copy-cutoff SizeSuffix                    Cutoff for switching to multipart copy (default 4Gi)
-      --b2-disable-checksum                          Disable checksums for large (> upload cutoff) files
-      --b2-download-auth-duration Duration           Time before the authorization token will expire in s or suffix ms|s|m|h|d (default 1w)
-      --b2-download-url string                       Custom endpoint for downloads
-      --b2-encoding MultiEncoder                     The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
-      --b2-endpoint string                           Endpoint for the service
-      --b2-hard-delete                               Permanently delete files on remote removal, otherwise hide files
-      --b2-key string                                Application Key
-      --b2-memory-pool-flush-time Duration           How often internal memory buffer pools will be flushed (default 1m0s)
-      --b2-memory-pool-use-mmap                      Whether to use mmap buffers in internal memory pool
-      --b2-test-mode string                          A flag string for X-Bz-Test-Mode header for debugging
-      --b2-upload-cutoff SizeSuffix                  Cutoff for switching to chunked upload (default 200Mi)
-      --b2-version-at Time                           Show file versions as they were at the specified time (default off)
-      --b2-versions                                  Include old versions in directory listings
-      --box-access-token string                      Box App Primary Access Token
-      --box-auth-url string                          Auth server URL
-      --box-box-config-file string                   Box App config.json location
-      --box-box-sub-type string                       (default "user")
-      --box-client-id string                         OAuth Client Id
-      --box-client-secret string                     OAuth Client Secret
-      --box-commit-retries int                       Max number of times to try committing a multipart file (default 100)
-      --box-encoding MultiEncoder                    The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot)
-      --box-list-chunk int                           Size of listing chunk 1-1000 (default 1000)
-      --box-owned-by string                          Only show items owned by the login (email address) passed in
-      --box-root-folder-id string                    Fill in for rclone to use a non root folder as its starting point
-      --box-token string                             OAuth Access Token as a JSON blob
-      --box-token-url string                         Token server url
-      --box-upload-cutoff SizeSuffix                 Cutoff for switching to multipart upload (>= 50 MiB) (default 50Mi)
-      --cache-chunk-clean-interval Duration          How often should the cache perform cleanups of the chunk storage (default 1m0s)
-      --cache-chunk-no-memory                        Disable the in-memory cache for storing chunks during streaming
-      --cache-chunk-path string                      Directory to cache chunk files (default "$HOME/.cache/rclone/cache-backend")
-      --cache-chunk-size SizeSuffix                  The size of a chunk (partial file data) (default 5Mi)
-      --cache-chunk-total-size SizeSuffix            The total size that the chunks can take up on the local disk (default 10Gi)
-      --cache-db-path string                         Directory to store file structure metadata DB (default "$HOME/.cache/rclone/cache-backend")
-      --cache-db-purge                               Clear all the cached data for this remote on start
-      --cache-db-wait-time Duration                  How long to wait for the DB to be available - 0 is unlimited (default 1s)
-      --cache-info-age Duration                      How long to cache file structure information (directory listings, file size, times, etc.) (default 6h0m0s)
-      --cache-plex-insecure string                   Skip all certificate verification when connecting to the Plex server
-      --cache-plex-password string                   The password of the Plex user (obscured)
-      --cache-plex-url string                        The URL of the Plex server
-      --cache-plex-username string                   The username of the Plex user
-      --cache-read-retries int                       How many times to retry a read from a cache storage (default 10)
-      --cache-remote string                          Remote to cache
-      --cache-rps int                                Limits the number of requests per second to the source FS (-1 to disable) (default -1)
-      --cache-tmp-upload-path string                 Directory to keep temporary files until they are uploaded
-      --cache-tmp-wait-time Duration                 How long should files be stored in local cache before being uploaded (default 15s)
-      --cache-workers int                            How many workers should run in parallel to download chunks (default 4)
-      --cache-writes                                 Cache file data on writes through the FS
-      --chunker-chunk-size SizeSuffix                Files larger than chunk size will be split in chunks (default 2Gi)
-      --chunker-fail-hard                            Choose how chunker should handle files with missing or invalid chunks
-      --chunker-hash-type string                     Choose how chunker handles hash sums (default "md5")
-      --chunker-remote string                        Remote to chunk/unchunk
-      --combine-upstreams SpaceSepList               Upstreams for combining
-      --compress-level int                           GZIP compression level (-2 to 9) (default -1)
-      --compress-mode string                         Compression mode (default "gzip")
-      --compress-ram-cache-limit SizeSuffix          Some remotes don't allow the upload of files with unknown size (default 20Mi)
-      --compress-remote string                       Remote to compress
-  -L, --copy-links                                   Follow symlinks and copy the pointed to item
-      --crypt-directory-name-encryption              Option to either encrypt directory names or leave them intact (default true)
-      --crypt-filename-encoding string               How to encode the encrypted filename to text string (default "base32")
-      --crypt-filename-encryption string             How to encrypt the filenames (default "standard")
-      --crypt-no-data-encryption                     Option to either encrypt file data or leave it unencrypted
-      --crypt-password string                        Password or pass phrase for encryption (obscured)
-      --crypt-password2 string                       Password or pass phrase for salt (obscured)
-      --crypt-remote string                          Remote to encrypt/decrypt
-      --crypt-server-side-across-configs             Allow server-side operations (e.g. copy) to work across different crypt configs
-      --crypt-show-mapping                           For all files listed show how the names encrypt
-      --drive-acknowledge-abuse                      Set to allow files which return cannotDownloadAbusiveFile to be downloaded
-      --drive-allow-import-name-change               Allow the filetype to change when uploading Google docs
-      --drive-auth-owner-only                        Only consider files owned by the authenticated user
-      --drive-auth-url string                        Auth server URL
-      --drive-chunk-size SizeSuffix                  Upload chunk size (default 8Mi)
-      --drive-client-id string                       Google Application Client Id
-      --drive-client-secret string                   OAuth Client Secret
-      --drive-copy-shortcut-content                  Server side copy contents of shortcuts instead of the shortcut
-      --drive-disable-http2                          Disable drive using http2 (default true)
-      --drive-encoding MultiEncoder                  The encoding for the backend (default InvalidUtf8)
-      --drive-export-formats string                  Comma separated list of preferred formats for downloading Google docs (default "docx,xlsx,pptx,svg")
-      --drive-formats string                         Deprecated: See export_formats
-      --drive-impersonate string                     Impersonate this user when using a service account
-      --drive-import-formats string                  Comma separated list of preferred formats for uploading Google docs
-      --drive-keep-revision-forever                  Keep new head revision of each file forever
-      --drive-list-chunk int                         Size of listing chunk 100-1000, 0 to disable (default 1000)
-      --drive-pacer-burst int                        Number of API calls to allow without sleeping (default 100)
-      --drive-pacer-min-sleep Duration               Minimum time to sleep between API calls (default 100ms)
-      --drive-resource-key string                    Resource key for accessing a link-shared file
-      --drive-root-folder-id string                  ID of the root folder
-      --drive-scope string                           Scope that rclone should use when requesting access from drive
-      --drive-server-side-across-configs             Allow server-side operations (e.g. copy) to work across different drive configs
-      --drive-service-account-credentials string     Service Account Credentials JSON blob
-      --drive-service-account-file string            Service Account Credentials JSON file path
-      --drive-shared-with-me                         Only show files that are shared with me
-      --drive-size-as-quota                          Show sizes as storage quota usage, not actual size
-      --drive-skip-checksum-gphotos                  Skip MD5 checksum on Google photos and videos only
-      --drive-skip-dangling-shortcuts                If set skip dangling shortcut files
-      --drive-skip-gdocs                             Skip google documents in all listings
-      --drive-skip-shortcuts                         If set skip shortcut files
-      --drive-starred-only                           Only show files that are starred
-      --drive-stop-on-download-limit                 Make download limit errors be fatal
-      --drive-stop-on-upload-limit                   Make upload limit errors be fatal
-      --drive-team-drive string                      ID of the Shared Drive (Team Drive)
-      --drive-token string                           OAuth Access Token as a JSON blob
-      --drive-token-url string                       Token server url
-      --drive-trashed-only                           Only show files that are in the trash
-      --drive-upload-cutoff SizeSuffix               Cutoff for switching to chunked upload (default 8Mi)
-      --drive-use-created-date                       Use file created date instead of modified date
-      --drive-use-shared-date                        Use date file was shared instead of modified date
-      --drive-use-trash                              Send files to the trash instead of deleting permanently (default true)
-      --drive-v2-download-min-size SizeSuffix        If Object's are greater, use drive v2 API to download (default off)
-      --dropbox-auth-url string                      Auth server URL
-      --dropbox-batch-commit-timeout Duration        Max time to wait for a batch to finish committing (default 10m0s)
-      --dropbox-batch-mode string                    Upload file batching sync|async|off (default "sync")
-      --dropbox-batch-size int                       Max number of files in upload batch
-      --dropbox-batch-timeout Duration               Max time to allow an idle upload batch before uploading (default 0s)
-      --dropbox-chunk-size SizeSuffix                Upload chunk size (< 150Mi) (default 48Mi)
-      --dropbox-client-id string                     OAuth Client Id
-      --dropbox-client-secret string                 OAuth Client Secret
-      --dropbox-encoding MultiEncoder                The encoding for the backend (default Slash,BackSlash,Del,RightSpace,InvalidUtf8,Dot)
-      --dropbox-impersonate string                   Impersonate this user when using a business account
-      --dropbox-shared-files                         Instructs rclone to work on individual shared files
-      --dropbox-shared-folders                       Instructs rclone to work on shared folders
-      --dropbox-token string                         OAuth Access Token as a JSON blob
-      --dropbox-token-url string                     Token server url
-      --fichier-api-key string                       Your API Key, get it from https://1fichier.com/console/params.pl
-      --fichier-encoding MultiEncoder                The encoding for the backend (default Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot)
-      --fichier-file-password string                 If you want to download a shared file that is password protected, add this parameter (obscured)
-      --fichier-folder-password string               If you want to list the files in a shared folder that is password protected, add this parameter (obscured)
-      --fichier-shared-folder string                 If you want to download a shared folder, add this parameter
-      --filefabric-encoding MultiEncoder             The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot)
-      --filefabric-permanent-token string            Permanent Authentication Token
-      --filefabric-root-folder-id string             ID of the root folder
-      --filefabric-token string                      Session Token
-      --filefabric-token-expiry string               Token expiry time
-      --filefabric-url string                        URL of the Enterprise File Fabric to connect to
-      --filefabric-version string                    Version read from the file fabric
-      --ftp-ask-password                             Allow asking for FTP password when needed
-      --ftp-close-timeout Duration                   Maximum time to wait for a response to close (default 1m0s)
-      --ftp-concurrency int                          Maximum number of FTP simultaneous connections, 0 for unlimited
-      --ftp-disable-epsv                             Disable using EPSV even if server advertises support
-      --ftp-disable-mlsd                             Disable using MLSD even if server advertises support
-      --ftp-disable-tls13                            Disable TLS 1.3 (workaround for FTP servers with buggy TLS)
-      --ftp-disable-utf8                             Disable using UTF-8 even if server advertises support
-      --ftp-encoding MultiEncoder                    The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
-      --ftp-explicit-tls                             Use Explicit FTPS (FTP over TLS)
-      --ftp-force-list-hidden                        Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
-      --ftp-host string                              FTP host to connect to
-      --ftp-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
-      --ftp-no-check-certificate                     Do not verify the TLS certificate of the server
-      --ftp-pass string                              FTP password (obscured)
-      --ftp-port int                                 FTP port number (default 21)
-      --ftp-shut-timeout Duration                    Maximum time to wait for data connection closing status (default 1m0s)
-      --ftp-tls                                      Use Implicit FTPS (FTP over TLS)
-      --ftp-tls-cache-size int                       Size of TLS session cache for all control and data connections (default 32)
-      --ftp-user string                              FTP username (default "$USER")
-      --ftp-writing-mdtm                             Use MDTM to set modification time (VsFtpd quirk)
-      --gcs-anonymous                                Access public buckets and objects without credentials
-      --gcs-auth-url string                          Auth server URL
-      --gcs-bucket-acl string                        Access Control List for new buckets
-      --gcs-bucket-policy-only                       Access checks should use bucket-level IAM policies
-      --gcs-client-id string                         OAuth Client Id
-      --gcs-client-secret string                     OAuth Client Secret
-      --gcs-decompress                               If set this will decompress gzip encoded objects
-      --gcs-encoding MultiEncoder                    The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot)
-      --gcs-endpoint string                          Endpoint for the service
-      --gcs-location string                          Location for the newly created buckets
-      --gcs-no-check-bucket                          If set, don't attempt to check the bucket exists or create it
-      --gcs-object-acl string                        Access Control List for new objects
-      --gcs-project-number string                    Project number
-      --gcs-service-account-file string              Service Account Credentials JSON file path
-      --gcs-storage-class string                     The storage class to use when storing objects in Google Cloud Storage
-      --gcs-token string                             OAuth Access Token as a JSON blob
-      --gcs-token-url string                         Token server url
-      --gphotos-auth-url string                      Auth server URL
-      --gphotos-client-id string                     OAuth Client Id
-      --gphotos-client-secret string                 OAuth Client Secret
-      --gphotos-encoding MultiEncoder                The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot)
-      --gphotos-include-archived                     Also view and download archived media
-      --gphotos-read-only                            Set to make the Google Photos backend read only
-      --gphotos-read-size                            Set to read the size of media items
-      --gphotos-start-year int                       Year limits the photos to be downloaded to those which are uploaded after the given year (default 2000)
-      --gphotos-token string                         OAuth Access Token as a JSON blob
-      --gphotos-token-url string                     Token server url
-      --hasher-auto-size SizeSuffix                  Auto-update checksum for files smaller than this size (disabled by default)
-      --hasher-hashes CommaSepList                   Comma separated list of supported checksum types (default md5,sha1)
-      --hasher-max-age Duration                      Maximum time to keep checksums in cache (0 = no cache, off = cache forever) (default off)
-      --hasher-remote string                         Remote to cache checksums for (e.g. myRemote:path)
-      --hdfs-data-transfer-protection string         Kerberos data transfer protection: authentication|integrity|privacy
-      --hdfs-encoding MultiEncoder                   The encoding for the backend (default Slash,Colon,Del,Ctl,InvalidUtf8,Dot)
-      --hdfs-namenode string                         Hadoop name node and port
-      --hdfs-service-principal-name string           Kerberos service principal name for the namenode
-      --hdfs-username string                         Hadoop user name
-      --hidrive-auth-url string                      Auth server URL
-      --hidrive-chunk-size SizeSuffix                Chunksize for chunked uploads (default 48Mi)
-      --hidrive-client-id string                     OAuth Client Id
-      --hidrive-client-secret string                 OAuth Client Secret
-      --hidrive-disable-fetching-member-count        Do not fetch number of objects in directories unless it is absolutely necessary
-      --hidrive-encoding MultiEncoder                The encoding for the backend (default Slash,Dot)
-      --hidrive-endpoint string                      Endpoint for the service (default "https://api.hidrive.strato.com/2.1")
-      --hidrive-root-prefix string                   The root/parent folder for all paths (default "/")
-      --hidrive-scope-access string                  Access permissions that rclone should use when requesting access from HiDrive (default "rw")
-      --hidrive-scope-role string                    User-level that rclone should use when requesting access from HiDrive (default "user")
-      --hidrive-token string                         OAuth Access Token as a JSON blob
-      --hidrive-token-url string                     Token server url
-      --hidrive-upload-concurrency int               Concurrency for chunked uploads (default 4)
-      --hidrive-upload-cutoff SizeSuffix             Cutoff/Threshold for chunked uploads (default 96Mi)
-      --http-headers CommaSepList                    Set HTTP headers for all transactions
-      --http-no-head                                 Don't use HEAD requests
-      --http-no-slash                                Set this if the site doesn't end directories with /
-      --http-url string                              URL of HTTP host to connect to
-      --internetarchive-access-key-id string         IAS3 Access Key
-      --internetarchive-disable-checksum             Don't ask the server to test against MD5 checksum calculated by rclone (default true)
-      --internetarchive-encoding MultiEncoder        The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot)
-      --internetarchive-endpoint string              IAS3 Endpoint (default "https://s3.us.archive.org")
-      --internetarchive-front-endpoint string        Host of InternetArchive Frontend (default "https://archive.org")
-      --internetarchive-secret-access-key string     IAS3 Secret Key (password)
-      --internetarchive-wait-archive Duration        Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s)
-      --jottacloud-encoding MultiEncoder             The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot)
-      --jottacloud-hard-delete                       Delete files permanently rather than putting them into the trash
-      --jottacloud-md5-memory-limit SizeSuffix       Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi)
-      --jottacloud-no-versions                       Avoid server side versioning by deleting files and recreating files instead of overwriting them
-      --jottacloud-trashed-only                      Only show files that are in the trash
-      --jottacloud-upload-resume-limit SizeSuffix    Files bigger than this can be resumed if the upload fail's (default 10Mi)
-      --koofr-encoding MultiEncoder                  The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
-      --koofr-endpoint string                        The Koofr API endpoint to use
-      --koofr-mountid string                         Mount ID of the mount to use
-      --koofr-password string                        Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured)
-      --koofr-provider string                        Choose your storage provider
-      --koofr-setmtime                               Does the backend support setting modification time (default true)
-      --koofr-user string                            Your user name
-  -l, --links                                        Translate symlinks to/from regular files with a '.rclonelink' extension
-      --local-case-insensitive                       Force the filesystem to report itself as case insensitive
-      --local-case-sensitive                         Force the filesystem to report itself as case sensitive
-      --local-encoding MultiEncoder                  The encoding for the backend (default Slash,Dot)
-      --local-no-check-updated                       Don't check to see if the files change during upload
-      --local-no-preallocate                         Disable preallocation of disk space for transferred files
-      --local-no-set-modtime                         Disable setting modtime
-      --local-no-sparse                              Disable sparse files for multi-thread downloads
-      --local-nounc                                  Disable UNC (long path names) conversion on Windows
-      --local-unicode-normalization                  Apply unicode NFC normalization to paths and filenames
-      --local-zero-size-links                        Assume the Stat size of links is zero (and read them instead) (deprecated)
-      --mailru-check-hash                            What should copy do if file checksum is mismatched or invalid (default true)
-      --mailru-encoding MultiEncoder                 The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot)
-      --mailru-pass string                           Password (obscured)
-      --mailru-speedup-enable                        Skip full upload if there is another file with same data hash (default true)
-      --mailru-speedup-file-patterns string          Comma separated list of file name patterns eligible for speedup (put by hash) (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf")
-      --mailru-speedup-max-disk SizeSuffix           This option allows you to disable speedup (put by hash) for large files (default 3Gi)
-      --mailru-speedup-max-memory SizeSuffix         Files larger than the size given below will always be hashed on disk (default 32Mi)
-      --mailru-user string                           User name (usually email)
-      --mega-debug                                   Output more debug from Mega
-      --mega-encoding MultiEncoder                   The encoding for the backend (default Slash,InvalidUtf8,Dot)
-      --mega-hard-delete                             Delete files permanently rather than putting them into the trash
-      --mega-pass string                             Password (obscured)
-      --mega-user string                             User name
-      --netstorage-account string                    Set the NetStorage account name
-      --netstorage-host string                       Domain+path of NetStorage host to connect to
-      --netstorage-protocol string                   Select between HTTP or HTTPS protocol (default "https")
-      --netstorage-secret string                     Set the NetStorage account secret/G2O key for authentication (obscured)
-  -x, --one-file-system                              Don't cross filesystem boundaries (unix/macOS only)
-      --onedrive-access-scopes SpaceSepList          Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access)
-      --onedrive-auth-url string                     Auth server URL
-      --onedrive-chunk-size SizeSuffix               Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi)
-      --onedrive-client-id string                    OAuth Client Id
-      --onedrive-client-secret string                OAuth Client Secret
-      --onedrive-drive-id string                     The ID of the drive to use
-      --onedrive-drive-type string                   The type of the drive (personal | business | documentLibrary)
-      --onedrive-encoding MultiEncoder               The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot)
-      --onedrive-expose-onenote-files                Set to make OneNote files show up in directory listings
-      --onedrive-link-password string                Set the password for links created by the link command
-      --onedrive-link-scope string                   Set the scope of the links created by the link command (default "anonymous")
-      --onedrive-link-type string                    Set the type of the links created by the link command (default "view")
-      --onedrive-list-chunk int                      Size of listing chunk (default 1000)
-      --onedrive-no-versions                         Remove all versions on modifying operations
-      --onedrive-region string                       Choose national cloud region for OneDrive (default "global")
-      --onedrive-root-folder-id string               ID of the root folder
-      --onedrive-server-side-across-configs          Allow server-side operations (e.g. copy) to work across different onedrive configs
-      --onedrive-token string                        OAuth Access Token as a JSON blob
-      --onedrive-token-url string                    Token server url
-      --oos-chunk-size SizeSuffix                    Chunk size to use for uploading (default 5Mi)
-      --oos-compartment string                       Object storage compartment OCID
-      --oos-config-file string                       Path to OCI config file (default "~/.oci/config")
-      --oos-config-profile string                    Profile name inside the oci config file (default "Default")
-      --oos-copy-cutoff SizeSuffix                   Cutoff for switching to multipart copy (default 4.656Gi)
-      --oos-copy-timeout Duration                    Timeout for copy (default 1m0s)
-      --oos-disable-checksum                         Don't store MD5 checksum with object metadata
-      --oos-encoding MultiEncoder                    The encoding for the backend (default Slash,InvalidUtf8,Dot)
-      --oos-endpoint string                          Endpoint for Object storage API
-      --oos-leave-parts-on-error                     If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery
-      --oos-namespace string                         Object storage namespace
-      --oos-no-check-bucket                          If set, don't attempt to check the bucket exists or create it
-      --oos-provider string                          Choose your Auth Provider (default "env_auth")
-      --oos-region string                            Object storage Region
-      --oos-upload-concurrency int                   Concurrency for multipart uploads (default 10)
-      --oos-upload-cutoff SizeSuffix                 Cutoff for switching to chunked upload (default 200Mi)
-      --opendrive-chunk-size SizeSuffix              Files will be uploaded in chunks this size (default 10Mi)
-      --opendrive-encoding MultiEncoder              The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot)
-      --opendrive-password string                    Password (obscured)
-      --opendrive-username string                    Username
-      --pcloud-auth-url string                       Auth server URL
-      --pcloud-client-id string                      OAuth Client Id
-      --pcloud-client-secret string                  OAuth Client Secret
-      --pcloud-encoding MultiEncoder                 The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
-      --pcloud-hostname string                       Hostname to connect to (default "api.pcloud.com")
-      --pcloud-password string                       Your pcloud password (obscured)
-      --pcloud-root-folder-id string                 Fill in for rclone to use a non root folder as its starting point (default "d0")
-      --pcloud-token string                          OAuth Access Token as a JSON blob
-      --pcloud-token-url string                      Token server url
-      --pcloud-username string                       Your pcloud username
-      --premiumizeme-encoding MultiEncoder           The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot)
-      --putio-encoding MultiEncoder                  The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
-      --qingstor-access-key-id string                QingStor Access Key ID
-      --qingstor-chunk-size SizeSuffix               Chunk size to use for uploading (default 4Mi)
-      --qingstor-connection-retries int              Number of connection retries (default 3)
-      --qingstor-encoding MultiEncoder               The encoding for the backend (default Slash,Ctl,InvalidUtf8)
-      --qingstor-endpoint string                     Enter an endpoint URL to connection QingStor API
-      --qingstor-env-auth                            Get QingStor credentials from runtime
-      --qingstor-secret-access-key string            QingStor Secret Access Key (password)
-      --qingstor-upload-concurrency int              Concurrency for multipart uploads (default 1)
-      --qingstor-upload-cutoff SizeSuffix            Cutoff for switching to chunked upload (default 200Mi)
-      --qingstor-zone string                         Zone to connect to
-      --s3-access-key-id string                      AWS Access Key ID
-      --s3-acl string                                Canned ACL used when creating buckets and storing or copying objects
-      --s3-bucket-acl string                         Canned ACL used when creating buckets
-      --s3-chunk-size SizeSuffix                     Chunk size to use for uploading (default 5Mi)
-      --s3-copy-cutoff SizeSuffix                    Cutoff for switching to multipart copy (default 4.656Gi)
-      --s3-decompress                                If set this will decompress gzip encoded objects
-      --s3-disable-checksum                          Don't store MD5 checksum with object metadata
-      --s3-disable-http2                             Disable usage of http2 for S3 backends
-      --s3-download-url string                       Custom endpoint for downloads
-      --s3-encoding MultiEncoder                     The encoding for the backend (default Slash,InvalidUtf8,Dot)
-      --s3-endpoint string                           Endpoint for S3 API
-      --s3-env-auth                                  Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars)
-      --s3-force-path-style                          If true use path style access if false use virtual hosted style (default true)
-      --s3-leave-parts-on-error                      If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery
-      --s3-list-chunk int                            Size of listing chunk (response list for each ListObject S3 request) (default 1000)
-      --s3-list-url-encode Tristate                  Whether to url encode listings: true/false/unset (default unset)
-      --s3-list-version int                          Version of ListObjects to use: 1,2 or 0 for auto
-      --s3-location-constraint string                Location constraint - must be set to match the Region
-      --s3-max-upload-parts int                      Maximum number of parts in a multipart upload (default 10000)
-      --s3-memory-pool-flush-time Duration           How often internal memory buffer pools will be flushed (default 1m0s)
-      --s3-memory-pool-use-mmap                      Whether to use mmap buffers in internal memory pool
-      --s3-no-check-bucket                           If set, don't attempt to check the bucket exists or create it
-      --s3-no-head                                   If set, don't HEAD uploaded objects to check integrity
-      --s3-no-head-object                            If set, do not do HEAD before GET when getting objects
-      --s3-no-system-metadata                        Suppress setting and reading of system metadata
-      --s3-profile string                            Profile to use in the shared credentials file
-      --s3-provider string                           Choose your S3 provider
-      --s3-region string                             Region to connect to
-      --s3-requester-pays                            Enables requester pays option when interacting with S3 bucket
-      --s3-secret-access-key string                  AWS Secret Access Key (password)
-      --s3-server-side-encryption string             The server-side encryption algorithm used when storing this object in S3
-      --s3-session-token string                      An AWS session token
-      --s3-shared-credentials-file string            Path to the shared credentials file
-      --s3-sse-customer-algorithm string             If using SSE-C, the server-side encryption algorithm used when storing this object in S3
-      --s3-sse-customer-key string                   To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data
-      --s3-sse-customer-key-base64 string            If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data
-      --s3-sse-customer-key-md5 string               If using SSE-C you may provide the secret encryption key MD5 checksum (optional)
-      --s3-sse-kms-key-id string                     If using KMS ID you must provide the ARN of Key
-      --s3-storage-class string                      The storage class to use when storing new objects in S3
-      --s3-upload-concurrency int                    Concurrency for multipart uploads (default 4)
-      --s3-upload-cutoff SizeSuffix                  Cutoff for switching to chunked upload (default 200Mi)
-      --s3-use-accelerate-endpoint                   If true use the AWS S3 accelerated endpoint
-      --s3-use-multipart-etag Tristate               Whether to use ETag in multipart uploads for verification (default unset)
-      --s3-use-presigned-request                     Whether to use a presigned request or PutObject for single part uploads
-      --s3-v2-auth                                   If true use v2 authentication
-      --s3-version-at Time                           Show file versions as they were at the specified time (default off)
-      --s3-versions                                  Include old versions in directory listings
-      --seafile-2fa                                  Two-factor authentication ('true' if the account has 2FA enabled)
-      --seafile-create-library                       Should rclone create a library if it doesn't exist
-      --seafile-encoding MultiEncoder                The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
-      --seafile-library string                       Name of the library
-      --seafile-library-key string                   Library password (for encrypted libraries only) (obscured)
-      --seafile-pass string                          Password (obscured)
-      --seafile-url string                           URL of seafile host to connect to
-      --seafile-user string                          User name (usually email address)
-      --sftp-ask-password                            Allow asking for SFTP password when needed
-      --sftp-chunk-size SizeSuffix                   Upload and download chunk size (default 32Ki)
-      --sftp-concurrency int                         The maximum number of outstanding requests for one file (default 64)
-      --sftp-disable-concurrent-reads                If set don't use concurrent reads
-      --sftp-disable-concurrent-writes               If set don't use concurrent writes
-      --sftp-disable-hashcheck                       Disable the execution of SSH commands to determine if remote file hashing is available
-      --sftp-host string                             SSH host to connect to
-      --sftp-idle-timeout Duration                   Max time before closing idle connections (default 1m0s)
-      --sftp-key-file string                         Path to PEM-encoded private key file
-      --sftp-key-file-pass string                    The passphrase to decrypt the PEM-encoded private key file (obscured)
-      --sftp-key-pem string                          Raw PEM-encoded private key
-      --sftp-key-use-agent                           When set forces the usage of the ssh-agent
-      --sftp-known-hosts-file string                 Optional path to known_hosts file
-      --sftp-md5sum-command string                   The command used to read md5 hashes
-      --sftp-pass string                             SSH password, leave blank to use ssh-agent (obscured)
-      --sftp-path-override string                    Override path used by SSH shell commands
-      --sftp-port int                                SSH port number (default 22)
-      --sftp-pubkey-file string                      Optional path to public key file
-      --sftp-server-command string                   Specifies the path or command to run a sftp server on the remote host
-      --sftp-set-env SpaceSepList                    Environment variables to pass to sftp and commands
-      --sftp-set-modtime                             Set the modified time on the remote if set (default true)
-      --sftp-sha1sum-command string                  The command used to read sha1 hashes
-      --sftp-shell-type string                       The type of SSH shell on remote server, if any
-      --sftp-skip-links                              Set to skip any symlinks and any other non regular files
-      --sftp-subsystem string                        Specifies the SSH2 subsystem on the remote host (default "sftp")
-      --sftp-use-fstat                               If set use fstat instead of stat
-      --sftp-use-insecure-cipher                     Enable the use of insecure ciphers and key exchange methods
-      --sftp-user string                             SSH username (default "$USER")
-      --sharefile-chunk-size SizeSuffix              Upload chunk size (default 64Mi)
-      --sharefile-encoding MultiEncoder              The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,LeftSpace,LeftPeriod,RightSpace,RightPeriod,InvalidUtf8,Dot)
-      --sharefile-endpoint string                    Endpoint for API calls
-      --sharefile-root-folder-id string              ID of the root folder
-      --sharefile-upload-cutoff SizeSuffix           Cutoff for switching to multipart upload (default 128Mi)
-      --sia-api-password string                      Sia Daemon API Password (obscured)
-      --sia-api-url string                           Sia daemon API URL, like http://sia.daemon.host:9980 (default "http://127.0.0.1:9980")
-      --sia-encoding MultiEncoder                    The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot)
-      --sia-user-agent string                        Siad User Agent (default "Sia-Agent")
-      --skip-links                                   Don't warn about skipped symlinks
-      --smb-case-insensitive                         Whether the server is configured to be case-insensitive (default true)
-      --smb-domain string                            Domain name for NTLM authentication (default "WORKGROUP")
-      --smb-encoding MultiEncoder                    The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot)
-      --smb-hide-special-share                       Hide special shares (e.g. print$) which users aren't supposed to access (default true)
-      --smb-host string                              SMB server hostname to connect to
-      --smb-idle-timeout Duration                    Max time before closing idle connections (default 1m0s)
-      --smb-pass string                              SMB password (obscured)
-      --smb-port int                                 SMB port number (default 445)
-      --smb-user string                              SMB username (default "$USER")
-      --storj-access-grant string                    Access grant
-      --storj-api-key string                         API key
-      --storj-passphrase string                      Encryption passphrase
-      --storj-provider string                        Choose an authentication method (default "existing")
-      --storj-satellite-address string               Satellite address (default "us-central-1.storj.io")
-      --sugarsync-access-key-id string               Sugarsync Access Key ID
-      --sugarsync-app-id string                      Sugarsync App ID
-      --sugarsync-authorization string               Sugarsync authorization
-      --sugarsync-authorization-expiry string        Sugarsync authorization expiry
-      --sugarsync-deleted-id string                  Sugarsync deleted folder id
-      --sugarsync-encoding MultiEncoder              The encoding for the backend (default Slash,Ctl,InvalidUtf8,Dot)
-      --sugarsync-hard-delete                        Permanently delete files if true
-      --sugarsync-private-access-key string          Sugarsync Private Access Key
-      --sugarsync-refresh-token string               Sugarsync refresh token
-      --sugarsync-root-id string                     Sugarsync root id
-      --sugarsync-user string                        Sugarsync user
-      --swift-application-credential-id string       Application Credential ID (OS_APPLICATION_CREDENTIAL_ID)
-      --swift-application-credential-name string     Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME)
-      --swift-application-credential-secret string   Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET)
-      --swift-auth string                            Authentication URL for server (OS_AUTH_URL)
-      --swift-auth-token string                      Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)
-      --swift-auth-version int                       AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)
-      --swift-chunk-size SizeSuffix                  Above this size files will be chunked into a _segments container (default 5Gi)
-      --swift-domain string                          User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME)
-      --swift-encoding MultiEncoder                  The encoding for the backend (default Slash,InvalidUtf8)
-      --swift-endpoint-type string                   Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default "public")
-      --swift-env-auth                               Get swift credentials from environment variables in standard OpenStack form
-      --swift-key string                             API key or password (OS_PASSWORD)
-      --swift-leave-parts-on-error                   If true avoid calling abort upload on a failure
-      --swift-no-chunk                               Don't chunk files during streaming upload
-      --swift-no-large-objects                       Disable support for static and dynamic large objects
-      --swift-region string                          Region name - optional (OS_REGION_NAME)
-      --swift-storage-policy string                  The storage policy to use when creating a new container
-      --swift-storage-url string                     Storage URL - optional (OS_STORAGE_URL)
-      --swift-tenant string                          Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME)
-      --swift-tenant-domain string                   Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME)
-      --swift-tenant-id string                       Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID)
-      --swift-user string                            User name to log in (OS_USERNAME)
-      --swift-user-id string                         User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID)
-      --union-action-policy string                   Policy to choose upstream on ACTION category (default "epall")
-      --union-cache-time int                         Cache time of usage and free space (in seconds) (default 120)
-      --union-create-policy string                   Policy to choose upstream on CREATE category (default "epmfs")
-      --union-min-free-space SizeSuffix              Minimum viable free space for lfs/eplfs policies (default 1Gi)
-      --union-search-policy string                   Policy to choose upstream on SEARCH category (default "ff")
-      --union-upstreams string                       List of space separated upstreams
-      --uptobox-access-token string                  Your access token
-      --uptobox-encoding MultiEncoder                The encoding for the backend (default Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot)
-      --webdav-bearer-token string                   Bearer token instead of user/pass (e.g. a Macaroon)
-      --webdav-bearer-token-command string           Command to run to get a bearer token
-      --webdav-encoding string                       The encoding for the backend
-      --webdav-headers CommaSepList                  Set HTTP headers for all transactions
-      --webdav-pass string                           Password (obscured)
-      --webdav-url string                            URL of http host to connect to
-      --webdav-user string                           User name
-      --webdav-vendor string                         Name of the WebDAV site/service/software you are using
-      --yandex-auth-url string                       Auth server URL
-      --yandex-client-id string                      OAuth Client Id
-      --yandex-client-secret string                  OAuth Client Secret
-      --yandex-encoding MultiEncoder                 The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot)
-      --yandex-hard-delete                           Delete files permanently rather than putting them into the trash
-      --yandex-token string                          OAuth Access Token as a JSON blob
-      --yandex-token-url string                      Token server url
-      --zoho-auth-url string                         Auth server URL
-      --zoho-client-id string                        OAuth Client Id
-      --zoho-client-secret string                    OAuth Client Secret
-      --zoho-encoding MultiEncoder                   The encoding for the backend (default Del,Ctl,InvalidUtf8)
-      --zoho-region string                           Zoho region to connect to
-      --zoho-token string                            OAuth Access Token as a JSON blob
-      --zoho-token-url string                        Token server url
              +
                    --acd-auth-url string                            Auth server URL
+      --acd-client-id string                           OAuth Client Id
+      --acd-client-secret string                       OAuth Client Secret
+      --acd-encoding MultiEncoder                      The encoding for the backend (default Slash,InvalidUtf8,Dot)
+      --acd-templink-threshold SizeSuffix              Files >= this size will be downloaded via their tempLink (default 9Gi)
+      --acd-token string                               OAuth Access Token as a JSON blob
+      --acd-token-url string                           Token server url
+      --acd-upload-wait-per-gb Duration                Additional time per GiB to wait after a failed complete upload to see if it appears (default 3m0s)
+      --alias-remote string                            Remote or path to alias
+      --azureblob-access-tier string                   Access tier of blob: hot, cool or archive
+      --azureblob-account string                       Azure Storage Account Name
+      --azureblob-archive-tier-delete                  Delete archive tier blobs before overwriting
+      --azureblob-chunk-size SizeSuffix                Upload chunk size (default 4Mi)
+      --azureblob-client-certificate-password string   Password for the certificate file (optional) (obscured)
+      --azureblob-client-certificate-path string       Path to a PEM or PKCS12 certificate file including the private key
+      --azureblob-client-id string                     The ID of the client in use
+      --azureblob-client-secret string                 One of the service principal's client secrets
+      --azureblob-client-send-certificate-chain        Send the certificate chain when using certificate auth
+      --azureblob-disable-checksum                     Don't store MD5 checksum with object metadata
+      --azureblob-encoding MultiEncoder                The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8)
+      --azureblob-endpoint string                      Endpoint for the service
+      --azureblob-env-auth                             Read credentials from runtime (environment variables, CLI or MSI)
+      --azureblob-key string                           Storage Account Shared Key
+      --azureblob-list-chunk int                       Size of blob list (default 5000)
+      --azureblob-memory-pool-flush-time Duration      How often internal memory buffer pools will be flushed (default 1m0s)
+      --azureblob-memory-pool-use-mmap                 Whether to use mmap buffers in internal memory pool
+      --azureblob-msi-client-id string                 Object ID of the user-assigned MSI to use, if any
+      --azureblob-msi-mi-res-id string                 Azure resource ID of the user-assigned MSI to use, if any
+      --azureblob-msi-object-id string                 Object ID of the user-assigned MSI to use, if any
+      --azureblob-no-check-container                   If set, don't attempt to check the container exists or create it
+      --azureblob-no-head-object                       If set, do not do HEAD before GET when getting objects
+      --azureblob-password string                      The user's password (obscured)
+      --azureblob-public-access string                 Public access level of a container: blob or container
+      --azureblob-sas-url string                       SAS URL for container level access only
+      --azureblob-service-principal-file string        Path to file containing credentials for use with a service principal
+      --azureblob-tenant string                        ID of the service principal's tenant. Also called its directory ID
+      --azureblob-upload-concurrency int               Concurrency for multipart uploads (default 16)
+      --azureblob-upload-cutoff string                 Cutoff for switching to chunked upload (<= 256 MiB) (deprecated)
+      --azureblob-use-emulator                         Uses local storage emulator if provided as 'true'
+      --azureblob-use-msi                              Use a managed service identity to authenticate (only works in Azure)
+      --azureblob-username string                      User name (usually an email address)
+      --b2-account string                              Account ID or Application Key ID
+      --b2-chunk-size SizeSuffix                       Upload chunk size (default 96Mi)
+      --b2-copy-cutoff SizeSuffix                      Cutoff for switching to multipart copy (default 4Gi)
+      --b2-disable-checksum                            Disable checksums for large (> upload cutoff) files
+      --b2-download-auth-duration Duration             Time before the authorization token will expire in s or suffix ms|s|m|h|d (default 1w)
+      --b2-download-url string                         Custom endpoint for downloads
+      --b2-encoding MultiEncoder                       The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
+      --b2-endpoint string                             Endpoint for the service
+      --b2-hard-delete                                 Permanently delete files on remote removal, otherwise hide files
+      --b2-key string                                  Application Key
+      --b2-memory-pool-flush-time Duration             How often internal memory buffer pools will be flushed (default 1m0s)
+      --b2-memory-pool-use-mmap                        Whether to use mmap buffers in internal memory pool
+      --b2-test-mode string                            A flag string for X-Bz-Test-Mode header for debugging
+      --b2-upload-cutoff SizeSuffix                    Cutoff for switching to chunked upload (default 200Mi)
+      --b2-version-at Time                             Show file versions as they were at the specified time (default off)
+      --b2-versions                                    Include old versions in directory listings
+      --box-access-token string                        Box App Primary Access Token
+      --box-auth-url string                            Auth server URL
+      --box-box-config-file string                     Box App config.json location
+      --box-box-sub-type string                         (default "user")
+      --box-client-id string                           OAuth Client Id
+      --box-client-secret string                       OAuth Client Secret
+      --box-commit-retries int                         Max number of times to try committing a multipart file (default 100)
+      --box-encoding MultiEncoder                      The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot)
+      --box-list-chunk int                             Size of listing chunk 1-1000 (default 1000)
+      --box-owned-by string                            Only show items owned by the login (email address) passed in
+      --box-root-folder-id string                      Fill in for rclone to use a non root folder as its starting point
+      --box-token string                               OAuth Access Token as a JSON blob
+      --box-token-url string                           Token server url
+      --box-upload-cutoff SizeSuffix                   Cutoff for switching to multipart upload (>= 50 MiB) (default 50Mi)
+      --cache-chunk-clean-interval Duration            How often should the cache perform cleanups of the chunk storage (default 1m0s)
+      --cache-chunk-no-memory                          Disable the in-memory cache for storing chunks during streaming
+      --cache-chunk-path string                        Directory to cache chunk files (default "$HOME/.cache/rclone/cache-backend")
+      --cache-chunk-size SizeSuffix                    The size of a chunk (partial file data) (default 5Mi)
+      --cache-chunk-total-size SizeSuffix              The total size that the chunks can take up on the local disk (default 10Gi)
+      --cache-db-path string                           Directory to store file structure metadata DB (default "$HOME/.cache/rclone/cache-backend")
+      --cache-db-purge                                 Clear all the cached data for this remote on start
+      --cache-db-wait-time Duration                    How long to wait for the DB to be available - 0 is unlimited (default 1s)
+      --cache-info-age Duration                        How long to cache file structure information (directory listings, file size, times, etc.) (default 6h0m0s)
+      --cache-plex-insecure string                     Skip all certificate verification when connecting to the Plex server
+      --cache-plex-password string                     The password of the Plex user (obscured)
+      --cache-plex-url string                          The URL of the Plex server
+      --cache-plex-username string                     The username of the Plex user
+      --cache-read-retries int                         How many times to retry a read from a cache storage (default 10)
+      --cache-remote string                            Remote to cache
+      --cache-rps int                                  Limits the number of requests per second to the source FS (-1 to disable) (default -1)
+      --cache-tmp-upload-path string                   Directory to keep temporary files until they are uploaded
+      --cache-tmp-wait-time Duration                   How long should files be stored in local cache before being uploaded (default 15s)
+      --cache-workers int                              How many workers should run in parallel to download chunks (default 4)
+      --cache-writes                                   Cache file data on writes through the FS
+      --chunker-chunk-size SizeSuffix                  Files larger than chunk size will be split in chunks (default 2Gi)
+      --chunker-fail-hard                              Choose how chunker should handle files with missing or invalid chunks
+      --chunker-hash-type string                       Choose how chunker handles hash sums (default "md5")
+      --chunker-remote string                          Remote to chunk/unchunk
+      --combine-upstreams SpaceSepList                 Upstreams for combining
+      --compress-level int                             GZIP compression level (-2 to 9) (default -1)
+      --compress-mode string                           Compression mode (default "gzip")
+      --compress-ram-cache-limit SizeSuffix            Some remotes don't allow the upload of files with unknown size (default 20Mi)
+      --compress-remote string                         Remote to compress
+  -L, --copy-links                                     Follow symlinks and copy the pointed to item
+      --crypt-directory-name-encryption                Option to either encrypt directory names or leave them intact (default true)
+      --crypt-filename-encoding string                 How to encode the encrypted filename to text string (default "base32")
+      --crypt-filename-encryption string               How to encrypt the filenames (default "standard")
+      --crypt-no-data-encryption                       Option to either encrypt file data or leave it unencrypted
+      --crypt-password string                          Password or pass phrase for encryption (obscured)
+      --crypt-password2 string                         Password or pass phrase for salt (obscured)
+      --crypt-remote string                            Remote to encrypt/decrypt
+      --crypt-server-side-across-configs               Allow server-side operations (e.g. copy) to work across different crypt configs
+      --crypt-show-mapping                             For all files listed show how the names encrypt
+      --drive-acknowledge-abuse                        Set to allow files which return cannotDownloadAbusiveFile to be downloaded
+      --drive-allow-import-name-change                 Allow the filetype to change when uploading Google docs
+      --drive-auth-owner-only                          Only consider files owned by the authenticated user
+      --drive-auth-url string                          Auth server URL
+      --drive-chunk-size SizeSuffix                    Upload chunk size (default 8Mi)
+      --drive-client-id string                         Google Application Client Id
+      --drive-client-secret string                     OAuth Client Secret
+      --drive-copy-shortcut-content                    Server side copy contents of shortcuts instead of the shortcut
+      --drive-disable-http2                            Disable drive using http2 (default true)
+      --drive-encoding MultiEncoder                    The encoding for the backend (default InvalidUtf8)
+      --drive-export-formats string                    Comma separated list of preferred formats for downloading Google docs (default "docx,xlsx,pptx,svg")
+      --drive-formats string                           Deprecated: See export_formats
+      --drive-impersonate string                       Impersonate this user when using a service account
+      --drive-import-formats string                    Comma separated list of preferred formats for uploading Google docs
+      --drive-keep-revision-forever                    Keep new head revision of each file forever
+      --drive-list-chunk int                           Size of listing chunk 100-1000, 0 to disable (default 1000)
+      --drive-pacer-burst int                          Number of API calls to allow without sleeping (default 100)
+      --drive-pacer-min-sleep Duration                 Minimum time to sleep between API calls (default 100ms)
+      --drive-resource-key string                      Resource key for accessing a link-shared file
+      --drive-root-folder-id string                    ID of the root folder
+      --drive-scope string                             Scope that rclone should use when requesting access from drive
+      --drive-server-side-across-configs               Allow server-side operations (e.g. copy) to work across different drive configs
+      --drive-service-account-credentials string       Service Account Credentials JSON blob
+      --drive-service-account-file string              Service Account Credentials JSON file path
+      --drive-shared-with-me                           Only show files that are shared with me
+      --drive-size-as-quota                            Show sizes as storage quota usage, not actual size
+      --drive-skip-checksum-gphotos                    Skip MD5 checksum on Google photos and videos only
+      --drive-skip-dangling-shortcuts                  If set skip dangling shortcut files
+      --drive-skip-gdocs                               Skip google documents in all listings
+      --drive-skip-shortcuts                           If set skip shortcut files
+      --drive-starred-only                             Only show files that are starred
+      --drive-stop-on-download-limit                   Make download limit errors be fatal
+      --drive-stop-on-upload-limit                     Make upload limit errors be fatal
+      --drive-team-drive string                        ID of the Shared Drive (Team Drive)
+      --drive-token string                             OAuth Access Token as a JSON blob
+      --drive-token-url string                         Token server url
+      --drive-trashed-only                             Only show files that are in the trash
+      --drive-upload-cutoff SizeSuffix                 Cutoff for switching to chunked upload (default 8Mi)
+      --drive-use-created-date                         Use file created date instead of modified date
+      --drive-use-shared-date                          Use date file was shared instead of modified date
+      --drive-use-trash                                Send files to the trash instead of deleting permanently (default true)
+      --drive-v2-download-min-size SizeSuffix          If Object's are greater, use drive v2 API to download (default off)
+      --dropbox-auth-url string                        Auth server URL
+      --dropbox-batch-commit-timeout Duration          Max time to wait for a batch to finish committing (default 10m0s)
+      --dropbox-batch-mode string                      Upload file batching sync|async|off (default "sync")
+      --dropbox-batch-size int                         Max number of files in upload batch
+      --dropbox-batch-timeout Duration                 Max time to allow an idle upload batch before uploading (default 0s)
+      --dropbox-chunk-size SizeSuffix                  Upload chunk size (< 150Mi) (default 48Mi)
+      --dropbox-client-id string                       OAuth Client Id
+      --dropbox-client-secret string                   OAuth Client Secret
+      --dropbox-encoding MultiEncoder                  The encoding for the backend (default Slash,BackSlash,Del,RightSpace,InvalidUtf8,Dot)
+      --dropbox-impersonate string                     Impersonate this user when using a business account
+      --dropbox-shared-files                           Instructs rclone to work on individual shared files
+      --dropbox-shared-folders                         Instructs rclone to work on shared folders
+      --dropbox-token string                           OAuth Access Token as a JSON blob
+      --dropbox-token-url string                       Token server url
+      --fichier-api-key string                         Your API Key, get it from https://1fichier.com/console/params.pl
+      --fichier-encoding MultiEncoder                  The encoding for the backend (default Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot)
+      --fichier-file-password string                   If you want to download a shared file that is password protected, add this parameter (obscured)
+      --fichier-folder-password string                 If you want to list the files in a shared folder that is password protected, add this parameter (obscured)
+      --fichier-shared-folder string                   If you want to download a shared folder, add this parameter
+      --filefabric-encoding MultiEncoder               The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot)
+      --filefabric-permanent-token string              Permanent Authentication Token
+      --filefabric-root-folder-id string               ID of the root folder
+      --filefabric-token string                        Session Token
+      --filefabric-token-expiry string                 Token expiry time
+      --filefabric-url string                          URL of the Enterprise File Fabric to connect to
+      --filefabric-version string                      Version read from the file fabric
+      --ftp-ask-password                               Allow asking for FTP password when needed
+      --ftp-close-timeout Duration                     Maximum time to wait for a response to close (default 1m0s)
+      --ftp-concurrency int                            Maximum number of FTP simultaneous connections, 0 for unlimited
+      --ftp-disable-epsv                               Disable using EPSV even if server advertises support
+      --ftp-disable-mlsd                               Disable using MLSD even if server advertises support
+      --ftp-disable-tls13                              Disable TLS 1.3 (workaround for FTP servers with buggy TLS)
+      --ftp-disable-utf8                               Disable using UTF-8 even if server advertises support
+      --ftp-encoding MultiEncoder                      The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
+      --ftp-explicit-tls                               Use Explicit FTPS (FTP over TLS)
+      --ftp-force-list-hidden                          Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD
+      --ftp-host string                                FTP host to connect to
+      --ftp-idle-timeout Duration                      Max time before closing idle connections (default 1m0s)
+      --ftp-no-check-certificate                       Do not verify the TLS certificate of the server
+      --ftp-pass string                                FTP password (obscured)
+      --ftp-port int                                   FTP port number (default 21)
+      --ftp-shut-timeout Duration                      Maximum time to wait for data connection closing status (default 1m0s)
+      --ftp-tls                                        Use Implicit FTPS (FTP over TLS)
+      --ftp-tls-cache-size int                         Size of TLS session cache for all control and data connections (default 32)
+      --ftp-user string                                FTP username (default "$USER")
+      --ftp-writing-mdtm                               Use MDTM to set modification time (VsFtpd quirk)
+      --gcs-anonymous                                  Access public buckets and objects without credentials
+      --gcs-auth-url string                            Auth server URL
+      --gcs-bucket-acl string                          Access Control List for new buckets
+      --gcs-bucket-policy-only                         Access checks should use bucket-level IAM policies
+      --gcs-client-id string                           OAuth Client Id
+      --gcs-client-secret string                       OAuth Client Secret
+      --gcs-decompress                                 If set this will decompress gzip encoded objects
+      --gcs-encoding MultiEncoder                      The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot)
+      --gcs-endpoint string                            Endpoint for the service
+      --gcs-location string                            Location for the newly created buckets
+      --gcs-no-check-bucket                            If set, don't attempt to check the bucket exists or create it
+      --gcs-object-acl string                          Access Control List for new objects
+      --gcs-project-number string                      Project number
+      --gcs-service-account-file string                Service Account Credentials JSON file path
+      --gcs-storage-class string                       The storage class to use when storing objects in Google Cloud Storage
+      --gcs-token string                               OAuth Access Token as a JSON blob
+      --gcs-token-url string                           Token server url
+      --gphotos-auth-url string                        Auth server URL
+      --gphotos-client-id string                       OAuth Client Id
+      --gphotos-client-secret string                   OAuth Client Secret
+      --gphotos-encoding MultiEncoder                  The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot)
+      --gphotos-include-archived                       Also view and download archived media
+      --gphotos-read-only                              Set to make the Google Photos backend read only
+      --gphotos-read-size                              Set to read the size of media items
+      --gphotos-start-year int                         Year limits the photos to be downloaded to those which are uploaded after the given year (default 2000)
+      --gphotos-token string                           OAuth Access Token as a JSON blob
+      --gphotos-token-url string                       Token server url
+      --hasher-auto-size SizeSuffix                    Auto-update checksum for files smaller than this size (disabled by default)
+      --hasher-hashes CommaSepList                     Comma separated list of supported checksum types (default md5,sha1)
+      --hasher-max-age Duration                        Maximum time to keep checksums in cache (0 = no cache, off = cache forever) (default off)
+      --hasher-remote string                           Remote to cache checksums for (e.g. myRemote:path)
+      --hdfs-data-transfer-protection string           Kerberos data transfer protection: authentication|integrity|privacy
+      --hdfs-encoding MultiEncoder                     The encoding for the backend (default Slash,Colon,Del,Ctl,InvalidUtf8,Dot)
+      --hdfs-namenode string                           Hadoop name node and port
+      --hdfs-service-principal-name string             Kerberos service principal name for the namenode
+      --hdfs-username string                           Hadoop user name
+      --hidrive-auth-url string                        Auth server URL
+      --hidrive-chunk-size SizeSuffix                  Chunksize for chunked uploads (default 48Mi)
+      --hidrive-client-id string                       OAuth Client Id
+      --hidrive-client-secret string                   OAuth Client Secret
+      --hidrive-disable-fetching-member-count          Do not fetch number of objects in directories unless it is absolutely necessary
+      --hidrive-encoding MultiEncoder                  The encoding for the backend (default Slash,Dot)
+      --hidrive-endpoint string                        Endpoint for the service (default "https://api.hidrive.strato.com/2.1")
+      --hidrive-root-prefix string                     The root/parent folder for all paths (default "/")
+      --hidrive-scope-access string                    Access permissions that rclone should use when requesting access from HiDrive (default "rw")
+      --hidrive-scope-role string                      User-level that rclone should use when requesting access from HiDrive (default "user")
+      --hidrive-token string                           OAuth Access Token as a JSON blob
+      --hidrive-token-url string                       Token server url
+      --hidrive-upload-concurrency int                 Concurrency for chunked uploads (default 4)
+      --hidrive-upload-cutoff SizeSuffix               Cutoff/Threshold for chunked uploads (default 96Mi)
+      --http-headers CommaSepList                      Set HTTP headers for all transactions
+      --http-no-head                                   Don't use HEAD requests
+      --http-no-slash                                  Set this if the site doesn't end directories with /
+      --http-url string                                URL of HTTP host to connect to
+      --internetarchive-access-key-id string           IAS3 Access Key
+      --internetarchive-disable-checksum               Don't ask the server to test against MD5 checksum calculated by rclone (default true)
+      --internetarchive-encoding MultiEncoder          The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot)
+      --internetarchive-endpoint string                IAS3 Endpoint (default "https://s3.us.archive.org")
+      --internetarchive-front-endpoint string          Host of InternetArchive Frontend (default "https://archive.org")
+      --internetarchive-secret-access-key string       IAS3 Secret Key (password)
+      --internetarchive-wait-archive Duration          Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s)
+      --jottacloud-encoding MultiEncoder               The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot)
+      --jottacloud-hard-delete                         Delete files permanently rather than putting them into the trash
+      --jottacloud-md5-memory-limit SizeSuffix         Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi)
+      --jottacloud-no-versions                         Avoid server side versioning by deleting files and recreating files instead of overwriting them
+      --jottacloud-trashed-only                        Only show files that are in the trash
+      --jottacloud-upload-resume-limit SizeSuffix      Files bigger than this can be resumed if the upload fail's (default 10Mi)
+      --koofr-encoding MultiEncoder                    The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
+      --koofr-endpoint string                          The Koofr API endpoint to use
+      --koofr-mountid string                           Mount ID of the mount to use
+      --koofr-password string                          Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured)
+      --koofr-provider string                          Choose your storage provider
+      --koofr-setmtime                                 Does the backend support setting modification time (default true)
+      --koofr-user string                              Your user name
+  -l, --links                                          Translate symlinks to/from regular files with a '.rclonelink' extension
+      --local-case-insensitive                         Force the filesystem to report itself as case insensitive
+      --local-case-sensitive                           Force the filesystem to report itself as case sensitive
+      --local-encoding MultiEncoder                    The encoding for the backend (default Slash,Dot)
+      --local-no-check-updated                         Don't check to see if the files change during upload
+      --local-no-preallocate                           Disable preallocation of disk space for transferred files
+      --local-no-set-modtime                           Disable setting modtime
+      --local-no-sparse                                Disable sparse files for multi-thread downloads
+      --local-nounc                                    Disable UNC (long path names) conversion on Windows
+      --local-unicode-normalization                    Apply unicode NFC normalization to paths and filenames
+      --local-zero-size-links                          Assume the Stat size of links is zero (and read them instead) (deprecated)
+      --mailru-check-hash                              What should copy do if file checksum is mismatched or invalid (default true)
+      --mailru-encoding MultiEncoder                   The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot)
+      --mailru-pass string                             Password (obscured)
+      --mailru-speedup-enable                          Skip full upload if there is another file with same data hash (default true)
+      --mailru-speedup-file-patterns string            Comma separated list of file name patterns eligible for speedup (put by hash) (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf")
+      --mailru-speedup-max-disk SizeSuffix             This option allows you to disable speedup (put by hash) for large files (default 3Gi)
+      --mailru-speedup-max-memory SizeSuffix           Files larger than the size given below will always be hashed on disk (default 32Mi)
+      --mailru-user string                             User name (usually email)
+      --mega-debug                                     Output more debug from Mega
+      --mega-encoding MultiEncoder                     The encoding for the backend (default Slash,InvalidUtf8,Dot)
+      --mega-hard-delete                               Delete files permanently rather than putting them into the trash
+      --mega-pass string                               Password (obscured)
+      --mega-user string                               User name
+      --netstorage-account string                      Set the NetStorage account name
+      --netstorage-host string                         Domain+path of NetStorage host to connect to
+      --netstorage-protocol string                     Select between HTTP or HTTPS protocol (default "https")
+      --netstorage-secret string                       Set the NetStorage account secret/G2O key for authentication (obscured)
+  -x, --one-file-system                                Don't cross filesystem boundaries (unix/macOS only)
+      --onedrive-access-scopes SpaceSepList            Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access)
+      --onedrive-auth-url string                       Auth server URL
+      --onedrive-chunk-size SizeSuffix                 Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi)
+      --onedrive-client-id string                      OAuth Client Id
+      --onedrive-client-secret string                  OAuth Client Secret
+      --onedrive-drive-id string                       The ID of the drive to use
+      --onedrive-drive-type string                     The type of the drive (personal | business | documentLibrary)
+      --onedrive-encoding MultiEncoder                 The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot)
+      --onedrive-expose-onenote-files                  Set to make OneNote files show up in directory listings
+      --onedrive-link-password string                  Set the password for links created by the link command
+      --onedrive-link-scope string                     Set the scope of the links created by the link command (default "anonymous")
+      --onedrive-link-type string                      Set the type of the links created by the link command (default "view")
+      --onedrive-list-chunk int                        Size of listing chunk (default 1000)
+      --onedrive-no-versions                           Remove all versions on modifying operations
+      --onedrive-region string                         Choose national cloud region for OneDrive (default "global")
+      --onedrive-root-folder-id string                 ID of the root folder
+      --onedrive-server-side-across-configs            Allow server-side operations (e.g. copy) to work across different onedrive configs
+      --onedrive-token string                          OAuth Access Token as a JSON blob
+      --onedrive-token-url string                      Token server url
+      --oos-chunk-size SizeSuffix                      Chunk size to use for uploading (default 5Mi)
+      --oos-compartment string                         Object storage compartment OCID
+      --oos-config-file string                         Path to OCI config file (default "~/.oci/config")
+      --oos-config-profile string                      Profile name inside the oci config file (default "Default")
+      --oos-copy-cutoff SizeSuffix                     Cutoff for switching to multipart copy (default 4.656Gi)
+      --oos-copy-timeout Duration                      Timeout for copy (default 1m0s)
+      --oos-disable-checksum                           Don't store MD5 checksum with object metadata
+      --oos-encoding MultiEncoder                      The encoding for the backend (default Slash,InvalidUtf8,Dot)
+      --oos-endpoint string                            Endpoint for Object storage API
+      --oos-leave-parts-on-error                       If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery
+      --oos-namespace string                           Object storage namespace
+      --oos-no-check-bucket                            If set, don't attempt to check the bucket exists or create it
+      --oos-provider string                            Choose your Auth Provider (default "env_auth")
+      --oos-region string                              Object storage Region
+      --oos-upload-concurrency int                     Concurrency for multipart uploads (default 10)
+      --oos-upload-cutoff SizeSuffix                   Cutoff for switching to chunked upload (default 200Mi)
+      --opendrive-chunk-size SizeSuffix                Files will be uploaded in chunks this size (default 10Mi)
+      --opendrive-encoding MultiEncoder                The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot)
+      --opendrive-password string                      Password (obscured)
+      --opendrive-username string                      Username
+      --pcloud-auth-url string                         Auth server URL
+      --pcloud-client-id string                        OAuth Client Id
+      --pcloud-client-secret string                    OAuth Client Secret
+      --pcloud-encoding MultiEncoder                   The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
+      --pcloud-hostname string                         Hostname to connect to (default "api.pcloud.com")
+      --pcloud-password string                         Your pcloud password (obscured)
+      --pcloud-root-folder-id string                   Fill in for rclone to use a non root folder as its starting point (default "d0")
+      --pcloud-token string                            OAuth Access Token as a JSON blob
+      --pcloud-token-url string                        Token server url
+      --pcloud-username string                         Your pcloud username
+      --premiumizeme-encoding MultiEncoder             The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot)
+      --putio-encoding MultiEncoder                    The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
+      --qingstor-access-key-id string                  QingStor Access Key ID
+      --qingstor-chunk-size SizeSuffix                 Chunk size to use for uploading (default 4Mi)
+      --qingstor-connection-retries int                Number of connection retries (default 3)
+      --qingstor-encoding MultiEncoder                 The encoding for the backend (default Slash,Ctl,InvalidUtf8)
+      --qingstor-endpoint string                       Enter an endpoint URL to connection QingStor API
+      --qingstor-env-auth                              Get QingStor credentials from runtime
+      --qingstor-secret-access-key string              QingStor Secret Access Key (password)
+      --qingstor-upload-concurrency int                Concurrency for multipart uploads (default 1)
+      --qingstor-upload-cutoff SizeSuffix              Cutoff for switching to chunked upload (default 200Mi)
+      --qingstor-zone string                           Zone to connect to
+      --s3-access-key-id string                        AWS Access Key ID
+      --s3-acl string                                  Canned ACL used when creating buckets and storing or copying objects
+      --s3-bucket-acl string                           Canned ACL used when creating buckets
+      --s3-chunk-size SizeSuffix                       Chunk size to use for uploading (default 5Mi)
+      --s3-copy-cutoff SizeSuffix                      Cutoff for switching to multipart copy (default 4.656Gi)
+      --s3-decompress                                  If set this will decompress gzip encoded objects
+      --s3-disable-checksum                            Don't store MD5 checksum with object metadata
+      --s3-disable-http2                               Disable usage of http2 for S3 backends
+      --s3-download-url string                         Custom endpoint for downloads
+      --s3-encoding MultiEncoder                       The encoding for the backend (default Slash,InvalidUtf8,Dot)
+      --s3-endpoint string                             Endpoint for S3 API
+      --s3-env-auth                                    Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars)
+      --s3-force-path-style                            If true use path style access if false use virtual hosted style (default true)
+      --s3-leave-parts-on-error                        If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery
+      --s3-list-chunk int                              Size of listing chunk (response list for each ListObject S3 request) (default 1000)
+      --s3-list-url-encode Tristate                    Whether to url encode listings: true/false/unset (default unset)
+      --s3-list-version int                            Version of ListObjects to use: 1,2 or 0 for auto
+      --s3-location-constraint string                  Location constraint - must be set to match the Region
+      --s3-max-upload-parts int                        Maximum number of parts in a multipart upload (default 10000)
+      --s3-memory-pool-flush-time Duration             How often internal memory buffer pools will be flushed (default 1m0s)
+      --s3-memory-pool-use-mmap                        Whether to use mmap buffers in internal memory pool
+      --s3-might-gzip Tristate                         Set this if the backend might gzip objects (default unset)
+      --s3-no-check-bucket                             If set, don't attempt to check the bucket exists or create it
+      --s3-no-head                                     If set, don't HEAD uploaded objects to check integrity
+      --s3-no-head-object                              If set, do not do HEAD before GET when getting objects
+      --s3-no-system-metadata                          Suppress setting and reading of system metadata
+      --s3-profile string                              Profile to use in the shared credentials file
+      --s3-provider string                             Choose your S3 provider
+      --s3-region string                               Region to connect to
+      --s3-requester-pays                              Enables requester pays option when interacting with S3 bucket
+      --s3-secret-access-key string                    AWS Secret Access Key (password)
+      --s3-server-side-encryption string               The server-side encryption algorithm used when storing this object in S3
+      --s3-session-token string                        An AWS session token
+      --s3-shared-credentials-file string              Path to the shared credentials file
+      --s3-sse-customer-algorithm string               If using SSE-C, the server-side encryption algorithm used when storing this object in S3
+      --s3-sse-customer-key string                     To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data
+      --s3-sse-customer-key-base64 string              If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data
+      --s3-sse-customer-key-md5 string                 If using SSE-C you may provide the secret encryption key MD5 checksum (optional)
+      --s3-sse-kms-key-id string                       If using KMS ID you must provide the ARN of Key
+      --s3-storage-class string                        The storage class to use when storing new objects in S3
+      --s3-upload-concurrency int                      Concurrency for multipart uploads (default 4)
+      --s3-upload-cutoff SizeSuffix                    Cutoff for switching to chunked upload (default 200Mi)
+      --s3-use-accelerate-endpoint                     If true use the AWS S3 accelerated endpoint
+      --s3-use-multipart-etag Tristate                 Whether to use ETag in multipart uploads for verification (default unset)
+      --s3-use-presigned-request                       Whether to use a presigned request or PutObject for single part uploads
+      --s3-v2-auth                                     If true use v2 authentication
+      --s3-version-at Time                             Show file versions as they were at the specified time (default off)
+      --s3-versions                                    Include old versions in directory listings
+      --seafile-2fa                                    Two-factor authentication ('true' if the account has 2FA enabled)
+      --seafile-create-library                         Should rclone create a library if it doesn't exist
+      --seafile-encoding MultiEncoder                  The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
+      --seafile-library string                         Name of the library
+      --seafile-library-key string                     Library password (for encrypted libraries only) (obscured)
+      --seafile-pass string                            Password (obscured)
+      --seafile-url string                             URL of seafile host to connect to
+      --seafile-user string                            User name (usually email address)
+      --sftp-ask-password                              Allow asking for SFTP password when needed
+      --sftp-chunk-size SizeSuffix                     Upload and download chunk size (default 32Ki)
+      --sftp-ciphers SpaceSepList                      Space separated list of ciphers to be used for session encryption, ordered by preference
+      --sftp-concurrency int                           The maximum number of outstanding requests for one file (default 64)
+      --sftp-disable-concurrent-reads                  If set don't use concurrent reads
+      --sftp-disable-concurrent-writes                 If set don't use concurrent writes
+      --sftp-disable-hashcheck                         Disable the execution of SSH commands to determine if remote file hashing is available
+      --sftp-host string                               SSH host to connect to
+      --sftp-idle-timeout Duration                     Max time before closing idle connections (default 1m0s)
+      --sftp-key-exchange SpaceSepList                 Space separated list of key exchange algorithms, ordered by preference
+      --sftp-key-file string                           Path to PEM-encoded private key file
+      --sftp-key-file-pass string                      The passphrase to decrypt the PEM-encoded private key file (obscured)
+      --sftp-key-pem string                            Raw PEM-encoded private key
+      --sftp-key-use-agent                             When set forces the usage of the ssh-agent
+      --sftp-known-hosts-file string                   Optional path to known_hosts file
+      --sftp-macs SpaceSepList                         Space separated list of MACs (message authentication code) algorithms, ordered by preference
+      --sftp-md5sum-command string                     The command used to read md5 hashes
+      --sftp-pass string                               SSH password, leave blank to use ssh-agent (obscured)
+      --sftp-path-override string                      Override path used by SSH shell commands
+      --sftp-port int                                  SSH port number (default 22)
+      --sftp-pubkey-file string                        Optional path to public key file
+      --sftp-server-command string                     Specifies the path or command to run a sftp server on the remote host
+      --sftp-set-env SpaceSepList                      Environment variables to pass to sftp and commands
+      --sftp-set-modtime                               Set the modified time on the remote if set (default true)
+      --sftp-sha1sum-command string                    The command used to read sha1 hashes
+      --sftp-shell-type string                         The type of SSH shell on remote server, if any
+      --sftp-skip-links                                Set to skip any symlinks and any other non regular files
+      --sftp-subsystem string                          Specifies the SSH2 subsystem on the remote host (default "sftp")
+      --sftp-use-fstat                                 If set use fstat instead of stat
+      --sftp-use-insecure-cipher                       Enable the use of insecure ciphers and key exchange methods
+      --sftp-user string                               SSH username (default "$USER")
+      --sharefile-chunk-size SizeSuffix                Upload chunk size (default 64Mi)
+      --sharefile-encoding MultiEncoder                The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,LeftSpace,LeftPeriod,RightSpace,RightPeriod,InvalidUtf8,Dot)
+      --sharefile-endpoint string                      Endpoint for API calls
+      --sharefile-root-folder-id string                ID of the root folder
+      --sharefile-upload-cutoff SizeSuffix             Cutoff for switching to multipart upload (default 128Mi)
+      --sia-api-password string                        Sia Daemon API Password (obscured)
+      --sia-api-url string                             Sia daemon API URL, like http://sia.daemon.host:9980 (default "http://127.0.0.1:9980")
+      --sia-encoding MultiEncoder                      The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot)
+      --sia-user-agent string                          Siad User Agent (default "Sia-Agent")
+      --skip-links                                     Don't warn about skipped symlinks
+      --smb-case-insensitive                           Whether the server is configured to be case-insensitive (default true)
+      --smb-domain string                              Domain name for NTLM authentication (default "WORKGROUP")
+      --smb-encoding MultiEncoder                      The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot)
+      --smb-hide-special-share                         Hide special shares (e.g. print$) which users aren't supposed to access (default true)
+      --smb-host string                                SMB server hostname to connect to
+      --smb-idle-timeout Duration                      Max time before closing idle connections (default 1m0s)
+      --smb-pass string                                SMB password (obscured)
+      --smb-port int                                   SMB port number (default 445)
+      --smb-user string                                SMB username (default "$USER")
+      --storj-access-grant string                      Access grant
+      --storj-api-key string                           API key
+      --storj-passphrase string                        Encryption passphrase
+      --storj-provider string                          Choose an authentication method (default "existing")
+      --storj-satellite-address string                 Satellite address (default "us-central-1.storj.io")
+      --sugarsync-access-key-id string                 Sugarsync Access Key ID
+      --sugarsync-app-id string                        Sugarsync App ID
+      --sugarsync-authorization string                 Sugarsync authorization
+      --sugarsync-authorization-expiry string          Sugarsync authorization expiry
+      --sugarsync-deleted-id string                    Sugarsync deleted folder id
+      --sugarsync-encoding MultiEncoder                The encoding for the backend (default Slash,Ctl,InvalidUtf8,Dot)
+      --sugarsync-hard-delete                          Permanently delete files if true
+      --sugarsync-private-access-key string            Sugarsync Private Access Key
+      --sugarsync-refresh-token string                 Sugarsync refresh token
+      --sugarsync-root-id string                       Sugarsync root id
+      --sugarsync-user string                          Sugarsync user
+      --swift-application-credential-id string         Application Credential ID (OS_APPLICATION_CREDENTIAL_ID)
+      --swift-application-credential-name string       Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME)
+      --swift-application-credential-secret string     Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET)
+      --swift-auth string                              Authentication URL for server (OS_AUTH_URL)
+      --swift-auth-token string                        Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)
+      --swift-auth-version int                         AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)
+      --swift-chunk-size SizeSuffix                    Above this size files will be chunked into a _segments container (default 5Gi)
+      --swift-domain string                            User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME)
+      --swift-encoding MultiEncoder                    The encoding for the backend (default Slash,InvalidUtf8)
+      --swift-endpoint-type string                     Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default "public")
+      --swift-env-auth                                 Get swift credentials from environment variables in standard OpenStack form
+      --swift-key string                               API key or password (OS_PASSWORD)
+      --swift-leave-parts-on-error                     If true avoid calling abort upload on a failure
+      --swift-no-chunk                                 Don't chunk files during streaming upload
+      --swift-no-large-objects                         Disable support for static and dynamic large objects
+      --swift-region string                            Region name - optional (OS_REGION_NAME)
+      --swift-storage-policy string                    The storage policy to use when creating a new container
+      --swift-storage-url string                       Storage URL - optional (OS_STORAGE_URL)
+      --swift-tenant string                            Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME)
+      --swift-tenant-domain string                     Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME)
+      --swift-tenant-id string                         Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID)
+      --swift-user string                              User name to log in (OS_USERNAME)
+      --swift-user-id string                           User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID)
+      --union-action-policy string                     Policy to choose upstream on ACTION category (default "epall")
+      --union-cache-time int                           Cache time of usage and free space (in seconds) (default 120)
+      --union-create-policy string                     Policy to choose upstream on CREATE category (default "epmfs")
+      --union-min-free-space SizeSuffix                Minimum viable free space for lfs/eplfs policies (default 1Gi)
+      --union-search-policy string                     Policy to choose upstream on SEARCH category (default "ff")
+      --union-upstreams string                         List of space separated upstreams
+      --uptobox-access-token string                    Your access token
+      --uptobox-encoding MultiEncoder                  The encoding for the backend (default Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot)
+      --webdav-bearer-token string                     Bearer token instead of user/pass (e.g. a Macaroon)
+      --webdav-bearer-token-command string             Command to run to get a bearer token
+      --webdav-encoding string                         The encoding for the backend
+      --webdav-headers CommaSepList                    Set HTTP headers for all transactions
+      --webdav-pass string                             Password (obscured)
+      --webdav-url string                              URL of http host to connect to
+      --webdav-user string                             User name
+      --webdav-vendor string                           Name of the WebDAV site/service/software you are using
+      --yandex-auth-url string                         Auth server URL
+      --yandex-client-id string                        OAuth Client Id
+      --yandex-client-secret string                    OAuth Client Secret
+      --yandex-encoding MultiEncoder                   The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot)
+      --yandex-hard-delete                             Delete files permanently rather than putting them into the trash
+      --yandex-token string                            OAuth Access Token as a JSON blob
+      --yandex-token-url string                        Token server url
+      --zoho-auth-url string                           Auth server URL
+      --zoho-client-id string                          OAuth Client Id
+      --zoho-client-secret string                      OAuth Client Secret
+      --zoho-encoding MultiEncoder                     The encoding for the backend (default Del,Ctl,InvalidUtf8)
+      --zoho-region string                             Zoho region to connect to
+      --zoho-token string                              OAuth Access Token as a JSON blob
+      --zoho-token-url string                          Token server url

              Docker Volume Plugin

              Introduction

              Docker 1.9 has added support for creating named volumes via command-line interface and mounting them in containers as a way to share data between them. Since Docker 1.10 you can create named volumes with Docker Compose by descriptions in docker-compose.yml files for use by container groups on a single host. As of Docker 1.12 volumes are supported by Docker Swarm included with Docker Engine and created from descriptions in swarm compose v3 files for use with swarm stacks across multiple cluster nodes.

              @@ -10111,9 +10228,10 @@

              Configuration

              token_url> Optional token URL Remote config Make sure your Redirect URL is set to "http://127.0.0.1:53682/" in your custom config. -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -10292,6 +10410,7 @@

              Amazon S3 Storage Providers

            • IBM COS S3
            • IDrive e2
            • IONOS Cloud
              • +
            • Liara Object Storage
            • Minio
            • Qiniu Cloud Object Storage (Kodo)
            • RackCorp Object Storage
              • @@ -10327,7 +10446,7 @@

              Configuration

              Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS +XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Liara, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -10337,7 +10456,7 @@

              Configuration

              \ "AWS" 2 / Ceph Object Storage \ "Ceph" - 3 / Digital Ocean Spaces + 3 / DigitalOcean Spaces \ "DigitalOcean" 4 / Dreamhost DreamObjects \ "Dreamhost" @@ -10653,7 +10772,7 @@

              Multipart uploads

              Increasing --s3-upload-concurrency will increase throughput (8 would be a sensible value) and increasing --s3-chunk-size also increases throughput (16M would be sensible). Increasing either of these will use more memory. The default values are high enough to gain most of the possible performance without using too much memory.

              Buckets and Regions

              With Amazon S3 you can list buckets (rclone lsd) using any region, but you can only access the content of a bucket from the region it was created in. If you attempt to access a bucket from the wrong region, you will get an error, incorrect region, the bucket is not in 'XXX' region.

              -

              Authentication

              +

              Authentication

              There are a number of ways to supply rclone with a set of AWS credentials, with and without using the environment.

              The different authentication methods are tried in this order:

                @@ -10743,7 +10862,7 @@

                Object-lock enabled S3 bucket

                As mentioned in the Hashes section, small files that are not uploaded as multipart, use a different tag, causing the upload to fail. A simple solution is to set the --s3-upload-cutoff 0 and force all the files to be uploaded as multipart.

                Standard options

                -

                Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).

                +

                Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).

                --s3-provider

                Choose your S3 provider.

                Properties:

                @@ -10780,7 +10899,7 @@

                --s3-provider

            • "DigitalOcean"
                -
              • Digital Ocean Spaces
                • +
              • DigitalOcean Spaces
            • "Dreamhost"
                @@ -10806,6 +10925,10 @@

                --s3-provider

                • Seagate Lyve Cloud
                +
              • "Liara" +
                  +
                • Liara Object Storage
                  • +
              • "Minio"
                • Minio Object Storage
                  • @@ -11317,7 +11440,7 @@

                  --s3-region

                  • Config: region
                  • Env Var: RCLONE_S3_REGION
                    • -
                  • Provider: !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
                    • +
                  • Provider: !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Liara,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
                  • Type: string
                  • Required: false
                  • Examples: @@ -11790,6 +11913,24 @@

                    --s3-endpoint

                --s3-endpoint

                +

                Endpoint for Liara Object Storage API.

                +

                Properties:

                +
                  +
                • Config: endpoint
                  • +
                • Env Var: RCLONE_S3_ENDPOINT
                  • +
                • Provider: Liara
                  • +
                • Type: string
                  • +
                • Required: false
                  • +
                • Examples: +
                    +
                  • "storage.iran.liara.space" +
                      +
                    • The default endpoint
                      • +
                    • Iran
                      • +
                    • +
                  • +
                +

                --s3-endpoint

                Endpoint for OSS API.

                Properties:

                  @@ -11902,7 +12043,7 @@

                  --s3-endpoint

            -

            --s3-endpoint

            +

            --s3-endpoint

            Endpoint for OBS API.

            Properties:

              @@ -11975,7 +12116,7 @@

              --s3-endpoint

        -

        --s3-endpoint

        +

        --s3-endpoint

        Endpoint for Scaleway Object Storage.

        Properties:

          @@ -12000,7 +12141,7 @@

          --s3-endpoint

      -

      --s3-endpoint

      +

      --s3-endpoint

      Endpoint for StackPath Object Storage.

      Properties:

        @@ -12025,7 +12166,7 @@

        --s3-endpoint

      -

      --s3-endpoint

      +

      --s3-endpoint

      Endpoint of the Shared Gateway.

      Properties:

        @@ -12050,7 +12191,7 @@

        --s3-endpoint

      -

      --s3-endpoint

      +

      --s3-endpoint

      Endpoint for Tencent COS API.

      Properties:

        @@ -12139,7 +12280,7 @@

        --s3-endpoint

      -

      --s3-endpoint

      +

      --s3-endpoint

      Endpoint for RackCorp Object Storage.

      Properties:

        @@ -12228,7 +12369,7 @@

        --s3-endpoint

      -

      --s3-endpoint

      +

      --s3-endpoint

      Endpoint for Qiniu Object Storage.

      Properties:

        @@ -12269,14 +12410,14 @@

        --s3-endpoint

      -

      --s3-endpoint

      +

      --s3-endpoint

      Endpoint for S3 API.

      Required when using an S3 clone.

      Properties:

      • Config: endpoint
      • Env Var: RCLONE_S3_ENDPOINT
        • -
      • Provider: !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu
        • +
      • Provider: !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu
      • Type: string
      • Required: false
      • Examples: @@ -12285,17 +12426,29 @@

        --s3-endpoint

        • Dream Objects endpoint
        • +
      • "syd1.digitaloceanspaces.com" +
          +
        • DigitalOcean Spaces Sydney 1
          • +
        • +
      • "sfo3.digitaloceanspaces.com" +
          +
        • DigitalOcean Spaces San Francisco 3
          • +
        • +
      • "fra1.digitaloceanspaces.com" +
          +
        • DigitalOcean Spaces Frankfurt 1
          • +
      • "nyc3.digitaloceanspaces.com"
          -
        • Digital Ocean Spaces New York 3
          • +
        • DigitalOcean Spaces New York 3
      • "ams3.digitaloceanspaces.com"
          -
        • Digital Ocean Spaces Amsterdam 3
          • +
        • DigitalOcean Spaces Amsterdam 3
      • "sgp1.digitaloceanspaces.com"
          -
        • Digital Ocean Spaces Singapore 1
          • +
        • DigitalOcean Spaces Singapore 1
      • "localhost:8333"
          @@ -12315,15 +12468,39 @@

          --s3-endpoint

      • "s3.wasabisys.com"
          -
        • Wasabi US East endpoint
          • +
        • Wasabi US East 1 (N. Virginia)
          • +
        • +
      • "s3.us-east-2.wasabisys.com" +
          +
        • Wasabi US East 2 (N. Virginia)
          • +
        • +
      • "s3.us-central-1.wasabisys.com" +
          +
        • Wasabi US Central 1 (Texas)
      • "s3.us-west-1.wasabisys.com"
          -
        • Wasabi US West endpoint
          • +
        • Wasabi US West 1 (Oregon)
          • +
        • +
      • "s3.ca-central-1.wasabisys.com" +
          +
        • Wasabi CA Central 1 (Toronto)
      • "s3.eu-central-1.wasabisys.com"
          -
        • Wasabi EU Central endpoint
          • +
        • Wasabi EU Central 1 (Amsterdam)
          • +
        • +
      • "s3.eu-central-2.wasabisys.com" +
          +
        • Wasabi EU Central 2 (Frankfurt)
          • +
        • +
      • "s3.eu-west-1.wasabisys.com" +
          +
        • Wasabi EU West 1 (London)
          • +
        • +
      • "s3.eu-west-2.wasabisys.com" +
          +
        • Wasabi EU West 2 (Paris)
      • "s3.ap-northeast-1.wasabisys.com"
          @@ -12333,6 +12510,18 @@

          --s3-endpoint

          • Wasabi AP Northeast 2 (Osaka) endpoint
          +
        • "s3.ap-southeast-1.wasabisys.com" +
            +
          • Wasabi AP Southeast 1 (Singapore)
            • +
          • +
        • "s3.ap-southeast-2.wasabisys.com" +
            +
          • Wasabi AP Southeast 2 (Sydney)
            • +
          • +
        • "storage.iran.liara.space" +
            +
          • Liara Iran endpoint
            • +
        • "s3.ir-thr-at1.arvanstorage.com"
          • ArvanCloud Tehran Iran (Asiatech) endpoint
            • @@ -12889,7 +13078,7 @@

            --s3-location-constraint

            • Config: location_constraint
            • Env Var: RCLONE_S3_LOCATION_CONSTRAINT
              • -
            • Provider: !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS
              • +
            • Provider: !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Liara,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS
            • Type: string
            • Required: false
            @@ -12898,6 +13087,7 @@

            --s3-acl

            This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.

            For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl

            Note that this ACL is applied when server-side copying objects as S3 doesn't copy the ACL from the source but rather writes a fresh one.

            +

            If the acl is an empty string then no X-Amz-Acl: header is added and the default (private) will be used.

            Properties:

            • Config: acl
              • @@ -13125,6 +13315,23 @@

              --s3-storage-class

          --s3-storage-class

          +

          The storage class to use when storing new objects in Liara

          +

          Properties:

          +
            +
          • Config: storage_class
            • +
          • Env Var: RCLONE_S3_STORAGE_CLASS
            • +
          • Provider: Liara
            • +
          • Type: string
            • +
          • Required: false
            • +
          • Examples: +
              +
            • "STANDARD" +
                +
              • Standard storage class
                • +
              • +
            • +
          +

          --s3-storage-class

          The storage class to use when storing new objects in ArvanCloud.

          Properties:

            @@ -13141,7 +13348,7 @@

            --s3-storage-class

      -

      --s3-storage-class

      +

      --s3-storage-class

      The storage class to use when storing new objects in Tencent COS.

      Properties:

        @@ -13170,7 +13377,7 @@

        --s3-storage-class

      -

      --s3-storage-class

      +

      --s3-storage-class

      The storage class to use when storing new objects in S3.

      Properties:

        @@ -13197,7 +13404,7 @@

        --s3-storage-class

      -

      --s3-storage-class

      +

      --s3-storage-class

      The storage class to use when storing new objects in Qiniu.

      Properties:

        @@ -13227,11 +13434,12 @@

        --s3-storage-class

      Advanced options

      -

      Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).

      +

      Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).

      --s3-bucket-acl

      Canned ACL used when creating buckets.

      For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl

      Note that this ACL is applied when only when creating buckets. If it isn't set then "acl" is used instead.

      +

      If the "acl" and "bucket_acl" are empty strings then no X-Amz-Acl: header is added and the default (private) will be used.

      Properties:

      • Config: bucket_acl
        • @@ -13673,6 +13881,21 @@

        --s3-decompress

      • Type: bool
      • Default: false
      +

      --s3-might-gzip

      +

      Set this if the backend might gzip objects.

      +

      Normally providers will not alter objects when they are downloaded. If an object was not uploaded with Content-Encoding: gzip then it won't be set on download.

      +

      However some providers may gzip objects even if they weren't uploaded with Content-Encoding: gzip (eg Cloudflare).

      +

      A symptom of this would be receiving errors like

      +
      ERROR corrupted on transfer: sizes differ NNN vs MMM
      +

      If you set this flag and rclone downloads an object with Content-Encoding: gzip set and chunked transfer encoding, then rclone will decompress the object on the fly.

      +

      If this is set to unset (the default) then rclone will choose according to the provider setting what to apply, but you can override rclone's choice here.

      +

      Properties:

      +
        +
      • Config: might_gzip
        • +
      • Env Var: RCLONE_S3_MIGHT_GZIP
        • +
      • Type: Tristate
        • +
      • Default: unset
        • +

      --s3-no-system-metadata

      Suppress setting and reading of system metadata

      Properties:

      @@ -13930,7 +14153,7 @@

      Cloudflare R2

      Type of storage to configure. Choose a number from below, or type in your own value. ... -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) ... Storage> s3 @@ -14061,7 +14284,7 @@

      Huawei OBS

      Type of storage to configure. Choose a number from below, or type in your own value. [snip] - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) [snip] Storage> 5 @@ -14183,7 +14406,7 @@

      IBM COS (S3)

      \ "alias" 2 / Amazon Drive \ "amazon cloud drive" - 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS) + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, Liara, ArvanCloud, Minio, IBM COS) \ "s3" 4 / Backblaze B2 \ "b2" @@ -14333,7 +14556,7 @@

      IDrive e2

      Type of storage to configure. Choose a number from below, or type in your own value. [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) [snip] Storage> s3 @@ -14428,7 +14651,7 @@

      IONOS Cloud

      Type of storage to configure. Choose a number from below, or type in your own value. [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) [snip] Storage> s3       @@ -14618,7 +14841,7 @@

      Qiniu Cloud Object Storage (Kodo)

      \ (alias) 4 / Amazon Drive \ (amazon cloud drive) - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi \ (s3) [snip] Storage> s3 @@ -14818,7 +15041,7 @@

      Seagate Lyve Cloud

      Type of storage to configure.
 Choose a number from below, or type in your own value.
 [snip]
-XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
    \ (s3)
 [snip]
 Storage> s3
      @@ -14943,7 +15166,7 @@

      Wasabi

      Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, Liara) \ "s3" [snip] Storage> s3 @@ -15045,7 +15268,7 @@

      Alibaba OSS

      Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [snip] - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -15147,7 +15370,7 @@

      China Mobile Ecloud Elastic Object Storage (EOS Type of storage to configure. Choose a number from below, or type in your own value. ... - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \ (s3) ... Storage> s3 @@ -15375,6 +15598,95 @@

      China Mobile Ecloud Elastic Object Storage (EOS e) Edit this remote d) Delete this remote y/e/d> y +

      Liara

      +

      Here is an example of making a Liara Object Storage configuration. First run:

      +
      rclone config
      +

      This will guide you through an interactive setup process.

      +
      No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+n/s> n
+name> Liara
+Type of storage to configure.
+Choose a number from below, or type in your own value
+[snip]
+XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Liara, Minio)
+   \ "s3"
+[snip]
+Storage> s3
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+AWS Access Key ID - leave blank for anonymous access or runtime credentials.
+access_key_id> YOURACCESSKEY
+AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
+secret_access_key> YOURSECRETACCESSKEY
+Region to connect to.
+Choose a number from below, or type in your own value
+   / The default endpoint
+ 1 | US Region, Northern Virginia, or Pacific Northwest.
+   | Leave location constraint empty.
+   \ "us-east-1"
+[snip]
+region>
+Endpoint for S3 API.
+Leave blank if using Liara to use the default endpoint for the region.
+Specify if using an S3 clone such as Ceph.
+endpoint> storage.iran.liara.space
+Canned ACL used when creating buckets and/or storing objects in S3.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Choose a number from below, or type in your own value
+ 1 / Owner gets FULL_CONTROL. No one else has access rights (default).
+   \ "private"
+[snip]
+acl>
+The server-side encryption algorithm used when storing this object in S3.
+Choose a number from below, or type in your own value
+ 1 / None
+   \ ""
+ 2 / AES256
+   \ "AES256"
+server_side_encryption>
+The storage class to use when storing objects in S3.
+Choose a number from below, or type in your own value
+ 1 / Default
+   \ ""
+ 2 / Standard storage class
+   \ "STANDARD"
+storage_class>
+Remote config
+--------------------
+[Liara]
+env_auth = false
+access_key_id = YOURACCESSKEY
+secret_access_key = YOURSECRETACCESSKEY
+endpoint = storage.iran.liara.space
+location_constraint =
+acl =
+server_side_encryption =
+storage_class =
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
      +

      This will leave the config file looking like this.

      +
      [Liara]
+type = s3
+provider = Liara
+env_auth = false
+access_key_id = YOURACCESSKEY
+secret_access_key = YOURSECRETACCESSKEY
+region =
+endpoint = storage.iran.liara.space
+location_constraint =
+acl =
+server_side_encryption =
+storage_class =

      ArvanCloud

      ArvanCloud ArvanCloud Object Storage goes beyond the limited traditional file storage. It gives you access to backup and archived files and allows sharing. Files like profile image in the app, images sent by users or scanned documents can be stored securely and easily in our Object Storage service.

      ArvanCloud provides an S3 interface which can be configured for use with rclone like this.

      @@ -15386,7 +15698,7 @@

      ArvanCloud

      Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Liara, Minio) \ "s3" [snip] Storage> s3 @@ -15496,7 +15808,7 @@

      Tencent COS

      \ "alias" 3 / Amazon Drive \ "amazon cloud drive" - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -16031,9 +16343,10 @@

      Configuration

      \ "enterprise" box_sub_type> Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -16116,9 +16429,10 @@

      Invalid refresh token

      y) Yes n) No y/n> y -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -16337,9 +16651,10 @@

      Limitations

      Note that Box is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

      Box file names can't have the \ character in. rclone maps this to and from an identical looking unicode equivalent ＼ (U+FF3C Fullwidth Reverse Solidus).

      Box only supports filenames up to 255 characters in length.

      +

      Box has API rate limits that sometimes reduce the speed of rclone.

      rclone about is not supported by the Box backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.

      See List of backends that do not support rclone about and rclone about

      -

      Cache (DEPRECATED)

      +

      Cache

      The cache remote wraps another existing remote and stores file structure and its data for long running tasks like rclone mount.

      Status

      The cache backend code is working but it currently doesn't have a maintainer so there are outstanding bugs which aren't getting fixed.

      @@ -16773,7 +17088,7 @@

      Backend commands

      stats

      Print stats on the cache backend in JSON format.

      rclone backend stats remote: [options] [<arguments>+]
      -

      Chunker (BETA)

      +

      Chunker

      The chunker overlay transparently splits large files into smaller chunks during upload to wrapped remote and transparently assembles them back when the file is downloaded. This allows to effectively overcome size limits imposed by storage providers.

      Configuration

      To use it, first set up the underlying remote following the configuration instructions for that remote. You can also use a local pathname instead of a remote.

      @@ -17091,9 +17406,10 @@

      Configuration

      n) No y/n> n Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -17724,7 +18040,7 @@

      SEE ALSO

      -

      Compress (Experimental)

      +

      Compress

      Warning

      This remote is currently experimental. Things may break and data may be lost. Anything you do with this remote is at your own risk. Please understand the risks associated with using experimental code and don't use this remote in critical applications.

      The Compress remote adds compression to another remote. It is best used with remotes containing many large compressible files.

      @@ -17972,6 +18288,8 @@

      Configuration

      e) Edit this remote d) Delete this remote y/e/d> y +

      See the remote setup docs for how to set it up on a machine with no Internet browser available.

      +

      Note that rclone runs a webserver on your local machine to collect the token as returned from Dropbox. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and it may require you to unblock it temporarily if you are running a host firewall, or use manual mode.

      You can then use it like this,

      List directories in top level of your dropbox

      rclone lsd remote:
      @@ -18595,7 +18913,7 @@

      --ftp-pass

      --ftp-tls

      Use Implicit FTPS (FTP over TLS).

      -

      When using implicit FTP over TLS the client connects using TLS right from the start which breaks compatibility with non-TLS-aware servers. This is usually served over port 990 rather than port 21. Cannot be used in combination with explicit FTP.

      +

      When using implicit FTP over TLS the client connects using TLS right from the start which breaks compatibility with non-TLS-aware servers. This is usually served over port 990 rather than port 21. Cannot be used in combination with explicit FTPS.

      Properties:

      • Config: tls
        • @@ -18605,7 +18923,7 @@

        --ftp-tls

      --ftp-explicit-tls

      Use Explicit FTPS (FTP over TLS).

      -

      When using explicit FTP over TLS the client explicitly requests security from the server in order to upgrade a plain text connection to an encrypted one. Cannot be used in combination with implicit FTP.

      +

      When using explicit FTP over TLS the client explicitly requests security from the server in order to upgrade a plain text connection to an encrypted one. Cannot be used in combination with implicit FTPS.

      Properties:

      • Config: explicit_tls
        • @@ -18881,9 +19199,10 @@

        Configuration

        \ "DURABLE_REDUCED_AVAILABILITY" storage_class> 5 Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine or Y didn't work +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -18905,7 +19224,8 @@

        Configuration

        e) Edit this remote d) Delete this remote y/e/d> y -

        Note that rclone runs a webserver on your local machine to collect the token as returned from Google if you use auto config mode. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall, or use manual mode.

        +

        See the remote setup docs for how to set it up on a machine with no Internet browser available.

        +

        Note that rclone runs a webserver on your local machine to collect the token as returned from Google if using web browser to automatically authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall, or use manual mode.

        This remote is called remote and can now be used like this

        See all the buckets in your project

        rclone lsd remote:
        @@ -19452,9 +19772,10 @@

        Configuration

        Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. service_account_file> Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine or Y didn't work +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -19479,7 +19800,8 @@

        Configuration

        e) Edit this remote d) Delete this remote y/e/d> y -

        Note that rclone runs a webserver on your local machine to collect the token as returned from Google if you use auto config mode. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and it may require you to unblock it temporarily if you are running a host firewall, or use manual mode.

        +

        See the remote setup docs for how to set it up on a machine with no Internet browser available.

        +

        Note that rclone runs a webserver on your local machine to collect the token as returned from Google if using web browser to automatically authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and it may require you to unblock it temporarily if you are running a host firewall, or use manual mode.

        You can then use it like this,

        List directories in top level of your drive

        rclone lsd remote:
        @@ -20557,9 +20879,10 @@

        Configuration

        n) No y/n> n Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -20581,7 +20904,8 @@

        Configuration

        e) Edit this remote d) Delete this remote y/e/d> y -

        Note that rclone runs a webserver on your local machine to collect the token as returned from Google if you use auto config mode. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this may require you to unblock it temporarily if you are running a host firewall, or use manual mode.

        +

        See the remote setup docs for how to set it up on a machine with no Internet browser available.

        +

        Note that rclone runs a webserver on your local machine to collect the token as returned from Google if using web browser to automatically authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this may require you to unblock it temporarily if you are running a host firewall, or use manual mode.

        This remote is called remote and can now be used like this

        See all the albums in your photos

        rclone lsd remote:album
        @@ -20803,7 +21127,7 @@

        Deleting files

        Rclone cannot delete files anywhere except under album.

        Deleting albums

        The Google Photos API does not support deleting albums - see bug #135714733.

        -

        Hasher (EXPERIMENTAL)

        +

        Hasher

        Hasher is a special overlay backend to create remotes which handle checksums for other remotes. It's main functions include: - Emulate hash types unimplemented by backends - Cache checksums to help with slow hashing of large local or (S)FTP files - Warm up checksum cache from external SUM files

        Getting started

        To use Hasher, first set up the underlying remote following the configuration instructions for that remote. You can also use a local pathname instead of a remote. Check that your base remote is working.

        @@ -21191,7 +21515,10 @@

        Configuration

        scope_access> Edit advanced config? y/n> n -Use auto config? +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y/n> y If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx Log in and authorize rclone for access @@ -21606,6 +21933,11 @@

        About metadata

        rclone reserves all the keys starting with rclone-. Setting value for these keys will give you warnings, but values are set according to request.

        If there are multiple values for a key, only the first one is returned. This is a limitation of rclone, that supports one value per one key. It can be triggered when you did a server-side copy.

        Reading metadata will also provide custom (non-standard nor reserved) ones.

        +

        Filtering auto generated files

        +

        The Internet Archive automatically creates metadata files after upload. These can cause problems when doing an rclone sync as rclone will try, and fail, to delete them. These metadata files are not changeable, as they are created by the Internet Archive automatically.

        +

        These auto-created files can be excluded from the sync using metadata filtering.

        +
        rclone sync ... --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata"
        +

        Which excludes from the sync any files which have the source=metadata or format=Metadata flags which are added to Internet Archive auto-created files.

        Configuration

        Here is an example of making an internetarchive configuration. Most applies to the other providers as well, any differences are described below.

        First run

        @@ -22458,7 +22790,17 @@

        Features highlights

      • If a particular file is already present in storage, one can quickly submit file hash instead of long file upload (this optimization is supported by rclone)

      Configuration

      -

      Here is an example of making a mailru configuration. First create a Mail.ru Cloud account and choose a tariff, then run

      +

      Here is an example of making a mailru configuration.

      +

      First create a Mail.ru Cloud account and choose a tariff.

      +

      You will need to log in and create an app password for rclone. Rclone will not work with your normal username and password - it will give an error like oauth2: server response missing access_token.

      +
        +
      • Click on your user icon in the top right
        • +
      • Go to Security / "Пароль и безопасность"
        • +
      • Click password for apps / "Пароли для внешних приложений"
        • +
      • Add the password - give it a name - eg "rclone"
        • +
      • Copy the password and use this password below - your normal login password won't work.
        • +
      +

      Now run

      rclone config

      This will guide you through an interactive setup process:

      No remotes found, make a new one?
@@ -22480,6 +22822,10 @@ 
      Configuration
      
 Enter a string value. Press Enter for the default ("").
 user> username@mail.ru
 Password
+
+This must be an app password - rclone will not work with your normal
+password. See the Configuration section in the docs for how to make an
+app password.
 y) Yes type in my own password
 g) Generate random password
 y/g> y
@@ -22598,6 +22944,7 @@ 
      --mailru-user
      
 
 
      --mailru-pass
      
 
      Password.
      
+
      This must be an app password - rclone will not work with your normal password. See the Configuration section in the docs for how to make an app password.
      
 
      NB Input to this must be obscured - see rclone obscure.
      
 
      Properties:
      
 
        
@@ -23190,7 +23537,8 @@ 
        Configuration
        
 
        --fast-list
        
 
        This remote supports --fast-list which allows you to use fewer transactions in exchange for more memory. See the rclone docs for more details.
        
 
        Modified time
        
-
        The modified time is stored as metadata on the object with the mtime key. It is stored using RFC3339 Format time with nanosecond precision. The metadata is supplied during directory listings so there is no overhead to using it.
        
+
        The modified time is stored as metadata on the object with the mtime key. It is stored using RFC3339 Format time with nanosecond precision. The metadata is supplied during directory listings so there is no performance overhead to using it.
        
+
        If you wish to use the Azure standard LastModified time stored on the object as the modified time, then use the --use-server-modtime flag. Note that rclone can't set LastModified, so using the --update flag when syncing is recommended if using --use-server-modtime.
        
 
        Performance
        
 
        When uploading large files, increasing the value of --azureblob-upload-concurrency will increase performance at the cost of using more memory. The default of 16 is set quite conservatively to use less memory. It maybe be necessary raise it to 64 or higher to fully utilize a 1 GBit/s link with a single file transfer.
        
 
        Restricted filename characters
        
@@ -23236,13 +23584,60 @@ 
        Restricted filename characters
        
 
        Invalid UTF-8 bytes will also be replaced, as they can't be used in JSON strings.
        
 
        Hashes
        
 
        MD5 hashes are stored with blobs. However blobs that were uploaded in chunks only have an MD5 if the source remote was capable of MD5 hashes, e.g. the local disk.
        
-
        Authenticating with Azure Blob Storage
        
-
        Rclone has 3 ways of authenticating with Azure Blob Storage:
        
-
        Account and Key
        
+
        Authentication
        
+
        There are a number of ways of supplying credentials for Azure Blob Storage. Rclone tries them in the order of the sections below.
        
+
        Env Auth
        
+
        If the env_auth config parameter is true then rclone will pull credentials from the environment or runtime.
        
+
        It tries these authentication methods in this order:
        
+
          
+
        1. Environment Variables
          2. 
+
        2. Managed Service Identity Credentials
          3. 
+
        3. Azure CLI credentials (as used by the az tool)
          4. 
+
        
+
        These are described in the following sections
        
+
        Env Auth: 1. Environment Variables
        
+
        If env_auth is set and environment variables are present rclone authenticates a service principal with a secret or certificate, or a user with a password, depending on which environment variable are set. It reads configuration from these variables, in the following order:
        
+
          
+
        1. Service principal with client secret
+
            
+
          • AZURE_TENANT_ID: ID of the service principal's tenant. Also called its "directory" ID.
            • 
+
          • AZURE_CLIENT_ID: the service principal's client ID
            • 
+
          • AZURE_CLIENT_SECRET: one of the service principal's client secrets
            • 
+
          2. 
+
        2. Service principal with certificate
+
            
+
          • AZURE_TENANT_ID: ID of the service principal's tenant. Also called its "directory" ID.
            • 
+
          • AZURE_CLIENT_ID: the service principal's client ID
            • 
+
          • AZURE_CLIENT_CERTIFICATE_PATH: path to a PEM or PKCS12 certificate file including the private key.
            • 
+
          • AZURE_CLIENT_CERTIFICATE_PASSWORD: (optional) password for the certificate file.
            • 
+
          • AZURE_CLIENT_SEND_CERTIFICATE_CHAIN: (optional) Specifies whether an authentication request will include an x5c header to support subject name / issuer based authentication. When set to "true" or "1", authentication requests include the x5c header.
            • 
+
          3. 
+
        3. User with username and password
+
            
+
          • AZURE_TENANT_ID: (optional) tenant to authenticate in. Defaults to "organizations".
            • 
+
          • AZURE_CLIENT_ID: client ID of the application the user will authenticate to
            • 
+
          • AZURE_USERNAME: a username (usually an email address)
            • 
+
          • AZURE_PASSWORD: the user's password
            • 
+
          4. 
+
        
+
        Env Auth: 2. Managed Service Identity Credentials
        
+
        When using Managed Service Identity if the VM(SS) on which this program is running has a system-assigned identity, it will be used by default. If the resource has no system-assigned but exactly one user-assigned identity, the user-assigned identity will be used by default.
        
+
        If the resource has multiple user-assigned identities you will need to unset env_auth and set use_msi instead. See the use_msi section.
        
+
        Env Auth: 3. Azure CLI credentials (as used by the az tool)
        
+
        Credentials created with the az tool can be picked up using env_auth.
        
+
        For example if you were to login with a service principal like this:
        
+
        az login --service-principal -u XXX -p XXX --tenant XXX
        
+
        Then you could access rclone resources like this:
        
+
        rclone lsf :azureblob,env_auth,account=ACCOUNT:CONTAINER
        
+
        Or
        
+
        rclone lsf --azureblob-env-auth --azureblob-acccount=ACCOUNT :azureblob:CONTAINER
        
+
        Which is analogous to using the az tool:
        
+
        az storage blob list --container-name CONTAINER --account-name ACCOUNT --auth-mode login
        
+
        Account and Shared Key
        
 
        This is the most straight forward and least flexible way. Just fill in the account and key lines and leave the rest blank.
        
 
        SAS URL
        
 
        This can be an account level SAS URL or container level SAS URL.
        
-
        To use it leave account, key blank and fill in sas_url.
        
+
        To use it leave account and key blank and fill in sas_url.
        
 
        An account level SAS URL or container level SAS URL can be obtained from the Azure portal or the Azure Storage Explorer. To get a container level SAS URL right click on a container in the Azure Blob explorer in the Azure portal.
        
 
        If you use a container level SAS URL, rclone operations are permitted only on a particular container, e.g.
        
 
        rclone ls azureblob:container
        
@@ -23252,11 +23647,45 @@ 
        SAS URL
        
 
        Note that you can't see or access any other containers - this will fail
        
 
        rclone ls azureblob:othercontainer
        
 
        Container level SAS URLs are useful for temporarily allowing third parties access to a single container or putting credentials into an untrusted environment such as a CI build server.
        
+
        Service principal with client secret
        
+
        If these variables are set, rclone will authenticate with a service principal with a client secret.
        
+
          
+
        • tenant: ID of the service principal's tenant. Also called its "directory" ID.
          • 
+
        • client_id: the service principal's client ID
          • 
+
        • client_secret: one of the service principal's client secrets
          • 
+
        
+
        The credentials can also be placed in a file using the service_principal_file configuration option.
        
+
        Service principal with certificate
        
+
        If these variables are set, rclone will authenticate with a service principal with certificate.
        
+
          
+
        • tenant: ID of the service principal's tenant. Also called its "directory" ID.
          • 
+
        • client_id: the service principal's client ID
          • 
+
        • client_certificate_path: path to a PEM or PKCS12 certificate file including the private key.
          • 
+
        • client_certificate_password: (optional) password for the certificate file.
          • 
+
        • client_send_certificate_chain: (optional) Specifies whether an authentication request will include an x5c header to support subject name / issuer based authentication. When set to "true" or "1", authentication requests include the x5c header.
          • 
+
        
+
        NB client_certificate_password must be obscured - see rclone obscure.
        
+
        User with username and password
        
+
        If these variables are set, rclone will authenticate with username and password.
        
+
          
+
        • tenant: (optional) tenant to authenticate in. Defaults to "organizations".
          • 
+
        • client_id: client ID of the application the user will authenticate to
          • 
+
        • username: a username (usually an email address)
          • 
+
        • password: the user's password
          • 
+
        
+
        Microsoft doesn't recommend this kind of authentication, because it's less secure than other authentication flows. This method is not interactive, so it isn't compatible with any form of multi-factor authentication, and the application must already have user or admin consent. This credential can only authenticate work and school accounts; it can't authenticate Microsoft accounts.
        
+
        NB password must be obscured - see rclone obscure.
        
+
        Managed Service Identity Credentials
        
+
        If use_msi is set then managed service identity credentials are used. This authentication only works when running in an Azure service. env_auth needs to be unset to use this.
        
+
        However if you have multiple user identities to choose from these must be explicitly specified using exactly one of the msi_object_id, msi_client_id, or msi_mi_res_id parameters.
        
+
        If none of msi_object_id, msi_client_id, or msi_mi_res_id is set, this is is equivalent to using env_auth.
        
 
        Standard options
        
 
        Here are the Standard options specific to azureblob (Microsoft Azure Blob Storage).
        
 
        --azureblob-account
        
-
        Storage Account Name.
        
-
        Leave blank to use SAS URL or Emulator.
        
+
        Azure Storage Account Name.
        
+
        Set this to the Azure Storage Account Name in use.
        
+
        Leave blank to use SAS URL or Emulator, otherwise it needs to be set.
        
+
        If this is blank and if env_auth is set it will be read from the environment variable AZURE_STORAGE_ACCOUNT_NAME if possible.
        
 
        Properties:
        
 
          
 
        • Config: account
          • 
@@ -23264,23 +23693,18 @@ 
          --azureblob-account
          
 
        • Type: string
          • 
 
        • Required: false
          • 
 
        
-
        --azureblob-service-principal-file
        
-
        Path to file containing credentials for use with a service principal.
        
-
        Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
        
-
        $ az ad sp create-for-rbac --name "<name>" \
-  --role "Storage Blob Data Owner" \
-  --scopes "/subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Storage/storageAccounts/<storage-account>/blobServices/default/containers/<container>" \
-  > azure-principal.json
        
-
        See "Create an Azure service principal" and "Assign an Azure role for access to blob data" pages for more details.
        
+
        --azureblob-env-auth
        
+
        Read credentials from runtime (environment variables, CLI or MSI).
        
+
        See the authentication docs for full info.
        
 
        Properties:
        
 
          
-
        • Config: service_principal_file
          • 
-
        • Env Var: RCLONE_AZUREBLOB_SERVICE_PRINCIPAL_FILE
          • 
-
        • Type: string
          • 
-
        • Required: false
          • 
+
        • Config: env_auth
          • 
+
        • Env Var: RCLONE_AZUREBLOB_ENV_AUTH
          • 
+
        • Type: bool
          • 
+
        • Default: false
          • 
 
        
 
        --azureblob-key
        
-
        Storage Account Key.
        
+
        Storage Account Shared Key.
        
 
        Leave blank to use SAS URL or Emulator.
        
 
        Properties:
        
 
          
@@ -23299,6 +23723,108 @@ 
          --azureblob-sas-url
          
 
        • Type: string
          • 
 
        • Required: false
          • 
 
        
+
        --azureblob-tenant
        
+
        ID of the service principal's tenant. Also called its directory ID.
        
+
        Set this if using - Service principal with client secret - Service principal with certificate - User with username and password
        
+
        Properties:
        
+
          
+
        • Config: tenant
          • 
+
        • Env Var: RCLONE_AZUREBLOB_TENANT
          • 
+
        • Type: string
          • 
+
        • Required: false
          • 
+
        
+
        --azureblob-client-id
        
+
        The ID of the client in use.
        
+
        Set this if using - Service principal with client secret - Service principal with certificate - User with username and password
        
+
        Properties:
        
+
          
+
        • Config: client_id
          • 
+
        • Env Var: RCLONE_AZUREBLOB_CLIENT_ID
          • 
+
        • Type: string
          • 
+
        • Required: false
          • 
+
        
+
        --azureblob-client-secret
        
+
        One of the service principal's client secrets
        
+
        Set this if using - Service principal with client secret
        
+
        Properties:
        
+
          
+
        • Config: client_secret
          • 
+
        • Env Var: RCLONE_AZUREBLOB_CLIENT_SECRET
          • 
+
        • Type: string
          • 
+
        • Required: false
          • 
+
        
+
        --azureblob-client-certificate-path
        
+
        Path to a PEM or PKCS12 certificate file including the private key.
        
+
        Set this if using - Service principal with certificate
        
+
        Properties:
        
+
          
+
        • Config: client_certificate_path
          • 
+
        • Env Var: RCLONE_AZUREBLOB_CLIENT_CERTIFICATE_PATH
          • 
+
        • Type: string
          • 
+
        • Required: false
          • 
+
        
+
        --azureblob-client-certificate-password
        
+
        Password for the certificate file (optional).
        
+
        Optionally set this if using - Service principal with certificate
        
+
        And the certificate has a password.
        
+
        NB Input to this must be obscured - see rclone obscure.
        
+
        Properties:
        
+
          
+
        • Config: client_certificate_password
          • 
+
        • Env Var: RCLONE_AZUREBLOB_CLIENT_CERTIFICATE_PASSWORD
          • 
+
        • Type: string
          • 
+
        • Required: false
          • 
+
        
+
        Advanced options
        
+
        Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage).
        
+
        --azureblob-client-send-certificate-chain
        
+
        Send the certificate chain when using certificate auth.
        
+
        Specifies whether an authentication request will include an x5c header to support subject name / issuer based authentication. When set to true, authentication requests include the x5c header.
        
+
        Optionally set this if using - Service principal with certificate
        
+
        Properties:
        
+
          
+
        • Config: client_send_certificate_chain
          • 
+
        • Env Var: RCLONE_AZUREBLOB_CLIENT_SEND_CERTIFICATE_CHAIN
          • 
+
        • Type: bool
          • 
+
        • Default: false
          • 
+
        
+
        --azureblob-username
        
+
        User name (usually an email address)
        
+
        Set this if using - User with username and password
        
+
        Properties:
        
+
          
+
        • Config: username
          • 
+
        • Env Var: RCLONE_AZUREBLOB_USERNAME
          • 
+
        • Type: string
          • 
+
        • Required: false
          • 
+
        
+
        --azureblob-password
        
+
        The user's password
        
+
        Set this if using - User with username and password
        
+
        NB Input to this must be obscured - see rclone obscure.
        
+
        Properties:
        
+
          
+
        • Config: password
          • 
+
        • Env Var: RCLONE_AZUREBLOB_PASSWORD
          • 
+
        • Type: string
          • 
+
        • Required: false
          • 
+
        
+
        --azureblob-service-principal-file
        
+
        Path to file containing credentials for use with a service principal.
        
+
        Leave blank normally. Needed only if you want to use a service principal instead of interactive login.
        
+
        $ az ad sp create-for-rbac --name "<name>" \
+  --role "Storage Blob Data Owner" \
+  --scopes "/subscriptions/<subscription>/resourceGroups/<resource-group>/providers/Microsoft.Storage/storageAccounts/<storage-account>/blobServices/default/containers/<container>" \
+  > azure-principal.json
        
+
        See "Create an Azure service principal" and "Assign an Azure role for access to blob data" pages for more details.
        
+
        It may be more convenient to put the credentials directly into the rclone config file under the client_id, tenant and client_secret keys instead of setting service_principal_file.
        
+
        Properties:
        
+
          
+
        • Config: service_principal_file
          • 
+
        • Env Var: RCLONE_AZUREBLOB_SERVICE_PRINCIPAL_FILE
          • 
+
        • Type: string
          • 
+
        • Required: false
          • 
+
        
 
        --azureblob-use-msi
        
 
        Use a managed service identity to authenticate (only works in Azure).
        
 
        When true, use a managed service identity to authenticate to Azure Storage instead of a SAS token or account key.
        
@@ -23310,18 +23836,6 @@ 
        --azureblob-use-msi
        
 
      • Type: bool
        • 
 
      • Default: false
        • 
 
      
-
      --azureblob-use-emulator
      
-
      Uses local storage emulator if provided as 'true'.
      
-
      Leave blank if using real azure storage endpoint.
      
-
      Properties:
      
-
        
-
      • Config: use_emulator
        • 
-
      • Env Var: RCLONE_AZUREBLOB_USE_EMULATOR
        • 
-
      • Type: bool
        • 
-
      • Default: false
        • 
-
      
-
      Advanced options
      
-
      Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage).
      
 
      --azureblob-msi-object-id
      
 
      Object ID of the user-assigned MSI to use, if any.
      
 
      Leave blank if msi_client_id or msi_mi_res_id specified.
      
@@ -23352,6 +23866,16 @@ 
      --azureblob-msi-mi-res-id
      
 
    96. Type: string
      97. 
 
    97. Required: false
      98. 
 
+
      --azureblob-use-emulator
      
+
      Uses local storage emulator if provided as 'true'.
      
+
      Leave blank if using real azure storage endpoint.
      
+
      Properties:
      
+
        
+
      • Config: use_emulator
        • 
+
      • Env Var: RCLONE_AZUREBLOB_USE_EMULATOR
        • 
+
      • Type: bool
        • 
+
      • Default: false
        • 
+
      
 
      --azureblob-endpoint
      
 
      Endpoint for the service.
      
 
      Leave blank normally.
      
@@ -23491,6 +24015,16 @@ 
      --azureblob-public-access
      
 
 
 
+
      --azureblob-no-check-container
      
+
      If set, don't attempt to check the container exists or create it.
      
+
      This can be useful when trying to minimise the number of transactions rclone does if you know the container exists already.
      
+
      Properties:
      
+
        
+
      • Config: no_check_container
        • 
+
      • Env Var: RCLONE_AZUREBLOB_NO_CHECK_CONTAINER
        • 
+
      • Type: bool
        • 
+
      • Default: false
        • 
+
      
 
      --azureblob-no-head-object
      
 
      If set, do not do HEAD before GET when getting objects.
      
 
      Properties:
      
@@ -23500,14 +24034,24 @@ 
      --azureblob-no-head-object
      
 
    98. Type: bool
      99. 
 
    99. Default: false
      100. 
 
+
      Custom upload headers
      
+
      You can set custom upload headers with the --header-upload flag.
      
+
        
+
      • Cache-Control
        • 
+
      • Content-Disposition
        • 
+
      • Content-Encoding
        • 
+
      • Content-Language
        • 
+
      • Content-Type
        • 
+
      
+
      Eg --header-upload "Content-Type: text/potato"
      
 
      Limitations
      
 
      MD5 sums are only uploaded with chunked files if the source has an MD5 sum. This will always be the case for a local to azure copy.
      
 
      rclone about is not supported by the Microsoft Azure Blob storage backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote.
      
 
      See List of backends that do not support rclone about and rclone about
      
 
      Azure Storage Emulator Support
      
-
      You can run rclone with storage emulator (usually azurite).
      
-
      To do this, just set up a new remote with rclone config following instructions described in introduction and set use_emulator config as true. You do not need to provide default account name neither an account key.
      
-
      Also, if you want to access a storage emulator instance running on a different machine, you can override Endpoint parameter in advanced settings, setting it to http(s)://<host>:<port>/devstoreaccount1 (e.g. http://10.254.2.5:10000/devstoreaccount1).
      
+
      You can run rclone with the storage emulator (usually azurite).
      
+
      To do this, just set up a new remote with rclone config following the instructions in the introduction and set use_emulator in the advanced settings as true. You do not need to provide a default account name nor an account key. But you can override them in the account and key options. (Prior to v1.61 they were hard coded to azurite's devstoreaccount1.)
      
+
      Also, if you want to access a storage emulator instance running on a different machine, you can override the endpoint parameter in the advanced settings, setting it to http(s)://<host>:<port>/devstoreaccount1 (e.g. http://10.254.2.5:10000/devstoreaccount1).
      
 
      Microsoft OneDrive
      
 
      Paths are specified as remote:path
      
 
      Paths may be as deep as required, e.g. remote:directory/subdirectory.
      
@@ -23546,9 +24090,10 @@ 
      Configuration
      
 n) No
 y/n> n
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
@@ -24059,6 +24604,29 @@ 
      invalid_grant (AADSTS50076)
      
 
      If you see the error above after enabling multi-factor authentication for your account, you can fix it by refreshing your OAuth refresh token. To do that, run rclone config, and choose to edit your OneDrive backend. Then, you don't need to actually make any changes until you reach this question: Already have a token - refresh?. For this question, answer y and go through the process to refresh your token, just like the first time the backend is configured. After this, rclone should work again for this backend.
      
 
 
      On Sharepoint and OneDrive for Business, rclone link may return an "Invalid request" error. A possible cause is that the organisation admin didn't allow public links to be made for the organisation/sharepoint library. To fix the permissions as an admin, take a look at the docs: 1, 2.
      
+
      Can not access Shared with me files
      
+
      Shared with me files is not supported by rclone currently, but there is a workaround:
      
+
        
+
      1. Visit https://onedrive.live.com
        2. 
+
      2. Right click a item in Shared, then click Add shortcut to My files in the context
        
+
        
+
        Screenshot (Shared with me)
        
+
        
+
        make_shortcut
        
+
        
+
        3. 
+
      3. The shortcut will appear in My files, you can access it with rclone, it behaves like a normal folder/file.
        
+
        
+
        Screenshot (My Files)
        
+
        
+
        in_my_files
        
+
        
+
        4. 
+
      
+
      
+
      Screenshot (rclone mount)
      
+rclone_mount
+
      
 
      OpenDrive
      
 
      Paths are specified as remote:path
      
 
      Paths may be as deep as required, e.g. remote:directory/subdirectory.
      
@@ -24707,7 +25275,7 @@ 
      Multipart uploads
      
 
      Note that incomplete multipart uploads older than 24 hours can be removed with rclone cleanup remote:bucket just for one bucket rclone cleanup remote: for all buckets. QingStor does not ever remove incomplete multipart uploads so it may be necessary to run this from time to time.
      
 
      Buckets and Zone
      
 
      With QingStor you can list buckets (rclone lsd) using any zone, but you can only access the content of a bucket from the zone it was created in. If you attempt to access a bucket from the wrong zone, you will get an error, incorrect zone, the bucket is not in 'XXX' zone.
      
-
      Authentication
      
+
      Authentication
      
 
      There are two ways to supply rclone with a set of QingStor credentials. In order of precedence:
      
 
        
 
      • Directly in the rclone configuration file (as configured by rclone config)
@@ -25493,9 +26061,10 @@ 
        Configuration
        
 Pcloud App Client Secret - leave blank normally.
 client_secret> 
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
@@ -25692,9 +26261,10 @@ 
        Configuration
        
 ** See help for premiumizeme backend at: https://rclone.org/premiumizeme/ **
 
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
@@ -25799,9 +26369,10 @@ 
        Configuration
        
 ** See help for putio backend at: https://rclone.org/putio/ **
 
 Remote config
-Use auto config?
- * Say Y if not sure
- * Say N if you are working on a remote or headless machine
+Use web browser to automatically authenticate rclone with remote?
+ * Say Y if the machine running rclone has a web browser you can use
+ * Say N if running rclone on a (remote) machine without web browser access
+If not sure try Y. If Y failed, try N.
 y) Yes
 n) No
 y/n> y
@@ -25832,7 +26403,8 @@ 
        Configuration
        
 s) Set configuration password
 q) Quit config
 e/n/d/r/c/s/q> q
      -

      Note that rclone runs a webserver on your local machine to collect the token as returned from Google if you use auto config mode. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall, or use manual mode.

      +

      See the remote setup docs for how to set it up on a machine with no Internet browser available.

      +

      Note that rclone runs a webserver on your local machine to collect the token as returned from put.io if using web browser to automatically authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on http://127.0.0.1:53682/ and this it may require you to unblock it temporarily if you are running a host firewall, or use manual mode.

      You can then use it like this,

      List directories in top level of your put.io

      rclone lsd remote:
      @@ -26432,6 +27004,7 @@

      --sftp-use-insecure-cipher

    100. diffie-hellman-group-exchange-sha1

      101. Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.

      +

      This must be false if you use either ciphers or key_exchange advanced options.

      Properties:

      • Config: use_insecure_cipher
        • @@ -26674,6 +27247,44 @@

        --sftp-set-env

      • Type: SpaceSepList
      • Default:
      +

      --sftp-ciphers

      +

      Space separated list of ciphers to be used for session encryption, ordered by preference.

      +

      At least one must match with server configuration. This can be checked for example using ssh -Q cipher.

      +

      This must not be set if use_insecure_cipher is true.

      +

      Example:

      +
      aes128-ctr aes192-ctr aes256-ctr aes128-gcm@openssh.com aes256-gcm@openssh.com
      +

      Properties:

      +
        +
      • Config: ciphers
        • +
      • Env Var: RCLONE_SFTP_CIPHERS
        • +
      • Type: SpaceSepList
        • +
      • Default:
        • +
      +

      --sftp-key-exchange

      +

      Space separated list of key exchange algorithms, ordered by preference.

      +

      At least one must match with server configuration. This can be checked for example using ssh -Q kex.

      +

      This must not be set if use_insecure_cipher is true.

      +

      Example:

      +
      sntrup761x25519-sha512@openssh.com curve25519-sha256 curve25519-sha256@libssh.org ecdh-sha2-nistp256
      +

      Properties:

      +
        +
      • Config: key_exchange
        • +
      • Env Var: RCLONE_SFTP_KEY_EXCHANGE
        • +
      • Type: SpaceSepList
        • +
      • Default:
        • +
      +

      --sftp-macs

      +

      Space separated list of MACs (message authentication code) algorithms, ordered by preference.

      +

      At least one must match with server configuration. This can be checked for example using ssh -Q mac.

      +

      Example:

      +
      umac-64-etm@openssh.com umac-128-etm@openssh.com hmac-sha2-256-etm@openssh.com
      +

      Properties:

      +
        +
      • Config: macs
        • +
      • Env Var: RCLONE_SFTP_MACS
        • +
      • Type: SpaceSepList
        • +
      • Default:
        • +

      Limitations

      On some SFTP servers (e.g. Synology) the paths are different for SSH and SFTP so the hashes can't be calculated properly. For them using disable_hashcheck is a good idea.

      The only ssh agent supported under Windows is Putty's pageant.

      @@ -28029,9 +28640,10 @@

      Configuration

      Yandex Client Secret - leave blank normally. client_secret> Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -28182,9 +28794,10 @@

      Configuration

      n) No (default) y/n> n Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes (default) n) No y/n> @@ -28967,6 +29580,193 @@

      noop

    101. "error": return an error based on option value

      102. Changelog

      +

      v1.61.0 - 2022-12-20

      +

      See commits

      +
        +
      • New backends +
          +
        • New S3 providers +
          • +
        • +
      • New Features +
          +
        • build: Add vulnerability testing using govulncheck (albertony)
          • +
        • cmd: Enable SIGINFO (Ctrl-T) handler on FreeBSD, NetBSD, OpenBSD and Dragonfly BSD (x3-apptech)
          • +
        • config: Add config/setpath for setting config path via rc/librclone (Nick Craig-Wood)
          • +
        • dedupe +
            +
          • Count Checks in the stats while scanning for duplicates (Nick Craig-Wood)
            • +
          • Make dedupe obey the filters (Nick Craig-Wood)
            • +
          • +
        • dlna: Properly attribute code used from https://github.com/anacrolix/dms (Nick Craig-Wood)
          • +
        • docs +
            +
          • Add minimum versions and status badges to backend and command docs (Nick Craig-Wood, albertony)
            • +
          • Remote names may not start or end with space (albertony)
            • +
          • +
        • filter: Add metadata filters --metadata-include/exclude/filter and friends (Nick Craig-Wood)
          • +
        • fs +
            +
          • Make all duration flags take y, M, w, d etc suffixes (Nick Craig-Wood)
            • +
          • Add global flag --color to control terminal colors (Kevin Verstaen)
            • +
          • +
        • fspath: Allow unicode numbers and letters in remote names (albertony)
          • +
        • lib/file: Improve error message for creating dir on non-existent network host on windows (albertony)
          • +
        • lib/http: Finish port of rclone servers to lib/http (Tom Mombourquette, Nick Craig-Wood)
          • +
        • lib/oauthutil: Improved usability of config flows needing web browser (Ole Frost)
          • +
        • ncdu +
            +
          • Add support for modification time (albertony)
            • +
          • Fallback to sort by name also for sort by average size (albertony)
            • +
          • Rework to use tcell directly instead of the termbox wrapper (eNV25)
            • +
          • +
        • rc: Add commands to set GC Percent & Memory Limit (go 1.19+) (Anagh Kumar Baranwal)
          • +
        • rcat: Preserve metadata when Copy falls back to Rcat (Nick Craig-Wood)
          • +
        • rcd: Refactor rclone rc server to use lib/http (Nick Craig-Wood)
          • +
        • rcserver: Avoid generating default credentials with htpasswd (Kamui)
          • +
        • restic: Refactor to use lib/http (Nolan Woods)
          • +
        • serve http: Support unix sockets and multiple listeners (Tom Mombourquette)
          • +
        • serve webdav: Refactor to use lib/http (Nick Craig-Wood)
          • +
        • test: Replace defer cleanup with t.Cleanup (Eng Zer Jun)
          • +
        • test memory: Read metadata if -M flag is specified (Nick Craig-Wood)
          • +
        • wasm: Comply with wasm_exec.js licence terms (Matthew Vernon)
          • +
        • +
      • Bug Fixes +
          +
        • build: Update golang.org/x/net/http2 to fix GO-2022-1144 (Nick Craig-Wood)
          • +
        • restic: Fix typo in docs 'remove' should be 'remote' (asdffdsazqqq)
          • +
        • serve dlna: Fix panic: Logger uninitialized. (Nick Craig-Wood)
          • +
        • +
      • Mount +
          +
        • Update cgofuse for FUSE-T support for mounting volumes on Mac (Nick Craig-Wood)
          • +
        • +
      • VFS +
          +
        • Windows: fix slow opening of exe files by not truncating files when not necessary (Nick Craig-Wood)
          • +
        • Fix IO Error opening a file with O_CREATE|O_RDONLY in --vfs-cache-mode not full (Nick Craig-Wood)
          • +
        • +
      • Crypt +
          +
        • Fix compress wrapping crypt giving upload errors (Nick Craig-Wood)
          • +
        • +
      • Azure Blob +
          +
        • Port to new SDK (Nick Craig-Wood) +
            +
          • Revamp authentication to include all methods and docs (Nick Craig-Wood)
            • +
          • Port old authentication methods to new SDK (Nick Craig-Wood, Brad Ackerman)
            • +
          • Thanks to Stonebranch for sponsoring this work.
            • +
          • +
        • Add --azureblob-no-check-container to assume container exists (Nick Craig-Wood)
          • +
        • Add --use-server-modtime support (Abdullah Saglam)
          • +
        • Add support for custom upload headers (rkettelerij)
          • +
        • Allow emulator account/key override (Roel Arents)
          • +
        • Support simple "environment credentials" (Nathaniel Wesley Filardo)
          • +
        • Ignore AuthorizationFailure when trying to create a create a container (Nick Craig-Wood)
          • +
        • +
      • Box +
          +
        • Added note on Box API rate limits (Ole Frost)
          • +
        • +
      • Drive +
          +
        • Handle shared drives with leading/trailing space in name (related to) (albertony)
          • +
        • +
      • FTP +
          +
        • Update help text of implicit/explicit TLS options to refer to FTPS instead of FTP (ycdtosa)
          • +
        • Improve performance to speed up --files-from and NewObject (Anthony Pessy)
          • +
        • +
      • HTTP +
          +
        • Parse GET responses when no_head is set (Arnie97)
          • +
        • Do not update object size based on Range requests (Arnie97)
          • +
        • Support Content-Range response header (Arnie97)
          • +
        • +
      • Onedrive +
          +
        • Document workaround for shared with me files (vanplus)
          • +
        • +
      • S3 +
          +
        • Add Liara LOS to provider list (MohammadReza)
          • +
        • Add DigitalOcean Spaces regions sfo3, fra1, syd1 (Jack)
          • +
        • Avoid privileged GetBucketLocation to resolve s3 region (Anthony Pessy)
          • +
        • Stop setting object and bucket ACL to private if it is an empty string (Philip Harvey)
          • +
        • If bucket or object ACL is empty string then don't add X-Amz-Acl: header (Nick Craig-Wood)
          • +
        • Reduce memory consumption for s3 objects (Erik Agterdenbos)
          • +
        • Fix listing loop when using v2 listing on v1 server (Nick Craig-Wood)
          • +
        • Fix nil pointer exception when using Versions (Nick Craig-Wood)
          • +
        • Fix excess memory usage when using versions (Nick Craig-Wood)
          • +
        • Ignore versionIDs from uploads unless using --s3-versions or --s3-versions-at (Nick Craig-Wood)
          • +
        • +
      • SFTP +
          +
        • Add configuration options to set ssh Ciphers / MACs / KeyExchange (dgouju)
          • +
        • Auto-detect shell type for fish (albertony)
          • +
        • Fix NewObject with leading / (Nick Craig-Wood)
          • +
        • +
      • Smb +
          +
        • Fix issue where spurious dot directory is created (albertony)
          • +
        • +
      • Storj +
          +
        • Implement server side Copy (Kaloyan Raev)
          • +
        • +
      +

      v1.60.1 - 2022-11-17

      +

      See commits

      +
        +
      • Bug Fixes +
          +
        • lib/cache: Fix alias backend shutting down too soon (Nick Craig-Wood)
          • +
        • wasm: Fix walltime link error by adding up-to-date wasm_exec.js (João Henrique Franco)
          • +
        • docs +
            +
          • Update faq.md with bisync (Samuel Johnson)
            • +
          • Corrected download links in windows install docs (coultonluke)
            • +
          • Add direct download link for windows arm64 (albertony)
            • +
          • Remove link to rclone slack as it is no longer supported (Nick Craig-Wood)
            • +
          • Faq: how to use a proxy server that requires a username and password (asdffdsazqqq)
            • +
          • Oracle-object-storage: doc fix (Manoj Ghosh)
            • +
          • Fix typo remove in rclone_serve_restic command (Joda Stößer)
            • +
          • Fix character that was incorrectly interpreted as markdown (Clément Notin)
            • +
          • +
        • +
      • VFS +
          +
        • Fix deadlock caused by cache cleaner and upload finishing (Nick Craig-Wood)
          • +
        • +
      • Local +
          +
        • Clean absolute paths (albertony)
          • +
        • Fix -L/--copy-links with filters missing directories (Nick Craig-Wood)
          • +
        • +
      • Mailru +
          +
        • Note that an app password is now needed (Nick Craig-Wood)
          • +
        • Allow timestamps to be before the epoch 1970-01-01 (Nick Craig-Wood)
          • +
        • +
      • S3 +
          +
        • Add provider quirk --s3-might-gzip to fix corrupted on transfer: sizes differ (Nick Craig-Wood)
          • +
        • Allow Storj to server side copy since it seems to work now (Nick Craig-Wood)
          • +
        • Fix for unchecked err value in s3 listv2 (Aaron Gokaslan)
          • +
        • Add additional Wasabi locations (techknowlogick)
          • +
        • +
      • Smb +
          +
        • Fix Failed to sync: context canceled at the end of syncs (Nick Craig-Wood)
          • +
        • +
      • WebDAV +
          +
        • Fix Move/Copy/DirMove when using -server-side-across-configs (Nick Craig-Wood)
          • +
        • +

      v1.60.0 - 2022-10-21

      See commits

      Contact the rclone project

      Forum

      diff --git a/MANUAL.md b/MANUAL.md index c7030525ba61d..59c2cef308626 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -1,6 +1,6 @@ % rclone(1) User Manual % Nick Craig-Wood -% Oct 21, 2022 +% Dec 20, 2022 # Rclone syncs your files to cloud storage @@ -132,6 +132,7 @@ WebDAV or S3, that work out of the box.) - IDrive e2 - IONOS Cloud - Koofr +- Liara Object Storage - Mail.ru Cloud - Memset Memstore - Mega @@ -315,9 +316,9 @@ The simplest fix is to run Fetch the correct binary for your processor type by clicking on these links. If not sure, use the first link. -- [Intel/AMD - 64 Bit](https://downloads.rclone.org/rclone-current-linux-amd64.zip) -- [Intel/AMD - 32 Bit](https://downloads.rclone.org/rclone-current-linux-386.zip) -- [ARM - 64 Bit](https://downloads.rclone.org/rclone-current-linux-arm64.zip) +- [Intel/AMD - 64 Bit](https://downloads.rclone.org/rclone-current-windows-amd64.zip) +- [Intel/AMD - 32 Bit](https://downloads.rclone.org/rclone-current-windows-386.zip) +- [ARM - 64 Bit](https://downloads.rclone.org/rclone-current-windows-arm64.zip) Open this file in the Explorer and extract `rclone.exe`. Rclone is a portable executable so you can place it wherever is convenient. @@ -2365,7 +2366,7 @@ This will look something like (some irrelevant detail removed): "State": "*oauth-islocal,teamdrive,,", "Option": { "Name": "config_is_local", - "Help": "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n", + "Help": "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n", "Default": true, "Examples": [ { @@ -2760,7 +2761,7 @@ This will look something like (some irrelevant detail removed): "State": "*oauth-islocal,teamdrive,,", "Option": { "Name": "config_is_local", - "Help": "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n", + "Help": "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n", "Default": true, "Examples": [ { @@ -3689,6 +3690,7 @@ rclone lsjson remote:path [flags] --hash Include hashes in the output (may take longer) --hash-type stringArray Show only this hash type (may be repeated) -h, --help help for lsjson + -M, --metadata Add metadata to the listing --no-mimetype Don't read the mime type (can speed things up) --no-modtime Don't read the modification time (can speed things up) --original Show the ID of the underlying Object @@ -4419,14 +4421,14 @@ rclone mount remote:path /path/to/mountpoint [flags] --allow-other Allow access to other users (not supported on Windows) --allow-root Allow access to root user (not supported on Windows) --async-read Use asynchronous reads (not supported on Windows) (default true) - --attr-timeout duration Time for which file/directory attributes are cached (default 1s) + --attr-timeout Duration Time for which file/directory attributes are cached (default 1s) --daemon Run mount in background and exit parent process (as background output is suppressed, use --log-file with --log-format=pid,... to monitor) (not supported on Windows) - --daemon-timeout duration Time limit for rclone to respond to kernel (not supported on Windows) - --daemon-wait duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) + --daemon-timeout Duration Time limit for rclone to respond to kernel (not supported on Windows) (default 0s) + --daemon-wait Duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) --debug-fuse Debug the FUSE internals - needs -v --default-permissions Makes kernel enforce access control based on the file mode (not supported on Windows) --devname string Set the device name - default is remote:path - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --fuse-flag stringArray Flags or arguments to be passed direct to libfuse/WinFsp (repeat if required) @@ -4440,24 +4442,24 @@ rclone mount remote:path /path/to/mountpoint [flags] --noappledouble Ignore Apple Double (._) and .DS_Store files (supported on OSX only) (default true) --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) --volname string Set the volume name (supported on Windows and OSX only) --write-back-cache Makes kernel buffer writes before sending them to rclone (without this, writethrough caching is used) (not supported on Windows) ``` @@ -4547,11 +4549,12 @@ press '?' to toggle the help on and off. The supported keys are: ↑,↓ or k,j to Move →,l to enter ←,h to return - c toggle counts g toggle graph + c toggle counts a toggle average size in directory + m toggle modified time u toggle human-readable format - n,s,C,A sort by name,size,count,average size + n,s,C,A,M sort by name,size,count,asize,mtime d delete file/directory v select file/directory V enter visual select mode @@ -4803,6 +4806,101 @@ the browser when rclone is run. See the [rc documentation](https://rclone.org/rc/) for more info on the rc flags. +## Server options + +Use `--addr` to specify which IP address and port the server should +listen on, eg `--addr 1.2.3.4:8000` or `--addr :8080` to listen to all +IPs. By default it only listens on localhost. You can use port +:0 to let the OS choose an available port. + +If you set `--addr` to listen on a public or LAN accessible IP address +then using Authentication is advised - see the next section for info. + +You can use a unix socket by setting the url to `unix:///path/to/socket` +or just by using an absolute path name. Note that unix sockets bypass the +authentication - this is expected to be done with file system permissions. + +`--addr` may be repeated to listen on multiple IPs/ports/sockets. + +`--server-read-timeout` and `--server-write-timeout` can be used to +control the timeouts on the server. Note that this is the total time +for a transfer. + +`--max-header-bytes` controls the maximum number of bytes the server will +accept in the HTTP header. + +`--baseurl` controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used `--baseurl "/rclone"` then +rclone would serve from a URL starting with "/rclone/". This is +useful if you wish to proxy rclone serve. Rclone automatically +inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, +`--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated +identically. + +### TLS (SSL) + +By default this will serve over http. If you want you can serve over +https. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. + +`--cert` should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client +certificate authority certificate. + +--min-tls-version is minimum TLS version that is acceptable. Valid + values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default + "tls1.0"). + +### Template + +`--template` allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup +to be used within the template to server pages: + +| Parameter | Description | +| :---------- | :---------- | +| .Name | The full path of a file/directory. | +| .Title | Directory listing of .Name | +| .Sort | The current sort used. This is changeable via ?sort= parameter | +| | Sort Options: namedirfirst,name,size,time (default namedirfirst) | +| .Order | The current ordering used. This is changeable via ?order= parameter | +| | Order Options: asc,desc (default asc) | +| .Query | Currently unused. | +| .Breadcrumb | Allows for creating a relative navigation | +|-- .Link | The relative to the root link of the Text. | +|-- .Text | The Name of the directory. | +| .Entries | Information about a specific file/directory. | +|-- .URL | The 'url' of an entry. | +|-- .Leaf | Currently same as 'URL' but intended to be 'just' the name. | +|-- .IsDir | Boolean for if an entry is a directory or not. | +|-- .Size | Size in Bytes of the entry. | +|-- .ModTime | The UTC timestamp of an entry. | + +### Authentication + +By default this will serve files without needing a login. + +You can either use an htpasswd file which can take lots of users, or +set a single username and password with the `--user` and `--pass` flags. + +Use `--htpasswd /path/to/htpasswd` to provide an htpasswd file. This is +in standard apache format and supports MD5, SHA1 and BCrypt for basic +authentication. Bcrypt is recommended. + +To create an htpasswd file: + + touch htpasswd + htpasswd -B htpasswd user + htpasswd -B htpasswd anotherUser + +The password file can be updated while rclone is running. + +Use `--realm` to set the authentication realm. + +Use `--salt` to change the password hashing salt from the default. + ``` rclone rcd * [flags] @@ -5332,8 +5430,8 @@ rclone serve dlna remote:path [flags] ``` --addr string The ip:port or :port to bind the DLNA http server to (default ":7879") - --announce-interval duration The interval between SSDP announcements (default 12m0s) - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --announce-interval Duration The interval between SSDP announcements (default 12m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -5344,24 +5442,24 @@ rclone serve dlna remote:path [flags] --no-checksum Don't compare checksums on up/download --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) ``` See the [global flags page](https://rclone.org/flags/) for global options not listed here. @@ -5746,15 +5844,15 @@ rclone serve docker [flags] --allow-other Allow access to other users (not supported on Windows) --allow-root Allow access to root user (not supported on Windows) --async-read Use asynchronous reads (not supported on Windows) (default true) - --attr-timeout duration Time for which file/directory attributes are cached (default 1s) + --attr-timeout Duration Time for which file/directory attributes are cached (default 1s) --base-dir string Base directory for volumes (default "/var/lib/docker-volumes/rclone") --daemon Run mount in background and exit parent process (as background output is suppressed, use --log-file with --log-format=pid,... to monitor) (not supported on Windows) - --daemon-timeout duration Time limit for rclone to respond to kernel (not supported on Windows) - --daemon-wait duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) + --daemon-timeout Duration Time limit for rclone to respond to kernel (not supported on Windows) (default 0s) + --daemon-wait Duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) --debug-fuse Debug the FUSE internals - needs -v --default-permissions Makes kernel enforce access control based on the file mode (not supported on Windows) --devname string Set the device name - default is remote:path - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --forget-state Skip restoring previous state @@ -5770,26 +5868,26 @@ rclone serve docker [flags] --noappledouble Ignore Apple Double (._) and .DS_Store files (supported on OSX only) (default true) --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --socket-addr string Address or absolute path (default: /run/docker/plugins/rclone.sock) --socket-gid int GID for unix socket (default: current process GID) (default 1000) --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) --volname string Set the volume name (supported on Windows and OSX only) --write-back-cache Makes kernel buffer writes before sending them to rclone (without this, writethrough caching is used) (not supported on Windows) ``` @@ -6237,7 +6335,7 @@ rclone serve ftp remote:path [flags] --addr string IPaddress:Port or :Port to bind server to (default "localhost:2121") --auth-proxy string A program to use to create the backend from the auth --cert string TLS PEM key (concatenation of certificate and CA certificate) - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -6248,26 +6346,26 @@ rclone serve ftp remote:path [flags] --no-seek Don't allow seeking in files --pass string Password for authentication (empty value allow every password) --passive-port string Passive port range to use (default "30000-32000") - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --public-ip string Public IP address to advertise for passive connections --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication (default "anonymous") - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) ``` See the [global flags page](https://rclone.org/flags/) for global options not listed here. @@ -6304,6 +6402,12 @@ IPs. By default it only listens on localhost. You can use port If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. +You can use a unix socket by setting the url to `unix:///path/to/socket` +or just by using an absolute path name. Note that unix sockets bypass the +authentication - this is expected to be done with file system permissions. + +`--addr` may be repeated to listen on multiple IPs/ports/sockets. + `--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -6319,7 +6423,7 @@ inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, `--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. -### SSL/TLS +### TLS (SSL) By default this will serve over http. If you want you can serve over https. You will need to supply the `--cert` and `--key` flags. @@ -6709,47 +6813,47 @@ rclone serve http remote:path [flags] ## Options ``` - --addr string IPaddress:Port or :Port to bind server to (default "127.0.0.1:8080") + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --baseurl string Prefix for URLs - leave blank for root - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) -h, --help help for http --htpasswd string A htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --no-checksum Don't compare checksums on up/download --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --realm string Realm for authentication --salt string Password hashing salt (default "dlPL2MqE") - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --template string User-specified template --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) ``` See the [global flags page](https://rclone.org/flags/) for global options not listed here. @@ -6764,7 +6868,7 @@ Serve the remote for restic's REST API. ## Synopsis -Run a basic web server to serve a remove over restic's REST backend +Run a basic web server to serve a remote over restic's REST backend API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly. @@ -6849,13 +6953,19 @@ with a path of `//`. ## Server options Use `--addr` to specify which IP address and port the server should -listen on, e.g. `--addr 1.2.3.4:8000` or `--addr :8080` to -listen to all IPs. By default it only listens on localhost. You can use port +listen on, eg `--addr 1.2.3.4:8000` or `--addr :8080` to listen to all +IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. +You can use a unix socket by setting the url to `unix:///path/to/socket` +or just by using an absolute path name. Note that unix sockets bypass the +authentication - this is expected to be done with file system permissions. + +`--addr` may be repeated to listen on multiple IPs/ports/sockets. + `--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -6871,28 +6981,21 @@ inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, `--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. -`--template` allows a user to specify a custom markup template for HTTP -and WebDAV serve functions. The server exports the following markup -to be used within the template to server pages: +### TLS (SSL) -| Parameter | Description | -| :---------- | :---------- | -| .Name | The full path of a file/directory. | -| .Title | Directory listing of .Name | -| .Sort | The current sort used. This is changeable via ?sort= parameter | -| | Sort Options: namedirfirst,name,size,time (default namedirfirst) | -| .Order | The current ordering used. This is changeable via ?order= parameter | -| | Order Options: asc,desc (default asc) | -| .Query | Currently unused. | -| .Breadcrumb | Allows for creating a relative navigation | -|-- .Link | The relative to the root link of the Text. | -|-- .Text | The Name of the directory. | -| .Entries | Information about a specific file/directory. | -|-- .URL | The 'url' of an entry. | -|-- .Leaf | Currently same as 'URL' but intended to be 'just' the name. | -|-- .IsDir | Boolean for if an entry is a directory or not. | -|-- .Size | Size in Bytes of the entry. | -|-- .ModTime | The UTC timestamp of an entry. | +By default this will serve over http. If you want you can serve over +https. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. + +`--cert` should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client +certificate authority certificate. + +--min-tls-version is minimum TLS version that is acceptable. Valid + values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default + "tls1.0"). ### Authentication @@ -6915,21 +7018,7 @@ The password file can be updated while rclone is running. Use `--realm` to set the authentication realm. -### SSL/TLS - -By default this will serve over HTTP. If you want you can serve over -HTTPS. You will need to supply the `--cert` and `--key` flags. -If you wish to do client side certificate validation then you will need to -supply `--client-ca` also. - -`--cert` should be either a PEM encoded certificate or a concatenation -of that with the CA certificate. `--key` should be the PEM encoded -private key and `--client-ca` should be the PEM encoded client -certificate authority certificate. - ---min-tls-version is minimum TLS version that is acceptable. Valid - values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default - "tls1.0"). +Use `--salt` to change the password hashing salt from the default. ``` @@ -6939,24 +7028,24 @@ rclone serve restic remote:path [flags] ## Options ``` - --addr string IPaddress:Port or :Port to bind server to (default "localhost:8080") + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --append-only Disallow deletion of repository data --baseurl string Prefix for URLs - leave blank for root --cache-objects Cache listed objects (default true) - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with -h, --help help for restic - --htpasswd string htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --htpasswd string A htpasswd file - if not provided no authentication is done + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --pass string Password for authentication --private-repos Users can only access their private repo - --realm string Realm for authentication (default "rclone") - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --realm string Realm for authentication + --salt string Password hashing salt (default "dlPL2MqE") + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --stdio Run an HTTP2 server on stdin/stdout - --template string User-specified template --user string User name for authentication ``` @@ -7435,7 +7524,7 @@ rclone serve sftp remote:path [flags] --addr string IPaddress:Port or :Port to bind server to (default "localhost:2022") --auth-proxy string A program to use to create the backend from the auth --authorized-keys string Authorized keys file (default "~/.ssh/authorized_keys") - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -7446,26 +7535,26 @@ rclone serve sftp remote:path [flags] --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --stdio Run an sftp server on stdin/stdout --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) ``` See the [global flags page](https://rclone.org/flags/) for global options not listed here. @@ -7500,13 +7589,19 @@ to see the full list. ## Server options Use `--addr` to specify which IP address and port the server should -listen on, e.g. `--addr 1.2.3.4:8000` or `--addr :8080` to -listen to all IPs. By default it only listens on localhost. You can use port +listen on, eg `--addr 1.2.3.4:8000` or `--addr :8080` to listen to all +IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. +You can use a unix socket by setting the url to `unix:///path/to/socket` +or just by using an absolute path name. Note that unix sockets bypass the +authentication - this is expected to be done with file system permissions. + +`--addr` may be repeated to listen on multiple IPs/ports/sockets. + `--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -7522,6 +7617,24 @@ inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, `--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. +### TLS (SSL) + +By default this will serve over http. If you want you can serve over +https. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. + +`--cert` should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client +certificate authority certificate. + +--min-tls-version is minimum TLS version that is acceptable. Valid + values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default + "tls1.0"). + +### Template + `--template` allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: @@ -7566,21 +7679,7 @@ The password file can be updated while rclone is running. Use `--realm` to set the authentication realm. -### SSL/TLS - -By default this will serve over HTTP. If you want you can serve over -HTTPS. You will need to supply the `--cert` and `--key` flags. -If you wish to do client side certificate validation then you will need to -supply `--client-ca` also. - -`--cert` should be either a PEM encoded certificate or a concatenation -of that with the CA certificate. `--key` should be the PEM encoded -private key and `--client-ca` should be the PEM encoded client -certificate authority certificate. - ---min-tls-version is minimum TLS version that is acceptable. Valid - values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default - "tls1.0"). +Use `--salt` to change the password hashing salt from the default. ## VFS - Virtual File System @@ -7989,49 +8088,50 @@ rclone serve webdav remote:path [flags] ## Options ``` - --addr string IPaddress:Port or :Port to bind server to (default "localhost:8080") + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --auth-proxy string A program to use to create the backend from the auth --baseurl string Prefix for URLs - leave blank for root - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --disable-dir-list Disable HTML directory list on GET request for a directory --etag-hash string Which hash to use for the ETag, or auto or blank for off --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) -h, --help help for webdav - --htpasswd string htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --htpasswd string A htpasswd file - if not provided no authentication is done + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --no-checksum Don't compare checksums on up/download --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access - --realm string Realm for authentication (default "rclone") - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --realm string Realm for authentication + --salt string Password hashing salt (default "dlPL2MqE") + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --template string User-specified template --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) ``` See the [global flags page](https://rclone.org/flags/) for global options not listed here. @@ -8134,7 +8234,7 @@ rclone test changenotify remote: [flags] ``` -h, --help help for changenotify - --poll-interval duration Time to wait between polling for changes (default 10s) + --poll-interval Duration Time to wait between polling for changes (default 10s) ``` See the [global flags page](https://rclone.org/flags/) for global options not listed here. @@ -8199,7 +8299,7 @@ rclone test info [remote:path]+ [flags] --check-normalization Check UTF-8 Normalization --check-streaming Check uploads with indeterminate file size -h, --help help for info - --upload-wait duration Wait after writing a file + --upload-wait Duration Wait after writing a file (default 0s) --write-json string Write results to file ``` @@ -8251,6 +8351,7 @@ rclone test makefiles [flags] --files int Number of files to create (default 1000) --files-per-directory int Average number of files per directory (default 10) -h, --help help for makefiles + --max-depth int Maximum depth of directory hierarchy (default 10) --max-file-size SizeSuffix Maximum size of files to create (default 100) --max-name-length int Maximum size of file names (default 12) --min-file-size SizeSuffix Minimum size of file to create @@ -8378,7 +8479,6 @@ rclone tree remote:path [flags] ``` -a, --all All files are listed (list . files too) - -C, --color Turn colorization on always -d, --dirs-only List directories only --dirsfirst List directories before files (-U disables) --full-path Print the full path prefix for each file @@ -8602,8 +8702,17 @@ Will get their own names ### Valid remote names Remote names are case sensitive, and must adhere to the following rules: - - May only contain `0`-`9`, `A`-`Z`, `a`-`z`, `_`, `-`, `.` and space. + - May contain number, letter, `_`, `-`, `.` and space. - May not start with `-` or space. + - May not end with space. + +Starting with rclone version 1.61, any Unicode numbers and letters are allowed, +while in older versions it was limited to plain ASCII (0-9, A-Z, a-z). If you use +the same rclone configuration from different shells, which may be configured with +different character encoding, you must be cautious to use characters that are +possible to write in all of them. This is mostly a problem on Windows, where +the console traditionally uses a non-Unicode character set - defined +by the so-called "code page". Quoting and the shell --------------------- @@ -9087,6 +9196,16 @@ quicker than without the `--checksum` flag. When using this flag, rclone won't update mtimes of remote files if they are incorrect as it would normally. +### --color WHEN ### + +Specifiy when colors (and other ANSI codes) should be added to the output. + +`AUTO` (default) only allows ANSI codes when the output is a terminal + +`NEVER` never allow ANSI codes + +`ALWAYS` always add ANSI codes, regardless of the output format (terminal or file) + ### --compare-dest=DIR ### When using `sync`, `copy` or `move` DIR is checked in addition to the @@ -10600,6 +10719,12 @@ For the filtering options * `--min-age` * `--max-age` * `--dump filters` + * `--metadata-include` + * `--metadata-include-from` + * `--metadata-exclude` + * `--metadata-exclude-from` + * `--metadata-filter` + * `--metadata-filter-from` See the [filtering section](https://rclone.org/filtering/). @@ -10805,15 +10930,16 @@ two ways of doing it, described below. ## Configuring using rclone authorize ## -On the headless box run `rclone` config but answer `N` to the `Use -auto config?` question. +On the headless box run `rclone` config but answer `N` to the `Use web browser +to automatically authenticate?` question. ``` ... Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes (default) n) No y/n> n @@ -10889,15 +11015,16 @@ Linux and MacOS users can utilize SSH Tunnel to redirect the headless box port 5 ``` ssh -L localhost:53682:localhost:53682 username@remote_server ``` -Then on the headless box run `rclone` config and answer `Y` to the `Use -auto config?` question. +Then on the headless box run `rclone` config and answer `Y` to the `Use web +browser to automatically authenticate?` question. ``` ... Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes (default) n) No y/n> y @@ -10932,7 +11059,7 @@ you expect. Instead use a `--filter...` flag. ## Patterns for matching path/file names -### Pattern syntax +### Pattern syntax {#patterns} Here is a formal definition of the pattern syntax, [examples](#examples) are below. @@ -11094,7 +11221,7 @@ them into regular expressions. | Rooted Regexp | `/{{.*\.jpe?g}}` | `/file.jpeg` | `/file.png` | | | | `/file.jpg` | `/dir/file.jpg` | -## How filter rules are applied to files +## How filter rules are applied to files {#how-filter-rules-work} Rclone path/file name filters are made up of one or more of the following flags: @@ -11657,6 +11784,43 @@ E.g. for the following directory structure: The command `rclone ls --exclude-if-present .ignore dir1` does not list `dir3`, `file3` or `.ignore`. +## Metadata filters {#metadata} + +The metadata filters work in a very similar way to the normal file +name filters, except they match [metadata](https://rclone.org/docs/#metadata) on the +object. + +The metadata should be specified as `key=value` patterns. This may be +wildcarded using the normal [filter patterns](#patterns) or [regular +expressions](#regexp). + +For example if you wished to list only local files with a mode of +`100664` you could do that with: + + rclone lsf -M --files-only --metadata-include "mode=100664" . + +Or if you wished to show files with an `atime`, `mtime` or `btime` at a given date: + + rclone lsf -M --files-only --metadata-include "[abm]time=2022-12-16*" . + +Like file filtering, metadata filtering only applies to files not to +directories. + +The filters can be applied using these flags. + +- `--metadata-include` - Include metadatas matching pattern +- `--metadata-include-from` - Read metadata include patterns from file (use - to read from stdin) +- `--metadata-exclude` - Exclude metadatas matching pattern +- `--metadata-exclude-from` - Read metadata exclude patterns from file (use - to read from stdin) +- `--metadata-filter` - Add a metadata filtering rule +- `--metadata-filter-from` - Read metadata filtering patterns from a file (use - to read from stdin) + +Each flag can be repeated. See the section on [how filter rules are +applied](#how-filter-rules-work) for more details - these flags work +in an identical way to the file name filtering flags, but instead of +file name patterns have metadata patterns. + + ## Common pitfalls The most frequent filter support issues on @@ -12382,6 +12546,14 @@ See the [config providers](https://rclone.org/commands/rclone_config_providers/) **Authentication is required for this call.** +### config/setpath: Set the path of the config file {#config-setpath} + +Parameters: + +- path - path to the config file to use + +**Authentication is required for this call.** + ### config/update: update the config for a remote. {#config-update} This takes the following parameters: @@ -12480,7 +12652,7 @@ Returns: "result": "" } -OR +OR { "error": true, "result": "" @@ -12675,6 +12847,22 @@ Parameters: - rate - int +### debug/set-gc-percent: Call runtime/debug.SetGCPercent for setting the garbage collection target percentage. {#debug-set-gc-percent} + +SetGCPercent sets the garbage collection target percentage: a collection is triggered +when the ratio of freshly allocated data to live data remaining after the previous collection +reaches this percentage. SetGCPercent returns the previous setting. The initial setting is the +value of the GOGC environment variable at startup, or 100 if the variable is not set. + +This setting may be effectively reduced in order to maintain a memory limit. +A negative percentage effectively disables garbage collection, unless the memory limit is reached. + +See https://pkg.go.dev/runtime/debug#SetMemoryLimit for more details. + +Parameters: + +- gc-percent - int + ### debug/set-mutex-profile-fraction: Set runtime.SetMutexProfileFraction for mutex profiling. {#debug-set-mutex-profile-fraction} SetMutexProfileFraction controls the fraction of mutex contention @@ -12696,6 +12884,38 @@ Results: - previousRate - int +### debug/set-soft-memory-limit: Call runtime/debug.SetMemoryLimit for setting a soft memory limit for the runtime. {#debug-set-soft-memory-limit} + +SetMemoryLimit provides the runtime with a soft memory limit. + +The runtime undertakes several processes to try to respect this memory limit, including +adjustments to the frequency of garbage collections and returning memory to the underlying +system more aggressively. This limit will be respected even if GOGC=off (or, if SetGCPercent(-1) is executed). + +The input limit is provided as bytes, and includes all memory mapped, managed, and not +released by the Go runtime. Notably, it does not account for space used by the Go binary +and memory external to Go, such as memory managed by the underlying system on behalf of +the process, or memory managed by non-Go code inside the same process. +Examples of excluded memory sources include: OS kernel memory held on behalf of the process, +memory allocated by C code, and memory mapped by syscall.Mmap (because it is not managed by the Go runtime). + +A zero limit or a limit that's lower than the amount of memory used by the Go runtime may cause +the garbage collector to run nearly continuously. However, the application may still make progress. + +The memory limit is always respected by the Go runtime, so to effectively disable this behavior, +set the limit very high. math.MaxInt64 is the canonical value for disabling the limit, but values +much greater than the available memory on the underlying system work just as well. + +See https://go.dev/doc/gc-guide for a detailed guide explaining the soft memory limit in more detail, +as well as a variety of common use-cases and scenarios. + +SetMemoryLimit returns the previously set memory limit. A negative input does not adjust the limit, +and allows for retrieval of the currently set memory limit. + +Parameters: + +- mem-limit - int + ### fscache/clear: Clear the Fs cache. {#fscache-clear} This clears the fs cache. This is where remotes created from backends @@ -14241,7 +14461,7 @@ upon backend-specific capabilities. | Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | | OpenStack Swift | Yes † | Yes | No | No | No | Yes | Yes | No | Yes | No | -| Oracle Object Storage | Yes | Yes | No | No | Yes | Yes | No | No | No | No | +| Oracle Object Storage | No | Yes | No | No | Yes | Yes | Yes | No | No | No | | pCloud | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | premiumize.me | Yes | No | Yes | Yes | No | No | No | Yes | Yes | Yes | | put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | Yes | Yes | @@ -14251,7 +14471,7 @@ upon backend-specific capabilities. | Sia | No | No | No | No | No | No | Yes | No | No | Yes | | SMB | No | No | Yes | Yes | No | No | Yes | No | No | Yes | | SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes | -| Storj | Yes † | No | Yes | No | No | Yes | Yes | No | No | No | +| Storj | Yes † | Yes | Yes | No | No | Yes | Yes | No | No | No | | Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | | WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No | Yes | Yes | | Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | Yes | @@ -14366,9 +14586,10 @@ These flags are available for every command. -c, --checksum Skip based on checksum (if available) & size, not mod-time & size --client-cert string Client SSL certificate (PEM) for mutual TLS auth --client-key string Client SSL private key (PEM) for mutual TLS auth + --color string When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS (default "AUTO") --compare-dest stringArray Include additional comma separated server-side paths during comparison --config string Config file (default "$HOME/.config/rclone/rclone.conf") - --contimeout duration Connect timeout (default 1m0s) + --contimeout Duration Connect timeout (default 1m0s) --copy-dest stringArray Implies --compare-dest but also copies files from paths into destination --cpuprofile string Write cpu profile to file --cutoff-mode string Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS (default "HARD") @@ -14386,16 +14607,16 @@ These flags are available for every command. --dump-headers Dump HTTP headers - may contain sensitive info --error-on-no-transfer Sets exit code 9 if no files are transferred, useful in scripts --exclude stringArray Exclude files matching pattern - --exclude-from stringArray Read exclude patterns from file (use - to read from stdin) + --exclude-from stringArray Read file exclude patterns from file (use - to read from stdin) --exclude-if-present stringArray Exclude directories if filename is present - --expect-continue-timeout duration Timeout when using expect / 100-continue in HTTP (default 1s) + --expect-continue-timeout Duration Timeout when using expect / 100-continue in HTTP (default 1s) --fast-list Use recursive list if available; uses more memory but fewer transactions --files-from stringArray Read list of source-file names from file (use - to read from stdin) --files-from-raw stringArray Read list of source-file names from file without any processing of lines (use - to read from stdin) - -f, --filter stringArray Add a file-filtering rule - --filter-from stringArray Read filtering patterns from a file (use - to read from stdin) - --fs-cache-expire-duration duration Cache remotes for this long (0 to disable caching) (default 5m0s) - --fs-cache-expire-interval duration Interval to check for expired remotes (default 1m0s) + -f, --filter stringArray Add a file filtering rule + --filter-from stringArray Read file filtering patterns from a file (use - to read from stdin) + --fs-cache-expire-duration Duration Cache remotes for this long (0 to disable caching) (default 5m0s) + --fs-cache-expire-interval Duration Interval to check for expired remotes (default 1m0s) --header stringArray Set HTTP header for all transactions --header-download stringArray Set HTTP header for download transactions --header-upload stringArray Set HTTP header for upload transactions @@ -14409,9 +14630,9 @@ These flags are available for every command. -I, --ignore-times Don't skip files that match size and time - transfer all files --immutable Do not modify files, fail if existing files have been modified --include stringArray Include files matching pattern - --include-from stringArray Read include patterns from file (use - to read from stdin) + --include-from stringArray Read file include patterns from file (use - to read from stdin) -i, --interactive Enable interactive mode - --kv-lock-time duration Maximum time to keep key-value database locked by process (default 1s) + --kv-lock-time Duration Maximum time to keep key-value database locked by process (default 1s) --log-file string Log everything to this file --log-format string Comma separated list of log format options (default "date,time") --log-level string Log level DEBUG|INFO|NOTICE|ERROR (default "NOTICE") @@ -14421,16 +14642,22 @@ These flags are available for every command. --max-backlog int Maximum number of objects in sync or check backlog (default 10000) --max-delete int When synchronizing, limit the number of deletes (default -1) --max-depth int If set limits the recursion depth to this (default -1) - --max-duration duration Maximum duration rclone will transfer data for + --max-duration Duration Maximum duration rclone will transfer data for (default 0s) --max-size SizeSuffix Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P (default off) --max-stats-groups int Maximum number of stats groups to keep in memory, on max oldest is discarded (default 1000) --max-transfer SizeSuffix Maximum size of data to transfer (default off) --memprofile string Write memory profile to file -M, --metadata If set, preserve metadata when copying objects + --metadata-exclude stringArray Exclude metadatas matching pattern + --metadata-exclude-from stringArray Read metadata exclude patterns from file (use - to read from stdin) + --metadata-filter stringArray Add a metadata filtering rule + --metadata-filter-from stringArray Read metadata filtering patterns from a file (use - to read from stdin) + --metadata-include stringArray Include metadatas matching pattern + --metadata-include-from stringArray Read metadata include patterns from file (use - to read from stdin) --metadata-set stringArray Add metadata key=value when uploading --min-age Duration Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off) --min-size SizeSuffix Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off) - --modify-window duration Max time diff to be considered the same (default 1ns) + --modify-window Duration Max time diff to be considered the same (default 1ns) --multi-thread-cutoff SizeSuffix Use multi-thread downloads for files above this size (default 250Mi) --multi-thread-streams int Max number of streams to use for multi-thread downloads (default 4) --no-check-certificate Do not verify the server SSL certificate (insecure) @@ -14446,25 +14673,26 @@ These flags are available for every command. --progress-terminal-title Show progress on the terminal title (requires -P/--progress) -q, --quiet Print as little stuff as possible --rc Enable the remote control server - --rc-addr string IPaddress:Port or :Port to bind server to (default "localhost:5572") + --rc-addr stringArray IPaddress:Port or :Port to bind server to (default [localhost:5572]) --rc-allow-origin string Set the allowed origin for CORS --rc-baseurl string Prefix for URLs - leave blank for root - --rc-cert string SSL PEM key (concatenation of certificate and CA certificate) + --rc-cert string TLS PEM key (concatenation of certificate and CA certificate) --rc-client-ca string Client certificate authority to verify clients with --rc-enable-metrics Enable prometheus metrics on /metrics --rc-files string Path to local files to serve on the HTTP server - --rc-htpasswd string htpasswd file - if not provided no authentication is done - --rc-job-expire-duration duration Expire finished async jobs older than this value (default 1m0s) - --rc-job-expire-interval duration Interval to check for expired async jobs (default 10s) - --rc-key string SSL PEM Private key + --rc-htpasswd string A htpasswd file - if not provided no authentication is done + --rc-job-expire-duration Duration Expire finished async jobs older than this value (default 1m0s) + --rc-job-expire-interval Duration Interval to check for expired async jobs (default 10s) + --rc-key string TLS PEM Private key --rc-max-header-bytes int Maximum size of request header (default 4096) --rc-min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --rc-no-auth Don't require auth for certain methods --rc-pass string Password for authentication - --rc-realm string Realm for authentication (default "rclone") + --rc-realm string Realm for authentication + --rc-salt string Password hashing salt (default "dlPL2MqE") --rc-serve Enable the serving of remote objects - --rc-server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --rc-server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --rc-server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --rc-server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --rc-template string User-specified template --rc-user string User name for authentication --rc-web-fetch-url string URL to fetch the releases for webgui (default "https://api.github.com/repos/rclone/rclone-webui-react/releases/latest") @@ -14474,10 +14702,10 @@ These flags are available for every command. --rc-web-gui-update Check and update to latest version of web gui --refresh-times Refresh the modtime of remote files --retries int Retry operations this many times if they fail (default 3) - --retries-sleep duration Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) + --retries-sleep Duration Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) (default 0s) --server-side-across-configs Allow server-side operations (e.g. copy) to work across different configs --size-only Skip based on size only, not mod-time or checksum - --stats duration Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s) + --stats Duration Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s) --stats-file-name-length int Max file name length in stats (0 for no limit) (default 45) --stats-log-level string Log level to show --stats output DEBUG|INFO|NOTICE|ERROR (default "INFO") --stats-one-line Make the stats fit on one line @@ -14490,7 +14718,7 @@ These flags are available for every command. --syslog Use Syslog for logging --syslog-facility string Facility for syslog, e.g. KERN,USER,... (default "DAEMON") --temp-dir string Directory rclone will use for temporary files (default "/tmp") - --timeout duration IO idle timeout (default 5m0s) + --timeout Duration IO idle timeout (default 5m0s) --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --track-renames When synchronizing, track file renames and do a server-side move if possible @@ -14501,7 +14729,7 @@ These flags are available for every command. --use-json-log Use json log format --use-mmap Use mmap allocator (see docs) --use-server-modtime Use server modified time instead of object metadata - --user-agent string Set the user-agent to a specified string (default "rclone/v1.60.0") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.61.0") -v, --verbose count Print lots more stuff (repeat for more) ``` @@ -14511,529 +14739,543 @@ These flags are available for every command. They control the backends and may be set in the config file. ``` - --acd-auth-url string Auth server URL - --acd-client-id string OAuth Client Id - --acd-client-secret string OAuth Client Secret - --acd-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --acd-templink-threshold SizeSuffix Files >= this size will be downloaded via their tempLink (default 9Gi) - --acd-token string OAuth Access Token as a JSON blob - --acd-token-url string Token server url - --acd-upload-wait-per-gb Duration Additional time per GiB to wait after a failed complete upload to see if it appears (default 3m0s) - --alias-remote string Remote or path to alias - --azureblob-access-tier string Access tier of blob: hot, cool or archive - --azureblob-account string Storage Account Name - --azureblob-archive-tier-delete Delete archive tier blobs before overwriting - --azureblob-chunk-size SizeSuffix Upload chunk size (default 4Mi) - --azureblob-disable-checksum Don't store MD5 checksum with object metadata - --azureblob-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8) - --azureblob-endpoint string Endpoint for the service - --azureblob-key string Storage Account Key - --azureblob-list-chunk int Size of blob list (default 5000) - --azureblob-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --azureblob-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --azureblob-msi-client-id string Object ID of the user-assigned MSI to use, if any - --azureblob-msi-mi-res-id string Azure resource ID of the user-assigned MSI to use, if any - --azureblob-msi-object-id string Object ID of the user-assigned MSI to use, if any - --azureblob-no-head-object If set, do not do HEAD before GET when getting objects - --azureblob-public-access string Public access level of a container: blob or container - --azureblob-sas-url string SAS URL for container level access only - --azureblob-service-principal-file string Path to file containing credentials for use with a service principal - --azureblob-upload-concurrency int Concurrency for multipart uploads (default 16) - --azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256 MiB) (deprecated) - --azureblob-use-emulator Uses local storage emulator if provided as 'true' - --azureblob-use-msi Use a managed service identity to authenticate (only works in Azure) - --b2-account string Account ID or Application Key ID - --b2-chunk-size SizeSuffix Upload chunk size (default 96Mi) - --b2-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4Gi) - --b2-disable-checksum Disable checksums for large (> upload cutoff) files - --b2-download-auth-duration Duration Time before the authorization token will expire in s or suffix ms|s|m|h|d (default 1w) - --b2-download-url string Custom endpoint for downloads - --b2-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --b2-endpoint string Endpoint for the service - --b2-hard-delete Permanently delete files on remote removal, otherwise hide files - --b2-key string Application Key - --b2-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging - --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --b2-version-at Time Show file versions as they were at the specified time (default off) - --b2-versions Include old versions in directory listings - --box-access-token string Box App Primary Access Token - --box-auth-url string Auth server URL - --box-box-config-file string Box App config.json location - --box-box-sub-type string (default "user") - --box-client-id string OAuth Client Id - --box-client-secret string OAuth Client Secret - --box-commit-retries int Max number of times to try committing a multipart file (default 100) - --box-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot) - --box-list-chunk int Size of listing chunk 1-1000 (default 1000) - --box-owned-by string Only show items owned by the login (email address) passed in - --box-root-folder-id string Fill in for rclone to use a non root folder as its starting point - --box-token string OAuth Access Token as a JSON blob - --box-token-url string Token server url - --box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50 MiB) (default 50Mi) - --cache-chunk-clean-interval Duration How often should the cache perform cleanups of the chunk storage (default 1m0s) - --cache-chunk-no-memory Disable the in-memory cache for storing chunks during streaming - --cache-chunk-path string Directory to cache chunk files (default "$HOME/.cache/rclone/cache-backend") - --cache-chunk-size SizeSuffix The size of a chunk (partial file data) (default 5Mi) - --cache-chunk-total-size SizeSuffix The total size that the chunks can take up on the local disk (default 10Gi) - --cache-db-path string Directory to store file structure metadata DB (default "$HOME/.cache/rclone/cache-backend") - --cache-db-purge Clear all the cached data for this remote on start - --cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s) - --cache-info-age Duration How long to cache file structure information (directory listings, file size, times, etc.) (default 6h0m0s) - --cache-plex-insecure string Skip all certificate verification when connecting to the Plex server - --cache-plex-password string The password of the Plex user (obscured) - --cache-plex-url string The URL of the Plex server - --cache-plex-username string The username of the Plex user - --cache-read-retries int How many times to retry a read from a cache storage (default 10) - --cache-remote string Remote to cache - --cache-rps int Limits the number of requests per second to the source FS (-1 to disable) (default -1) - --cache-tmp-upload-path string Directory to keep temporary files until they are uploaded - --cache-tmp-wait-time Duration How long should files be stored in local cache before being uploaded (default 15s) - --cache-workers int How many workers should run in parallel to download chunks (default 4) - --cache-writes Cache file data on writes through the FS - --chunker-chunk-size SizeSuffix Files larger than chunk size will be split in chunks (default 2Gi) - --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks - --chunker-hash-type string Choose how chunker handles hash sums (default "md5") - --chunker-remote string Remote to chunk/unchunk - --combine-upstreams SpaceSepList Upstreams for combining - --compress-level int GZIP compression level (-2 to 9) (default -1) - --compress-mode string Compression mode (default "gzip") - --compress-ram-cache-limit SizeSuffix Some remotes don't allow the upload of files with unknown size (default 20Mi) - --compress-remote string Remote to compress - -L, --copy-links Follow symlinks and copy the pointed to item - --crypt-directory-name-encryption Option to either encrypt directory names or leave them intact (default true) - --crypt-filename-encoding string How to encode the encrypted filename to text string (default "base32") - --crypt-filename-encryption string How to encrypt the filenames (default "standard") - --crypt-no-data-encryption Option to either encrypt file data or leave it unencrypted - --crypt-password string Password or pass phrase for encryption (obscured) - --crypt-password2 string Password or pass phrase for salt (obscured) - --crypt-remote string Remote to encrypt/decrypt - --crypt-server-side-across-configs Allow server-side operations (e.g. copy) to work across different crypt configs - --crypt-show-mapping For all files listed show how the names encrypt - --drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded - --drive-allow-import-name-change Allow the filetype to change when uploading Google docs - --drive-auth-owner-only Only consider files owned by the authenticated user - --drive-auth-url string Auth server URL - --drive-chunk-size SizeSuffix Upload chunk size (default 8Mi) - --drive-client-id string Google Application Client Id - --drive-client-secret string OAuth Client Secret - --drive-copy-shortcut-content Server side copy contents of shortcuts instead of the shortcut - --drive-disable-http2 Disable drive using http2 (default true) - --drive-encoding MultiEncoder The encoding for the backend (default InvalidUtf8) - --drive-export-formats string Comma separated list of preferred formats for downloading Google docs (default "docx,xlsx,pptx,svg") - --drive-formats string Deprecated: See export_formats - --drive-impersonate string Impersonate this user when using a service account - --drive-import-formats string Comma separated list of preferred formats for uploading Google docs - --drive-keep-revision-forever Keep new head revision of each file forever - --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) - --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) - --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) - --drive-resource-key string Resource key for accessing a link-shared file - --drive-root-folder-id string ID of the root folder - --drive-scope string Scope that rclone should use when requesting access from drive - --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs - --drive-service-account-credentials string Service Account Credentials JSON blob - --drive-service-account-file string Service Account Credentials JSON file path - --drive-shared-with-me Only show files that are shared with me - --drive-size-as-quota Show sizes as storage quota usage, not actual size - --drive-skip-checksum-gphotos Skip MD5 checksum on Google photos and videos only - --drive-skip-dangling-shortcuts If set skip dangling shortcut files - --drive-skip-gdocs Skip google documents in all listings - --drive-skip-shortcuts If set skip shortcut files - --drive-starred-only Only show files that are starred - --drive-stop-on-download-limit Make download limit errors be fatal - --drive-stop-on-upload-limit Make upload limit errors be fatal - --drive-team-drive string ID of the Shared Drive (Team Drive) - --drive-token string OAuth Access Token as a JSON blob - --drive-token-url string Token server url - --drive-trashed-only Only show files that are in the trash - --drive-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 8Mi) - --drive-use-created-date Use file created date instead of modified date - --drive-use-shared-date Use date file was shared instead of modified date - --drive-use-trash Send files to the trash instead of deleting permanently (default true) - --drive-v2-download-min-size SizeSuffix If Object's are greater, use drive v2 API to download (default off) - --dropbox-auth-url string Auth server URL - --dropbox-batch-commit-timeout Duration Max time to wait for a batch to finish committing (default 10m0s) - --dropbox-batch-mode string Upload file batching sync|async|off (default "sync") - --dropbox-batch-size int Max number of files in upload batch - --dropbox-batch-timeout Duration Max time to allow an idle upload batch before uploading (default 0s) - --dropbox-chunk-size SizeSuffix Upload chunk size (< 150Mi) (default 48Mi) - --dropbox-client-id string OAuth Client Id - --dropbox-client-secret string OAuth Client Secret - --dropbox-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,RightSpace,InvalidUtf8,Dot) - --dropbox-impersonate string Impersonate this user when using a business account - --dropbox-shared-files Instructs rclone to work on individual shared files - --dropbox-shared-folders Instructs rclone to work on shared folders - --dropbox-token string OAuth Access Token as a JSON blob - --dropbox-token-url string Token server url - --fichier-api-key string Your API Key, get it from https://1fichier.com/console/params.pl - --fichier-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot) - --fichier-file-password string If you want to download a shared file that is password protected, add this parameter (obscured) - --fichier-folder-password string If you want to list the files in a shared folder that is password protected, add this parameter (obscured) - --fichier-shared-folder string If you want to download a shared folder, add this parameter - --filefabric-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) - --filefabric-permanent-token string Permanent Authentication Token - --filefabric-root-folder-id string ID of the root folder - --filefabric-token string Session Token - --filefabric-token-expiry string Token expiry time - --filefabric-url string URL of the Enterprise File Fabric to connect to - --filefabric-version string Version read from the file fabric - --ftp-ask-password Allow asking for FTP password when needed - --ftp-close-timeout Duration Maximum time to wait for a response to close (default 1m0s) - --ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited - --ftp-disable-epsv Disable using EPSV even if server advertises support - --ftp-disable-mlsd Disable using MLSD even if server advertises support - --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) - --ftp-disable-utf8 Disable using UTF-8 even if server advertises support - --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) - --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) - --ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD - --ftp-host string FTP host to connect to - --ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --ftp-no-check-certificate Do not verify the TLS certificate of the server - --ftp-pass string FTP password (obscured) - --ftp-port int FTP port number (default 21) - --ftp-shut-timeout Duration Maximum time to wait for data connection closing status (default 1m0s) - --ftp-tls Use Implicit FTPS (FTP over TLS) - --ftp-tls-cache-size int Size of TLS session cache for all control and data connections (default 32) - --ftp-user string FTP username (default "$USER") - --ftp-writing-mdtm Use MDTM to set modification time (VsFtpd quirk) - --gcs-anonymous Access public buckets and objects without credentials - --gcs-auth-url string Auth server URL - --gcs-bucket-acl string Access Control List for new buckets - --gcs-bucket-policy-only Access checks should use bucket-level IAM policies - --gcs-client-id string OAuth Client Id - --gcs-client-secret string OAuth Client Secret - --gcs-decompress If set this will decompress gzip encoded objects - --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) - --gcs-endpoint string Endpoint for the service - --gcs-location string Location for the newly created buckets - --gcs-no-check-bucket If set, don't attempt to check the bucket exists or create it - --gcs-object-acl string Access Control List for new objects - --gcs-project-number string Project number - --gcs-service-account-file string Service Account Credentials JSON file path - --gcs-storage-class string The storage class to use when storing objects in Google Cloud Storage - --gcs-token string OAuth Access Token as a JSON blob - --gcs-token-url string Token server url - --gphotos-auth-url string Auth server URL - --gphotos-client-id string OAuth Client Id - --gphotos-client-secret string OAuth Client Secret - --gphotos-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) - --gphotos-include-archived Also view and download archived media - --gphotos-read-only Set to make the Google Photos backend read only - --gphotos-read-size Set to read the size of media items - --gphotos-start-year int Year limits the photos to be downloaded to those which are uploaded after the given year (default 2000) - --gphotos-token string OAuth Access Token as a JSON blob - --gphotos-token-url string Token server url - --hasher-auto-size SizeSuffix Auto-update checksum for files smaller than this size (disabled by default) - --hasher-hashes CommaSepList Comma separated list of supported checksum types (default md5,sha1) - --hasher-max-age Duration Maximum time to keep checksums in cache (0 = no cache, off = cache forever) (default off) - --hasher-remote string Remote to cache checksums for (e.g. myRemote:path) - --hdfs-data-transfer-protection string Kerberos data transfer protection: authentication|integrity|privacy - --hdfs-encoding MultiEncoder The encoding for the backend (default Slash,Colon,Del,Ctl,InvalidUtf8,Dot) - --hdfs-namenode string Hadoop name node and port - --hdfs-service-principal-name string Kerberos service principal name for the namenode - --hdfs-username string Hadoop user name - --hidrive-auth-url string Auth server URL - --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) - --hidrive-client-id string OAuth Client Id - --hidrive-client-secret string OAuth Client Secret - --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary - --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) - --hidrive-endpoint string Endpoint for the service (default "https://api.hidrive.strato.com/2.1") - --hidrive-root-prefix string The root/parent folder for all paths (default "/") - --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default "rw") - --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default "user") - --hidrive-token string OAuth Access Token as a JSON blob - --hidrive-token-url string Token server url - --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) - --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) - --http-headers CommaSepList Set HTTP headers for all transactions - --http-no-head Don't use HEAD requests - --http-no-slash Set this if the site doesn't end directories with / - --http-url string URL of HTTP host to connect to - --internetarchive-access-key-id string IAS3 Access Key - --internetarchive-disable-checksum Don't ask the server to test against MD5 checksum calculated by rclone (default true) - --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) - --internetarchive-endpoint string IAS3 Endpoint (default "https://s3.us.archive.org") - --internetarchive-front-endpoint string Host of InternetArchive Frontend (default "https://archive.org") - --internetarchive-secret-access-key string IAS3 Secret Key (password) - --internetarchive-wait-archive Duration Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s) - --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) - --jottacloud-hard-delete Delete files permanently rather than putting them into the trash - --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) - --jottacloud-no-versions Avoid server side versioning by deleting files and recreating files instead of overwriting them - --jottacloud-trashed-only Only show files that are in the trash - --jottacloud-upload-resume-limit SizeSuffix Files bigger than this can be resumed if the upload fail's (default 10Mi) - --koofr-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --koofr-endpoint string The Koofr API endpoint to use - --koofr-mountid string Mount ID of the mount to use - --koofr-password string Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured) - --koofr-provider string Choose your storage provider - --koofr-setmtime Does the backend support setting modification time (default true) - --koofr-user string Your user name - -l, --links Translate symlinks to/from regular files with a '.rclonelink' extension - --local-case-insensitive Force the filesystem to report itself as case insensitive - --local-case-sensitive Force the filesystem to report itself as case sensitive - --local-encoding MultiEncoder The encoding for the backend (default Slash,Dot) - --local-no-check-updated Don't check to see if the files change during upload - --local-no-preallocate Disable preallocation of disk space for transferred files - --local-no-set-modtime Disable setting modtime - --local-no-sparse Disable sparse files for multi-thread downloads - --local-nounc Disable UNC (long path names) conversion on Windows - --local-unicode-normalization Apply unicode NFC normalization to paths and filenames - --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) - --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) - --mailru-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --mailru-pass string Password (obscured) - --mailru-speedup-enable Skip full upload if there is another file with same data hash (default true) - --mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash) (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf") - --mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3Gi) - --mailru-speedup-max-memory SizeSuffix Files larger than the size given below will always be hashed on disk (default 32Mi) - --mailru-user string User name (usually email) - --mega-debug Output more debug from Mega - --mega-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --mega-hard-delete Delete files permanently rather than putting them into the trash - --mega-pass string Password (obscured) - --mega-user string User name - --netstorage-account string Set the NetStorage account name - --netstorage-host string Domain+path of NetStorage host to connect to - --netstorage-protocol string Select between HTTP or HTTPS protocol (default "https") - --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) - -x, --one-file-system Don't cross filesystem boundaries (unix/macOS only) - --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) - --onedrive-auth-url string Auth server URL - --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) - --onedrive-client-id string OAuth Client Id - --onedrive-client-secret string OAuth Client Secret - --onedrive-drive-id string The ID of the drive to use - --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) - --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) - --onedrive-expose-onenote-files Set to make OneNote files show up in directory listings - --onedrive-link-password string Set the password for links created by the link command - --onedrive-link-scope string Set the scope of the links created by the link command (default "anonymous") - --onedrive-link-type string Set the type of the links created by the link command (default "view") - --onedrive-list-chunk int Size of listing chunk (default 1000) - --onedrive-no-versions Remove all versions on modifying operations - --onedrive-region string Choose national cloud region for OneDrive (default "global") - --onedrive-root-folder-id string ID of the root folder - --onedrive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different onedrive configs - --onedrive-token string OAuth Access Token as a JSON blob - --onedrive-token-url string Token server url - --oos-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) - --oos-compartment string Object storage compartment OCID - --oos-config-file string Path to OCI config file (default "~/.oci/config") - --oos-config-profile string Profile name inside the oci config file (default "Default") - --oos-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) - --oos-copy-timeout Duration Timeout for copy (default 1m0s) - --oos-disable-checksum Don't store MD5 checksum with object metadata - --oos-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --oos-endpoint string Endpoint for Object storage API - --oos-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery - --oos-namespace string Object storage namespace - --oos-no-check-bucket If set, don't attempt to check the bucket exists or create it - --oos-provider string Choose your Auth Provider (default "env_auth") - --oos-region string Object storage Region - --oos-upload-concurrency int Concurrency for multipart uploads (default 10) - --oos-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size (default 10Mi) - --opendrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot) - --opendrive-password string Password (obscured) - --opendrive-username string Username - --pcloud-auth-url string Auth server URL - --pcloud-client-id string OAuth Client Id - --pcloud-client-secret string OAuth Client Secret - --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --pcloud-hostname string Hostname to connect to (default "api.pcloud.com") - --pcloud-password string Your pcloud password (obscured) - --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default "d0") - --pcloud-token string OAuth Access Token as a JSON blob - --pcloud-token-url string Token server url - --pcloud-username string Your pcloud username - --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --qingstor-access-key-id string QingStor Access Key ID - --qingstor-chunk-size SizeSuffix Chunk size to use for uploading (default 4Mi) - --qingstor-connection-retries int Number of connection retries (default 3) - --qingstor-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8) - --qingstor-endpoint string Enter an endpoint URL to connection QingStor API - --qingstor-env-auth Get QingStor credentials from runtime - --qingstor-secret-access-key string QingStor Secret Access Key (password) - --qingstor-upload-concurrency int Concurrency for multipart uploads (default 1) - --qingstor-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --qingstor-zone string Zone to connect to - --s3-access-key-id string AWS Access Key ID - --s3-acl string Canned ACL used when creating buckets and storing or copying objects - --s3-bucket-acl string Canned ACL used when creating buckets - --s3-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) - --s3-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) - --s3-decompress If set this will decompress gzip encoded objects - --s3-disable-checksum Don't store MD5 checksum with object metadata - --s3-disable-http2 Disable usage of http2 for S3 backends - --s3-download-url string Custom endpoint for downloads - --s3-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --s3-endpoint string Endpoint for S3 API - --s3-env-auth Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars) - --s3-force-path-style If true use path style access if false use virtual hosted style (default true) - --s3-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery - --s3-list-chunk int Size of listing chunk (response list for each ListObject S3 request) (default 1000) - --s3-list-url-encode Tristate Whether to url encode listings: true/false/unset (default unset) - --s3-list-version int Version of ListObjects to use: 1,2 or 0 for auto - --s3-location-constraint string Location constraint - must be set to match the Region - --s3-max-upload-parts int Maximum number of parts in a multipart upload (default 10000) - --s3-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --s3-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --s3-no-check-bucket If set, don't attempt to check the bucket exists or create it - --s3-no-head If set, don't HEAD uploaded objects to check integrity - --s3-no-head-object If set, do not do HEAD before GET when getting objects - --s3-no-system-metadata Suppress setting and reading of system metadata - --s3-profile string Profile to use in the shared credentials file - --s3-provider string Choose your S3 provider - --s3-region string Region to connect to - --s3-requester-pays Enables requester pays option when interacting with S3 bucket - --s3-secret-access-key string AWS Secret Access Key (password) - --s3-server-side-encryption string The server-side encryption algorithm used when storing this object in S3 - --s3-session-token string An AWS session token - --s3-shared-credentials-file string Path to the shared credentials file - --s3-sse-customer-algorithm string If using SSE-C, the server-side encryption algorithm used when storing this object in S3 - --s3-sse-customer-key string To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data - --s3-sse-customer-key-base64 string If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data - --s3-sse-customer-key-md5 string If using SSE-C you may provide the secret encryption key MD5 checksum (optional) - --s3-sse-kms-key-id string If using KMS ID you must provide the ARN of Key - --s3-storage-class string The storage class to use when storing new objects in S3 - --s3-upload-concurrency int Concurrency for multipart uploads (default 4) - --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint - --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) - --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads - --s3-v2-auth If true use v2 authentication - --s3-version-at Time Show file versions as they were at the specified time (default off) - --s3-versions Include old versions in directory listings - --seafile-2fa Two-factor authentication ('true' if the account has 2FA enabled) - --seafile-create-library Should rclone create a library if it doesn't exist - --seafile-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8) - --seafile-library string Name of the library - --seafile-library-key string Library password (for encrypted libraries only) (obscured) - --seafile-pass string Password (obscured) - --seafile-url string URL of seafile host to connect to - --seafile-user string User name (usually email address) - --sftp-ask-password Allow asking for SFTP password when needed - --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) - --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) - --sftp-disable-concurrent-reads If set don't use concurrent reads - --sftp-disable-concurrent-writes If set don't use concurrent writes - --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available - --sftp-host string SSH host to connect to - --sftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --sftp-key-file string Path to PEM-encoded private key file - --sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file (obscured) - --sftp-key-pem string Raw PEM-encoded private key - --sftp-key-use-agent When set forces the usage of the ssh-agent - --sftp-known-hosts-file string Optional path to known_hosts file - --sftp-md5sum-command string The command used to read md5 hashes - --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) - --sftp-path-override string Override path used by SSH shell commands - --sftp-port int SSH port number (default 22) - --sftp-pubkey-file string Optional path to public key file - --sftp-server-command string Specifies the path or command to run a sftp server on the remote host - --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands - --sftp-set-modtime Set the modified time on the remote if set (default true) - --sftp-sha1sum-command string The command used to read sha1 hashes - --sftp-shell-type string The type of SSH shell on remote server, if any - --sftp-skip-links Set to skip any symlinks and any other non regular files - --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default "sftp") - --sftp-use-fstat If set use fstat instead of stat - --sftp-use-insecure-cipher Enable the use of insecure ciphers and key exchange methods - --sftp-user string SSH username (default "$USER") - --sharefile-chunk-size SizeSuffix Upload chunk size (default 64Mi) - --sharefile-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,LeftSpace,LeftPeriod,RightSpace,RightPeriod,InvalidUtf8,Dot) - --sharefile-endpoint string Endpoint for API calls - --sharefile-root-folder-id string ID of the root folder - --sharefile-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (default 128Mi) - --sia-api-password string Sia Daemon API Password (obscured) - --sia-api-url string Sia daemon API URL, like http://sia.daemon.host:9980 (default "http://127.0.0.1:9980") - --sia-encoding MultiEncoder The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot) - --sia-user-agent string Siad User Agent (default "Sia-Agent") - --skip-links Don't warn about skipped symlinks - --smb-case-insensitive Whether the server is configured to be case-insensitive (default true) - --smb-domain string Domain name for NTLM authentication (default "WORKGROUP") - --smb-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot) - --smb-hide-special-share Hide special shares (e.g. print$) which users aren't supposed to access (default true) - --smb-host string SMB server hostname to connect to - --smb-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --smb-pass string SMB password (obscured) - --smb-port int SMB port number (default 445) - --smb-user string SMB username (default "$USER") - --storj-access-grant string Access grant - --storj-api-key string API key - --storj-passphrase string Encryption passphrase - --storj-provider string Choose an authentication method (default "existing") - --storj-satellite-address string Satellite address (default "us-central-1.storj.io") - --sugarsync-access-key-id string Sugarsync Access Key ID - --sugarsync-app-id string Sugarsync App ID - --sugarsync-authorization string Sugarsync authorization - --sugarsync-authorization-expiry string Sugarsync authorization expiry - --sugarsync-deleted-id string Sugarsync deleted folder id - --sugarsync-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8,Dot) - --sugarsync-hard-delete Permanently delete files if true - --sugarsync-private-access-key string Sugarsync Private Access Key - --sugarsync-refresh-token string Sugarsync refresh token - --sugarsync-root-id string Sugarsync root id - --sugarsync-user string Sugarsync user - --swift-application-credential-id string Application Credential ID (OS_APPLICATION_CREDENTIAL_ID) - --swift-application-credential-name string Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME) - --swift-application-credential-secret string Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET) - --swift-auth string Authentication URL for server (OS_AUTH_URL) - --swift-auth-token string Auth Token from alternate authentication - optional (OS_AUTH_TOKEN) - --swift-auth-version int AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION) - --swift-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) - --swift-domain string User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME) - --swift-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8) - --swift-endpoint-type string Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default "public") - --swift-env-auth Get swift credentials from environment variables in standard OpenStack form - --swift-key string API key or password (OS_PASSWORD) - --swift-leave-parts-on-error If true avoid calling abort upload on a failure - --swift-no-chunk Don't chunk files during streaming upload - --swift-no-large-objects Disable support for static and dynamic large objects - --swift-region string Region name - optional (OS_REGION_NAME) - --swift-storage-policy string The storage policy to use when creating a new container - --swift-storage-url string Storage URL - optional (OS_STORAGE_URL) - --swift-tenant string Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME) - --swift-tenant-domain string Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME) - --swift-tenant-id string Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID) - --swift-user string User name to log in (OS_USERNAME) - --swift-user-id string User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID) - --union-action-policy string Policy to choose upstream on ACTION category (default "epall") - --union-cache-time int Cache time of usage and free space (in seconds) (default 120) - --union-create-policy string Policy to choose upstream on CREATE category (default "epmfs") - --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) - --union-search-policy string Policy to choose upstream on SEARCH category (default "ff") - --union-upstreams string List of space separated upstreams - --uptobox-access-token string Your access token - --uptobox-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot) - --webdav-bearer-token string Bearer token instead of user/pass (e.g. a Macaroon) - --webdav-bearer-token-command string Command to run to get a bearer token - --webdav-encoding string The encoding for the backend - --webdav-headers CommaSepList Set HTTP headers for all transactions - --webdav-pass string Password (obscured) - --webdav-url string URL of http host to connect to - --webdav-user string User name - --webdav-vendor string Name of the WebDAV site/service/software you are using - --yandex-auth-url string Auth server URL - --yandex-client-id string OAuth Client Id - --yandex-client-secret string OAuth Client Secret - --yandex-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) - --yandex-hard-delete Delete files permanently rather than putting them into the trash - --yandex-token string OAuth Access Token as a JSON blob - --yandex-token-url string Token server url - --zoho-auth-url string Auth server URL - --zoho-client-id string OAuth Client Id - --zoho-client-secret string OAuth Client Secret - --zoho-encoding MultiEncoder The encoding for the backend (default Del,Ctl,InvalidUtf8) - --zoho-region string Zoho region to connect to - --zoho-token string OAuth Access Token as a JSON blob - --zoho-token-url string Token server url + --acd-auth-url string Auth server URL + --acd-client-id string OAuth Client Id + --acd-client-secret string OAuth Client Secret + --acd-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --acd-templink-threshold SizeSuffix Files >= this size will be downloaded via their tempLink (default 9Gi) + --acd-token string OAuth Access Token as a JSON blob + --acd-token-url string Token server url + --acd-upload-wait-per-gb Duration Additional time per GiB to wait after a failed complete upload to see if it appears (default 3m0s) + --alias-remote string Remote or path to alias + --azureblob-access-tier string Access tier of blob: hot, cool or archive + --azureblob-account string Azure Storage Account Name + --azureblob-archive-tier-delete Delete archive tier blobs before overwriting + --azureblob-chunk-size SizeSuffix Upload chunk size (default 4Mi) + --azureblob-client-certificate-password string Password for the certificate file (optional) (obscured) + --azureblob-client-certificate-path string Path to a PEM or PKCS12 certificate file including the private key + --azureblob-client-id string The ID of the client in use + --azureblob-client-secret string One of the service principal's client secrets + --azureblob-client-send-certificate-chain Send the certificate chain when using certificate auth + --azureblob-disable-checksum Don't store MD5 checksum with object metadata + --azureblob-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8) + --azureblob-endpoint string Endpoint for the service + --azureblob-env-auth Read credentials from runtime (environment variables, CLI or MSI) + --azureblob-key string Storage Account Shared Key + --azureblob-list-chunk int Size of blob list (default 5000) + --azureblob-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --azureblob-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --azureblob-msi-client-id string Object ID of the user-assigned MSI to use, if any + --azureblob-msi-mi-res-id string Azure resource ID of the user-assigned MSI to use, if any + --azureblob-msi-object-id string Object ID of the user-assigned MSI to use, if any + --azureblob-no-check-container If set, don't attempt to check the container exists or create it + --azureblob-no-head-object If set, do not do HEAD before GET when getting objects + --azureblob-password string The user's password (obscured) + --azureblob-public-access string Public access level of a container: blob or container + --azureblob-sas-url string SAS URL for container level access only + --azureblob-service-principal-file string Path to file containing credentials for use with a service principal + --azureblob-tenant string ID of the service principal's tenant. Also called its directory ID + --azureblob-upload-concurrency int Concurrency for multipart uploads (default 16) + --azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256 MiB) (deprecated) + --azureblob-use-emulator Uses local storage emulator if provided as 'true' + --azureblob-use-msi Use a managed service identity to authenticate (only works in Azure) + --azureblob-username string User name (usually an email address) + --b2-account string Account ID or Application Key ID + --b2-chunk-size SizeSuffix Upload chunk size (default 96Mi) + --b2-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4Gi) + --b2-disable-checksum Disable checksums for large (> upload cutoff) files + --b2-download-auth-duration Duration Time before the authorization token will expire in s or suffix ms|s|m|h|d (default 1w) + --b2-download-url string Custom endpoint for downloads + --b2-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --b2-endpoint string Endpoint for the service + --b2-hard-delete Permanently delete files on remote removal, otherwise hide files + --b2-key string Application Key + --b2-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging + --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --b2-version-at Time Show file versions as they were at the specified time (default off) + --b2-versions Include old versions in directory listings + --box-access-token string Box App Primary Access Token + --box-auth-url string Auth server URL + --box-box-config-file string Box App config.json location + --box-box-sub-type string (default "user") + --box-client-id string OAuth Client Id + --box-client-secret string OAuth Client Secret + --box-commit-retries int Max number of times to try committing a multipart file (default 100) + --box-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot) + --box-list-chunk int Size of listing chunk 1-1000 (default 1000) + --box-owned-by string Only show items owned by the login (email address) passed in + --box-root-folder-id string Fill in for rclone to use a non root folder as its starting point + --box-token string OAuth Access Token as a JSON blob + --box-token-url string Token server url + --box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50 MiB) (default 50Mi) + --cache-chunk-clean-interval Duration How often should the cache perform cleanups of the chunk storage (default 1m0s) + --cache-chunk-no-memory Disable the in-memory cache for storing chunks during streaming + --cache-chunk-path string Directory to cache chunk files (default "$HOME/.cache/rclone/cache-backend") + --cache-chunk-size SizeSuffix The size of a chunk (partial file data) (default 5Mi) + --cache-chunk-total-size SizeSuffix The total size that the chunks can take up on the local disk (default 10Gi) + --cache-db-path string Directory to store file structure metadata DB (default "$HOME/.cache/rclone/cache-backend") + --cache-db-purge Clear all the cached data for this remote on start + --cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s) + --cache-info-age Duration How long to cache file structure information (directory listings, file size, times, etc.) (default 6h0m0s) + --cache-plex-insecure string Skip all certificate verification when connecting to the Plex server + --cache-plex-password string The password of the Plex user (obscured) + --cache-plex-url string The URL of the Plex server + --cache-plex-username string The username of the Plex user + --cache-read-retries int How many times to retry a read from a cache storage (default 10) + --cache-remote string Remote to cache + --cache-rps int Limits the number of requests per second to the source FS (-1 to disable) (default -1) + --cache-tmp-upload-path string Directory to keep temporary files until they are uploaded + --cache-tmp-wait-time Duration How long should files be stored in local cache before being uploaded (default 15s) + --cache-workers int How many workers should run in parallel to download chunks (default 4) + --cache-writes Cache file data on writes through the FS + --chunker-chunk-size SizeSuffix Files larger than chunk size will be split in chunks (default 2Gi) + --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks + --chunker-hash-type string Choose how chunker handles hash sums (default "md5") + --chunker-remote string Remote to chunk/unchunk + --combine-upstreams SpaceSepList Upstreams for combining + --compress-level int GZIP compression level (-2 to 9) (default -1) + --compress-mode string Compression mode (default "gzip") + --compress-ram-cache-limit SizeSuffix Some remotes don't allow the upload of files with unknown size (default 20Mi) + --compress-remote string Remote to compress + -L, --copy-links Follow symlinks and copy the pointed to item + --crypt-directory-name-encryption Option to either encrypt directory names or leave them intact (default true) + --crypt-filename-encoding string How to encode the encrypted filename to text string (default "base32") + --crypt-filename-encryption string How to encrypt the filenames (default "standard") + --crypt-no-data-encryption Option to either encrypt file data or leave it unencrypted + --crypt-password string Password or pass phrase for encryption (obscured) + --crypt-password2 string Password or pass phrase for salt (obscured) + --crypt-remote string Remote to encrypt/decrypt + --crypt-server-side-across-configs Allow server-side operations (e.g. copy) to work across different crypt configs + --crypt-show-mapping For all files listed show how the names encrypt + --drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded + --drive-allow-import-name-change Allow the filetype to change when uploading Google docs + --drive-auth-owner-only Only consider files owned by the authenticated user + --drive-auth-url string Auth server URL + --drive-chunk-size SizeSuffix Upload chunk size (default 8Mi) + --drive-client-id string Google Application Client Id + --drive-client-secret string OAuth Client Secret + --drive-copy-shortcut-content Server side copy contents of shortcuts instead of the shortcut + --drive-disable-http2 Disable drive using http2 (default true) + --drive-encoding MultiEncoder The encoding for the backend (default InvalidUtf8) + --drive-export-formats string Comma separated list of preferred formats for downloading Google docs (default "docx,xlsx,pptx,svg") + --drive-formats string Deprecated: See export_formats + --drive-impersonate string Impersonate this user when using a service account + --drive-import-formats string Comma separated list of preferred formats for uploading Google docs + --drive-keep-revision-forever Keep new head revision of each file forever + --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) + --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) + --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) + --drive-resource-key string Resource key for accessing a link-shared file + --drive-root-folder-id string ID of the root folder + --drive-scope string Scope that rclone should use when requesting access from drive + --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs + --drive-service-account-credentials string Service Account Credentials JSON blob + --drive-service-account-file string Service Account Credentials JSON file path + --drive-shared-with-me Only show files that are shared with me + --drive-size-as-quota Show sizes as storage quota usage, not actual size + --drive-skip-checksum-gphotos Skip MD5 checksum on Google photos and videos only + --drive-skip-dangling-shortcuts If set skip dangling shortcut files + --drive-skip-gdocs Skip google documents in all listings + --drive-skip-shortcuts If set skip shortcut files + --drive-starred-only Only show files that are starred + --drive-stop-on-download-limit Make download limit errors be fatal + --drive-stop-on-upload-limit Make upload limit errors be fatal + --drive-team-drive string ID of the Shared Drive (Team Drive) + --drive-token string OAuth Access Token as a JSON blob + --drive-token-url string Token server url + --drive-trashed-only Only show files that are in the trash + --drive-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 8Mi) + --drive-use-created-date Use file created date instead of modified date + --drive-use-shared-date Use date file was shared instead of modified date + --drive-use-trash Send files to the trash instead of deleting permanently (default true) + --drive-v2-download-min-size SizeSuffix If Object's are greater, use drive v2 API to download (default off) + --dropbox-auth-url string Auth server URL + --dropbox-batch-commit-timeout Duration Max time to wait for a batch to finish committing (default 10m0s) + --dropbox-batch-mode string Upload file batching sync|async|off (default "sync") + --dropbox-batch-size int Max number of files in upload batch + --dropbox-batch-timeout Duration Max time to allow an idle upload batch before uploading (default 0s) + --dropbox-chunk-size SizeSuffix Upload chunk size (< 150Mi) (default 48Mi) + --dropbox-client-id string OAuth Client Id + --dropbox-client-secret string OAuth Client Secret + --dropbox-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,RightSpace,InvalidUtf8,Dot) + --dropbox-impersonate string Impersonate this user when using a business account + --dropbox-shared-files Instructs rclone to work on individual shared files + --dropbox-shared-folders Instructs rclone to work on shared folders + --dropbox-token string OAuth Access Token as a JSON blob + --dropbox-token-url string Token server url + --fichier-api-key string Your API Key, get it from https://1fichier.com/console/params.pl + --fichier-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot) + --fichier-file-password string If you want to download a shared file that is password protected, add this parameter (obscured) + --fichier-folder-password string If you want to list the files in a shared folder that is password protected, add this parameter (obscured) + --fichier-shared-folder string If you want to download a shared folder, add this parameter + --filefabric-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) + --filefabric-permanent-token string Permanent Authentication Token + --filefabric-root-folder-id string ID of the root folder + --filefabric-token string Session Token + --filefabric-token-expiry string Token expiry time + --filefabric-url string URL of the Enterprise File Fabric to connect to + --filefabric-version string Version read from the file fabric + --ftp-ask-password Allow asking for FTP password when needed + --ftp-close-timeout Duration Maximum time to wait for a response to close (default 1m0s) + --ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited + --ftp-disable-epsv Disable using EPSV even if server advertises support + --ftp-disable-mlsd Disable using MLSD even if server advertises support + --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) + --ftp-disable-utf8 Disable using UTF-8 even if server advertises support + --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) + --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) + --ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD + --ftp-host string FTP host to connect to + --ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --ftp-no-check-certificate Do not verify the TLS certificate of the server + --ftp-pass string FTP password (obscured) + --ftp-port int FTP port number (default 21) + --ftp-shut-timeout Duration Maximum time to wait for data connection closing status (default 1m0s) + --ftp-tls Use Implicit FTPS (FTP over TLS) + --ftp-tls-cache-size int Size of TLS session cache for all control and data connections (default 32) + --ftp-user string FTP username (default "$USER") + --ftp-writing-mdtm Use MDTM to set modification time (VsFtpd quirk) + --gcs-anonymous Access public buckets and objects without credentials + --gcs-auth-url string Auth server URL + --gcs-bucket-acl string Access Control List for new buckets + --gcs-bucket-policy-only Access checks should use bucket-level IAM policies + --gcs-client-id string OAuth Client Id + --gcs-client-secret string OAuth Client Secret + --gcs-decompress If set this will decompress gzip encoded objects + --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) + --gcs-endpoint string Endpoint for the service + --gcs-location string Location for the newly created buckets + --gcs-no-check-bucket If set, don't attempt to check the bucket exists or create it + --gcs-object-acl string Access Control List for new objects + --gcs-project-number string Project number + --gcs-service-account-file string Service Account Credentials JSON file path + --gcs-storage-class string The storage class to use when storing objects in Google Cloud Storage + --gcs-token string OAuth Access Token as a JSON blob + --gcs-token-url string Token server url + --gphotos-auth-url string Auth server URL + --gphotos-client-id string OAuth Client Id + --gphotos-client-secret string OAuth Client Secret + --gphotos-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) + --gphotos-include-archived Also view and download archived media + --gphotos-read-only Set to make the Google Photos backend read only + --gphotos-read-size Set to read the size of media items + --gphotos-start-year int Year limits the photos to be downloaded to those which are uploaded after the given year (default 2000) + --gphotos-token string OAuth Access Token as a JSON blob + --gphotos-token-url string Token server url + --hasher-auto-size SizeSuffix Auto-update checksum for files smaller than this size (disabled by default) + --hasher-hashes CommaSepList Comma separated list of supported checksum types (default md5,sha1) + --hasher-max-age Duration Maximum time to keep checksums in cache (0 = no cache, off = cache forever) (default off) + --hasher-remote string Remote to cache checksums for (e.g. myRemote:path) + --hdfs-data-transfer-protection string Kerberos data transfer protection: authentication|integrity|privacy + --hdfs-encoding MultiEncoder The encoding for the backend (default Slash,Colon,Del,Ctl,InvalidUtf8,Dot) + --hdfs-namenode string Hadoop name node and port + --hdfs-service-principal-name string Kerberos service principal name for the namenode + --hdfs-username string Hadoop user name + --hidrive-auth-url string Auth server URL + --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) + --hidrive-client-id string OAuth Client Id + --hidrive-client-secret string OAuth Client Secret + --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary + --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --hidrive-endpoint string Endpoint for the service (default "https://api.hidrive.strato.com/2.1") + --hidrive-root-prefix string The root/parent folder for all paths (default "/") + --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default "rw") + --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default "user") + --hidrive-token string OAuth Access Token as a JSON blob + --hidrive-token-url string Token server url + --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) + --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) + --http-headers CommaSepList Set HTTP headers for all transactions + --http-no-head Don't use HEAD requests + --http-no-slash Set this if the site doesn't end directories with / + --http-url string URL of HTTP host to connect to + --internetarchive-access-key-id string IAS3 Access Key + --internetarchive-disable-checksum Don't ask the server to test against MD5 checksum calculated by rclone (default true) + --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) + --internetarchive-endpoint string IAS3 Endpoint (default "https://s3.us.archive.org") + --internetarchive-front-endpoint string Host of InternetArchive Frontend (default "https://archive.org") + --internetarchive-secret-access-key string IAS3 Secret Key (password) + --internetarchive-wait-archive Duration Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s) + --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) + --jottacloud-hard-delete Delete files permanently rather than putting them into the trash + --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) + --jottacloud-no-versions Avoid server side versioning by deleting files and recreating files instead of overwriting them + --jottacloud-trashed-only Only show files that are in the trash + --jottacloud-upload-resume-limit SizeSuffix Files bigger than this can be resumed if the upload fail's (default 10Mi) + --koofr-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --koofr-endpoint string The Koofr API endpoint to use + --koofr-mountid string Mount ID of the mount to use + --koofr-password string Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured) + --koofr-provider string Choose your storage provider + --koofr-setmtime Does the backend support setting modification time (default true) + --koofr-user string Your user name + -l, --links Translate symlinks to/from regular files with a '.rclonelink' extension + --local-case-insensitive Force the filesystem to report itself as case insensitive + --local-case-sensitive Force the filesystem to report itself as case sensitive + --local-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --local-no-check-updated Don't check to see if the files change during upload + --local-no-preallocate Disable preallocation of disk space for transferred files + --local-no-set-modtime Disable setting modtime + --local-no-sparse Disable sparse files for multi-thread downloads + --local-nounc Disable UNC (long path names) conversion on Windows + --local-unicode-normalization Apply unicode NFC normalization to paths and filenames + --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) + --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) + --mailru-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --mailru-pass string Password (obscured) + --mailru-speedup-enable Skip full upload if there is another file with same data hash (default true) + --mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash) (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf") + --mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3Gi) + --mailru-speedup-max-memory SizeSuffix Files larger than the size given below will always be hashed on disk (default 32Mi) + --mailru-user string User name (usually email) + --mega-debug Output more debug from Mega + --mega-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --mega-hard-delete Delete files permanently rather than putting them into the trash + --mega-pass string Password (obscured) + --mega-user string User name + --netstorage-account string Set the NetStorage account name + --netstorage-host string Domain+path of NetStorage host to connect to + --netstorage-protocol string Select between HTTP or HTTPS protocol (default "https") + --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) + -x, --one-file-system Don't cross filesystem boundaries (unix/macOS only) + --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) + --onedrive-auth-url string Auth server URL + --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) + --onedrive-client-id string OAuth Client Id + --onedrive-client-secret string OAuth Client Secret + --onedrive-drive-id string The ID of the drive to use + --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) + --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) + --onedrive-expose-onenote-files Set to make OneNote files show up in directory listings + --onedrive-link-password string Set the password for links created by the link command + --onedrive-link-scope string Set the scope of the links created by the link command (default "anonymous") + --onedrive-link-type string Set the type of the links created by the link command (default "view") + --onedrive-list-chunk int Size of listing chunk (default 1000) + --onedrive-no-versions Remove all versions on modifying operations + --onedrive-region string Choose national cloud region for OneDrive (default "global") + --onedrive-root-folder-id string ID of the root folder + --onedrive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different onedrive configs + --onedrive-token string OAuth Access Token as a JSON blob + --onedrive-token-url string Token server url + --oos-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) + --oos-compartment string Object storage compartment OCID + --oos-config-file string Path to OCI config file (default "~/.oci/config") + --oos-config-profile string Profile name inside the oci config file (default "Default") + --oos-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) + --oos-copy-timeout Duration Timeout for copy (default 1m0s) + --oos-disable-checksum Don't store MD5 checksum with object metadata + --oos-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --oos-endpoint string Endpoint for Object storage API + --oos-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery + --oos-namespace string Object storage namespace + --oos-no-check-bucket If set, don't attempt to check the bucket exists or create it + --oos-provider string Choose your Auth Provider (default "env_auth") + --oos-region string Object storage Region + --oos-upload-concurrency int Concurrency for multipart uploads (default 10) + --oos-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size (default 10Mi) + --opendrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot) + --opendrive-password string Password (obscured) + --opendrive-username string Username + --pcloud-auth-url string Auth server URL + --pcloud-client-id string OAuth Client Id + --pcloud-client-secret string OAuth Client Secret + --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --pcloud-hostname string Hostname to connect to (default "api.pcloud.com") + --pcloud-password string Your pcloud password (obscured) + --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default "d0") + --pcloud-token string OAuth Access Token as a JSON blob + --pcloud-token-url string Token server url + --pcloud-username string Your pcloud username + --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --qingstor-access-key-id string QingStor Access Key ID + --qingstor-chunk-size SizeSuffix Chunk size to use for uploading (default 4Mi) + --qingstor-connection-retries int Number of connection retries (default 3) + --qingstor-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8) + --qingstor-endpoint string Enter an endpoint URL to connection QingStor API + --qingstor-env-auth Get QingStor credentials from runtime + --qingstor-secret-access-key string QingStor Secret Access Key (password) + --qingstor-upload-concurrency int Concurrency for multipart uploads (default 1) + --qingstor-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --qingstor-zone string Zone to connect to + --s3-access-key-id string AWS Access Key ID + --s3-acl string Canned ACL used when creating buckets and storing or copying objects + --s3-bucket-acl string Canned ACL used when creating buckets + --s3-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) + --s3-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) + --s3-decompress If set this will decompress gzip encoded objects + --s3-disable-checksum Don't store MD5 checksum with object metadata + --s3-disable-http2 Disable usage of http2 for S3 backends + --s3-download-url string Custom endpoint for downloads + --s3-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --s3-endpoint string Endpoint for S3 API + --s3-env-auth Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars) + --s3-force-path-style If true use path style access if false use virtual hosted style (default true) + --s3-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery + --s3-list-chunk int Size of listing chunk (response list for each ListObject S3 request) (default 1000) + --s3-list-url-encode Tristate Whether to url encode listings: true/false/unset (default unset) + --s3-list-version int Version of ListObjects to use: 1,2 or 0 for auto + --s3-location-constraint string Location constraint - must be set to match the Region + --s3-max-upload-parts int Maximum number of parts in a multipart upload (default 10000) + --s3-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --s3-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --s3-might-gzip Tristate Set this if the backend might gzip objects (default unset) + --s3-no-check-bucket If set, don't attempt to check the bucket exists or create it + --s3-no-head If set, don't HEAD uploaded objects to check integrity + --s3-no-head-object If set, do not do HEAD before GET when getting objects + --s3-no-system-metadata Suppress setting and reading of system metadata + --s3-profile string Profile to use in the shared credentials file + --s3-provider string Choose your S3 provider + --s3-region string Region to connect to + --s3-requester-pays Enables requester pays option when interacting with S3 bucket + --s3-secret-access-key string AWS Secret Access Key (password) + --s3-server-side-encryption string The server-side encryption algorithm used when storing this object in S3 + --s3-session-token string An AWS session token + --s3-shared-credentials-file string Path to the shared credentials file + --s3-sse-customer-algorithm string If using SSE-C, the server-side encryption algorithm used when storing this object in S3 + --s3-sse-customer-key string To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data + --s3-sse-customer-key-base64 string If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data + --s3-sse-customer-key-md5 string If using SSE-C you may provide the secret encryption key MD5 checksum (optional) + --s3-sse-kms-key-id string If using KMS ID you must provide the ARN of Key + --s3-storage-class string The storage class to use when storing new objects in S3 + --s3-upload-concurrency int Concurrency for multipart uploads (default 4) + --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint + --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) + --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads + --s3-v2-auth If true use v2 authentication + --s3-version-at Time Show file versions as they were at the specified time (default off) + --s3-versions Include old versions in directory listings + --seafile-2fa Two-factor authentication ('true' if the account has 2FA enabled) + --seafile-create-library Should rclone create a library if it doesn't exist + --seafile-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8) + --seafile-library string Name of the library + --seafile-library-key string Library password (for encrypted libraries only) (obscured) + --seafile-pass string Password (obscured) + --seafile-url string URL of seafile host to connect to + --seafile-user string User name (usually email address) + --sftp-ask-password Allow asking for SFTP password when needed + --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) + --sftp-ciphers SpaceSepList Space separated list of ciphers to be used for session encryption, ordered by preference + --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) + --sftp-disable-concurrent-reads If set don't use concurrent reads + --sftp-disable-concurrent-writes If set don't use concurrent writes + --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available + --sftp-host string SSH host to connect to + --sftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --sftp-key-exchange SpaceSepList Space separated list of key exchange algorithms, ordered by preference + --sftp-key-file string Path to PEM-encoded private key file + --sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file (obscured) + --sftp-key-pem string Raw PEM-encoded private key + --sftp-key-use-agent When set forces the usage of the ssh-agent + --sftp-known-hosts-file string Optional path to known_hosts file + --sftp-macs SpaceSepList Space separated list of MACs (message authentication code) algorithms, ordered by preference + --sftp-md5sum-command string The command used to read md5 hashes + --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) + --sftp-path-override string Override path used by SSH shell commands + --sftp-port int SSH port number (default 22) + --sftp-pubkey-file string Optional path to public key file + --sftp-server-command string Specifies the path or command to run a sftp server on the remote host + --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands + --sftp-set-modtime Set the modified time on the remote if set (default true) + --sftp-sha1sum-command string The command used to read sha1 hashes + --sftp-shell-type string The type of SSH shell on remote server, if any + --sftp-skip-links Set to skip any symlinks and any other non regular files + --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default "sftp") + --sftp-use-fstat If set use fstat instead of stat + --sftp-use-insecure-cipher Enable the use of insecure ciphers and key exchange methods + --sftp-user string SSH username (default "$USER") + --sharefile-chunk-size SizeSuffix Upload chunk size (default 64Mi) + --sharefile-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,LeftSpace,LeftPeriod,RightSpace,RightPeriod,InvalidUtf8,Dot) + --sharefile-endpoint string Endpoint for API calls + --sharefile-root-folder-id string ID of the root folder + --sharefile-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (default 128Mi) + --sia-api-password string Sia Daemon API Password (obscured) + --sia-api-url string Sia daemon API URL, like http://sia.daemon.host:9980 (default "http://127.0.0.1:9980") + --sia-encoding MultiEncoder The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot) + --sia-user-agent string Siad User Agent (default "Sia-Agent") + --skip-links Don't warn about skipped symlinks + --smb-case-insensitive Whether the server is configured to be case-insensitive (default true) + --smb-domain string Domain name for NTLM authentication (default "WORKGROUP") + --smb-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot) + --smb-hide-special-share Hide special shares (e.g. print$) which users aren't supposed to access (default true) + --smb-host string SMB server hostname to connect to + --smb-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --smb-pass string SMB password (obscured) + --smb-port int SMB port number (default 445) + --smb-user string SMB username (default "$USER") + --storj-access-grant string Access grant + --storj-api-key string API key + --storj-passphrase string Encryption passphrase + --storj-provider string Choose an authentication method (default "existing") + --storj-satellite-address string Satellite address (default "us-central-1.storj.io") + --sugarsync-access-key-id string Sugarsync Access Key ID + --sugarsync-app-id string Sugarsync App ID + --sugarsync-authorization string Sugarsync authorization + --sugarsync-authorization-expiry string Sugarsync authorization expiry + --sugarsync-deleted-id string Sugarsync deleted folder id + --sugarsync-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8,Dot) + --sugarsync-hard-delete Permanently delete files if true + --sugarsync-private-access-key string Sugarsync Private Access Key + --sugarsync-refresh-token string Sugarsync refresh token + --sugarsync-root-id string Sugarsync root id + --sugarsync-user string Sugarsync user + --swift-application-credential-id string Application Credential ID (OS_APPLICATION_CREDENTIAL_ID) + --swift-application-credential-name string Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME) + --swift-application-credential-secret string Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET) + --swift-auth string Authentication URL for server (OS_AUTH_URL) + --swift-auth-token string Auth Token from alternate authentication - optional (OS_AUTH_TOKEN) + --swift-auth-version int AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION) + --swift-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) + --swift-domain string User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME) + --swift-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8) + --swift-endpoint-type string Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default "public") + --swift-env-auth Get swift credentials from environment variables in standard OpenStack form + --swift-key string API key or password (OS_PASSWORD) + --swift-leave-parts-on-error If true avoid calling abort upload on a failure + --swift-no-chunk Don't chunk files during streaming upload + --swift-no-large-objects Disable support for static and dynamic large objects + --swift-region string Region name - optional (OS_REGION_NAME) + --swift-storage-policy string The storage policy to use when creating a new container + --swift-storage-url string Storage URL - optional (OS_STORAGE_URL) + --swift-tenant string Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME) + --swift-tenant-domain string Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME) + --swift-tenant-id string Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID) + --swift-user string User name to log in (OS_USERNAME) + --swift-user-id string User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID) + --union-action-policy string Policy to choose upstream on ACTION category (default "epall") + --union-cache-time int Cache time of usage and free space (in seconds) (default 120) + --union-create-policy string Policy to choose upstream on CREATE category (default "epmfs") + --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) + --union-search-policy string Policy to choose upstream on SEARCH category (default "ff") + --union-upstreams string List of space separated upstreams + --uptobox-access-token string Your access token + --uptobox-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot) + --webdav-bearer-token string Bearer token instead of user/pass (e.g. a Macaroon) + --webdav-bearer-token-command string Command to run to get a bearer token + --webdav-encoding string The encoding for the backend + --webdav-headers CommaSepList Set HTTP headers for all transactions + --webdav-pass string Password (obscured) + --webdav-url string URL of http host to connect to + --webdav-user string User name + --webdav-vendor string Name of the WebDAV site/service/software you are using + --yandex-auth-url string Auth server URL + --yandex-client-id string OAuth Client Id + --yandex-client-secret string OAuth Client Secret + --yandex-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) + --yandex-hard-delete Delete files permanently rather than putting them into the trash + --yandex-token string OAuth Access Token as a JSON blob + --yandex-token-url string Token server url + --zoho-auth-url string Auth server URL + --zoho-client-id string OAuth Client Id + --zoho-client-secret string OAuth Client Secret + --zoho-encoding MultiEncoder The encoding for the backend (default Del,Ctl,InvalidUtf8) + --zoho-region string Zoho region to connect to + --zoho-token string OAuth Access Token as a JSON blob + --zoho-token-url string Token server url ``` # Docker Volume Plugin @@ -17054,9 +17296,10 @@ Token server url - leave blank to use Amazon's. token_url> Optional token URL Remote config Make sure your Redirect URL is set to "http://127.0.0.1:53682/" in your custom config. -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -17329,6 +17572,7 @@ The S3 backend can be used with a number of different providers: - IBM COS S3 - IDrive e2 - IONOS Cloud +- Liara Object Storage - Minio - Qiniu Cloud Object Storage (Kodo) - RackCorp Object Storage @@ -17385,7 +17629,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS +XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Liara, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -17395,7 +17639,7 @@ Choose a number from below, or type in your own value \ "AWS" 2 / Ceph Object Storage \ "Ceph" - 3 / Digital Ocean Spaces + 3 / DigitalOcean Spaces \ "DigitalOcean" 4 / Dreamhost DreamObjects \ "Dreamhost" @@ -17950,7 +18194,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t ### Standard options -Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). +Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). #### --s3-provider @@ -17976,7 +18220,7 @@ Properties: - "ArvanCloud" - Arvan Cloud Object Storage (AOS) - "DigitalOcean" - - Digital Ocean Spaces + - DigitalOcean Spaces - "Dreamhost" - Dreamhost DreamObjects - "HuaweiOBS" @@ -17989,6 +18233,8 @@ Properties: - IONOS Cloud - "LyveCloud" - Seagate Lyve Cloud + - "Liara" + - Liara Object Storage - "Minio" - Minio Object Storage - "Netease" @@ -18340,7 +18586,7 @@ Properties: - Config: region - Env Var: RCLONE_S3_REGION -- Provider: !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive +- Provider: !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Liara,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive - Type: string - Required: false - Examples: @@ -18619,6 +18865,22 @@ Properties: #### --s3-endpoint +Endpoint for Liara Object Storage API. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Provider: Liara +- Type: string +- Required: false +- Examples: + - "storage.iran.liara.space" + - The default endpoint + - Iran + +#### --s3-endpoint + Endpoint for OSS API. Properties: @@ -18919,18 +19181,24 @@ Properties: - Config: endpoint - Env Var: RCLONE_S3_ENDPOINT -- Provider: !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu +- Provider: !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu - Type: string - Required: false - Examples: - "objects-us-east-1.dream.io" - Dream Objects endpoint + - "syd1.digitaloceanspaces.com" + - DigitalOcean Spaces Sydney 1 + - "sfo3.digitaloceanspaces.com" + - DigitalOcean Spaces San Francisco 3 + - "fra1.digitaloceanspaces.com" + - DigitalOcean Spaces Frankfurt 1 - "nyc3.digitaloceanspaces.com" - - Digital Ocean Spaces New York 3 + - DigitalOcean Spaces New York 3 - "ams3.digitaloceanspaces.com" - - Digital Ocean Spaces Amsterdam 3 + - DigitalOcean Spaces Amsterdam 3 - "sgp1.digitaloceanspaces.com" - - Digital Ocean Spaces Singapore 1 + - DigitalOcean Spaces Singapore 1 - "localhost:8333" - SeaweedFS S3 localhost - "s3.us-east-1.lyvecloud.seagate.com" @@ -18940,15 +19208,33 @@ Properties: - "s3.ap-southeast-1.lyvecloud.seagate.com" - Seagate Lyve Cloud AP Southeast 1 (Singapore) - "s3.wasabisys.com" - - Wasabi US East endpoint + - Wasabi US East 1 (N. Virginia) + - "s3.us-east-2.wasabisys.com" + - Wasabi US East 2 (N. Virginia) + - "s3.us-central-1.wasabisys.com" + - Wasabi US Central 1 (Texas) - "s3.us-west-1.wasabisys.com" - - Wasabi US West endpoint + - Wasabi US West 1 (Oregon) + - "s3.ca-central-1.wasabisys.com" + - Wasabi CA Central 1 (Toronto) - "s3.eu-central-1.wasabisys.com" - - Wasabi EU Central endpoint + - Wasabi EU Central 1 (Amsterdam) + - "s3.eu-central-2.wasabisys.com" + - Wasabi EU Central 2 (Frankfurt) + - "s3.eu-west-1.wasabisys.com" + - Wasabi EU West 1 (London) + - "s3.eu-west-2.wasabisys.com" + - Wasabi EU West 2 (Paris) - "s3.ap-northeast-1.wasabisys.com" - Wasabi AP Northeast 1 (Tokyo) endpoint - "s3.ap-northeast-2.wasabisys.com" - Wasabi AP Northeast 2 (Osaka) endpoint + - "s3.ap-southeast-1.wasabisys.com" + - Wasabi AP Southeast 1 (Singapore) + - "s3.ap-southeast-2.wasabisys.com" + - Wasabi AP Southeast 2 (Sydney) + - "storage.iran.liara.space" + - Liara Iran endpoint - "s3.ir-thr-at1.arvanstorage.com" - ArvanCloud Tehran Iran (Asiatech) endpoint @@ -19281,7 +19567,7 @@ Properties: - Config: location_constraint - Env Var: RCLONE_S3_LOCATION_CONSTRAINT -- Provider: !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS +- Provider: !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Liara,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS - Type: string - Required: false @@ -19296,6 +19582,10 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview Note that this ACL is applied when server-side copying objects as S3 doesn't copy the ACL from the source but rather writes a fresh one. +If the acl is an empty string then no X-Amz-Acl: header is added and +the default (private) will be used. + + Properties: - Config: acl @@ -19456,6 +19746,21 @@ Properties: #### --s3-storage-class +The storage class to use when storing new objects in Liara + +Properties: + +- Config: storage_class +- Env Var: RCLONE_S3_STORAGE_CLASS +- Provider: Liara +- Type: string +- Required: false +- Examples: + - "STANDARD" + - Standard storage class + +#### --s3-storage-class + The storage class to use when storing new objects in ArvanCloud. Properties: @@ -19534,7 +19839,7 @@ Properties: ### Advanced options -Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). +Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). #### --s3-bucket-acl @@ -19545,6 +19850,10 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview Note that this ACL is applied when only when creating buckets. If it isn't set then "acl" is used instead. +If the "acl" and "bucket_acl" are empty strings then no X-Amz-Acl: +header is added and the default (private) will be used. + + Properties: - Config: bucket_acl @@ -20167,6 +20476,37 @@ Properties: - Type: bool - Default: false +#### --s3-might-gzip + +Set this if the backend might gzip objects. + +Normally providers will not alter objects when they are downloaded. If +an object was not uploaded with `Content-Encoding: gzip` then it won't +be set on download. + +However some providers may gzip objects even if they weren't uploaded +with `Content-Encoding: gzip` (eg Cloudflare). + +A symptom of this would be receiving errors like + + ERROR corrupted on transfer: sizes differ NNN vs MMM + +If you set this flag and rclone downloads an object with +Content-Encoding: gzip set and chunked transfer encoding, then rclone +will decompress the object on the fly. + +If this is set to unset (the default) then rclone will choose +according to the provider setting what to apply, but you can override +rclone's choice here. + + +Properties: + +- Config: might_gzip +- Env Var: RCLONE_S3_MIGHT_GZIP +- Type: Tristate +- Default: unset + #### --s3-no-system-metadata Suppress setting and reading of system metadata @@ -20499,7 +20839,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. ... -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) ... Storage> s3 @@ -20668,7 +21008,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. [snip] - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) [snip] Storage> 5 @@ -20796,7 +21136,7 @@ Choose a number from below, or type in your own value \ "alias" 2 / Amazon Drive \ "amazon cloud drive" - 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS) + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, Liara, ArvanCloud, Minio, IBM COS) \ "s3" 4 / Backblaze B2 \ "b2" @@ -20963,7 +21303,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) [snip] Storage> s3 @@ -21069,7 +21409,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) [snip] Storage> s3 @@ -21315,7 +21655,7 @@ Choose a number from below, or type in your own value \ (alias) 4 / Amazon Drive \ (amazon cloud drive) - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi \ (s3) [snip] Storage> s3 @@ -21555,7 +21895,7 @@ Choose `s3` backend Type of storage to configure. Choose a number from below, or type in your own value. [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \ (s3) [snip] Storage> s3 @@ -21742,7 +22082,7 @@ name> wasabi Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, Liara) \ "s3" [snip] Storage> s3 @@ -21856,7 +22196,7 @@ Type of storage to configure. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [snip] - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -21966,7 +22306,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. ... - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \ (s3) ... Storage> s3 @@ -22196,6 +22536,107 @@ d) Delete this remote y/e/d> y ``` +### Liara {#liara-cloud} + +Here is an example of making a [Liara Object Storage](https://liara.ir/landing/object-storage) +configuration. First run: + + rclone config + +This will guide you through an interactive setup process. + +``` +No remotes found, make a new one? +n) New remote +s) Set configuration password +n/s> n +name> Liara +Type of storage to configure. +Choose a number from below, or type in your own value +[snip] +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Liara, Minio) + \ "s3" +[snip] +Storage> s3 +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" +env_auth> 1 +AWS Access Key ID - leave blank for anonymous access or runtime credentials. +access_key_id> YOURACCESSKEY +AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. +secret_access_key> YOURSECRETACCESSKEY +Region to connect to. +Choose a number from below, or type in your own value + / The default endpoint + 1 | US Region, Northern Virginia, or Pacific Northwest. + | Leave location constraint empty. + \ "us-east-1" +[snip] +region> +Endpoint for S3 API. +Leave blank if using Liara to use the default endpoint for the region. +Specify if using an S3 clone such as Ceph. +endpoint> storage.iran.liara.space +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" +[snip] +acl> +The server-side encryption algorithm used when storing this object in S3. +Choose a number from below, or type in your own value + 1 / None + \ "" + 2 / AES256 + \ "AES256" +server_side_encryption> +The storage class to use when storing objects in S3. +Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Standard storage class + \ "STANDARD" +storage_class> +Remote config +-------------------- +[Liara] +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +endpoint = storage.iran.liara.space +location_constraint = +acl = +server_side_encryption = +storage_class = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +``` + +This will leave the config file looking like this. + +``` +[Liara] +type = s3 +provider = Liara +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +region = +endpoint = storage.iran.liara.space +location_constraint = +acl = +server_side_encryption = +storage_class = +``` + ### ArvanCloud {#arvan-cloud} [ArvanCloud](https://www.arvancloud.com/en/products/cloud-storage) ArvanCloud Object Storage goes beyond the limited traditional file storage. @@ -22214,7 +22655,7 @@ name> ArvanCloud Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Liara, Minio) \ "s3" [snip] Storage> s3 @@ -22337,7 +22778,7 @@ Choose a number from below, or type in your own value \ "alias" 3 / Amazon Drive \ "amazon cloud drive" - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -23184,9 +23625,10 @@ Choose a number from below, or type in your own value \ "enterprise" box_sub_type> Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -23303,9 +23745,10 @@ Already have a token - refresh? y) Yes n) No y/n> y -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -23589,6 +24032,8 @@ Reverse Solidus). Box only supports filenames up to 255 characters in length. +Box has [API rate limits](https://developer.box.com/guides/api-calls/permissions-and-errors/rate-limits/) that sometimes reduce the speed of rclone. + `rclone about` is not supported by the Box backend. Backends without this capability cannot determine free space for an rclone mount or use policy `mfs` (most free space) as a member of an rclone union @@ -23596,7 +24041,7 @@ remote. See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) -# Cache (DEPRECATED) +# Cache The `cache` remote wraps another existing remote and stores file structure and its data for long running tasks like `rclone mount`. @@ -24279,7 +24724,7 @@ Print stats on the cache backend in JSON format. -# Chunker (BETA) +# Chunker The `chunker` overlay transparently splits large files into smaller chunks during upload to wrapped remote and transparently assembles them back @@ -24798,9 +25243,10 @@ y) Yes n) No y/n> n Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -24995,7 +25441,7 @@ remote. See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/) -# Crypt +# Crypt Rclone `crypt` remotes encrypt and decrypt other remotes. @@ -25761,7 +26207,7 @@ a salt. * [rclone cryptdecode](https://rclone.org/commands/rclone_cryptdecode/) - Show forward/reverse mapping of encrypted filenames -# Compress (Experimental) +# Compress ## Warning @@ -26132,6 +26578,16 @@ d) Delete this remote y/e/d> y ``` +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Dropbox. This only +runs from the moment it opens your browser to the moment you get back +the verification code. This is on `http://127.0.0.1:53682/` and it +may require you to unblock it temporarily if you are running a host +firewall, or use manual mode. + You can then use it like this, List directories in top level of your dropbox @@ -27046,7 +27502,7 @@ Use Implicit FTPS (FTP over TLS). When using implicit FTP over TLS the client connects using TLS right from the start which breaks compatibility with non-TLS-aware servers. This is usually served over port 990 rather -than port 21. Cannot be used in combination with explicit FTP. +than port 21. Cannot be used in combination with explicit FTPS. Properties: @@ -27061,7 +27517,7 @@ Use Explicit FTPS (FTP over TLS). When using explicit FTP over TLS the client explicitly requests security from the server in order to upgrade a plain text connection -to an encrypted one. Cannot be used in combination with implicit FTP. +to an encrypted one. Cannot be used in combination with implicit FTPS. Properties: @@ -27430,9 +27886,10 @@ Choose a number from below, or type in your own value \ "DURABLE_REDUCED_AVAILABILITY" storage_class> 5 Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine or Y didn't work +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -27456,8 +27913,12 @@ d) Delete this remote y/e/d> y ``` +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. This only +token as returned from Google if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on `http://127.0.0.1:53682/` and this it may require you to unblock it temporarily if you are running a host @@ -28030,9 +28491,10 @@ scope> 1 Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. service_account_file> Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine or Y didn't work +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -28059,8 +28521,12 @@ d) Delete this remote y/e/d> y ``` +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. This only +token as returned from Google if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on `http://127.0.0.1:53682/` and it may require you to unblock it temporarily if you are running a host @@ -29572,9 +30038,10 @@ y) Yes n) No y/n> n Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -29598,8 +30065,12 @@ d) Delete this remote y/e/d> y ``` +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. This only +token as returned from Google if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on `http://127.0.0.1:53682/` and this may require you to unblock it temporarily if you are running a host @@ -29978,7 +30449,7 @@ Rclone cannot delete files anywhere except under `album`. The Google Photos API does not support deleting albums - see [bug #135714733](https://issuetracker.google.com/issues/135714733). -# Hasher (EXPERIMENTAL) +# Hasher Hasher is a special overlay backend to create remotes which handle checksums for other remotes. It's main functions include: @@ -30594,7 +31065,10 @@ Leave blank normally. scope_access> Edit advanced config? y/n> n -Use auto config? +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y/n> y If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx Log in and authorize rclone for access @@ -31272,6 +31746,22 @@ It can be triggered when you did a server-side copy. Reading metadata will also provide custom (non-standard nor reserved) ones. +## Filtering auto generated files + +The Internet Archive automatically creates metadata files after +upload. These can cause problems when doing an `rclone sync` as rclone +will try, and fail, to delete them. These metadata files are not +changeable, as they are created by the Internet Archive automatically. + +These auto-created files can be excluded from the sync using [metadata +filtering](https://rclone.org/filtering/#metadata). + + rclone sync ... --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata" + +Which excludes from the sync any files which have the +`source=metadata` or `format=Metadata` flags which are added to +Internet Archive auto-created files. + ## Configuration Here is an example of making an internetarchive configuration. @@ -32271,8 +32761,21 @@ Currently it is recommended to disable 2FA on Mail.ru accounts intended for rclo ## Configuration -Here is an example of making a mailru configuration. First create a Mail.ru Cloud -account and choose a tariff, then run +Here is an example of making a mailru configuration. + +First create a Mail.ru Cloud account and choose a tariff. + +You will need to log in and create an app password for rclone. Rclone +**will not work** with your normal username and password - it will +give an error like `oauth2: server response missing access_token`. + +- Click on your user icon in the top right +- Go to Security / "Пароль и безопасность" +- Click password for apps / "Пароли для внешних приложений" +- Add the password - give it a name - eg "rclone" +- Copy the password and use this password below - your normal login password won't work. + +Now run rclone config @@ -32298,6 +32801,10 @@ User name (usually email) Enter a string value. Press Enter for the default (""). user> username@mail.ru Password + +This must be an app password - rclone will not work with your normal +password. See the Configuration section in the docs for how to make an +app password. y) Yes type in my own password g) Generate random password y/g> y @@ -32420,6 +32927,11 @@ Properties: Password. +This must be an app password - rclone will not work with your normal +password. See the Configuration section in the docs for how to make an +app password. + + **NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/). Properties: @@ -33274,7 +33786,13 @@ docs](https://rclone.org/docs/#fast-list) for more details. The modified time is stored as metadata on the object with the `mtime` key. It is stored using RFC3339 Format time with nanosecond precision. The metadata is supplied during directory listings so -there is no overhead to using it. +there is no performance overhead to using it. + +If you wish to use the Azure standard `LastModified` time stored on +the object as the modified time, then use the `--use-server-modtime` +flag. Note that rclone can't set `LastModified`, so using the +`--update` flag when syncing is recommended if using +`--use-server-modtime`. ### Performance @@ -33310,11 +33828,80 @@ MD5 hashes are stored with blobs. However blobs that were uploaded in chunks only have an MD5 if the source remote was capable of MD5 hashes, e.g. the local disk. -### Authenticating with Azure Blob Storage +### Authentication {#authentication} + +There are a number of ways of supplying credentials for Azure Blob +Storage. Rclone tries them in the order of the sections below. + +#### Env Auth + +If the `env_auth` config parameter is `true` then rclone will pull +credentials from the environment or runtime. + +It tries these authentication methods in this order: + +1. Environment Variables +2. Managed Service Identity Credentials +3. Azure CLI credentials (as used by the az tool) + +These are described in the following sections + +##### Env Auth: 1. Environment Variables + +If `env_auth` is set and environment variables are present rclone +authenticates a service principal with a secret or certificate, or a +user with a password, depending on which environment variable are set. +It reads configuration from these variables, in the following order: + +1. Service principal with client secret + - `AZURE_TENANT_ID`: ID of the service principal's tenant. Also called its "directory" ID. + - `AZURE_CLIENT_ID`: the service principal's client ID + - `AZURE_CLIENT_SECRET`: one of the service principal's client secrets +2. Service principal with certificate + - `AZURE_TENANT_ID`: ID of the service principal's tenant. Also called its "directory" ID. + - `AZURE_CLIENT_ID`: the service principal's client ID + - `AZURE_CLIENT_CERTIFICATE_PATH`: path to a PEM or PKCS12 certificate file including the private key. + - `AZURE_CLIENT_CERTIFICATE_PASSWORD`: (optional) password for the certificate file. + - `AZURE_CLIENT_SEND_CERTIFICATE_CHAIN`: (optional) Specifies whether an authentication request will include an x5c header to support subject name / issuer based authentication. When set to "true" or "1", authentication requests include the x5c header. +3. User with username and password + - `AZURE_TENANT_ID`: (optional) tenant to authenticate in. Defaults to "organizations". + - `AZURE_CLIENT_ID`: client ID of the application the user will authenticate to + - `AZURE_USERNAME`: a username (usually an email address) + - `AZURE_PASSWORD`: the user's password + +##### Env Auth: 2. Managed Service Identity Credentials + +When using Managed Service Identity if the VM(SS) on which this +program is running has a system-assigned identity, it will be used by +default. If the resource has no system-assigned but exactly one +user-assigned identity, the user-assigned identity will be used by +default. + +If the resource has multiple user-assigned identities you will need to +unset `env_auth` and set `use_msi` instead. See the [`use_msi` +section](#use_msi). + +##### Env Auth: 3. Azure CLI credentials (as used by the az tool) + +Credentials created with the `az` tool can be picked up using `env_auth`. -Rclone has 3 ways of authenticating with Azure Blob Storage: +For example if you were to login with a service principal like this: -#### Account and Key + az login --service-principal -u XXX -p XXX --tenant XXX + +Then you could access rclone resources like this: + + rclone lsf :azureblob,env_auth,account=ACCOUNT:CONTAINER + +Or + + rclone lsf --azureblob-env-auth --azureblob-acccount=ACCOUNT :azureblob:CONTAINER + +Which is analogous to using the `az` tool: + + az storage blob list --container-name CONTAINER --account-name ACCOUNT --auth-mode login + +#### Account and Shared Key This is the most straight forward and least flexible way. Just fill in the `account` and `key` lines and leave the rest blank. @@ -33323,7 +33910,7 @@ in the `account` and `key` lines and leave the rest blank. This can be an account level SAS URL or container level SAS URL. -To use it leave `account`, `key` blank and fill in `sas_url`. +To use it leave `account` and `key` blank and fill in `sas_url`. An account level SAS URL or container level SAS URL can be obtained from the Azure portal or the Azure Storage Explorer. To get a @@ -33350,6 +33937,60 @@ Container level SAS URLs are useful for temporarily allowing third parties access to a single container or putting credentials into an untrusted environment such as a CI build server. +#### Service principal with client secret + +If these variables are set, rclone will authenticate with a service principal with a client secret. + +- `tenant`: ID of the service principal's tenant. Also called its "directory" ID. +- `client_id`: the service principal's client ID +- `client_secret`: one of the service principal's client secrets + +The credentials can also be placed in a file using the +`service_principal_file` configuration option. + +#### Service principal with certificate + +If these variables are set, rclone will authenticate with a service principal with certificate. + +- `tenant`: ID of the service principal's tenant. Also called its "directory" ID. +- `client_id`: the service principal's client ID +- `client_certificate_path`: path to a PEM or PKCS12 certificate file including the private key. +- `client_certificate_password`: (optional) password for the certificate file. +- `client_send_certificate_chain`: (optional) Specifies whether an authentication request will include an x5c header to support subject name / issuer based authentication. When set to "true" or "1", authentication requests include the x5c header. + +**NB** `client_certificate_password` must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/). + +#### User with username and password + +If these variables are set, rclone will authenticate with username and password. + +- `tenant`: (optional) tenant to authenticate in. Defaults to "organizations". +- `client_id`: client ID of the application the user will authenticate to +- `username`: a username (usually an email address) +- `password`: the user's password + +Microsoft doesn't recommend this kind of authentication, because it's +less secure than other authentication flows. This method is not +interactive, so it isn't compatible with any form of multi-factor +authentication, and the application must already have user or admin +consent. This credential can only authenticate work and school +accounts; it can't authenticate Microsoft accounts. + +**NB** `password` must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/). + +#### Managed Service Identity Credentials {#use_msi} + +If `use_msi` is set then managed service identity credentials are +used. This authentication only works when running in an Azure service. +`env_auth` needs to be unset to use this. + +However if you have multiple user identities to choose from these must +be explicitly specified using exactly one of the `msi_object_id`, +`msi_client_id`, or `msi_mi_res_id` parameters. + +If none of `msi_object_id`, `msi_client_id`, or `msi_mi_res_id` is +set, this is is equivalent to using `env_auth`. + ### Standard options @@ -33357,9 +33998,15 @@ Here are the Standard options specific to azureblob (Microsoft Azure Blob Storag #### --azureblob-account -Storage Account Name. +Azure Storage Account Name. + +Set this to the Azure Storage Account Name in use. + +Leave blank to use SAS URL or Emulator, otherwise it needs to be set. + +If this is blank and if env_auth is set it will be read from the +environment variable `AZURE_STORAGE_ACCOUNT_NAME` if possible. -Leave blank to use SAS URL or Emulator. Properties: @@ -33368,30 +34015,22 @@ Properties: - Type: string - Required: false -#### --azureblob-service-principal-file - -Path to file containing credentials for use with a service principal. - -Leave blank normally. Needed only if you want to use a service principal instead of interactive login. - - $ az ad sp create-for-rbac --name "" \ - --role "Storage Blob Data Owner" \ - --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \ - > azure-principal.json +#### --azureblob-env-auth -See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details. +Read credentials from runtime (environment variables, CLI or MSI). +See the [authentication docs](/azureblob#authentication) for full info. Properties: -- Config: service_principal_file -- Env Var: RCLONE_AZUREBLOB_SERVICE_PRINCIPAL_FILE -- Type: string -- Required: false +- Config: env_auth +- Env Var: RCLONE_AZUREBLOB_ENV_AUTH +- Type: bool +- Default: false #### --azureblob-key -Storage Account Key. +Storage Account Shared Key. Leave blank to use SAS URL or Emulator. @@ -33415,6 +34054,169 @@ Properties: - Type: string - Required: false +#### --azureblob-tenant + +ID of the service principal's tenant. Also called its directory ID. + +Set this if using +- Service principal with client secret +- Service principal with certificate +- User with username and password + + +Properties: + +- Config: tenant +- Env Var: RCLONE_AZUREBLOB_TENANT +- Type: string +- Required: false + +#### --azureblob-client-id + +The ID of the client in use. + +Set this if using +- Service principal with client secret +- Service principal with certificate +- User with username and password + + +Properties: + +- Config: client_id +- Env Var: RCLONE_AZUREBLOB_CLIENT_ID +- Type: string +- Required: false + +#### --azureblob-client-secret + +One of the service principal's client secrets + +Set this if using +- Service principal with client secret + + +Properties: + +- Config: client_secret +- Env Var: RCLONE_AZUREBLOB_CLIENT_SECRET +- Type: string +- Required: false + +#### --azureblob-client-certificate-path + +Path to a PEM or PKCS12 certificate file including the private key. + +Set this if using +- Service principal with certificate + + +Properties: + +- Config: client_certificate_path +- Env Var: RCLONE_AZUREBLOB_CLIENT_CERTIFICATE_PATH +- Type: string +- Required: false + +#### --azureblob-client-certificate-password + +Password for the certificate file (optional). + +Optionally set this if using +- Service principal with certificate + +And the certificate has a password. + + +**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/). + +Properties: + +- Config: client_certificate_password +- Env Var: RCLONE_AZUREBLOB_CLIENT_CERTIFICATE_PASSWORD +- Type: string +- Required: false + +### Advanced options + +Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage). + +#### --azureblob-client-send-certificate-chain + +Send the certificate chain when using certificate auth. + +Specifies whether an authentication request will include an x5c header +to support subject name / issuer based authentication. When set to +true, authentication requests include the x5c header. + +Optionally set this if using +- Service principal with certificate + + +Properties: + +- Config: client_send_certificate_chain +- Env Var: RCLONE_AZUREBLOB_CLIENT_SEND_CERTIFICATE_CHAIN +- Type: bool +- Default: false + +#### --azureblob-username + +User name (usually an email address) + +Set this if using +- User with username and password + + +Properties: + +- Config: username +- Env Var: RCLONE_AZUREBLOB_USERNAME +- Type: string +- Required: false + +#### --azureblob-password + +The user's password + +Set this if using +- User with username and password + + +**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/). + +Properties: + +- Config: password +- Env Var: RCLONE_AZUREBLOB_PASSWORD +- Type: string +- Required: false + +#### --azureblob-service-principal-file + +Path to file containing credentials for use with a service principal. + +Leave blank normally. Needed only if you want to use a service principal instead of interactive login. + + $ az ad sp create-for-rbac --name "" \ + --role "Storage Blob Data Owner" \ + --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \ + > azure-principal.json + +See ["Create an Azure service principal"](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) and ["Assign an Azure role for access to blob data"](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) pages for more details. + +It may be more convenient to put the credentials directly into the +rclone config file under the `client_id`, `tenant` and `client_secret` +keys instead of setting `service_principal_file`. + + +Properties: + +- Config: service_principal_file +- Env Var: RCLONE_AZUREBLOB_SERVICE_PRINCIPAL_FILE +- Type: string +- Required: false + #### --azureblob-use-msi Use a managed service identity to authenticate (only works in Azure). @@ -33435,23 +34237,6 @@ Properties: - Type: bool - Default: false -#### --azureblob-use-emulator - -Uses local storage emulator if provided as 'true'. - -Leave blank if using real azure storage endpoint. - -Properties: - -- Config: use_emulator -- Env Var: RCLONE_AZUREBLOB_USE_EMULATOR -- Type: bool -- Default: false - -### Advanced options - -Here are the Advanced options specific to azureblob (Microsoft Azure Blob Storage). - #### --azureblob-msi-object-id Object ID of the user-assigned MSI to use, if any. @@ -33491,6 +34276,19 @@ Properties: - Type: string - Required: false +#### --azureblob-use-emulator + +Uses local storage emulator if provided as 'true'. + +Leave blank if using real azure storage endpoint. + +Properties: + +- Config: use_emulator +- Env Var: RCLONE_AZUREBLOB_USE_EMULATOR +- Type: bool +- Default: false + #### --azureblob-endpoint Endpoint for the service. @@ -33694,6 +34492,21 @@ Properties: - "container" - Allow full public read access for container and blob data. +#### --azureblob-no-check-container + +If set, don't attempt to check the container exists or create it. + +This can be useful when trying to minimise the number of transactions +rclone does if you know the container exists already. + + +Properties: + +- Config: no_check_container +- Env Var: RCLONE_AZUREBLOB_NO_CHECK_CONTAINER +- Type: bool +- Default: false + #### --azureblob-no-head-object If set, do not do HEAD before GET when getting objects. @@ -33707,6 +34520,18 @@ Properties: +### Custom upload headers + +You can set custom upload headers with the `--header-upload` flag. + +- Cache-Control +- Content-Disposition +- Content-Encoding +- Content-Language +- Content-Type + +Eg `--header-upload "Content-Type: text/potato"` + ## Limitations MD5 sums are only uploaded with chunked files if the source has an MD5 @@ -33721,11 +34546,20 @@ See [List of backends that do not support rclone about](https://rclone.org/overv ## Azure Storage Emulator Support -You can run rclone with storage emulator (usually _azurite_). +You can run rclone with the storage emulator (usually _azurite_). -To do this, just set up a new remote with `rclone config` following instructions described in introduction and set `use_emulator` config as `true`. You do not need to provide default account name neither an account key. +To do this, just set up a new remote with `rclone config` following +the instructions in the introduction and set `use_emulator` in the +advanced settings as `true`. You do not need to provide a default +account name nor an account key. But you can override them in the +`account` and `key` options. (Prior to v1.61 they were hard coded to +_azurite_'s `devstoreaccount1`.) -Also, if you want to access a storage emulator instance running on a different machine, you can override _Endpoint_ parameter in advanced settings, setting it to `http(s)://:/devstoreaccount1` (e.g. `http://10.254.2.5:10000/devstoreaccount1`). +Also, if you want to access a storage emulator instance running on a +different machine, you can override the `endpoint` parameter in the +advanced settings, setting it to +`http(s)://:/devstoreaccount1` +(e.g. `http://10.254.2.5:10000/devstoreaccount1`). # Microsoft OneDrive @@ -33776,9 +34610,10 @@ y) Yes n) No y/n> n Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -34437,6 +35272,31 @@ permissions as an admin, take a look at the docs: [1](https://docs.microsoft.com/en-us/sharepoint/turn-external-sharing-on-or-off), [2](https://support.microsoft.com/en-us/office/set-up-and-manage-access-requests-94b26e0b-2822-49d4-929a-8455698654b3). +### Can not access `Shared` with me files + +Shared with me files is not supported by rclone [currently](https://github.com/rclone/rclone/issues/4062), but there is a workaround: + +1. Visit [https://onedrive.live.com](https://onedrive.live.com/) +2. Right click a item in `Shared`, then click `Add shortcut to My files` in the context +
      + Screenshot (Shared with me) + + ![make_shortcut](https://user-images.githubusercontent.com/60313789/206118040-7e762b3b-aa61-41a1-8649-cc18889f3572.png) +
      + +3. The shortcut will appear in `My files`, you can access it with rclone, it behaves like a normal folder/file. +
      + Screenshot (My Files) + + ![in_my_files](https://i.imgur.com/0S8H3li.png) +
      + +
      + Screenshot (rclone mount) + + ![rclone_mount](https://i.imgur.com/2Iq66sW.png) +
      + # OpenDrive Paths are specified as `remote:path` @@ -35661,7 +36521,7 @@ Properties: signs in file names. rclone will transparently [encode](https://rclone.org/overview/#encoding) them for you, but you'd better be aware -# Swift +# Swift Swift refers to [OpenStack Object Storage](https://docs.openstack.org/swift/latest/). Commercial implementations of that being: @@ -36316,9 +37176,10 @@ client_id> Pcloud App Client Secret - leave blank normally. client_secret> Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -36600,9 +37461,10 @@ Storage> premiumizeme ** See help for premiumizeme backend at: https://rclone.org/premiumizeme/ ** Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -36749,9 +37611,10 @@ Storage> putio ** See help for putio backend at: https://rclone.org/putio/ ** Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -36784,8 +37647,12 @@ q) Quit config e/n/d/r/c/s/q> q ``` +See the [remote setup docs](https://rclone.org/remote_setup/) for how to set it up on a +machine with no Internet browser available. + Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. This only +token as returned from put.io if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on `http://127.0.0.1:53682/` and this it may require you to unblock it temporarily if you are running a host @@ -36846,7 +37713,7 @@ If you want to avoid ever hitting these limits, you may use the `--tpslimit` flag with a low number. Note that the imposed limits may be different for different operations, and may change over time. -# Seafile +# Seafile This is a backend for the [Seafile](https://www.seafile.com/) storage service: - It works with both the free community edition or the professional edition. @@ -37753,6 +38620,9 @@ This enables the use of the following insecure ciphers and key exchange methods: Those algorithms are insecure and may allow plaintext data to be recovered by an attacker. +This must be false if you use either ciphers or key_exchange advanced options. + + Properties: - Config: use_insecure_cipher @@ -38085,6 +38955,64 @@ Properties: - Type: SpaceSepList - Default: +#### --sftp-ciphers + +Space separated list of ciphers to be used for session encryption, ordered by preference. + +At least one must match with server configuration. This can be checked for example using ssh -Q cipher. + +This must not be set if use_insecure_cipher is true. + +Example: + + aes128-ctr aes192-ctr aes256-ctr aes128-gcm@openssh.com aes256-gcm@openssh.com + + +Properties: + +- Config: ciphers +- Env Var: RCLONE_SFTP_CIPHERS +- Type: SpaceSepList +- Default: + +#### --sftp-key-exchange + +Space separated list of key exchange algorithms, ordered by preference. + +At least one must match with server configuration. This can be checked for example using ssh -Q kex. + +This must not be set if use_insecure_cipher is true. + +Example: + + sntrup761x25519-sha512@openssh.com curve25519-sha256 curve25519-sha256@libssh.org ecdh-sha2-nistp256 + + +Properties: + +- Config: key_exchange +- Env Var: RCLONE_SFTP_KEY_EXCHANGE +- Type: SpaceSepList +- Default: + +#### --sftp-macs + +Space separated list of MACs (message authentication code) algorithms, ordered by preference. + +At least one must match with server configuration. This can be checked for example using ssh -Q mac. + +Example: + + umac-64-etm@openssh.com umac-128-etm@openssh.com hmac-sha2-256-etm@openssh.com + + +Properties: + +- Config: macs +- Env Var: RCLONE_SFTP_MACS +- Type: SpaceSepList +- Default: + ## Limitations @@ -39860,7 +40788,7 @@ vendor = other bearer_token_command = oidc-token XDC ``` -# Yandex Disk +# Yandex Disk [Yandex Disk](https://disk.yandex.com) is a cloud storage solution created by [Yandex](https://yandex.com). @@ -39890,9 +40818,10 @@ client_id> Yandex Client Secret - leave blank normally. client_secret> Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -40087,7 +41016,7 @@ Token generation will work without a mail account, but Rclone won't be able to c [403 - DiskUnsupportedUserAccountTypeError] User account type is not supported. ``` -# Zoho Workdrive +# Zoho Workdrive [Zoho WorkDrive](https://www.zoho.com/workdrive/) is a cloud storage solution created by [Zoho](https://zoho.com). @@ -40128,9 +41057,10 @@ y) Yes n) No (default) y/n> n Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes (default) n) No y/n> @@ -40760,8 +41690,8 @@ Properties: Don't check to see if the files change during upload. Normally rclone checks the size and modification time of files as they -are being uploaded and aborts with a message which starts "can't copy -- source file is being updated" if the file changes during upload. +are being uploaded and aborts with a message which starts "can't copy - +source file is being updated" if the file changes during upload. However on some file systems this modification time check may fail (e.g. [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this @@ -40954,6 +41884,135 @@ Options: # Changelog +## v1.61.0 - 2022-12-20 + +[See commits](https://github.com/rclone/rclone/compare/v1.60.0...v1.61.0) + +* New backends + * New S3 providers + * [Liara LOS](https://rclone.org/s3/#liara-cloud) (MohammadReza) +* New Features + * build: Add vulnerability testing using govulncheck (albertony) + * cmd: Enable `SIGINFO` (Ctrl-T) handler on FreeBSD, NetBSD, OpenBSD and Dragonfly BSD (x3-apptech) + * config: Add [config/setpath](https://rclone.org/rc/#config-setpath) for setting config path via rc/librclone (Nick Craig-Wood) + * dedupe + * Count Checks in the stats while scanning for duplicates (Nick Craig-Wood) + * Make dedupe obey the filters (Nick Craig-Wood) + * dlna: Properly attribute code used from https://github.com/anacrolix/dms (Nick Craig-Wood) + * docs + * Add minimum versions and status badges to backend and command docs (Nick Craig-Wood, albertony) + * Remote names may not start or end with space (albertony) + * filter: Add metadata filters [--metadata-include/exclude/filter](https://rclone.org/filtering/#metadata) and friends (Nick Craig-Wood) + * fs + * Make all duration flags take `y`, `M`, `w`, `d` etc suffixes (Nick Craig-Wood) + * Add global flag `--color` to control terminal colors (Kevin Verstaen) + * fspath: Allow unicode numbers and letters in remote names (albertony) + * lib/file: Improve error message for creating dir on non-existent network host on windows (albertony) + * lib/http: Finish port of rclone servers to `lib/http` (Tom Mombourquette, Nick Craig-Wood) + * lib/oauthutil: Improved usability of config flows needing web browser (Ole Frost) + * ncdu + * Add support for modification time (albertony) + * Fallback to sort by name also for sort by average size (albertony) + * Rework to use tcell directly instead of the termbox wrapper (eNV25) + * rc: Add commands to set [GC Percent](https://rclone.org/rc/#debug-set-gc-percent) & [Memory Limit](/rc/#debug-set-soft-memory-limit) (go 1.19+) (Anagh Kumar Baranwal) + * rcat: Preserve metadata when Copy falls back to Rcat (Nick Craig-Wood) + * rcd: Refactor rclone rc server to use `lib/http` (Nick Craig-Wood) + * rcserver: Avoid generating default credentials with htpasswd (Kamui) + * restic: Refactor to use `lib/http` (Nolan Woods) + * serve http: Support unix sockets and multiple listeners (Tom Mombourquette) + * serve webdav: Refactor to use `lib/http` (Nick Craig-Wood) + * test: Replace defer cleanup with `t.Cleanup` (Eng Zer Jun) + * test memory: Read metadata if `-M` flag is specified (Nick Craig-Wood) + * wasm: Comply with `wasm_exec.js` licence terms (Matthew Vernon) +* Bug Fixes + * build: Update `golang.org/x/net/http2` to fix GO-2022-1144 (Nick Craig-Wood) + * restic: Fix typo in docs 'remove' should be 'remote' (asdffdsazqqq) + * serve dlna: Fix panic: Logger uninitialized. (Nick Craig-Wood) +* Mount + * Update cgofuse for FUSE-T support for mounting volumes on Mac (Nick Craig-Wood) +* VFS + * Windows: fix slow opening of exe files by not truncating files when not necessary (Nick Craig-Wood) + * Fix IO Error opening a file with `O_CREATE|O_RDONLY` in `--vfs-cache-mode` not full (Nick Craig-Wood) +* Crypt + * Fix compress wrapping crypt giving upload errors (Nick Craig-Wood) +* Azure Blob + * Port to new SDK (Nick Craig-Wood) + * Revamp authentication to include all methods and docs (Nick Craig-Wood) + * Port old authentication methods to new SDK (Nick Craig-Wood, Brad Ackerman) + * Thanks to [Stonebranch](https://www.stonebranch.com/) for sponsoring this work. + * Add `--azureblob-no-check-container` to assume container exists (Nick Craig-Wood) + * Add `--use-server-modtime` support (Abdullah Saglam) + * Add support for custom upload headers (rkettelerij) + * Allow emulator account/key override (Roel Arents) + * Support simple "environment credentials" (Nathaniel Wesley Filardo) + * Ignore `AuthorizationFailure` when trying to create a create a container (Nick Craig-Wood) +* Box + * Added note on Box API rate limits (Ole Frost) +* Drive + * Handle shared drives with leading/trailing space in name (related to) (albertony) +* FTP + * Update help text of implicit/explicit TLS options to refer to FTPS instead of FTP (ycdtosa) + * Improve performance to speed up `--files-from` and `NewObject` (Anthony Pessy) +* HTTP + * Parse GET responses when `no_head` is set (Arnie97) + * Do not update object size based on `Range` requests (Arnie97) + * Support `Content-Range` response header (Arnie97) +* Onedrive + * Document workaround for shared with me files (vanplus) +* S3 + * Add Liara LOS to provider list (MohammadReza) + * Add DigitalOcean Spaces regions `sfo3`, `fra1`, `syd1` (Jack) + * Avoid privileged `GetBucketLocation` to resolve s3 region (Anthony Pessy) + * Stop setting object and bucket ACL to `private` if it is an empty string (Philip Harvey) + * If bucket or object ACL is empty string then don't add `X-Amz-Acl:` header (Nick Craig-Wood) + * Reduce memory consumption for s3 objects (Erik Agterdenbos) + * Fix listing loop when using v2 listing on v1 server (Nick Craig-Wood) + * Fix nil pointer exception when using Versions (Nick Craig-Wood) + * Fix excess memory usage when using versions (Nick Craig-Wood) + * Ignore versionIDs from uploads unless using `--s3-versions` or `--s3-versions-at` (Nick Craig-Wood) +* SFTP + * Add configuration options to set ssh Ciphers / MACs / KeyExchange (dgouju) + * Auto-detect shell type for fish (albertony) + * Fix NewObject with leading / (Nick Craig-Wood) +* Smb + * Fix issue where spurious dot directory is created (albertony) +* Storj + * Implement server side Copy (Kaloyan Raev) + +## v1.60.1 - 2022-11-17 + +[See commits](https://github.com/rclone/rclone/compare/v1.60.0...v1.60.1) + +* Bug Fixes + * lib/cache: Fix alias backend shutting down too soon (Nick Craig-Wood) + * wasm: Fix walltime link error by adding up-to-date wasm_exec.js (João Henrique Franco) + * docs + * Update faq.md with bisync (Samuel Johnson) + * Corrected download links in windows install docs (coultonluke) + * Add direct download link for windows arm64 (albertony) + * Remove link to rclone slack as it is no longer supported (Nick Craig-Wood) + * Faq: how to use a proxy server that requires a username and password (asdffdsazqqq) + * Oracle-object-storage: doc fix (Manoj Ghosh) + * Fix typo `remove` in rclone_serve_restic command (Joda Stößer) + * Fix character that was incorrectly interpreted as markdown (Clément Notin) +* VFS + * Fix deadlock caused by cache cleaner and upload finishing (Nick Craig-Wood) +* Local + * Clean absolute paths (albertony) + * Fix -L/--copy-links with filters missing directories (Nick Craig-Wood) +* Mailru + * Note that an app password is now needed (Nick Craig-Wood) + * Allow timestamps to be before the epoch 1970-01-01 (Nick Craig-Wood) +* S3 + * Add provider quirk `--s3-might-gzip` to fix corrupted on transfer: sizes differ (Nick Craig-Wood) + * Allow Storj to server side copy since it seems to work now (Nick Craig-Wood) + * Fix for unchecked err value in s3 listv2 (Aaron Gokaslan) + * Add additional Wasabi locations (techknowlogick) +* Smb + * Fix `Failed to sync: context canceled` at the end of syncs (Nick Craig-Wood) +* WebDAV + * Fix Move/Copy/DirMove when using -server-side-across-configs (Nick Craig-Wood) + ## v1.60.0 - 2022-10-21 [See commits](https://github.com/rclone/rclone/compare/v1.59.0...v1.60.0) @@ -45237,9 +46296,8 @@ of metadata, which breaks the desired 1:1 mapping of files to objects. ### Can rclone do bi-directional sync? ### -No, not at present. rclone only does uni-directional sync from A -> -B. It may do in the future though since it has all the primitives - it -just requires writing the algorithm to do it. +Yes, since rclone v1.58.0, [bidirectional cloud sync](https://rclone.org/bisync/) is +available. ### Can I use rclone with an HTTP proxy? ### @@ -45264,6 +46322,14 @@ possibilities. So, on Linux, you may end up with code similar to export HTTP_PROXY=$http_proxy export HTTPS_PROXY=$http_proxy + +Note: If the proxy server requires a username and password, then use + + export http_proxy=http://username:password@proxyserver:12345 + export https_proxy=$http_proxy + export HTTP_PROXY=$http_proxy + export HTTPS_PROXY=$http_proxy + The `NO_PROXY` allows you to disable the proxy for specific hosts. Hosts must be comma separated, and can contain domains or parts. For instance "foo.com" also matches "bar.foo.com". @@ -45638,6 +46704,7 @@ put them back in again.` >}} * Jay * andrea rota * nicolov + * Matt Joiner * Dario Guzik * qip * yair@unicorn @@ -46057,6 +47124,28 @@ put them back in again.` >}} * Manoj Ghosh * Tom Mombourquette * Robert Newson + * Samuel Johnson + * coultonluke + * Anthony Pessy + * Philip Harvey + * dgouju + * Clément Notin + * x3-apptech <66947598+x3-apptech@users.noreply.github.com> + * Arnie97 + * Roel Arents <2691308+roelarents@users.noreply.github.com> + * Aaron Gokaslan + * techknowlogick + * rkettelerij + * Kamui + * asdffdsazqqq <90116442+asdffdsazqqq@users.noreply.github.com> + * Nathaniel Wesley Filardo + * ycdtosa + * Erik Agterdenbos + * Kevin Verstaen <48050031+kverstae@users.noreply.github.com> + * MohammadReza + * vanplus <60313789+vanplus@users.noreply.github.com> + * Jack <16779171+jkpe@users.noreply.github.com> + * Abdullah Saglam # Contact the rclone project # diff --git a/MANUAL.txt b/MANUAL.txt index 6cdd6ee0a3a09..81320892b3979 100644 --- a/MANUAL.txt +++ b/MANUAL.txt @@ -1,6 +1,6 @@ rclone(1) User Manual Nick Craig-Wood -Oct 21, 2022 +Dec 20, 2022 Rclone syncs your files to cloud storage @@ -121,6 +121,7 @@ S3, that work out of the box.) - IDrive e2 - IONOS Cloud - Koofr +- Liara Object Storage - Mail.ru Cloud - Memset Memstore - Mega @@ -2181,7 +2182,7 @@ This will look something like (some irrelevant detail removed): "State": "*oauth-islocal,teamdrive,,", "Option": { "Name": "config_is_local", - "Help": "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n", + "Help": "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n", "Default": true, "Examples": [ { @@ -2518,7 +2519,7 @@ This will look something like (some irrelevant detail removed): "State": "*oauth-islocal,teamdrive,,", "Option": { "Name": "config_is_local", - "Help": "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n", + "Help": "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n", "Default": true, "Examples": [ { @@ -3360,6 +3361,7 @@ Options --hash Include hashes in the output (may take longer) --hash-type stringArray Show only this hash type (may be repeated) -h, --help help for lsjson + -M, --metadata Add metadata to the listing --no-mimetype Don't read the mime type (can speed things up) --no-modtime Don't read the modification time (can speed things up) --original Show the ID of the underlying Object @@ -4086,14 +4088,14 @@ Options --allow-other Allow access to other users (not supported on Windows) --allow-root Allow access to root user (not supported on Windows) --async-read Use asynchronous reads (not supported on Windows) (default true) - --attr-timeout duration Time for which file/directory attributes are cached (default 1s) + --attr-timeout Duration Time for which file/directory attributes are cached (default 1s) --daemon Run mount in background and exit parent process (as background output is suppressed, use --log-file with --log-format=pid,... to monitor) (not supported on Windows) - --daemon-timeout duration Time limit for rclone to respond to kernel (not supported on Windows) - --daemon-wait duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) + --daemon-timeout Duration Time limit for rclone to respond to kernel (not supported on Windows) (default 0s) + --daemon-wait Duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) --debug-fuse Debug the FUSE internals - needs -v --default-permissions Makes kernel enforce access control based on the file mode (not supported on Windows) --devname string Set the device name - default is remote:path - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --fuse-flag stringArray Flags or arguments to be passed direct to libfuse/WinFsp (repeat if required) @@ -4107,24 +4109,24 @@ Options --noappledouble Ignore Apple Double (._) and .DS_Store files (supported on OSX only) (default true) --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) --volname string Set the volume name (supported on Windows and OSX only) --write-back-cache Makes kernel buffer writes before sending them to rclone (without this, writethrough caching is used) (not supported on Windows) @@ -4204,11 +4206,12 @@ toggle the help on and off. The supported keys are: ↑,↓ or k,j to Move →,l to enter ←,h to return - c toggle counts g toggle graph + c toggle counts a toggle average size in directory + m toggle modified time u toggle human-readable format - n,s,C,A sort by name,size,count,average size + n,s,C,A,M sort by name,size,count,asize,mtime d delete file/directory v select file/directory V enter visual select mode @@ -4435,6 +4438,128 @@ browser when rclone is run. See the rc documentation for more info on the rc flags. +Server options + +Use --addr to specify which IP address and port the server should listen +on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By +default it only listens on localhost. You can use port :0 to let the OS +choose an available port. + +If you set --addr to listen on a public or LAN accessible IP address +then using Authentication is advised - see the next section for info. + +You can use a unix socket by setting the url to unix:///path/to/socket +or just by using an absolute path name. Note that unix sockets bypass +the authentication - this is expected to be done with file system +permissions. + +--addr may be repeated to listen on multiple IPs/ports/sockets. + +--server-read-timeout and --server-write-timeout can be used to control +the timeouts on the server. Note that this is the total time for a +transfer. + +--max-header-bytes controls the maximum number of bytes the server will +accept in the HTTP header. + +--baseurl controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used --baseurl "/rclone" then +rclone would serve from a URL starting with "/rclone/". This is useful +if you wish to proxy rclone serve. Rclone automatically inserts leading +and trailing "/" on --baseurl, so --baseurl "rclone", +--baseurl "/rclone" and --baseurl "/rclone/" are all treated +identically. + +TLS (SSL) + +By default this will serve over http. If you want you can serve over +https. You will need to supply the --cert and --key flags. If you wish +to do client side certificate validation then you will need to supply +--client-ca also. + +--cert should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. --key should be the PEM encoded private +key and --client-ca should be the PEM encoded client certificate +authority certificate. + +--min-tls-version is minimum TLS version that is acceptable. Valid +values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0"). + +Template + +--template allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup to +be used within the template to server pages: + + ----------------------------------------------------------------------- + Parameter Description + ----------------------------------- ----------------------------------- + .Name The full path of a file/directory. + + .Title Directory listing of .Name + + .Sort The current sort used. This is + changeable via ?sort= parameter + + Sort Options: + namedirfirst,name,size,time + (default namedirfirst) + + .Order The current ordering used. This is + changeable via ?order= parameter + + Order Options: asc,desc (default + asc) + + .Query Currently unused. + + .Breadcrumb Allows for creating a relative + navigation + + -- .Link The relative to the root link of + the Text. + + -- .Text The Name of the directory. + + .Entries Information about a specific + file/directory. + + -- .URL The 'url' of an entry. + + -- .Leaf Currently same as 'URL' but + intended to be 'just' the name. + + -- .IsDir Boolean for if an entry is a + directory or not. + + -- .Size Size in Bytes of the entry. + + -- .ModTime The UTC timestamp of an entry. + ----------------------------------------------------------------------- + +Authentication + +By default this will serve files without needing a login. + +You can either use an htpasswd file which can take lots of users, or set +a single username and password with the --user and --pass flags. + +Use --htpasswd /path/to/htpasswd to provide an htpasswd file. This is in +standard apache format and supports MD5, SHA1 and BCrypt for basic +authentication. Bcrypt is recommended. + +To create an htpasswd file: + + touch htpasswd + htpasswd -B htpasswd user + htpasswd -B htpasswd anotherUser + +The password file can be updated while rclone is running. + +Use --realm to set the authentication realm. + +Use --salt to change the password hashing salt from the default. + rclone rcd * [flags] Options @@ -4937,8 +5062,8 @@ only with caching. Options --addr string The ip:port or :port to bind the DLNA http server to (default ":7879") - --announce-interval duration The interval between SSDP announcements (default 12m0s) - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --announce-interval Duration The interval between SSDP announcements (default 12m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -4949,24 +5074,24 @@ Options --no-checksum Don't compare checksums on up/download --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) See the global flags page for global options not listed here. @@ -5345,15 +5470,15 @@ Options --allow-other Allow access to other users (not supported on Windows) --allow-root Allow access to root user (not supported on Windows) --async-read Use asynchronous reads (not supported on Windows) (default true) - --attr-timeout duration Time for which file/directory attributes are cached (default 1s) + --attr-timeout Duration Time for which file/directory attributes are cached (default 1s) --base-dir string Base directory for volumes (default "/var/lib/docker-volumes/rclone") --daemon Run mount in background and exit parent process (as background output is suppressed, use --log-file with --log-format=pid,... to monitor) (not supported on Windows) - --daemon-timeout duration Time limit for rclone to respond to kernel (not supported on Windows) - --daemon-wait duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) + --daemon-timeout Duration Time limit for rclone to respond to kernel (not supported on Windows) (default 0s) + --daemon-wait Duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) --debug-fuse Debug the FUSE internals - needs -v --default-permissions Makes kernel enforce access control based on the file mode (not supported on Windows) --devname string Set the device name - default is remote:path - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --forget-state Skip restoring previous state @@ -5369,26 +5494,26 @@ Options --noappledouble Ignore Apple Double (._) and .DS_Store files (supported on OSX only) (default true) --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --socket-addr string Address or absolute path (default: /run/docker/plugins/rclone.sock) --socket-gid int GID for unix socket (default: current process GID) (default 1000) --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) --volname string Set the volume name (supported on Windows and OSX only) --write-back-cache Makes kernel buffer writes before sending them to rclone (without this, writethrough caching is used) (not supported on Windows) @@ -5822,7 +5947,7 @@ Options --addr string IPaddress:Port or :Port to bind server to (default "localhost:2121") --auth-proxy string A program to use to create the backend from the auth --cert string TLS PEM key (concatenation of certificate and CA certificate) - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -5833,26 +5958,26 @@ Options --no-seek Don't allow seeking in files --pass string Password for authentication (empty value allow every password) --passive-port string Passive port range to use (default "30000-32000") - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --public-ip string Public IP address to advertise for passive connections --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication (default "anonymous") - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) See the global flags page for global options not listed here. @@ -5887,6 +6012,13 @@ choose an available port. If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. +You can use a unix socket by setting the url to unix:///path/to/socket +or just by using an absolute path name. Note that unix sockets bypass +the authentication - this is expected to be done with file system +permissions. + +--addr may be repeated to listen on multiple IPs/ports/sockets. + --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -5902,7 +6034,7 @@ and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically. -SSL/TLS +TLS (SSL) By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish @@ -6315,47 +6447,47 @@ only with caching. Options - --addr string IPaddress:Port or :Port to bind server to (default "127.0.0.1:8080") + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --baseurl string Prefix for URLs - leave blank for root - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) -h, --help help for http --htpasswd string A htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --no-checksum Don't compare checksums on up/download --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --realm string Realm for authentication --salt string Password hashing salt (default "dlPL2MqE") - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --template string User-specified template --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) See the global flags page for global options not listed here. @@ -6369,7 +6501,7 @@ Serve the remote for restic's REST API. Synopsis -Run a basic web server to serve a remove over restic's REST backend API +Run a basic web server to serve a remote over restic's REST backend API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly. @@ -6451,13 +6583,20 @@ starting with a path of //. Server options Use --addr to specify which IP address and port the server should listen -on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By +on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. +You can use a unix socket by setting the url to unix:///path/to/socket +or just by using an absolute path name. Note that unix sockets bypass +the authentication - this is expected to be done with file system +permissions. + +--addr may be repeated to listen on multiple IPs/ports/sockets. + --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -6473,55 +6612,20 @@ and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically. ---template allows a user to specify a custom markup template for HTTP -and WebDAV serve functions. The server exports the following markup to -be used within the template to server pages: - - ----------------------------------------------------------------------- - Parameter Description - ----------------------------------- ----------------------------------- - .Name The full path of a file/directory. - - .Title Directory listing of .Name - - .Sort The current sort used. This is - changeable via ?sort= parameter - - Sort Options: - namedirfirst,name,size,time - (default namedirfirst) - - .Order The current ordering used. This is - changeable via ?order= parameter - - Order Options: asc,desc (default - asc) - - .Query Currently unused. - - .Breadcrumb Allows for creating a relative - navigation - - -- .Link The relative to the root link of - the Text. - - -- .Text The Name of the directory. - - .Entries Information about a specific - file/directory. - - -- .URL The 'url' of an entry. +TLS (SSL) - -- .Leaf Currently same as 'URL' but - intended to be 'just' the name. - - -- .IsDir Boolean for if an entry is a - directory or not. +By default this will serve over http. If you want you can serve over +https. You will need to supply the --cert and --key flags. If you wish +to do client side certificate validation then you will need to supply +--client-ca also. - -- .Size Size in Bytes of the entry. +--cert should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. --key should be the PEM encoded private +key and --client-ca should be the PEM encoded client certificate +authority certificate. - -- .ModTime The UTC timestamp of an entry. - ----------------------------------------------------------------------- +--min-tls-version is minimum TLS version that is acceptable. Valid +values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0"). Authentication @@ -6544,43 +6648,30 @@ The password file can be updated while rclone is running. Use --realm to set the authentication realm. -SSL/TLS - -By default this will serve over HTTP. If you want you can serve over -HTTPS. You will need to supply the --cert and --key flags. If you wish -to do client side certificate validation then you will need to supply ---client-ca also. - ---cert should be either a PEM encoded certificate or a concatenation of -that with the CA certificate. --key should be the PEM encoded private -key and --client-ca should be the PEM encoded client certificate -authority certificate. - ---min-tls-version is minimum TLS version that is acceptable. Valid -values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0"). +Use --salt to change the password hashing salt from the default. rclone serve restic remote:path [flags] Options - --addr string IPaddress:Port or :Port to bind server to (default "localhost:8080") + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --append-only Disallow deletion of repository data --baseurl string Prefix for URLs - leave blank for root --cache-objects Cache listed objects (default true) - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with -h, --help help for restic - --htpasswd string htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --htpasswd string A htpasswd file - if not provided no authentication is done + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --pass string Password for authentication --private-repos Users can only access their private repo - --realm string Realm for authentication (default "rclone") - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --realm string Realm for authentication + --salt string Password hashing salt (default "dlPL2MqE") + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --stdio Run an HTTP2 server on stdin/stdout - --template string User-specified template --user string User name for authentication See the global flags page for global options not listed here. @@ -7044,7 +7135,7 @@ Options --addr string IPaddress:Port or :Port to bind server to (default "localhost:2022") --auth-proxy string A program to use to create the backend from the auth --authorized-keys string Authorized keys file (default "~/.ssh/authorized_keys") - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -7055,26 +7146,26 @@ Options --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --stdio Run an sftp server on stdin/stdout --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) See the global flags page for global options not listed here. @@ -7106,13 +7197,20 @@ or "SHA-1". Use the hashsum command to see the full list. Server options Use --addr to specify which IP address and port the server should listen -on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By +on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. If you set --addr to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. +You can use a unix socket by setting the url to unix:///path/to/socket +or just by using an absolute path name. Note that unix sockets bypass +the authentication - this is expected to be done with file system +permissions. + +--addr may be repeated to listen on multiple IPs/ports/sockets. + --server-read-timeout and --server-write-timeout can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -7128,6 +7226,23 @@ and trailing "/" on --baseurl, so --baseurl "rclone", --baseurl "/rclone" and --baseurl "/rclone/" are all treated identically. +TLS (SSL) + +By default this will serve over http. If you want you can serve over +https. You will need to supply the --cert and --key flags. If you wish +to do client side certificate validation then you will need to supply +--client-ca also. + +--cert should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. --key should be the PEM encoded private +key and --client-ca should be the PEM encoded client certificate +authority certificate. + +--min-tls-version is minimum TLS version that is acceptable. Valid +values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0"). + +Template + --template allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: @@ -7199,20 +7314,7 @@ The password file can be updated while rclone is running. Use --realm to set the authentication realm. -SSL/TLS - -By default this will serve over HTTP. If you want you can serve over -HTTPS. You will need to supply the --cert and --key flags. If you wish -to do client side certificate validation then you will need to supply ---client-ca also. - ---cert should be either a PEM encoded certificate or a concatenation of -that with the CA certificate. --key should be the PEM encoded private -key and --client-ca should be the PEM encoded client certificate -authority certificate. - ---min-tls-version is minimum TLS version that is acceptable. Valid -values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default "tls1.0"). +Use --salt to change the password hashing salt from the default. VFS - Virtual File System @@ -7608,49 +7710,50 @@ that rclone supports. Options - --addr string IPaddress:Port or :Port to bind server to (default "localhost:8080") + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --auth-proxy string A program to use to create the backend from the auth --baseurl string Prefix for URLs - leave blank for root - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --disable-dir-list Disable HTML directory list on GET request for a directory --etag-hash string Which hash to use for the ETag, or auto or blank for off --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) -h, --help help for webdav - --htpasswd string htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --htpasswd string A htpasswd file - if not provided no authentication is done + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --no-checksum Don't compare checksums on up/download --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access - --realm string Realm for authentication (default "rclone") - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --realm string Realm for authentication + --salt string Password hashing salt (default "dlPL2MqE") + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --template string User-specified template --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) See the global flags page for global options not listed here. @@ -7745,7 +7848,7 @@ Log any change notify requests for the remote passed in. Options -h, --help help for changenotify - --poll-interval duration Time to wait between polling for changes (default 10s) + --poll-interval Duration Time to wait between polling for changes (default 10s) See the global flags page for global options not listed here. @@ -7800,7 +7903,7 @@ Options --check-normalization Check UTF-8 Normalization --check-streaming Check uploads with indeterminate file size -h, --help help for info - --upload-wait duration Wait after writing a file + --upload-wait Duration Wait after writing a file (default 0s) --write-json string Write results to file See the global flags page for global options not listed here. @@ -7844,6 +7947,7 @@ Options --files int Number of files to create (default 1000) --files-per-directory int Average number of files per directory (default 10) -h, --help help for makefiles + --max-depth int Maximum depth of directory hierarchy (default 10) --max-file-size SizeSuffix Maximum size of files to create (default 100) --max-name-length int Maximum size of file names (default 12) --min-file-size SizeSuffix Minimum size of file to create @@ -7954,7 +8058,6 @@ For a more interactive navigation of the remote see the ncdu command. Options -a, --all All files are listed (list . files too) - -C, --color Turn colorization on always -d, --dirs-only List directories only --dirsfirst List directories before files (-U disables) --full-path Print the full path prefix for each file @@ -8174,8 +8277,17 @@ Will get their own names Valid remote names Remote names are case sensitive, and must adhere to the following rules: -- May only contain 0-9, A-Z, a-z, _, -, . and space. - May not start -with - or space. +- May contain number, letter, _, -, . and space. - May not start with - +or space. - May not end with space. + +Starting with rclone version 1.61, any Unicode numbers and letters are +allowed, while in older versions it was limited to plain ASCII (0-9, +A-Z, a-z). If you use the same rclone configuration from different +shells, which may be configured with different character encoding, you +must be cautious to use characters that are possible to write in all of +them. This is mostly a problem on Windows, where the console +traditionally uses a non-Unicode character set - defined by the +so-called "code page". Quoting and the shell @@ -8672,6 +8784,18 @@ quicker than without the --checksum flag. When using this flag, rclone won't update mtimes of remote files if they are incorrect as it would normally. +--color WHEN + +Specifiy when colors (and other ANSI codes) should be added to the +output. + +AUTO (default) only allows ANSI codes when the output is a terminal + +NEVER never allow ANSI codes + +ALWAYS always add ANSI codes, regardless of the output format (terminal +or file) + --compare-dest=DIR When using sync, copy or move DIR is checked in addition to the @@ -10157,6 +10281,12 @@ For the filtering options - --min-age - --max-age - --dump filters +- --metadata-include +- --metadata-include-from +- --metadata-exclude +- --metadata-exclude-from +- --metadata-filter +- --metadata-filter-from See the filtering section. @@ -10369,13 +10499,14 @@ two ways of doing it, described below. Configuring using rclone authorize On the headless box run rclone config but answer N to the -Use auto config? question. +Use web browser to automatically authenticate? question. ... Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes (default) n) No y/n> n @@ -10446,13 +10577,14 @@ box port 53682 to local machine by using the following command: ssh -L localhost:53682:localhost:53682 username@remote_server Then on the headless box run rclone config and answer Y to the -Use auto config? question. +Use web browser to automatically authenticate? question. ... Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes (default) n) No y/n> y @@ -11207,6 +11339,44 @@ E.g. for the following directory structure: The command rclone ls --exclude-if-present .ignore dir1 does not list dir3, file3 or .ignore. +Metadata filters + +The metadata filters work in a very similar way to the normal file name +filters, except they match metadata on the object. + +The metadata should be specified as key=value patterns. This may be +wildcarded using the normal filter patterns or regular expressions. + +For example if you wished to list only local files with a mode of 100664 +you could do that with: + + rclone lsf -M --files-only --metadata-include "mode=100664" . + +Or if you wished to show files with an atime, mtime or btime at a given +date: + + rclone lsf -M --files-only --metadata-include "[abm]time=2022-12-16*" . + +Like file filtering, metadata filtering only applies to files not to +directories. + +The filters can be applied using these flags. + +- --metadata-include - Include metadatas matching pattern +- --metadata-include-from - Read metadata include patterns from file + (use - to read from stdin) +- --metadata-exclude - Exclude metadatas matching pattern +- --metadata-exclude-from - Read metadata exclude patterns from file + (use - to read from stdin) +- --metadata-filter - Add a metadata filtering rule +- --metadata-filter-from - Read metadata filtering patterns from a + file (use - to read from stdin) + +Each flag can be repeated. See the section on how filter rules are +applied for more details - these flags work in an identical way to the +file name filtering flags, but instead of file name patterns have +metadata patterns. + Common pitfalls The most frequent filter support issues on the rclone forum are: @@ -11910,6 +12080,14 @@ See the config providers command for more information on the above. Authentication is required for this call. +config/setpath: Set the path of the config file + +Parameters: + +- path - path to the config file to use + +Authentication is required for this call. + config/update: update the config for a remote. This takes the following parameters: @@ -12010,7 +12188,7 @@ Returns: "result": "" } - OR + OR { "error": true, "result": "" @@ -12201,6 +12379,25 @@ Parameters: - rate - int +debug/set-gc-percent: Call runtime/debug.SetGCPercent for setting the garbage collection target percentage. + +SetGCPercent sets the garbage collection target percentage: a collection +is triggered when the ratio of freshly allocated data to live data +remaining after the previous collection reaches this percentage. +SetGCPercent returns the previous setting. The initial setting is the +value of the GOGC environment variable at startup, or 100 if the +variable is not set. + +This setting may be effectively reduced in order to maintain a memory +limit. A negative percentage effectively disables garbage collection, +unless the memory limit is reached. + +See https://pkg.go.dev/runtime/debug#SetMemoryLimit for more details. + +Parameters: + +- gc-percent - int + debug/set-mutex-profile-fraction: Set runtime.SetMutexProfileFraction for mutex profiling. SetMutexProfileFraction controls the fraction of mutex contention events @@ -12222,6 +12419,47 @@ Results: - previousRate - int +debug/set-soft-memory-limit: Call runtime/debug.SetMemoryLimit for setting a soft memory limit for the runtime. + +SetMemoryLimit provides the runtime with a soft memory limit. + +The runtime undertakes several processes to try to respect this memory +limit, including adjustments to the frequency of garbage collections and +returning memory to the underlying system more aggressively. This limit +will be respected even if GOGC=off (or, if SetGCPercent(-1) is +executed). + +The input limit is provided as bytes, and includes all memory mapped, +managed, and not released by the Go runtime. Notably, it does not +account for space used by the Go binary and memory external to Go, such +as memory managed by the underlying system on behalf of the process, or +memory managed by non-Go code inside the same process. Examples of +excluded memory sources include: OS kernel memory held on behalf of the +process, memory allocated by C code, and memory mapped by syscall.Mmap +(because it is not managed by the Go runtime). + +A zero limit or a limit that's lower than the amount of memory used by +the Go runtime may cause the garbage collector to run nearly +continuously. However, the application may still make progress. + +The memory limit is always respected by the Go runtime, so to +effectively disable this behavior, set the limit very high. +math.MaxInt64 is the canonical value for disabling the limit, but values +much greater than the available memory on the underlying system work +just as well. + +See https://go.dev/doc/gc-guide for a detailed guide explaining the soft +memory limit in more detail, as well as a variety of common use-cases +and scenarios. + +SetMemoryLimit returns the previously set memory limit. A negative input +does not adjust the limit, and allows for retrieval of the currently set +memory limit. + +Parameters: + +- mem-limit - int + fscache/clear: Clear the Fs cache. This clears the fs cache. This is where remotes created from backends @@ -13781,7 +14019,7 @@ upon backend-specific capabilities. Microsoft OneDrive Yes Yes Yes Yes Yes No No Yes Yes Yes OpenDrive Yes Yes Yes Yes No No No No No Yes OpenStack Swift Yes † Yes No No No Yes Yes No Yes No - Oracle Object Storage Yes Yes No No Yes Yes No No No No + Oracle Object Storage No Yes No No Yes Yes Yes No No No pCloud Yes Yes Yes Yes Yes No No Yes Yes Yes premiumize.me Yes No Yes Yes No No No Yes Yes Yes put.io Yes No Yes Yes Yes No Yes No Yes Yes @@ -13791,7 +14029,7 @@ upon backend-specific capabilities. Sia No No No No No No Yes No No Yes SMB No No Yes Yes No No Yes No No Yes SugarSync Yes Yes Yes Yes No No Yes Yes No Yes - Storj Yes † No Yes No No Yes Yes No No No + Storj Yes † Yes Yes No No Yes Yes No No No Uptobox No Yes Yes Yes No No No No No No WebDAV Yes Yes Yes Yes No No Yes ‡ No Yes Yes Yandex Disk Yes Yes Yes Yes Yes No Yes Yes Yes Yes @@ -13903,9 +14141,10 @@ These flags are available for every command. -c, --checksum Skip based on checksum (if available) & size, not mod-time & size --client-cert string Client SSL certificate (PEM) for mutual TLS auth --client-key string Client SSL private key (PEM) for mutual TLS auth + --color string When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS (default "AUTO") --compare-dest stringArray Include additional comma separated server-side paths during comparison --config string Config file (default "$HOME/.config/rclone/rclone.conf") - --contimeout duration Connect timeout (default 1m0s) + --contimeout Duration Connect timeout (default 1m0s) --copy-dest stringArray Implies --compare-dest but also copies files from paths into destination --cpuprofile string Write cpu profile to file --cutoff-mode string Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS (default "HARD") @@ -13923,16 +14162,16 @@ These flags are available for every command. --dump-headers Dump HTTP headers - may contain sensitive info --error-on-no-transfer Sets exit code 9 if no files are transferred, useful in scripts --exclude stringArray Exclude files matching pattern - --exclude-from stringArray Read exclude patterns from file (use - to read from stdin) + --exclude-from stringArray Read file exclude patterns from file (use - to read from stdin) --exclude-if-present stringArray Exclude directories if filename is present - --expect-continue-timeout duration Timeout when using expect / 100-continue in HTTP (default 1s) + --expect-continue-timeout Duration Timeout when using expect / 100-continue in HTTP (default 1s) --fast-list Use recursive list if available; uses more memory but fewer transactions --files-from stringArray Read list of source-file names from file (use - to read from stdin) --files-from-raw stringArray Read list of source-file names from file without any processing of lines (use - to read from stdin) - -f, --filter stringArray Add a file-filtering rule - --filter-from stringArray Read filtering patterns from a file (use - to read from stdin) - --fs-cache-expire-duration duration Cache remotes for this long (0 to disable caching) (default 5m0s) - --fs-cache-expire-interval duration Interval to check for expired remotes (default 1m0s) + -f, --filter stringArray Add a file filtering rule + --filter-from stringArray Read file filtering patterns from a file (use - to read from stdin) + --fs-cache-expire-duration Duration Cache remotes for this long (0 to disable caching) (default 5m0s) + --fs-cache-expire-interval Duration Interval to check for expired remotes (default 1m0s) --header stringArray Set HTTP header for all transactions --header-download stringArray Set HTTP header for download transactions --header-upload stringArray Set HTTP header for upload transactions @@ -13946,9 +14185,9 @@ These flags are available for every command. -I, --ignore-times Don't skip files that match size and time - transfer all files --immutable Do not modify files, fail if existing files have been modified --include stringArray Include files matching pattern - --include-from stringArray Read include patterns from file (use - to read from stdin) + --include-from stringArray Read file include patterns from file (use - to read from stdin) -i, --interactive Enable interactive mode - --kv-lock-time duration Maximum time to keep key-value database locked by process (default 1s) + --kv-lock-time Duration Maximum time to keep key-value database locked by process (default 1s) --log-file string Log everything to this file --log-format string Comma separated list of log format options (default "date,time") --log-level string Log level DEBUG|INFO|NOTICE|ERROR (default "NOTICE") @@ -13958,16 +14197,22 @@ These flags are available for every command. --max-backlog int Maximum number of objects in sync or check backlog (default 10000) --max-delete int When synchronizing, limit the number of deletes (default -1) --max-depth int If set limits the recursion depth to this (default -1) - --max-duration duration Maximum duration rclone will transfer data for + --max-duration Duration Maximum duration rclone will transfer data for (default 0s) --max-size SizeSuffix Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P (default off) --max-stats-groups int Maximum number of stats groups to keep in memory, on max oldest is discarded (default 1000) --max-transfer SizeSuffix Maximum size of data to transfer (default off) --memprofile string Write memory profile to file -M, --metadata If set, preserve metadata when copying objects + --metadata-exclude stringArray Exclude metadatas matching pattern + --metadata-exclude-from stringArray Read metadata exclude patterns from file (use - to read from stdin) + --metadata-filter stringArray Add a metadata filtering rule + --metadata-filter-from stringArray Read metadata filtering patterns from a file (use - to read from stdin) + --metadata-include stringArray Include metadatas matching pattern + --metadata-include-from stringArray Read metadata include patterns from file (use - to read from stdin) --metadata-set stringArray Add metadata key=value when uploading --min-age Duration Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off) --min-size SizeSuffix Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off) - --modify-window duration Max time diff to be considered the same (default 1ns) + --modify-window Duration Max time diff to be considered the same (default 1ns) --multi-thread-cutoff SizeSuffix Use multi-thread downloads for files above this size (default 250Mi) --multi-thread-streams int Max number of streams to use for multi-thread downloads (default 4) --no-check-certificate Do not verify the server SSL certificate (insecure) @@ -13983,25 +14228,26 @@ These flags are available for every command. --progress-terminal-title Show progress on the terminal title (requires -P/--progress) -q, --quiet Print as little stuff as possible --rc Enable the remote control server - --rc-addr string IPaddress:Port or :Port to bind server to (default "localhost:5572") + --rc-addr stringArray IPaddress:Port or :Port to bind server to (default [localhost:5572]) --rc-allow-origin string Set the allowed origin for CORS --rc-baseurl string Prefix for URLs - leave blank for root - --rc-cert string SSL PEM key (concatenation of certificate and CA certificate) + --rc-cert string TLS PEM key (concatenation of certificate and CA certificate) --rc-client-ca string Client certificate authority to verify clients with --rc-enable-metrics Enable prometheus metrics on /metrics --rc-files string Path to local files to serve on the HTTP server - --rc-htpasswd string htpasswd file - if not provided no authentication is done - --rc-job-expire-duration duration Expire finished async jobs older than this value (default 1m0s) - --rc-job-expire-interval duration Interval to check for expired async jobs (default 10s) - --rc-key string SSL PEM Private key + --rc-htpasswd string A htpasswd file - if not provided no authentication is done + --rc-job-expire-duration Duration Expire finished async jobs older than this value (default 1m0s) + --rc-job-expire-interval Duration Interval to check for expired async jobs (default 10s) + --rc-key string TLS PEM Private key --rc-max-header-bytes int Maximum size of request header (default 4096) --rc-min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --rc-no-auth Don't require auth for certain methods --rc-pass string Password for authentication - --rc-realm string Realm for authentication (default "rclone") + --rc-realm string Realm for authentication + --rc-salt string Password hashing salt (default "dlPL2MqE") --rc-serve Enable the serving of remote objects - --rc-server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --rc-server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --rc-server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --rc-server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --rc-template string User-specified template --rc-user string User name for authentication --rc-web-fetch-url string URL to fetch the releases for webgui (default "https://api.github.com/repos/rclone/rclone-webui-react/releases/latest") @@ -14011,10 +14257,10 @@ These flags are available for every command. --rc-web-gui-update Check and update to latest version of web gui --refresh-times Refresh the modtime of remote files --retries int Retry operations this many times if they fail (default 3) - --retries-sleep duration Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) + --retries-sleep Duration Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) (default 0s) --server-side-across-configs Allow server-side operations (e.g. copy) to work across different configs --size-only Skip based on size only, not mod-time or checksum - --stats duration Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s) + --stats Duration Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s) --stats-file-name-length int Max file name length in stats (0 for no limit) (default 45) --stats-log-level string Log level to show --stats output DEBUG|INFO|NOTICE|ERROR (default "INFO") --stats-one-line Make the stats fit on one line @@ -14027,7 +14273,7 @@ These flags are available for every command. --syslog Use Syslog for logging --syslog-facility string Facility for syslog, e.g. KERN,USER,... (default "DAEMON") --temp-dir string Directory rclone will use for temporary files (default "/tmp") - --timeout duration IO idle timeout (default 5m0s) + --timeout Duration IO idle timeout (default 5m0s) --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --track-renames When synchronizing, track file renames and do a server-side move if possible @@ -14038,7 +14284,7 @@ These flags are available for every command. --use-json-log Use json log format --use-mmap Use mmap allocator (see docs) --use-server-modtime Use server modified time instead of object metadata - --user-agent string Set the user-agent to a specified string (default "rclone/v1.60.0") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.61.0") -v, --verbose count Print lots more stuff (repeat for more) Backend Flags @@ -14046,529 +14292,543 @@ Backend Flags These flags are available for every command. They control the backends and may be set in the config file. - --acd-auth-url string Auth server URL - --acd-client-id string OAuth Client Id - --acd-client-secret string OAuth Client Secret - --acd-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --acd-templink-threshold SizeSuffix Files >= this size will be downloaded via their tempLink (default 9Gi) - --acd-token string OAuth Access Token as a JSON blob - --acd-token-url string Token server url - --acd-upload-wait-per-gb Duration Additional time per GiB to wait after a failed complete upload to see if it appears (default 3m0s) - --alias-remote string Remote or path to alias - --azureblob-access-tier string Access tier of blob: hot, cool or archive - --azureblob-account string Storage Account Name - --azureblob-archive-tier-delete Delete archive tier blobs before overwriting - --azureblob-chunk-size SizeSuffix Upload chunk size (default 4Mi) - --azureblob-disable-checksum Don't store MD5 checksum with object metadata - --azureblob-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8) - --azureblob-endpoint string Endpoint for the service - --azureblob-key string Storage Account Key - --azureblob-list-chunk int Size of blob list (default 5000) - --azureblob-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --azureblob-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --azureblob-msi-client-id string Object ID of the user-assigned MSI to use, if any - --azureblob-msi-mi-res-id string Azure resource ID of the user-assigned MSI to use, if any - --azureblob-msi-object-id string Object ID of the user-assigned MSI to use, if any - --azureblob-no-head-object If set, do not do HEAD before GET when getting objects - --azureblob-public-access string Public access level of a container: blob or container - --azureblob-sas-url string SAS URL for container level access only - --azureblob-service-principal-file string Path to file containing credentials for use with a service principal - --azureblob-upload-concurrency int Concurrency for multipart uploads (default 16) - --azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256 MiB) (deprecated) - --azureblob-use-emulator Uses local storage emulator if provided as 'true' - --azureblob-use-msi Use a managed service identity to authenticate (only works in Azure) - --b2-account string Account ID or Application Key ID - --b2-chunk-size SizeSuffix Upload chunk size (default 96Mi) - --b2-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4Gi) - --b2-disable-checksum Disable checksums for large (> upload cutoff) files - --b2-download-auth-duration Duration Time before the authorization token will expire in s or suffix ms|s|m|h|d (default 1w) - --b2-download-url string Custom endpoint for downloads - --b2-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --b2-endpoint string Endpoint for the service - --b2-hard-delete Permanently delete files on remote removal, otherwise hide files - --b2-key string Application Key - --b2-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging - --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --b2-version-at Time Show file versions as they were at the specified time (default off) - --b2-versions Include old versions in directory listings - --box-access-token string Box App Primary Access Token - --box-auth-url string Auth server URL - --box-box-config-file string Box App config.json location - --box-box-sub-type string (default "user") - --box-client-id string OAuth Client Id - --box-client-secret string OAuth Client Secret - --box-commit-retries int Max number of times to try committing a multipart file (default 100) - --box-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot) - --box-list-chunk int Size of listing chunk 1-1000 (default 1000) - --box-owned-by string Only show items owned by the login (email address) passed in - --box-root-folder-id string Fill in for rclone to use a non root folder as its starting point - --box-token string OAuth Access Token as a JSON blob - --box-token-url string Token server url - --box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50 MiB) (default 50Mi) - --cache-chunk-clean-interval Duration How often should the cache perform cleanups of the chunk storage (default 1m0s) - --cache-chunk-no-memory Disable the in-memory cache for storing chunks during streaming - --cache-chunk-path string Directory to cache chunk files (default "$HOME/.cache/rclone/cache-backend") - --cache-chunk-size SizeSuffix The size of a chunk (partial file data) (default 5Mi) - --cache-chunk-total-size SizeSuffix The total size that the chunks can take up on the local disk (default 10Gi) - --cache-db-path string Directory to store file structure metadata DB (default "$HOME/.cache/rclone/cache-backend") - --cache-db-purge Clear all the cached data for this remote on start - --cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s) - --cache-info-age Duration How long to cache file structure information (directory listings, file size, times, etc.) (default 6h0m0s) - --cache-plex-insecure string Skip all certificate verification when connecting to the Plex server - --cache-plex-password string The password of the Plex user (obscured) - --cache-plex-url string The URL of the Plex server - --cache-plex-username string The username of the Plex user - --cache-read-retries int How many times to retry a read from a cache storage (default 10) - --cache-remote string Remote to cache - --cache-rps int Limits the number of requests per second to the source FS (-1 to disable) (default -1) - --cache-tmp-upload-path string Directory to keep temporary files until they are uploaded - --cache-tmp-wait-time Duration How long should files be stored in local cache before being uploaded (default 15s) - --cache-workers int How many workers should run in parallel to download chunks (default 4) - --cache-writes Cache file data on writes through the FS - --chunker-chunk-size SizeSuffix Files larger than chunk size will be split in chunks (default 2Gi) - --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks - --chunker-hash-type string Choose how chunker handles hash sums (default "md5") - --chunker-remote string Remote to chunk/unchunk - --combine-upstreams SpaceSepList Upstreams for combining - --compress-level int GZIP compression level (-2 to 9) (default -1) - --compress-mode string Compression mode (default "gzip") - --compress-ram-cache-limit SizeSuffix Some remotes don't allow the upload of files with unknown size (default 20Mi) - --compress-remote string Remote to compress - -L, --copy-links Follow symlinks and copy the pointed to item - --crypt-directory-name-encryption Option to either encrypt directory names or leave them intact (default true) - --crypt-filename-encoding string How to encode the encrypted filename to text string (default "base32") - --crypt-filename-encryption string How to encrypt the filenames (default "standard") - --crypt-no-data-encryption Option to either encrypt file data or leave it unencrypted - --crypt-password string Password or pass phrase for encryption (obscured) - --crypt-password2 string Password or pass phrase for salt (obscured) - --crypt-remote string Remote to encrypt/decrypt - --crypt-server-side-across-configs Allow server-side operations (e.g. copy) to work across different crypt configs - --crypt-show-mapping For all files listed show how the names encrypt - --drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded - --drive-allow-import-name-change Allow the filetype to change when uploading Google docs - --drive-auth-owner-only Only consider files owned by the authenticated user - --drive-auth-url string Auth server URL - --drive-chunk-size SizeSuffix Upload chunk size (default 8Mi) - --drive-client-id string Google Application Client Id - --drive-client-secret string OAuth Client Secret - --drive-copy-shortcut-content Server side copy contents of shortcuts instead of the shortcut - --drive-disable-http2 Disable drive using http2 (default true) - --drive-encoding MultiEncoder The encoding for the backend (default InvalidUtf8) - --drive-export-formats string Comma separated list of preferred formats for downloading Google docs (default "docx,xlsx,pptx,svg") - --drive-formats string Deprecated: See export_formats - --drive-impersonate string Impersonate this user when using a service account - --drive-import-formats string Comma separated list of preferred formats for uploading Google docs - --drive-keep-revision-forever Keep new head revision of each file forever - --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) - --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) - --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) - --drive-resource-key string Resource key for accessing a link-shared file - --drive-root-folder-id string ID of the root folder - --drive-scope string Scope that rclone should use when requesting access from drive - --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs - --drive-service-account-credentials string Service Account Credentials JSON blob - --drive-service-account-file string Service Account Credentials JSON file path - --drive-shared-with-me Only show files that are shared with me - --drive-size-as-quota Show sizes as storage quota usage, not actual size - --drive-skip-checksum-gphotos Skip MD5 checksum on Google photos and videos only - --drive-skip-dangling-shortcuts If set skip dangling shortcut files - --drive-skip-gdocs Skip google documents in all listings - --drive-skip-shortcuts If set skip shortcut files - --drive-starred-only Only show files that are starred - --drive-stop-on-download-limit Make download limit errors be fatal - --drive-stop-on-upload-limit Make upload limit errors be fatal - --drive-team-drive string ID of the Shared Drive (Team Drive) - --drive-token string OAuth Access Token as a JSON blob - --drive-token-url string Token server url - --drive-trashed-only Only show files that are in the trash - --drive-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 8Mi) - --drive-use-created-date Use file created date instead of modified date - --drive-use-shared-date Use date file was shared instead of modified date - --drive-use-trash Send files to the trash instead of deleting permanently (default true) - --drive-v2-download-min-size SizeSuffix If Object's are greater, use drive v2 API to download (default off) - --dropbox-auth-url string Auth server URL - --dropbox-batch-commit-timeout Duration Max time to wait for a batch to finish committing (default 10m0s) - --dropbox-batch-mode string Upload file batching sync|async|off (default "sync") - --dropbox-batch-size int Max number of files in upload batch - --dropbox-batch-timeout Duration Max time to allow an idle upload batch before uploading (default 0s) - --dropbox-chunk-size SizeSuffix Upload chunk size (< 150Mi) (default 48Mi) - --dropbox-client-id string OAuth Client Id - --dropbox-client-secret string OAuth Client Secret - --dropbox-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,RightSpace,InvalidUtf8,Dot) - --dropbox-impersonate string Impersonate this user when using a business account - --dropbox-shared-files Instructs rclone to work on individual shared files - --dropbox-shared-folders Instructs rclone to work on shared folders - --dropbox-token string OAuth Access Token as a JSON blob - --dropbox-token-url string Token server url - --fichier-api-key string Your API Key, get it from https://1fichier.com/console/params.pl - --fichier-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot) - --fichier-file-password string If you want to download a shared file that is password protected, add this parameter (obscured) - --fichier-folder-password string If you want to list the files in a shared folder that is password protected, add this parameter (obscured) - --fichier-shared-folder string If you want to download a shared folder, add this parameter - --filefabric-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) - --filefabric-permanent-token string Permanent Authentication Token - --filefabric-root-folder-id string ID of the root folder - --filefabric-token string Session Token - --filefabric-token-expiry string Token expiry time - --filefabric-url string URL of the Enterprise File Fabric to connect to - --filefabric-version string Version read from the file fabric - --ftp-ask-password Allow asking for FTP password when needed - --ftp-close-timeout Duration Maximum time to wait for a response to close (default 1m0s) - --ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited - --ftp-disable-epsv Disable using EPSV even if server advertises support - --ftp-disable-mlsd Disable using MLSD even if server advertises support - --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) - --ftp-disable-utf8 Disable using UTF-8 even if server advertises support - --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) - --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) - --ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD - --ftp-host string FTP host to connect to - --ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --ftp-no-check-certificate Do not verify the TLS certificate of the server - --ftp-pass string FTP password (obscured) - --ftp-port int FTP port number (default 21) - --ftp-shut-timeout Duration Maximum time to wait for data connection closing status (default 1m0s) - --ftp-tls Use Implicit FTPS (FTP over TLS) - --ftp-tls-cache-size int Size of TLS session cache for all control and data connections (default 32) - --ftp-user string FTP username (default "$USER") - --ftp-writing-mdtm Use MDTM to set modification time (VsFtpd quirk) - --gcs-anonymous Access public buckets and objects without credentials - --gcs-auth-url string Auth server URL - --gcs-bucket-acl string Access Control List for new buckets - --gcs-bucket-policy-only Access checks should use bucket-level IAM policies - --gcs-client-id string OAuth Client Id - --gcs-client-secret string OAuth Client Secret - --gcs-decompress If set this will decompress gzip encoded objects - --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) - --gcs-endpoint string Endpoint for the service - --gcs-location string Location for the newly created buckets - --gcs-no-check-bucket If set, don't attempt to check the bucket exists or create it - --gcs-object-acl string Access Control List for new objects - --gcs-project-number string Project number - --gcs-service-account-file string Service Account Credentials JSON file path - --gcs-storage-class string The storage class to use when storing objects in Google Cloud Storage - --gcs-token string OAuth Access Token as a JSON blob - --gcs-token-url string Token server url - --gphotos-auth-url string Auth server URL - --gphotos-client-id string OAuth Client Id - --gphotos-client-secret string OAuth Client Secret - --gphotos-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) - --gphotos-include-archived Also view and download archived media - --gphotos-read-only Set to make the Google Photos backend read only - --gphotos-read-size Set to read the size of media items - --gphotos-start-year int Year limits the photos to be downloaded to those which are uploaded after the given year (default 2000) - --gphotos-token string OAuth Access Token as a JSON blob - --gphotos-token-url string Token server url - --hasher-auto-size SizeSuffix Auto-update checksum for files smaller than this size (disabled by default) - --hasher-hashes CommaSepList Comma separated list of supported checksum types (default md5,sha1) - --hasher-max-age Duration Maximum time to keep checksums in cache (0 = no cache, off = cache forever) (default off) - --hasher-remote string Remote to cache checksums for (e.g. myRemote:path) - --hdfs-data-transfer-protection string Kerberos data transfer protection: authentication|integrity|privacy - --hdfs-encoding MultiEncoder The encoding for the backend (default Slash,Colon,Del,Ctl,InvalidUtf8,Dot) - --hdfs-namenode string Hadoop name node and port - --hdfs-service-principal-name string Kerberos service principal name for the namenode - --hdfs-username string Hadoop user name - --hidrive-auth-url string Auth server URL - --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) - --hidrive-client-id string OAuth Client Id - --hidrive-client-secret string OAuth Client Secret - --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary - --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) - --hidrive-endpoint string Endpoint for the service (default "https://api.hidrive.strato.com/2.1") - --hidrive-root-prefix string The root/parent folder for all paths (default "/") - --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default "rw") - --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default "user") - --hidrive-token string OAuth Access Token as a JSON blob - --hidrive-token-url string Token server url - --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) - --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) - --http-headers CommaSepList Set HTTP headers for all transactions - --http-no-head Don't use HEAD requests - --http-no-slash Set this if the site doesn't end directories with / - --http-url string URL of HTTP host to connect to - --internetarchive-access-key-id string IAS3 Access Key - --internetarchive-disable-checksum Don't ask the server to test against MD5 checksum calculated by rclone (default true) - --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) - --internetarchive-endpoint string IAS3 Endpoint (default "https://s3.us.archive.org") - --internetarchive-front-endpoint string Host of InternetArchive Frontend (default "https://archive.org") - --internetarchive-secret-access-key string IAS3 Secret Key (password) - --internetarchive-wait-archive Duration Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s) - --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) - --jottacloud-hard-delete Delete files permanently rather than putting them into the trash - --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) - --jottacloud-no-versions Avoid server side versioning by deleting files and recreating files instead of overwriting them - --jottacloud-trashed-only Only show files that are in the trash - --jottacloud-upload-resume-limit SizeSuffix Files bigger than this can be resumed if the upload fail's (default 10Mi) - --koofr-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --koofr-endpoint string The Koofr API endpoint to use - --koofr-mountid string Mount ID of the mount to use - --koofr-password string Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured) - --koofr-provider string Choose your storage provider - --koofr-setmtime Does the backend support setting modification time (default true) - --koofr-user string Your user name - -l, --links Translate symlinks to/from regular files with a '.rclonelink' extension - --local-case-insensitive Force the filesystem to report itself as case insensitive - --local-case-sensitive Force the filesystem to report itself as case sensitive - --local-encoding MultiEncoder The encoding for the backend (default Slash,Dot) - --local-no-check-updated Don't check to see if the files change during upload - --local-no-preallocate Disable preallocation of disk space for transferred files - --local-no-set-modtime Disable setting modtime - --local-no-sparse Disable sparse files for multi-thread downloads - --local-nounc Disable UNC (long path names) conversion on Windows - --local-unicode-normalization Apply unicode NFC normalization to paths and filenames - --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) - --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) - --mailru-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --mailru-pass string Password (obscured) - --mailru-speedup-enable Skip full upload if there is another file with same data hash (default true) - --mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash) (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf") - --mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3Gi) - --mailru-speedup-max-memory SizeSuffix Files larger than the size given below will always be hashed on disk (default 32Mi) - --mailru-user string User name (usually email) - --mega-debug Output more debug from Mega - --mega-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --mega-hard-delete Delete files permanently rather than putting them into the trash - --mega-pass string Password (obscured) - --mega-user string User name - --netstorage-account string Set the NetStorage account name - --netstorage-host string Domain+path of NetStorage host to connect to - --netstorage-protocol string Select between HTTP or HTTPS protocol (default "https") - --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) - -x, --one-file-system Don't cross filesystem boundaries (unix/macOS only) - --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) - --onedrive-auth-url string Auth server URL - --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) - --onedrive-client-id string OAuth Client Id - --onedrive-client-secret string OAuth Client Secret - --onedrive-drive-id string The ID of the drive to use - --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) - --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) - --onedrive-expose-onenote-files Set to make OneNote files show up in directory listings - --onedrive-link-password string Set the password for links created by the link command - --onedrive-link-scope string Set the scope of the links created by the link command (default "anonymous") - --onedrive-link-type string Set the type of the links created by the link command (default "view") - --onedrive-list-chunk int Size of listing chunk (default 1000) - --onedrive-no-versions Remove all versions on modifying operations - --onedrive-region string Choose national cloud region for OneDrive (default "global") - --onedrive-root-folder-id string ID of the root folder - --onedrive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different onedrive configs - --onedrive-token string OAuth Access Token as a JSON blob - --onedrive-token-url string Token server url - --oos-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) - --oos-compartment string Object storage compartment OCID - --oos-config-file string Path to OCI config file (default "~/.oci/config") - --oos-config-profile string Profile name inside the oci config file (default "Default") - --oos-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) - --oos-copy-timeout Duration Timeout for copy (default 1m0s) - --oos-disable-checksum Don't store MD5 checksum with object metadata - --oos-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --oos-endpoint string Endpoint for Object storage API - --oos-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery - --oos-namespace string Object storage namespace - --oos-no-check-bucket If set, don't attempt to check the bucket exists or create it - --oos-provider string Choose your Auth Provider (default "env_auth") - --oos-region string Object storage Region - --oos-upload-concurrency int Concurrency for multipart uploads (default 10) - --oos-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size (default 10Mi) - --opendrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot) - --opendrive-password string Password (obscured) - --opendrive-username string Username - --pcloud-auth-url string Auth server URL - --pcloud-client-id string OAuth Client Id - --pcloud-client-secret string OAuth Client Secret - --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --pcloud-hostname string Hostname to connect to (default "api.pcloud.com") - --pcloud-password string Your pcloud password (obscured) - --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default "d0") - --pcloud-token string OAuth Access Token as a JSON blob - --pcloud-token-url string Token server url - --pcloud-username string Your pcloud username - --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --qingstor-access-key-id string QingStor Access Key ID - --qingstor-chunk-size SizeSuffix Chunk size to use for uploading (default 4Mi) - --qingstor-connection-retries int Number of connection retries (default 3) - --qingstor-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8) - --qingstor-endpoint string Enter an endpoint URL to connection QingStor API - --qingstor-env-auth Get QingStor credentials from runtime - --qingstor-secret-access-key string QingStor Secret Access Key (password) - --qingstor-upload-concurrency int Concurrency for multipart uploads (default 1) - --qingstor-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --qingstor-zone string Zone to connect to - --s3-access-key-id string AWS Access Key ID - --s3-acl string Canned ACL used when creating buckets and storing or copying objects - --s3-bucket-acl string Canned ACL used when creating buckets - --s3-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) - --s3-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) - --s3-decompress If set this will decompress gzip encoded objects - --s3-disable-checksum Don't store MD5 checksum with object metadata - --s3-disable-http2 Disable usage of http2 for S3 backends - --s3-download-url string Custom endpoint for downloads - --s3-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --s3-endpoint string Endpoint for S3 API - --s3-env-auth Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars) - --s3-force-path-style If true use path style access if false use virtual hosted style (default true) - --s3-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery - --s3-list-chunk int Size of listing chunk (response list for each ListObject S3 request) (default 1000) - --s3-list-url-encode Tristate Whether to url encode listings: true/false/unset (default unset) - --s3-list-version int Version of ListObjects to use: 1,2 or 0 for auto - --s3-location-constraint string Location constraint - must be set to match the Region - --s3-max-upload-parts int Maximum number of parts in a multipart upload (default 10000) - --s3-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --s3-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --s3-no-check-bucket If set, don't attempt to check the bucket exists or create it - --s3-no-head If set, don't HEAD uploaded objects to check integrity - --s3-no-head-object If set, do not do HEAD before GET when getting objects - --s3-no-system-metadata Suppress setting and reading of system metadata - --s3-profile string Profile to use in the shared credentials file - --s3-provider string Choose your S3 provider - --s3-region string Region to connect to - --s3-requester-pays Enables requester pays option when interacting with S3 bucket - --s3-secret-access-key string AWS Secret Access Key (password) - --s3-server-side-encryption string The server-side encryption algorithm used when storing this object in S3 - --s3-session-token string An AWS session token - --s3-shared-credentials-file string Path to the shared credentials file - --s3-sse-customer-algorithm string If using SSE-C, the server-side encryption algorithm used when storing this object in S3 - --s3-sse-customer-key string To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data - --s3-sse-customer-key-base64 string If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data - --s3-sse-customer-key-md5 string If using SSE-C you may provide the secret encryption key MD5 checksum (optional) - --s3-sse-kms-key-id string If using KMS ID you must provide the ARN of Key - --s3-storage-class string The storage class to use when storing new objects in S3 - --s3-upload-concurrency int Concurrency for multipart uploads (default 4) - --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint - --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) - --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads - --s3-v2-auth If true use v2 authentication - --s3-version-at Time Show file versions as they were at the specified time (default off) - --s3-versions Include old versions in directory listings - --seafile-2fa Two-factor authentication ('true' if the account has 2FA enabled) - --seafile-create-library Should rclone create a library if it doesn't exist - --seafile-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8) - --seafile-library string Name of the library - --seafile-library-key string Library password (for encrypted libraries only) (obscured) - --seafile-pass string Password (obscured) - --seafile-url string URL of seafile host to connect to - --seafile-user string User name (usually email address) - --sftp-ask-password Allow asking for SFTP password when needed - --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) - --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) - --sftp-disable-concurrent-reads If set don't use concurrent reads - --sftp-disable-concurrent-writes If set don't use concurrent writes - --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available - --sftp-host string SSH host to connect to - --sftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --sftp-key-file string Path to PEM-encoded private key file - --sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file (obscured) - --sftp-key-pem string Raw PEM-encoded private key - --sftp-key-use-agent When set forces the usage of the ssh-agent - --sftp-known-hosts-file string Optional path to known_hosts file - --sftp-md5sum-command string The command used to read md5 hashes - --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) - --sftp-path-override string Override path used by SSH shell commands - --sftp-port int SSH port number (default 22) - --sftp-pubkey-file string Optional path to public key file - --sftp-server-command string Specifies the path or command to run a sftp server on the remote host - --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands - --sftp-set-modtime Set the modified time on the remote if set (default true) - --sftp-sha1sum-command string The command used to read sha1 hashes - --sftp-shell-type string The type of SSH shell on remote server, if any - --sftp-skip-links Set to skip any symlinks and any other non regular files - --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default "sftp") - --sftp-use-fstat If set use fstat instead of stat - --sftp-use-insecure-cipher Enable the use of insecure ciphers and key exchange methods - --sftp-user string SSH username (default "$USER") - --sharefile-chunk-size SizeSuffix Upload chunk size (default 64Mi) - --sharefile-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,LeftSpace,LeftPeriod,RightSpace,RightPeriod,InvalidUtf8,Dot) - --sharefile-endpoint string Endpoint for API calls - --sharefile-root-folder-id string ID of the root folder - --sharefile-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (default 128Mi) - --sia-api-password string Sia Daemon API Password (obscured) - --sia-api-url string Sia daemon API URL, like http://sia.daemon.host:9980 (default "http://127.0.0.1:9980") - --sia-encoding MultiEncoder The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot) - --sia-user-agent string Siad User Agent (default "Sia-Agent") - --skip-links Don't warn about skipped symlinks - --smb-case-insensitive Whether the server is configured to be case-insensitive (default true) - --smb-domain string Domain name for NTLM authentication (default "WORKGROUP") - --smb-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot) - --smb-hide-special-share Hide special shares (e.g. print$) which users aren't supposed to access (default true) - --smb-host string SMB server hostname to connect to - --smb-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --smb-pass string SMB password (obscured) - --smb-port int SMB port number (default 445) - --smb-user string SMB username (default "$USER") - --storj-access-grant string Access grant - --storj-api-key string API key - --storj-passphrase string Encryption passphrase - --storj-provider string Choose an authentication method (default "existing") - --storj-satellite-address string Satellite address (default "us-central-1.storj.io") - --sugarsync-access-key-id string Sugarsync Access Key ID - --sugarsync-app-id string Sugarsync App ID - --sugarsync-authorization string Sugarsync authorization - --sugarsync-authorization-expiry string Sugarsync authorization expiry - --sugarsync-deleted-id string Sugarsync deleted folder id - --sugarsync-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8,Dot) - --sugarsync-hard-delete Permanently delete files if true - --sugarsync-private-access-key string Sugarsync Private Access Key - --sugarsync-refresh-token string Sugarsync refresh token - --sugarsync-root-id string Sugarsync root id - --sugarsync-user string Sugarsync user - --swift-application-credential-id string Application Credential ID (OS_APPLICATION_CREDENTIAL_ID) - --swift-application-credential-name string Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME) - --swift-application-credential-secret string Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET) - --swift-auth string Authentication URL for server (OS_AUTH_URL) - --swift-auth-token string Auth Token from alternate authentication - optional (OS_AUTH_TOKEN) - --swift-auth-version int AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION) - --swift-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) - --swift-domain string User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME) - --swift-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8) - --swift-endpoint-type string Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default "public") - --swift-env-auth Get swift credentials from environment variables in standard OpenStack form - --swift-key string API key or password (OS_PASSWORD) - --swift-leave-parts-on-error If true avoid calling abort upload on a failure - --swift-no-chunk Don't chunk files during streaming upload - --swift-no-large-objects Disable support for static and dynamic large objects - --swift-region string Region name - optional (OS_REGION_NAME) - --swift-storage-policy string The storage policy to use when creating a new container - --swift-storage-url string Storage URL - optional (OS_STORAGE_URL) - --swift-tenant string Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME) - --swift-tenant-domain string Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME) - --swift-tenant-id string Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID) - --swift-user string User name to log in (OS_USERNAME) - --swift-user-id string User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID) - --union-action-policy string Policy to choose upstream on ACTION category (default "epall") - --union-cache-time int Cache time of usage and free space (in seconds) (default 120) - --union-create-policy string Policy to choose upstream on CREATE category (default "epmfs") - --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) - --union-search-policy string Policy to choose upstream on SEARCH category (default "ff") - --union-upstreams string List of space separated upstreams - --uptobox-access-token string Your access token - --uptobox-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot) - --webdav-bearer-token string Bearer token instead of user/pass (e.g. a Macaroon) - --webdav-bearer-token-command string Command to run to get a bearer token - --webdav-encoding string The encoding for the backend - --webdav-headers CommaSepList Set HTTP headers for all transactions - --webdav-pass string Password (obscured) - --webdav-url string URL of http host to connect to - --webdav-user string User name - --webdav-vendor string Name of the WebDAV site/service/software you are using - --yandex-auth-url string Auth server URL - --yandex-client-id string OAuth Client Id - --yandex-client-secret string OAuth Client Secret - --yandex-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) - --yandex-hard-delete Delete files permanently rather than putting them into the trash - --yandex-token string OAuth Access Token as a JSON blob - --yandex-token-url string Token server url - --zoho-auth-url string Auth server URL - --zoho-client-id string OAuth Client Id - --zoho-client-secret string OAuth Client Secret - --zoho-encoding MultiEncoder The encoding for the backend (default Del,Ctl,InvalidUtf8) - --zoho-region string Zoho region to connect to - --zoho-token string OAuth Access Token as a JSON blob - --zoho-token-url string Token server url + --acd-auth-url string Auth server URL + --acd-client-id string OAuth Client Id + --acd-client-secret string OAuth Client Secret + --acd-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --acd-templink-threshold SizeSuffix Files >= this size will be downloaded via their tempLink (default 9Gi) + --acd-token string OAuth Access Token as a JSON blob + --acd-token-url string Token server url + --acd-upload-wait-per-gb Duration Additional time per GiB to wait after a failed complete upload to see if it appears (default 3m0s) + --alias-remote string Remote or path to alias + --azureblob-access-tier string Access tier of blob: hot, cool or archive + --azureblob-account string Azure Storage Account Name + --azureblob-archive-tier-delete Delete archive tier blobs before overwriting + --azureblob-chunk-size SizeSuffix Upload chunk size (default 4Mi) + --azureblob-client-certificate-password string Password for the certificate file (optional) (obscured) + --azureblob-client-certificate-path string Path to a PEM or PKCS12 certificate file including the private key + --azureblob-client-id string The ID of the client in use + --azureblob-client-secret string One of the service principal's client secrets + --azureblob-client-send-certificate-chain Send the certificate chain when using certificate auth + --azureblob-disable-checksum Don't store MD5 checksum with object metadata + --azureblob-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8) + --azureblob-endpoint string Endpoint for the service + --azureblob-env-auth Read credentials from runtime (environment variables, CLI or MSI) + --azureblob-key string Storage Account Shared Key + --azureblob-list-chunk int Size of blob list (default 5000) + --azureblob-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --azureblob-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --azureblob-msi-client-id string Object ID of the user-assigned MSI to use, if any + --azureblob-msi-mi-res-id string Azure resource ID of the user-assigned MSI to use, if any + --azureblob-msi-object-id string Object ID of the user-assigned MSI to use, if any + --azureblob-no-check-container If set, don't attempt to check the container exists or create it + --azureblob-no-head-object If set, do not do HEAD before GET when getting objects + --azureblob-password string The user's password (obscured) + --azureblob-public-access string Public access level of a container: blob or container + --azureblob-sas-url string SAS URL for container level access only + --azureblob-service-principal-file string Path to file containing credentials for use with a service principal + --azureblob-tenant string ID of the service principal's tenant. Also called its directory ID + --azureblob-upload-concurrency int Concurrency for multipart uploads (default 16) + --azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256 MiB) (deprecated) + --azureblob-use-emulator Uses local storage emulator if provided as 'true' + --azureblob-use-msi Use a managed service identity to authenticate (only works in Azure) + --azureblob-username string User name (usually an email address) + --b2-account string Account ID or Application Key ID + --b2-chunk-size SizeSuffix Upload chunk size (default 96Mi) + --b2-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4Gi) + --b2-disable-checksum Disable checksums for large (> upload cutoff) files + --b2-download-auth-duration Duration Time before the authorization token will expire in s or suffix ms|s|m|h|d (default 1w) + --b2-download-url string Custom endpoint for downloads + --b2-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --b2-endpoint string Endpoint for the service + --b2-hard-delete Permanently delete files on remote removal, otherwise hide files + --b2-key string Application Key + --b2-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging + --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --b2-version-at Time Show file versions as they were at the specified time (default off) + --b2-versions Include old versions in directory listings + --box-access-token string Box App Primary Access Token + --box-auth-url string Auth server URL + --box-box-config-file string Box App config.json location + --box-box-sub-type string (default "user") + --box-client-id string OAuth Client Id + --box-client-secret string OAuth Client Secret + --box-commit-retries int Max number of times to try committing a multipart file (default 100) + --box-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot) + --box-list-chunk int Size of listing chunk 1-1000 (default 1000) + --box-owned-by string Only show items owned by the login (email address) passed in + --box-root-folder-id string Fill in for rclone to use a non root folder as its starting point + --box-token string OAuth Access Token as a JSON blob + --box-token-url string Token server url + --box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50 MiB) (default 50Mi) + --cache-chunk-clean-interval Duration How often should the cache perform cleanups of the chunk storage (default 1m0s) + --cache-chunk-no-memory Disable the in-memory cache for storing chunks during streaming + --cache-chunk-path string Directory to cache chunk files (default "$HOME/.cache/rclone/cache-backend") + --cache-chunk-size SizeSuffix The size of a chunk (partial file data) (default 5Mi) + --cache-chunk-total-size SizeSuffix The total size that the chunks can take up on the local disk (default 10Gi) + --cache-db-path string Directory to store file structure metadata DB (default "$HOME/.cache/rclone/cache-backend") + --cache-db-purge Clear all the cached data for this remote on start + --cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s) + --cache-info-age Duration How long to cache file structure information (directory listings, file size, times, etc.) (default 6h0m0s) + --cache-plex-insecure string Skip all certificate verification when connecting to the Plex server + --cache-plex-password string The password of the Plex user (obscured) + --cache-plex-url string The URL of the Plex server + --cache-plex-username string The username of the Plex user + --cache-read-retries int How many times to retry a read from a cache storage (default 10) + --cache-remote string Remote to cache + --cache-rps int Limits the number of requests per second to the source FS (-1 to disable) (default -1) + --cache-tmp-upload-path string Directory to keep temporary files until they are uploaded + --cache-tmp-wait-time Duration How long should files be stored in local cache before being uploaded (default 15s) + --cache-workers int How many workers should run in parallel to download chunks (default 4) + --cache-writes Cache file data on writes through the FS + --chunker-chunk-size SizeSuffix Files larger than chunk size will be split in chunks (default 2Gi) + --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks + --chunker-hash-type string Choose how chunker handles hash sums (default "md5") + --chunker-remote string Remote to chunk/unchunk + --combine-upstreams SpaceSepList Upstreams for combining + --compress-level int GZIP compression level (-2 to 9) (default -1) + --compress-mode string Compression mode (default "gzip") + --compress-ram-cache-limit SizeSuffix Some remotes don't allow the upload of files with unknown size (default 20Mi) + --compress-remote string Remote to compress + -L, --copy-links Follow symlinks and copy the pointed to item + --crypt-directory-name-encryption Option to either encrypt directory names or leave them intact (default true) + --crypt-filename-encoding string How to encode the encrypted filename to text string (default "base32") + --crypt-filename-encryption string How to encrypt the filenames (default "standard") + --crypt-no-data-encryption Option to either encrypt file data or leave it unencrypted + --crypt-password string Password or pass phrase for encryption (obscured) + --crypt-password2 string Password or pass phrase for salt (obscured) + --crypt-remote string Remote to encrypt/decrypt + --crypt-server-side-across-configs Allow server-side operations (e.g. copy) to work across different crypt configs + --crypt-show-mapping For all files listed show how the names encrypt + --drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded + --drive-allow-import-name-change Allow the filetype to change when uploading Google docs + --drive-auth-owner-only Only consider files owned by the authenticated user + --drive-auth-url string Auth server URL + --drive-chunk-size SizeSuffix Upload chunk size (default 8Mi) + --drive-client-id string Google Application Client Id + --drive-client-secret string OAuth Client Secret + --drive-copy-shortcut-content Server side copy contents of shortcuts instead of the shortcut + --drive-disable-http2 Disable drive using http2 (default true) + --drive-encoding MultiEncoder The encoding for the backend (default InvalidUtf8) + --drive-export-formats string Comma separated list of preferred formats for downloading Google docs (default "docx,xlsx,pptx,svg") + --drive-formats string Deprecated: See export_formats + --drive-impersonate string Impersonate this user when using a service account + --drive-import-formats string Comma separated list of preferred formats for uploading Google docs + --drive-keep-revision-forever Keep new head revision of each file forever + --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) + --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) + --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) + --drive-resource-key string Resource key for accessing a link-shared file + --drive-root-folder-id string ID of the root folder + --drive-scope string Scope that rclone should use when requesting access from drive + --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs + --drive-service-account-credentials string Service Account Credentials JSON blob + --drive-service-account-file string Service Account Credentials JSON file path + --drive-shared-with-me Only show files that are shared with me + --drive-size-as-quota Show sizes as storage quota usage, not actual size + --drive-skip-checksum-gphotos Skip MD5 checksum on Google photos and videos only + --drive-skip-dangling-shortcuts If set skip dangling shortcut files + --drive-skip-gdocs Skip google documents in all listings + --drive-skip-shortcuts If set skip shortcut files + --drive-starred-only Only show files that are starred + --drive-stop-on-download-limit Make download limit errors be fatal + --drive-stop-on-upload-limit Make upload limit errors be fatal + --drive-team-drive string ID of the Shared Drive (Team Drive) + --drive-token string OAuth Access Token as a JSON blob + --drive-token-url string Token server url + --drive-trashed-only Only show files that are in the trash + --drive-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 8Mi) + --drive-use-created-date Use file created date instead of modified date + --drive-use-shared-date Use date file was shared instead of modified date + --drive-use-trash Send files to the trash instead of deleting permanently (default true) + --drive-v2-download-min-size SizeSuffix If Object's are greater, use drive v2 API to download (default off) + --dropbox-auth-url string Auth server URL + --dropbox-batch-commit-timeout Duration Max time to wait for a batch to finish committing (default 10m0s) + --dropbox-batch-mode string Upload file batching sync|async|off (default "sync") + --dropbox-batch-size int Max number of files in upload batch + --dropbox-batch-timeout Duration Max time to allow an idle upload batch before uploading (default 0s) + --dropbox-chunk-size SizeSuffix Upload chunk size (< 150Mi) (default 48Mi) + --dropbox-client-id string OAuth Client Id + --dropbox-client-secret string OAuth Client Secret + --dropbox-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,RightSpace,InvalidUtf8,Dot) + --dropbox-impersonate string Impersonate this user when using a business account + --dropbox-shared-files Instructs rclone to work on individual shared files + --dropbox-shared-folders Instructs rclone to work on shared folders + --dropbox-token string OAuth Access Token as a JSON blob + --dropbox-token-url string Token server url + --fichier-api-key string Your API Key, get it from https://1fichier.com/console/params.pl + --fichier-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot) + --fichier-file-password string If you want to download a shared file that is password protected, add this parameter (obscured) + --fichier-folder-password string If you want to list the files in a shared folder that is password protected, add this parameter (obscured) + --fichier-shared-folder string If you want to download a shared folder, add this parameter + --filefabric-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) + --filefabric-permanent-token string Permanent Authentication Token + --filefabric-root-folder-id string ID of the root folder + --filefabric-token string Session Token + --filefabric-token-expiry string Token expiry time + --filefabric-url string URL of the Enterprise File Fabric to connect to + --filefabric-version string Version read from the file fabric + --ftp-ask-password Allow asking for FTP password when needed + --ftp-close-timeout Duration Maximum time to wait for a response to close (default 1m0s) + --ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited + --ftp-disable-epsv Disable using EPSV even if server advertises support + --ftp-disable-mlsd Disable using MLSD even if server advertises support + --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) + --ftp-disable-utf8 Disable using UTF-8 even if server advertises support + --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) + --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) + --ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD + --ftp-host string FTP host to connect to + --ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --ftp-no-check-certificate Do not verify the TLS certificate of the server + --ftp-pass string FTP password (obscured) + --ftp-port int FTP port number (default 21) + --ftp-shut-timeout Duration Maximum time to wait for data connection closing status (default 1m0s) + --ftp-tls Use Implicit FTPS (FTP over TLS) + --ftp-tls-cache-size int Size of TLS session cache for all control and data connections (default 32) + --ftp-user string FTP username (default "$USER") + --ftp-writing-mdtm Use MDTM to set modification time (VsFtpd quirk) + --gcs-anonymous Access public buckets and objects without credentials + --gcs-auth-url string Auth server URL + --gcs-bucket-acl string Access Control List for new buckets + --gcs-bucket-policy-only Access checks should use bucket-level IAM policies + --gcs-client-id string OAuth Client Id + --gcs-client-secret string OAuth Client Secret + --gcs-decompress If set this will decompress gzip encoded objects + --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) + --gcs-endpoint string Endpoint for the service + --gcs-location string Location for the newly created buckets + --gcs-no-check-bucket If set, don't attempt to check the bucket exists or create it + --gcs-object-acl string Access Control List for new objects + --gcs-project-number string Project number + --gcs-service-account-file string Service Account Credentials JSON file path + --gcs-storage-class string The storage class to use when storing objects in Google Cloud Storage + --gcs-token string OAuth Access Token as a JSON blob + --gcs-token-url string Token server url + --gphotos-auth-url string Auth server URL + --gphotos-client-id string OAuth Client Id + --gphotos-client-secret string OAuth Client Secret + --gphotos-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) + --gphotos-include-archived Also view and download archived media + --gphotos-read-only Set to make the Google Photos backend read only + --gphotos-read-size Set to read the size of media items + --gphotos-start-year int Year limits the photos to be downloaded to those which are uploaded after the given year (default 2000) + --gphotos-token string OAuth Access Token as a JSON blob + --gphotos-token-url string Token server url + --hasher-auto-size SizeSuffix Auto-update checksum for files smaller than this size (disabled by default) + --hasher-hashes CommaSepList Comma separated list of supported checksum types (default md5,sha1) + --hasher-max-age Duration Maximum time to keep checksums in cache (0 = no cache, off = cache forever) (default off) + --hasher-remote string Remote to cache checksums for (e.g. myRemote:path) + --hdfs-data-transfer-protection string Kerberos data transfer protection: authentication|integrity|privacy + --hdfs-encoding MultiEncoder The encoding for the backend (default Slash,Colon,Del,Ctl,InvalidUtf8,Dot) + --hdfs-namenode string Hadoop name node and port + --hdfs-service-principal-name string Kerberos service principal name for the namenode + --hdfs-username string Hadoop user name + --hidrive-auth-url string Auth server URL + --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) + --hidrive-client-id string OAuth Client Id + --hidrive-client-secret string OAuth Client Secret + --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary + --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --hidrive-endpoint string Endpoint for the service (default "https://api.hidrive.strato.com/2.1") + --hidrive-root-prefix string The root/parent folder for all paths (default "/") + --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default "rw") + --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default "user") + --hidrive-token string OAuth Access Token as a JSON blob + --hidrive-token-url string Token server url + --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) + --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) + --http-headers CommaSepList Set HTTP headers for all transactions + --http-no-head Don't use HEAD requests + --http-no-slash Set this if the site doesn't end directories with / + --http-url string URL of HTTP host to connect to + --internetarchive-access-key-id string IAS3 Access Key + --internetarchive-disable-checksum Don't ask the server to test against MD5 checksum calculated by rclone (default true) + --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) + --internetarchive-endpoint string IAS3 Endpoint (default "https://s3.us.archive.org") + --internetarchive-front-endpoint string Host of InternetArchive Frontend (default "https://archive.org") + --internetarchive-secret-access-key string IAS3 Secret Key (password) + --internetarchive-wait-archive Duration Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s) + --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) + --jottacloud-hard-delete Delete files permanently rather than putting them into the trash + --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) + --jottacloud-no-versions Avoid server side versioning by deleting files and recreating files instead of overwriting them + --jottacloud-trashed-only Only show files that are in the trash + --jottacloud-upload-resume-limit SizeSuffix Files bigger than this can be resumed if the upload fail's (default 10Mi) + --koofr-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --koofr-endpoint string The Koofr API endpoint to use + --koofr-mountid string Mount ID of the mount to use + --koofr-password string Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured) + --koofr-provider string Choose your storage provider + --koofr-setmtime Does the backend support setting modification time (default true) + --koofr-user string Your user name + -l, --links Translate symlinks to/from regular files with a '.rclonelink' extension + --local-case-insensitive Force the filesystem to report itself as case insensitive + --local-case-sensitive Force the filesystem to report itself as case sensitive + --local-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --local-no-check-updated Don't check to see if the files change during upload + --local-no-preallocate Disable preallocation of disk space for transferred files + --local-no-set-modtime Disable setting modtime + --local-no-sparse Disable sparse files for multi-thread downloads + --local-nounc Disable UNC (long path names) conversion on Windows + --local-unicode-normalization Apply unicode NFC normalization to paths and filenames + --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) + --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) + --mailru-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --mailru-pass string Password (obscured) + --mailru-speedup-enable Skip full upload if there is another file with same data hash (default true) + --mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash) (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf") + --mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3Gi) + --mailru-speedup-max-memory SizeSuffix Files larger than the size given below will always be hashed on disk (default 32Mi) + --mailru-user string User name (usually email) + --mega-debug Output more debug from Mega + --mega-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --mega-hard-delete Delete files permanently rather than putting them into the trash + --mega-pass string Password (obscured) + --mega-user string User name + --netstorage-account string Set the NetStorage account name + --netstorage-host string Domain+path of NetStorage host to connect to + --netstorage-protocol string Select between HTTP or HTTPS protocol (default "https") + --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) + -x, --one-file-system Don't cross filesystem boundaries (unix/macOS only) + --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) + --onedrive-auth-url string Auth server URL + --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) + --onedrive-client-id string OAuth Client Id + --onedrive-client-secret string OAuth Client Secret + --onedrive-drive-id string The ID of the drive to use + --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) + --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) + --onedrive-expose-onenote-files Set to make OneNote files show up in directory listings + --onedrive-link-password string Set the password for links created by the link command + --onedrive-link-scope string Set the scope of the links created by the link command (default "anonymous") + --onedrive-link-type string Set the type of the links created by the link command (default "view") + --onedrive-list-chunk int Size of listing chunk (default 1000) + --onedrive-no-versions Remove all versions on modifying operations + --onedrive-region string Choose national cloud region for OneDrive (default "global") + --onedrive-root-folder-id string ID of the root folder + --onedrive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different onedrive configs + --onedrive-token string OAuth Access Token as a JSON blob + --onedrive-token-url string Token server url + --oos-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) + --oos-compartment string Object storage compartment OCID + --oos-config-file string Path to OCI config file (default "~/.oci/config") + --oos-config-profile string Profile name inside the oci config file (default "Default") + --oos-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) + --oos-copy-timeout Duration Timeout for copy (default 1m0s) + --oos-disable-checksum Don't store MD5 checksum with object metadata + --oos-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --oos-endpoint string Endpoint for Object storage API + --oos-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery + --oos-namespace string Object storage namespace + --oos-no-check-bucket If set, don't attempt to check the bucket exists or create it + --oos-provider string Choose your Auth Provider (default "env_auth") + --oos-region string Object storage Region + --oos-upload-concurrency int Concurrency for multipart uploads (default 10) + --oos-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size (default 10Mi) + --opendrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot) + --opendrive-password string Password (obscured) + --opendrive-username string Username + --pcloud-auth-url string Auth server URL + --pcloud-client-id string OAuth Client Id + --pcloud-client-secret string OAuth Client Secret + --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --pcloud-hostname string Hostname to connect to (default "api.pcloud.com") + --pcloud-password string Your pcloud password (obscured) + --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default "d0") + --pcloud-token string OAuth Access Token as a JSON blob + --pcloud-token-url string Token server url + --pcloud-username string Your pcloud username + --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --qingstor-access-key-id string QingStor Access Key ID + --qingstor-chunk-size SizeSuffix Chunk size to use for uploading (default 4Mi) + --qingstor-connection-retries int Number of connection retries (default 3) + --qingstor-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8) + --qingstor-endpoint string Enter an endpoint URL to connection QingStor API + --qingstor-env-auth Get QingStor credentials from runtime + --qingstor-secret-access-key string QingStor Secret Access Key (password) + --qingstor-upload-concurrency int Concurrency for multipart uploads (default 1) + --qingstor-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --qingstor-zone string Zone to connect to + --s3-access-key-id string AWS Access Key ID + --s3-acl string Canned ACL used when creating buckets and storing or copying objects + --s3-bucket-acl string Canned ACL used when creating buckets + --s3-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) + --s3-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) + --s3-decompress If set this will decompress gzip encoded objects + --s3-disable-checksum Don't store MD5 checksum with object metadata + --s3-disable-http2 Disable usage of http2 for S3 backends + --s3-download-url string Custom endpoint for downloads + --s3-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --s3-endpoint string Endpoint for S3 API + --s3-env-auth Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars) + --s3-force-path-style If true use path style access if false use virtual hosted style (default true) + --s3-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery + --s3-list-chunk int Size of listing chunk (response list for each ListObject S3 request) (default 1000) + --s3-list-url-encode Tristate Whether to url encode listings: true/false/unset (default unset) + --s3-list-version int Version of ListObjects to use: 1,2 or 0 for auto + --s3-location-constraint string Location constraint - must be set to match the Region + --s3-max-upload-parts int Maximum number of parts in a multipart upload (default 10000) + --s3-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --s3-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --s3-might-gzip Tristate Set this if the backend might gzip objects (default unset) + --s3-no-check-bucket If set, don't attempt to check the bucket exists or create it + --s3-no-head If set, don't HEAD uploaded objects to check integrity + --s3-no-head-object If set, do not do HEAD before GET when getting objects + --s3-no-system-metadata Suppress setting and reading of system metadata + --s3-profile string Profile to use in the shared credentials file + --s3-provider string Choose your S3 provider + --s3-region string Region to connect to + --s3-requester-pays Enables requester pays option when interacting with S3 bucket + --s3-secret-access-key string AWS Secret Access Key (password) + --s3-server-side-encryption string The server-side encryption algorithm used when storing this object in S3 + --s3-session-token string An AWS session token + --s3-shared-credentials-file string Path to the shared credentials file + --s3-sse-customer-algorithm string If using SSE-C, the server-side encryption algorithm used when storing this object in S3 + --s3-sse-customer-key string To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data + --s3-sse-customer-key-base64 string If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data + --s3-sse-customer-key-md5 string If using SSE-C you may provide the secret encryption key MD5 checksum (optional) + --s3-sse-kms-key-id string If using KMS ID you must provide the ARN of Key + --s3-storage-class string The storage class to use when storing new objects in S3 + --s3-upload-concurrency int Concurrency for multipart uploads (default 4) + --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint + --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) + --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads + --s3-v2-auth If true use v2 authentication + --s3-version-at Time Show file versions as they were at the specified time (default off) + --s3-versions Include old versions in directory listings + --seafile-2fa Two-factor authentication ('true' if the account has 2FA enabled) + --seafile-create-library Should rclone create a library if it doesn't exist + --seafile-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8) + --seafile-library string Name of the library + --seafile-library-key string Library password (for encrypted libraries only) (obscured) + --seafile-pass string Password (obscured) + --seafile-url string URL of seafile host to connect to + --seafile-user string User name (usually email address) + --sftp-ask-password Allow asking for SFTP password when needed + --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) + --sftp-ciphers SpaceSepList Space separated list of ciphers to be used for session encryption, ordered by preference + --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) + --sftp-disable-concurrent-reads If set don't use concurrent reads + --sftp-disable-concurrent-writes If set don't use concurrent writes + --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available + --sftp-host string SSH host to connect to + --sftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --sftp-key-exchange SpaceSepList Space separated list of key exchange algorithms, ordered by preference + --sftp-key-file string Path to PEM-encoded private key file + --sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file (obscured) + --sftp-key-pem string Raw PEM-encoded private key + --sftp-key-use-agent When set forces the usage of the ssh-agent + --sftp-known-hosts-file string Optional path to known_hosts file + --sftp-macs SpaceSepList Space separated list of MACs (message authentication code) algorithms, ordered by preference + --sftp-md5sum-command string The command used to read md5 hashes + --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) + --sftp-path-override string Override path used by SSH shell commands + --sftp-port int SSH port number (default 22) + --sftp-pubkey-file string Optional path to public key file + --sftp-server-command string Specifies the path or command to run a sftp server on the remote host + --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands + --sftp-set-modtime Set the modified time on the remote if set (default true) + --sftp-sha1sum-command string The command used to read sha1 hashes + --sftp-shell-type string The type of SSH shell on remote server, if any + --sftp-skip-links Set to skip any symlinks and any other non regular files + --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default "sftp") + --sftp-use-fstat If set use fstat instead of stat + --sftp-use-insecure-cipher Enable the use of insecure ciphers and key exchange methods + --sftp-user string SSH username (default "$USER") + --sharefile-chunk-size SizeSuffix Upload chunk size (default 64Mi) + --sharefile-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,LeftSpace,LeftPeriod,RightSpace,RightPeriod,InvalidUtf8,Dot) + --sharefile-endpoint string Endpoint for API calls + --sharefile-root-folder-id string ID of the root folder + --sharefile-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (default 128Mi) + --sia-api-password string Sia Daemon API Password (obscured) + --sia-api-url string Sia daemon API URL, like http://sia.daemon.host:9980 (default "http://127.0.0.1:9980") + --sia-encoding MultiEncoder The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot) + --sia-user-agent string Siad User Agent (default "Sia-Agent") + --skip-links Don't warn about skipped symlinks + --smb-case-insensitive Whether the server is configured to be case-insensitive (default true) + --smb-domain string Domain name for NTLM authentication (default "WORKGROUP") + --smb-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot) + --smb-hide-special-share Hide special shares (e.g. print$) which users aren't supposed to access (default true) + --smb-host string SMB server hostname to connect to + --smb-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --smb-pass string SMB password (obscured) + --smb-port int SMB port number (default 445) + --smb-user string SMB username (default "$USER") + --storj-access-grant string Access grant + --storj-api-key string API key + --storj-passphrase string Encryption passphrase + --storj-provider string Choose an authentication method (default "existing") + --storj-satellite-address string Satellite address (default "us-central-1.storj.io") + --sugarsync-access-key-id string Sugarsync Access Key ID + --sugarsync-app-id string Sugarsync App ID + --sugarsync-authorization string Sugarsync authorization + --sugarsync-authorization-expiry string Sugarsync authorization expiry + --sugarsync-deleted-id string Sugarsync deleted folder id + --sugarsync-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8,Dot) + --sugarsync-hard-delete Permanently delete files if true + --sugarsync-private-access-key string Sugarsync Private Access Key + --sugarsync-refresh-token string Sugarsync refresh token + --sugarsync-root-id string Sugarsync root id + --sugarsync-user string Sugarsync user + --swift-application-credential-id string Application Credential ID (OS_APPLICATION_CREDENTIAL_ID) + --swift-application-credential-name string Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME) + --swift-application-credential-secret string Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET) + --swift-auth string Authentication URL for server (OS_AUTH_URL) + --swift-auth-token string Auth Token from alternate authentication - optional (OS_AUTH_TOKEN) + --swift-auth-version int AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION) + --swift-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) + --swift-domain string User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME) + --swift-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8) + --swift-endpoint-type string Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default "public") + --swift-env-auth Get swift credentials from environment variables in standard OpenStack form + --swift-key string API key or password (OS_PASSWORD) + --swift-leave-parts-on-error If true avoid calling abort upload on a failure + --swift-no-chunk Don't chunk files during streaming upload + --swift-no-large-objects Disable support for static and dynamic large objects + --swift-region string Region name - optional (OS_REGION_NAME) + --swift-storage-policy string The storage policy to use when creating a new container + --swift-storage-url string Storage URL - optional (OS_STORAGE_URL) + --swift-tenant string Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME) + --swift-tenant-domain string Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME) + --swift-tenant-id string Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID) + --swift-user string User name to log in (OS_USERNAME) + --swift-user-id string User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID) + --union-action-policy string Policy to choose upstream on ACTION category (default "epall") + --union-cache-time int Cache time of usage and free space (in seconds) (default 120) + --union-create-policy string Policy to choose upstream on CREATE category (default "epmfs") + --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) + --union-search-policy string Policy to choose upstream on SEARCH category (default "ff") + --union-upstreams string List of space separated upstreams + --uptobox-access-token string Your access token + --uptobox-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot) + --webdav-bearer-token string Bearer token instead of user/pass (e.g. a Macaroon) + --webdav-bearer-token-command string Command to run to get a bearer token + --webdav-encoding string The encoding for the backend + --webdav-headers CommaSepList Set HTTP headers for all transactions + --webdav-pass string Password (obscured) + --webdav-url string URL of http host to connect to + --webdav-user string User name + --webdav-vendor string Name of the WebDAV site/service/software you are using + --yandex-auth-url string Auth server URL + --yandex-client-id string OAuth Client Id + --yandex-client-secret string OAuth Client Secret + --yandex-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) + --yandex-hard-delete Delete files permanently rather than putting them into the trash + --yandex-token string OAuth Access Token as a JSON blob + --yandex-token-url string Token server url + --zoho-auth-url string Auth server URL + --zoho-client-id string OAuth Client Id + --zoho-client-secret string OAuth Client Secret + --zoho-encoding MultiEncoder The encoding for the backend (default Del,Ctl,InvalidUtf8) + --zoho-region string Zoho region to connect to + --zoho-token string OAuth Access Token as a JSON blob + --zoho-token-url string Token server url Docker Volume Plugin @@ -16542,9 +16802,10 @@ This will guide you through an interactive setup process: token_url> Optional token URL Remote config Make sure your Redirect URL is set to "http://127.0.0.1:53682/" in your custom config. - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -16815,6 +17076,7 @@ The S3 backend can be used with a number of different providers: - IBM COS S3 - IDrive e2 - IONOS Cloud +- Liara Object Storage - Minio - Qiniu Cloud Object Storage (Kodo) - RackCorp Object Storage @@ -16870,7 +17132,7 @@ This will guide you through an interactive setup process. Type of storage to configure. Choose a number from below, or type in your own value [snip] - XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS + XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Liara, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -16880,7 +17142,7 @@ This will guide you through an interactive setup process. \ "AWS" 2 / Ceph Object Storage \ "Ceph" - 3 / Digital Ocean Spaces + 3 / DigitalOcean Spaces \ "DigitalOcean" 4 / Dreamhost DreamObjects \ "Dreamhost" @@ -17430,9 +17692,9 @@ Standard options Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, -Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, -IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, -SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). +Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, +IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, +Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). --s3-provider @@ -17458,7 +17720,7 @@ Properties: - "ArvanCloud" - Arvan Cloud Object Storage (AOS) - "DigitalOcean" - - Digital Ocean Spaces + - DigitalOcean Spaces - "Dreamhost" - Dreamhost DreamObjects - "HuaweiOBS" @@ -17471,6 +17733,8 @@ Properties: - IONOS Cloud - "LyveCloud" - Seagate Lyve Cloud + - "Liara" + - Liara Object Storage - "Minio" - Minio Object Storage - "Netease" @@ -17824,7 +18088,7 @@ Properties: - Config: region - Env Var: RCLONE_S3_REGION - Provider: - !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive + !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Liara,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive - Type: string - Required: false - Examples: @@ -18103,6 +18367,22 @@ Properties: --s3-endpoint +Endpoint for Liara Object Storage API. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Provider: Liara +- Type: string +- Required: false +- Examples: + - "storage.iran.liara.space" + - The default endpoint + - Iran + +--s3-endpoint + Endpoint for OSS API. Properties: @@ -18404,18 +18684,24 @@ Properties: - Config: endpoint - Env Var: RCLONE_S3_ENDPOINT - Provider: - !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu + !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu - Type: string - Required: false - Examples: - "objects-us-east-1.dream.io" - Dream Objects endpoint + - "syd1.digitaloceanspaces.com" + - DigitalOcean Spaces Sydney 1 + - "sfo3.digitaloceanspaces.com" + - DigitalOcean Spaces San Francisco 3 + - "fra1.digitaloceanspaces.com" + - DigitalOcean Spaces Frankfurt 1 - "nyc3.digitaloceanspaces.com" - - Digital Ocean Spaces New York 3 + - DigitalOcean Spaces New York 3 - "ams3.digitaloceanspaces.com" - - Digital Ocean Spaces Amsterdam 3 + - DigitalOcean Spaces Amsterdam 3 - "sgp1.digitaloceanspaces.com" - - Digital Ocean Spaces Singapore 1 + - DigitalOcean Spaces Singapore 1 - "localhost:8333" - SeaweedFS S3 localhost - "s3.us-east-1.lyvecloud.seagate.com" @@ -18425,15 +18711,33 @@ Properties: - "s3.ap-southeast-1.lyvecloud.seagate.com" - Seagate Lyve Cloud AP Southeast 1 (Singapore) - "s3.wasabisys.com" - - Wasabi US East endpoint + - Wasabi US East 1 (N. Virginia) + - "s3.us-east-2.wasabisys.com" + - Wasabi US East 2 (N. Virginia) + - "s3.us-central-1.wasabisys.com" + - Wasabi US Central 1 (Texas) - "s3.us-west-1.wasabisys.com" - - Wasabi US West endpoint + - Wasabi US West 1 (Oregon) + - "s3.ca-central-1.wasabisys.com" + - Wasabi CA Central 1 (Toronto) - "s3.eu-central-1.wasabisys.com" - - Wasabi EU Central endpoint + - Wasabi EU Central 1 (Amsterdam) + - "s3.eu-central-2.wasabisys.com" + - Wasabi EU Central 2 (Frankfurt) + - "s3.eu-west-1.wasabisys.com" + - Wasabi EU West 1 (London) + - "s3.eu-west-2.wasabisys.com" + - Wasabi EU West 2 (Paris) - "s3.ap-northeast-1.wasabisys.com" - Wasabi AP Northeast 1 (Tokyo) endpoint - "s3.ap-northeast-2.wasabisys.com" - Wasabi AP Northeast 2 (Osaka) endpoint + - "s3.ap-southeast-1.wasabisys.com" + - Wasabi AP Southeast 1 (Singapore) + - "s3.ap-southeast-2.wasabisys.com" + - Wasabi AP Southeast 2 (Sydney) + - "storage.iran.liara.space" + - Liara Iran endpoint - "s3.ir-thr-at1.arvanstorage.com" - ArvanCloud Tehran Iran (Asiatech) endpoint @@ -18767,7 +19071,7 @@ Properties: - Config: location_constraint - Env Var: RCLONE_S3_LOCATION_CONSTRAINT - Provider: - !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS + !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Liara,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS - Type: string - Required: false @@ -18784,6 +19088,9 @@ https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Note that this ACL is applied when server-side copying objects as S3 doesn't copy the ACL from the source but rather writes a fresh one. +If the acl is an empty string then no X-Amz-Acl: header is added and the +default (private) will be used. + Properties: - Config: acl @@ -18952,6 +19259,21 @@ Properties: --s3-storage-class +The storage class to use when storing new objects in Liara + +Properties: + +- Config: storage_class +- Env Var: RCLONE_S3_STORAGE_CLASS +- Provider: Liara +- Type: string +- Required: false +- Examples: + - "STANDARD" + - Standard storage class + +--s3-storage-class + The storage class to use when storing new objects in ArvanCloud. Properties: @@ -19033,9 +19355,9 @@ Advanced options Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, -Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, -IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, -SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). +Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, +IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, +Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). --s3-bucket-acl @@ -19047,6 +19369,9 @@ https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Note that this ACL is applied when only when creating buckets. If it isn't set then "acl" is used instead. +If the "acl" and "bucket_acl" are empty strings then no X-Amz-Acl: +header is added and the default (private) will be used. + Properties: - Config: bucket_acl @@ -19665,6 +19990,36 @@ Properties: - Type: bool - Default: false +--s3-might-gzip + +Set this if the backend might gzip objects. + +Normally providers will not alter objects when they are downloaded. If +an object was not uploaded with Content-Encoding: gzip then it won't be +set on download. + +However some providers may gzip objects even if they weren't uploaded +with Content-Encoding: gzip (eg Cloudflare). + +A symptom of this would be receiving errors like + + ERROR corrupted on transfer: sizes differ NNN vs MMM + +If you set this flag and rclone downloads an object with +Content-Encoding: gzip set and chunked transfer encoding, then rclone +will decompress the object on the fly. + +If this is set to unset (the default) then rclone will choose according +to the provider setting what to apply, but you can override rclone's +choice here. + +Properties: + +- Config: might_gzip +- Env Var: RCLONE_S3_MIGHT_GZIP +- Type: Tristate +- Default: unset + --s3-no-system-metadata Suppress setting and reading of system metadata @@ -19995,7 +20350,7 @@ of a bucket publicly. Type of storage to configure. Choose a number from below, or type in your own value. ... - XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) ... Storage> s3 @@ -20164,7 +20519,7 @@ Or you can also configure via the interactive command line: Type of storage to configure. Choose a number from below, or type in your own value. [snip] - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) [snip] Storage> 5 @@ -20294,7 +20649,7 @@ To configure access to IBM COS S3, follow the steps below: \ "alias" 2 / Amazon Drive \ "amazon cloud drive" - 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS) + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, Liara, ArvanCloud, Minio, IBM COS) \ "s3" 4 / Backblaze B2 \ "b2" @@ -20455,7 +20810,7 @@ This will guide you through an interactive setup process. Type of storage to configure. Choose a number from below, or type in your own value. [snip] - XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) [snip] Storage> s3 @@ -20561,7 +20916,7 @@ Type s3 to choose the connection type: Type of storage to configure. Choose a number from below, or type in your own value. [snip] - XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \ (s3) [snip] Storage> s3 @@ -20795,7 +21150,7 @@ To configure access to Qiniu Kodo, follow the steps below: \ (alias) 4 / Amazon Drive \ (amazon cloud drive) - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi \ (s3) [snip] Storage> s3 @@ -21032,7 +21387,7 @@ Choose s3 backend Type of storage to configure. Choose a number from below, or type in your own value. [snip] - XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS + XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \ (s3) [snip] Storage> s3 @@ -21196,7 +21551,7 @@ rclone like this. Type of storage to configure. Choose a number from below, or type in your own value [snip] - XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) + XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, Liara) \ "s3" [snip] Storage> s3 @@ -21306,7 +21661,7 @@ This will guide you through an interactive setup process. Enter a string value. Press Enter for the default (""). Choose a number from below, or type in your own value [snip] - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -21414,7 +21769,7 @@ This will guide you through an interactive setup process. Type of storage to configure. Choose a number from below, or type in your own value. ... - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \ (s3) ... Storage> s3 @@ -21643,6 +21998,103 @@ This will guide you through an interactive setup process. d) Delete this remote y/e/d> y +Liara + +Here is an example of making a Liara Object Storage configuration. First +run: + + rclone config + +This will guide you through an interactive setup process. + + No remotes found, make a new one? + n) New remote + s) Set configuration password + n/s> n + name> Liara + Type of storage to configure. + Choose a number from below, or type in your own value + [snip] + XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Liara, Minio) + \ "s3" + [snip] + Storage> s3 + Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. + Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \ "false" + 2 / Get AWS credentials from the environment (env vars or IAM) + \ "true" + env_auth> 1 + AWS Access Key ID - leave blank for anonymous access or runtime credentials. + access_key_id> YOURACCESSKEY + AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. + secret_access_key> YOURSECRETACCESSKEY + Region to connect to. + Choose a number from below, or type in your own value + / The default endpoint + 1 | US Region, Northern Virginia, or Pacific Northwest. + | Leave location constraint empty. + \ "us-east-1" + [snip] + region> + Endpoint for S3 API. + Leave blank if using Liara to use the default endpoint for the region. + Specify if using an S3 clone such as Ceph. + endpoint> storage.iran.liara.space + Canned ACL used when creating buckets and/or storing objects in S3. + For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl + Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \ "private" + [snip] + acl> + The server-side encryption algorithm used when storing this object in S3. + Choose a number from below, or type in your own value + 1 / None + \ "" + 2 / AES256 + \ "AES256" + server_side_encryption> + The storage class to use when storing objects in S3. + Choose a number from below, or type in your own value + 1 / Default + \ "" + 2 / Standard storage class + \ "STANDARD" + storage_class> + Remote config + -------------------- + [Liara] + env_auth = false + access_key_id = YOURACCESSKEY + secret_access_key = YOURSECRETACCESSKEY + endpoint = storage.iran.liara.space + location_constraint = + acl = + server_side_encryption = + storage_class = + -------------------- + y) Yes this is OK + e) Edit this remote + d) Delete this remote + y/e/d> y + +This will leave the config file looking like this. + + [Liara] + type = s3 + provider = Liara + env_auth = false + access_key_id = YOURACCESSKEY + secret_access_key = YOURSECRETACCESSKEY + region = + endpoint = storage.iran.liara.space + location_constraint = + acl = + server_side_encryption = + storage_class = + ArvanCloud ArvanCloud ArvanCloud Object Storage goes beyond the limited traditional @@ -21662,7 +22114,7 @@ rclone like this. Type of storage to configure. Choose a number from below, or type in your own value [snip] - XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) + XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Liara, Minio) \ "s3" [snip] Storage> s3 @@ -21779,7 +22231,7 @@ To configure access to Tencent COS, follow the steps below: \ "alias" 3 / Amazon Drive \ "amazon cloud drive" - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS \ "s3" [snip] Storage> s3 @@ -22584,9 +23036,10 @@ This will guide you through an interactive setup process: \ "enterprise" box_sub_type> Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -22701,9 +23154,10 @@ Here is how to do it. y) Yes n) No y/n> y - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -22977,13 +23431,15 @@ Reverse Solidus). Box only supports filenames up to 255 characters in length. +Box has API rate limits that sometimes reduce the speed of rclone. + rclone about is not supported by the Box backend. Backends without this capability cannot determine free space for an rclone mount or use policy mfs (most free space) as a member of an rclone union remote. See List of backends that do not support rclone about and rclone about -Cache (DEPRECATED) +Cache The cache remote wraps another existing remote and stores file structure and its data for long running tasks like rclone mount. @@ -23678,7 +24134,7 @@ Print stats on the cache backend in JSON format. rclone backend stats remote: [options] [+] -Chunker (BETA) +Chunker The chunker overlay transparently splits large files into smaller chunks during upload to wrapped remote and transparently assembles them back @@ -24200,9 +24656,10 @@ This will guide you through an interactive setup process: n) No y/n> n Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -25137,7 +25594,7 @@ SEE ALSO - rclone cryptdecode - Show forward/reverse mapping of encrypted filenames -Compress (Experimental) +Compress Warning @@ -25494,6 +25951,15 @@ This will guide you through an interactive setup process: d) Delete this remote y/e/d> y +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + +Note that rclone runs a webserver on your local machine to collect the +token as returned from Dropbox. This only runs from the moment it opens +your browser to the moment you get back the verification code. This is +on http://127.0.0.1:53682/ and it may require you to unblock it +temporarily if you are running a host firewall, or use manual mode. + You can then use it like this, List directories in top level of your dropbox @@ -26391,7 +26857,7 @@ Use Implicit FTPS (FTP over TLS). When using implicit FTP over TLS the client connects using TLS right from the start which breaks compatibility with non-TLS-aware servers. This is usually served over port 990 rather than port 21. Cannot be used -in combination with explicit FTP. +in combination with explicit FTPS. Properties: @@ -26406,7 +26872,7 @@ Use Explicit FTPS (FTP over TLS). When using explicit FTP over TLS the client explicitly requests security from the server in order to upgrade a plain text connection to an -encrypted one. Cannot be used in combination with implicit FTP. +encrypted one. Cannot be used in combination with implicit FTPS. Properties: @@ -26772,9 +27238,10 @@ This will guide you through an interactive setup process: \ "DURABLE_REDUCED_AVAILABILITY" storage_class> 5 Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine or Y didn't work + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -26797,12 +27264,15 @@ This will guide you through an interactive setup process: d) Delete this remote y/e/d> y +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. This only -runs from the moment it opens your browser to the moment you get back -the verification code. This is on http://127.0.0.1:53682/ and this it -may require you to unblock it temporarily if you are running a host -firewall, or use manual mode. +token as returned from Google if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to +the moment you get back the verification code. This is on +http://127.0.0.1:53682/ and this it may require you to unblock it +temporarily if you are running a host firewall, or use manual mode. This remote is called remote and can now be used like this @@ -27366,9 +27836,10 @@ This will guide you through an interactive setup process: Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. service_account_file> Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine or Y didn't work + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -27394,12 +27865,15 @@ This will guide you through an interactive setup process: d) Delete this remote y/e/d> y +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. This only -runs from the moment it opens your browser to the moment you get back -the verification code. This is on http://127.0.0.1:53682/ and it may -require you to unblock it temporarily if you are running a host -firewall, or use manual mode. +token as returned from Google if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to +the moment you get back the verification code. This is on +http://127.0.0.1:53682/ and it may require you to unblock it temporarily +if you are running a host firewall, or use manual mode. You can then use it like this, @@ -28939,9 +29413,10 @@ This will guide you through an interactive setup process: n) No y/n> n Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -28964,12 +29439,15 @@ This will guide you through an interactive setup process: d) Delete this remote y/e/d> y +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. This only -runs from the moment it opens your browser to the moment you get back -the verification code. This is on http://127.0.0.1:53682/ and this may -require you to unblock it temporarily if you are running a host -firewall, or use manual mode. +token as returned from Google if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to +the moment you get back the verification code. This is on +http://127.0.0.1:53682/ and this may require you to unblock it +temporarily if you are running a host firewall, or use manual mode. This remote is called remote and can now be used like this @@ -29341,7 +29819,7 @@ Deleting albums The Google Photos API does not support deleting albums - see bug #135714733. -Hasher (EXPERIMENTAL) +Hasher Hasher is a special overlay backend to create remotes which handle checksums for other remotes. It's main functions include: - Emulate hash @@ -29934,7 +30412,10 @@ This will guide you through an interactive setup process: scope_access> Edit advanced config? y/n> n - Use auto config? + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y/n> y If your browser doesn't open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx Log in and authorize rclone for access @@ -30632,6 +31113,22 @@ can be triggered when you did a server-side copy. Reading metadata will also provide custom (non-standard nor reserved) ones. +Filtering auto generated files + +The Internet Archive automatically creates metadata files after upload. +These can cause problems when doing an rclone sync as rclone will try, +and fail, to delete them. These metadata files are not changeable, as +they are created by the Internet Archive automatically. + +These auto-created files can be excluded from the sync using metadata +filtering. + + rclone sync ... --metadata-exclude "source=metadata" --metadata-exclude "format=Metadata" + +Which excludes from the sync any files which have the source=metadata or +format=Metadata flags which are added to Internet Archive auto-created +files. + Configuration Here is an example of making an internetarchive configuration. Most @@ -31679,8 +32176,22 @@ Features highlights Configuration -Here is an example of making a mailru configuration. First create a -Mail.ru Cloud account and choose a tariff, then run +Here is an example of making a mailru configuration. + +First create a Mail.ru Cloud account and choose a tariff. + +You will need to log in and create an app password for rclone. Rclone +will not work with your normal username and password - it will give an +error like oauth2: server response missing access_token. + +- Click on your user icon in the top right +- Go to Security / "Пароль и безопасность" +- Click password for apps / "Пароли для внешних приложений" +- Add the password - give it a name - eg "rclone" +- Copy the password and use this password below - your normal login + password won't work. + +Now run rclone config @@ -31705,6 +32216,10 @@ This will guide you through an interactive setup process: Enter a string value. Press Enter for the default (""). user> username@mail.ru Password + + This must be an app password - rclone will not work with your normal + password. See the Configuration section in the docs for how to make an + app password. y) Yes type in my own password g) Generate random password y/g> y @@ -31826,6 +32341,10 @@ Properties: Password. +This must be an app password - rclone will not work with your normal +password. See the Configuration section in the docs for how to make an +app password. + NB Input to this must be obscured - see rclone obscure. Properties: @@ -32710,7 +33229,12 @@ Modified time The modified time is stored as metadata on the object with the mtime key. It is stored using RFC3339 Format time with nanosecond precision. The metadata is supplied during directory listings so there is no -overhead to using it. +performance overhead to using it. + +If you wish to use the Azure standard LastModified time stored on the +object as the modified time, then use the --use-server-modtime flag. +Note that rclone can't set LastModified, so using the --update flag when +syncing is recommended if using --use-server-modtime. Performance @@ -32746,11 +33270,88 @@ MD5 hashes are stored with blobs. However blobs that were uploaded in chunks only have an MD5 if the source remote was capable of MD5 hashes, e.g. the local disk. -Authenticating with Azure Blob Storage +Authentication + +There are a number of ways of supplying credentials for Azure Blob +Storage. Rclone tries them in the order of the sections below. + +Env Auth + +If the env_auth config parameter is true then rclone will pull +credentials from the environment or runtime. + +It tries these authentication methods in this order: + +1. Environment Variables +2. Managed Service Identity Credentials +3. Azure CLI credentials (as used by the az tool) + +These are described in the following sections + +Env Auth: 1. Environment Variables + +If env_auth is set and environment variables are present rclone +authenticates a service principal with a secret or certificate, or a +user with a password, depending on which environment variable are set. +It reads configuration from these variables, in the following order: + +1. Service principal with client secret + - AZURE_TENANT_ID: ID of the service principal's tenant. Also + called its "directory" ID. + - AZURE_CLIENT_ID: the service principal's client ID + - AZURE_CLIENT_SECRET: one of the service principal's client + secrets +2. Service principal with certificate + - AZURE_TENANT_ID: ID of the service principal's tenant. Also + called its "directory" ID. + - AZURE_CLIENT_ID: the service principal's client ID + - AZURE_CLIENT_CERTIFICATE_PATH: path to a PEM or PKCS12 + certificate file including the private key. + - AZURE_CLIENT_CERTIFICATE_PASSWORD: (optional) password for the + certificate file. + - AZURE_CLIENT_SEND_CERTIFICATE_CHAIN: (optional) Specifies + whether an authentication request will include an x5c header to + support subject name / issuer based authentication. When set to + "true" or "1", authentication requests include the x5c header. +3. User with username and password + - AZURE_TENANT_ID: (optional) tenant to authenticate in. Defaults + to "organizations". + - AZURE_CLIENT_ID: client ID of the application the user will + authenticate to + - AZURE_USERNAME: a username (usually an email address) + - AZURE_PASSWORD: the user's password + +Env Auth: 2. Managed Service Identity Credentials + +When using Managed Service Identity if the VM(SS) on which this program +is running has a system-assigned identity, it will be used by default. +If the resource has no system-assigned but exactly one user-assigned +identity, the user-assigned identity will be used by default. + +If the resource has multiple user-assigned identities you will need to +unset env_auth and set use_msi instead. See the use_msi section. + +Env Auth: 3. Azure CLI credentials (as used by the az tool) + +Credentials created with the az tool can be picked up using env_auth. + +For example if you were to login with a service principal like this: + + az login --service-principal -u XXX -p XXX --tenant XXX + +Then you could access rclone resources like this: + + rclone lsf :azureblob,env_auth,account=ACCOUNT:CONTAINER + +Or -Rclone has 3 ways of authenticating with Azure Blob Storage: + rclone lsf --azureblob-env-auth --azureblob-acccount=ACCOUNT :azureblob:CONTAINER -Account and Key +Which is analogous to using the az tool: + + az storage blob list --container-name CONTAINER --account-name ACCOUNT --auth-mode login + +Account and Shared Key This is the most straight forward and least flexible way. Just fill in the account and key lines and leave the rest blank. @@ -32759,7 +33360,7 @@ SAS URL This can be an account level SAS URL or container level SAS URL. -To use it leave account, key blank and fill in sas_url. +To use it leave account and key blank and fill in sas_url. An account level SAS URL or container level SAS URL can be obtained from the Azure portal or the Azure Storage Explorer. To get a container level @@ -32785,6 +33386,72 @@ Container level SAS URLs are useful for temporarily allowing third parties access to a single container or putting credentials into an untrusted environment such as a CI build server. +Service principal with client secret + +If these variables are set, rclone will authenticate with a service +principal with a client secret. + +- tenant: ID of the service principal's tenant. Also called its + "directory" ID. +- client_id: the service principal's client ID +- client_secret: one of the service principal's client secrets + +The credentials can also be placed in a file using the +service_principal_file configuration option. + +Service principal with certificate + +If these variables are set, rclone will authenticate with a service +principal with certificate. + +- tenant: ID of the service principal's tenant. Also called its + "directory" ID. +- client_id: the service principal's client ID +- client_certificate_path: path to a PEM or PKCS12 certificate file + including the private key. +- client_certificate_password: (optional) password for the certificate + file. +- client_send_certificate_chain: (optional) Specifies whether an + authentication request will include an x5c header to support subject + name / issuer based authentication. When set to "true" or "1", + authentication requests include the x5c header. + +NB client_certificate_password must be obscured - see rclone obscure. + +User with username and password + +If these variables are set, rclone will authenticate with username and +password. + +- tenant: (optional) tenant to authenticate in. Defaults to + "organizations". +- client_id: client ID of the application the user will authenticate + to +- username: a username (usually an email address) +- password: the user's password + +Microsoft doesn't recommend this kind of authentication, because it's +less secure than other authentication flows. This method is not +interactive, so it isn't compatible with any form of multi-factor +authentication, and the application must already have user or admin +consent. This credential can only authenticate work and school accounts; +it can't authenticate Microsoft accounts. + +NB password must be obscured - see rclone obscure. + +Managed Service Identity Credentials + +If use_msi is set then managed service identity credentials are used. +This authentication only works when running in an Azure service. +env_auth needs to be unset to use this. + +However if you have multiple user identities to choose from these must +be explicitly specified using exactly one of the msi_object_id, +msi_client_id, or msi_mi_res_id parameters. + +If none of msi_object_id, msi_client_id, or msi_mi_res_id is set, this +is is equivalent to using env_auth. + Standard options Here are the Standard options specific to azureblob (Microsoft Azure @@ -32792,9 +33459,14 @@ Blob Storage). --azureblob-account -Storage Account Name. +Azure Storage Account Name. -Leave blank to use SAS URL or Emulator. +Set this to the Azure Storage Account Name in use. + +Leave blank to use SAS URL or Emulator, otherwise it needs to be set. + +If this is blank and if env_auth is set it will be read from the +environment variable AZURE_STORAGE_ACCOUNT_NAME if possible. Properties: @@ -32803,31 +33475,22 @@ Properties: - Type: string - Required: false ---azureblob-service-principal-file +--azureblob-env-auth -Path to file containing credentials for use with a service principal. +Read credentials from runtime (environment variables, CLI or MSI). -Leave blank normally. Needed only if you want to use a service principal -instead of interactive login. - - $ az ad sp create-for-rbac --name "" \ - --role "Storage Blob Data Owner" \ - --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \ - > azure-principal.json - -See "Create an Azure service principal" and "Assign an Azure role for -access to blob data" pages for more details. +See the authentication docs for full info. Properties: -- Config: service_principal_file -- Env Var: RCLONE_AZUREBLOB_SERVICE_PRINCIPAL_FILE -- Type: string -- Required: false +- Config: env_auth +- Env Var: RCLONE_AZUREBLOB_ENV_AUTH +- Type: bool +- Default: false --azureblob-key -Storage Account Key. +Storage Account Shared Key. Leave blank to use SAS URL or Emulator. @@ -32851,6 +33514,153 @@ Properties: - Type: string - Required: false +--azureblob-tenant + +ID of the service principal's tenant. Also called its directory ID. + +Set this if using - Service principal with client secret - Service +principal with certificate - User with username and password + +Properties: + +- Config: tenant +- Env Var: RCLONE_AZUREBLOB_TENANT +- Type: string +- Required: false + +--azureblob-client-id + +The ID of the client in use. + +Set this if using - Service principal with client secret - Service +principal with certificate - User with username and password + +Properties: + +- Config: client_id +- Env Var: RCLONE_AZUREBLOB_CLIENT_ID +- Type: string +- Required: false + +--azureblob-client-secret + +One of the service principal's client secrets + +Set this if using - Service principal with client secret + +Properties: + +- Config: client_secret +- Env Var: RCLONE_AZUREBLOB_CLIENT_SECRET +- Type: string +- Required: false + +--azureblob-client-certificate-path + +Path to a PEM or PKCS12 certificate file including the private key. + +Set this if using - Service principal with certificate + +Properties: + +- Config: client_certificate_path +- Env Var: RCLONE_AZUREBLOB_CLIENT_CERTIFICATE_PATH +- Type: string +- Required: false + +--azureblob-client-certificate-password + +Password for the certificate file (optional). + +Optionally set this if using - Service principal with certificate + +And the certificate has a password. + +NB Input to this must be obscured - see rclone obscure. + +Properties: + +- Config: client_certificate_password +- Env Var: RCLONE_AZUREBLOB_CLIENT_CERTIFICATE_PASSWORD +- Type: string +- Required: false + +Advanced options + +Here are the Advanced options specific to azureblob (Microsoft Azure +Blob Storage). + +--azureblob-client-send-certificate-chain + +Send the certificate chain when using certificate auth. + +Specifies whether an authentication request will include an x5c header +to support subject name / issuer based authentication. When set to true, +authentication requests include the x5c header. + +Optionally set this if using - Service principal with certificate + +Properties: + +- Config: client_send_certificate_chain +- Env Var: RCLONE_AZUREBLOB_CLIENT_SEND_CERTIFICATE_CHAIN +- Type: bool +- Default: false + +--azureblob-username + +User name (usually an email address) + +Set this if using - User with username and password + +Properties: + +- Config: username +- Env Var: RCLONE_AZUREBLOB_USERNAME +- Type: string +- Required: false + +--azureblob-password + +The user's password + +Set this if using - User with username and password + +NB Input to this must be obscured - see rclone obscure. + +Properties: + +- Config: password +- Env Var: RCLONE_AZUREBLOB_PASSWORD +- Type: string +- Required: false + +--azureblob-service-principal-file + +Path to file containing credentials for use with a service principal. + +Leave blank normally. Needed only if you want to use a service principal +instead of interactive login. + + $ az ad sp create-for-rbac --name "" \ + --role "Storage Blob Data Owner" \ + --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \ + > azure-principal.json + +See "Create an Azure service principal" and "Assign an Azure role for +access to blob data" pages for more details. + +It may be more convenient to put the credentials directly into the +rclone config file under the client_id, tenant and client_secret keys +instead of setting service_principal_file. + +Properties: + +- Config: service_principal_file +- Env Var: RCLONE_AZUREBLOB_SERVICE_PRINCIPAL_FILE +- Type: string +- Required: false + --azureblob-use-msi Use a managed service identity to authenticate (only works in Azure). @@ -32873,24 +33683,6 @@ Properties: - Type: bool - Default: false ---azureblob-use-emulator - -Uses local storage emulator if provided as 'true'. - -Leave blank if using real azure storage endpoint. - -Properties: - -- Config: use_emulator -- Env Var: RCLONE_AZUREBLOB_USE_EMULATOR -- Type: bool -- Default: false - -Advanced options - -Here are the Advanced options specific to azureblob (Microsoft Azure -Blob Storage). - --azureblob-msi-object-id Object ID of the user-assigned MSI to use, if any. @@ -32930,6 +33722,19 @@ Properties: - Type: string - Required: false +--azureblob-use-emulator + +Uses local storage emulator if provided as 'true'. + +Leave blank if using real azure storage endpoint. + +Properties: + +- Config: use_emulator +- Env Var: RCLONE_AZUREBLOB_USE_EMULATOR +- Type: bool +- Default: false + --azureblob-endpoint Endpoint for the service. @@ -33132,6 +33937,20 @@ Properties: - "container" - Allow full public read access for container and blob data. +--azureblob-no-check-container + +If set, don't attempt to check the container exists or create it. + +This can be useful when trying to minimise the number of transactions +rclone does if you know the container exists already. + +Properties: + +- Config: no_check_container +- Env Var: RCLONE_AZUREBLOB_NO_CHECK_CONTAINER +- Type: bool +- Default: false + --azureblob-no-head-object If set, do not do HEAD before GET when getting objects. @@ -33143,6 +33962,18 @@ Properties: - Type: bool - Default: false +Custom upload headers + +You can set custom upload headers with the --header-upload flag. + +- Cache-Control +- Content-Disposition +- Content-Encoding +- Content-Language +- Content-Type + +Eg --header-upload "Content-Type: text/potato" + Limitations MD5 sums are only uploaded with chunked files if the source has an MD5 @@ -33157,16 +33988,19 @@ See List of backends that do not support rclone about and rclone about Azure Storage Emulator Support -You can run rclone with storage emulator (usually azurite). +You can run rclone with the storage emulator (usually azurite). -To do this, just set up a new remote with rclone config following -instructions described in introduction and set use_emulator config as -true. You do not need to provide default account name neither an account -key. +To do this, just set up a new remote with rclone config following the +instructions in the introduction and set use_emulator in the advanced +settings as true. You do not need to provide a default account name nor +an account key. But you can override them in the account and key +options. (Prior to v1.61 they were hard coded to azurite's +devstoreaccount1.) Also, if you want to access a storage emulator instance running on a -different machine, you can override Endpoint parameter in advanced -settings, setting it to http(s)://:/devstoreaccount1 (e.g. +different machine, you can override the endpoint parameter in the +advanced settings, setting it to +http(s)://:/devstoreaccount1 (e.g. http://10.254.2.5:10000/devstoreaccount1). Microsoft OneDrive @@ -33217,9 +34051,10 @@ This will guide you through an interactive setup process: n) No y/n> n Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -33926,6 +34761,30 @@ didn't allow public links to be made for the organisation/sharepoint library. To fix the permissions as an admin, take a look at the docs: 1, 2. +Can not access Shared with me files + +Shared with me files is not supported by rclone currently, but there is +a workaround: + +1. Visit https://onedrive.live.com + +2. Right click a item in Shared, then click Add shortcut to My files in + the context + + Screenshot (Shared with me) + + [make_shortcut] + +3. The shortcut will appear in My files, you can access it with rclone, + it behaves like a normal folder/file. + + Screenshot (My Files) + + [in_my_files] + +Screenshot (rclone mount) + +[rclone_mount] OpenDrive Paths are specified as remote:path @@ -35782,9 +36641,10 @@ This will guide you through an interactive setup process: Pcloud App Client Secret - leave blank normally. client_secret> Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -36062,9 +36922,10 @@ This will guide you through an interactive setup process: ** See help for premiumizeme backend at: https://rclone.org/premiumizeme/ ** Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -36204,9 +37065,10 @@ This will guide you through an interactive setup process: ** See help for putio backend at: https://rclone.org/putio/ ** Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -36238,12 +37100,15 @@ This will guide you through an interactive setup process: q) Quit config e/n/d/r/c/s/q> q +See the remote setup docs for how to set it up on a machine with no +Internet browser available. + Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. This only -runs from the moment it opens your browser to the moment you get back -the verification code. This is on http://127.0.0.1:53682/ and this it -may require you to unblock it temporarily if you are running a host -firewall, or use manual mode. +token as returned from put.io if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to +the moment you get back the verification code. This is on +http://127.0.0.1:53682/ and this it may require you to unblock it +temporarily if you are running a host firewall, or use manual mode. You can then use it like this, @@ -37192,6 +38057,9 @@ methods: Those algorithms are insecure and may allow plaintext data to be recovered by an attacker. +This must be false if you use either ciphers or key_exchange advanced +options. + Properties: - Config: use_insecure_cipher @@ -37519,6 +38387,66 @@ Properties: - Type: SpaceSepList - Default: +--sftp-ciphers + +Space separated list of ciphers to be used for session encryption, +ordered by preference. + +At least one must match with server configuration. This can be checked +for example using ssh -Q cipher. + +This must not be set if use_insecure_cipher is true. + +Example: + + aes128-ctr aes192-ctr aes256-ctr aes128-gcm@openssh.com aes256-gcm@openssh.com + +Properties: + +- Config: ciphers +- Env Var: RCLONE_SFTP_CIPHERS +- Type: SpaceSepList +- Default: + +--sftp-key-exchange + +Space separated list of key exchange algorithms, ordered by preference. + +At least one must match with server configuration. This can be checked +for example using ssh -Q kex. + +This must not be set if use_insecure_cipher is true. + +Example: + + sntrup761x25519-sha512@openssh.com curve25519-sha256 curve25519-sha256@libssh.org ecdh-sha2-nistp256 + +Properties: + +- Config: key_exchange +- Env Var: RCLONE_SFTP_KEY_EXCHANGE +- Type: SpaceSepList +- Default: + +--sftp-macs + +Space separated list of MACs (message authentication code) algorithms, +ordered by preference. + +At least one must match with server configuration. This can be checked +for example using ssh -Q mac. + +Example: + + umac-64-etm@openssh.com umac-128-etm@openssh.com hmac-sha2-256-etm@openssh.com + +Properties: + +- Config: macs +- Env Var: RCLONE_SFTP_MACS +- Type: SpaceSepList +- Default: + Limitations On some SFTP servers (e.g. Synology) the paths are different for SSH and @@ -39385,9 +40313,10 @@ This will guide you through an interactive setup process: Yandex Client Secret - leave blank normally. client_secret> Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -39617,9 +40546,10 @@ This will guide you through an interactive setup process: n) No (default) y/n> n Remote config - Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine + Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access + If not sure try Y. If Y failed, try N. y) Yes (default) n) No y/n> @@ -40428,6 +41358,187 @@ Options: Changelog +v1.61.0 - 2022-12-20 + +See commits + +- New backends + - New S3 providers + - Liara LOS (MohammadReza) +- New Features + - build: Add vulnerability testing using govulncheck (albertony) + - cmd: Enable SIGINFO (Ctrl-T) handler on FreeBSD, NetBSD, OpenBSD + and Dragonfly BSD (x3-apptech) + - config: Add config/setpath for setting config path via + rc/librclone (Nick Craig-Wood) + - dedupe + - Count Checks in the stats while scanning for duplicates + (Nick Craig-Wood) + - Make dedupe obey the filters (Nick Craig-Wood) + - dlna: Properly attribute code used from + https://github.com/anacrolix/dms (Nick Craig-Wood) + - docs + - Add minimum versions and status badges to backend and + command docs (Nick Craig-Wood, albertony) + - Remote names may not start or end with space (albertony) + - filter: Add metadata filters --metadata-include/exclude/filter + and friends (Nick Craig-Wood) + - fs + - Make all duration flags take y, M, w, d etc suffixes (Nick + Craig-Wood) + - Add global flag --color to control terminal colors (Kevin + Verstaen) + - fspath: Allow unicode numbers and letters in remote names + (albertony) + - lib/file: Improve error message for creating dir on non-existent + network host on windows (albertony) + - lib/http: Finish port of rclone servers to lib/http (Tom + Mombourquette, Nick Craig-Wood) + - lib/oauthutil: Improved usability of config flows needing web + browser (Ole Frost) + - ncdu + - Add support for modification time (albertony) + - Fallback to sort by name also for sort by average size + (albertony) + - Rework to use tcell directly instead of the termbox wrapper + (eNV25) + - rc: Add commands to set GC Percent & Memory Limit (go 1.19+) + (Anagh Kumar Baranwal) + - rcat: Preserve metadata when Copy falls back to Rcat (Nick + Craig-Wood) + - rcd: Refactor rclone rc server to use lib/http (Nick Craig-Wood) + - rcserver: Avoid generating default credentials with htpasswd + (Kamui) + - restic: Refactor to use lib/http (Nolan Woods) + - serve http: Support unix sockets and multiple listeners (Tom + Mombourquette) + - serve webdav: Refactor to use lib/http (Nick Craig-Wood) + - test: Replace defer cleanup with t.Cleanup (Eng Zer Jun) + - test memory: Read metadata if -M flag is specified (Nick + Craig-Wood) + - wasm: Comply with wasm_exec.js licence terms (Matthew Vernon) +- Bug Fixes + - build: Update golang.org/x/net/http2 to fix GO-2022-1144 (Nick + Craig-Wood) + - restic: Fix typo in docs 'remove' should be 'remote' + (asdffdsazqqq) + - serve dlna: Fix panic: Logger uninitialized. (Nick Craig-Wood) +- Mount + - Update cgofuse for FUSE-T support for mounting volumes on Mac + (Nick Craig-Wood) +- VFS + - Windows: fix slow opening of exe files by not truncating files + when not necessary (Nick Craig-Wood) + - Fix IO Error opening a file with O_CREATE|O_RDONLY in + --vfs-cache-mode not full (Nick Craig-Wood) +- Crypt + - Fix compress wrapping crypt giving upload errors (Nick + Craig-Wood) +- Azure Blob + - Port to new SDK (Nick Craig-Wood) + - Revamp authentication to include all methods and docs (Nick + Craig-Wood) + - Port old authentication methods to new SDK (Nick Craig-Wood, + Brad Ackerman) + - Thanks to Stonebranch for sponsoring this work. + - Add --azureblob-no-check-container to assume container exists + (Nick Craig-Wood) + - Add --use-server-modtime support (Abdullah Saglam) + - Add support for custom upload headers (rkettelerij) + - Allow emulator account/key override (Roel Arents) + - Support simple "environment credentials" (Nathaniel Wesley + Filardo) + - Ignore AuthorizationFailure when trying to create a create a + container (Nick Craig-Wood) +- Box + - Added note on Box API rate limits (Ole Frost) +- Drive + - Handle shared drives with leading/trailing space in name + (related to) (albertony) +- FTP + - Update help text of implicit/explicit TLS options to refer to + FTPS instead of FTP (ycdtosa) + - Improve performance to speed up --files-from and NewObject + (Anthony Pessy) +- HTTP + - Parse GET responses when no_head is set (Arnie97) + - Do not update object size based on Range requests (Arnie97) + - Support Content-Range response header (Arnie97) +- Onedrive + - Document workaround for shared with me files (vanplus) +- S3 + - Add Liara LOS to provider list (MohammadReza) + - Add DigitalOcean Spaces regions sfo3, fra1, syd1 (Jack) + - Avoid privileged GetBucketLocation to resolve s3 region (Anthony + Pessy) + - Stop setting object and bucket ACL to private if it is an empty + string (Philip Harvey) + - If bucket or object ACL is empty string then don't add + X-Amz-Acl: header (Nick Craig-Wood) + - Reduce memory consumption for s3 objects (Erik Agterdenbos) + - Fix listing loop when using v2 listing on v1 server (Nick + Craig-Wood) + - Fix nil pointer exception when using Versions (Nick Craig-Wood) + - Fix excess memory usage when using versions (Nick Craig-Wood) + - Ignore versionIDs from uploads unless using --s3-versions or + --s3-versions-at (Nick Craig-Wood) +- SFTP + - Add configuration options to set ssh Ciphers / MACs / + KeyExchange (dgouju) + - Auto-detect shell type for fish (albertony) + - Fix NewObject with leading / (Nick Craig-Wood) +- Smb + - Fix issue where spurious dot directory is created (albertony) +- Storj + - Implement server side Copy (Kaloyan Raev) + +v1.60.1 - 2022-11-17 + +See commits + +- Bug Fixes + - lib/cache: Fix alias backend shutting down too soon (Nick + Craig-Wood) + - wasm: Fix walltime link error by adding up-to-date wasm_exec.js + (João Henrique Franco) + - docs + - Update faq.md with bisync (Samuel Johnson) + - Corrected download links in windows install docs + (coultonluke) + - Add direct download link for windows arm64 (albertony) + - Remove link to rclone slack as it is no longer supported + (Nick Craig-Wood) + - Faq: how to use a proxy server that requires a username and + password (asdffdsazqqq) + - Oracle-object-storage: doc fix (Manoj Ghosh) + - Fix typo remove in rclone_serve_restic command (Joda Stößer) + - Fix character that was incorrectly interpreted as markdown + (Clément Notin) +- VFS + - Fix deadlock caused by cache cleaner and upload finishing (Nick + Craig-Wood) +- Local + - Clean absolute paths (albertony) + - Fix -L/--copy-links with filters missing directories (Nick + Craig-Wood) +- Mailru + - Note that an app password is now needed (Nick Craig-Wood) + - Allow timestamps to be before the epoch 1970-01-01 (Nick + Craig-Wood) +- S3 + - Add provider quirk --s3-might-gzip to fix corrupted on transfer: + sizes differ (Nick Craig-Wood) + - Allow Storj to server side copy since it seems to work now (Nick + Craig-Wood) + - Fix for unchecked err value in s3 listv2 (Aaron Gokaslan) + - Add additional Wasabi locations (techknowlogick) +- Smb + - Fix Failed to sync: context canceled at the end of syncs (Nick + Craig-Wood) +- WebDAV + - Fix Move/Copy/DirMove when using -server-side-across-configs + (Nick Craig-Wood) + v1.60.0 - 2022-10-21 See commits @@ -45966,9 +47077,7 @@ metadata, which breaks the desired 1:1 mapping of files to objects. Can rclone do bi-directional sync? -No, not at present. rclone only does uni-directional sync from A -> B. -It may do in the future though since it has all the primitives - it just -requires writing the algorithm to do it. +Yes, since rclone v1.58.0, bidirectional cloud sync is available. Can I use rclone with an HTTP proxy? @@ -45993,6 +47102,13 @@ set all possibilities. So, on Linux, you may end up with code similar to export HTTP_PROXY=$http_proxy export HTTPS_PROXY=$http_proxy +Note: If the proxy server requires a username and password, then use + + export http_proxy=http://username:password@proxyserver:12345 + export https_proxy=$http_proxy + export HTTP_PROXY=$http_proxy + export HTTPS_PROXY=$http_proxy + The NO_PROXY allows you to disable the proxy for specific hosts. Hosts must be comma separated, and can contain domains or parts. For instance "foo.com" also matches "bar.foo.com". @@ -46363,6 +47479,7 @@ email addresses removed from here need to be addeed to bin/.ignore-emails to mak - Jay dev@jaygoel.com - andrea rota a@xelera.eu - nicolov nicolov@users.noreply.github.com +- Matt Joiner anacrolix@gmail.com - Dario Guzik dario@guzik.com.ar - qip qip@users.noreply.github.com - yair@unicorn yair@unicorn @@ -46787,6 +47904,28 @@ email addresses removed from here need to be addeed to bin/.ignore-emails to mak - Manoj Ghosh manoj.ghosh@oracle.com - Tom Mombourquette tom@devnode.com - Robert Newson rnewson@apache.org +- Samuel Johnson esamueljohnson@gmail.com +- coultonluke luke@luke.org.uk +- Anthony Pessy anthony@cogniteev.com +- Philip Harvey pharvey@battelleecology.org +- dgouju dgouju@users.noreply.github.com +- Clément Notin clement.notin@gmail.com +- x3-apptech 66947598+x3-apptech@users.noreply.github.com +- Arnie97 arnie97@gmail.com +- Roel Arents 2691308+roelarents@users.noreply.github.com +- Aaron Gokaslan aaronGokaslan@gmail.com +- techknowlogick matti@mdranta.net +- rkettelerij richard@mindloops.nl +- Kamui fin-kamui@pm.me +- asdffdsazqqq 90116442+asdffdsazqqq@users.noreply.github.com +- Nathaniel Wesley Filardo nfilardo@microsoft.com +- ycdtosa ycdtosa@users.noreply.github.com +- Erik Agterdenbos agterdenbos@users.noreply.github.com +- Kevin Verstaen 48050031+kverstae@users.noreply.github.com +- MohammadReza mrvashian@gmail.com +- vanplus 60313789+vanplus@users.noreply.github.com +- Jack 16779171+jkpe@users.noreply.github.com +- Abdullah Saglam abdullah.saglam@stonebranch.com Contact the rclone project diff --git a/docs/content/changelog.md b/docs/content/changelog.md index 2c844f25621f2..2ba5494eac099 100644 --- a/docs/content/changelog.md +++ b/docs/content/changelog.md @@ -5,6 +5,101 @@ description: "Rclone Changelog" # Changelog +## v1.61.0 - 2022-12-20 + +[See commits](https://github.com/rclone/rclone/compare/v1.60.0...v1.61.0) + +* New backends + * New S3 providers + * [Liara LOS](/s3/#liara-cloud) (MohammadReza) +* New Features + * build: Add vulnerability testing using govulncheck (albertony) + * cmd: Enable `SIGINFO` (Ctrl-T) handler on FreeBSD, NetBSD, OpenBSD and Dragonfly BSD (x3-apptech) + * config: Add [config/setpath](/rc/#config-setpath) for setting config path via rc/librclone (Nick Craig-Wood) + * dedupe + * Count Checks in the stats while scanning for duplicates (Nick Craig-Wood) + * Make dedupe obey the filters (Nick Craig-Wood) + * dlna: Properly attribute code used from https://github.com/anacrolix/dms (Nick Craig-Wood) + * docs + * Add minimum versions and status badges to backend and command docs (Nick Craig-Wood, albertony) + * Remote names may not start or end with space (albertony) + * filter: Add metadata filters [--metadata-include/exclude/filter](/filtering/#metadata) and friends (Nick Craig-Wood) + * fs + * Make all duration flags take `y`, `M`, `w`, `d` etc suffixes (Nick Craig-Wood) + * Add global flag `--color` to control terminal colors (Kevin Verstaen) + * fspath: Allow unicode numbers and letters in remote names (albertony) + * lib/file: Improve error message for creating dir on non-existent network host on windows (albertony) + * lib/http: Finish port of rclone servers to `lib/http` (Tom Mombourquette, Nick Craig-Wood) + * lib/oauthutil: Improved usability of config flows needing web browser (Ole Frost) + * ncdu + * Add support for modification time (albertony) + * Fallback to sort by name also for sort by average size (albertony) + * Rework to use tcell directly instead of the termbox wrapper (eNV25) + * rc: Add commands to set [GC Percent](/rc/#debug-set-gc-percent) & [Memory Limit](/rc/#debug-set-soft-memory-limit) (go 1.19+) (Anagh Kumar Baranwal) + * rcat: Preserve metadata when Copy falls back to Rcat (Nick Craig-Wood) + * rcd: Refactor rclone rc server to use `lib/http` (Nick Craig-Wood) + * rcserver: Avoid generating default credentials with htpasswd (Kamui) + * restic: Refactor to use `lib/http` (Nolan Woods) + * serve http: Support unix sockets and multiple listeners (Tom Mombourquette) + * serve webdav: Refactor to use `lib/http` (Nick Craig-Wood) + * test: Replace defer cleanup with `t.Cleanup` (Eng Zer Jun) + * test memory: Read metadata if `-M` flag is specified (Nick Craig-Wood) + * wasm: Comply with `wasm_exec.js` licence terms (Matthew Vernon) +* Bug Fixes + * build: Update `golang.org/x/net/http2` to fix GO-2022-1144 (Nick Craig-Wood) + * restic: Fix typo in docs 'remove' should be 'remote' (asdffdsazqqq) + * serve dlna: Fix panic: Logger uninitialized. (Nick Craig-Wood) +* Mount + * Update cgofuse for FUSE-T support for mounting volumes on Mac (Nick Craig-Wood) +* VFS + * Windows: fix slow opening of exe files by not truncating files when not necessary (Nick Craig-Wood) + * Fix IO Error opening a file with `O_CREATE|O_RDONLY` in `--vfs-cache-mode` not full (Nick Craig-Wood) +* Crypt + * Fix compress wrapping crypt giving upload errors (Nick Craig-Wood) +* Azure Blob + * Port to new SDK (Nick Craig-Wood) + * Revamp authentication to include all methods and docs (Nick Craig-Wood) + * Port old authentication methods to new SDK (Nick Craig-Wood, Brad Ackerman) + * Thanks to [Stonebranch](https://www.stonebranch.com/) for sponsoring this work. + * Add `--azureblob-no-check-container` to assume container exists (Nick Craig-Wood) + * Add `--use-server-modtime` support (Abdullah Saglam) + * Add support for custom upload headers (rkettelerij) + * Allow emulator account/key override (Roel Arents) + * Support simple "environment credentials" (Nathaniel Wesley Filardo) + * Ignore `AuthorizationFailure` when trying to create a create a container (Nick Craig-Wood) +* Box + * Added note on Box API rate limits (Ole Frost) +* Drive + * Handle shared drives with leading/trailing space in name (related to) (albertony) +* FTP + * Update help text of implicit/explicit TLS options to refer to FTPS instead of FTP (ycdtosa) + * Improve performance to speed up `--files-from` and `NewObject` (Anthony Pessy) +* HTTP + * Parse GET responses when `no_head` is set (Arnie97) + * Do not update object size based on `Range` requests (Arnie97) + * Support `Content-Range` response header (Arnie97) +* Onedrive + * Document workaround for shared with me files (vanplus) +* S3 + * Add Liara LOS to provider list (MohammadReza) + * Add DigitalOcean Spaces regions `sfo3`, `fra1`, `syd1` (Jack) + * Avoid privileged `GetBucketLocation` to resolve s3 region (Anthony Pessy) + * Stop setting object and bucket ACL to `private` if it is an empty string (Philip Harvey) + * If bucket or object ACL is empty string then don't add `X-Amz-Acl:` header (Nick Craig-Wood) + * Reduce memory consumption for s3 objects (Erik Agterdenbos) + * Fix listing loop when using v2 listing on v1 server (Nick Craig-Wood) + * Fix nil pointer exception when using Versions (Nick Craig-Wood) + * Fix excess memory usage when using versions (Nick Craig-Wood) + * Ignore versionIDs from uploads unless using `--s3-versions` or `--s3-versions-at` (Nick Craig-Wood) +* SFTP + * Add configuration options to set ssh Ciphers / MACs / KeyExchange (dgouju) + * Auto-detect shell type for fish (albertony) + * Fix NewObject with leading / (Nick Craig-Wood) +* Smb + * Fix issue where spurious dot directory is created (albertony) +* Storj + * Implement server side Copy (Kaloyan Raev) + ## v1.60.1 - 2022-11-17 [See commits](https://github.com/rclone/rclone/compare/v1.60.0...v1.60.1) diff --git a/docs/content/commands/rclone_about.md b/docs/content/commands/rclone_about.md index 07a01d4105e96..4464aa80a634c 100644 --- a/docs/content/commands/rclone_about.md +++ b/docs/content/commands/rclone_about.md @@ -3,6 +3,7 @@ title: "rclone about" description: "Get quota information from the remote." slug: rclone_about url: /commands/rclone_about/ +versionIntroduced: v1.41 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/about/ and as part of making a release run "make commanddocs" --- # rclone about diff --git a/docs/content/commands/rclone_authorize.md b/docs/content/commands/rclone_authorize.md index 4e8fd0ba0fe82..3620829eb722c 100644 --- a/docs/content/commands/rclone_authorize.md +++ b/docs/content/commands/rclone_authorize.md @@ -3,6 +3,7 @@ title: "rclone authorize" description: "Remote authorization." slug: rclone_authorize url: /commands/rclone_authorize/ +versionIntroduced: v1.27 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/authorize/ and as part of making a release run "make commanddocs" --- # rclone authorize diff --git a/docs/content/commands/rclone_backend.md b/docs/content/commands/rclone_backend.md index 9887db60d7315..8e29f42ede74d 100644 --- a/docs/content/commands/rclone_backend.md +++ b/docs/content/commands/rclone_backend.md @@ -3,6 +3,7 @@ title: "rclone backend" description: "Run a backend-specific command." slug: rclone_backend url: /commands/rclone_backend/ +versionIntroduced: v1.52 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/backend/ and as part of making a release run "make commanddocs" --- # rclone backend diff --git a/docs/content/commands/rclone_bisync.md b/docs/content/commands/rclone_bisync.md index 4a7127329b32c..dacda99370a81 100644 --- a/docs/content/commands/rclone_bisync.md +++ b/docs/content/commands/rclone_bisync.md @@ -3,6 +3,7 @@ title: "rclone bisync" description: "Perform bidirectional synchronization between two paths." slug: rclone_bisync url: /commands/rclone_bisync/ +versionIntroduced: v1.58 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/bisync/ and as part of making a release run "make commanddocs" --- # rclone bisync diff --git a/docs/content/commands/rclone_cat.md b/docs/content/commands/rclone_cat.md index 0b3b40f71c815..6ad41423d65c3 100644 --- a/docs/content/commands/rclone_cat.md +++ b/docs/content/commands/rclone_cat.md @@ -3,6 +3,7 @@ title: "rclone cat" description: "Concatenates any files and sends them to stdout." slug: rclone_cat url: /commands/rclone_cat/ +versionIntroduced: v1.33 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/cat/ and as part of making a release run "make commanddocs" --- # rclone cat diff --git a/docs/content/commands/rclone_checksum.md b/docs/content/commands/rclone_checksum.md index e285358bdeed0..661dab6b2d929 100644 --- a/docs/content/commands/rclone_checksum.md +++ b/docs/content/commands/rclone_checksum.md @@ -3,6 +3,7 @@ title: "rclone checksum" description: "Checks the files in the source against a SUM file." slug: rclone_checksum url: /commands/rclone_checksum/ +versionIntroduced: v1.56 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/checksum/ and as part of making a release run "make commanddocs" --- # rclone checksum diff --git a/docs/content/commands/rclone_cleanup.md b/docs/content/commands/rclone_cleanup.md index 680b4f981f760..0b7b0b189bb76 100644 --- a/docs/content/commands/rclone_cleanup.md +++ b/docs/content/commands/rclone_cleanup.md @@ -3,6 +3,7 @@ title: "rclone cleanup" description: "Clean up the remote if possible." slug: rclone_cleanup url: /commands/rclone_cleanup/ +versionIntroduced: v1.31 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/cleanup/ and as part of making a release run "make commanddocs" --- # rclone cleanup diff --git a/docs/content/commands/rclone_config.md b/docs/content/commands/rclone_config.md index f6ca5e41985b6..186d90d2651dd 100644 --- a/docs/content/commands/rclone_config.md +++ b/docs/content/commands/rclone_config.md @@ -3,6 +3,7 @@ title: "rclone config" description: "Enter an interactive configuration session." slug: rclone_config url: /commands/rclone_config/ +versionIntroduced: v1.39 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/config/ and as part of making a release run "make commanddocs" --- # rclone config diff --git a/docs/content/commands/rclone_config_create.md b/docs/content/commands/rclone_config_create.md index 8d12ea86b512a..a923b246dad5d 100644 --- a/docs/content/commands/rclone_config_create.md +++ b/docs/content/commands/rclone_config_create.md @@ -3,6 +3,7 @@ title: "rclone config create" description: "Create a new remote with name, type and options." slug: rclone_config_create url: /commands/rclone_config_create/ +versionIntroduced: v1.39 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/config/create/ and as part of making a release run "make commanddocs" --- # rclone config create @@ -57,7 +58,7 @@ This will look something like (some irrelevant detail removed): "State": "*oauth-islocal,teamdrive,,", "Option": { "Name": "config_is_local", - "Help": "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n", + "Help": "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n", "Default": true, "Examples": [ { diff --git a/docs/content/commands/rclone_config_delete.md b/docs/content/commands/rclone_config_delete.md index 6b82e80069d24..c58bed4e42c2e 100644 --- a/docs/content/commands/rclone_config_delete.md +++ b/docs/content/commands/rclone_config_delete.md @@ -3,6 +3,7 @@ title: "rclone config delete" description: "Delete an existing remote." slug: rclone_config_delete url: /commands/rclone_config_delete/ +versionIntroduced: v1.39 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/config/delete/ and as part of making a release run "make commanddocs" --- # rclone config delete diff --git a/docs/content/commands/rclone_config_dump.md b/docs/content/commands/rclone_config_dump.md index 5f471c40bc1dd..e0119936dbc93 100644 --- a/docs/content/commands/rclone_config_dump.md +++ b/docs/content/commands/rclone_config_dump.md @@ -3,6 +3,7 @@ title: "rclone config dump" description: "Dump the config file as JSON." slug: rclone_config_dump url: /commands/rclone_config_dump/ +versionIntroduced: v1.39 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/config/dump/ and as part of making a release run "make commanddocs" --- # rclone config dump diff --git a/docs/content/commands/rclone_config_file.md b/docs/content/commands/rclone_config_file.md index 0b289a4e5f5a1..8585b77f933a1 100644 --- a/docs/content/commands/rclone_config_file.md +++ b/docs/content/commands/rclone_config_file.md @@ -3,6 +3,7 @@ title: "rclone config file" description: "Show path of configuration file in use." slug: rclone_config_file url: /commands/rclone_config_file/ +versionIntroduced: v1.38 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/config/file/ and as part of making a release run "make commanddocs" --- # rclone config file diff --git a/docs/content/commands/rclone_config_password.md b/docs/content/commands/rclone_config_password.md index ee7fd3d3d3014..f69301811e0ae 100644 --- a/docs/content/commands/rclone_config_password.md +++ b/docs/content/commands/rclone_config_password.md @@ -3,6 +3,7 @@ title: "rclone config password" description: "Update password in an existing remote." slug: rclone_config_password url: /commands/rclone_config_password/ +versionIntroduced: v1.39 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/config/password/ and as part of making a release run "make commanddocs" --- # rclone config password diff --git a/docs/content/commands/rclone_config_paths.md b/docs/content/commands/rclone_config_paths.md index 7171828dd1008..7762ce0903b96 100644 --- a/docs/content/commands/rclone_config_paths.md +++ b/docs/content/commands/rclone_config_paths.md @@ -3,6 +3,7 @@ title: "rclone config paths" description: "Show paths used for configuration, cache, temp etc." slug: rclone_config_paths url: /commands/rclone_config_paths/ +versionIntroduced: v1.57 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/config/paths/ and as part of making a release run "make commanddocs" --- # rclone config paths diff --git a/docs/content/commands/rclone_config_providers.md b/docs/content/commands/rclone_config_providers.md index ed89ea5fe3f03..43441af08832e 100644 --- a/docs/content/commands/rclone_config_providers.md +++ b/docs/content/commands/rclone_config_providers.md @@ -3,6 +3,7 @@ title: "rclone config providers" description: "List in JSON format all the providers and options." slug: rclone_config_providers url: /commands/rclone_config_providers/ +versionIntroduced: v1.39 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/config/providers/ and as part of making a release run "make commanddocs" --- # rclone config providers diff --git a/docs/content/commands/rclone_config_show.md b/docs/content/commands/rclone_config_show.md index 97b818f361f07..beab56349cbfa 100644 --- a/docs/content/commands/rclone_config_show.md +++ b/docs/content/commands/rclone_config_show.md @@ -3,6 +3,7 @@ title: "rclone config show" description: "Print (decrypted) config file, or the config for a single remote." slug: rclone_config_show url: /commands/rclone_config_show/ +versionIntroduced: v1.38 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/config/show/ and as part of making a release run "make commanddocs" --- # rclone config show diff --git a/docs/content/commands/rclone_config_touch.md b/docs/content/commands/rclone_config_touch.md index ebea6e257ee33..d96c3702782a3 100644 --- a/docs/content/commands/rclone_config_touch.md +++ b/docs/content/commands/rclone_config_touch.md @@ -3,6 +3,7 @@ title: "rclone config touch" description: "Ensure configuration file exists." slug: rclone_config_touch url: /commands/rclone_config_touch/ +versionIntroduced: v1.56 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/config/touch/ and as part of making a release run "make commanddocs" --- # rclone config touch diff --git a/docs/content/commands/rclone_config_update.md b/docs/content/commands/rclone_config_update.md index 468a683076cfb..9e3fa9a38a1f9 100644 --- a/docs/content/commands/rclone_config_update.md +++ b/docs/content/commands/rclone_config_update.md @@ -3,6 +3,7 @@ title: "rclone config update" description: "Update options in an existing remote." slug: rclone_config_update url: /commands/rclone_config_update/ +versionIntroduced: v1.39 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/config/update/ and as part of making a release run "make commanddocs" --- # rclone config update @@ -57,7 +58,7 @@ This will look something like (some irrelevant detail removed): "State": "*oauth-islocal,teamdrive,,", "Option": { "Name": "config_is_local", - "Help": "Use auto config?\n * Say Y if not sure\n * Say N if you are working on a remote or headless machine\n", + "Help": "Use web browser to automatically authenticate rclone with remote?\n * Say Y if the machine running rclone has a web browser you can use\n * Say N if running rclone on a (remote) machine without web browser access\nIf not sure try Y. If Y failed, try N.\n", "Default": true, "Examples": [ { diff --git a/docs/content/commands/rclone_copyto.md b/docs/content/commands/rclone_copyto.md index a5cfa974e4d93..bfd8b5e72fb15 100644 --- a/docs/content/commands/rclone_copyto.md +++ b/docs/content/commands/rclone_copyto.md @@ -3,6 +3,7 @@ title: "rclone copyto" description: "Copy files from source to dest, skipping identical files." slug: rclone_copyto url: /commands/rclone_copyto/ +versionIntroduced: v1.35 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/copyto/ and as part of making a release run "make commanddocs" --- # rclone copyto diff --git a/docs/content/commands/rclone_copyurl.md b/docs/content/commands/rclone_copyurl.md index 1188f0647b9ce..2d13cd9a8633c 100644 --- a/docs/content/commands/rclone_copyurl.md +++ b/docs/content/commands/rclone_copyurl.md @@ -3,6 +3,7 @@ title: "rclone copyurl" description: "Copy url content to dest." slug: rclone_copyurl url: /commands/rclone_copyurl/ +versionIntroduced: v1.43 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/copyurl/ and as part of making a release run "make commanddocs" --- # rclone copyurl diff --git a/docs/content/commands/rclone_cryptcheck.md b/docs/content/commands/rclone_cryptcheck.md index 3e81e53e9b8be..21528b029ff79 100644 --- a/docs/content/commands/rclone_cryptcheck.md +++ b/docs/content/commands/rclone_cryptcheck.md @@ -3,6 +3,7 @@ title: "rclone cryptcheck" description: "Cryptcheck checks the integrity of a crypted remote." slug: rclone_cryptcheck url: /commands/rclone_cryptcheck/ +versionIntroduced: v1.36 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/cryptcheck/ and as part of making a release run "make commanddocs" --- # rclone cryptcheck diff --git a/docs/content/commands/rclone_cryptdecode.md b/docs/content/commands/rclone_cryptdecode.md index c4836ec8d583a..11be5a7d1c2a0 100644 --- a/docs/content/commands/rclone_cryptdecode.md +++ b/docs/content/commands/rclone_cryptdecode.md @@ -3,6 +3,7 @@ title: "rclone cryptdecode" description: "Cryptdecode returns unencrypted file names." slug: rclone_cryptdecode url: /commands/rclone_cryptdecode/ +versionIntroduced: v1.38 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/cryptdecode/ and as part of making a release run "make commanddocs" --- # rclone cryptdecode diff --git a/docs/content/commands/rclone_dedupe.md b/docs/content/commands/rclone_dedupe.md index 6b77f17cbf317..a7a711b4576be 100644 --- a/docs/content/commands/rclone_dedupe.md +++ b/docs/content/commands/rclone_dedupe.md @@ -3,6 +3,7 @@ title: "rclone dedupe" description: "Interactively find duplicate filenames and delete/rename them." slug: rclone_dedupe url: /commands/rclone_dedupe/ +versionIntroduced: v1.27 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/dedupe/ and as part of making a release run "make commanddocs" --- # rclone dedupe diff --git a/docs/content/commands/rclone_delete.md b/docs/content/commands/rclone_delete.md index 09076a46ae028..08ec20f98faa0 100644 --- a/docs/content/commands/rclone_delete.md +++ b/docs/content/commands/rclone_delete.md @@ -3,6 +3,7 @@ title: "rclone delete" description: "Remove the files in path." slug: rclone_delete url: /commands/rclone_delete/ +versionIntroduced: v1.27 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/delete/ and as part of making a release run "make commanddocs" --- # rclone delete diff --git a/docs/content/commands/rclone_deletefile.md b/docs/content/commands/rclone_deletefile.md index 865498d1fc79c..0cdaaf860be06 100644 --- a/docs/content/commands/rclone_deletefile.md +++ b/docs/content/commands/rclone_deletefile.md @@ -3,6 +3,7 @@ title: "rclone deletefile" description: "Remove a single file from remote." slug: rclone_deletefile url: /commands/rclone_deletefile/ +versionIntroduced: v1.42 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/deletefile/ and as part of making a release run "make commanddocs" --- # rclone deletefile diff --git a/docs/content/commands/rclone_genautocomplete.md b/docs/content/commands/rclone_genautocomplete.md index 3838dda4a8bd7..efaa38b07db30 100644 --- a/docs/content/commands/rclone_genautocomplete.md +++ b/docs/content/commands/rclone_genautocomplete.md @@ -3,6 +3,7 @@ title: "rclone genautocomplete" description: "Output completion script for a given shell." slug: rclone_genautocomplete url: /commands/rclone_genautocomplete/ +versionIntroduced: v1.33 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/genautocomplete/ and as part of making a release run "make commanddocs" --- # rclone genautocomplete diff --git a/docs/content/commands/rclone_gendocs.md b/docs/content/commands/rclone_gendocs.md index b5d8bef45fa7e..5389340c6e08b 100644 --- a/docs/content/commands/rclone_gendocs.md +++ b/docs/content/commands/rclone_gendocs.md @@ -3,6 +3,7 @@ title: "rclone gendocs" description: "Output markdown docs for rclone to the directory supplied." slug: rclone_gendocs url: /commands/rclone_gendocs/ +versionIntroduced: v1.33 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/gendocs/ and as part of making a release run "make commanddocs" --- # rclone gendocs diff --git a/docs/content/commands/rclone_hashsum.md b/docs/content/commands/rclone_hashsum.md index f9b96e6a4af4e..e1a78e3be5abb 100644 --- a/docs/content/commands/rclone_hashsum.md +++ b/docs/content/commands/rclone_hashsum.md @@ -3,6 +3,7 @@ title: "rclone hashsum" description: "Produces a hashsum file for all the objects in the path." slug: rclone_hashsum url: /commands/rclone_hashsum/ +versionIntroduced: v1.41 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/hashsum/ and as part of making a release run "make commanddocs" --- # rclone hashsum diff --git a/docs/content/commands/rclone_link.md b/docs/content/commands/rclone_link.md index 00a2d0202bb04..45684987307b6 100644 --- a/docs/content/commands/rclone_link.md +++ b/docs/content/commands/rclone_link.md @@ -3,6 +3,7 @@ title: "rclone link" description: "Generate public link to file/folder." slug: rclone_link url: /commands/rclone_link/ +versionIntroduced: v1.41 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/link/ and as part of making a release run "make commanddocs" --- # rclone link diff --git a/docs/content/commands/rclone_listremotes.md b/docs/content/commands/rclone_listremotes.md index 54e4317c8e954..fdf20b88c8c99 100644 --- a/docs/content/commands/rclone_listremotes.md +++ b/docs/content/commands/rclone_listremotes.md @@ -3,6 +3,7 @@ title: "rclone listremotes" description: "List all the remotes in the config file." slug: rclone_listremotes url: /commands/rclone_listremotes/ +versionIntroduced: v1.34 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/listremotes/ and as part of making a release run "make commanddocs" --- # rclone listremotes diff --git a/docs/content/commands/rclone_lsf.md b/docs/content/commands/rclone_lsf.md index 0a34277fae3fb..8567cbc9e653e 100644 --- a/docs/content/commands/rclone_lsf.md +++ b/docs/content/commands/rclone_lsf.md @@ -3,6 +3,7 @@ title: "rclone lsf" description: "List directories and objects in remote:path formatted for parsing." slug: rclone_lsf url: /commands/rclone_lsf/ +versionIntroduced: v1.40 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/lsf/ and as part of making a release run "make commanddocs" --- # rclone lsf diff --git a/docs/content/commands/rclone_lsjson.md b/docs/content/commands/rclone_lsjson.md index e3cac5d21165a..fbb979363ce3f 100644 --- a/docs/content/commands/rclone_lsjson.md +++ b/docs/content/commands/rclone_lsjson.md @@ -3,6 +3,7 @@ title: "rclone lsjson" description: "List directories and objects in the path in JSON format." slug: rclone_lsjson url: /commands/rclone_lsjson/ +versionIntroduced: v1.37 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/lsjson/ and as part of making a release run "make commanddocs" --- # rclone lsjson @@ -120,6 +121,7 @@ rclone lsjson remote:path [flags] --hash Include hashes in the output (may take longer) --hash-type stringArray Show only this hash type (may be repeated) -h, --help help for lsjson + -M, --metadata Add metadata to the listing --no-mimetype Don't read the mime type (can speed things up) --no-modtime Don't read the modification time (can speed things up) --original Show the ID of the underlying Object diff --git a/docs/content/commands/rclone_lsl.md b/docs/content/commands/rclone_lsl.md index f493916a92d03..b7419e7ac8e40 100644 --- a/docs/content/commands/rclone_lsl.md +++ b/docs/content/commands/rclone_lsl.md @@ -3,6 +3,7 @@ title: "rclone lsl" description: "List the objects in path with modification time, size and path." slug: rclone_lsl url: /commands/rclone_lsl/ +versionIntroduced: v1.02 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/lsl/ and as part of making a release run "make commanddocs" --- # rclone lsl diff --git a/docs/content/commands/rclone_md5sum.md b/docs/content/commands/rclone_md5sum.md index 9cd53cad08d20..c805064295ce0 100644 --- a/docs/content/commands/rclone_md5sum.md +++ b/docs/content/commands/rclone_md5sum.md @@ -3,6 +3,7 @@ title: "rclone md5sum" description: "Produces an md5sum file for all the objects in the path." slug: rclone_md5sum url: /commands/rclone_md5sum/ +versionIntroduced: v1.02 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/md5sum/ and as part of making a release run "make commanddocs" --- # rclone md5sum diff --git a/docs/content/commands/rclone_mount.md b/docs/content/commands/rclone_mount.md index b322b23f89646..2c8fdb16912e1 100644 --- a/docs/content/commands/rclone_mount.md +++ b/docs/content/commands/rclone_mount.md @@ -3,6 +3,7 @@ title: "rclone mount" description: "Mount the remote as file system on a mountpoint." slug: rclone_mount url: /commands/rclone_mount/ +versionIntroduced: v1.33 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/mount/ and as part of making a release run "make commanddocs" --- # rclone mount @@ -722,14 +723,14 @@ rclone mount remote:path /path/to/mountpoint [flags] --allow-other Allow access to other users (not supported on Windows) --allow-root Allow access to root user (not supported on Windows) --async-read Use asynchronous reads (not supported on Windows) (default true) - --attr-timeout duration Time for which file/directory attributes are cached (default 1s) + --attr-timeout Duration Time for which file/directory attributes are cached (default 1s) --daemon Run mount in background and exit parent process (as background output is suppressed, use --log-file with --log-format=pid,... to monitor) (not supported on Windows) - --daemon-timeout duration Time limit for rclone to respond to kernel (not supported on Windows) - --daemon-wait duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) + --daemon-timeout Duration Time limit for rclone to respond to kernel (not supported on Windows) (default 0s) + --daemon-wait Duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) --debug-fuse Debug the FUSE internals - needs -v --default-permissions Makes kernel enforce access control based on the file mode (not supported on Windows) --devname string Set the device name - default is remote:path - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --fuse-flag stringArray Flags or arguments to be passed direct to libfuse/WinFsp (repeat if required) @@ -743,24 +744,24 @@ rclone mount remote:path /path/to/mountpoint [flags] --noappledouble Ignore Apple Double (._) and .DS_Store files (supported on OSX only) (default true) --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) --volname string Set the volume name (supported on Windows and OSX only) --write-back-cache Makes kernel buffer writes before sending them to rclone (without this, writethrough caching is used) (not supported on Windows) ``` diff --git a/docs/content/commands/rclone_move.md b/docs/content/commands/rclone_move.md index ce97800a2aeea..cd2e96fa35654 100644 --- a/docs/content/commands/rclone_move.md +++ b/docs/content/commands/rclone_move.md @@ -3,6 +3,7 @@ title: "rclone move" description: "Move files from source to dest." slug: rclone_move url: /commands/rclone_move/ +versionIntroduced: v1.19 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/move/ and as part of making a release run "make commanddocs" --- # rclone move diff --git a/docs/content/commands/rclone_moveto.md b/docs/content/commands/rclone_moveto.md index 6f8488f83a4fb..4a236fe93f23c 100644 --- a/docs/content/commands/rclone_moveto.md +++ b/docs/content/commands/rclone_moveto.md @@ -3,6 +3,7 @@ title: "rclone moveto" description: "Move file or directory from source to dest." slug: rclone_moveto url: /commands/rclone_moveto/ +versionIntroduced: v1.35 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/moveto/ and as part of making a release run "make commanddocs" --- # rclone moveto diff --git a/docs/content/commands/rclone_ncdu.md b/docs/content/commands/rclone_ncdu.md index 154eb219b3c2b..a92250cf25969 100644 --- a/docs/content/commands/rclone_ncdu.md +++ b/docs/content/commands/rclone_ncdu.md @@ -3,6 +3,7 @@ title: "rclone ncdu" description: "Explore a remote with a text based user interface." slug: rclone_ncdu url: /commands/rclone_ncdu/ +versionIntroduced: v1.37 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/ncdu/ and as part of making a release run "make commanddocs" --- # rclone ncdu @@ -29,11 +30,12 @@ press '?' to toggle the help on and off. The supported keys are: ↑,↓ or k,j to Move →,l to enter ←,h to return - c toggle counts g toggle graph + c toggle counts a toggle average size in directory + m toggle modified time u toggle human-readable format - n,s,C,A sort by name,size,count,average size + n,s,C,A,M sort by name,size,count,asize,mtime d delete file/directory v select file/directory V enter visual select mode diff --git a/docs/content/commands/rclone_obscure.md b/docs/content/commands/rclone_obscure.md index 0a77772a889ce..06f556430ad7c 100644 --- a/docs/content/commands/rclone_obscure.md +++ b/docs/content/commands/rclone_obscure.md @@ -3,6 +3,7 @@ title: "rclone obscure" description: "Obscure password for use in the rclone config file." slug: rclone_obscure url: /commands/rclone_obscure/ +versionIntroduced: v1.36 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/obscure/ and as part of making a release run "make commanddocs" --- # rclone obscure diff --git a/docs/content/commands/rclone_rc.md b/docs/content/commands/rclone_rc.md index 4d2dbb07f7bc2..1cdd9b135d6e6 100644 --- a/docs/content/commands/rclone_rc.md +++ b/docs/content/commands/rclone_rc.md @@ -3,6 +3,7 @@ title: "rclone rc" description: "Run a command against a running rclone." slug: rclone_rc url: /commands/rclone_rc/ +versionIntroduced: v1.40 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/rc/ and as part of making a release run "make commanddocs" --- # rclone rc diff --git a/docs/content/commands/rclone_rcat.md b/docs/content/commands/rclone_rcat.md index 86d4c793e9c4a..a51eda8515973 100644 --- a/docs/content/commands/rclone_rcat.md +++ b/docs/content/commands/rclone_rcat.md @@ -3,6 +3,7 @@ title: "rclone rcat" description: "Copies standard input to file on remote." slug: rclone_rcat url: /commands/rclone_rcat/ +versionIntroduced: v1.38 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/rcat/ and as part of making a release run "make commanddocs" --- # rclone rcat diff --git a/docs/content/commands/rclone_rcd.md b/docs/content/commands/rclone_rcd.md index 7e3508d3fe2a2..d179b606abddf 100644 --- a/docs/content/commands/rclone_rcd.md +++ b/docs/content/commands/rclone_rcd.md @@ -3,6 +3,7 @@ title: "rclone rcd" description: "Run rclone listening to remote control commands only." slug: rclone_rcd url: /commands/rclone_rcd/ +versionIntroduced: v1.45 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/rcd/ and as part of making a release run "make commanddocs" --- # rclone rcd @@ -22,6 +23,101 @@ the browser when rclone is run. See the [rc documentation](/rc/) for more info on the rc flags. +## Server options + +Use `--addr` to specify which IP address and port the server should +listen on, eg `--addr 1.2.3.4:8000` or `--addr :8080` to listen to all +IPs. By default it only listens on localhost. You can use port +:0 to let the OS choose an available port. + +If you set `--addr` to listen on a public or LAN accessible IP address +then using Authentication is advised - see the next section for info. + +You can use a unix socket by setting the url to `unix:///path/to/socket` +or just by using an absolute path name. Note that unix sockets bypass the +authentication - this is expected to be done with file system permissions. + +`--addr` may be repeated to listen on multiple IPs/ports/sockets. + +`--server-read-timeout` and `--server-write-timeout` can be used to +control the timeouts on the server. Note that this is the total time +for a transfer. + +`--max-header-bytes` controls the maximum number of bytes the server will +accept in the HTTP header. + +`--baseurl` controls the URL prefix that rclone serves from. By default +rclone will serve from the root. If you used `--baseurl "/rclone"` then +rclone would serve from a URL starting with "/rclone/". This is +useful if you wish to proxy rclone serve. Rclone automatically +inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, +`--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated +identically. + +### TLS (SSL) + +By default this will serve over http. If you want you can serve over +https. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. + +`--cert` should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client +certificate authority certificate. + +--min-tls-version is minimum TLS version that is acceptable. Valid + values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default + "tls1.0"). + +### Template + +`--template` allows a user to specify a custom markup template for HTTP +and WebDAV serve functions. The server exports the following markup +to be used within the template to server pages: + +| Parameter | Description | +| :---------- | :---------- | +| .Name | The full path of a file/directory. | +| .Title | Directory listing of .Name | +| .Sort | The current sort used. This is changeable via ?sort= parameter | +| | Sort Options: namedirfirst,name,size,time (default namedirfirst) | +| .Order | The current ordering used. This is changeable via ?order= parameter | +| | Order Options: asc,desc (default asc) | +| .Query | Currently unused. | +| .Breadcrumb | Allows for creating a relative navigation | +|-- .Link | The relative to the root link of the Text. | +|-- .Text | The Name of the directory. | +| .Entries | Information about a specific file/directory. | +|-- .URL | The 'url' of an entry. | +|-- .Leaf | Currently same as 'URL' but intended to be 'just' the name. | +|-- .IsDir | Boolean for if an entry is a directory or not. | +|-- .Size | Size in Bytes of the entry. | +|-- .ModTime | The UTC timestamp of an entry. | + +### Authentication + +By default this will serve files without needing a login. + +You can either use an htpasswd file which can take lots of users, or +set a single username and password with the `--user` and `--pass` flags. + +Use `--htpasswd /path/to/htpasswd` to provide an htpasswd file. This is +in standard apache format and supports MD5, SHA1 and BCrypt for basic +authentication. Bcrypt is recommended. + +To create an htpasswd file: + + touch htpasswd + htpasswd -B htpasswd user + htpasswd -B htpasswd anotherUser + +The password file can be updated while rclone is running. + +Use `--realm` to set the authentication realm. + +Use `--salt` to change the password hashing salt from the default. + ``` rclone rcd * [flags] diff --git a/docs/content/commands/rclone_rmdirs.md b/docs/content/commands/rclone_rmdirs.md index ba5dc56fc1fec..22196b848ccdd 100644 --- a/docs/content/commands/rclone_rmdirs.md +++ b/docs/content/commands/rclone_rmdirs.md @@ -3,6 +3,7 @@ title: "rclone rmdirs" description: "Remove empty directories under the path." slug: rclone_rmdirs url: /commands/rclone_rmdirs/ +versionIntroduced: v1.35 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/rmdirs/ and as part of making a release run "make commanddocs" --- # rclone rmdirs diff --git a/docs/content/commands/rclone_selfupdate.md b/docs/content/commands/rclone_selfupdate.md index 2709c63f18c03..8a12883ad4ad0 100644 --- a/docs/content/commands/rclone_selfupdate.md +++ b/docs/content/commands/rclone_selfupdate.md @@ -3,6 +3,7 @@ title: "rclone selfupdate" description: "Update the rclone binary." slug: rclone_selfupdate url: /commands/rclone_selfupdate/ +versionIntroduced: v1.55 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/selfupdate/ and as part of making a release run "make commanddocs" --- # rclone selfupdate diff --git a/docs/content/commands/rclone_serve.md b/docs/content/commands/rclone_serve.md index 12d8141a82a04..ff08cc8b77afa 100644 --- a/docs/content/commands/rclone_serve.md +++ b/docs/content/commands/rclone_serve.md @@ -3,6 +3,7 @@ title: "rclone serve" description: "Serve a remote over a protocol." slug: rclone_serve url: /commands/rclone_serve/ +versionIntroduced: v1.39 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/serve/ and as part of making a release run "make commanddocs" --- # rclone serve diff --git a/docs/content/commands/rclone_serve_dlna.md b/docs/content/commands/rclone_serve_dlna.md index 0cb85400e51fb..00718ef47a82b 100644 --- a/docs/content/commands/rclone_serve_dlna.md +++ b/docs/content/commands/rclone_serve_dlna.md @@ -3,6 +3,7 @@ title: "rclone serve dlna" description: "Serve remote:path over DLNA" slug: rclone_serve_dlna url: /commands/rclone_serve_dlna/ +versionIntroduced: v1.46 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/serve/dlna/ and as part of making a release run "make commanddocs" --- # rclone serve dlna @@ -362,8 +363,8 @@ rclone serve dlna remote:path [flags] ``` --addr string The ip:port or :port to bind the DLNA http server to (default ":7879") - --announce-interval duration The interval between SSDP announcements (default 12m0s) - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --announce-interval Duration The interval between SSDP announcements (default 12m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -374,24 +375,24 @@ rclone serve dlna remote:path [flags] --no-checksum Don't compare checksums on up/download --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) ``` See the [global flags page](/flags/) for global options not listed here. diff --git a/docs/content/commands/rclone_serve_docker.md b/docs/content/commands/rclone_serve_docker.md index b294968a080fe..a7d02fc0351f0 100644 --- a/docs/content/commands/rclone_serve_docker.md +++ b/docs/content/commands/rclone_serve_docker.md @@ -3,6 +3,7 @@ title: "rclone serve docker" description: "Serve any remote on docker's volume plugin API." slug: rclone_serve_docker url: /commands/rclone_serve_docker/ +versionIntroduced: v1.56 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/serve/docker/ and as part of making a release run "make commanddocs" --- # rclone serve docker @@ -381,15 +382,15 @@ rclone serve docker [flags] --allow-other Allow access to other users (not supported on Windows) --allow-root Allow access to root user (not supported on Windows) --async-read Use asynchronous reads (not supported on Windows) (default true) - --attr-timeout duration Time for which file/directory attributes are cached (default 1s) + --attr-timeout Duration Time for which file/directory attributes are cached (default 1s) --base-dir string Base directory for volumes (default "/var/lib/docker-volumes/rclone") --daemon Run mount in background and exit parent process (as background output is suppressed, use --log-file with --log-format=pid,... to monitor) (not supported on Windows) - --daemon-timeout duration Time limit for rclone to respond to kernel (not supported on Windows) - --daemon-wait duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) + --daemon-timeout Duration Time limit for rclone to respond to kernel (not supported on Windows) (default 0s) + --daemon-wait Duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) --debug-fuse Debug the FUSE internals - needs -v --default-permissions Makes kernel enforce access control based on the file mode (not supported on Windows) --devname string Set the device name - default is remote:path - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --forget-state Skip restoring previous state @@ -405,26 +406,26 @@ rclone serve docker [flags] --noappledouble Ignore Apple Double (._) and .DS_Store files (supported on OSX only) (default true) --noapplexattr Ignore all "com.apple.*" extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --socket-addr string Address or absolute path (default: /run/docker/plugins/rclone.sock) --socket-gid int GID for unix socket (default: current process GID) (default 1000) --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) --volname string Set the volume name (supported on Windows and OSX only) --write-back-cache Makes kernel buffer writes before sending them to rclone (without this, writethrough caching is used) (not supported on Windows) ``` diff --git a/docs/content/commands/rclone_serve_ftp.md b/docs/content/commands/rclone_serve_ftp.md index 3274a92c491d1..475e7bb2136e2 100644 --- a/docs/content/commands/rclone_serve_ftp.md +++ b/docs/content/commands/rclone_serve_ftp.md @@ -3,6 +3,7 @@ title: "rclone serve ftp" description: "Serve remote:path over FTP." slug: rclone_serve_ftp url: /commands/rclone_serve_ftp/ +versionIntroduced: v1.44 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/serve/ftp/ and as part of making a release run "make commanddocs" --- # rclone serve ftp @@ -442,7 +443,7 @@ rclone serve ftp remote:path [flags] --addr string IPaddress:Port or :Port to bind server to (default "localhost:2121") --auth-proxy string A program to use to create the backend from the auth --cert string TLS PEM key (concatenation of certificate and CA certificate) - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -453,26 +454,26 @@ rclone serve ftp remote:path [flags] --no-seek Don't allow seeking in files --pass string Password for authentication (empty value allow every password) --passive-port string Passive port range to use (default "30000-32000") - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --public-ip string Public IP address to advertise for passive connections --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication (default "anonymous") - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) ``` See the [global flags page](/flags/) for global options not listed here. diff --git a/docs/content/commands/rclone_serve_http.md b/docs/content/commands/rclone_serve_http.md index 5ad079b41d007..fe9eeaa13c6ea 100644 --- a/docs/content/commands/rclone_serve_http.md +++ b/docs/content/commands/rclone_serve_http.md @@ -3,6 +3,7 @@ title: "rclone serve http" description: "Serve the remote over HTTP." slug: rclone_serve_http url: /commands/rclone_serve_http/ +versionIntroduced: v1.39 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/serve/http/ and as part of making a release run "make commanddocs" --- # rclone serve http @@ -33,6 +34,12 @@ IPs. By default it only listens on localhost. You can use port If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. +You can use a unix socket by setting the url to `unix:///path/to/socket` +or just by using an absolute path name. Note that unix sockets bypass the +authentication - this is expected to be done with file system permissions. + +`--addr` may be repeated to listen on multiple IPs/ports/sockets. + `--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -48,7 +55,7 @@ inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, `--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. -### SSL/TLS +### TLS (SSL) By default this will serve over http. If you want you can serve over https. You will need to supply the `--cert` and `--key` flags. @@ -438,47 +445,47 @@ rclone serve http remote:path [flags] ## Options ``` - --addr string IPaddress:Port or :Port to bind server to (default "127.0.0.1:8080") + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --baseurl string Prefix for URLs - leave blank for root - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) -h, --help help for http --htpasswd string A htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --no-checksum Don't compare checksums on up/download --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --realm string Realm for authentication --salt string Password hashing salt (default "dlPL2MqE") - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --template string User-specified template --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) ``` See the [global flags page](/flags/) for global options not listed here. diff --git a/docs/content/commands/rclone_serve_restic.md b/docs/content/commands/rclone_serve_restic.md index 40a4ed6d803f4..cbd8cd5ac18a5 100644 --- a/docs/content/commands/rclone_serve_restic.md +++ b/docs/content/commands/rclone_serve_restic.md @@ -3,6 +3,7 @@ title: "rclone serve restic" description: "Serve the remote for restic's REST API." slug: rclone_serve_restic url: /commands/rclone_serve_restic/ +versionIntroduced: v1.40 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/serve/restic/ and as part of making a release run "make commanddocs" --- # rclone serve restic @@ -96,13 +97,19 @@ with a path of `//`. ## Server options Use `--addr` to specify which IP address and port the server should -listen on, e.g. `--addr 1.2.3.4:8000` or `--addr :8080` to -listen to all IPs. By default it only listens on localhost. You can use port +listen on, eg `--addr 1.2.3.4:8000` or `--addr :8080` to listen to all +IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. +You can use a unix socket by setting the url to `unix:///path/to/socket` +or just by using an absolute path name. Note that unix sockets bypass the +authentication - this is expected to be done with file system permissions. + +`--addr` may be repeated to listen on multiple IPs/ports/sockets. + `--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -118,28 +125,21 @@ inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, `--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. -`--template` allows a user to specify a custom markup template for HTTP -and WebDAV serve functions. The server exports the following markup -to be used within the template to server pages: - -| Parameter | Description | -| :---------- | :---------- | -| .Name | The full path of a file/directory. | -| .Title | Directory listing of .Name | -| .Sort | The current sort used. This is changeable via ?sort= parameter | -| | Sort Options: namedirfirst,name,size,time (default namedirfirst) | -| .Order | The current ordering used. This is changeable via ?order= parameter | -| | Order Options: asc,desc (default asc) | -| .Query | Currently unused. | -| .Breadcrumb | Allows for creating a relative navigation | -|-- .Link | The relative to the root link of the Text. | -|-- .Text | The Name of the directory. | -| .Entries | Information about a specific file/directory. | -|-- .URL | The 'url' of an entry. | -|-- .Leaf | Currently same as 'URL' but intended to be 'just' the name. | -|-- .IsDir | Boolean for if an entry is a directory or not. | -|-- .Size | Size in Bytes of the entry. | -|-- .ModTime | The UTC timestamp of an entry. | +### TLS (SSL) + +By default this will serve over http. If you want you can serve over +https. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. + +`--cert` should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client +certificate authority certificate. + +--min-tls-version is minimum TLS version that is acceptable. Valid + values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default + "tls1.0"). ### Authentication @@ -162,21 +162,7 @@ The password file can be updated while rclone is running. Use `--realm` to set the authentication realm. -### SSL/TLS - -By default this will serve over HTTP. If you want you can serve over -HTTPS. You will need to supply the `--cert` and `--key` flags. -If you wish to do client side certificate validation then you will need to -supply `--client-ca` also. - -`--cert` should be either a PEM encoded certificate or a concatenation -of that with the CA certificate. `--key` should be the PEM encoded -private key and `--client-ca` should be the PEM encoded client -certificate authority certificate. - ---min-tls-version is minimum TLS version that is acceptable. Valid - values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default - "tls1.0"). +Use `--salt` to change the password hashing salt from the default. ``` @@ -186,24 +172,24 @@ rclone serve restic remote:path [flags] ## Options ``` - --addr string IPaddress:Port or :Port to bind server to (default "localhost:8080") + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --append-only Disallow deletion of repository data --baseurl string Prefix for URLs - leave blank for root --cache-objects Cache listed objects (default true) - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with -h, --help help for restic - --htpasswd string htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --htpasswd string A htpasswd file - if not provided no authentication is done + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --pass string Password for authentication --private-repos Users can only access their private repo - --realm string Realm for authentication (default "rclone") - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --realm string Realm for authentication + --salt string Password hashing salt (default "dlPL2MqE") + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --stdio Run an HTTP2 server on stdin/stdout - --template string User-specified template --user string User name for authentication ``` diff --git a/docs/content/commands/rclone_serve_sftp.md b/docs/content/commands/rclone_serve_sftp.md index 9bed264fbebbf..1e878c058bd0e 100644 --- a/docs/content/commands/rclone_serve_sftp.md +++ b/docs/content/commands/rclone_serve_sftp.md @@ -3,6 +3,7 @@ title: "rclone serve sftp" description: "Serve the remote over SFTP." slug: rclone_serve_sftp url: /commands/rclone_serve_sftp/ +versionIntroduced: v1.48 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/serve/sftp/ and as part of making a release run "make commanddocs" --- # rclone serve sftp @@ -474,7 +475,7 @@ rclone serve sftp remote:path [flags] --addr string IPaddress:Port or :Port to bind server to (default "localhost:2022") --auth-proxy string A program to use to create the backend from the auth --authorized-keys string Authorized keys file (default "~/.ssh/authorized_keys") - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -485,26 +486,26 @@ rclone serve sftp remote:path [flags] --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --stdio Run an sftp server on stdin/stdout --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) ``` See the [global flags page](/flags/) for global options not listed here. diff --git a/docs/content/commands/rclone_serve_webdav.md b/docs/content/commands/rclone_serve_webdav.md index 21c2491ea06b9..88e6973dbed74 100644 --- a/docs/content/commands/rclone_serve_webdav.md +++ b/docs/content/commands/rclone_serve_webdav.md @@ -3,6 +3,7 @@ title: "rclone serve webdav" description: "Serve remote:path over WebDAV." slug: rclone_serve_webdav url: /commands/rclone_serve_webdav/ +versionIntroduced: v1.39 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/serve/webdav/ and as part of making a release run "make commanddocs" --- # rclone serve webdav @@ -31,13 +32,19 @@ to see the full list. ## Server options Use `--addr` to specify which IP address and port the server should -listen on, e.g. `--addr 1.2.3.4:8000` or `--addr :8080` to -listen to all IPs. By default it only listens on localhost. You can use port +listen on, eg `--addr 1.2.3.4:8000` or `--addr :8080` to listen to all +IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. If you set `--addr` to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. +You can use a unix socket by setting the url to `unix:///path/to/socket` +or just by using an absolute path name. Note that unix sockets bypass the +authentication - this is expected to be done with file system permissions. + +`--addr` may be repeated to listen on multiple IPs/ports/sockets. + `--server-read-timeout` and `--server-write-timeout` can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -53,6 +60,24 @@ inserts leading and trailing "/" on `--baseurl`, so `--baseurl "rclone"`, `--baseurl "/rclone"` and `--baseurl "/rclone/"` are all treated identically. +### TLS (SSL) + +By default this will serve over http. If you want you can serve over +https. You will need to supply the `--cert` and `--key` flags. +If you wish to do client side certificate validation then you will need to +supply `--client-ca` also. + +`--cert` should be a either a PEM encoded certificate or a concatenation +of that with the CA certificate. `--key` should be the PEM encoded +private key and `--client-ca` should be the PEM encoded client +certificate authority certificate. + +--min-tls-version is minimum TLS version that is acceptable. Valid + values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default + "tls1.0"). + +### Template + `--template` allows a user to specify a custom markup template for HTTP and WebDAV serve functions. The server exports the following markup to be used within the template to server pages: @@ -97,21 +122,7 @@ The password file can be updated while rclone is running. Use `--realm` to set the authentication realm. -### SSL/TLS - -By default this will serve over HTTP. If you want you can serve over -HTTPS. You will need to supply the `--cert` and `--key` flags. -If you wish to do client side certificate validation then you will need to -supply `--client-ca` also. - -`--cert` should be either a PEM encoded certificate or a concatenation -of that with the CA certificate. `--key` should be the PEM encoded -private key and `--client-ca` should be the PEM encoded client -certificate authority certificate. - ---min-tls-version is minimum TLS version that is acceptable. Valid - values are "tls1.0", "tls1.1", "tls1.2" and "tls1.3" (default - "tls1.0"). +Use `--salt` to change the password hashing salt from the default. ## VFS - Virtual File System @@ -520,49 +531,50 @@ rclone serve webdav remote:path [flags] ## Options ``` - --addr string IPaddress:Port or :Port to bind server to (default "localhost:8080") + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --auth-proxy string A program to use to create the backend from the auth --baseurl string Prefix for URLs - leave blank for root - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --disable-dir-list Disable HTML directory list on GET request for a directory --etag-hash string Which hash to use for the ETag, or auto or blank for off --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) -h, --help help for webdav - --htpasswd string htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --htpasswd string A htpasswd file - if not provided no authentication is done + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --no-checksum Don't compare checksums on up/download --no-modtime Don't read/write the modification time (can speed things up) --no-seek Don't allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access - --realm string Realm for authentication (default "rclone") - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --realm string Realm for authentication + --salt string Password hashing salt (default "dlPL2MqE") + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --template string User-specified template --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached ('off' is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) ``` See the [global flags page](/flags/) for global options not listed here. diff --git a/docs/content/commands/rclone_settier.md b/docs/content/commands/rclone_settier.md index 014a4fce1f9ef..b69f9f302803d 100644 --- a/docs/content/commands/rclone_settier.md +++ b/docs/content/commands/rclone_settier.md @@ -3,6 +3,7 @@ title: "rclone settier" description: "Changes storage class/tier of objects in remote." slug: rclone_settier url: /commands/rclone_settier/ +versionIntroduced: v1.44 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/settier/ and as part of making a release run "make commanddocs" --- # rclone settier diff --git a/docs/content/commands/rclone_sha1sum.md b/docs/content/commands/rclone_sha1sum.md index 49a09ec29ef35..3d02f6d41ab95 100644 --- a/docs/content/commands/rclone_sha1sum.md +++ b/docs/content/commands/rclone_sha1sum.md @@ -3,6 +3,7 @@ title: "rclone sha1sum" description: "Produces an sha1sum file for all the objects in the path." slug: rclone_sha1sum url: /commands/rclone_sha1sum/ +versionIntroduced: v1.27 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/sha1sum/ and as part of making a release run "make commanddocs" --- # rclone sha1sum diff --git a/docs/content/commands/rclone_size.md b/docs/content/commands/rclone_size.md index 7f75fe981d148..ad58f6f11e686 100644 --- a/docs/content/commands/rclone_size.md +++ b/docs/content/commands/rclone_size.md @@ -3,6 +3,7 @@ title: "rclone size" description: "Prints the total size and number of objects in remote:path." slug: rclone_size url: /commands/rclone_size/ +versionIntroduced: v1.23 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/size/ and as part of making a release run "make commanddocs" --- # rclone size diff --git a/docs/content/commands/rclone_test.md b/docs/content/commands/rclone_test.md index d04ccb803b159..4eed2b262e958 100644 --- a/docs/content/commands/rclone_test.md +++ b/docs/content/commands/rclone_test.md @@ -3,6 +3,7 @@ title: "rclone test" description: "Run a test command" slug: rclone_test url: /commands/rclone_test/ +versionIntroduced: v1.55 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/test/ and as part of making a release run "make commanddocs" --- # rclone test diff --git a/docs/content/commands/rclone_test_changenotify.md b/docs/content/commands/rclone_test_changenotify.md index e5f1024debe15..1f87bc68240e0 100644 --- a/docs/content/commands/rclone_test_changenotify.md +++ b/docs/content/commands/rclone_test_changenotify.md @@ -3,6 +3,7 @@ title: "rclone test changenotify" description: "Log any change notify requests for the remote passed in." slug: rclone_test_changenotify url: /commands/rclone_test_changenotify/ +versionIntroduced: v1.56 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/test/changenotify/ and as part of making a release run "make commanddocs" --- # rclone test changenotify @@ -17,7 +18,7 @@ rclone test changenotify remote: [flags] ``` -h, --help help for changenotify - --poll-interval duration Time to wait between polling for changes (default 10s) + --poll-interval Duration Time to wait between polling for changes (default 10s) ``` See the [global flags page](/flags/) for global options not listed here. diff --git a/docs/content/commands/rclone_test_histogram.md b/docs/content/commands/rclone_test_histogram.md index b87ac83a0c289..493007d99bb86 100644 --- a/docs/content/commands/rclone_test_histogram.md +++ b/docs/content/commands/rclone_test_histogram.md @@ -3,6 +3,7 @@ title: "rclone test histogram" description: "Makes a histogram of file name characters." slug: rclone_test_histogram url: /commands/rclone_test_histogram/ +versionIntroduced: v1.55 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/test/histogram/ and as part of making a release run "make commanddocs" --- # rclone test histogram diff --git a/docs/content/commands/rclone_test_info.md b/docs/content/commands/rclone_test_info.md index d8791d7c13aff..1a16e62423381 100644 --- a/docs/content/commands/rclone_test_info.md +++ b/docs/content/commands/rclone_test_info.md @@ -3,6 +3,7 @@ title: "rclone test info" description: "Discovers file name or other limitations for paths." slug: rclone_test_info url: /commands/rclone_test_info/ +versionIntroduced: v1.55 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/test/info/ and as part of making a release run "make commanddocs" --- # rclone test info @@ -32,7 +33,7 @@ rclone test info [remote:path]+ [flags] --check-normalization Check UTF-8 Normalization --check-streaming Check uploads with indeterminate file size -h, --help help for info - --upload-wait duration Wait after writing a file + --upload-wait Duration Wait after writing a file (default 0s) --write-json string Write results to file ``` diff --git a/docs/content/commands/rclone_test_makefile.md b/docs/content/commands/rclone_test_makefile.md index 5acddb5c19256..4eb0977e31634 100644 --- a/docs/content/commands/rclone_test_makefile.md +++ b/docs/content/commands/rclone_test_makefile.md @@ -3,6 +3,7 @@ title: "rclone test makefile" description: "Make files with random contents of the size given" slug: rclone_test_makefile url: /commands/rclone_test_makefile/ +versionIntroduced: v1.59 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/test/makefile/ and as part of making a release run "make commanddocs" --- # rclone test makefile diff --git a/docs/content/commands/rclone_test_makefiles.md b/docs/content/commands/rclone_test_makefiles.md index ad8e3f14b819b..4211bed963d56 100644 --- a/docs/content/commands/rclone_test_makefiles.md +++ b/docs/content/commands/rclone_test_makefiles.md @@ -3,6 +3,7 @@ title: "rclone test makefiles" description: "Make a random file hierarchy in a directory" slug: rclone_test_makefiles url: /commands/rclone_test_makefiles/ +versionIntroduced: v1.55 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/test/makefiles/ and as part of making a release run "make commanddocs" --- # rclone test makefiles @@ -21,6 +22,7 @@ rclone test makefiles [flags] --files int Number of files to create (default 1000) --files-per-directory int Average number of files per directory (default 10) -h, --help help for makefiles + --max-depth int Maximum depth of directory hierarchy (default 10) --max-file-size SizeSuffix Maximum size of files to create (default 100) --max-name-length int Maximum size of file names (default 12) --min-file-size SizeSuffix Minimum size of file to create diff --git a/docs/content/commands/rclone_test_memory.md b/docs/content/commands/rclone_test_memory.md index 104e9f24da65c..5d527fe20d6ba 100644 --- a/docs/content/commands/rclone_test_memory.md +++ b/docs/content/commands/rclone_test_memory.md @@ -3,6 +3,7 @@ title: "rclone test memory" description: "Load all the objects at remote:path into memory and report memory stats." slug: rclone_test_memory url: /commands/rclone_test_memory/ +versionIntroduced: v1.55 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/test/memory/ and as part of making a release run "make commanddocs" --- # rclone test memory diff --git a/docs/content/commands/rclone_touch.md b/docs/content/commands/rclone_touch.md index 556a15fec399c..0882cc34a3887 100644 --- a/docs/content/commands/rclone_touch.md +++ b/docs/content/commands/rclone_touch.md @@ -3,6 +3,7 @@ title: "rclone touch" description: "Create new file or change file modification time." slug: rclone_touch url: /commands/rclone_touch/ +versionIntroduced: v1.39 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/touch/ and as part of making a release run "make commanddocs" --- # rclone touch diff --git a/docs/content/commands/rclone_tree.md b/docs/content/commands/rclone_tree.md index 3ecc450e515b5..277f05e80b733 100644 --- a/docs/content/commands/rclone_tree.md +++ b/docs/content/commands/rclone_tree.md @@ -3,6 +3,7 @@ title: "rclone tree" description: "List the contents of the remote in a tree like fashion." slug: rclone_tree url: /commands/rclone_tree/ +versionIntroduced: v1.38 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/tree/ and as part of making a release run "make commanddocs" --- # rclone tree diff --git a/docs/content/commands/rclone_version.md b/docs/content/commands/rclone_version.md index 3f16a70ffbff9..c2d424ee3e74a 100644 --- a/docs/content/commands/rclone_version.md +++ b/docs/content/commands/rclone_version.md @@ -3,6 +3,7 @@ title: "rclone version" description: "Show the version number." slug: rclone_version url: /commands/rclone_version/ +versionIntroduced: v1.33 # autogenerated - DO NOT EDIT, instead edit the source code in cmd/version/ and as part of making a release run "make commanddocs" --- # rclone version diff --git a/docs/content/flags.md b/docs/content/flags.md index c94a9e90441b2..48771a909a614 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -27,10 +27,10 @@ These flags are available for every command. -c, --checksum Skip based on checksum (if available) & size, not mod-time & size --client-cert string Client SSL certificate (PEM) for mutual TLS auth --client-key string Client SSL private key (PEM) for mutual TLS auth - --color Define when colors (and other ANSI codes) should be shown AUTO|ALWAYS|NEVER (default AUTO) + --color string When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS (default "AUTO") --compare-dest stringArray Include additional comma separated server-side paths during comparison --config string Config file (default "$HOME/.config/rclone/rclone.conf") - --contimeout duration Connect timeout (default 1m0s) + --contimeout Duration Connect timeout (default 1m0s) --copy-dest stringArray Implies --compare-dest but also copies files from paths into destination --cpuprofile string Write cpu profile to file --cutoff-mode string Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS (default "HARD") @@ -48,16 +48,16 @@ These flags are available for every command. --dump-headers Dump HTTP headers - may contain sensitive info --error-on-no-transfer Sets exit code 9 if no files are transferred, useful in scripts --exclude stringArray Exclude files matching pattern - --exclude-from stringArray Read exclude patterns from file (use - to read from stdin) + --exclude-from stringArray Read file exclude patterns from file (use - to read from stdin) --exclude-if-present stringArray Exclude directories if filename is present - --expect-continue-timeout duration Timeout when using expect / 100-continue in HTTP (default 1s) + --expect-continue-timeout Duration Timeout when using expect / 100-continue in HTTP (default 1s) --fast-list Use recursive list if available; uses more memory but fewer transactions --files-from stringArray Read list of source-file names from file (use - to read from stdin) --files-from-raw stringArray Read list of source-file names from file without any processing of lines (use - to read from stdin) - -f, --filter stringArray Add a file-filtering rule - --filter-from stringArray Read filtering patterns from a file (use - to read from stdin) - --fs-cache-expire-duration duration Cache remotes for this long (0 to disable caching) (default 5m0s) - --fs-cache-expire-interval duration Interval to check for expired remotes (default 1m0s) + -f, --filter stringArray Add a file filtering rule + --filter-from stringArray Read file filtering patterns from a file (use - to read from stdin) + --fs-cache-expire-duration Duration Cache remotes for this long (0 to disable caching) (default 5m0s) + --fs-cache-expire-interval Duration Interval to check for expired remotes (default 1m0s) --header stringArray Set HTTP header for all transactions --header-download stringArray Set HTTP header for download transactions --header-upload stringArray Set HTTP header for upload transactions @@ -71,9 +71,9 @@ These flags are available for every command. -I, --ignore-times Don't skip files that match size and time - transfer all files --immutable Do not modify files, fail if existing files have been modified --include stringArray Include files matching pattern - --include-from stringArray Read include patterns from file (use - to read from stdin) + --include-from stringArray Read file include patterns from file (use - to read from stdin) -i, --interactive Enable interactive mode - --kv-lock-time duration Maximum time to keep key-value database locked by process (default 1s) + --kv-lock-time Duration Maximum time to keep key-value database locked by process (default 1s) --log-file string Log everything to this file --log-format string Comma separated list of log format options (default "date,time") --log-level string Log level DEBUG|INFO|NOTICE|ERROR (default "NOTICE") @@ -83,16 +83,22 @@ These flags are available for every command. --max-backlog int Maximum number of objects in sync or check backlog (default 10000) --max-delete int When synchronizing, limit the number of deletes (default -1) --max-depth int If set limits the recursion depth to this (default -1) - --max-duration duration Maximum duration rclone will transfer data for + --max-duration Duration Maximum duration rclone will transfer data for (default 0s) --max-size SizeSuffix Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P (default off) --max-stats-groups int Maximum number of stats groups to keep in memory, on max oldest is discarded (default 1000) --max-transfer SizeSuffix Maximum size of data to transfer (default off) --memprofile string Write memory profile to file -M, --metadata If set, preserve metadata when copying objects + --metadata-exclude stringArray Exclude metadatas matching pattern + --metadata-exclude-from stringArray Read metadata exclude patterns from file (use - to read from stdin) + --metadata-filter stringArray Add a metadata filtering rule + --metadata-filter-from stringArray Read metadata filtering patterns from a file (use - to read from stdin) + --metadata-include stringArray Include metadatas matching pattern + --metadata-include-from stringArray Read metadata include patterns from file (use - to read from stdin) --metadata-set stringArray Add metadata key=value when uploading --min-age Duration Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off) --min-size SizeSuffix Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off) - --modify-window duration Max time diff to be considered the same (default 1ns) + --modify-window Duration Max time diff to be considered the same (default 1ns) --multi-thread-cutoff SizeSuffix Use multi-thread downloads for files above this size (default 250Mi) --multi-thread-streams int Max number of streams to use for multi-thread downloads (default 4) --no-check-certificate Do not verify the server SSL certificate (insecure) @@ -108,25 +114,26 @@ These flags are available for every command. --progress-terminal-title Show progress on the terminal title (requires -P/--progress) -q, --quiet Print as little stuff as possible --rc Enable the remote control server - --rc-addr string IPaddress:Port or :Port to bind server to (default "localhost:5572") + --rc-addr stringArray IPaddress:Port or :Port to bind server to (default [localhost:5572]) --rc-allow-origin string Set the allowed origin for CORS --rc-baseurl string Prefix for URLs - leave blank for root - --rc-cert string SSL PEM key (concatenation of certificate and CA certificate) + --rc-cert string TLS PEM key (concatenation of certificate and CA certificate) --rc-client-ca string Client certificate authority to verify clients with --rc-enable-metrics Enable prometheus metrics on /metrics --rc-files string Path to local files to serve on the HTTP server - --rc-htpasswd string htpasswd file - if not provided no authentication is done - --rc-job-expire-duration duration Expire finished async jobs older than this value (default 1m0s) - --rc-job-expire-interval duration Interval to check for expired async jobs (default 10s) - --rc-key string SSL PEM Private key + --rc-htpasswd string A htpasswd file - if not provided no authentication is done + --rc-job-expire-duration Duration Expire finished async jobs older than this value (default 1m0s) + --rc-job-expire-interval Duration Interval to check for expired async jobs (default 10s) + --rc-key string TLS PEM Private key --rc-max-header-bytes int Maximum size of request header (default 4096) --rc-min-tls-version string Minimum TLS version that is acceptable (default "tls1.0") --rc-no-auth Don't require auth for certain methods --rc-pass string Password for authentication - --rc-realm string Realm for authentication (default "rclone") + --rc-realm string Realm for authentication + --rc-salt string Password hashing salt (default "dlPL2MqE") --rc-serve Enable the serving of remote objects - --rc-server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --rc-server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --rc-server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --rc-server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --rc-template string User-specified template --rc-user string User name for authentication --rc-web-fetch-url string URL to fetch the releases for webgui (default "https://api.github.com/repos/rclone/rclone-webui-react/releases/latest") @@ -136,10 +143,10 @@ These flags are available for every command. --rc-web-gui-update Check and update to latest version of web gui --refresh-times Refresh the modtime of remote files --retries int Retry operations this many times if they fail (default 3) - --retries-sleep duration Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) + --retries-sleep Duration Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) (default 0s) --server-side-across-configs Allow server-side operations (e.g. copy) to work across different configs --size-only Skip based on size only, not mod-time or checksum - --stats duration Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s) + --stats Duration Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s) --stats-file-name-length int Max file name length in stats (0 for no limit) (default 45) --stats-log-level string Log level to show --stats output DEBUG|INFO|NOTICE|ERROR (default "INFO") --stats-one-line Make the stats fit on one line @@ -152,7 +159,7 @@ These flags are available for every command. --syslog Use Syslog for logging --syslog-facility string Facility for syslog, e.g. KERN,USER,... (default "DAEMON") --temp-dir string Directory rclone will use for temporary files (default "/tmp") - --timeout duration IO idle timeout (default 5m0s) + --timeout Duration IO idle timeout (default 5m0s) --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --track-renames When synchronizing, track file renames and do a server-side move if possible @@ -163,7 +170,7 @@ These flags are available for every command. --use-json-log Use json log format --use-mmap Use mmap allocator (see docs) --use-server-modtime Use server modified time instead of object metadata - --user-agent string Set the user-agent to a specified string (default "rclone/v1.60.0") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.61.0") -v, --verbose count Print lots more stuff (repeat for more) ``` @@ -173,527 +180,541 @@ These flags are available for every command. They control the backends and may be set in the config file. ``` - --acd-auth-url string Auth server URL - --acd-client-id string OAuth Client Id - --acd-client-secret string OAuth Client Secret - --acd-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --acd-templink-threshold SizeSuffix Files >= this size will be downloaded via their tempLink (default 9Gi) - --acd-token string OAuth Access Token as a JSON blob - --acd-token-url string Token server url - --acd-upload-wait-per-gb Duration Additional time per GiB to wait after a failed complete upload to see if it appears (default 3m0s) - --alias-remote string Remote or path to alias - --azureblob-access-tier string Access tier of blob: hot, cool or archive - --azureblob-account string Storage Account Name - --azureblob-archive-tier-delete Delete archive tier blobs before overwriting - --azureblob-chunk-size SizeSuffix Upload chunk size (default 4Mi) - --azureblob-disable-checksum Don't store MD5 checksum with object metadata - --azureblob-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8) - --azureblob-endpoint string Endpoint for the service - --azureblob-key string Storage Account Key - --azureblob-list-chunk int Size of blob list (default 5000) - --azureblob-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --azureblob-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --azureblob-msi-client-id string Object ID of the user-assigned MSI to use, if any - --azureblob-msi-mi-res-id string Azure resource ID of the user-assigned MSI to use, if any - --azureblob-msi-object-id string Object ID of the user-assigned MSI to use, if any - --azureblob-no-head-object If set, do not do HEAD before GET when getting objects - --azureblob-public-access string Public access level of a container: blob or container - --azureblob-sas-url string SAS URL for container level access only - --azureblob-service-principal-file string Path to file containing credentials for use with a service principal - --azureblob-upload-concurrency int Concurrency for multipart uploads (default 16) - --azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256 MiB) (deprecated) - --azureblob-use-emulator Uses local storage emulator if provided as 'true' - --azureblob-use-msi Use a managed service identity to authenticate (only works in Azure) - --b2-account string Account ID or Application Key ID - --b2-chunk-size SizeSuffix Upload chunk size (default 96Mi) - --b2-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4Gi) - --b2-disable-checksum Disable checksums for large (> upload cutoff) files - --b2-download-auth-duration Duration Time before the authorization token will expire in s or suffix ms|s|m|h|d (default 1w) - --b2-download-url string Custom endpoint for downloads - --b2-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --b2-endpoint string Endpoint for the service - --b2-hard-delete Permanently delete files on remote removal, otherwise hide files - --b2-key string Application Key - --b2-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging - --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --b2-version-at Time Show file versions as they were at the specified time (default off) - --b2-versions Include old versions in directory listings - --box-access-token string Box App Primary Access Token - --box-auth-url string Auth server URL - --box-box-config-file string Box App config.json location - --box-box-sub-type string (default "user") - --box-client-id string OAuth Client Id - --box-client-secret string OAuth Client Secret - --box-commit-retries int Max number of times to try committing a multipart file (default 100) - --box-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot) - --box-list-chunk int Size of listing chunk 1-1000 (default 1000) - --box-owned-by string Only show items owned by the login (email address) passed in - --box-root-folder-id string Fill in for rclone to use a non root folder as its starting point - --box-token string OAuth Access Token as a JSON blob - --box-token-url string Token server url - --box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50 MiB) (default 50Mi) - --cache-chunk-clean-interval Duration How often should the cache perform cleanups of the chunk storage (default 1m0s) - --cache-chunk-no-memory Disable the in-memory cache for storing chunks during streaming - --cache-chunk-path string Directory to cache chunk files (default "$HOME/.cache/rclone/cache-backend") - --cache-chunk-size SizeSuffix The size of a chunk (partial file data) (default 5Mi) - --cache-chunk-total-size SizeSuffix The total size that the chunks can take up on the local disk (default 10Gi) - --cache-db-path string Directory to store file structure metadata DB (default "$HOME/.cache/rclone/cache-backend") - --cache-db-purge Clear all the cached data for this remote on start - --cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s) - --cache-info-age Duration How long to cache file structure information (directory listings, file size, times, etc.) (default 6h0m0s) - --cache-plex-insecure string Skip all certificate verification when connecting to the Plex server - --cache-plex-password string The password of the Plex user (obscured) - --cache-plex-url string The URL of the Plex server - --cache-plex-username string The username of the Plex user - --cache-read-retries int How many times to retry a read from a cache storage (default 10) - --cache-remote string Remote to cache - --cache-rps int Limits the number of requests per second to the source FS (-1 to disable) (default -1) - --cache-tmp-upload-path string Directory to keep temporary files until they are uploaded - --cache-tmp-wait-time Duration How long should files be stored in local cache before being uploaded (default 15s) - --cache-workers int How many workers should run in parallel to download chunks (default 4) - --cache-writes Cache file data on writes through the FS - --chunker-chunk-size SizeSuffix Files larger than chunk size will be split in chunks (default 2Gi) - --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks - --chunker-hash-type string Choose how chunker handles hash sums (default "md5") - --chunker-remote string Remote to chunk/unchunk - --combine-upstreams SpaceSepList Upstreams for combining - --compress-level int GZIP compression level (-2 to 9) (default -1) - --compress-mode string Compression mode (default "gzip") - --compress-ram-cache-limit SizeSuffix Some remotes don't allow the upload of files with unknown size (default 20Mi) - --compress-remote string Remote to compress - -L, --copy-links Follow symlinks and copy the pointed to item - --crypt-directory-name-encryption Option to either encrypt directory names or leave them intact (default true) - --crypt-filename-encoding string How to encode the encrypted filename to text string (default "base32") - --crypt-filename-encryption string How to encrypt the filenames (default "standard") - --crypt-no-data-encryption Option to either encrypt file data or leave it unencrypted - --crypt-password string Password or pass phrase for encryption (obscured) - --crypt-password2 string Password or pass phrase for salt (obscured) - --crypt-remote string Remote to encrypt/decrypt - --crypt-server-side-across-configs Allow server-side operations (e.g. copy) to work across different crypt configs - --crypt-show-mapping For all files listed show how the names encrypt - --drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded - --drive-allow-import-name-change Allow the filetype to change when uploading Google docs - --drive-auth-owner-only Only consider files owned by the authenticated user - --drive-auth-url string Auth server URL - --drive-chunk-size SizeSuffix Upload chunk size (default 8Mi) - --drive-client-id string Google Application Client Id - --drive-client-secret string OAuth Client Secret - --drive-copy-shortcut-content Server side copy contents of shortcuts instead of the shortcut - --drive-disable-http2 Disable drive using http2 (default true) - --drive-encoding MultiEncoder The encoding for the backend (default InvalidUtf8) - --drive-export-formats string Comma separated list of preferred formats for downloading Google docs (default "docx,xlsx,pptx,svg") - --drive-formats string Deprecated: See export_formats - --drive-impersonate string Impersonate this user when using a service account - --drive-import-formats string Comma separated list of preferred formats for uploading Google docs - --drive-keep-revision-forever Keep new head revision of each file forever - --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) - --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) - --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) - --drive-resource-key string Resource key for accessing a link-shared file - --drive-root-folder-id string ID of the root folder - --drive-scope string Scope that rclone should use when requesting access from drive - --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs - --drive-service-account-credentials string Service Account Credentials JSON blob - --drive-service-account-file string Service Account Credentials JSON file path - --drive-shared-with-me Only show files that are shared with me - --drive-size-as-quota Show sizes as storage quota usage, not actual size - --drive-skip-checksum-gphotos Skip MD5 checksum on Google photos and videos only - --drive-skip-dangling-shortcuts If set skip dangling shortcut files - --drive-skip-gdocs Skip google documents in all listings - --drive-skip-shortcuts If set skip shortcut files - --drive-starred-only Only show files that are starred - --drive-stop-on-download-limit Make download limit errors be fatal - --drive-stop-on-upload-limit Make upload limit errors be fatal - --drive-team-drive string ID of the Shared Drive (Team Drive) - --drive-token string OAuth Access Token as a JSON blob - --drive-token-url string Token server url - --drive-trashed-only Only show files that are in the trash - --drive-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 8Mi) - --drive-use-created-date Use file created date instead of modified date - --drive-use-shared-date Use date file was shared instead of modified date - --drive-use-trash Send files to the trash instead of deleting permanently (default true) - --drive-v2-download-min-size SizeSuffix If Object's are greater, use drive v2 API to download (default off) - --dropbox-auth-url string Auth server URL - --dropbox-batch-commit-timeout Duration Max time to wait for a batch to finish committing (default 10m0s) - --dropbox-batch-mode string Upload file batching sync|async|off (default "sync") - --dropbox-batch-size int Max number of files in upload batch - --dropbox-batch-timeout Duration Max time to allow an idle upload batch before uploading (default 0s) - --dropbox-chunk-size SizeSuffix Upload chunk size (< 150Mi) (default 48Mi) - --dropbox-client-id string OAuth Client Id - --dropbox-client-secret string OAuth Client Secret - --dropbox-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,RightSpace,InvalidUtf8,Dot) - --dropbox-impersonate string Impersonate this user when using a business account - --dropbox-shared-files Instructs rclone to work on individual shared files - --dropbox-shared-folders Instructs rclone to work on shared folders - --dropbox-token string OAuth Access Token as a JSON blob - --dropbox-token-url string Token server url - --fichier-api-key string Your API Key, get it from https://1fichier.com/console/params.pl - --fichier-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot) - --fichier-file-password string If you want to download a shared file that is password protected, add this parameter (obscured) - --fichier-folder-password string If you want to list the files in a shared folder that is password protected, add this parameter (obscured) - --fichier-shared-folder string If you want to download a shared folder, add this parameter - --filefabric-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) - --filefabric-permanent-token string Permanent Authentication Token - --filefabric-root-folder-id string ID of the root folder - --filefabric-token string Session Token - --filefabric-token-expiry string Token expiry time - --filefabric-url string URL of the Enterprise File Fabric to connect to - --filefabric-version string Version read from the file fabric - --ftp-ask-password Allow asking for FTP password when needed - --ftp-close-timeout Duration Maximum time to wait for a response to close (default 1m0s) - --ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited - --ftp-disable-epsv Disable using EPSV even if server advertises support - --ftp-disable-mlsd Disable using MLSD even if server advertises support - --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) - --ftp-disable-utf8 Disable using UTF-8 even if server advertises support - --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) - --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) - --ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD - --ftp-host string FTP host to connect to - --ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --ftp-no-check-certificate Do not verify the TLS certificate of the server - --ftp-pass string FTP password (obscured) - --ftp-port int FTP port number (default 21) - --ftp-shut-timeout Duration Maximum time to wait for data connection closing status (default 1m0s) - --ftp-tls Use Implicit FTPS (FTP over TLS) - --ftp-tls-cache-size int Size of TLS session cache for all control and data connections (default 32) - --ftp-user string FTP username (default "$USER") - --ftp-writing-mdtm Use MDTM to set modification time (VsFtpd quirk) - --gcs-anonymous Access public buckets and objects without credentials - --gcs-auth-url string Auth server URL - --gcs-bucket-acl string Access Control List for new buckets - --gcs-bucket-policy-only Access checks should use bucket-level IAM policies - --gcs-client-id string OAuth Client Id - --gcs-client-secret string OAuth Client Secret - --gcs-decompress If set this will decompress gzip encoded objects - --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) - --gcs-endpoint string Endpoint for the service - --gcs-location string Location for the newly created buckets - --gcs-no-check-bucket If set, don't attempt to check the bucket exists or create it - --gcs-object-acl string Access Control List for new objects - --gcs-project-number string Project number - --gcs-service-account-file string Service Account Credentials JSON file path - --gcs-storage-class string The storage class to use when storing objects in Google Cloud Storage - --gcs-token string OAuth Access Token as a JSON blob - --gcs-token-url string Token server url - --gphotos-auth-url string Auth server URL - --gphotos-client-id string OAuth Client Id - --gphotos-client-secret string OAuth Client Secret - --gphotos-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) - --gphotos-include-archived Also view and download archived media - --gphotos-read-only Set to make the Google Photos backend read only - --gphotos-read-size Set to read the size of media items - --gphotos-start-year int Year limits the photos to be downloaded to those which are uploaded after the given year (default 2000) - --gphotos-token string OAuth Access Token as a JSON blob - --gphotos-token-url string Token server url - --hasher-auto-size SizeSuffix Auto-update checksum for files smaller than this size (disabled by default) - --hasher-hashes CommaSepList Comma separated list of supported checksum types (default md5,sha1) - --hasher-max-age Duration Maximum time to keep checksums in cache (0 = no cache, off = cache forever) (default off) - --hasher-remote string Remote to cache checksums for (e.g. myRemote:path) - --hdfs-data-transfer-protection string Kerberos data transfer protection: authentication|integrity|privacy - --hdfs-encoding MultiEncoder The encoding for the backend (default Slash,Colon,Del,Ctl,InvalidUtf8,Dot) - --hdfs-namenode string Hadoop name node and port - --hdfs-service-principal-name string Kerberos service principal name for the namenode - --hdfs-username string Hadoop user name - --hidrive-auth-url string Auth server URL - --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) - --hidrive-client-id string OAuth Client Id - --hidrive-client-secret string OAuth Client Secret - --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary - --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) - --hidrive-endpoint string Endpoint for the service (default "https://api.hidrive.strato.com/2.1") - --hidrive-root-prefix string The root/parent folder for all paths (default "/") - --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default "rw") - --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default "user") - --hidrive-token string OAuth Access Token as a JSON blob - --hidrive-token-url string Token server url - --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) - --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) - --http-headers CommaSepList Set HTTP headers for all transactions - --http-no-head Don't use HEAD requests - --http-no-slash Set this if the site doesn't end directories with / - --http-url string URL of HTTP host to connect to - --internetarchive-access-key-id string IAS3 Access Key - --internetarchive-disable-checksum Don't ask the server to test against MD5 checksum calculated by rclone (default true) - --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) - --internetarchive-endpoint string IAS3 Endpoint (default "https://s3.us.archive.org") - --internetarchive-front-endpoint string Host of InternetArchive Frontend (default "https://archive.org") - --internetarchive-secret-access-key string IAS3 Secret Key (password) - --internetarchive-wait-archive Duration Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s) - --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) - --jottacloud-hard-delete Delete files permanently rather than putting them into the trash - --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) - --jottacloud-no-versions Avoid server side versioning by deleting files and recreating files instead of overwriting them - --jottacloud-trashed-only Only show files that are in the trash - --jottacloud-upload-resume-limit SizeSuffix Files bigger than this can be resumed if the upload fail's (default 10Mi) - --koofr-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --koofr-endpoint string The Koofr API endpoint to use - --koofr-mountid string Mount ID of the mount to use - --koofr-password string Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured) - --koofr-provider string Choose your storage provider - --koofr-setmtime Does the backend support setting modification time (default true) - --koofr-user string Your user name - -l, --links Translate symlinks to/from regular files with a '.rclonelink' extension - --local-case-insensitive Force the filesystem to report itself as case insensitive - --local-case-sensitive Force the filesystem to report itself as case sensitive - --local-encoding MultiEncoder The encoding for the backend (default Slash,Dot) - --local-no-check-updated Don't check to see if the files change during upload - --local-no-preallocate Disable preallocation of disk space for transferred files - --local-no-set-modtime Disable setting modtime - --local-no-sparse Disable sparse files for multi-thread downloads - --local-nounc Disable UNC (long path names) conversion on Windows - --local-unicode-normalization Apply unicode NFC normalization to paths and filenames - --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) - --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) - --mailru-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --mailru-pass string Password (obscured) - --mailru-speedup-enable Skip full upload if there is another file with same data hash (default true) - --mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash) (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf") - --mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3Gi) - --mailru-speedup-max-memory SizeSuffix Files larger than the size given below will always be hashed on disk (default 32Mi) - --mailru-user string User name (usually email) - --mega-debug Output more debug from Mega - --mega-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --mega-hard-delete Delete files permanently rather than putting them into the trash - --mega-pass string Password (obscured) - --mega-user string User name - --netstorage-account string Set the NetStorage account name - --netstorage-host string Domain+path of NetStorage host to connect to - --netstorage-protocol string Select between HTTP or HTTPS protocol (default "https") - --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) - -x, --one-file-system Don't cross filesystem boundaries (unix/macOS only) - --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) - --onedrive-auth-url string Auth server URL - --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) - --onedrive-client-id string OAuth Client Id - --onedrive-client-secret string OAuth Client Secret - --onedrive-drive-id string The ID of the drive to use - --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) - --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) - --onedrive-expose-onenote-files Set to make OneNote files show up in directory listings - --onedrive-link-password string Set the password for links created by the link command - --onedrive-link-scope string Set the scope of the links created by the link command (default "anonymous") - --onedrive-link-type string Set the type of the links created by the link command (default "view") - --onedrive-list-chunk int Size of listing chunk (default 1000) - --onedrive-no-versions Remove all versions on modifying operations - --onedrive-region string Choose national cloud region for OneDrive (default "global") - --onedrive-root-folder-id string ID of the root folder - --onedrive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different onedrive configs - --onedrive-token string OAuth Access Token as a JSON blob - --onedrive-token-url string Token server url - --oos-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) - --oos-compartment string Object storage compartment OCID - --oos-config-file string Path to OCI config file (default "~/.oci/config") - --oos-config-profile string Profile name inside the oci config file (default "Default") - --oos-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) - --oos-copy-timeout Duration Timeout for copy (default 1m0s) - --oos-disable-checksum Don't store MD5 checksum with object metadata - --oos-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --oos-endpoint string Endpoint for Object storage API - --oos-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery - --oos-namespace string Object storage namespace - --oos-no-check-bucket If set, don't attempt to check the bucket exists or create it - --oos-provider string Choose your Auth Provider (default "env_auth") - --oos-region string Object storage Region - --oos-upload-concurrency int Concurrency for multipart uploads (default 10) - --oos-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size (default 10Mi) - --opendrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot) - --opendrive-password string Password (obscured) - --opendrive-username string Username - --pcloud-auth-url string Auth server URL - --pcloud-client-id string OAuth Client Id - --pcloud-client-secret string OAuth Client Secret - --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --pcloud-hostname string Hostname to connect to (default "api.pcloud.com") - --pcloud-password string Your pcloud password (obscured) - --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default "d0") - --pcloud-token string OAuth Access Token as a JSON blob - --pcloud-token-url string Token server url - --pcloud-username string Your pcloud username - --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --qingstor-access-key-id string QingStor Access Key ID - --qingstor-chunk-size SizeSuffix Chunk size to use for uploading (default 4Mi) - --qingstor-connection-retries int Number of connection retries (default 3) - --qingstor-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8) - --qingstor-endpoint string Enter an endpoint URL to connection QingStor API - --qingstor-env-auth Get QingStor credentials from runtime - --qingstor-secret-access-key string QingStor Secret Access Key (password) - --qingstor-upload-concurrency int Concurrency for multipart uploads (default 1) - --qingstor-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --qingstor-zone string Zone to connect to - --s3-access-key-id string AWS Access Key ID - --s3-acl string Canned ACL used when creating buckets and storing or copying objects - --s3-bucket-acl string Canned ACL used when creating buckets - --s3-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) - --s3-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) - --s3-decompress If set this will decompress gzip encoded objects - --s3-disable-checksum Don't store MD5 checksum with object metadata - --s3-disable-http2 Disable usage of http2 for S3 backends - --s3-download-url string Custom endpoint for downloads - --s3-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --s3-endpoint string Endpoint for S3 API - --s3-env-auth Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars) - --s3-force-path-style If true use path style access if false use virtual hosted style (default true) - --s3-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery - --s3-list-chunk int Size of listing chunk (response list for each ListObject S3 request) (default 1000) - --s3-list-url-encode Tristate Whether to url encode listings: true/false/unset (default unset) - --s3-list-version int Version of ListObjects to use: 1,2 or 0 for auto - --s3-location-constraint string Location constraint - must be set to match the Region - --s3-max-upload-parts int Maximum number of parts in a multipart upload (default 10000) - --s3-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --s3-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --s3-no-check-bucket If set, don't attempt to check the bucket exists or create it - --s3-no-head If set, don't HEAD uploaded objects to check integrity - --s3-no-head-object If set, do not do HEAD before GET when getting objects - --s3-no-system-metadata Suppress setting and reading of system metadata - --s3-profile string Profile to use in the shared credentials file - --s3-provider string Choose your S3 provider - --s3-region string Region to connect to - --s3-requester-pays Enables requester pays option when interacting with S3 bucket - --s3-secret-access-key string AWS Secret Access Key (password) - --s3-server-side-encryption string The server-side encryption algorithm used when storing this object in S3 - --s3-session-token string An AWS session token - --s3-shared-credentials-file string Path to the shared credentials file - --s3-sse-customer-algorithm string If using SSE-C, the server-side encryption algorithm used when storing this object in S3 - --s3-sse-customer-key string To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data - --s3-sse-customer-key-base64 string If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data - --s3-sse-customer-key-md5 string If using SSE-C you may provide the secret encryption key MD5 checksum (optional) - --s3-sse-kms-key-id string If using KMS ID you must provide the ARN of Key - --s3-storage-class string The storage class to use when storing new objects in S3 - --s3-upload-concurrency int Concurrency for multipart uploads (default 4) - --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint - --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) - --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads - --s3-v2-auth If true use v2 authentication - --s3-version-at Time Show file versions as they were at the specified time (default off) - --s3-versions Include old versions in directory listings - --seafile-2fa Two-factor authentication ('true' if the account has 2FA enabled) - --seafile-create-library Should rclone create a library if it doesn't exist - --seafile-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8) - --seafile-library string Name of the library - --seafile-library-key string Library password (for encrypted libraries only) (obscured) - --seafile-pass string Password (obscured) - --seafile-url string URL of seafile host to connect to - --seafile-user string User name (usually email address) - --sftp-ask-password Allow asking for SFTP password when needed - --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) - --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) - --sftp-disable-concurrent-reads If set don't use concurrent reads - --sftp-disable-concurrent-writes If set don't use concurrent writes - --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available - --sftp-host string SSH host to connect to - --sftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --sftp-key-file string Path to PEM-encoded private key file - --sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file (obscured) - --sftp-key-pem string Raw PEM-encoded private key - --sftp-key-use-agent When set forces the usage of the ssh-agent - --sftp-known-hosts-file string Optional path to known_hosts file - --sftp-md5sum-command string The command used to read md5 hashes - --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) - --sftp-path-override string Override path used by SSH shell commands - --sftp-port int SSH port number (default 22) - --sftp-pubkey-file string Optional path to public key file - --sftp-server-command string Specifies the path or command to run a sftp server on the remote host - --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands - --sftp-set-modtime Set the modified time on the remote if set (default true) - --sftp-sha1sum-command string The command used to read sha1 hashes - --sftp-shell-type string The type of SSH shell on remote server, if any - --sftp-skip-links Set to skip any symlinks and any other non regular files - --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default "sftp") - --sftp-use-fstat If set use fstat instead of stat - --sftp-use-insecure-cipher Enable the use of insecure ciphers and key exchange methods - --sftp-user string SSH username (default "$USER") - --sharefile-chunk-size SizeSuffix Upload chunk size (default 64Mi) - --sharefile-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,LeftSpace,LeftPeriod,RightSpace,RightPeriod,InvalidUtf8,Dot) - --sharefile-endpoint string Endpoint for API calls - --sharefile-root-folder-id string ID of the root folder - --sharefile-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (default 128Mi) - --sia-api-password string Sia Daemon API Password (obscured) - --sia-api-url string Sia daemon API URL, like http://sia.daemon.host:9980 (default "http://127.0.0.1:9980") - --sia-encoding MultiEncoder The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot) - --sia-user-agent string Siad User Agent (default "Sia-Agent") - --skip-links Don't warn about skipped symlinks - --smb-case-insensitive Whether the server is configured to be case-insensitive (default true) - --smb-domain string Domain name for NTLM authentication (default "WORKGROUP") - --smb-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot) - --smb-hide-special-share Hide special shares (e.g. print$) which users aren't supposed to access (default true) - --smb-host string SMB server hostname to connect to - --smb-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --smb-pass string SMB password (obscured) - --smb-port int SMB port number (default 445) - --smb-user string SMB username (default "$USER") - --storj-access-grant string Access grant - --storj-api-key string API key - --storj-passphrase string Encryption passphrase - --storj-provider string Choose an authentication method (default "existing") - --storj-satellite-address string Satellite address (default "us-central-1.storj.io") - --sugarsync-access-key-id string Sugarsync Access Key ID - --sugarsync-app-id string Sugarsync App ID - --sugarsync-authorization string Sugarsync authorization - --sugarsync-authorization-expiry string Sugarsync authorization expiry - --sugarsync-deleted-id string Sugarsync deleted folder id - --sugarsync-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8,Dot) - --sugarsync-hard-delete Permanently delete files if true - --sugarsync-private-access-key string Sugarsync Private Access Key - --sugarsync-refresh-token string Sugarsync refresh token - --sugarsync-root-id string Sugarsync root id - --sugarsync-user string Sugarsync user - --swift-application-credential-id string Application Credential ID (OS_APPLICATION_CREDENTIAL_ID) - --swift-application-credential-name string Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME) - --swift-application-credential-secret string Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET) - --swift-auth string Authentication URL for server (OS_AUTH_URL) - --swift-auth-token string Auth Token from alternate authentication - optional (OS_AUTH_TOKEN) - --swift-auth-version int AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION) - --swift-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) - --swift-domain string User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME) - --swift-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8) - --swift-endpoint-type string Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default "public") - --swift-env-auth Get swift credentials from environment variables in standard OpenStack form - --swift-key string API key or password (OS_PASSWORD) - --swift-leave-parts-on-error If true avoid calling abort upload on a failure - --swift-no-chunk Don't chunk files during streaming upload - --swift-no-large-objects Disable support for static and dynamic large objects - --swift-region string Region name - optional (OS_REGION_NAME) - --swift-storage-policy string The storage policy to use when creating a new container - --swift-storage-url string Storage URL - optional (OS_STORAGE_URL) - --swift-tenant string Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME) - --swift-tenant-domain string Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME) - --swift-tenant-id string Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID) - --swift-user string User name to log in (OS_USERNAME) - --swift-user-id string User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID) - --union-action-policy string Policy to choose upstream on ACTION category (default "epall") - --union-cache-time int Cache time of usage and free space (in seconds) (default 120) - --union-create-policy string Policy to choose upstream on CREATE category (default "epmfs") - --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) - --union-search-policy string Policy to choose upstream on SEARCH category (default "ff") - --union-upstreams string List of space separated upstreams - --uptobox-access-token string Your access token - --uptobox-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot) - --webdav-bearer-token string Bearer token instead of user/pass (e.g. a Macaroon) - --webdav-bearer-token-command string Command to run to get a bearer token - --webdav-encoding string The encoding for the backend - --webdav-headers CommaSepList Set HTTP headers for all transactions - --webdav-pass string Password (obscured) - --webdav-url string URL of http host to connect to - --webdav-user string User name - --webdav-vendor string Name of the WebDAV site/service/software you are using - --yandex-auth-url string Auth server URL - --yandex-client-id string OAuth Client Id - --yandex-client-secret string OAuth Client Secret - --yandex-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) - --yandex-hard-delete Delete files permanently rather than putting them into the trash - --yandex-token string OAuth Access Token as a JSON blob - --yandex-token-url string Token server url - --zoho-auth-url string Auth server URL - --zoho-client-id string OAuth Client Id - --zoho-client-secret string OAuth Client Secret - --zoho-encoding MultiEncoder The encoding for the backend (default Del,Ctl,InvalidUtf8) - --zoho-region string Zoho region to connect to - --zoho-token string OAuth Access Token as a JSON blob - --zoho-token-url string Token server url + --acd-auth-url string Auth server URL + --acd-client-id string OAuth Client Id + --acd-client-secret string OAuth Client Secret + --acd-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --acd-templink-threshold SizeSuffix Files >= this size will be downloaded via their tempLink (default 9Gi) + --acd-token string OAuth Access Token as a JSON blob + --acd-token-url string Token server url + --acd-upload-wait-per-gb Duration Additional time per GiB to wait after a failed complete upload to see if it appears (default 3m0s) + --alias-remote string Remote or path to alias + --azureblob-access-tier string Access tier of blob: hot, cool or archive + --azureblob-account string Azure Storage Account Name + --azureblob-archive-tier-delete Delete archive tier blobs before overwriting + --azureblob-chunk-size SizeSuffix Upload chunk size (default 4Mi) + --azureblob-client-certificate-password string Password for the certificate file (optional) (obscured) + --azureblob-client-certificate-path string Path to a PEM or PKCS12 certificate file including the private key + --azureblob-client-id string The ID of the client in use + --azureblob-client-secret string One of the service principal's client secrets + --azureblob-client-send-certificate-chain Send the certificate chain when using certificate auth + --azureblob-disable-checksum Don't store MD5 checksum with object metadata + --azureblob-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8) + --azureblob-endpoint string Endpoint for the service + --azureblob-env-auth Read credentials from runtime (environment variables, CLI or MSI) + --azureblob-key string Storage Account Shared Key + --azureblob-list-chunk int Size of blob list (default 5000) + --azureblob-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --azureblob-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --azureblob-msi-client-id string Object ID of the user-assigned MSI to use, if any + --azureblob-msi-mi-res-id string Azure resource ID of the user-assigned MSI to use, if any + --azureblob-msi-object-id string Object ID of the user-assigned MSI to use, if any + --azureblob-no-check-container If set, don't attempt to check the container exists or create it + --azureblob-no-head-object If set, do not do HEAD before GET when getting objects + --azureblob-password string The user's password (obscured) + --azureblob-public-access string Public access level of a container: blob or container + --azureblob-sas-url string SAS URL for container level access only + --azureblob-service-principal-file string Path to file containing credentials for use with a service principal + --azureblob-tenant string ID of the service principal's tenant. Also called its directory ID + --azureblob-upload-concurrency int Concurrency for multipart uploads (default 16) + --azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256 MiB) (deprecated) + --azureblob-use-emulator Uses local storage emulator if provided as 'true' + --azureblob-use-msi Use a managed service identity to authenticate (only works in Azure) + --azureblob-username string User name (usually an email address) + --b2-account string Account ID or Application Key ID + --b2-chunk-size SizeSuffix Upload chunk size (default 96Mi) + --b2-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4Gi) + --b2-disable-checksum Disable checksums for large (> upload cutoff) files + --b2-download-auth-duration Duration Time before the authorization token will expire in s or suffix ms|s|m|h|d (default 1w) + --b2-download-url string Custom endpoint for downloads + --b2-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --b2-endpoint string Endpoint for the service + --b2-hard-delete Permanently delete files on remote removal, otherwise hide files + --b2-key string Application Key + --b2-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging + --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --b2-version-at Time Show file versions as they were at the specified time (default off) + --b2-versions Include old versions in directory listings + --box-access-token string Box App Primary Access Token + --box-auth-url string Auth server URL + --box-box-config-file string Box App config.json location + --box-box-sub-type string (default "user") + --box-client-id string OAuth Client Id + --box-client-secret string OAuth Client Secret + --box-commit-retries int Max number of times to try committing a multipart file (default 100) + --box-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot) + --box-list-chunk int Size of listing chunk 1-1000 (default 1000) + --box-owned-by string Only show items owned by the login (email address) passed in + --box-root-folder-id string Fill in for rclone to use a non root folder as its starting point + --box-token string OAuth Access Token as a JSON blob + --box-token-url string Token server url + --box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50 MiB) (default 50Mi) + --cache-chunk-clean-interval Duration How often should the cache perform cleanups of the chunk storage (default 1m0s) + --cache-chunk-no-memory Disable the in-memory cache for storing chunks during streaming + --cache-chunk-path string Directory to cache chunk files (default "$HOME/.cache/rclone/cache-backend") + --cache-chunk-size SizeSuffix The size of a chunk (partial file data) (default 5Mi) + --cache-chunk-total-size SizeSuffix The total size that the chunks can take up on the local disk (default 10Gi) + --cache-db-path string Directory to store file structure metadata DB (default "$HOME/.cache/rclone/cache-backend") + --cache-db-purge Clear all the cached data for this remote on start + --cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s) + --cache-info-age Duration How long to cache file structure information (directory listings, file size, times, etc.) (default 6h0m0s) + --cache-plex-insecure string Skip all certificate verification when connecting to the Plex server + --cache-plex-password string The password of the Plex user (obscured) + --cache-plex-url string The URL of the Plex server + --cache-plex-username string The username of the Plex user + --cache-read-retries int How many times to retry a read from a cache storage (default 10) + --cache-remote string Remote to cache + --cache-rps int Limits the number of requests per second to the source FS (-1 to disable) (default -1) + --cache-tmp-upload-path string Directory to keep temporary files until they are uploaded + --cache-tmp-wait-time Duration How long should files be stored in local cache before being uploaded (default 15s) + --cache-workers int How many workers should run in parallel to download chunks (default 4) + --cache-writes Cache file data on writes through the FS + --chunker-chunk-size SizeSuffix Files larger than chunk size will be split in chunks (default 2Gi) + --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks + --chunker-hash-type string Choose how chunker handles hash sums (default "md5") + --chunker-remote string Remote to chunk/unchunk + --combine-upstreams SpaceSepList Upstreams for combining + --compress-level int GZIP compression level (-2 to 9) (default -1) + --compress-mode string Compression mode (default "gzip") + --compress-ram-cache-limit SizeSuffix Some remotes don't allow the upload of files with unknown size (default 20Mi) + --compress-remote string Remote to compress + -L, --copy-links Follow symlinks and copy the pointed to item + --crypt-directory-name-encryption Option to either encrypt directory names or leave them intact (default true) + --crypt-filename-encoding string How to encode the encrypted filename to text string (default "base32") + --crypt-filename-encryption string How to encrypt the filenames (default "standard") + --crypt-no-data-encryption Option to either encrypt file data or leave it unencrypted + --crypt-password string Password or pass phrase for encryption (obscured) + --crypt-password2 string Password or pass phrase for salt (obscured) + --crypt-remote string Remote to encrypt/decrypt + --crypt-server-side-across-configs Allow server-side operations (e.g. copy) to work across different crypt configs + --crypt-show-mapping For all files listed show how the names encrypt + --drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded + --drive-allow-import-name-change Allow the filetype to change when uploading Google docs + --drive-auth-owner-only Only consider files owned by the authenticated user + --drive-auth-url string Auth server URL + --drive-chunk-size SizeSuffix Upload chunk size (default 8Mi) + --drive-client-id string Google Application Client Id + --drive-client-secret string OAuth Client Secret + --drive-copy-shortcut-content Server side copy contents of shortcuts instead of the shortcut + --drive-disable-http2 Disable drive using http2 (default true) + --drive-encoding MultiEncoder The encoding for the backend (default InvalidUtf8) + --drive-export-formats string Comma separated list of preferred formats for downloading Google docs (default "docx,xlsx,pptx,svg") + --drive-formats string Deprecated: See export_formats + --drive-impersonate string Impersonate this user when using a service account + --drive-import-formats string Comma separated list of preferred formats for uploading Google docs + --drive-keep-revision-forever Keep new head revision of each file forever + --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) + --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) + --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) + --drive-resource-key string Resource key for accessing a link-shared file + --drive-root-folder-id string ID of the root folder + --drive-scope string Scope that rclone should use when requesting access from drive + --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs + --drive-service-account-credentials string Service Account Credentials JSON blob + --drive-service-account-file string Service Account Credentials JSON file path + --drive-shared-with-me Only show files that are shared with me + --drive-size-as-quota Show sizes as storage quota usage, not actual size + --drive-skip-checksum-gphotos Skip MD5 checksum on Google photos and videos only + --drive-skip-dangling-shortcuts If set skip dangling shortcut files + --drive-skip-gdocs Skip google documents in all listings + --drive-skip-shortcuts If set skip shortcut files + --drive-starred-only Only show files that are starred + --drive-stop-on-download-limit Make download limit errors be fatal + --drive-stop-on-upload-limit Make upload limit errors be fatal + --drive-team-drive string ID of the Shared Drive (Team Drive) + --drive-token string OAuth Access Token as a JSON blob + --drive-token-url string Token server url + --drive-trashed-only Only show files that are in the trash + --drive-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 8Mi) + --drive-use-created-date Use file created date instead of modified date + --drive-use-shared-date Use date file was shared instead of modified date + --drive-use-trash Send files to the trash instead of deleting permanently (default true) + --drive-v2-download-min-size SizeSuffix If Object's are greater, use drive v2 API to download (default off) + --dropbox-auth-url string Auth server URL + --dropbox-batch-commit-timeout Duration Max time to wait for a batch to finish committing (default 10m0s) + --dropbox-batch-mode string Upload file batching sync|async|off (default "sync") + --dropbox-batch-size int Max number of files in upload batch + --dropbox-batch-timeout Duration Max time to allow an idle upload batch before uploading (default 0s) + --dropbox-chunk-size SizeSuffix Upload chunk size (< 150Mi) (default 48Mi) + --dropbox-client-id string OAuth Client Id + --dropbox-client-secret string OAuth Client Secret + --dropbox-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,RightSpace,InvalidUtf8,Dot) + --dropbox-impersonate string Impersonate this user when using a business account + --dropbox-shared-files Instructs rclone to work on individual shared files + --dropbox-shared-folders Instructs rclone to work on shared folders + --dropbox-token string OAuth Access Token as a JSON blob + --dropbox-token-url string Token server url + --fichier-api-key string Your API Key, get it from https://1fichier.com/console/params.pl + --fichier-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot) + --fichier-file-password string If you want to download a shared file that is password protected, add this parameter (obscured) + --fichier-folder-password string If you want to list the files in a shared folder that is password protected, add this parameter (obscured) + --fichier-shared-folder string If you want to download a shared folder, add this parameter + --filefabric-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) + --filefabric-permanent-token string Permanent Authentication Token + --filefabric-root-folder-id string ID of the root folder + --filefabric-token string Session Token + --filefabric-token-expiry string Token expiry time + --filefabric-url string URL of the Enterprise File Fabric to connect to + --filefabric-version string Version read from the file fabric + --ftp-ask-password Allow asking for FTP password when needed + --ftp-close-timeout Duration Maximum time to wait for a response to close (default 1m0s) + --ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited + --ftp-disable-epsv Disable using EPSV even if server advertises support + --ftp-disable-mlsd Disable using MLSD even if server advertises support + --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) + --ftp-disable-utf8 Disable using UTF-8 even if server advertises support + --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) + --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) + --ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD + --ftp-host string FTP host to connect to + --ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --ftp-no-check-certificate Do not verify the TLS certificate of the server + --ftp-pass string FTP password (obscured) + --ftp-port int FTP port number (default 21) + --ftp-shut-timeout Duration Maximum time to wait for data connection closing status (default 1m0s) + --ftp-tls Use Implicit FTPS (FTP over TLS) + --ftp-tls-cache-size int Size of TLS session cache for all control and data connections (default 32) + --ftp-user string FTP username (default "$USER") + --ftp-writing-mdtm Use MDTM to set modification time (VsFtpd quirk) + --gcs-anonymous Access public buckets and objects without credentials + --gcs-auth-url string Auth server URL + --gcs-bucket-acl string Access Control List for new buckets + --gcs-bucket-policy-only Access checks should use bucket-level IAM policies + --gcs-client-id string OAuth Client Id + --gcs-client-secret string OAuth Client Secret + --gcs-decompress If set this will decompress gzip encoded objects + --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) + --gcs-endpoint string Endpoint for the service + --gcs-location string Location for the newly created buckets + --gcs-no-check-bucket If set, don't attempt to check the bucket exists or create it + --gcs-object-acl string Access Control List for new objects + --gcs-project-number string Project number + --gcs-service-account-file string Service Account Credentials JSON file path + --gcs-storage-class string The storage class to use when storing objects in Google Cloud Storage + --gcs-token string OAuth Access Token as a JSON blob + --gcs-token-url string Token server url + --gphotos-auth-url string Auth server URL + --gphotos-client-id string OAuth Client Id + --gphotos-client-secret string OAuth Client Secret + --gphotos-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) + --gphotos-include-archived Also view and download archived media + --gphotos-read-only Set to make the Google Photos backend read only + --gphotos-read-size Set to read the size of media items + --gphotos-start-year int Year limits the photos to be downloaded to those which are uploaded after the given year (default 2000) + --gphotos-token string OAuth Access Token as a JSON blob + --gphotos-token-url string Token server url + --hasher-auto-size SizeSuffix Auto-update checksum for files smaller than this size (disabled by default) + --hasher-hashes CommaSepList Comma separated list of supported checksum types (default md5,sha1) + --hasher-max-age Duration Maximum time to keep checksums in cache (0 = no cache, off = cache forever) (default off) + --hasher-remote string Remote to cache checksums for (e.g. myRemote:path) + --hdfs-data-transfer-protection string Kerberos data transfer protection: authentication|integrity|privacy + --hdfs-encoding MultiEncoder The encoding for the backend (default Slash,Colon,Del,Ctl,InvalidUtf8,Dot) + --hdfs-namenode string Hadoop name node and port + --hdfs-service-principal-name string Kerberos service principal name for the namenode + --hdfs-username string Hadoop user name + --hidrive-auth-url string Auth server URL + --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) + --hidrive-client-id string OAuth Client Id + --hidrive-client-secret string OAuth Client Secret + --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary + --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --hidrive-endpoint string Endpoint for the service (default "https://api.hidrive.strato.com/2.1") + --hidrive-root-prefix string The root/parent folder for all paths (default "/") + --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default "rw") + --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default "user") + --hidrive-token string OAuth Access Token as a JSON blob + --hidrive-token-url string Token server url + --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) + --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) + --http-headers CommaSepList Set HTTP headers for all transactions + --http-no-head Don't use HEAD requests + --http-no-slash Set this if the site doesn't end directories with / + --http-url string URL of HTTP host to connect to + --internetarchive-access-key-id string IAS3 Access Key + --internetarchive-disable-checksum Don't ask the server to test against MD5 checksum calculated by rclone (default true) + --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) + --internetarchive-endpoint string IAS3 Endpoint (default "https://s3.us.archive.org") + --internetarchive-front-endpoint string Host of InternetArchive Frontend (default "https://archive.org") + --internetarchive-secret-access-key string IAS3 Secret Key (password) + --internetarchive-wait-archive Duration Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s) + --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) + --jottacloud-hard-delete Delete files permanently rather than putting them into the trash + --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) + --jottacloud-no-versions Avoid server side versioning by deleting files and recreating files instead of overwriting them + --jottacloud-trashed-only Only show files that are in the trash + --jottacloud-upload-resume-limit SizeSuffix Files bigger than this can be resumed if the upload fail's (default 10Mi) + --koofr-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --koofr-endpoint string The Koofr API endpoint to use + --koofr-mountid string Mount ID of the mount to use + --koofr-password string Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured) + --koofr-provider string Choose your storage provider + --koofr-setmtime Does the backend support setting modification time (default true) + --koofr-user string Your user name + -l, --links Translate symlinks to/from regular files with a '.rclonelink' extension + --local-case-insensitive Force the filesystem to report itself as case insensitive + --local-case-sensitive Force the filesystem to report itself as case sensitive + --local-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --local-no-check-updated Don't check to see if the files change during upload + --local-no-preallocate Disable preallocation of disk space for transferred files + --local-no-set-modtime Disable setting modtime + --local-no-sparse Disable sparse files for multi-thread downloads + --local-nounc Disable UNC (long path names) conversion on Windows + --local-unicode-normalization Apply unicode NFC normalization to paths and filenames + --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) + --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) + --mailru-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --mailru-pass string Password (obscured) + --mailru-speedup-enable Skip full upload if there is another file with same data hash (default true) + --mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash) (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf") + --mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3Gi) + --mailru-speedup-max-memory SizeSuffix Files larger than the size given below will always be hashed on disk (default 32Mi) + --mailru-user string User name (usually email) + --mega-debug Output more debug from Mega + --mega-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --mega-hard-delete Delete files permanently rather than putting them into the trash + --mega-pass string Password (obscured) + --mega-user string User name + --netstorage-account string Set the NetStorage account name + --netstorage-host string Domain+path of NetStorage host to connect to + --netstorage-protocol string Select between HTTP or HTTPS protocol (default "https") + --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) + -x, --one-file-system Don't cross filesystem boundaries (unix/macOS only) + --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) + --onedrive-auth-url string Auth server URL + --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) + --onedrive-client-id string OAuth Client Id + --onedrive-client-secret string OAuth Client Secret + --onedrive-drive-id string The ID of the drive to use + --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) + --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) + --onedrive-expose-onenote-files Set to make OneNote files show up in directory listings + --onedrive-link-password string Set the password for links created by the link command + --onedrive-link-scope string Set the scope of the links created by the link command (default "anonymous") + --onedrive-link-type string Set the type of the links created by the link command (default "view") + --onedrive-list-chunk int Size of listing chunk (default 1000) + --onedrive-no-versions Remove all versions on modifying operations + --onedrive-region string Choose national cloud region for OneDrive (default "global") + --onedrive-root-folder-id string ID of the root folder + --onedrive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different onedrive configs + --onedrive-token string OAuth Access Token as a JSON blob + --onedrive-token-url string Token server url + --oos-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) + --oos-compartment string Object storage compartment OCID + --oos-config-file string Path to OCI config file (default "~/.oci/config") + --oos-config-profile string Profile name inside the oci config file (default "Default") + --oos-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) + --oos-copy-timeout Duration Timeout for copy (default 1m0s) + --oos-disable-checksum Don't store MD5 checksum with object metadata + --oos-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --oos-endpoint string Endpoint for Object storage API + --oos-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery + --oos-namespace string Object storage namespace + --oos-no-check-bucket If set, don't attempt to check the bucket exists or create it + --oos-provider string Choose your Auth Provider (default "env_auth") + --oos-region string Object storage Region + --oos-upload-concurrency int Concurrency for multipart uploads (default 10) + --oos-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size (default 10Mi) + --opendrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot) + --opendrive-password string Password (obscured) + --opendrive-username string Username + --pcloud-auth-url string Auth server URL + --pcloud-client-id string OAuth Client Id + --pcloud-client-secret string OAuth Client Secret + --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --pcloud-hostname string Hostname to connect to (default "api.pcloud.com") + --pcloud-password string Your pcloud password (obscured) + --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default "d0") + --pcloud-token string OAuth Access Token as a JSON blob + --pcloud-token-url string Token server url + --pcloud-username string Your pcloud username + --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --qingstor-access-key-id string QingStor Access Key ID + --qingstor-chunk-size SizeSuffix Chunk size to use for uploading (default 4Mi) + --qingstor-connection-retries int Number of connection retries (default 3) + --qingstor-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8) + --qingstor-endpoint string Enter an endpoint URL to connection QingStor API + --qingstor-env-auth Get QingStor credentials from runtime + --qingstor-secret-access-key string QingStor Secret Access Key (password) + --qingstor-upload-concurrency int Concurrency for multipart uploads (default 1) + --qingstor-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --qingstor-zone string Zone to connect to + --s3-access-key-id string AWS Access Key ID + --s3-acl string Canned ACL used when creating buckets and storing or copying objects + --s3-bucket-acl string Canned ACL used when creating buckets + --s3-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) + --s3-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) + --s3-decompress If set this will decompress gzip encoded objects + --s3-disable-checksum Don't store MD5 checksum with object metadata + --s3-disable-http2 Disable usage of http2 for S3 backends + --s3-download-url string Custom endpoint for downloads + --s3-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --s3-endpoint string Endpoint for S3 API + --s3-env-auth Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars) + --s3-force-path-style If true use path style access if false use virtual hosted style (default true) + --s3-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery + --s3-list-chunk int Size of listing chunk (response list for each ListObject S3 request) (default 1000) + --s3-list-url-encode Tristate Whether to url encode listings: true/false/unset (default unset) + --s3-list-version int Version of ListObjects to use: 1,2 or 0 for auto + --s3-location-constraint string Location constraint - must be set to match the Region + --s3-max-upload-parts int Maximum number of parts in a multipart upload (default 10000) + --s3-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --s3-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --s3-might-gzip Tristate Set this if the backend might gzip objects (default unset) + --s3-no-check-bucket If set, don't attempt to check the bucket exists or create it + --s3-no-head If set, don't HEAD uploaded objects to check integrity + --s3-no-head-object If set, do not do HEAD before GET when getting objects + --s3-no-system-metadata Suppress setting and reading of system metadata + --s3-profile string Profile to use in the shared credentials file + --s3-provider string Choose your S3 provider + --s3-region string Region to connect to + --s3-requester-pays Enables requester pays option when interacting with S3 bucket + --s3-secret-access-key string AWS Secret Access Key (password) + --s3-server-side-encryption string The server-side encryption algorithm used when storing this object in S3 + --s3-session-token string An AWS session token + --s3-shared-credentials-file string Path to the shared credentials file + --s3-sse-customer-algorithm string If using SSE-C, the server-side encryption algorithm used when storing this object in S3 + --s3-sse-customer-key string To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data + --s3-sse-customer-key-base64 string If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data + --s3-sse-customer-key-md5 string If using SSE-C you may provide the secret encryption key MD5 checksum (optional) + --s3-sse-kms-key-id string If using KMS ID you must provide the ARN of Key + --s3-storage-class string The storage class to use when storing new objects in S3 + --s3-upload-concurrency int Concurrency for multipart uploads (default 4) + --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint + --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) + --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads + --s3-v2-auth If true use v2 authentication + --s3-version-at Time Show file versions as they were at the specified time (default off) + --s3-versions Include old versions in directory listings + --seafile-2fa Two-factor authentication ('true' if the account has 2FA enabled) + --seafile-create-library Should rclone create a library if it doesn't exist + --seafile-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8) + --seafile-library string Name of the library + --seafile-library-key string Library password (for encrypted libraries only) (obscured) + --seafile-pass string Password (obscured) + --seafile-url string URL of seafile host to connect to + --seafile-user string User name (usually email address) + --sftp-ask-password Allow asking for SFTP password when needed + --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) + --sftp-ciphers SpaceSepList Space separated list of ciphers to be used for session encryption, ordered by preference + --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) + --sftp-disable-concurrent-reads If set don't use concurrent reads + --sftp-disable-concurrent-writes If set don't use concurrent writes + --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available + --sftp-host string SSH host to connect to + --sftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --sftp-key-exchange SpaceSepList Space separated list of key exchange algorithms, ordered by preference + --sftp-key-file string Path to PEM-encoded private key file + --sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file (obscured) + --sftp-key-pem string Raw PEM-encoded private key + --sftp-key-use-agent When set forces the usage of the ssh-agent + --sftp-known-hosts-file string Optional path to known_hosts file + --sftp-macs SpaceSepList Space separated list of MACs (message authentication code) algorithms, ordered by preference + --sftp-md5sum-command string The command used to read md5 hashes + --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) + --sftp-path-override string Override path used by SSH shell commands + --sftp-port int SSH port number (default 22) + --sftp-pubkey-file string Optional path to public key file + --sftp-server-command string Specifies the path or command to run a sftp server on the remote host + --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands + --sftp-set-modtime Set the modified time on the remote if set (default true) + --sftp-sha1sum-command string The command used to read sha1 hashes + --sftp-shell-type string The type of SSH shell on remote server, if any + --sftp-skip-links Set to skip any symlinks and any other non regular files + --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default "sftp") + --sftp-use-fstat If set use fstat instead of stat + --sftp-use-insecure-cipher Enable the use of insecure ciphers and key exchange methods + --sftp-user string SSH username (default "$USER") + --sharefile-chunk-size SizeSuffix Upload chunk size (default 64Mi) + --sharefile-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,LeftSpace,LeftPeriod,RightSpace,RightPeriod,InvalidUtf8,Dot) + --sharefile-endpoint string Endpoint for API calls + --sharefile-root-folder-id string ID of the root folder + --sharefile-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (default 128Mi) + --sia-api-password string Sia Daemon API Password (obscured) + --sia-api-url string Sia daemon API URL, like http://sia.daemon.host:9980 (default "http://127.0.0.1:9980") + --sia-encoding MultiEncoder The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot) + --sia-user-agent string Siad User Agent (default "Sia-Agent") + --skip-links Don't warn about skipped symlinks + --smb-case-insensitive Whether the server is configured to be case-insensitive (default true) + --smb-domain string Domain name for NTLM authentication (default "WORKGROUP") + --smb-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot) + --smb-hide-special-share Hide special shares (e.g. print$) which users aren't supposed to access (default true) + --smb-host string SMB server hostname to connect to + --smb-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --smb-pass string SMB password (obscured) + --smb-port int SMB port number (default 445) + --smb-user string SMB username (default "$USER") + --storj-access-grant string Access grant + --storj-api-key string API key + --storj-passphrase string Encryption passphrase + --storj-provider string Choose an authentication method (default "existing") + --storj-satellite-address string Satellite address (default "us-central-1.storj.io") + --sugarsync-access-key-id string Sugarsync Access Key ID + --sugarsync-app-id string Sugarsync App ID + --sugarsync-authorization string Sugarsync authorization + --sugarsync-authorization-expiry string Sugarsync authorization expiry + --sugarsync-deleted-id string Sugarsync deleted folder id + --sugarsync-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8,Dot) + --sugarsync-hard-delete Permanently delete files if true + --sugarsync-private-access-key string Sugarsync Private Access Key + --sugarsync-refresh-token string Sugarsync refresh token + --sugarsync-root-id string Sugarsync root id + --sugarsync-user string Sugarsync user + --swift-application-credential-id string Application Credential ID (OS_APPLICATION_CREDENTIAL_ID) + --swift-application-credential-name string Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME) + --swift-application-credential-secret string Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET) + --swift-auth string Authentication URL for server (OS_AUTH_URL) + --swift-auth-token string Auth Token from alternate authentication - optional (OS_AUTH_TOKEN) + --swift-auth-version int AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION) + --swift-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) + --swift-domain string User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME) + --swift-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8) + --swift-endpoint-type string Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default "public") + --swift-env-auth Get swift credentials from environment variables in standard OpenStack form + --swift-key string API key or password (OS_PASSWORD) + --swift-leave-parts-on-error If true avoid calling abort upload on a failure + --swift-no-chunk Don't chunk files during streaming upload + --swift-no-large-objects Disable support for static and dynamic large objects + --swift-region string Region name - optional (OS_REGION_NAME) + --swift-storage-policy string The storage policy to use when creating a new container + --swift-storage-url string Storage URL - optional (OS_STORAGE_URL) + --swift-tenant string Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME) + --swift-tenant-domain string Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME) + --swift-tenant-id string Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID) + --swift-user string User name to log in (OS_USERNAME) + --swift-user-id string User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID) + --union-action-policy string Policy to choose upstream on ACTION category (default "epall") + --union-cache-time int Cache time of usage and free space (in seconds) (default 120) + --union-create-policy string Policy to choose upstream on CREATE category (default "epmfs") + --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) + --union-search-policy string Policy to choose upstream on SEARCH category (default "ff") + --union-upstreams string List of space separated upstreams + --uptobox-access-token string Your access token + --uptobox-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot) + --webdav-bearer-token string Bearer token instead of user/pass (e.g. a Macaroon) + --webdav-bearer-token-command string Command to run to get a bearer token + --webdav-encoding string The encoding for the backend + --webdav-headers CommaSepList Set HTTP headers for all transactions + --webdav-pass string Password (obscured) + --webdav-url string URL of http host to connect to + --webdav-user string User name + --webdav-vendor string Name of the WebDAV site/service/software you are using + --yandex-auth-url string Auth server URL + --yandex-client-id string OAuth Client Id + --yandex-client-secret string OAuth Client Secret + --yandex-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) + --yandex-hard-delete Delete files permanently rather than putting them into the trash + --yandex-token string OAuth Access Token as a JSON blob + --yandex-token-url string Token server url + --zoho-auth-url string Auth server URL + --zoho-client-id string OAuth Client Id + --zoho-client-secret string OAuth Client Secret + --zoho-encoding MultiEncoder The encoding for the backend (default Del,Ctl,InvalidUtf8) + --zoho-region string Zoho region to connect to + --zoho-token string OAuth Access Token as a JSON blob + --zoho-token-url string Token server url ``` diff --git a/docs/content/ftp.md b/docs/content/ftp.md index 563de41322ea4..ca919fd377985 100644 --- a/docs/content/ftp.md +++ b/docs/content/ftp.md @@ -217,7 +217,7 @@ Use Implicit FTPS (FTP over TLS). When using implicit FTP over TLS the client connects using TLS right from the start which breaks compatibility with non-TLS-aware servers. This is usually served over port 990 rather -than port 21. Cannot be used in combination with explicit FTP. +than port 21. Cannot be used in combination with explicit FTPS. Properties: @@ -232,7 +232,7 @@ Use Explicit FTPS (FTP over TLS). When using explicit FTP over TLS the client explicitly requests security from the server in order to upgrade a plain text connection -to an encrypted one. Cannot be used in combination with implicit FTP. +to an encrypted one. Cannot be used in combination with implicit FTPS. Properties: diff --git a/docs/content/local.md b/docs/content/local.md index 70fbf9315f468..15c577e83f3a2 100644 --- a/docs/content/local.md +++ b/docs/content/local.md @@ -429,7 +429,7 @@ Properties: Don't check to see if the files change during upload. Normally rclone checks the size and modification time of files as they -are being uploaded and aborts with a message which starts "can't copy - +are being uploaded and aborts with a message which starts "can't copy - source file is being updated" if the file changes during upload. However on some file systems this modification time check may fail (e.g. diff --git a/docs/content/mailru.md b/docs/content/mailru.md index 758cb4b9e5d5f..0f2f106c265df 100644 --- a/docs/content/mailru.md +++ b/docs/content/mailru.md @@ -191,6 +191,11 @@ Properties: Password. +This must be an app password - rclone will not work with your normal +password. See the Configuration section in the docs for how to make an +app password. + + **NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/). Properties: diff --git a/docs/content/rc.md b/docs/content/rc.md index 3143b956d8b06..1cb4dd55ba9f6 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -616,6 +616,14 @@ See the [config providers](/commands/rclone_config_providers/) command for more **Authentication is required for this call.** +### config/setpath: Set the path of the config file {#config-setpath} + +Parameters: + +- path - path to the config file to use + +**Authentication is required for this call.** + ### config/update: update the config for a remote. {#config-update} This takes the following parameters: @@ -714,7 +722,7 @@ Returns: "result": "" } -OR +OR { "error": true, "result": "" @@ -909,6 +917,22 @@ Parameters: - rate - int +### debug/set-gc-percent: Call runtime/debug.SetGCPercent for setting the garbage collection target percentage. {#debug-set-gc-percent} + +SetGCPercent sets the garbage collection target percentage: a collection is triggered +when the ratio of freshly allocated data to live data remaining after the previous collection +reaches this percentage. SetGCPercent returns the previous setting. The initial setting is the +value of the GOGC environment variable at startup, or 100 if the variable is not set. + +This setting may be effectively reduced in order to maintain a memory limit. +A negative percentage effectively disables garbage collection, unless the memory limit is reached. + +See https://pkg.go.dev/runtime/debug#SetMemoryLimit for more details. + +Parameters: + +- gc-percent - int + ### debug/set-mutex-profile-fraction: Set runtime.SetMutexProfileFraction for mutex profiling. {#debug-set-mutex-profile-fraction} SetMutexProfileFraction controls the fraction of mutex contention @@ -930,6 +954,38 @@ Results: - previousRate - int +### debug/set-soft-memory-limit: Call runtime/debug.SetMemoryLimit for setting a soft memory limit for the runtime. {#debug-set-soft-memory-limit} + +SetMemoryLimit provides the runtime with a soft memory limit. + +The runtime undertakes several processes to try to respect this memory limit, including +adjustments to the frequency of garbage collections and returning memory to the underlying +system more aggressively. This limit will be respected even if GOGC=off (or, if SetGCPercent(-1) is executed). + +The input limit is provided as bytes, and includes all memory mapped, managed, and not +released by the Go runtime. Notably, it does not account for space used by the Go binary +and memory external to Go, such as memory managed by the underlying system on behalf of +the process, or memory managed by non-Go code inside the same process. +Examples of excluded memory sources include: OS kernel memory held on behalf of the process, +memory allocated by C code, and memory mapped by syscall.Mmap (because it is not managed by the Go runtime). + +A zero limit or a limit that's lower than the amount of memory used by the Go runtime may cause +the garbage collector to run nearly continuously. However, the application may still make progress. + +The memory limit is always respected by the Go runtime, so to effectively disable this behavior, +set the limit very high. math.MaxInt64 is the canonical value for disabling the limit, but values +much greater than the available memory on the underlying system work just as well. + +See https://go.dev/doc/gc-guide for a detailed guide explaining the soft memory limit in more detail, +as well as a variety of common use-cases and scenarios. + +SetMemoryLimit returns the previously set memory limit. A negative input does not adjust the limit, +and allows for retrieval of the currently set memory limit. + +Parameters: + +- mem-limit - int + ### fscache/clear: Clear the Fs cache. {#fscache-clear} This clears the fs cache. This is where remotes created from backends diff --git a/docs/content/s3.md b/docs/content/s3.md index 34e34230fdb07..172365e24c69d 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -643,7 +643,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}} ### Standard options -Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). +Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). #### --s3-provider @@ -682,6 +682,8 @@ Properties: - IONOS Cloud - "LyveCloud" - Seagate Lyve Cloud + - "Liara" + - Liara Object Storage - "Minio" - Minio Object Storage - "Netease" @@ -1033,7 +1035,7 @@ Properties: - Config: region - Env Var: RCLONE_S3_REGION -- Provider: !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive +- Provider: !AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Liara,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive - Type: string - Required: false - Examples: @@ -1312,6 +1314,22 @@ Properties: #### --s3-endpoint +Endpoint for Liara Object Storage API. + +Properties: + +- Config: endpoint +- Env Var: RCLONE_S3_ENDPOINT +- Provider: Liara +- Type: string +- Required: false +- Examples: + - "storage.iran.liara.space" + - The default endpoint + - Iran + +#### --s3-endpoint + Endpoint for OSS API. Properties: @@ -1612,7 +1630,7 @@ Properties: - Config: endpoint - Env Var: RCLONE_S3_ENDPOINT -- Provider: !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu +- Provider: !AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu - Type: string - Required: false - Examples: @@ -1639,15 +1657,33 @@ Properties: - "s3.ap-southeast-1.lyvecloud.seagate.com" - Seagate Lyve Cloud AP Southeast 1 (Singapore) - "s3.wasabisys.com" - - Wasabi US East endpoint + - Wasabi US East 1 (N. Virginia) + - "s3.us-east-2.wasabisys.com" + - Wasabi US East 2 (N. Virginia) + - "s3.us-central-1.wasabisys.com" + - Wasabi US Central 1 (Texas) - "s3.us-west-1.wasabisys.com" - - Wasabi US West endpoint + - Wasabi US West 1 (Oregon) + - "s3.ca-central-1.wasabisys.com" + - Wasabi CA Central 1 (Toronto) - "s3.eu-central-1.wasabisys.com" - - Wasabi EU Central endpoint + - Wasabi EU Central 1 (Amsterdam) + - "s3.eu-central-2.wasabisys.com" + - Wasabi EU Central 2 (Frankfurt) + - "s3.eu-west-1.wasabisys.com" + - Wasabi EU West 1 (London) + - "s3.eu-west-2.wasabisys.com" + - Wasabi EU West 2 (Paris) - "s3.ap-northeast-1.wasabisys.com" - Wasabi AP Northeast 1 (Tokyo) endpoint - "s3.ap-northeast-2.wasabisys.com" - Wasabi AP Northeast 2 (Osaka) endpoint + - "s3.ap-southeast-1.wasabisys.com" + - Wasabi AP Southeast 1 (Singapore) + - "s3.ap-southeast-2.wasabisys.com" + - Wasabi AP Southeast 2 (Sydney) + - "storage.iran.liara.space" + - Liara Iran endpoint - "s3.ir-thr-at1.arvanstorage.com" - ArvanCloud Tehran Iran (Asiatech) endpoint @@ -1980,7 +2016,7 @@ Properties: - Config: location_constraint - Env Var: RCLONE_S3_LOCATION_CONSTRAINT -- Provider: !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS +- Provider: !AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Liara,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS - Type: string - Required: false @@ -1995,6 +2031,10 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview Note that this ACL is applied when server-side copying objects as S3 doesn't copy the ACL from the source but rather writes a fresh one. +If the acl is an empty string then no X-Amz-Acl: header is added and +the default (private) will be used. + + Properties: - Config: acl @@ -2155,6 +2195,21 @@ Properties: #### --s3-storage-class +The storage class to use when storing new objects in Liara + +Properties: + +- Config: storage_class +- Env Var: RCLONE_S3_STORAGE_CLASS +- Provider: Liara +- Type: string +- Required: false +- Examples: + - "STANDARD" + - Standard storage class + +#### --s3-storage-class + The storage class to use when storing new objects in ArvanCloud. Properties: @@ -2233,7 +2288,7 @@ Properties: ### Advanced options -Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). +Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). #### --s3-bucket-acl @@ -2244,6 +2299,10 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview Note that this ACL is applied when only when creating buckets. If it isn't set then "acl" is used instead. +If the "acl" and "bucket_acl" are empty strings then no X-Amz-Acl: +header is added and the default (private) will be used. + + Properties: - Config: bucket_acl @@ -2866,6 +2925,37 @@ Properties: - Type: bool - Default: false +#### --s3-might-gzip + +Set this if the backend might gzip objects. + +Normally providers will not alter objects when they are downloaded. If +an object was not uploaded with `Content-Encoding: gzip` then it won't +be set on download. + +However some providers may gzip objects even if they weren't uploaded +with `Content-Encoding: gzip` (eg Cloudflare). + +A symptom of this would be receiving errors like + + ERROR corrupted on transfer: sizes differ NNN vs MMM + +If you set this flag and rclone downloads an object with +Content-Encoding: gzip set and chunked transfer encoding, then rclone +will decompress the object on the fly. + +If this is set to unset (the default) then rclone will choose +according to the provider setting what to apply, but you can override +rclone's choice here. + + +Properties: + +- Config: might_gzip +- Env Var: RCLONE_S3_MIGHT_GZIP +- Type: Tristate +- Default: unset + #### --s3-no-system-metadata Suppress setting and reading of system metadata diff --git a/docs/content/sftp.md b/docs/content/sftp.md index bbbbcf7637fc5..d7b23ee0f34ec 100644 --- a/docs/content/sftp.md +++ b/docs/content/sftp.md @@ -528,6 +528,9 @@ This enables the use of the following insecure ciphers and key exchange methods: Those algorithms are insecure and may allow plaintext data to be recovered by an attacker. +This must be false if you use either ciphers or key_exchange advanced options. + + Properties: - Config: use_insecure_cipher @@ -860,6 +863,64 @@ Properties: - Type: SpaceSepList - Default: +#### --sftp-ciphers + +Space separated list of ciphers to be used for session encryption, ordered by preference. + +At least one must match with server configuration. This can be checked for example using ssh -Q cipher. + +This must not be set if use_insecure_cipher is true. + +Example: + + aes128-ctr aes192-ctr aes256-ctr aes128-gcm@openssh.com aes256-gcm@openssh.com + + +Properties: + +- Config: ciphers +- Env Var: RCLONE_SFTP_CIPHERS +- Type: SpaceSepList +- Default: + +#### --sftp-key-exchange + +Space separated list of key exchange algorithms, ordered by preference. + +At least one must match with server configuration. This can be checked for example using ssh -Q kex. + +This must not be set if use_insecure_cipher is true. + +Example: + + sntrup761x25519-sha512@openssh.com curve25519-sha256 curve25519-sha256@libssh.org ecdh-sha2-nistp256 + + +Properties: + +- Config: key_exchange +- Env Var: RCLONE_SFTP_KEY_EXCHANGE +- Type: SpaceSepList +- Default: + +#### --sftp-macs + +Space separated list of MACs (message authentication code) algorithms, ordered by preference. + +At least one must match with server configuration. This can be checked for example using ssh -Q mac. + +Example: + + umac-64-etm@openssh.com umac-128-etm@openssh.com hmac-sha2-256-etm@openssh.com + + +Properties: + +- Config: macs +- Env Var: RCLONE_SFTP_MACS +- Type: SpaceSepList +- Default: + {{< rem autogenerated options stop >}} ## Limitations diff --git a/go.mod b/go.mod index 873b24fb978e5..8cf92794b4cc6 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3 github.com/abbot/go-http-auth v0.4.0 github.com/anacrolix/dms v1.5.0 + github.com/anacrolix/log v0.13.1 github.com/artyom/mtab v1.0.0 github.com/atotto/clipboard v0.1.4 github.com/aws/aws-sdk-go v1.44.145 @@ -48,6 +49,7 @@ require ( github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6 github.com/rfjakob/eme v1.1.2 + github.com/rivo/uniseg v0.2.0 github.com/shirou/gopsutil/v3 v3.22.10 github.com/sirupsen/logrus v1.9.0 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 @@ -78,7 +80,6 @@ require ( cloud.google.com/go/compute/metadata v0.2.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect - github.com/anacrolix/log v0.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/calebcase/tmpfile v1.0.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -116,7 +117,6 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - github.com/rivo/uniseg v0.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/spacemonkeygo/monkit/v3 v3.0.17 // indirect diff --git a/go.sum b/go.sum index 38a637db50897..e1e85c49cb0c3 100644 --- a/go.sum +++ b/go.sum @@ -1033,7 +1033,6 @@ golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= @@ -1175,7 +1174,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1183,7 +1181,6 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= @@ -1197,7 +1194,6 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= diff --git a/rclone.1 b/rclone.1 index fd0219cddcc37..2b14cd4ffceba 100644 --- a/rclone.1 +++ b/rclone.1 @@ -1,7 +1,7 @@ .\"t .\" Automatically generated by Pandoc 2.9.2.1 .\" -.TH "rclone" "1" "Oct 21, 2022" "User Manual" "" +.TH "rclone" "1" "Dec 20, 2022" "User Manual" "" .hy .SH Rclone syncs your files to cloud storage .PP @@ -203,6 +203,8 @@ IONOS Cloud .IP \[bu] 2 Koofr .IP \[bu] 2 +Liara Object Storage +.IP \[bu] 2 Mail.ru Cloud .IP \[bu] 2 Memset Memstore @@ -490,13 +492,13 @@ links. If not sure, use the first link. .IP \[bu] 2 Intel/AMD - 64 -Bit (https://downloads.rclone.org/rclone-current-linux-amd64.zip) +Bit (https://downloads.rclone.org/rclone-current-windows-amd64.zip) .IP \[bu] 2 Intel/AMD - 32 -Bit (https://downloads.rclone.org/rclone-current-linux-386.zip) +Bit (https://downloads.rclone.org/rclone-current-windows-386.zip) .IP \[bu] 2 ARM - 64 -Bit (https://downloads.rclone.org/rclone-current-linux-arm64.zip) +Bit (https://downloads.rclone.org/rclone-current-windows-arm64.zip) .PP Open this file in the Explorer and extract \f[C]rclone.exe\f[R]. Rclone is a portable executable so you can place it wherever is @@ -3035,7 +3037,7 @@ This will look something like (some irrelevant detail removed): \[dq]State\[dq]: \[dq]*oauth-islocal,teamdrive,,\[dq], \[dq]Option\[dq]: { \[dq]Name\[dq]: \[dq]config_is_local\[dq], - \[dq]Help\[dq]: \[dq]Use auto config?\[rs]n * Say Y if not sure\[rs]n * Say N if you are working on a remote or headless machine\[rs]n\[dq], + \[dq]Help\[dq]: \[dq]Use web browser to automatically authenticate rclone with remote?\[rs]n * Say Y if the machine running rclone has a web browser you can use\[rs]n * Say N if running rclone on a (remote) machine without web browser access\[rs]nIf not sure try Y. If Y failed, try N.\[rs]n\[dq], \[dq]Default\[dq]: true, \[dq]Examples\[dq]: [ { @@ -3490,7 +3492,7 @@ This will look something like (some irrelevant detail removed): \[dq]State\[dq]: \[dq]*oauth-islocal,teamdrive,,\[dq], \[dq]Option\[dq]: { \[dq]Name\[dq]: \[dq]config_is_local\[dq], - \[dq]Help\[dq]: \[dq]Use auto config?\[rs]n * Say Y if not sure\[rs]n * Say N if you are working on a remote or headless machine\[rs]n\[dq], + \[dq]Help\[dq]: \[dq]Use web browser to automatically authenticate rclone with remote?\[rs]n * Say Y if the machine running rclone has a web browser you can use\[rs]n * Say N if running rclone on a (remote) machine without web browser access\[rs]nIf not sure try Y. If Y failed, try N.\[rs]n\[dq], \[dq]Default\[dq]: true, \[dq]Examples\[dq]: [ { @@ -4605,6 +4607,7 @@ rclone lsjson remote:path [flags] --hash Include hashes in the output (may take longer) --hash-type stringArray Show only this hash type (may be repeated) -h, --help help for lsjson + -M, --metadata Add metadata to the listing --no-mimetype Don\[aq]t read the mime type (can speed things up) --no-modtime Don\[aq]t read the modification time (can speed things up) --original Show the ID of the underlying Object @@ -5518,14 +5521,14 @@ rclone mount remote:path /path/to/mountpoint [flags] --allow-other Allow access to other users (not supported on Windows) --allow-root Allow access to root user (not supported on Windows) --async-read Use asynchronous reads (not supported on Windows) (default true) - --attr-timeout duration Time for which file/directory attributes are cached (default 1s) + --attr-timeout Duration Time for which file/directory attributes are cached (default 1s) --daemon Run mount in background and exit parent process (as background output is suppressed, use --log-file with --log-format=pid,... to monitor) (not supported on Windows) - --daemon-timeout duration Time limit for rclone to respond to kernel (not supported on Windows) - --daemon-wait duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) + --daemon-timeout Duration Time limit for rclone to respond to kernel (not supported on Windows) (default 0s) + --daemon-wait Duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) --debug-fuse Debug the FUSE internals - needs -v --default-permissions Makes kernel enforce access control based on the file mode (not supported on Windows) --devname string Set the device name - default is remote:path - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --fuse-flag stringArray Flags or arguments to be passed direct to libfuse/WinFsp (repeat if required) @@ -5539,24 +5542,24 @@ rclone mount remote:path /path/to/mountpoint [flags] --noappledouble Ignore Apple Double (._) and .DS_Store files (supported on OSX only) (default true) --noapplexattr Ignore all \[dq]com.apple.*\[dq] extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) --volname string Set the volume name (supported on Windows and OSX only) --write-back-cache Makes kernel buffer writes before sending them to rclone (without this, writethrough caching is used) (not supported on Windows) \f[R] @@ -5657,11 +5660,12 @@ The supported keys are: \[ua],\[da] or k,j to Move \[->],l to enter \[<-],h to return - c toggle counts g toggle graph + c toggle counts a toggle average size in directory + m toggle modified time u toggle human-readable format - n,s,C,A sort by name,size,count,average size + n,s,C,A,M sort by name,size,count,asize,mtime d delete file/directory v select file/directory V enter visual select mode @@ -5965,6 +5969,186 @@ It will also open the URL in the browser when rclone is run. .PP See the rc documentation (https://rclone.org/rc/) for more info on the rc flags. +.SS Server options +.PP +Use \f[C]--addr\f[R] to specify which IP address and port the server +should listen on, eg \f[C]--addr 1.2.3.4:8000\f[R] or +\f[C]--addr :8080\f[R] to listen to all IPs. +By default it only listens on localhost. +You can use port :0 to let the OS choose an available port. +.PP +If you set \f[C]--addr\f[R] to listen on a public or LAN accessible IP +address then using Authentication is advised - see the next section for +info. +.PP +You can use a unix socket by setting the url to +\f[C]unix:///path/to/socket\f[R] or just by using an absolute path name. +Note that unix sockets bypass the authentication - this is expected to +be done with file system permissions. +.PP +\f[C]--addr\f[R] may be repeated to listen on multiple +IPs/ports/sockets. +.PP +\f[C]--server-read-timeout\f[R] and \f[C]--server-write-timeout\f[R] can +be used to control the timeouts on the server. +Note that this is the total time for a transfer. +.PP +\f[C]--max-header-bytes\f[R] controls the maximum number of bytes the +server will accept in the HTTP header. +.PP +\f[C]--baseurl\f[R] controls the URL prefix that rclone serves from. +By default rclone will serve from the root. +If you used \f[C]--baseurl \[dq]/rclone\[dq]\f[R] then rclone would +serve from a URL starting with \[dq]/rclone/\[dq]. +This is useful if you wish to proxy rclone serve. +Rclone automatically inserts leading and trailing \[dq]/\[dq] on +\f[C]--baseurl\f[R], so \f[C]--baseurl \[dq]rclone\[dq]\f[R], +\f[C]--baseurl \[dq]/rclone\[dq]\f[R] and +\f[C]--baseurl \[dq]/rclone/\[dq]\f[R] are all treated identically. +.SS TLS (SSL) +.PP +By default this will serve over http. +If you want you can serve over https. +You will need to supply the \f[C]--cert\f[R] and \f[C]--key\f[R] flags. +If you wish to do client side certificate validation then you will need +to supply \f[C]--client-ca\f[R] also. +.PP +\f[C]--cert\f[R] should be a either a PEM encoded certificate or a +concatenation of that with the CA certificate. +\f[C]--key\f[R] should be the PEM encoded private key and +\f[C]--client-ca\f[R] should be the PEM encoded client certificate +authority certificate. +.PP +--min-tls-version is minimum TLS version that is acceptable. +Valid values are \[dq]tls1.0\[dq], \[dq]tls1.1\[dq], \[dq]tls1.2\[dq] +and \[dq]tls1.3\[dq] (default \[dq]tls1.0\[dq]). +.SS Template +.PP +\f[C]--template\f[R] allows a user to specify a custom markup template +for HTTP and WebDAV serve functions. +The server exports the following markup to be used within the template +to server pages: +.PP +.TS +tab(@); +lw(35.0n) lw(35.0n). +T{ +Parameter +T}@T{ +Description +T} +_ +T{ +\&.Name +T}@T{ +The full path of a file/directory. +T} +T{ +\&.Title +T}@T{ +Directory listing of .Name +T} +T{ +\&.Sort +T}@T{ +The current sort used. +This is changeable via ?sort= parameter +T} +T{ +T}@T{ +Sort Options: namedirfirst,name,size,time (default namedirfirst) +T} +T{ +\&.Order +T}@T{ +The current ordering used. +This is changeable via ?order= parameter +T} +T{ +T}@T{ +Order Options: asc,desc (default asc) +T} +T{ +\&.Query +T}@T{ +Currently unused. +T} +T{ +\&.Breadcrumb +T}@T{ +Allows for creating a relative navigation +T} +T{ +-- .Link +T}@T{ +The relative to the root link of the Text. +T} +T{ +-- .Text +T}@T{ +The Name of the directory. +T} +T{ +\&.Entries +T}@T{ +Information about a specific file/directory. +T} +T{ +-- .URL +T}@T{ +The \[aq]url\[aq] of an entry. +T} +T{ +-- .Leaf +T}@T{ +Currently same as \[aq]URL\[aq] but intended to be \[aq]just\[aq] the +name. +T} +T{ +-- .IsDir +T}@T{ +Boolean for if an entry is a directory or not. +T} +T{ +-- .Size +T}@T{ +Size in Bytes of the entry. +T} +T{ +-- .ModTime +T}@T{ +The UTC timestamp of an entry. +T} +.TE +.SS Authentication +.PP +By default this will serve files without needing a login. +.PP +You can either use an htpasswd file which can take lots of users, or set +a single username and password with the \f[C]--user\f[R] and +\f[C]--pass\f[R] flags. +.PP +Use \f[C]--htpasswd /path/to/htpasswd\f[R] to provide an htpasswd file. +This is in standard apache format and supports MD5, SHA1 and BCrypt for +basic authentication. +Bcrypt is recommended. +.PP +To create an htpasswd file: +.IP +.nf +\f[C] +touch htpasswd +htpasswd -B htpasswd user +htpasswd -B htpasswd anotherUser +\f[R] +.fi +.PP +The password file can be updated while rclone is running. +.PP +Use \f[C]--realm\f[R] to set the authentication realm. +.PP +Use \f[C]--salt\f[R] to change the password hashing salt from the +default. .IP .nf \f[C] @@ -6610,8 +6794,8 @@ rclone serve dlna remote:path [flags] .nf \f[C] --addr string The ip:port or :port to bind the DLNA http server to (default \[dq]:7879\[dq]) - --announce-interval duration The interval between SSDP announcements (default 12m0s) - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --announce-interval Duration The interval between SSDP announcements (default 12m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -6622,24 +6806,24 @@ rclone serve dlna remote:path [flags] --no-checksum Don\[aq]t compare checksums on up/download --no-modtime Don\[aq]t read/write the modification time (can speed things up) --no-seek Don\[aq]t allow seeking in files - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) \f[R] .fi .PP @@ -7109,15 +7293,15 @@ rclone serve docker [flags] --allow-other Allow access to other users (not supported on Windows) --allow-root Allow access to root user (not supported on Windows) --async-read Use asynchronous reads (not supported on Windows) (default true) - --attr-timeout duration Time for which file/directory attributes are cached (default 1s) + --attr-timeout Duration Time for which file/directory attributes are cached (default 1s) --base-dir string Base directory for volumes (default \[dq]/var/lib/docker-volumes/rclone\[dq]) --daemon Run mount in background and exit parent process (as background output is suppressed, use --log-file with --log-format=pid,... to monitor) (not supported on Windows) - --daemon-timeout duration Time limit for rclone to respond to kernel (not supported on Windows) - --daemon-wait duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) + --daemon-timeout Duration Time limit for rclone to respond to kernel (not supported on Windows) (default 0s) + --daemon-wait Duration Time to wait for ready mount from daemon (maximum time on Linux, constant sleep time on OSX/BSD) (not supported on Windows) (default 1m0s) --debug-fuse Debug the FUSE internals - needs -v --default-permissions Makes kernel enforce access control based on the file mode (not supported on Windows) --devname string Set the device name - default is remote:path - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --forget-state Skip restoring previous state @@ -7133,26 +7317,26 @@ rclone serve docker [flags] --noappledouble Ignore Apple Double (._) and .DS_Store files (supported on OSX only) (default true) --noapplexattr Ignore all \[dq]com.apple.*\[dq] extended attributes (supported on OSX only) -o, --option stringArray Option for libfuse/WinFsp (repeat if required) - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --socket-addr string Address or absolute path (default: /run/docker/plugins/rclone.sock) --socket-gid int GID for unix socket (default: current process GID) (default 1000) --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) --volname string Set the volume name (supported on Windows and OSX only) --write-back-cache Makes kernel buffer writes before sending them to rclone (without this, writethrough caching is used) (not supported on Windows) \f[R] @@ -7686,7 +7870,7 @@ rclone serve ftp remote:path [flags] --addr string IPaddress:Port or :Port to bind server to (default \[dq]localhost:2121\[dq]) --auth-proxy string A program to use to create the backend from the auth --cert string TLS PEM key (concatenation of certificate and CA certificate) - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -7697,26 +7881,26 @@ rclone serve ftp remote:path [flags] --no-seek Don\[aq]t allow seeking in files --pass string Password for authentication (empty value allow every password) --passive-port string Passive port range to use (default \[dq]30000-32000\[dq]) - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --public-ip string Public IP address to advertise for passive connections --read-only Only allow read-only access --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication (default \[dq]anonymous\[dq]) - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) \f[R] .fi .PP @@ -7755,6 +7939,14 @@ If you set \f[C]--addr\f[R] to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. .PP +You can use a unix socket by setting the url to +\f[C]unix:///path/to/socket\f[R] or just by using an absolute path name. +Note that unix sockets bypass the authentication - this is expected to +be done with file system permissions. +.PP +\f[C]--addr\f[R] may be repeated to listen on multiple +IPs/ports/sockets. +.PP \f[C]--server-read-timeout\f[R] and \f[C]--server-write-timeout\f[R] can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -7771,7 +7963,7 @@ Rclone automatically inserts leading and trailing \[dq]/\[dq] on \f[C]--baseurl\f[R], so \f[C]--baseurl \[dq]rclone\[dq]\f[R], \f[C]--baseurl \[dq]/rclone\[dq]\f[R] and \f[C]--baseurl \[dq]/rclone/\[dq]\f[R] are all treated identically. -.SS SSL/TLS +.SS TLS (SSL) .PP By default this will serve over http. If you want you can serve over https. @@ -8317,47 +8509,47 @@ rclone serve http remote:path [flags] .IP .nf \f[C] - --addr string IPaddress:Port or :Port to bind server to (default \[dq]127.0.0.1:8080\[dq]) + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --baseurl string Prefix for URLs - leave blank for root - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) -h, --help help for http --htpasswd string A htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default \[dq]tls1.0\[dq]) --no-checksum Don\[aq]t compare checksums on up/download --no-modtime Don\[aq]t read/write the modification time (can speed things up) --no-seek Don\[aq]t allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --realm string Realm for authentication --salt string Password hashing salt (default \[dq]dlPL2MqE\[dq]) - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --template string User-specified template --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) \f[R] .fi .PP @@ -8372,7 +8564,7 @@ remote over a protocol. Serve the remote for restic\[aq]s REST API. .SS Synopsis .PP -Run a basic web server to serve a remove over restic\[aq]s REST backend +Run a basic web server to serve a remote over restic\[aq]s REST backend API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly. @@ -8469,9 +8661,8 @@ repositories starting with a path of \f[C]//\f[R]. .SS Server options .PP Use \f[C]--addr\f[R] to specify which IP address and port the server -should listen on, e.g. -\f[C]--addr 1.2.3.4:8000\f[R] or \f[C]--addr :8080\f[R] to listen to all -IPs. +should listen on, eg \f[C]--addr 1.2.3.4:8000\f[R] or +\f[C]--addr :8080\f[R] to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. .PP @@ -8479,6 +8670,14 @@ If you set \f[C]--addr\f[R] to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. .PP +You can use a unix socket by setting the url to +\f[C]unix:///path/to/socket\f[R] or just by using an absolute path name. +Note that unix sockets bypass the authentication - this is expected to +be done with file system permissions. +.PP +\f[C]--addr\f[R] may be repeated to listen on multiple +IPs/ports/sockets. +.PP \f[C]--server-read-timeout\f[R] and \f[C]--server-write-timeout\f[R] can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -8495,103 +8694,23 @@ Rclone automatically inserts leading and trailing \[dq]/\[dq] on \f[C]--baseurl\f[R], so \f[C]--baseurl \[dq]rclone\[dq]\f[R], \f[C]--baseurl \[dq]/rclone\[dq]\f[R] and \f[C]--baseurl \[dq]/rclone/\[dq]\f[R] are all treated identically. +.SS TLS (SSL) .PP -\f[C]--template\f[R] allows a user to specify a custom markup template -for HTTP and WebDAV serve functions. -The server exports the following markup to be used within the template -to server pages: +By default this will serve over http. +If you want you can serve over https. +You will need to supply the \f[C]--cert\f[R] and \f[C]--key\f[R] flags. +If you wish to do client side certificate validation then you will need +to supply \f[C]--client-ca\f[R] also. .PP -.TS -tab(@); -lw(35.0n) lw(35.0n). -T{ -Parameter -T}@T{ -Description -T} -_ -T{ -\&.Name -T}@T{ -The full path of a file/directory. -T} -T{ -\&.Title -T}@T{ -Directory listing of .Name -T} -T{ -\&.Sort -T}@T{ -The current sort used. -This is changeable via ?sort= parameter -T} -T{ -T}@T{ -Sort Options: namedirfirst,name,size,time (default namedirfirst) -T} -T{ -\&.Order -T}@T{ -The current ordering used. -This is changeable via ?order= parameter -T} -T{ -T}@T{ -Order Options: asc,desc (default asc) -T} -T{ -\&.Query -T}@T{ -Currently unused. -T} -T{ -\&.Breadcrumb -T}@T{ -Allows for creating a relative navigation -T} -T{ --- .Link -T}@T{ -The relative to the root link of the Text. -T} -T{ --- .Text -T}@T{ -The Name of the directory. -T} -T{ -\&.Entries -T}@T{ -Information about a specific file/directory. -T} -T{ --- .URL -T}@T{ -The \[aq]url\[aq] of an entry. -T} -T{ --- .Leaf -T}@T{ -Currently same as \[aq]URL\[aq] but intended to be \[aq]just\[aq] the -name. -T} -T{ --- .IsDir -T}@T{ -Boolean for if an entry is a directory or not. -T} -T{ --- .Size -T}@T{ -Size in Bytes of the entry. -T} -T{ --- .ModTime -T}@T{ -The UTC timestamp of an entry. -T} -.TE +\f[C]--cert\f[R] should be a either a PEM encoded certificate or a +concatenation of that with the CA certificate. +\f[C]--key\f[R] should be the PEM encoded private key and +\f[C]--client-ca\f[R] should be the PEM encoded client certificate +authority certificate. +.PP +--min-tls-version is minimum TLS version that is acceptable. +Valid values are \[dq]tls1.0\[dq], \[dq]tls1.1\[dq], \[dq]tls1.2\[dq] +and \[dq]tls1.3\[dq] (default \[dq]tls1.0\[dq]). .SS Authentication .PP By default this will serve files without needing a login. @@ -8618,23 +8737,9 @@ htpasswd -B htpasswd anotherUser The password file can be updated while rclone is running. .PP Use \f[C]--realm\f[R] to set the authentication realm. -.SS SSL/TLS .PP -By default this will serve over HTTP. -If you want you can serve over HTTPS. -You will need to supply the \f[C]--cert\f[R] and \f[C]--key\f[R] flags. -If you wish to do client side certificate validation then you will need -to supply \f[C]--client-ca\f[R] also. -.PP -\f[C]--cert\f[R] should be either a PEM encoded certificate or a -concatenation of that with the CA certificate. -\f[C]--key\f[R] should be the PEM encoded private key and -\f[C]--client-ca\f[R] should be the PEM encoded client certificate -authority certificate. -.PP ---min-tls-version is minimum TLS version that is acceptable. -Valid values are \[dq]tls1.0\[dq], \[dq]tls1.1\[dq], \[dq]tls1.2\[dq] -and \[dq]tls1.3\[dq] (default \[dq]tls1.0\[dq]). +Use \f[C]--salt\f[R] to change the password hashing salt from the +default. .IP .nf \f[C] @@ -8645,24 +8750,24 @@ rclone serve restic remote:path [flags] .IP .nf \f[C] - --addr string IPaddress:Port or :Port to bind server to (default \[dq]localhost:8080\[dq]) + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --append-only Disallow deletion of repository data --baseurl string Prefix for URLs - leave blank for root --cache-objects Cache listed objects (default true) - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with -h, --help help for restic - --htpasswd string htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --htpasswd string A htpasswd file - if not provided no authentication is done + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default \[dq]tls1.0\[dq]) --pass string Password for authentication --private-repos Users can only access their private repo - --realm string Realm for authentication (default \[dq]rclone\[dq]) - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --realm string Realm for authentication + --salt string Password hashing salt (default \[dq]dlPL2MqE\[dq]) + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --stdio Run an HTTP2 server on stdin/stdout - --template string User-specified template --user string User name for authentication \f[R] .fi @@ -9237,7 +9342,7 @@ rclone serve sftp remote:path [flags] --addr string IPaddress:Port or :Port to bind server to (default \[dq]localhost:2022\[dq]) --auth-proxy string A program to use to create the backend from the auth --authorized-keys string Authorized keys file (default \[dq]\[ti]/.ssh/authorized_keys\[dq]) - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) @@ -9248,26 +9353,26 @@ rclone serve sftp remote:path [flags] --no-modtime Don\[aq]t read/write the modification time (can speed things up) --no-seek Don\[aq]t allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access --stdio Run an sftp server on stdin/stdout --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) \f[R] .fi .PP @@ -9301,9 +9406,8 @@ see the full list. .SS Server options .PP Use \f[C]--addr\f[R] to specify which IP address and port the server -should listen on, e.g. -\f[C]--addr 1.2.3.4:8000\f[R] or \f[C]--addr :8080\f[R] to listen to all -IPs. +should listen on, eg \f[C]--addr 1.2.3.4:8000\f[R] or +\f[C]--addr :8080\f[R] to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port. .PP @@ -9311,6 +9415,14 @@ If you set \f[C]--addr\f[R] to listen on a public or LAN accessible IP address then using Authentication is advised - see the next section for info. .PP +You can use a unix socket by setting the url to +\f[C]unix:///path/to/socket\f[R] or just by using an absolute path name. +Note that unix sockets bypass the authentication - this is expected to +be done with file system permissions. +.PP +\f[C]--addr\f[R] may be repeated to listen on multiple +IPs/ports/sockets. +.PP \f[C]--server-read-timeout\f[R] and \f[C]--server-write-timeout\f[R] can be used to control the timeouts on the server. Note that this is the total time for a transfer. @@ -9327,6 +9439,24 @@ Rclone automatically inserts leading and trailing \[dq]/\[dq] on \f[C]--baseurl\f[R], so \f[C]--baseurl \[dq]rclone\[dq]\f[R], \f[C]--baseurl \[dq]/rclone\[dq]\f[R] and \f[C]--baseurl \[dq]/rclone/\[dq]\f[R] are all treated identically. +.SS TLS (SSL) +.PP +By default this will serve over http. +If you want you can serve over https. +You will need to supply the \f[C]--cert\f[R] and \f[C]--key\f[R] flags. +If you wish to do client side certificate validation then you will need +to supply \f[C]--client-ca\f[R] also. +.PP +\f[C]--cert\f[R] should be a either a PEM encoded certificate or a +concatenation of that with the CA certificate. +\f[C]--key\f[R] should be the PEM encoded private key and +\f[C]--client-ca\f[R] should be the PEM encoded client certificate +authority certificate. +.PP +--min-tls-version is minimum TLS version that is acceptable. +Valid values are \[dq]tls1.0\[dq], \[dq]tls1.1\[dq], \[dq]tls1.2\[dq] +and \[dq]tls1.3\[dq] (default \[dq]tls1.0\[dq]). +.SS Template .PP \f[C]--template\f[R] allows a user to specify a custom markup template for HTTP and WebDAV serve functions. @@ -9450,23 +9580,9 @@ htpasswd -B htpasswd anotherUser The password file can be updated while rclone is running. .PP Use \f[C]--realm\f[R] to set the authentication realm. -.SS SSL/TLS .PP -By default this will serve over HTTP. -If you want you can serve over HTTPS. -You will need to supply the \f[C]--cert\f[R] and \f[C]--key\f[R] flags. -If you wish to do client side certificate validation then you will need -to supply \f[C]--client-ca\f[R] also. -.PP -\f[C]--cert\f[R] should be either a PEM encoded certificate or a -concatenation of that with the CA certificate. -\f[C]--key\f[R] should be the PEM encoded private key and -\f[C]--client-ca\f[R] should be the PEM encoded client certificate -authority certificate. -.PP ---min-tls-version is minimum TLS version that is acceptable. -Valid values are \[dq]tls1.0\[dq], \[dq]tls1.1\[dq], \[dq]tls1.2\[dq] -and \[dq]tls1.3\[dq] (default \[dq]tls1.0\[dq]). +Use \f[C]--salt\f[R] to change the password hashing salt from the +default. .SS VFS - Virtual File System .PP This command uses the VFS layer. @@ -9962,49 +10078,50 @@ rclone serve webdav remote:path [flags] .IP .nf \f[C] - --addr string IPaddress:Port or :Port to bind server to (default \[dq]localhost:8080\[dq]) + --addr stringArray IPaddress:Port or :Port to bind server to (default [127.0.0.1:8080]) --auth-proxy string A program to use to create the backend from the auth --baseurl string Prefix for URLs - leave blank for root - --cert string SSL PEM key (concatenation of certificate and CA certificate) + --cert string TLS PEM key (concatenation of certificate and CA certificate) --client-ca string Client certificate authority to verify clients with - --dir-cache-time duration Time to cache directory entries for (default 5m0s) + --dir-cache-time Duration Time to cache directory entries for (default 5m0s) --dir-perms FileMode Directory permissions (default 0777) --disable-dir-list Disable HTML directory list on GET request for a directory --etag-hash string Which hash to use for the ETag, or auto or blank for off --file-perms FileMode File permissions (default 0666) --gid uint32 Override the gid field set by the filesystem (not supported on Windows) (default 1000) -h, --help help for webdav - --htpasswd string htpasswd file - if not provided no authentication is done - --key string SSL PEM Private key + --htpasswd string A htpasswd file - if not provided no authentication is done + --key string TLS PEM Private key --max-header-bytes int Maximum size of request header (default 4096) --min-tls-version string Minimum TLS version that is acceptable (default \[dq]tls1.0\[dq]) --no-checksum Don\[aq]t compare checksums on up/download --no-modtime Don\[aq]t read/write the modification time (can speed things up) --no-seek Don\[aq]t allow seeking in files --pass string Password for authentication - --poll-interval duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) + --poll-interval Duration Time to wait between polling for changes, must be smaller than dir-cache-time and only on supported remotes (set 0 to disable) (default 1m0s) --read-only Only allow read-only access - --realm string Realm for authentication (default \[dq]rclone\[dq]) - --server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --realm string Realm for authentication + --salt string Password hashing salt (default \[dq]dlPL2MqE\[dq]) + --server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --template string User-specified template --uid uint32 Override the uid field set by the filesystem (not supported on Windows) (default 1000) --umask int Override the permission bits set by the filesystem (not supported on Windows) (default 2) --user string User name for authentication - --vfs-cache-max-age duration Max age of objects in the cache (default 1h0m0s) + --vfs-cache-max-age Duration Max age of objects in the cache (default 1h0m0s) --vfs-cache-max-size SizeSuffix Max total size of objects in the cache (default off) --vfs-cache-mode CacheMode Cache mode off|minimal|writes|full (default off) - --vfs-cache-poll-interval duration Interval to poll the cache for stale objects (default 1m0s) + --vfs-cache-poll-interval Duration Interval to poll the cache for stale objects (default 1m0s) --vfs-case-insensitive If a file name not found, find a case insensitive match --vfs-disk-space-total-size SizeSuffix Specify the total space of disk (default off) --vfs-fast-fingerprint Use fast (less accurate) fingerprints for change detection --vfs-read-ahead SizeSuffix Extra read ahead over --buffer-size when using cache-mode full --vfs-read-chunk-size SizeSuffix Read the source objects in chunks (default 128Mi) --vfs-read-chunk-size-limit SizeSuffix If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached (\[aq]off\[aq] is unlimited) (default off) - --vfs-read-wait duration Time to wait for in-sequence read before seeking (default 20ms) + --vfs-read-wait Duration Time to wait for in-sequence read before seeking (default 20ms) --vfs-used-is-size rclone size Use the rclone size algorithm for Used size - --vfs-write-back duration Time to writeback files after last use when using cache (default 5s) - --vfs-write-wait duration Time to wait for in-sequence write before giving error (default 1s) + --vfs-write-back Duration Time to writeback files after last use when using cache (default 5s) + --vfs-write-wait Duration Time to wait for in-sequence write before giving error (default 1s) \f[R] .fi .PP @@ -10143,7 +10260,7 @@ rclone test changenotify remote: [flags] .nf \f[C] -h, --help help for changenotify - --poll-interval duration Time to wait between polling for changes (default 10s) + --poll-interval Duration Time to wait between polling for changes (default 10s) \f[R] .fi .PP @@ -10212,7 +10329,7 @@ rclone test info [remote:path]+ [flags] --check-normalization Check UTF-8 Normalization --check-streaming Check uploads with indeterminate file size -h, --help help for info - --upload-wait duration Wait after writing a file + --upload-wait Duration Wait after writing a file (default 0s) --write-json string Write results to file \f[R] .fi @@ -10270,6 +10387,7 @@ rclone test makefiles [flags] --files int Number of files to create (default 1000) --files-per-directory int Average number of files per directory (default 10) -h, --help help for makefiles + --max-depth int Maximum depth of directory hierarchy (default 10) --max-file-size SizeSuffix Maximum size of files to create (default 100) --max-name-length int Maximum size of file names (default 12) --min-file-size SizeSuffix Minimum size of file to create @@ -10413,7 +10531,6 @@ rclone tree remote:path [flags] .nf \f[C] -a, --all All files are listed (list . files too) - -C, --color Turn colorization on always -d, --dirs-only List directories only --dirsfirst List directories before files (-U disables) --full-path Print the full path prefix for each file @@ -10735,10 +10852,20 @@ DEBUG : :s3: detected overridden config - adding \[dq]{YTu53}\[dq] suffix to nam .SS Valid remote names .PP Remote names are case sensitive, and must adhere to the following rules: -- May only contain \f[C]0\f[R]-\f[C]9\f[R], \f[C]A\f[R]-\f[C]Z\f[R], -\f[C]a\f[R]-\f[C]z\f[R], \f[C]_\f[R], \f[C]-\f[R], \f[C].\f[R] and +- May contain number, letter, \f[C]_\f[R], \f[C]-\f[R], \f[C].\f[R] and space. - May not start with \f[C]-\f[R] or space. +- May not end with space. +.PP +Starting with rclone version 1.61, any Unicode numbers and letters are +allowed, while in older versions it was limited to plain ASCII (0-9, +A-Z, a-z). +If you use the same rclone configuration from different shells, which +may be configured with different character encoding, you must be +cautious to use characters that are possible to write in all of them. +This is mostly a problem on Windows, where the console traditionally +uses a non-Unicode character set - defined by the so-called \[dq]code +page\[dq]. .SS Quoting and the shell .PP When you are typing commands to your computer you are using something @@ -11408,6 +11535,18 @@ much quicker than without the \f[C]--checksum\f[R] flag. .PP When using this flag, rclone won\[aq]t update mtimes of remote files if they are incorrect as it would normally. +.SS --color WHEN +.PP +Specifiy when colors (and other ANSI codes) should be added to the +output. +.PP +\f[C]AUTO\f[R] (default) only allows ANSI codes when the output is a +terminal +.PP +\f[C]NEVER\f[R] never allow ANSI codes +.PP +\f[C]ALWAYS\f[R] always add ANSI codes, regardless of the output format +(terminal or file) .SS --compare-dest=DIR .PP When using \f[C]sync\f[R], \f[C]copy\f[R] or \f[C]move\f[R] DIR is @@ -13101,6 +13240,18 @@ For the filtering options \f[C]--max-age\f[R] .IP \[bu] 2 \f[C]--dump filters\f[R] +.IP \[bu] 2 +\f[C]--metadata-include\f[R] +.IP \[bu] 2 +\f[C]--metadata-include-from\f[R] +.IP \[bu] 2 +\f[C]--metadata-exclude\f[R] +.IP \[bu] 2 +\f[C]--metadata-exclude-from\f[R] +.IP \[bu] 2 +\f[C]--metadata-filter\f[R] +.IP \[bu] 2 +\f[C]--metadata-filter-from\f[R] .PP See the filtering section (https://rclone.org/filtering/). .SS Remote control @@ -13369,15 +13520,17 @@ There are two ways of doing it, described below. .SS Configuring using rclone authorize .PP On the headless box run \f[C]rclone\f[R] config but answer \f[C]N\f[R] -to the \f[C]Use auto config?\f[R] question. +to the \f[C]Use web browser to automatically authenticate?\f[R] +question. .IP .nf \f[C] \&... Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes (default) n) No y/n> n @@ -13469,15 +13622,17 @@ ssh -L localhost:53682:localhost:53682 username\[at]remote_server .fi .PP Then on the headless box run \f[C]rclone\f[R] config and answer -\f[C]Y\f[R] to the \f[C]Use auto config?\f[R] question. +\f[C]Y\f[R] to the +\f[C]Use web browser to automatically authenticate?\f[R] question. .IP .nf \f[C] \&... Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes (default) n) No y/n> y @@ -14618,6 +14773,58 @@ dir1/dir2/dir3/.ignore .PP The command \f[C]rclone ls --exclude-if-present .ignore dir1\f[R] does not list \f[C]dir3\f[R], \f[C]file3\f[R] or \f[C].ignore\f[R]. +.SS Metadata filters +.PP +The metadata filters work in a very similar way to the normal file name +filters, except they match metadata (https://rclone.org/docs/#metadata) +on the object. +.PP +The metadata should be specified as \f[C]key=value\f[R] patterns. +This may be wildcarded using the normal filter patterns or regular +expressions. +.PP +For example if you wished to list only local files with a mode of +\f[C]100664\f[R] you could do that with: +.IP +.nf +\f[C] +rclone lsf -M --files-only --metadata-include \[dq]mode=100664\[dq] . +\f[R] +.fi +.PP +Or if you wished to show files with an \f[C]atime\f[R], \f[C]mtime\f[R] +or \f[C]btime\f[R] at a given date: +.IP +.nf +\f[C] +rclone lsf -M --files-only --metadata-include \[dq][abm]time=2022-12-16*\[dq] . +\f[R] +.fi +.PP +Like file filtering, metadata filtering only applies to files not to +directories. +.PP +The filters can be applied using these flags. +.IP \[bu] 2 +\f[C]--metadata-include\f[R] - Include metadatas matching pattern +.IP \[bu] 2 +\f[C]--metadata-include-from\f[R] - Read metadata include patterns from +file (use - to read from stdin) +.IP \[bu] 2 +\f[C]--metadata-exclude\f[R] - Exclude metadatas matching pattern +.IP \[bu] 2 +\f[C]--metadata-exclude-from\f[R] - Read metadata exclude patterns from +file (use - to read from stdin) +.IP \[bu] 2 +\f[C]--metadata-filter\f[R] - Add a metadata filtering rule +.IP \[bu] 2 +\f[C]--metadata-filter-from\f[R] - Read metadata filtering patterns from +a file (use - to read from stdin) +.PP +Each flag can be repeated. +See the section on how filter rules are applied for more details - these +flags work in an identical way to the file name filtering flags, but +instead of file name patterns have metadata patterns. .SS Common pitfalls .PP The most frequent filter support issues on the rclone @@ -15460,6 +15667,13 @@ providers (https://rclone.org/commands/rclone_config_providers/) command for more information on the above. .PP \f[B]Authentication is required for this call.\f[R] +.SS config/setpath: Set the path of the config file +.PP +Parameters: +.IP \[bu] 2 +path - path to the config file to use +.PP +\f[B]Authentication is required for this call.\f[R] .SS config/update: update the config for a remote. .PP This takes the following parameters: @@ -15599,7 +15813,7 @@ Returns: \[dq]result\[dq]: \[dq]\[dq] } -OR +OR { \[dq]error\[dq]: true, \[dq]result\[dq]: \[dq]\[dq] @@ -15818,6 +16032,25 @@ go tool pprof http://localhost:5572/debug/pprof/block Parameters: .IP \[bu] 2 rate - int +.SS debug/set-gc-percent: Call runtime/debug.SetGCPercent for setting the garbage collection target percentage. +.PP +SetGCPercent sets the garbage collection target percentage: a collection +is triggered when the ratio of freshly allocated data to live data +remaining after the previous collection reaches this percentage. +SetGCPercent returns the previous setting. +The initial setting is the value of the GOGC environment variable at +startup, or 100 if the variable is not set. +.PP +This setting may be effectively reduced in order to maintain a memory +limit. +A negative percentage effectively disables garbage collection, unless +the memory limit is reached. +.PP +See https://pkg.go.dev/runtime/debug#SetMemoryLimit for more details. +.PP +Parameters: +.IP \[bu] 2 +gc-percent - int .SS debug/set-mutex-profile-fraction: Set runtime.SetMutexProfileFraction for mutex profiling. .PP SetMutexProfileFraction controls the fraction of mutex contention events @@ -15844,6 +16077,48 @@ rate - int Results: .IP \[bu] 2 previousRate - int +.SS debug/set-soft-memory-limit: Call runtime/debug.SetMemoryLimit for setting a soft memory limit for the runtime. +.PP +SetMemoryLimit provides the runtime with a soft memory limit. +.PP +The runtime undertakes several processes to try to respect this memory +limit, including adjustments to the frequency of garbage collections and +returning memory to the underlying system more aggressively. +This limit will be respected even if GOGC=off (or, if SetGCPercent(-1) +is executed). +.PP +The input limit is provided as bytes, and includes all memory mapped, +managed, and not released by the Go runtime. +Notably, it does not account for space used by the Go binary and memory +external to Go, such as memory managed by the underlying system on +behalf of the process, or memory managed by non-Go code inside the same +process. +Examples of excluded memory sources include: OS kernel memory held on +behalf of the process, memory allocated by C code, and memory mapped by +syscall.Mmap (because it is not managed by the Go runtime). +.PP +A zero limit or a limit that\[aq]s lower than the amount of memory used +by the Go runtime may cause the garbage collector to run nearly +continuously. +However, the application may still make progress. +.PP +The memory limit is always respected by the Go runtime, so to +effectively disable this behavior, set the limit very high. +math.MaxInt64 is the canonical value for disabling the limit, but values +much greater than the available memory on the underlying system work +just as well. +.PP +See https://go.dev/doc/gc-guide for a detailed guide explaining the soft +memory limit in more detail, as well as a variety of common use-cases +and scenarios. +.PP +SetMemoryLimit returns the previously set memory limit. +A negative input does not adjust the limit, and allows for retrieval of +the currently set memory limit. +.PP +Parameters: +.IP \[bu] 2 +mem-limit - int .SS fscache/clear: Clear the Fs cache. .PP This clears the fs cache. @@ -19313,7 +19588,7 @@ T} T{ Oracle Object Storage T}@T{ -Yes +No T}@T{ Yes T}@T{ @@ -19325,7 +19600,7 @@ Yes T}@T{ Yes T}@T{ -No +Yes T}@T{ No T}@T{ @@ -19545,7 +19820,7 @@ Storj T}@T{ Yes \[dg] T}@T{ -No +Yes T}@T{ Yes T}@T{ @@ -19789,9 +20064,10 @@ These flags are available for every command. -c, --checksum Skip based on checksum (if available) & size, not mod-time & size --client-cert string Client SSL certificate (PEM) for mutual TLS auth --client-key string Client SSL private key (PEM) for mutual TLS auth + --color string When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS (default \[dq]AUTO\[dq]) --compare-dest stringArray Include additional comma separated server-side paths during comparison --config string Config file (default \[dq]$HOME/.config/rclone/rclone.conf\[dq]) - --contimeout duration Connect timeout (default 1m0s) + --contimeout Duration Connect timeout (default 1m0s) --copy-dest stringArray Implies --compare-dest but also copies files from paths into destination --cpuprofile string Write cpu profile to file --cutoff-mode string Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS (default \[dq]HARD\[dq]) @@ -19809,16 +20085,16 @@ These flags are available for every command. --dump-headers Dump HTTP headers - may contain sensitive info --error-on-no-transfer Sets exit code 9 if no files are transferred, useful in scripts --exclude stringArray Exclude files matching pattern - --exclude-from stringArray Read exclude patterns from file (use - to read from stdin) + --exclude-from stringArray Read file exclude patterns from file (use - to read from stdin) --exclude-if-present stringArray Exclude directories if filename is present - --expect-continue-timeout duration Timeout when using expect / 100-continue in HTTP (default 1s) + --expect-continue-timeout Duration Timeout when using expect / 100-continue in HTTP (default 1s) --fast-list Use recursive list if available; uses more memory but fewer transactions --files-from stringArray Read list of source-file names from file (use - to read from stdin) --files-from-raw stringArray Read list of source-file names from file without any processing of lines (use - to read from stdin) - -f, --filter stringArray Add a file-filtering rule - --filter-from stringArray Read filtering patterns from a file (use - to read from stdin) - --fs-cache-expire-duration duration Cache remotes for this long (0 to disable caching) (default 5m0s) - --fs-cache-expire-interval duration Interval to check for expired remotes (default 1m0s) + -f, --filter stringArray Add a file filtering rule + --filter-from stringArray Read file filtering patterns from a file (use - to read from stdin) + --fs-cache-expire-duration Duration Cache remotes for this long (0 to disable caching) (default 5m0s) + --fs-cache-expire-interval Duration Interval to check for expired remotes (default 1m0s) --header stringArray Set HTTP header for all transactions --header-download stringArray Set HTTP header for download transactions --header-upload stringArray Set HTTP header for upload transactions @@ -19832,9 +20108,9 @@ These flags are available for every command. -I, --ignore-times Don\[aq]t skip files that match size and time - transfer all files --immutable Do not modify files, fail if existing files have been modified --include stringArray Include files matching pattern - --include-from stringArray Read include patterns from file (use - to read from stdin) + --include-from stringArray Read file include patterns from file (use - to read from stdin) -i, --interactive Enable interactive mode - --kv-lock-time duration Maximum time to keep key-value database locked by process (default 1s) + --kv-lock-time Duration Maximum time to keep key-value database locked by process (default 1s) --log-file string Log everything to this file --log-format string Comma separated list of log format options (default \[dq]date,time\[dq]) --log-level string Log level DEBUG|INFO|NOTICE|ERROR (default \[dq]NOTICE\[dq]) @@ -19844,16 +20120,22 @@ These flags are available for every command. --max-backlog int Maximum number of objects in sync or check backlog (default 10000) --max-delete int When synchronizing, limit the number of deletes (default -1) --max-depth int If set limits the recursion depth to this (default -1) - --max-duration duration Maximum duration rclone will transfer data for + --max-duration Duration Maximum duration rclone will transfer data for (default 0s) --max-size SizeSuffix Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P (default off) --max-stats-groups int Maximum number of stats groups to keep in memory, on max oldest is discarded (default 1000) --max-transfer SizeSuffix Maximum size of data to transfer (default off) --memprofile string Write memory profile to file -M, --metadata If set, preserve metadata when copying objects + --metadata-exclude stringArray Exclude metadatas matching pattern + --metadata-exclude-from stringArray Read metadata exclude patterns from file (use - to read from stdin) + --metadata-filter stringArray Add a metadata filtering rule + --metadata-filter-from stringArray Read metadata filtering patterns from a file (use - to read from stdin) + --metadata-include stringArray Include metadatas matching pattern + --metadata-include-from stringArray Read metadata include patterns from file (use - to read from stdin) --metadata-set stringArray Add metadata key=value when uploading --min-age Duration Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off) --min-size SizeSuffix Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off) - --modify-window duration Max time diff to be considered the same (default 1ns) + --modify-window Duration Max time diff to be considered the same (default 1ns) --multi-thread-cutoff SizeSuffix Use multi-thread downloads for files above this size (default 250Mi) --multi-thread-streams int Max number of streams to use for multi-thread downloads (default 4) --no-check-certificate Do not verify the server SSL certificate (insecure) @@ -19869,25 +20151,26 @@ These flags are available for every command. --progress-terminal-title Show progress on the terminal title (requires -P/--progress) -q, --quiet Print as little stuff as possible --rc Enable the remote control server - --rc-addr string IPaddress:Port or :Port to bind server to (default \[dq]localhost:5572\[dq]) + --rc-addr stringArray IPaddress:Port or :Port to bind server to (default [localhost:5572]) --rc-allow-origin string Set the allowed origin for CORS --rc-baseurl string Prefix for URLs - leave blank for root - --rc-cert string SSL PEM key (concatenation of certificate and CA certificate) + --rc-cert string TLS PEM key (concatenation of certificate and CA certificate) --rc-client-ca string Client certificate authority to verify clients with --rc-enable-metrics Enable prometheus metrics on /metrics --rc-files string Path to local files to serve on the HTTP server - --rc-htpasswd string htpasswd file - if not provided no authentication is done - --rc-job-expire-duration duration Expire finished async jobs older than this value (default 1m0s) - --rc-job-expire-interval duration Interval to check for expired async jobs (default 10s) - --rc-key string SSL PEM Private key + --rc-htpasswd string A htpasswd file - if not provided no authentication is done + --rc-job-expire-duration Duration Expire finished async jobs older than this value (default 1m0s) + --rc-job-expire-interval Duration Interval to check for expired async jobs (default 10s) + --rc-key string TLS PEM Private key --rc-max-header-bytes int Maximum size of request header (default 4096) --rc-min-tls-version string Minimum TLS version that is acceptable (default \[dq]tls1.0\[dq]) --rc-no-auth Don\[aq]t require auth for certain methods --rc-pass string Password for authentication - --rc-realm string Realm for authentication (default \[dq]rclone\[dq]) + --rc-realm string Realm for authentication + --rc-salt string Password hashing salt (default \[dq]dlPL2MqE\[dq]) --rc-serve Enable the serving of remote objects - --rc-server-read-timeout duration Timeout for server reading data (default 1h0m0s) - --rc-server-write-timeout duration Timeout for server writing data (default 1h0m0s) + --rc-server-read-timeout Duration Timeout for server reading data (default 1h0m0s) + --rc-server-write-timeout Duration Timeout for server writing data (default 1h0m0s) --rc-template string User-specified template --rc-user string User name for authentication --rc-web-fetch-url string URL to fetch the releases for webgui (default \[dq]https://api.github.com/repos/rclone/rclone-webui-react/releases/latest\[dq]) @@ -19897,10 +20180,10 @@ These flags are available for every command. --rc-web-gui-update Check and update to latest version of web gui --refresh-times Refresh the modtime of remote files --retries int Retry operations this many times if they fail (default 3) - --retries-sleep duration Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) + --retries-sleep Duration Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) (default 0s) --server-side-across-configs Allow server-side operations (e.g. copy) to work across different configs --size-only Skip based on size only, not mod-time or checksum - --stats duration Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s) + --stats Duration Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s) --stats-file-name-length int Max file name length in stats (0 for no limit) (default 45) --stats-log-level string Log level to show --stats output DEBUG|INFO|NOTICE|ERROR (default \[dq]INFO\[dq]) --stats-one-line Make the stats fit on one line @@ -19913,7 +20196,7 @@ These flags are available for every command. --syslog Use Syslog for logging --syslog-facility string Facility for syslog, e.g. KERN,USER,... (default \[dq]DAEMON\[dq]) --temp-dir string Directory rclone will use for temporary files (default \[dq]/tmp\[dq]) - --timeout duration IO idle timeout (default 5m0s) + --timeout Duration IO idle timeout (default 5m0s) --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --track-renames When synchronizing, track file renames and do a server-side move if possible @@ -19924,7 +20207,7 @@ These flags are available for every command. --use-json-log Use json log format --use-mmap Use mmap allocator (see docs) --use-server-modtime Use server modified time instead of object metadata - --user-agent string Set the user-agent to a specified string (default \[dq]rclone/v1.60.0\[dq]) + --user-agent string Set the user-agent to a specified string (default \[dq]rclone/v1.61.0\[dq]) -v, --verbose count Print lots more stuff (repeat for more) \f[R] .fi @@ -19935,529 +20218,543 @@ They control the backends and may be set in the config file. .IP .nf \f[C] - --acd-auth-url string Auth server URL - --acd-client-id string OAuth Client Id - --acd-client-secret string OAuth Client Secret - --acd-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --acd-templink-threshold SizeSuffix Files >= this size will be downloaded via their tempLink (default 9Gi) - --acd-token string OAuth Access Token as a JSON blob - --acd-token-url string Token server url - --acd-upload-wait-per-gb Duration Additional time per GiB to wait after a failed complete upload to see if it appears (default 3m0s) - --alias-remote string Remote or path to alias - --azureblob-access-tier string Access tier of blob: hot, cool or archive - --azureblob-account string Storage Account Name - --azureblob-archive-tier-delete Delete archive tier blobs before overwriting - --azureblob-chunk-size SizeSuffix Upload chunk size (default 4Mi) - --azureblob-disable-checksum Don\[aq]t store MD5 checksum with object metadata - --azureblob-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8) - --azureblob-endpoint string Endpoint for the service - --azureblob-key string Storage Account Key - --azureblob-list-chunk int Size of blob list (default 5000) - --azureblob-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --azureblob-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --azureblob-msi-client-id string Object ID of the user-assigned MSI to use, if any - --azureblob-msi-mi-res-id string Azure resource ID of the user-assigned MSI to use, if any - --azureblob-msi-object-id string Object ID of the user-assigned MSI to use, if any - --azureblob-no-head-object If set, do not do HEAD before GET when getting objects - --azureblob-public-access string Public access level of a container: blob or container - --azureblob-sas-url string SAS URL for container level access only - --azureblob-service-principal-file string Path to file containing credentials for use with a service principal - --azureblob-upload-concurrency int Concurrency for multipart uploads (default 16) - --azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256 MiB) (deprecated) - --azureblob-use-emulator Uses local storage emulator if provided as \[aq]true\[aq] - --azureblob-use-msi Use a managed service identity to authenticate (only works in Azure) - --b2-account string Account ID or Application Key ID - --b2-chunk-size SizeSuffix Upload chunk size (default 96Mi) - --b2-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4Gi) - --b2-disable-checksum Disable checksums for large (> upload cutoff) files - --b2-download-auth-duration Duration Time before the authorization token will expire in s or suffix ms|s|m|h|d (default 1w) - --b2-download-url string Custom endpoint for downloads - --b2-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --b2-endpoint string Endpoint for the service - --b2-hard-delete Permanently delete files on remote removal, otherwise hide files - --b2-key string Application Key - --b2-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging - --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --b2-version-at Time Show file versions as they were at the specified time (default off) - --b2-versions Include old versions in directory listings - --box-access-token string Box App Primary Access Token - --box-auth-url string Auth server URL - --box-box-config-file string Box App config.json location - --box-box-sub-type string (default \[dq]user\[dq]) - --box-client-id string OAuth Client Id - --box-client-secret string OAuth Client Secret - --box-commit-retries int Max number of times to try committing a multipart file (default 100) - --box-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot) - --box-list-chunk int Size of listing chunk 1-1000 (default 1000) - --box-owned-by string Only show items owned by the login (email address) passed in - --box-root-folder-id string Fill in for rclone to use a non root folder as its starting point - --box-token string OAuth Access Token as a JSON blob - --box-token-url string Token server url - --box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50 MiB) (default 50Mi) - --cache-chunk-clean-interval Duration How often should the cache perform cleanups of the chunk storage (default 1m0s) - --cache-chunk-no-memory Disable the in-memory cache for storing chunks during streaming - --cache-chunk-path string Directory to cache chunk files (default \[dq]$HOME/.cache/rclone/cache-backend\[dq]) - --cache-chunk-size SizeSuffix The size of a chunk (partial file data) (default 5Mi) - --cache-chunk-total-size SizeSuffix The total size that the chunks can take up on the local disk (default 10Gi) - --cache-db-path string Directory to store file structure metadata DB (default \[dq]$HOME/.cache/rclone/cache-backend\[dq]) - --cache-db-purge Clear all the cached data for this remote on start - --cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s) - --cache-info-age Duration How long to cache file structure information (directory listings, file size, times, etc.) (default 6h0m0s) - --cache-plex-insecure string Skip all certificate verification when connecting to the Plex server - --cache-plex-password string The password of the Plex user (obscured) - --cache-plex-url string The URL of the Plex server - --cache-plex-username string The username of the Plex user - --cache-read-retries int How many times to retry a read from a cache storage (default 10) - --cache-remote string Remote to cache - --cache-rps int Limits the number of requests per second to the source FS (-1 to disable) (default -1) - --cache-tmp-upload-path string Directory to keep temporary files until they are uploaded - --cache-tmp-wait-time Duration How long should files be stored in local cache before being uploaded (default 15s) - --cache-workers int How many workers should run in parallel to download chunks (default 4) - --cache-writes Cache file data on writes through the FS - --chunker-chunk-size SizeSuffix Files larger than chunk size will be split in chunks (default 2Gi) - --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks - --chunker-hash-type string Choose how chunker handles hash sums (default \[dq]md5\[dq]) - --chunker-remote string Remote to chunk/unchunk - --combine-upstreams SpaceSepList Upstreams for combining - --compress-level int GZIP compression level (-2 to 9) (default -1) - --compress-mode string Compression mode (default \[dq]gzip\[dq]) - --compress-ram-cache-limit SizeSuffix Some remotes don\[aq]t allow the upload of files with unknown size (default 20Mi) - --compress-remote string Remote to compress - -L, --copy-links Follow symlinks and copy the pointed to item - --crypt-directory-name-encryption Option to either encrypt directory names or leave them intact (default true) - --crypt-filename-encoding string How to encode the encrypted filename to text string (default \[dq]base32\[dq]) - --crypt-filename-encryption string How to encrypt the filenames (default \[dq]standard\[dq]) - --crypt-no-data-encryption Option to either encrypt file data or leave it unencrypted - --crypt-password string Password or pass phrase for encryption (obscured) - --crypt-password2 string Password or pass phrase for salt (obscured) - --crypt-remote string Remote to encrypt/decrypt - --crypt-server-side-across-configs Allow server-side operations (e.g. copy) to work across different crypt configs - --crypt-show-mapping For all files listed show how the names encrypt - --drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded - --drive-allow-import-name-change Allow the filetype to change when uploading Google docs - --drive-auth-owner-only Only consider files owned by the authenticated user - --drive-auth-url string Auth server URL - --drive-chunk-size SizeSuffix Upload chunk size (default 8Mi) - --drive-client-id string Google Application Client Id - --drive-client-secret string OAuth Client Secret - --drive-copy-shortcut-content Server side copy contents of shortcuts instead of the shortcut - --drive-disable-http2 Disable drive using http2 (default true) - --drive-encoding MultiEncoder The encoding for the backend (default InvalidUtf8) - --drive-export-formats string Comma separated list of preferred formats for downloading Google docs (default \[dq]docx,xlsx,pptx,svg\[dq]) - --drive-formats string Deprecated: See export_formats - --drive-impersonate string Impersonate this user when using a service account - --drive-import-formats string Comma separated list of preferred formats for uploading Google docs - --drive-keep-revision-forever Keep new head revision of each file forever - --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) - --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) - --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) - --drive-resource-key string Resource key for accessing a link-shared file - --drive-root-folder-id string ID of the root folder - --drive-scope string Scope that rclone should use when requesting access from drive - --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs - --drive-service-account-credentials string Service Account Credentials JSON blob - --drive-service-account-file string Service Account Credentials JSON file path - --drive-shared-with-me Only show files that are shared with me - --drive-size-as-quota Show sizes as storage quota usage, not actual size - --drive-skip-checksum-gphotos Skip MD5 checksum on Google photos and videos only - --drive-skip-dangling-shortcuts If set skip dangling shortcut files - --drive-skip-gdocs Skip google documents in all listings - --drive-skip-shortcuts If set skip shortcut files - --drive-starred-only Only show files that are starred - --drive-stop-on-download-limit Make download limit errors be fatal - --drive-stop-on-upload-limit Make upload limit errors be fatal - --drive-team-drive string ID of the Shared Drive (Team Drive) - --drive-token string OAuth Access Token as a JSON blob - --drive-token-url string Token server url - --drive-trashed-only Only show files that are in the trash - --drive-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 8Mi) - --drive-use-created-date Use file created date instead of modified date - --drive-use-shared-date Use date file was shared instead of modified date - --drive-use-trash Send files to the trash instead of deleting permanently (default true) - --drive-v2-download-min-size SizeSuffix If Object\[aq]s are greater, use drive v2 API to download (default off) - --dropbox-auth-url string Auth server URL - --dropbox-batch-commit-timeout Duration Max time to wait for a batch to finish committing (default 10m0s) - --dropbox-batch-mode string Upload file batching sync|async|off (default \[dq]sync\[dq]) - --dropbox-batch-size int Max number of files in upload batch - --dropbox-batch-timeout Duration Max time to allow an idle upload batch before uploading (default 0s) - --dropbox-chunk-size SizeSuffix Upload chunk size (< 150Mi) (default 48Mi) - --dropbox-client-id string OAuth Client Id - --dropbox-client-secret string OAuth Client Secret - --dropbox-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,RightSpace,InvalidUtf8,Dot) - --dropbox-impersonate string Impersonate this user when using a business account - --dropbox-shared-files Instructs rclone to work on individual shared files - --dropbox-shared-folders Instructs rclone to work on shared folders - --dropbox-token string OAuth Access Token as a JSON blob - --dropbox-token-url string Token server url - --fichier-api-key string Your API Key, get it from https://1fichier.com/console/params.pl - --fichier-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot) - --fichier-file-password string If you want to download a shared file that is password protected, add this parameter (obscured) - --fichier-folder-password string If you want to list the files in a shared folder that is password protected, add this parameter (obscured) - --fichier-shared-folder string If you want to download a shared folder, add this parameter - --filefabric-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) - --filefabric-permanent-token string Permanent Authentication Token - --filefabric-root-folder-id string ID of the root folder - --filefabric-token string Session Token - --filefabric-token-expiry string Token expiry time - --filefabric-url string URL of the Enterprise File Fabric to connect to - --filefabric-version string Version read from the file fabric - --ftp-ask-password Allow asking for FTP password when needed - --ftp-close-timeout Duration Maximum time to wait for a response to close (default 1m0s) - --ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited - --ftp-disable-epsv Disable using EPSV even if server advertises support - --ftp-disable-mlsd Disable using MLSD even if server advertises support - --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) - --ftp-disable-utf8 Disable using UTF-8 even if server advertises support - --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) - --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) - --ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD - --ftp-host string FTP host to connect to - --ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --ftp-no-check-certificate Do not verify the TLS certificate of the server - --ftp-pass string FTP password (obscured) - --ftp-port int FTP port number (default 21) - --ftp-shut-timeout Duration Maximum time to wait for data connection closing status (default 1m0s) - --ftp-tls Use Implicit FTPS (FTP over TLS) - --ftp-tls-cache-size int Size of TLS session cache for all control and data connections (default 32) - --ftp-user string FTP username (default \[dq]$USER\[dq]) - --ftp-writing-mdtm Use MDTM to set modification time (VsFtpd quirk) - --gcs-anonymous Access public buckets and objects without credentials - --gcs-auth-url string Auth server URL - --gcs-bucket-acl string Access Control List for new buckets - --gcs-bucket-policy-only Access checks should use bucket-level IAM policies - --gcs-client-id string OAuth Client Id - --gcs-client-secret string OAuth Client Secret - --gcs-decompress If set this will decompress gzip encoded objects - --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) - --gcs-endpoint string Endpoint for the service - --gcs-location string Location for the newly created buckets - --gcs-no-check-bucket If set, don\[aq]t attempt to check the bucket exists or create it - --gcs-object-acl string Access Control List for new objects - --gcs-project-number string Project number - --gcs-service-account-file string Service Account Credentials JSON file path - --gcs-storage-class string The storage class to use when storing objects in Google Cloud Storage - --gcs-token string OAuth Access Token as a JSON blob - --gcs-token-url string Token server url - --gphotos-auth-url string Auth server URL - --gphotos-client-id string OAuth Client Id - --gphotos-client-secret string OAuth Client Secret - --gphotos-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) - --gphotos-include-archived Also view and download archived media - --gphotos-read-only Set to make the Google Photos backend read only - --gphotos-read-size Set to read the size of media items - --gphotos-start-year int Year limits the photos to be downloaded to those which are uploaded after the given year (default 2000) - --gphotos-token string OAuth Access Token as a JSON blob - --gphotos-token-url string Token server url - --hasher-auto-size SizeSuffix Auto-update checksum for files smaller than this size (disabled by default) - --hasher-hashes CommaSepList Comma separated list of supported checksum types (default md5,sha1) - --hasher-max-age Duration Maximum time to keep checksums in cache (0 = no cache, off = cache forever) (default off) - --hasher-remote string Remote to cache checksums for (e.g. myRemote:path) - --hdfs-data-transfer-protection string Kerberos data transfer protection: authentication|integrity|privacy - --hdfs-encoding MultiEncoder The encoding for the backend (default Slash,Colon,Del,Ctl,InvalidUtf8,Dot) - --hdfs-namenode string Hadoop name node and port - --hdfs-service-principal-name string Kerberos service principal name for the namenode - --hdfs-username string Hadoop user name - --hidrive-auth-url string Auth server URL - --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) - --hidrive-client-id string OAuth Client Id - --hidrive-client-secret string OAuth Client Secret - --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary - --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) - --hidrive-endpoint string Endpoint for the service (default \[dq]https://api.hidrive.strato.com/2.1\[dq]) - --hidrive-root-prefix string The root/parent folder for all paths (default \[dq]/\[dq]) - --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default \[dq]rw\[dq]) - --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default \[dq]user\[dq]) - --hidrive-token string OAuth Access Token as a JSON blob - --hidrive-token-url string Token server url - --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) - --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) - --http-headers CommaSepList Set HTTP headers for all transactions - --http-no-head Don\[aq]t use HEAD requests - --http-no-slash Set this if the site doesn\[aq]t end directories with / - --http-url string URL of HTTP host to connect to - --internetarchive-access-key-id string IAS3 Access Key - --internetarchive-disable-checksum Don\[aq]t ask the server to test against MD5 checksum calculated by rclone (default true) - --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) - --internetarchive-endpoint string IAS3 Endpoint (default \[dq]https://s3.us.archive.org\[dq]) - --internetarchive-front-endpoint string Host of InternetArchive Frontend (default \[dq]https://archive.org\[dq]) - --internetarchive-secret-access-key string IAS3 Secret Key (password) - --internetarchive-wait-archive Duration Timeout for waiting the server\[aq]s processing tasks (specifically archive and book_op) to finish (default 0s) - --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) - --jottacloud-hard-delete Delete files permanently rather than putting them into the trash - --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) - --jottacloud-no-versions Avoid server side versioning by deleting files and recreating files instead of overwriting them - --jottacloud-trashed-only Only show files that are in the trash - --jottacloud-upload-resume-limit SizeSuffix Files bigger than this can be resumed if the upload fail\[aq]s (default 10Mi) - --koofr-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --koofr-endpoint string The Koofr API endpoint to use - --koofr-mountid string Mount ID of the mount to use - --koofr-password string Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured) - --koofr-provider string Choose your storage provider - --koofr-setmtime Does the backend support setting modification time (default true) - --koofr-user string Your user name - -l, --links Translate symlinks to/from regular files with a \[aq].rclonelink\[aq] extension - --local-case-insensitive Force the filesystem to report itself as case insensitive - --local-case-sensitive Force the filesystem to report itself as case sensitive - --local-encoding MultiEncoder The encoding for the backend (default Slash,Dot) - --local-no-check-updated Don\[aq]t check to see if the files change during upload - --local-no-preallocate Disable preallocation of disk space for transferred files - --local-no-set-modtime Disable setting modtime - --local-no-sparse Disable sparse files for multi-thread downloads - --local-nounc Disable UNC (long path names) conversion on Windows - --local-unicode-normalization Apply unicode NFC normalization to paths and filenames - --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) - --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) - --mailru-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --mailru-pass string Password (obscured) - --mailru-speedup-enable Skip full upload if there is another file with same data hash (default true) - --mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash) (default \[dq]*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf\[dq]) - --mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3Gi) - --mailru-speedup-max-memory SizeSuffix Files larger than the size given below will always be hashed on disk (default 32Mi) - --mailru-user string User name (usually email) - --mega-debug Output more debug from Mega - --mega-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --mega-hard-delete Delete files permanently rather than putting them into the trash - --mega-pass string Password (obscured) - --mega-user string User name - --netstorage-account string Set the NetStorage account name - --netstorage-host string Domain+path of NetStorage host to connect to - --netstorage-protocol string Select between HTTP or HTTPS protocol (default \[dq]https\[dq]) - --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) - -x, --one-file-system Don\[aq]t cross filesystem boundaries (unix/macOS only) - --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) - --onedrive-auth-url string Auth server URL - --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) - --onedrive-client-id string OAuth Client Id - --onedrive-client-secret string OAuth Client Secret - --onedrive-drive-id string The ID of the drive to use - --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) - --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) - --onedrive-expose-onenote-files Set to make OneNote files show up in directory listings - --onedrive-link-password string Set the password for links created by the link command - --onedrive-link-scope string Set the scope of the links created by the link command (default \[dq]anonymous\[dq]) - --onedrive-link-type string Set the type of the links created by the link command (default \[dq]view\[dq]) - --onedrive-list-chunk int Size of listing chunk (default 1000) - --onedrive-no-versions Remove all versions on modifying operations - --onedrive-region string Choose national cloud region for OneDrive (default \[dq]global\[dq]) - --onedrive-root-folder-id string ID of the root folder - --onedrive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different onedrive configs - --onedrive-token string OAuth Access Token as a JSON blob - --onedrive-token-url string Token server url - --oos-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) - --oos-compartment string Object storage compartment OCID - --oos-config-file string Path to OCI config file (default \[dq]\[ti]/.oci/config\[dq]) - --oos-config-profile string Profile name inside the oci config file (default \[dq]Default\[dq]) - --oos-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) - --oos-copy-timeout Duration Timeout for copy (default 1m0s) - --oos-disable-checksum Don\[aq]t store MD5 checksum with object metadata - --oos-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --oos-endpoint string Endpoint for Object storage API - --oos-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery - --oos-namespace string Object storage namespace - --oos-no-check-bucket If set, don\[aq]t attempt to check the bucket exists or create it - --oos-provider string Choose your Auth Provider (default \[dq]env_auth\[dq]) - --oos-region string Object storage Region - --oos-upload-concurrency int Concurrency for multipart uploads (default 10) - --oos-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size (default 10Mi) - --opendrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot) - --opendrive-password string Password (obscured) - --opendrive-username string Username - --pcloud-auth-url string Auth server URL - --pcloud-client-id string OAuth Client Id - --pcloud-client-secret string OAuth Client Secret - --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --pcloud-hostname string Hostname to connect to (default \[dq]api.pcloud.com\[dq]) - --pcloud-password string Your pcloud password (obscured) - --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default \[dq]d0\[dq]) - --pcloud-token string OAuth Access Token as a JSON blob - --pcloud-token-url string Token server url - --pcloud-username string Your pcloud username - --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) - --qingstor-access-key-id string QingStor Access Key ID - --qingstor-chunk-size SizeSuffix Chunk size to use for uploading (default 4Mi) - --qingstor-connection-retries int Number of connection retries (default 3) - --qingstor-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8) - --qingstor-endpoint string Enter an endpoint URL to connection QingStor API - --qingstor-env-auth Get QingStor credentials from runtime - --qingstor-secret-access-key string QingStor Secret Access Key (password) - --qingstor-upload-concurrency int Concurrency for multipart uploads (default 1) - --qingstor-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --qingstor-zone string Zone to connect to - --s3-access-key-id string AWS Access Key ID - --s3-acl string Canned ACL used when creating buckets and storing or copying objects - --s3-bucket-acl string Canned ACL used when creating buckets - --s3-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) - --s3-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) - --s3-decompress If set this will decompress gzip encoded objects - --s3-disable-checksum Don\[aq]t store MD5 checksum with object metadata - --s3-disable-http2 Disable usage of http2 for S3 backends - --s3-download-url string Custom endpoint for downloads - --s3-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) - --s3-endpoint string Endpoint for S3 API - --s3-env-auth Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars) - --s3-force-path-style If true use path style access if false use virtual hosted style (default true) - --s3-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery - --s3-list-chunk int Size of listing chunk (response list for each ListObject S3 request) (default 1000) - --s3-list-url-encode Tristate Whether to url encode listings: true/false/unset (default unset) - --s3-list-version int Version of ListObjects to use: 1,2 or 0 for auto - --s3-location-constraint string Location constraint - must be set to match the Region - --s3-max-upload-parts int Maximum number of parts in a multipart upload (default 10000) - --s3-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) - --s3-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool - --s3-no-check-bucket If set, don\[aq]t attempt to check the bucket exists or create it - --s3-no-head If set, don\[aq]t HEAD uploaded objects to check integrity - --s3-no-head-object If set, do not do HEAD before GET when getting objects - --s3-no-system-metadata Suppress setting and reading of system metadata - --s3-profile string Profile to use in the shared credentials file - --s3-provider string Choose your S3 provider - --s3-region string Region to connect to - --s3-requester-pays Enables requester pays option when interacting with S3 bucket - --s3-secret-access-key string AWS Secret Access Key (password) - --s3-server-side-encryption string The server-side encryption algorithm used when storing this object in S3 - --s3-session-token string An AWS session token - --s3-shared-credentials-file string Path to the shared credentials file - --s3-sse-customer-algorithm string If using SSE-C, the server-side encryption algorithm used when storing this object in S3 - --s3-sse-customer-key string To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data - --s3-sse-customer-key-base64 string If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data - --s3-sse-customer-key-md5 string If using SSE-C you may provide the secret encryption key MD5 checksum (optional) - --s3-sse-kms-key-id string If using KMS ID you must provide the ARN of Key - --s3-storage-class string The storage class to use when storing new objects in S3 - --s3-upload-concurrency int Concurrency for multipart uploads (default 4) - --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) - --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint - --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) - --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads - --s3-v2-auth If true use v2 authentication - --s3-version-at Time Show file versions as they were at the specified time (default off) - --s3-versions Include old versions in directory listings - --seafile-2fa Two-factor authentication (\[aq]true\[aq] if the account has 2FA enabled) - --seafile-create-library Should rclone create a library if it doesn\[aq]t exist - --seafile-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8) - --seafile-library string Name of the library - --seafile-library-key string Library password (for encrypted libraries only) (obscured) - --seafile-pass string Password (obscured) - --seafile-url string URL of seafile host to connect to - --seafile-user string User name (usually email address) - --sftp-ask-password Allow asking for SFTP password when needed - --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) - --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) - --sftp-disable-concurrent-reads If set don\[aq]t use concurrent reads - --sftp-disable-concurrent-writes If set don\[aq]t use concurrent writes - --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available - --sftp-host string SSH host to connect to - --sftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --sftp-key-file string Path to PEM-encoded private key file - --sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file (obscured) - --sftp-key-pem string Raw PEM-encoded private key - --sftp-key-use-agent When set forces the usage of the ssh-agent - --sftp-known-hosts-file string Optional path to known_hosts file - --sftp-md5sum-command string The command used to read md5 hashes - --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) - --sftp-path-override string Override path used by SSH shell commands - --sftp-port int SSH port number (default 22) - --sftp-pubkey-file string Optional path to public key file - --sftp-server-command string Specifies the path or command to run a sftp server on the remote host - --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands - --sftp-set-modtime Set the modified time on the remote if set (default true) - --sftp-sha1sum-command string The command used to read sha1 hashes - --sftp-shell-type string The type of SSH shell on remote server, if any - --sftp-skip-links Set to skip any symlinks and any other non regular files - --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default \[dq]sftp\[dq]) - --sftp-use-fstat If set use fstat instead of stat - --sftp-use-insecure-cipher Enable the use of insecure ciphers and key exchange methods - --sftp-user string SSH username (default \[dq]$USER\[dq]) - --sharefile-chunk-size SizeSuffix Upload chunk size (default 64Mi) - --sharefile-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,LeftSpace,LeftPeriod,RightSpace,RightPeriod,InvalidUtf8,Dot) - --sharefile-endpoint string Endpoint for API calls - --sharefile-root-folder-id string ID of the root folder - --sharefile-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (default 128Mi) - --sia-api-password string Sia Daemon API Password (obscured) - --sia-api-url string Sia daemon API URL, like http://sia.daemon.host:9980 (default \[dq]http://127.0.0.1:9980\[dq]) - --sia-encoding MultiEncoder The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot) - --sia-user-agent string Siad User Agent (default \[dq]Sia-Agent\[dq]) - --skip-links Don\[aq]t warn about skipped symlinks - --smb-case-insensitive Whether the server is configured to be case-insensitive (default true) - --smb-domain string Domain name for NTLM authentication (default \[dq]WORKGROUP\[dq]) - --smb-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot) - --smb-hide-special-share Hide special shares (e.g. print$) which users aren\[aq]t supposed to access (default true) - --smb-host string SMB server hostname to connect to - --smb-idle-timeout Duration Max time before closing idle connections (default 1m0s) - --smb-pass string SMB password (obscured) - --smb-port int SMB port number (default 445) - --smb-user string SMB username (default \[dq]$USER\[dq]) - --storj-access-grant string Access grant - --storj-api-key string API key - --storj-passphrase string Encryption passphrase - --storj-provider string Choose an authentication method (default \[dq]existing\[dq]) - --storj-satellite-address string Satellite address (default \[dq]us-central-1.storj.io\[dq]) - --sugarsync-access-key-id string Sugarsync Access Key ID - --sugarsync-app-id string Sugarsync App ID - --sugarsync-authorization string Sugarsync authorization - --sugarsync-authorization-expiry string Sugarsync authorization expiry - --sugarsync-deleted-id string Sugarsync deleted folder id - --sugarsync-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8,Dot) - --sugarsync-hard-delete Permanently delete files if true - --sugarsync-private-access-key string Sugarsync Private Access Key - --sugarsync-refresh-token string Sugarsync refresh token - --sugarsync-root-id string Sugarsync root id - --sugarsync-user string Sugarsync user - --swift-application-credential-id string Application Credential ID (OS_APPLICATION_CREDENTIAL_ID) - --swift-application-credential-name string Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME) - --swift-application-credential-secret string Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET) - --swift-auth string Authentication URL for server (OS_AUTH_URL) - --swift-auth-token string Auth Token from alternate authentication - optional (OS_AUTH_TOKEN) - --swift-auth-version int AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION) - --swift-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) - --swift-domain string User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME) - --swift-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8) - --swift-endpoint-type string Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default \[dq]public\[dq]) - --swift-env-auth Get swift credentials from environment variables in standard OpenStack form - --swift-key string API key or password (OS_PASSWORD) - --swift-leave-parts-on-error If true avoid calling abort upload on a failure - --swift-no-chunk Don\[aq]t chunk files during streaming upload - --swift-no-large-objects Disable support for static and dynamic large objects - --swift-region string Region name - optional (OS_REGION_NAME) - --swift-storage-policy string The storage policy to use when creating a new container - --swift-storage-url string Storage URL - optional (OS_STORAGE_URL) - --swift-tenant string Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME) - --swift-tenant-domain string Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME) - --swift-tenant-id string Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID) - --swift-user string User name to log in (OS_USERNAME) - --swift-user-id string User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID) - --union-action-policy string Policy to choose upstream on ACTION category (default \[dq]epall\[dq]) - --union-cache-time int Cache time of usage and free space (in seconds) (default 120) - --union-create-policy string Policy to choose upstream on CREATE category (default \[dq]epmfs\[dq]) - --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) - --union-search-policy string Policy to choose upstream on SEARCH category (default \[dq]ff\[dq]) - --union-upstreams string List of space separated upstreams - --uptobox-access-token string Your access token - --uptobox-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot) - --webdav-bearer-token string Bearer token instead of user/pass (e.g. a Macaroon) - --webdav-bearer-token-command string Command to run to get a bearer token - --webdav-encoding string The encoding for the backend - --webdav-headers CommaSepList Set HTTP headers for all transactions - --webdav-pass string Password (obscured) - --webdav-url string URL of http host to connect to - --webdav-user string User name - --webdav-vendor string Name of the WebDAV site/service/software you are using - --yandex-auth-url string Auth server URL - --yandex-client-id string OAuth Client Id - --yandex-client-secret string OAuth Client Secret - --yandex-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) - --yandex-hard-delete Delete files permanently rather than putting them into the trash - --yandex-token string OAuth Access Token as a JSON blob - --yandex-token-url string Token server url - --zoho-auth-url string Auth server URL - --zoho-client-id string OAuth Client Id - --zoho-client-secret string OAuth Client Secret - --zoho-encoding MultiEncoder The encoding for the backend (default Del,Ctl,InvalidUtf8) - --zoho-region string Zoho region to connect to - --zoho-token string OAuth Access Token as a JSON blob - --zoho-token-url string Token server url + --acd-auth-url string Auth server URL + --acd-client-id string OAuth Client Id + --acd-client-secret string OAuth Client Secret + --acd-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --acd-templink-threshold SizeSuffix Files >= this size will be downloaded via their tempLink (default 9Gi) + --acd-token string OAuth Access Token as a JSON blob + --acd-token-url string Token server url + --acd-upload-wait-per-gb Duration Additional time per GiB to wait after a failed complete upload to see if it appears (default 3m0s) + --alias-remote string Remote or path to alias + --azureblob-access-tier string Access tier of blob: hot, cool or archive + --azureblob-account string Azure Storage Account Name + --azureblob-archive-tier-delete Delete archive tier blobs before overwriting + --azureblob-chunk-size SizeSuffix Upload chunk size (default 4Mi) + --azureblob-client-certificate-password string Password for the certificate file (optional) (obscured) + --azureblob-client-certificate-path string Path to a PEM or PKCS12 certificate file including the private key + --azureblob-client-id string The ID of the client in use + --azureblob-client-secret string One of the service principal\[aq]s client secrets + --azureblob-client-send-certificate-chain Send the certificate chain when using certificate auth + --azureblob-disable-checksum Don\[aq]t store MD5 checksum with object metadata + --azureblob-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8) + --azureblob-endpoint string Endpoint for the service + --azureblob-env-auth Read credentials from runtime (environment variables, CLI or MSI) + --azureblob-key string Storage Account Shared Key + --azureblob-list-chunk int Size of blob list (default 5000) + --azureblob-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --azureblob-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --azureblob-msi-client-id string Object ID of the user-assigned MSI to use, if any + --azureblob-msi-mi-res-id string Azure resource ID of the user-assigned MSI to use, if any + --azureblob-msi-object-id string Object ID of the user-assigned MSI to use, if any + --azureblob-no-check-container If set, don\[aq]t attempt to check the container exists or create it + --azureblob-no-head-object If set, do not do HEAD before GET when getting objects + --azureblob-password string The user\[aq]s password (obscured) + --azureblob-public-access string Public access level of a container: blob or container + --azureblob-sas-url string SAS URL for container level access only + --azureblob-service-principal-file string Path to file containing credentials for use with a service principal + --azureblob-tenant string ID of the service principal\[aq]s tenant. Also called its directory ID + --azureblob-upload-concurrency int Concurrency for multipart uploads (default 16) + --azureblob-upload-cutoff string Cutoff for switching to chunked upload (<= 256 MiB) (deprecated) + --azureblob-use-emulator Uses local storage emulator if provided as \[aq]true\[aq] + --azureblob-use-msi Use a managed service identity to authenticate (only works in Azure) + --azureblob-username string User name (usually an email address) + --b2-account string Account ID or Application Key ID + --b2-chunk-size SizeSuffix Upload chunk size (default 96Mi) + --b2-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4Gi) + --b2-disable-checksum Disable checksums for large (> upload cutoff) files + --b2-download-auth-duration Duration Time before the authorization token will expire in s or suffix ms|s|m|h|d (default 1w) + --b2-download-url string Custom endpoint for downloads + --b2-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --b2-endpoint string Endpoint for the service + --b2-hard-delete Permanently delete files on remote removal, otherwise hide files + --b2-key string Application Key + --b2-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --b2-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --b2-test-mode string A flag string for X-Bz-Test-Mode header for debugging + --b2-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --b2-version-at Time Show file versions as they were at the specified time (default off) + --b2-versions Include old versions in directory listings + --box-access-token string Box App Primary Access Token + --box-auth-url string Auth server URL + --box-box-config-file string Box App config.json location + --box-box-sub-type string (default \[dq]user\[dq]) + --box-client-id string OAuth Client Id + --box-client-secret string OAuth Client Secret + --box-commit-retries int Max number of times to try committing a multipart file (default 100) + --box-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot) + --box-list-chunk int Size of listing chunk 1-1000 (default 1000) + --box-owned-by string Only show items owned by the login (email address) passed in + --box-root-folder-id string Fill in for rclone to use a non root folder as its starting point + --box-token string OAuth Access Token as a JSON blob + --box-token-url string Token server url + --box-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (>= 50 MiB) (default 50Mi) + --cache-chunk-clean-interval Duration How often should the cache perform cleanups of the chunk storage (default 1m0s) + --cache-chunk-no-memory Disable the in-memory cache for storing chunks during streaming + --cache-chunk-path string Directory to cache chunk files (default \[dq]$HOME/.cache/rclone/cache-backend\[dq]) + --cache-chunk-size SizeSuffix The size of a chunk (partial file data) (default 5Mi) + --cache-chunk-total-size SizeSuffix The total size that the chunks can take up on the local disk (default 10Gi) + --cache-db-path string Directory to store file structure metadata DB (default \[dq]$HOME/.cache/rclone/cache-backend\[dq]) + --cache-db-purge Clear all the cached data for this remote on start + --cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s) + --cache-info-age Duration How long to cache file structure information (directory listings, file size, times, etc.) (default 6h0m0s) + --cache-plex-insecure string Skip all certificate verification when connecting to the Plex server + --cache-plex-password string The password of the Plex user (obscured) + --cache-plex-url string The URL of the Plex server + --cache-plex-username string The username of the Plex user + --cache-read-retries int How many times to retry a read from a cache storage (default 10) + --cache-remote string Remote to cache + --cache-rps int Limits the number of requests per second to the source FS (-1 to disable) (default -1) + --cache-tmp-upload-path string Directory to keep temporary files until they are uploaded + --cache-tmp-wait-time Duration How long should files be stored in local cache before being uploaded (default 15s) + --cache-workers int How many workers should run in parallel to download chunks (default 4) + --cache-writes Cache file data on writes through the FS + --chunker-chunk-size SizeSuffix Files larger than chunk size will be split in chunks (default 2Gi) + --chunker-fail-hard Choose how chunker should handle files with missing or invalid chunks + --chunker-hash-type string Choose how chunker handles hash sums (default \[dq]md5\[dq]) + --chunker-remote string Remote to chunk/unchunk + --combine-upstreams SpaceSepList Upstreams for combining + --compress-level int GZIP compression level (-2 to 9) (default -1) + --compress-mode string Compression mode (default \[dq]gzip\[dq]) + --compress-ram-cache-limit SizeSuffix Some remotes don\[aq]t allow the upload of files with unknown size (default 20Mi) + --compress-remote string Remote to compress + -L, --copy-links Follow symlinks and copy the pointed to item + --crypt-directory-name-encryption Option to either encrypt directory names or leave them intact (default true) + --crypt-filename-encoding string How to encode the encrypted filename to text string (default \[dq]base32\[dq]) + --crypt-filename-encryption string How to encrypt the filenames (default \[dq]standard\[dq]) + --crypt-no-data-encryption Option to either encrypt file data or leave it unencrypted + --crypt-password string Password or pass phrase for encryption (obscured) + --crypt-password2 string Password or pass phrase for salt (obscured) + --crypt-remote string Remote to encrypt/decrypt + --crypt-server-side-across-configs Allow server-side operations (e.g. copy) to work across different crypt configs + --crypt-show-mapping For all files listed show how the names encrypt + --drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded + --drive-allow-import-name-change Allow the filetype to change when uploading Google docs + --drive-auth-owner-only Only consider files owned by the authenticated user + --drive-auth-url string Auth server URL + --drive-chunk-size SizeSuffix Upload chunk size (default 8Mi) + --drive-client-id string Google Application Client Id + --drive-client-secret string OAuth Client Secret + --drive-copy-shortcut-content Server side copy contents of shortcuts instead of the shortcut + --drive-disable-http2 Disable drive using http2 (default true) + --drive-encoding MultiEncoder The encoding for the backend (default InvalidUtf8) + --drive-export-formats string Comma separated list of preferred formats for downloading Google docs (default \[dq]docx,xlsx,pptx,svg\[dq]) + --drive-formats string Deprecated: See export_formats + --drive-impersonate string Impersonate this user when using a service account + --drive-import-formats string Comma separated list of preferred formats for uploading Google docs + --drive-keep-revision-forever Keep new head revision of each file forever + --drive-list-chunk int Size of listing chunk 100-1000, 0 to disable (default 1000) + --drive-pacer-burst int Number of API calls to allow without sleeping (default 100) + --drive-pacer-min-sleep Duration Minimum time to sleep between API calls (default 100ms) + --drive-resource-key string Resource key for accessing a link-shared file + --drive-root-folder-id string ID of the root folder + --drive-scope string Scope that rclone should use when requesting access from drive + --drive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different drive configs + --drive-service-account-credentials string Service Account Credentials JSON blob + --drive-service-account-file string Service Account Credentials JSON file path + --drive-shared-with-me Only show files that are shared with me + --drive-size-as-quota Show sizes as storage quota usage, not actual size + --drive-skip-checksum-gphotos Skip MD5 checksum on Google photos and videos only + --drive-skip-dangling-shortcuts If set skip dangling shortcut files + --drive-skip-gdocs Skip google documents in all listings + --drive-skip-shortcuts If set skip shortcut files + --drive-starred-only Only show files that are starred + --drive-stop-on-download-limit Make download limit errors be fatal + --drive-stop-on-upload-limit Make upload limit errors be fatal + --drive-team-drive string ID of the Shared Drive (Team Drive) + --drive-token string OAuth Access Token as a JSON blob + --drive-token-url string Token server url + --drive-trashed-only Only show files that are in the trash + --drive-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 8Mi) + --drive-use-created-date Use file created date instead of modified date + --drive-use-shared-date Use date file was shared instead of modified date + --drive-use-trash Send files to the trash instead of deleting permanently (default true) + --drive-v2-download-min-size SizeSuffix If Object\[aq]s are greater, use drive v2 API to download (default off) + --dropbox-auth-url string Auth server URL + --dropbox-batch-commit-timeout Duration Max time to wait for a batch to finish committing (default 10m0s) + --dropbox-batch-mode string Upload file batching sync|async|off (default \[dq]sync\[dq]) + --dropbox-batch-size int Max number of files in upload batch + --dropbox-batch-timeout Duration Max time to allow an idle upload batch before uploading (default 0s) + --dropbox-chunk-size SizeSuffix Upload chunk size (< 150Mi) (default 48Mi) + --dropbox-client-id string OAuth Client Id + --dropbox-client-secret string OAuth Client Secret + --dropbox-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,RightSpace,InvalidUtf8,Dot) + --dropbox-impersonate string Impersonate this user when using a business account + --dropbox-shared-files Instructs rclone to work on individual shared files + --dropbox-shared-folders Instructs rclone to work on shared folders + --dropbox-token string OAuth Access Token as a JSON blob + --dropbox-token-url string Token server url + --fichier-api-key string Your API Key, get it from https://1fichier.com/console/params.pl + --fichier-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,BackSlash,Del,Ctl,LeftSpace,RightSpace,InvalidUtf8,Dot) + --fichier-file-password string If you want to download a shared file that is password protected, add this parameter (obscured) + --fichier-folder-password string If you want to list the files in a shared folder that is password protected, add this parameter (obscured) + --fichier-shared-folder string If you want to download a shared folder, add this parameter + --filefabric-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) + --filefabric-permanent-token string Permanent Authentication Token + --filefabric-root-folder-id string ID of the root folder + --filefabric-token string Session Token + --filefabric-token-expiry string Token expiry time + --filefabric-url string URL of the Enterprise File Fabric to connect to + --filefabric-version string Version read from the file fabric + --ftp-ask-password Allow asking for FTP password when needed + --ftp-close-timeout Duration Maximum time to wait for a response to close (default 1m0s) + --ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited + --ftp-disable-epsv Disable using EPSV even if server advertises support + --ftp-disable-mlsd Disable using MLSD even if server advertises support + --ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS) + --ftp-disable-utf8 Disable using UTF-8 even if server advertises support + --ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot) + --ftp-explicit-tls Use Explicit FTPS (FTP over TLS) + --ftp-force-list-hidden Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD + --ftp-host string FTP host to connect to + --ftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --ftp-no-check-certificate Do not verify the TLS certificate of the server + --ftp-pass string FTP password (obscured) + --ftp-port int FTP port number (default 21) + --ftp-shut-timeout Duration Maximum time to wait for data connection closing status (default 1m0s) + --ftp-tls Use Implicit FTPS (FTP over TLS) + --ftp-tls-cache-size int Size of TLS session cache for all control and data connections (default 32) + --ftp-user string FTP username (default \[dq]$USER\[dq]) + --ftp-writing-mdtm Use MDTM to set modification time (VsFtpd quirk) + --gcs-anonymous Access public buckets and objects without credentials + --gcs-auth-url string Auth server URL + --gcs-bucket-acl string Access Control List for new buckets + --gcs-bucket-policy-only Access checks should use bucket-level IAM policies + --gcs-client-id string OAuth Client Id + --gcs-client-secret string OAuth Client Secret + --gcs-decompress If set this will decompress gzip encoded objects + --gcs-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) + --gcs-endpoint string Endpoint for the service + --gcs-location string Location for the newly created buckets + --gcs-no-check-bucket If set, don\[aq]t attempt to check the bucket exists or create it + --gcs-object-acl string Access Control List for new objects + --gcs-project-number string Project number + --gcs-service-account-file string Service Account Credentials JSON file path + --gcs-storage-class string The storage class to use when storing objects in Google Cloud Storage + --gcs-token string OAuth Access Token as a JSON blob + --gcs-token-url string Token server url + --gphotos-auth-url string Auth server URL + --gphotos-client-id string OAuth Client Id + --gphotos-client-secret string OAuth Client Secret + --gphotos-encoding MultiEncoder The encoding for the backend (default Slash,CrLf,InvalidUtf8,Dot) + --gphotos-include-archived Also view and download archived media + --gphotos-read-only Set to make the Google Photos backend read only + --gphotos-read-size Set to read the size of media items + --gphotos-start-year int Year limits the photos to be downloaded to those which are uploaded after the given year (default 2000) + --gphotos-token string OAuth Access Token as a JSON blob + --gphotos-token-url string Token server url + --hasher-auto-size SizeSuffix Auto-update checksum for files smaller than this size (disabled by default) + --hasher-hashes CommaSepList Comma separated list of supported checksum types (default md5,sha1) + --hasher-max-age Duration Maximum time to keep checksums in cache (0 = no cache, off = cache forever) (default off) + --hasher-remote string Remote to cache checksums for (e.g. myRemote:path) + --hdfs-data-transfer-protection string Kerberos data transfer protection: authentication|integrity|privacy + --hdfs-encoding MultiEncoder The encoding for the backend (default Slash,Colon,Del,Ctl,InvalidUtf8,Dot) + --hdfs-namenode string Hadoop name node and port + --hdfs-service-principal-name string Kerberos service principal name for the namenode + --hdfs-username string Hadoop user name + --hidrive-auth-url string Auth server URL + --hidrive-chunk-size SizeSuffix Chunksize for chunked uploads (default 48Mi) + --hidrive-client-id string OAuth Client Id + --hidrive-client-secret string OAuth Client Secret + --hidrive-disable-fetching-member-count Do not fetch number of objects in directories unless it is absolutely necessary + --hidrive-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --hidrive-endpoint string Endpoint for the service (default \[dq]https://api.hidrive.strato.com/2.1\[dq]) + --hidrive-root-prefix string The root/parent folder for all paths (default \[dq]/\[dq]) + --hidrive-scope-access string Access permissions that rclone should use when requesting access from HiDrive (default \[dq]rw\[dq]) + --hidrive-scope-role string User-level that rclone should use when requesting access from HiDrive (default \[dq]user\[dq]) + --hidrive-token string OAuth Access Token as a JSON blob + --hidrive-token-url string Token server url + --hidrive-upload-concurrency int Concurrency for chunked uploads (default 4) + --hidrive-upload-cutoff SizeSuffix Cutoff/Threshold for chunked uploads (default 96Mi) + --http-headers CommaSepList Set HTTP headers for all transactions + --http-no-head Don\[aq]t use HEAD requests + --http-no-slash Set this if the site doesn\[aq]t end directories with / + --http-url string URL of HTTP host to connect to + --internetarchive-access-key-id string IAS3 Access Key + --internetarchive-disable-checksum Don\[aq]t ask the server to test against MD5 checksum calculated by rclone (default true) + --internetarchive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot) + --internetarchive-endpoint string IAS3 Endpoint (default \[dq]https://s3.us.archive.org\[dq]) + --internetarchive-front-endpoint string Host of InternetArchive Frontend (default \[dq]https://archive.org\[dq]) + --internetarchive-secret-access-key string IAS3 Secret Key (password) + --internetarchive-wait-archive Duration Timeout for waiting the server\[aq]s processing tasks (specifically archive and book_op) to finish (default 0s) + --jottacloud-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,Del,Ctl,InvalidUtf8,Dot) + --jottacloud-hard-delete Delete files permanently rather than putting them into the trash + --jottacloud-md5-memory-limit SizeSuffix Files bigger than this will be cached on disk to calculate the MD5 if required (default 10Mi) + --jottacloud-no-versions Avoid server side versioning by deleting files and recreating files instead of overwriting them + --jottacloud-trashed-only Only show files that are in the trash + --jottacloud-upload-resume-limit SizeSuffix Files bigger than this can be resumed if the upload fail\[aq]s (default 10Mi) + --koofr-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --koofr-endpoint string The Koofr API endpoint to use + --koofr-mountid string Mount ID of the mount to use + --koofr-password string Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured) + --koofr-provider string Choose your storage provider + --koofr-setmtime Does the backend support setting modification time (default true) + --koofr-user string Your user name + -l, --links Translate symlinks to/from regular files with a \[aq].rclonelink\[aq] extension + --local-case-insensitive Force the filesystem to report itself as case insensitive + --local-case-sensitive Force the filesystem to report itself as case sensitive + --local-encoding MultiEncoder The encoding for the backend (default Slash,Dot) + --local-no-check-updated Don\[aq]t check to see if the files change during upload + --local-no-preallocate Disable preallocation of disk space for transferred files + --local-no-set-modtime Disable setting modtime + --local-no-sparse Disable sparse files for multi-thread downloads + --local-nounc Disable UNC (long path names) conversion on Windows + --local-unicode-normalization Apply unicode NFC normalization to paths and filenames + --local-zero-size-links Assume the Stat size of links is zero (and read them instead) (deprecated) + --mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true) + --mailru-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --mailru-pass string Password (obscured) + --mailru-speedup-enable Skip full upload if there is another file with same data hash (default true) + --mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash) (default \[dq]*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf\[dq]) + --mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3Gi) + --mailru-speedup-max-memory SizeSuffix Files larger than the size given below will always be hashed on disk (default 32Mi) + --mailru-user string User name (usually email) + --mega-debug Output more debug from Mega + --mega-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --mega-hard-delete Delete files permanently rather than putting them into the trash + --mega-pass string Password (obscured) + --mega-user string User name + --netstorage-account string Set the NetStorage account name + --netstorage-host string Domain+path of NetStorage host to connect to + --netstorage-protocol string Select between HTTP or HTTPS protocol (default \[dq]https\[dq]) + --netstorage-secret string Set the NetStorage account secret/G2O key for authentication (obscured) + -x, --one-file-system Don\[aq]t cross filesystem boundaries (unix/macOS only) + --onedrive-access-scopes SpaceSepList Set scopes to be requested by rclone (default Files.Read Files.ReadWrite Files.Read.All Files.ReadWrite.All Sites.Read.All offline_access) + --onedrive-auth-url string Auth server URL + --onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes) (default 10Mi) + --onedrive-client-id string OAuth Client Id + --onedrive-client-secret string OAuth Client Secret + --onedrive-drive-id string The ID of the drive to use + --onedrive-drive-type string The type of the drive (personal | business | documentLibrary) + --onedrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,LeftSpace,LeftTilde,RightSpace,RightPeriod,InvalidUtf8,Dot) + --onedrive-expose-onenote-files Set to make OneNote files show up in directory listings + --onedrive-link-password string Set the password for links created by the link command + --onedrive-link-scope string Set the scope of the links created by the link command (default \[dq]anonymous\[dq]) + --onedrive-link-type string Set the type of the links created by the link command (default \[dq]view\[dq]) + --onedrive-list-chunk int Size of listing chunk (default 1000) + --onedrive-no-versions Remove all versions on modifying operations + --onedrive-region string Choose national cloud region for OneDrive (default \[dq]global\[dq]) + --onedrive-root-folder-id string ID of the root folder + --onedrive-server-side-across-configs Allow server-side operations (e.g. copy) to work across different onedrive configs + --onedrive-token string OAuth Access Token as a JSON blob + --onedrive-token-url string Token server url + --oos-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) + --oos-compartment string Object storage compartment OCID + --oos-config-file string Path to OCI config file (default \[dq]\[ti]/.oci/config\[dq]) + --oos-config-profile string Profile name inside the oci config file (default \[dq]Default\[dq]) + --oos-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) + --oos-copy-timeout Duration Timeout for copy (default 1m0s) + --oos-disable-checksum Don\[aq]t store MD5 checksum with object metadata + --oos-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --oos-endpoint string Endpoint for Object storage API + --oos-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery + --oos-namespace string Object storage namespace + --oos-no-check-bucket If set, don\[aq]t attempt to check the bucket exists or create it + --oos-provider string Choose your Auth Provider (default \[dq]env_auth\[dq]) + --oos-region string Object storage Region + --oos-upload-concurrency int Concurrency for multipart uploads (default 10) + --oos-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size (default 10Mi) + --opendrive-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot) + --opendrive-password string Password (obscured) + --opendrive-username string Username + --pcloud-auth-url string Auth server URL + --pcloud-client-id string OAuth Client Id + --pcloud-client-secret string OAuth Client Secret + --pcloud-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --pcloud-hostname string Hostname to connect to (default \[dq]api.pcloud.com\[dq]) + --pcloud-password string Your pcloud password (obscured) + --pcloud-root-folder-id string Fill in for rclone to use a non root folder as its starting point (default \[dq]d0\[dq]) + --pcloud-token string OAuth Access Token as a JSON blob + --pcloud-token-url string Token server url + --pcloud-username string Your pcloud username + --premiumizeme-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --putio-encoding MultiEncoder The encoding for the backend (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot) + --qingstor-access-key-id string QingStor Access Key ID + --qingstor-chunk-size SizeSuffix Chunk size to use for uploading (default 4Mi) + --qingstor-connection-retries int Number of connection retries (default 3) + --qingstor-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8) + --qingstor-endpoint string Enter an endpoint URL to connection QingStor API + --qingstor-env-auth Get QingStor credentials from runtime + --qingstor-secret-access-key string QingStor Secret Access Key (password) + --qingstor-upload-concurrency int Concurrency for multipart uploads (default 1) + --qingstor-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --qingstor-zone string Zone to connect to + --s3-access-key-id string AWS Access Key ID + --s3-acl string Canned ACL used when creating buckets and storing or copying objects + --s3-bucket-acl string Canned ACL used when creating buckets + --s3-chunk-size SizeSuffix Chunk size to use for uploading (default 5Mi) + --s3-copy-cutoff SizeSuffix Cutoff for switching to multipart copy (default 4.656Gi) + --s3-decompress If set this will decompress gzip encoded objects + --s3-disable-checksum Don\[aq]t store MD5 checksum with object metadata + --s3-disable-http2 Disable usage of http2 for S3 backends + --s3-download-url string Custom endpoint for downloads + --s3-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8,Dot) + --s3-endpoint string Endpoint for S3 API + --s3-env-auth Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars) + --s3-force-path-style If true use path style access if false use virtual hosted style (default true) + --s3-leave-parts-on-error If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery + --s3-list-chunk int Size of listing chunk (response list for each ListObject S3 request) (default 1000) + --s3-list-url-encode Tristate Whether to url encode listings: true/false/unset (default unset) + --s3-list-version int Version of ListObjects to use: 1,2 or 0 for auto + --s3-location-constraint string Location constraint - must be set to match the Region + --s3-max-upload-parts int Maximum number of parts in a multipart upload (default 10000) + --s3-memory-pool-flush-time Duration How often internal memory buffer pools will be flushed (default 1m0s) + --s3-memory-pool-use-mmap Whether to use mmap buffers in internal memory pool + --s3-might-gzip Tristate Set this if the backend might gzip objects (default unset) + --s3-no-check-bucket If set, don\[aq]t attempt to check the bucket exists or create it + --s3-no-head If set, don\[aq]t HEAD uploaded objects to check integrity + --s3-no-head-object If set, do not do HEAD before GET when getting objects + --s3-no-system-metadata Suppress setting and reading of system metadata + --s3-profile string Profile to use in the shared credentials file + --s3-provider string Choose your S3 provider + --s3-region string Region to connect to + --s3-requester-pays Enables requester pays option when interacting with S3 bucket + --s3-secret-access-key string AWS Secret Access Key (password) + --s3-server-side-encryption string The server-side encryption algorithm used when storing this object in S3 + --s3-session-token string An AWS session token + --s3-shared-credentials-file string Path to the shared credentials file + --s3-sse-customer-algorithm string If using SSE-C, the server-side encryption algorithm used when storing this object in S3 + --s3-sse-customer-key string To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data + --s3-sse-customer-key-base64 string If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data + --s3-sse-customer-key-md5 string If using SSE-C you may provide the secret encryption key MD5 checksum (optional) + --s3-sse-kms-key-id string If using KMS ID you must provide the ARN of Key + --s3-storage-class string The storage class to use when storing new objects in S3 + --s3-upload-concurrency int Concurrency for multipart uploads (default 4) + --s3-upload-cutoff SizeSuffix Cutoff for switching to chunked upload (default 200Mi) + --s3-use-accelerate-endpoint If true use the AWS S3 accelerated endpoint + --s3-use-multipart-etag Tristate Whether to use ETag in multipart uploads for verification (default unset) + --s3-use-presigned-request Whether to use a presigned request or PutObject for single part uploads + --s3-v2-auth If true use v2 authentication + --s3-version-at Time Show file versions as they were at the specified time (default off) + --s3-versions Include old versions in directory listings + --seafile-2fa Two-factor authentication (\[aq]true\[aq] if the account has 2FA enabled) + --seafile-create-library Should rclone create a library if it doesn\[aq]t exist + --seafile-encoding MultiEncoder The encoding for the backend (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8) + --seafile-library string Name of the library + --seafile-library-key string Library password (for encrypted libraries only) (obscured) + --seafile-pass string Password (obscured) + --seafile-url string URL of seafile host to connect to + --seafile-user string User name (usually email address) + --sftp-ask-password Allow asking for SFTP password when needed + --sftp-chunk-size SizeSuffix Upload and download chunk size (default 32Ki) + --sftp-ciphers SpaceSepList Space separated list of ciphers to be used for session encryption, ordered by preference + --sftp-concurrency int The maximum number of outstanding requests for one file (default 64) + --sftp-disable-concurrent-reads If set don\[aq]t use concurrent reads + --sftp-disable-concurrent-writes If set don\[aq]t use concurrent writes + --sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available + --sftp-host string SSH host to connect to + --sftp-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --sftp-key-exchange SpaceSepList Space separated list of key exchange algorithms, ordered by preference + --sftp-key-file string Path to PEM-encoded private key file + --sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file (obscured) + --sftp-key-pem string Raw PEM-encoded private key + --sftp-key-use-agent When set forces the usage of the ssh-agent + --sftp-known-hosts-file string Optional path to known_hosts file + --sftp-macs SpaceSepList Space separated list of MACs (message authentication code) algorithms, ordered by preference + --sftp-md5sum-command string The command used to read md5 hashes + --sftp-pass string SSH password, leave blank to use ssh-agent (obscured) + --sftp-path-override string Override path used by SSH shell commands + --sftp-port int SSH port number (default 22) + --sftp-pubkey-file string Optional path to public key file + --sftp-server-command string Specifies the path or command to run a sftp server on the remote host + --sftp-set-env SpaceSepList Environment variables to pass to sftp and commands + --sftp-set-modtime Set the modified time on the remote if set (default true) + --sftp-sha1sum-command string The command used to read sha1 hashes + --sftp-shell-type string The type of SSH shell on remote server, if any + --sftp-skip-links Set to skip any symlinks and any other non regular files + --sftp-subsystem string Specifies the SSH2 subsystem on the remote host (default \[dq]sftp\[dq]) + --sftp-use-fstat If set use fstat instead of stat + --sftp-use-insecure-cipher Enable the use of insecure ciphers and key exchange methods + --sftp-user string SSH username (default \[dq]$USER\[dq]) + --sharefile-chunk-size SizeSuffix Upload chunk size (default 64Mi) + --sharefile-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,LeftSpace,LeftPeriod,RightSpace,RightPeriod,InvalidUtf8,Dot) + --sharefile-endpoint string Endpoint for API calls + --sharefile-root-folder-id string ID of the root folder + --sharefile-upload-cutoff SizeSuffix Cutoff for switching to multipart upload (default 128Mi) + --sia-api-password string Sia Daemon API Password (obscured) + --sia-api-url string Sia daemon API URL, like http://sia.daemon.host:9980 (default \[dq]http://127.0.0.1:9980\[dq]) + --sia-encoding MultiEncoder The encoding for the backend (default Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot) + --sia-user-agent string Siad User Agent (default \[dq]Sia-Agent\[dq]) + --skip-links Don\[aq]t warn about skipped symlinks + --smb-case-insensitive Whether the server is configured to be case-insensitive (default true) + --smb-domain string Domain name for NTLM authentication (default \[dq]WORKGROUP\[dq]) + --smb-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Ctl,RightSpace,RightPeriod,InvalidUtf8,Dot) + --smb-hide-special-share Hide special shares (e.g. print$) which users aren\[aq]t supposed to access (default true) + --smb-host string SMB server hostname to connect to + --smb-idle-timeout Duration Max time before closing idle connections (default 1m0s) + --smb-pass string SMB password (obscured) + --smb-port int SMB port number (default 445) + --smb-user string SMB username (default \[dq]$USER\[dq]) + --storj-access-grant string Access grant + --storj-api-key string API key + --storj-passphrase string Encryption passphrase + --storj-provider string Choose an authentication method (default \[dq]existing\[dq]) + --storj-satellite-address string Satellite address (default \[dq]us-central-1.storj.io\[dq]) + --sugarsync-access-key-id string Sugarsync Access Key ID + --sugarsync-app-id string Sugarsync App ID + --sugarsync-authorization string Sugarsync authorization + --sugarsync-authorization-expiry string Sugarsync authorization expiry + --sugarsync-deleted-id string Sugarsync deleted folder id + --sugarsync-encoding MultiEncoder The encoding for the backend (default Slash,Ctl,InvalidUtf8,Dot) + --sugarsync-hard-delete Permanently delete files if true + --sugarsync-private-access-key string Sugarsync Private Access Key + --sugarsync-refresh-token string Sugarsync refresh token + --sugarsync-root-id string Sugarsync root id + --sugarsync-user string Sugarsync user + --swift-application-credential-id string Application Credential ID (OS_APPLICATION_CREDENTIAL_ID) + --swift-application-credential-name string Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME) + --swift-application-credential-secret string Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET) + --swift-auth string Authentication URL for server (OS_AUTH_URL) + --swift-auth-token string Auth Token from alternate authentication - optional (OS_AUTH_TOKEN) + --swift-auth-version int AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION) + --swift-chunk-size SizeSuffix Above this size files will be chunked into a _segments container (default 5Gi) + --swift-domain string User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME) + --swift-encoding MultiEncoder The encoding for the backend (default Slash,InvalidUtf8) + --swift-endpoint-type string Endpoint type to choose from the service catalogue (OS_ENDPOINT_TYPE) (default \[dq]public\[dq]) + --swift-env-auth Get swift credentials from environment variables in standard OpenStack form + --swift-key string API key or password (OS_PASSWORD) + --swift-leave-parts-on-error If true avoid calling abort upload on a failure + --swift-no-chunk Don\[aq]t chunk files during streaming upload + --swift-no-large-objects Disable support for static and dynamic large objects + --swift-region string Region name - optional (OS_REGION_NAME) + --swift-storage-policy string The storage policy to use when creating a new container + --swift-storage-url string Storage URL - optional (OS_STORAGE_URL) + --swift-tenant string Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME) + --swift-tenant-domain string Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME) + --swift-tenant-id string Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID) + --swift-user string User name to log in (OS_USERNAME) + --swift-user-id string User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID) + --union-action-policy string Policy to choose upstream on ACTION category (default \[dq]epall\[dq]) + --union-cache-time int Cache time of usage and free space (in seconds) (default 120) + --union-create-policy string Policy to choose upstream on CREATE category (default \[dq]epmfs\[dq]) + --union-min-free-space SizeSuffix Minimum viable free space for lfs/eplfs policies (default 1Gi) + --union-search-policy string Policy to choose upstream on SEARCH category (default \[dq]ff\[dq]) + --union-upstreams string List of space separated upstreams + --uptobox-access-token string Your access token + --uptobox-encoding MultiEncoder The encoding for the backend (default Slash,LtGt,DoubleQuote,BackQuote,Del,Ctl,LeftSpace,InvalidUtf8,Dot) + --webdav-bearer-token string Bearer token instead of user/pass (e.g. a Macaroon) + --webdav-bearer-token-command string Command to run to get a bearer token + --webdav-encoding string The encoding for the backend + --webdav-headers CommaSepList Set HTTP headers for all transactions + --webdav-pass string Password (obscured) + --webdav-url string URL of http host to connect to + --webdav-user string User name + --webdav-vendor string Name of the WebDAV site/service/software you are using + --yandex-auth-url string Auth server URL + --yandex-client-id string OAuth Client Id + --yandex-client-secret string OAuth Client Secret + --yandex-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,InvalidUtf8,Dot) + --yandex-hard-delete Delete files permanently rather than putting them into the trash + --yandex-token string OAuth Access Token as a JSON blob + --yandex-token-url string Token server url + --zoho-auth-url string Auth server URL + --zoho-client-id string OAuth Client Id + --zoho-client-secret string OAuth Client Secret + --zoho-encoding MultiEncoder The encoding for the backend (default Del,Ctl,InvalidUtf8) + --zoho-region string Zoho region to connect to + --zoho-token string OAuth Access Token as a JSON blob + --zoho-token-url string Token server url \f[R] .fi .SH Docker Volume Plugin @@ -23136,9 +23433,10 @@ Token server url - leave blank to use Amazon\[aq]s. token_url> Optional token URL Remote config Make sure your Redirect URL is set to \[dq]http://127.0.0.1:53682/\[dq] in your custom config. -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -23479,6 +23777,8 @@ IDrive e2 .IP \[bu] 2 IONOS Cloud .IP \[bu] 2 +Liara Object Storage +.IP \[bu] 2 Minio .IP \[bu] 2 Qiniu Cloud Object Storage (Kodo) @@ -23566,7 +23866,7 @@ name> remote Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS +XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Liara, Minio, and Tencent COS \[rs] \[dq]s3\[dq] [snip] Storage> s3 @@ -23576,7 +23876,7 @@ Choose a number from below, or type in your own value \[rs] \[dq]AWS\[dq] 2 / Ceph Object Storage \[rs] \[dq]Ceph\[dq] - 3 / Digital Ocean Spaces + 3 / DigitalOcean Spaces \[rs] \[dq]DigitalOcean\[dq] 4 / Dreamhost DreamObjects \[rs] \[dq]Dreamhost\[dq] @@ -24267,9 +24567,9 @@ all the files to be uploaded as multipart. .PP Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, -Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, -IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, -SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). +Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, +IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, +Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). .SS --s3-provider .PP Choose your S3 provider. @@ -24326,7 +24626,7 @@ Arvan Cloud Object Storage (AOS) \[dq]DigitalOcean\[dq] .RS 2 .IP \[bu] 2 -Digital Ocean Spaces +DigitalOcean Spaces .RE .IP \[bu] 2 \[dq]Dreamhost\[dq] @@ -24365,6 +24665,12 @@ IONOS Cloud Seagate Lyve Cloud .RE .IP \[bu] 2 +\[dq]Liara\[dq] +.RS 2 +.IP \[bu] 2 +Liara Object Storage +.RE +.IP \[bu] 2 \[dq]Minio\[dq] .RS 2 .IP \[bu] 2 @@ -25150,7 +25456,7 @@ Config: region Env Var: RCLONE_S3_REGION .IP \[bu] 2 Provider: -!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive +!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Liara,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive .IP \[bu] 2 Type: string .IP \[bu] 2 @@ -25861,6 +26167,33 @@ Logrono, Spain .RE .SS --s3-endpoint .PP +Endpoint for Liara Object Storage API. +.PP +Properties: +.IP \[bu] 2 +Config: endpoint +.IP \[bu] 2 +Env Var: RCLONE_S3_ENDPOINT +.IP \[bu] 2 +Provider: Liara +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]storage.iran.liara.space\[dq] +.RS 2 +.IP \[bu] 2 +The default endpoint +.IP \[bu] 2 +Iran +.RE +.RE +.SS --s3-endpoint +.PP Endpoint for OSS API. .PP Properties: @@ -26588,7 +26921,7 @@ Config: endpoint Env Var: RCLONE_S3_ENDPOINT .IP \[bu] 2 Provider: -!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu +!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu .IP \[bu] 2 Type: string .IP \[bu] 2 @@ -26603,22 +26936,40 @@ Examples: Dream Objects endpoint .RE .IP \[bu] 2 +\[dq]syd1.digitaloceanspaces.com\[dq] +.RS 2 +.IP \[bu] 2 +DigitalOcean Spaces Sydney 1 +.RE +.IP \[bu] 2 +\[dq]sfo3.digitaloceanspaces.com\[dq] +.RS 2 +.IP \[bu] 2 +DigitalOcean Spaces San Francisco 3 +.RE +.IP \[bu] 2 +\[dq]fra1.digitaloceanspaces.com\[dq] +.RS 2 +.IP \[bu] 2 +DigitalOcean Spaces Frankfurt 1 +.RE +.IP \[bu] 2 \[dq]nyc3.digitaloceanspaces.com\[dq] .RS 2 .IP \[bu] 2 -Digital Ocean Spaces New York 3 +DigitalOcean Spaces New York 3 .RE .IP \[bu] 2 \[dq]ams3.digitaloceanspaces.com\[dq] .RS 2 .IP \[bu] 2 -Digital Ocean Spaces Amsterdam 3 +DigitalOcean Spaces Amsterdam 3 .RE .IP \[bu] 2 \[dq]sgp1.digitaloceanspaces.com\[dq] .RS 2 .IP \[bu] 2 -Digital Ocean Spaces Singapore 1 +DigitalOcean Spaces Singapore 1 .RE .IP \[bu] 2 \[dq]localhost:8333\[dq] @@ -26648,19 +26999,57 @@ Seagate Lyve Cloud AP Southeast 1 (Singapore) \[dq]s3.wasabisys.com\[dq] .RS 2 .IP \[bu] 2 -Wasabi US East endpoint +Wasabi US East 1 (N. +Virginia) +.RE +.IP \[bu] 2 +\[dq]s3.us-east-2.wasabisys.com\[dq] +.RS 2 +.IP \[bu] 2 +Wasabi US East 2 (N. +Virginia) +.RE +.IP \[bu] 2 +\[dq]s3.us-central-1.wasabisys.com\[dq] +.RS 2 +.IP \[bu] 2 +Wasabi US Central 1 (Texas) .RE .IP \[bu] 2 \[dq]s3.us-west-1.wasabisys.com\[dq] .RS 2 .IP \[bu] 2 -Wasabi US West endpoint +Wasabi US West 1 (Oregon) +.RE +.IP \[bu] 2 +\[dq]s3.ca-central-1.wasabisys.com\[dq] +.RS 2 +.IP \[bu] 2 +Wasabi CA Central 1 (Toronto) .RE .IP \[bu] 2 \[dq]s3.eu-central-1.wasabisys.com\[dq] .RS 2 .IP \[bu] 2 -Wasabi EU Central endpoint +Wasabi EU Central 1 (Amsterdam) +.RE +.IP \[bu] 2 +\[dq]s3.eu-central-2.wasabisys.com\[dq] +.RS 2 +.IP \[bu] 2 +Wasabi EU Central 2 (Frankfurt) +.RE +.IP \[bu] 2 +\[dq]s3.eu-west-1.wasabisys.com\[dq] +.RS 2 +.IP \[bu] 2 +Wasabi EU West 1 (London) +.RE +.IP \[bu] 2 +\[dq]s3.eu-west-2.wasabisys.com\[dq] +.RS 2 +.IP \[bu] 2 +Wasabi EU West 2 (Paris) .RE .IP \[bu] 2 \[dq]s3.ap-northeast-1.wasabisys.com\[dq] @@ -26675,6 +27064,24 @@ Wasabi AP Northeast 1 (Tokyo) endpoint Wasabi AP Northeast 2 (Osaka) endpoint .RE .IP \[bu] 2 +\[dq]s3.ap-southeast-1.wasabisys.com\[dq] +.RS 2 +.IP \[bu] 2 +Wasabi AP Southeast 1 (Singapore) +.RE +.IP \[bu] 2 +\[dq]s3.ap-southeast-2.wasabisys.com\[dq] +.RS 2 +.IP \[bu] 2 +Wasabi AP Southeast 2 (Sydney) +.RE +.IP \[bu] 2 +\[dq]storage.iran.liara.space\[dq] +.RS 2 +.IP \[bu] 2 +Liara Iran endpoint +.RE +.IP \[bu] 2 \[dq]s3.ir-thr-at1.arvanstorage.com\[dq] .RS 2 .IP \[bu] 2 @@ -27510,7 +27917,7 @@ Config: location_constraint Env Var: RCLONE_S3_LOCATION_CONSTRAINT .IP \[bu] 2 Provider: -!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS +!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Liara,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS .IP \[bu] 2 Type: string .IP \[bu] 2 @@ -27528,6 +27935,9 @@ https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Note that this ACL is applied when server-side copying objects as S3 doesn\[aq]t copy the ACL from the source but rather writes a fresh one. .PP +If the acl is an empty string then no X-Amz-Acl: header is added and the +default (private) will be used. +.PP Properties: .IP \[bu] 2 Config: acl @@ -27880,6 +28290,31 @@ Infrequent access storage mode .RE .SS --s3-storage-class .PP +The storage class to use when storing new objects in Liara +.PP +Properties: +.IP \[bu] 2 +Config: storage_class +.IP \[bu] 2 +Env Var: RCLONE_S3_STORAGE_CLASS +.IP \[bu] 2 +Provider: Liara +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.IP \[bu] 2 +Examples: +.RS 2 +.IP \[bu] 2 +\[dq]STANDARD\[dq] +.RS 2 +.IP \[bu] 2 +Standard storage class +.RE +.RE +.SS --s3-storage-class +.PP The storage class to use when storing new objects in ArvanCloud. .PP Properties: @@ -28034,9 +28469,9 @@ Deep archive storage mode .PP Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, -Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, -IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, -SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). +Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, +IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, +Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi). .SS --s3-bucket-acl .PP Canned ACL used when creating buckets. @@ -28047,6 +28482,9 @@ https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Note that this ACL is applied when only when creating buckets. If it isn\[aq]t set then \[dq]acl\[dq] is used instead. .PP +If the \[dq]acl\[dq] and \[dq]bucket_acl\[dq] are empty strings then no +X-Amz-Acl: header is added and the default (private) will be used. +.PP Properties: .IP \[bu] 2 Config: bucket_acl @@ -28823,6 +29261,42 @@ Env Var: RCLONE_S3_DECOMPRESS Type: bool .IP \[bu] 2 Default: false +.SS --s3-might-gzip +.PP +Set this if the backend might gzip objects. +.PP +Normally providers will not alter objects when they are downloaded. +If an object was not uploaded with \f[C]Content-Encoding: gzip\f[R] then +it won\[aq]t be set on download. +.PP +However some providers may gzip objects even if they weren\[aq]t +uploaded with \f[C]Content-Encoding: gzip\f[R] (eg Cloudflare). +.PP +A symptom of this would be receiving errors like +.IP +.nf +\f[C] +ERROR corrupted on transfer: sizes differ NNN vs MMM +\f[R] +.fi +.PP +If you set this flag and rclone downloads an object with +Content-Encoding: gzip set and chunked transfer encoding, then rclone +will decompress the object on the fly. +.PP +If this is set to unset (the default) then rclone will choose according +to the provider setting what to apply, but you can override rclone\[aq]s +choice here. +.PP +Properties: +.IP \[bu] 2 +Config: might_gzip +.IP \[bu] 2 +Env Var: RCLONE_S3_MIGHT_GZIP +.IP \[bu] 2 +Type: Tristate +.IP \[bu] 2 +Default: unset .SS --s3-no-system-metadata .PP Suppress setting and reading of system metadata @@ -29318,7 +29792,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. \&... -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \[rs] (s3) \&... Storage> s3 @@ -29518,7 +29992,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. [snip] - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \[rs] (s3) [snip] Storage> 5 @@ -29659,7 +30133,7 @@ Choose a number from below, or type in your own value \[rs] \[dq]alias\[dq] 2 / Amazon Drive \[rs] \[dq]amazon cloud drive\[dq] - 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS) + 3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, Liara, ArvanCloud, Minio, IBM COS) \[rs] \[dq]s3\[dq] 4 / Backblaze B2 \[rs] \[dq]b2\[dq] @@ -29861,7 +30335,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \[rs] (s3) [snip] Storage> s3 @@ -29978,7 +30452,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi \[rs] (s3) [snip] Storage> s3 @@ -30295,7 +30769,7 @@ Choose a number from below, or type in your own value \[rs] (alias) 4 / Amazon Drive \[rs] (amazon cloud drive) - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi \[rs] (s3) [snip] Storage> s3 @@ -30571,7 +31045,7 @@ Choose \f[C]s3\f[R] backend Type of storage to configure. Choose a number from below, or type in your own value. [snip] -XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS +XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \[rs] (s3) [snip] Storage> s3 @@ -30788,7 +31262,7 @@ name> wasabi Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, Liara) \[rs] \[dq]s3\[dq] [snip] Storage> s3 @@ -30910,7 +31384,7 @@ Type of storage to configure. Enter a string value. Press Enter for the default (\[dq]\[dq]). Choose a number from below, or type in your own value [snip] - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS \[rs] \[dq]s3\[dq] [snip] Storage> s3 @@ -31027,7 +31501,7 @@ Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. ... - 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS + 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS \[rs] (s3) ... Storage> s3 @@ -31257,18 +31731,19 @@ d) Delete this remote y/e/d> y \f[R] .fi -.SS ArvanCloud +.SS Liara .PP -ArvanCloud (https://www.arvancloud.com/en/products/cloud-storage) -ArvanCloud Object Storage goes beyond the limited traditional file -storage. -It gives you access to backup and archived files and allows sharing. -Files like profile image in the app, images sent by users or scanned -documents can be stored securely and easily in our Object Storage -service. +Here is an example of making a Liara Object +Storage (https://liara.ir/landing/object-storage) configuration. +First run: +.IP +.nf +\f[C] +rclone config +\f[R] +.fi .PP -ArvanCloud provides an S3 interface which can be configured for use with -rclone like this. +This will guide you through an interactive setup process. .IP .nf \f[C] @@ -31276,11 +31751,11 @@ No remotes found, make a new one? n) New remote s) Set configuration password n/s> n -name> ArvanCloud +name> Liara Type of storage to configure. Choose a number from below, or type in your own value [snip] -XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio) +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Liara, Minio) \[rs] \[dq]s3\[dq] [snip] Storage> s3 @@ -31297,22 +31772,130 @@ AWS Secret Access Key (password) - leave blank for anonymous access or runtime c secret_access_key> YOURSECRETACCESSKEY Region to connect to. Choose a number from below, or type in your own value - / The default endpoint - a good choice if you are unsure. + / The default endpoint 1 | US Region, Northern Virginia, or Pacific Northwest. | Leave location constraint empty. \[rs] \[dq]us-east-1\[dq] [snip] -region> +region> Endpoint for S3 API. -Leave blank if using ArvanCloud to use the default endpoint for the region. +Leave blank if using Liara to use the default endpoint for the region. Specify if using an S3 clone such as Ceph. -endpoint> s3.arvanstorage.com -Location constraint - must be set to match the Region. Used when creating buckets only. -Choose a number from below, or type in your own value - 1 / Empty for Iran-Tehran Region. - \[rs] \[dq]\[dq] -[snip] -location_constraint> +endpoint> storage.iran.liara.space +Canned ACL used when creating buckets and/or storing objects in S3. +For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Choose a number from below, or type in your own value + 1 / Owner gets FULL_CONTROL. No one else has access rights (default). + \[rs] \[dq]private\[dq] +[snip] +acl> +The server-side encryption algorithm used when storing this object in S3. +Choose a number from below, or type in your own value + 1 / None + \[rs] \[dq]\[dq] + 2 / AES256 + \[rs] \[dq]AES256\[dq] +server_side_encryption> +The storage class to use when storing objects in S3. +Choose a number from below, or type in your own value + 1 / Default + \[rs] \[dq]\[dq] + 2 / Standard storage class + \[rs] \[dq]STANDARD\[dq] +storage_class> +Remote config +-------------------- +[Liara] +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +endpoint = storage.iran.liara.space +location_constraint = +acl = +server_side_encryption = +storage_class = +-------------------- +y) Yes this is OK +e) Edit this remote +d) Delete this remote +y/e/d> y +\f[R] +.fi +.PP +This will leave the config file looking like this. +.IP +.nf +\f[C] +[Liara] +type = s3 +provider = Liara +env_auth = false +access_key_id = YOURACCESSKEY +secret_access_key = YOURSECRETACCESSKEY +region = +endpoint = storage.iran.liara.space +location_constraint = +acl = +server_side_encryption = +storage_class = +\f[R] +.fi +.SS ArvanCloud +.PP +ArvanCloud (https://www.arvancloud.com/en/products/cloud-storage) +ArvanCloud Object Storage goes beyond the limited traditional file +storage. +It gives you access to backup and archived files and allows sharing. +Files like profile image in the app, images sent by users or scanned +documents can be stored securely and easily in our Object Storage +service. +.PP +ArvanCloud provides an S3 interface which can be configured for use with +rclone like this. +.IP +.nf +\f[C] +No remotes found, make a new one? +n) New remote +s) Set configuration password +n/s> n +name> ArvanCloud +Type of storage to configure. +Choose a number from below, or type in your own value +[snip] +XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Liara, Minio) + \[rs] \[dq]s3\[dq] +[snip] +Storage> s3 +Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. +Choose a number from below, or type in your own value + 1 / Enter AWS credentials in the next step + \[rs] \[dq]false\[dq] + 2 / Get AWS credentials from the environment (env vars or IAM) + \[rs] \[dq]true\[dq] +env_auth> 1 +AWS Access Key ID - leave blank for anonymous access or runtime credentials. +access_key_id> YOURACCESSKEY +AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. +secret_access_key> YOURSECRETACCESSKEY +Region to connect to. +Choose a number from below, or type in your own value + / The default endpoint - a good choice if you are unsure. + 1 | US Region, Northern Virginia, or Pacific Northwest. + | Leave location constraint empty. + \[rs] \[dq]us-east-1\[dq] +[snip] +region> +Endpoint for S3 API. +Leave blank if using ArvanCloud to use the default endpoint for the region. +Specify if using an S3 clone such as Ceph. +endpoint> s3.arvanstorage.com +Location constraint - must be set to match the Region. Used when creating buckets only. +Choose a number from below, or type in your own value + 1 / Empty for Iran-Tehran Region. + \[rs] \[dq]\[dq] +[snip] +location_constraint> Canned ACL used when creating buckets and/or storing objects in S3. For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Choose a number from below, or type in your own value @@ -31414,7 +31997,7 @@ Choose a number from below, or type in your own value \[rs] \[dq]alias\[dq] 3 / Amazon Drive \[rs] \[dq]amazon cloud drive\[dq] - 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, Minio, and Tencent COS + 4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Liara, Minio, and Tencent COS \[rs] \[dq]s3\[dq] [snip] Storage> s3 @@ -32406,9 +32989,10 @@ Choose a number from below, or type in your own value \[rs] \[dq]enterprise\[dq] box_sub_type> Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -32546,9 +33130,10 @@ Already have a token - refresh? y) Yes n) No y/n> y -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -32895,6 +33480,10 @@ rclone maps this to and from an identical looking unicode equivalent .PP Box only supports filenames up to 255 characters in length. .PP +Box has API rate +limits (https://developer.box.com/guides/api-calls/permissions-and-errors/rate-limits/) +that sometimes reduce the speed of rclone. +.PP \f[C]rclone about\f[R] is not supported by the Box backend. Backends without this capability cannot determine free space for an rclone mount or use policy \f[C]mfs\f[R] (most free space) as a member @@ -32903,7 +33492,7 @@ of an rclone union remote. See List of backends that do not support rclone about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) -.SH Cache (DEPRECATED) +.SH Cache .PP The \f[C]cache\f[R] remote wraps another existing remote and stores file structure and its data for long running tasks like @@ -33732,7 +34321,7 @@ Print stats on the cache backend in JSON format. rclone backend stats remote: [options] [+] \f[R] .fi -.SH Chunker (BETA) +.SH Chunker .PP The \f[C]chunker\f[R] overlay transparently splits large files into smaller chunks during upload to wrapped remote and transparently @@ -34395,9 +34984,10 @@ y) Yes n) No y/n> n Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -35643,7 +36233,7 @@ For full protection against this you should always use a salt. .IP \[bu] 2 rclone cryptdecode (https://rclone.org/commands/rclone_cryptdecode/) - Show forward/reverse mapping of encrypted filenames -.SH Compress (Experimental) +.SH Compress .SS Warning .PP This remote is currently \f[B]experimental\f[R]. @@ -36064,6 +36654,17 @@ y/e/d> y \f[R] .fi .PP +See the remote setup docs (https://rclone.org/remote_setup/) for how to +set it up on a machine with no Internet browser available. +.PP +Note that rclone runs a webserver on your local machine to collect the +token as returned from Dropbox. +This only runs from the moment it opens your browser to the moment you +get back the verification code. +This is on \f[C]http://127.0.0.1:53682/\f[R] and it may require you to +unblock it temporarily if you are running a host firewall, or use manual +mode. +.PP You can then use it like this, .PP List directories in top level of your dropbox @@ -37218,7 +37819,7 @@ Use Implicit FTPS (FTP over TLS). When using implicit FTP over TLS the client connects using TLS right from the start which breaks compatibility with non-TLS-aware servers. This is usually served over port 990 rather than port 21. -Cannot be used in combination with explicit FTP. +Cannot be used in combination with explicit FTPS. .PP Properties: .IP \[bu] 2 @@ -37236,7 +37837,7 @@ Use Explicit FTPS (FTP over TLS). When using explicit FTP over TLS the client explicitly requests security from the server in order to upgrade a plain text connection to an encrypted one. -Cannot be used in combination with implicit FTP. +Cannot be used in combination with implicit FTPS. .PP Properties: .IP \[bu] 2 @@ -37667,9 +38268,10 @@ Choose a number from below, or type in your own value \[rs] \[dq]DURABLE_REDUCED_AVAILABILITY\[dq] storage_class> 5 Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine or Y didn\[aq]t work +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -37694,8 +38296,12 @@ y/e/d> y \f[R] .fi .PP +See the remote setup docs (https://rclone.org/remote_setup/) for how to +set it up on a machine with no Internet browser available. +.PP Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. +token as returned from Google if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on \f[C]http://127.0.0.1:53682/\f[R] and this it may require you @@ -38617,9 +39223,10 @@ scope> 1 Service Account Credentials JSON file path - needed only if you want use SA instead of interactive login. service_account_file> Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine or Y didn\[aq]t work +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -38647,8 +39254,12 @@ y/e/d> y \f[R] .fi .PP +See the remote setup docs (https://rclone.org/remote_setup/) for how to +set it up on a machine with no Internet browser available. +.PP Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. +token as returned from Google if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on \f[C]http://127.0.0.1:53682/\f[R] and it may require you to @@ -40729,9 +41340,10 @@ y) Yes n) No y/n> n Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -40756,8 +41368,12 @@ y/e/d> y \f[R] .fi .PP +See the remote setup docs (https://rclone.org/remote_setup/) for how to +set it up on a machine with no Internet browser available. +.PP Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. +token as returned from Google if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on \f[C]http://127.0.0.1:53682/\f[R] and this may require you to @@ -41205,7 +41821,7 @@ Rclone cannot delete files anywhere except under \f[C]album\f[R]. .PP The Google Photos API does not support deleting albums - see bug #135714733 (https://issuetracker.google.com/issues/135714733). -.SH Hasher (EXPERIMENTAL) +.SH Hasher .PP Hasher is a special overlay backend to create remotes which handle checksums for other remotes. @@ -41950,7 +42566,10 @@ Leave blank normally. scope_access> Edit advanced config? y/n> n -Use auto config? +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y/n> y If your browser doesn\[aq]t open automatically go to the following link: http://127.0.0.1:53682/auth?state=xxxxxxxxxxxxxxxxxxxxxx Log in and authorize rclone for access @@ -42812,6 +43431,26 @@ It can be triggered when you did a server-side copy. .PP Reading metadata will also provide custom (non-standard nor reserved) ones. +.SS Filtering auto generated files +.PP +The Internet Archive automatically creates metadata files after upload. +These can cause problems when doing an \f[C]rclone sync\f[R] as rclone +will try, and fail, to delete them. +These metadata files are not changeable, as they are created by the +Internet Archive automatically. +.PP +These auto-created files can be excluded from the sync using metadata +filtering (https://rclone.org/filtering/#metadata). +.IP +.nf +\f[C] +rclone sync ... --metadata-exclude \[dq]source=metadata\[dq] --metadata-exclude \[dq]format=Metadata\[dq] +\f[R] +.fi +.PP +Which excludes from the sync any files which have the +\f[C]source=metadata\f[R] or \f[C]format=Metadata\f[R] flags which are +added to Internet Archive auto-created files. .SS Configuration .PP Here is an example of making an internetarchive configuration. @@ -44213,7 +44852,32 @@ supported by rclone) .SS Configuration .PP Here is an example of making a mailru configuration. -First create a Mail.ru Cloud account and choose a tariff, then run +.PP +First create a Mail.ru Cloud account and choose a tariff. +.PP +You will need to log in and create an app password for rclone. +Rclone \f[B]will not work\f[R] with your normal username and password - +it will give an error like +\f[C]oauth2: server response missing access_token\f[R]. +.IP \[bu] 2 +Click on your user icon in the top right +.IP \[bu] 2 +Go to Security / \[dq]\[u041F]\[u0430]\[u0440]\[u043E]\[u043B]\[u044C] +\[u0438] +\[u0431]\[u0435]\[u0437]\[u043E]\[u043F]\[u0430]\[u0441]\[u043D]\[u043E]\[u0441]\[u0442]\[u044C]\[dq] +.IP \[bu] 2 +Click password for apps / +\[dq]\[u041F]\[u0430]\[u0440]\[u043E]\[u043B]\[u0438] +\[u0434]\[u043B]\[u044F] +\[u0432]\[u043D]\[u0435]\[u0448]\[u043D]\[u0438]\[u0445] +\[u043F]\[u0440]\[u0438]\[u043B]\[u043E]\[u0436]\[u0435]\[u043D]\[u0438]\[u0439]\[dq] +.IP \[bu] 2 +Add the password - give it a name - eg \[dq]rclone\[dq] +.IP \[bu] 2 +Copy the password and use this password below - your normal login +password won\[aq]t work. +.PP +Now run .IP .nf \f[C] @@ -44244,6 +44908,10 @@ User name (usually email) Enter a string value. Press Enter for the default (\[dq]\[dq]). user> username\[at]mail.ru Password + +This must be an app password - rclone will not work with your normal +password. See the Configuration section in the docs for how to make an +app password. y) Yes type in my own password g) Generate random password y/g> y @@ -44441,6 +45109,11 @@ Required: true .PP Password. .PP +This must be an app password - rclone will not work with your normal +password. +See the Configuration section in the docs for how to make an app +password. +.PP \f[B]NB\f[R] Input to this must be obscured - see rclone obscure (https://rclone.org/commands/rclone_obscure/). .PP @@ -45608,7 +46281,14 @@ The modified time is stored as metadata on the object with the \f[C]mtime\f[R] key. It is stored using RFC3339 Format time with nanosecond precision. The metadata is supplied during directory listings so there is no -overhead to using it. +performance overhead to using it. +.PP +If you wish to use the Azure standard \f[C]LastModified\f[R] time stored +on the object as the modified time, then use the +\f[C]--use-server-modtime\f[R] flag. +Note that rclone can\[aq]t set \f[C]LastModified\f[R], so using the +\f[C]--update\f[R] flag when syncing is recommended if using +\f[C]--use-server-modtime\f[R]. .SS Performance .PP When uploading large files, increasing the value of @@ -45682,10 +46362,126 @@ MD5 hashes are stored with blobs. However blobs that were uploaded in chunks only have an MD5 if the source remote was capable of MD5 hashes, e.g. the local disk. -.SS Authenticating with Azure Blob Storage +.SS Authentication +.PP +There are a number of ways of supplying credentials for Azure Blob +Storage. +Rclone tries them in the order of the sections below. +.SS Env Auth +.PP +If the \f[C]env_auth\f[R] config parameter is \f[C]true\f[R] then rclone +will pull credentials from the environment or runtime. +.PP +It tries these authentication methods in this order: +.IP "1." 3 +Environment Variables +.IP "2." 3 +Managed Service Identity Credentials +.IP "3." 3 +Azure CLI credentials (as used by the az tool) +.PP +These are described in the following sections +.SS Env Auth: 1. Environment Variables +.PP +If \f[C]env_auth\f[R] is set and environment variables are present +rclone authenticates a service principal with a secret or certificate, +or a user with a password, depending on which environment variable are +set. +It reads configuration from these variables, in the following order: +.IP "1." 3 +Service principal with client secret +.RS 4 +.IP \[bu] 2 +\f[C]AZURE_TENANT_ID\f[R]: ID of the service principal\[aq]s tenant. +Also called its \[dq]directory\[dq] ID. +.IP \[bu] 2 +\f[C]AZURE_CLIENT_ID\f[R]: the service principal\[aq]s client ID +.IP \[bu] 2 +\f[C]AZURE_CLIENT_SECRET\f[R]: one of the service principal\[aq]s client +secrets +.RE +.IP "2." 3 +Service principal with certificate +.RS 4 +.IP \[bu] 2 +\f[C]AZURE_TENANT_ID\f[R]: ID of the service principal\[aq]s tenant. +Also called its \[dq]directory\[dq] ID. +.IP \[bu] 2 +\f[C]AZURE_CLIENT_ID\f[R]: the service principal\[aq]s client ID +.IP \[bu] 2 +\f[C]AZURE_CLIENT_CERTIFICATE_PATH\f[R]: path to a PEM or PKCS12 +certificate file including the private key. +.IP \[bu] 2 +\f[C]AZURE_CLIENT_CERTIFICATE_PASSWORD\f[R]: (optional) password for the +certificate file. +.IP \[bu] 2 +\f[C]AZURE_CLIENT_SEND_CERTIFICATE_CHAIN\f[R]: (optional) Specifies +whether an authentication request will include an x5c header to support +subject name / issuer based authentication. +When set to \[dq]true\[dq] or \[dq]1\[dq], authentication requests +include the x5c header. +.RE +.IP "3." 3 +User with username and password +.RS 4 +.IP \[bu] 2 +\f[C]AZURE_TENANT_ID\f[R]: (optional) tenant to authenticate in. +Defaults to \[dq]organizations\[dq]. +.IP \[bu] 2 +\f[C]AZURE_CLIENT_ID\f[R]: client ID of the application the user will +authenticate to +.IP \[bu] 2 +\f[C]AZURE_USERNAME\f[R]: a username (usually an email address) +.IP \[bu] 2 +\f[C]AZURE_PASSWORD\f[R]: the user\[aq]s password +.RE +.SS Env Auth: 2. Managed Service Identity Credentials +.PP +When using Managed Service Identity if the VM(SS) on which this program +is running has a system-assigned identity, it will be used by default. +If the resource has no system-assigned but exactly one user-assigned +identity, the user-assigned identity will be used by default. +.PP +If the resource has multiple user-assigned identities you will need to +unset \f[C]env_auth\f[R] and set \f[C]use_msi\f[R] instead. +See the \f[C]use_msi\f[R] section. +.SS Env Auth: 3. Azure CLI credentials (as used by the az tool) +.PP +Credentials created with the \f[C]az\f[R] tool can be picked up using +\f[C]env_auth\f[R]. +.PP +For example if you were to login with a service principal like this: +.IP +.nf +\f[C] +az login --service-principal -u XXX -p XXX --tenant XXX +\f[R] +.fi +.PP +Then you could access rclone resources like this: +.IP +.nf +\f[C] +rclone lsf :azureblob,env_auth,account=ACCOUNT:CONTAINER +\f[R] +.fi +.PP +Or +.IP +.nf +\f[C] +rclone lsf --azureblob-env-auth --azureblob-acccount=ACCOUNT :azureblob:CONTAINER +\f[R] +.fi .PP -Rclone has 3 ways of authenticating with Azure Blob Storage: -.SS Account and Key +Which is analogous to using the \f[C]az\f[R] tool: +.IP +.nf +\f[C] +az storage blob list --container-name CONTAINER --account-name ACCOUNT --auth-mode login +\f[R] +.fi +.SS Account and Shared Key .PP This is the most straight forward and least flexible way. Just fill in the \f[C]account\f[R] and \f[C]key\f[R] lines and leave the @@ -45694,7 +46490,7 @@ rest blank. .PP This can be an account level SAS URL or container level SAS URL. .PP -To use it leave \f[C]account\f[R], \f[C]key\f[R] blank and fill in +To use it leave \f[C]account\f[R] and \f[C]key\f[R] blank and fill in \f[C]sas_url\f[R]. .PP An account level SAS URL or container level SAS URL can be obtained from @@ -45733,15 +46529,99 @@ rclone ls azureblob:othercontainer Container level SAS URLs are useful for temporarily allowing third parties access to a single container or putting credentials into an untrusted environment such as a CI build server. +.SS Service principal with client secret +.PP +If these variables are set, rclone will authenticate with a service +principal with a client secret. +.IP \[bu] 2 +\f[C]tenant\f[R]: ID of the service principal\[aq]s tenant. +Also called its \[dq]directory\[dq] ID. +.IP \[bu] 2 +\f[C]client_id\f[R]: the service principal\[aq]s client ID +.IP \[bu] 2 +\f[C]client_secret\f[R]: one of the service principal\[aq]s client +secrets +.PP +The credentials can also be placed in a file using the +\f[C]service_principal_file\f[R] configuration option. +.SS Service principal with certificate +.PP +If these variables are set, rclone will authenticate with a service +principal with certificate. +.IP \[bu] 2 +\f[C]tenant\f[R]: ID of the service principal\[aq]s tenant. +Also called its \[dq]directory\[dq] ID. +.IP \[bu] 2 +\f[C]client_id\f[R]: the service principal\[aq]s client ID +.IP \[bu] 2 +\f[C]client_certificate_path\f[R]: path to a PEM or PKCS12 certificate +file including the private key. +.IP \[bu] 2 +\f[C]client_certificate_password\f[R]: (optional) password for the +certificate file. +.IP \[bu] 2 +\f[C]client_send_certificate_chain\f[R]: (optional) Specifies whether an +authentication request will include an x5c header to support subject +name / issuer based authentication. +When set to \[dq]true\[dq] or \[dq]1\[dq], authentication requests +include the x5c header. +.PP +\f[B]NB\f[R] \f[C]client_certificate_password\f[R] must be obscured - +see rclone obscure (https://rclone.org/commands/rclone_obscure/). +.SS User with username and password +.PP +If these variables are set, rclone will authenticate with username and +password. +.IP \[bu] 2 +\f[C]tenant\f[R]: (optional) tenant to authenticate in. +Defaults to \[dq]organizations\[dq]. +.IP \[bu] 2 +\f[C]client_id\f[R]: client ID of the application the user will +authenticate to +.IP \[bu] 2 +\f[C]username\f[R]: a username (usually an email address) +.IP \[bu] 2 +\f[C]password\f[R]: the user\[aq]s password +.PP +Microsoft doesn\[aq]t recommend this kind of authentication, because +it\[aq]s less secure than other authentication flows. +This method is not interactive, so it isn\[aq]t compatible with any form +of multi-factor authentication, and the application must already have +user or admin consent. +This credential can only authenticate work and school accounts; it +can\[aq]t authenticate Microsoft accounts. +.PP +\f[B]NB\f[R] \f[C]password\f[R] must be obscured - see rclone +obscure (https://rclone.org/commands/rclone_obscure/). +.SS Managed Service Identity Credentials +.PP +If \f[C]use_msi\f[R] is set then managed service identity credentials +are used. +This authentication only works when running in an Azure service. +\f[C]env_auth\f[R] needs to be unset to use this. +.PP +However if you have multiple user identities to choose from these must +be explicitly specified using exactly one of the +\f[C]msi_object_id\f[R], \f[C]msi_client_id\f[R], or +\f[C]msi_mi_res_id\f[R] parameters. +.PP +If none of \f[C]msi_object_id\f[R], \f[C]msi_client_id\f[R], or +\f[C]msi_mi_res_id\f[R] is set, this is is equivalent to using +\f[C]env_auth\f[R]. .SS Standard options .PP Here are the Standard options specific to azureblob (Microsoft Azure Blob Storage). .SS --azureblob-account .PP -Storage Account Name. +Azure Storage Account Name. .PP -Leave blank to use SAS URL or Emulator. +Set this to the Azure Storage Account Name in use. +.PP +Leave blank to use SAS URL or Emulator, otherwise it needs to be set. +.PP +If this is blank and if env_auth is set it will be read from the +environment variable \f[C]AZURE_STORAGE_ACCOUNT_NAME\f[R] if possible. .PP Properties: .IP \[bu] 2 @@ -45752,41 +46632,24 @@ Env Var: RCLONE_AZUREBLOB_ACCOUNT Type: string .IP \[bu] 2 Required: false -.SS --azureblob-service-principal-file -.PP -Path to file containing credentials for use with a service principal. +.SS --azureblob-env-auth .PP -Leave blank normally. -Needed only if you want to use a service principal instead of -interactive login. -.IP -.nf -\f[C] -$ az ad sp create-for-rbac --name \[dq]\[dq] \[rs] - --role \[dq]Storage Blob Data Owner\[dq] \[rs] - --scopes \[dq]/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/\[dq] \[rs] - > azure-principal.json -\f[R] -.fi +Read credentials from runtime (environment variables, CLI or MSI). .PP -See \[dq]Create an Azure service -principal\[dq] (https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) -and \[dq]Assign an Azure role for access to blob -data\[dq] (https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) -pages for more details. +See the authentication docs for full info. .PP Properties: .IP \[bu] 2 -Config: service_principal_file +Config: env_auth .IP \[bu] 2 -Env Var: RCLONE_AZUREBLOB_SERVICE_PRINCIPAL_FILE +Env Var: RCLONE_AZUREBLOB_ENV_AUTH .IP \[bu] 2 -Type: string +Type: bool .IP \[bu] 2 -Required: false +Default: false .SS --azureblob-key .PP -Storage Account Key. +Storage Account Shared Key. .PP Leave blank to use SAS URL or Emulator. .PP @@ -45814,6 +46677,182 @@ Env Var: RCLONE_AZUREBLOB_SAS_URL Type: string .IP \[bu] 2 Required: false +.SS --azureblob-tenant +.PP +ID of the service principal\[aq]s tenant. +Also called its directory ID. +.PP +Set this if using - Service principal with client secret - Service +principal with certificate - User with username and password +.PP +Properties: +.IP \[bu] 2 +Config: tenant +.IP \[bu] 2 +Env Var: RCLONE_AZUREBLOB_TENANT +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --azureblob-client-id +.PP +The ID of the client in use. +.PP +Set this if using - Service principal with client secret - Service +principal with certificate - User with username and password +.PP +Properties: +.IP \[bu] 2 +Config: client_id +.IP \[bu] 2 +Env Var: RCLONE_AZUREBLOB_CLIENT_ID +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --azureblob-client-secret +.PP +One of the service principal\[aq]s client secrets +.PP +Set this if using - Service principal with client secret +.PP +Properties: +.IP \[bu] 2 +Config: client_secret +.IP \[bu] 2 +Env Var: RCLONE_AZUREBLOB_CLIENT_SECRET +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --azureblob-client-certificate-path +.PP +Path to a PEM or PKCS12 certificate file including the private key. +.PP +Set this if using - Service principal with certificate +.PP +Properties: +.IP \[bu] 2 +Config: client_certificate_path +.IP \[bu] 2 +Env Var: RCLONE_AZUREBLOB_CLIENT_CERTIFICATE_PATH +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --azureblob-client-certificate-password +.PP +Password for the certificate file (optional). +.PP +Optionally set this if using - Service principal with certificate +.PP +And the certificate has a password. +.PP +\f[B]NB\f[R] Input to this must be obscured - see rclone +obscure (https://rclone.org/commands/rclone_obscure/). +.PP +Properties: +.IP \[bu] 2 +Config: client_certificate_password +.IP \[bu] 2 +Env Var: RCLONE_AZUREBLOB_CLIENT_CERTIFICATE_PASSWORD +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS Advanced options +.PP +Here are the Advanced options specific to azureblob (Microsoft Azure +Blob Storage). +.SS --azureblob-client-send-certificate-chain +.PP +Send the certificate chain when using certificate auth. +.PP +Specifies whether an authentication request will include an x5c header +to support subject name / issuer based authentication. +When set to true, authentication requests include the x5c header. +.PP +Optionally set this if using - Service principal with certificate +.PP +Properties: +.IP \[bu] 2 +Config: client_send_certificate_chain +.IP \[bu] 2 +Env Var: RCLONE_AZUREBLOB_CLIENT_SEND_CERTIFICATE_CHAIN +.IP \[bu] 2 +Type: bool +.IP \[bu] 2 +Default: false +.SS --azureblob-username +.PP +User name (usually an email address) +.PP +Set this if using - User with username and password +.PP +Properties: +.IP \[bu] 2 +Config: username +.IP \[bu] 2 +Env Var: RCLONE_AZUREBLOB_USERNAME +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --azureblob-password +.PP +The user\[aq]s password +.PP +Set this if using - User with username and password +.PP +\f[B]NB\f[R] Input to this must be obscured - see rclone +obscure (https://rclone.org/commands/rclone_obscure/). +.PP +Properties: +.IP \[bu] 2 +Config: password +.IP \[bu] 2 +Env Var: RCLONE_AZUREBLOB_PASSWORD +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false +.SS --azureblob-service-principal-file +.PP +Path to file containing credentials for use with a service principal. +.PP +Leave blank normally. +Needed only if you want to use a service principal instead of +interactive login. +.IP +.nf +\f[C] +$ az ad sp create-for-rbac --name \[dq]\[dq] \[rs] + --role \[dq]Storage Blob Data Owner\[dq] \[rs] + --scopes \[dq]/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/\[dq] \[rs] + > azure-principal.json +\f[R] +.fi +.PP +See \[dq]Create an Azure service +principal\[dq] (https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli) +and \[dq]Assign an Azure role for access to blob +data\[dq] (https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) +pages for more details. +.PP +It may be more convenient to put the credentials directly into the +rclone config file under the \f[C]client_id\f[R], \f[C]tenant\f[R] and +\f[C]client_secret\f[R] keys instead of setting +\f[C]service_principal_file\f[R]. +.PP +Properties: +.IP \[bu] 2 +Config: service_principal_file +.IP \[bu] 2 +Env Var: RCLONE_AZUREBLOB_SERVICE_PRINCIPAL_FILE +.IP \[bu] 2 +Type: string +.IP \[bu] 2 +Required: false .SS --azureblob-use-msi .PP Use a managed service identity to authenticate (only works in Azure). @@ -45839,25 +46878,6 @@ Env Var: RCLONE_AZUREBLOB_USE_MSI Type: bool .IP \[bu] 2 Default: false -.SS --azureblob-use-emulator -.PP -Uses local storage emulator if provided as \[aq]true\[aq]. -.PP -Leave blank if using real azure storage endpoint. -.PP -Properties: -.IP \[bu] 2 -Config: use_emulator -.IP \[bu] 2 -Env Var: RCLONE_AZUREBLOB_USE_EMULATOR -.IP \[bu] 2 -Type: bool -.IP \[bu] 2 -Default: false -.SS Advanced options -.PP -Here are the Advanced options specific to azureblob (Microsoft Azure -Blob Storage). .SS --azureblob-msi-object-id .PP Object ID of the user-assigned MSI to use, if any. @@ -45903,6 +46923,21 @@ Env Var: RCLONE_AZUREBLOB_MSI_MI_RES_ID Type: string .IP \[bu] 2 Required: false +.SS --azureblob-use-emulator +.PP +Uses local storage emulator if provided as \[aq]true\[aq]. +.PP +Leave blank if using real azure storage endpoint. +.PP +Properties: +.IP \[bu] 2 +Config: use_emulator +.IP \[bu] 2 +Env Var: RCLONE_AZUREBLOB_USE_EMULATOR +.IP \[bu] 2 +Type: bool +.IP \[bu] 2 +Default: false .SS --azureblob-endpoint .PP Endpoint for the service. @@ -46162,6 +47197,22 @@ Blob data within this container can be read via anonymous request. Allow full public read access for container and blob data. .RE .RE +.SS --azureblob-no-check-container +.PP +If set, don\[aq]t attempt to check the container exists or create it. +.PP +This can be useful when trying to minimise the number of transactions +rclone does if you know the container exists already. +.PP +Properties: +.IP \[bu] 2 +Config: no_check_container +.IP \[bu] 2 +Env Var: RCLONE_AZUREBLOB_NO_CHECK_CONTAINER +.IP \[bu] 2 +Type: bool +.IP \[bu] 2 +Default: false .SS --azureblob-no-head-object .PP If set, do not do HEAD before GET when getting objects. @@ -46175,6 +47226,22 @@ Env Var: RCLONE_AZUREBLOB_NO_HEAD_OBJECT Type: bool .IP \[bu] 2 Default: false +.SS Custom upload headers +.PP +You can set custom upload headers with the \f[C]--header-upload\f[R] +flag. +.IP \[bu] 2 +Cache-Control +.IP \[bu] 2 +Content-Disposition +.IP \[bu] 2 +Content-Encoding +.IP \[bu] 2 +Content-Language +.IP \[bu] 2 +Content-Type +.PP +Eg \f[C]--header-upload \[dq]Content-Type: text/potato\[dq]\f[R] .SS Limitations .PP MD5 sums are only uploaded with chunked files if the source has an MD5 @@ -46192,16 +47259,21 @@ about (https://rclone.org/overview/#optional-features) and rclone about (https://rclone.org/commands/rclone_about/) .SS Azure Storage Emulator Support .PP -You can run rclone with storage emulator (usually \f[I]azurite\f[R]). +You can run rclone with the storage emulator (usually +\f[I]azurite\f[R]). .PP To do this, just set up a new remote with \f[C]rclone config\f[R] -following instructions described in introduction and set -\f[C]use_emulator\f[R] config as \f[C]true\f[R]. -You do not need to provide default account name neither an account key. +following the instructions in the introduction and set +\f[C]use_emulator\f[R] in the advanced settings as \f[C]true\f[R]. +You do not need to provide a default account name nor an account key. +But you can override them in the \f[C]account\f[R] and \f[C]key\f[R] +options. +(Prior to v1.61 they were hard coded to \f[I]azurite\f[R]\[aq]s +\f[C]devstoreaccount1\f[R].) .PP Also, if you want to access a storage emulator instance running on a -different machine, you can override \f[I]Endpoint\f[R] parameter in -advanced settings, setting it to +different machine, you can override the \f[C]endpoint\f[R] parameter in +the advanced settings, setting it to \f[C]http(s)://:/devstoreaccount1\f[R] (e.g. \f[C]http://10.254.2.5:10000/devstoreaccount1\f[R]). .SH Microsoft OneDrive @@ -46259,9 +47331,10 @@ y) Yes n) No y/n> n Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -47259,6 +48332,34 @@ links to be made for the organisation/sharepoint library. To fix the permissions as an admin, take a look at the docs: 1 (https://docs.microsoft.com/en-us/sharepoint/turn-external-sharing-on-or-off), 2 (https://support.microsoft.com/en-us/office/set-up-and-manage-access-requests-94b26e0b-2822-49d4-929a-8455698654b3). +.SS Can not access \f[C]Shared\f[R] with me files +.PP +Shared with me files is not supported by rclone +currently (https://github.com/rclone/rclone/issues/4062), but there is a +workaround: +.IP "1." 3 +Visit https://onedrive.live.com (https://onedrive.live.com/) +.IP "2." 3 +Right click a item in \f[C]Shared\f[R], then click +\f[C]Add shortcut to My files\f[R] in the context +.RS 4 +.PP +Screenshot (Shared with me) +.PP +[IMAGE: make_shortcut (https://user-images.githubusercontent.com/60313789/206118040-7e762b3b-aa61-41a1-8649-cc18889f3572.png)] +.RE +.IP "3." 3 +The shortcut will appear in \f[C]My files\f[R], you can access it with +rclone, it behaves like a normal folder/file. +.RS 4 +.PP +Screenshot (My Files) +.PP +[IMAGE: in_my_files (https://i.imgur.com/0S8H3li.png)] +.RE +.PP +Screenshot (rclone mount) +[IMAGE: rclone_mount (https://i.imgur.com/2Iq66sW.png)] .SH OpenDrive .PP Paths are specified as \f[C]remote:path\f[R] @@ -49714,9 +50815,10 @@ client_id> Pcloud App Client Secret - leave blank normally. client_secret> Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -50063,9 +51165,10 @@ Storage> premiumizeme ** See help for premiumizeme backend at: https://rclone.org/premiumizeme/ ** Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -50251,9 +51354,10 @@ Storage> putio ** See help for putio backend at: https://rclone.org/putio/ ** Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -50287,8 +51391,12 @@ e/n/d/r/c/s/q> q \f[R] .fi .PP +See the remote setup docs (https://rclone.org/remote_setup/) for how to +set it up on a machine with no Internet browser available. +.PP Note that rclone runs a webserver on your local machine to collect the -token as returned from Google if you use auto config mode. +token as returned from put.io if using web browser to automatically +authenticate. This only runs from the moment it opens your browser to the moment you get back the verification code. This is on \f[C]http://127.0.0.1:53682/\f[R] and this it may require you @@ -51533,6 +52641,9 @@ diffie-hellman-group-exchange-sha1 Those algorithms are insecure and may allow plaintext data to be recovered by an attacker. .PP +This must be false if you use either ciphers or key_exchange advanced +options. +.PP Properties: .IP \[bu] 2 Config: use_insecure_cipher @@ -51962,6 +53073,84 @@ Env Var: RCLONE_SFTP_SET_ENV Type: SpaceSepList .IP \[bu] 2 Default: +.SS --sftp-ciphers +.PP +Space separated list of ciphers to be used for session encryption, +ordered by preference. +.PP +At least one must match with server configuration. +This can be checked for example using ssh -Q cipher. +.PP +This must not be set if use_insecure_cipher is true. +.PP +Example: +.IP +.nf +\f[C] +aes128-ctr aes192-ctr aes256-ctr aes128-gcm\[at]openssh.com aes256-gcm\[at]openssh.com +\f[R] +.fi +.PP +Properties: +.IP \[bu] 2 +Config: ciphers +.IP \[bu] 2 +Env Var: RCLONE_SFTP_CIPHERS +.IP \[bu] 2 +Type: SpaceSepList +.IP \[bu] 2 +Default: +.SS --sftp-key-exchange +.PP +Space separated list of key exchange algorithms, ordered by preference. +.PP +At least one must match with server configuration. +This can be checked for example using ssh -Q kex. +.PP +This must not be set if use_insecure_cipher is true. +.PP +Example: +.IP +.nf +\f[C] +sntrup761x25519-sha512\[at]openssh.com curve25519-sha256 curve25519-sha256\[at]libssh.org ecdh-sha2-nistp256 +\f[R] +.fi +.PP +Properties: +.IP \[bu] 2 +Config: key_exchange +.IP \[bu] 2 +Env Var: RCLONE_SFTP_KEY_EXCHANGE +.IP \[bu] 2 +Type: SpaceSepList +.IP \[bu] 2 +Default: +.SS --sftp-macs +.PP +Space separated list of MACs (message authentication code) algorithms, +ordered by preference. +.PP +At least one must match with server configuration. +This can be checked for example using ssh -Q mac. +.PP +Example: +.IP +.nf +\f[C] +umac-64-etm\[at]openssh.com umac-128-etm\[at]openssh.com hmac-sha2-256-etm\[at]openssh.com +\f[R] +.fi +.PP +Properties: +.IP \[bu] 2 +Config: macs +.IP \[bu] 2 +Env Var: RCLONE_SFTP_MACS +.IP \[bu] 2 +Type: SpaceSepList +.IP \[bu] 2 +Default: .SS Limitations .PP On some SFTP servers (e.g. @@ -54379,9 +55568,10 @@ client_id> Yandex Client Secret - leave blank normally. client_secret> Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes n) No y/n> y @@ -54656,9 +55846,10 @@ y) Yes n) No (default) y/n> n Remote config -Use auto config? - * Say Y if not sure - * Say N if you are working on a remote or headless machine +Use web browser to automatically authenticate rclone with remote? + * Say Y if the machine running rclone has a web browser you can use + * Say N if running rclone on a (remote) machine without web browser access +If not sure try Y. If Y failed, try N. y) Yes (default) n) No y/n> @@ -56024,6 +57215,357 @@ Options: .IP \[bu] 2 \[dq]error\[dq]: return an error based on option value .SH Changelog +.SS v1.61.0 - 2022-12-20 +.PP +See commits (https://github.com/rclone/rclone/compare/v1.60.0...v1.61.0) +.IP \[bu] 2 +New backends +.RS 2 +.IP \[bu] 2 +New S3 providers +.RS 2 +.IP \[bu] 2 +Liara LOS (https://rclone.org/s3/#liara-cloud) (MohammadReza) +.RE +.RE +.IP \[bu] 2 +New Features +.RS 2 +.IP \[bu] 2 +build: Add vulnerability testing using govulncheck (albertony) +.IP \[bu] 2 +cmd: Enable \f[C]SIGINFO\f[R] (Ctrl-T) handler on FreeBSD, NetBSD, +OpenBSD and Dragonfly BSD (x3-apptech) +.IP \[bu] 2 +config: Add config/setpath (https://rclone.org/rc/#config-setpath) for +setting config path via rc/librclone (Nick Craig-Wood) +.IP \[bu] 2 +dedupe +.RS 2 +.IP \[bu] 2 +Count Checks in the stats while scanning for duplicates (Nick +Craig-Wood) +.IP \[bu] 2 +Make dedupe obey the filters (Nick Craig-Wood) +.RE +.IP \[bu] 2 +dlna: Properly attribute code used from https://github.com/anacrolix/dms +(Nick Craig-Wood) +.IP \[bu] 2 +docs +.RS 2 +.IP \[bu] 2 +Add minimum versions and status badges to backend and command docs (Nick +Craig-Wood, albertony) +.IP \[bu] 2 +Remote names may not start or end with space (albertony) +.RE +.IP \[bu] 2 +filter: Add metadata filters +--metadata-include/exclude/filter (https://rclone.org/filtering/#metadata) +and friends (Nick Craig-Wood) +.IP \[bu] 2 +fs +.RS 2 +.IP \[bu] 2 +Make all duration flags take \f[C]y\f[R], \f[C]M\f[R], \f[C]w\f[R], +\f[C]d\f[R] etc suffixes (Nick Craig-Wood) +.IP \[bu] 2 +Add global flag \f[C]--color\f[R] to control terminal colors (Kevin +Verstaen) +.RE +.IP \[bu] 2 +fspath: Allow unicode numbers and letters in remote names (albertony) +.IP \[bu] 2 +lib/file: Improve error message for creating dir on non-existent network +host on windows (albertony) +.IP \[bu] 2 +lib/http: Finish port of rclone servers to \f[C]lib/http\f[R] (Tom +Mombourquette, Nick Craig-Wood) +.IP \[bu] 2 +lib/oauthutil: Improved usability of config flows needing web browser +(Ole Frost) +.IP \[bu] 2 +ncdu +.RS 2 +.IP \[bu] 2 +Add support for modification time (albertony) +.IP \[bu] 2 +Fallback to sort by name also for sort by average size (albertony) +.IP \[bu] 2 +Rework to use tcell directly instead of the termbox wrapper (eNV25) +.RE +.IP \[bu] 2 +rc: Add commands to set GC +Percent (https://rclone.org/rc/#debug-set-gc-percent) & Memory Limit (go +1.19+) (Anagh Kumar Baranwal) +.IP \[bu] 2 +rcat: Preserve metadata when Copy falls back to Rcat (Nick Craig-Wood) +.IP \[bu] 2 +rcd: Refactor rclone rc server to use \f[C]lib/http\f[R] (Nick +Craig-Wood) +.IP \[bu] 2 +rcserver: Avoid generating default credentials with htpasswd (Kamui) +.IP \[bu] 2 +restic: Refactor to use \f[C]lib/http\f[R] (Nolan Woods) +.IP \[bu] 2 +serve http: Support unix sockets and multiple listeners (Tom +Mombourquette) +.IP \[bu] 2 +serve webdav: Refactor to use \f[C]lib/http\f[R] (Nick Craig-Wood) +.IP \[bu] 2 +test: Replace defer cleanup with \f[C]t.Cleanup\f[R] (Eng Zer Jun) +.IP \[bu] 2 +test memory: Read metadata if \f[C]-M\f[R] flag is specified (Nick +Craig-Wood) +.IP \[bu] 2 +wasm: Comply with \f[C]wasm_exec.js\f[R] licence terms (Matthew Vernon) +.RE +.IP \[bu] 2 +Bug Fixes +.RS 2 +.IP \[bu] 2 +build: Update \f[C]golang.org/x/net/http2\f[R] to fix GO-2022-1144 (Nick +Craig-Wood) +.IP \[bu] 2 +restic: Fix typo in docs \[aq]remove\[aq] should be \[aq]remote\[aq] +(asdffdsazqqq) +.IP \[bu] 2 +serve dlna: Fix panic: Logger uninitialized. +(Nick Craig-Wood) +.RE +.IP \[bu] 2 +Mount +.RS 2 +.IP \[bu] 2 +Update cgofuse for FUSE-T support for mounting volumes on Mac (Nick +Craig-Wood) +.RE +.IP \[bu] 2 +VFS +.RS 2 +.IP \[bu] 2 +Windows: fix slow opening of exe files by not truncating files when not +necessary (Nick Craig-Wood) +.IP \[bu] 2 +Fix IO Error opening a file with \f[C]O_CREATE|O_RDONLY\f[R] in +\f[C]--vfs-cache-mode\f[R] not full (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Crypt +.RS 2 +.IP \[bu] 2 +Fix compress wrapping crypt giving upload errors (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Azure Blob +.RS 2 +.IP \[bu] 2 +Port to new SDK (Nick Craig-Wood) +.RS 2 +.IP \[bu] 2 +Revamp authentication to include all methods and docs (Nick Craig-Wood) +.IP \[bu] 2 +Port old authentication methods to new SDK (Nick Craig-Wood, Brad +Ackerman) +.IP \[bu] 2 +Thanks to Stonebranch (https://www.stonebranch.com/) for sponsoring this +work. +.RE +.IP \[bu] 2 +Add \f[C]--azureblob-no-check-container\f[R] to assume container exists +(Nick Craig-Wood) +.IP \[bu] 2 +Add \f[C]--use-server-modtime\f[R] support (Abdullah Saglam) +.IP \[bu] 2 +Add support for custom upload headers (rkettelerij) +.IP \[bu] 2 +Allow emulator account/key override (Roel Arents) +.IP \[bu] 2 +Support simple \[dq]environment credentials\[dq] (Nathaniel Wesley +Filardo) +.IP \[bu] 2 +Ignore \f[C]AuthorizationFailure\f[R] when trying to create a create a +container (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Box +.RS 2 +.IP \[bu] 2 +Added note on Box API rate limits (Ole Frost) +.RE +.IP \[bu] 2 +Drive +.RS 2 +.IP \[bu] 2 +Handle shared drives with leading/trailing space in name (related to) +(albertony) +.RE +.IP \[bu] 2 +FTP +.RS 2 +.IP \[bu] 2 +Update help text of implicit/explicit TLS options to refer to FTPS +instead of FTP (ycdtosa) +.IP \[bu] 2 +Improve performance to speed up \f[C]--files-from\f[R] and +\f[C]NewObject\f[R] (Anthony Pessy) +.RE +.IP \[bu] 2 +HTTP +.RS 2 +.IP \[bu] 2 +Parse GET responses when \f[C]no_head\f[R] is set (Arnie97) +.IP \[bu] 2 +Do not update object size based on \f[C]Range\f[R] requests (Arnie97) +.IP \[bu] 2 +Support \f[C]Content-Range\f[R] response header (Arnie97) +.RE +.IP \[bu] 2 +Onedrive +.RS 2 +.IP \[bu] 2 +Document workaround for shared with me files (vanplus) +.RE +.IP \[bu] 2 +S3 +.RS 2 +.IP \[bu] 2 +Add Liara LOS to provider list (MohammadReza) +.IP \[bu] 2 +Add DigitalOcean Spaces regions \f[C]sfo3\f[R], \f[C]fra1\f[R], +\f[C]syd1\f[R] (Jack) +.IP \[bu] 2 +Avoid privileged \f[C]GetBucketLocation\f[R] to resolve s3 region +(Anthony Pessy) +.IP \[bu] 2 +Stop setting object and bucket ACL to \f[C]private\f[R] if it is an +empty string (Philip Harvey) +.IP \[bu] 2 +If bucket or object ACL is empty string then don\[aq]t add +\f[C]X-Amz-Acl:\f[R] header (Nick Craig-Wood) +.IP \[bu] 2 +Reduce memory consumption for s3 objects (Erik Agterdenbos) +.IP \[bu] 2 +Fix listing loop when using v2 listing on v1 server (Nick Craig-Wood) +.IP \[bu] 2 +Fix nil pointer exception when using Versions (Nick Craig-Wood) +.IP \[bu] 2 +Fix excess memory usage when using versions (Nick Craig-Wood) +.IP \[bu] 2 +Ignore versionIDs from uploads unless using \f[C]--s3-versions\f[R] or +\f[C]--s3-versions-at\f[R] (Nick Craig-Wood) +.RE +.IP \[bu] 2 +SFTP +.RS 2 +.IP \[bu] 2 +Add configuration options to set ssh Ciphers / MACs / KeyExchange +(dgouju) +.IP \[bu] 2 +Auto-detect shell type for fish (albertony) +.IP \[bu] 2 +Fix NewObject with leading / (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Smb +.RS 2 +.IP \[bu] 2 +Fix issue where spurious dot directory is created (albertony) +.RE +.IP \[bu] 2 +Storj +.RS 2 +.IP \[bu] 2 +Implement server side Copy (Kaloyan Raev) +.RE +.SS v1.60.1 - 2022-11-17 +.PP +See commits (https://github.com/rclone/rclone/compare/v1.60.0...v1.60.1) +.IP \[bu] 2 +Bug Fixes +.RS 2 +.IP \[bu] 2 +lib/cache: Fix alias backend shutting down too soon (Nick Craig-Wood) +.IP \[bu] 2 +wasm: Fix walltime link error by adding up-to-date wasm_exec.js +(Jo\[~a]o Henrique Franco) +.IP \[bu] 2 +docs +.RS 2 +.IP \[bu] 2 +Update faq.md with bisync (Samuel Johnson) +.IP \[bu] 2 +Corrected download links in windows install docs (coultonluke) +.IP \[bu] 2 +Add direct download link for windows arm64 (albertony) +.IP \[bu] 2 +Remove link to rclone slack as it is no longer supported (Nick +Craig-Wood) +.IP \[bu] 2 +Faq: how to use a proxy server that requires a username and password +(asdffdsazqqq) +.IP \[bu] 2 +Oracle-object-storage: doc fix (Manoj Ghosh) +.IP \[bu] 2 +Fix typo \f[C]remove\f[R] in rclone_serve_restic command (Joda +St\[:o]\[ss]er) +.IP \[bu] 2 +Fix character that was incorrectly interpreted as markdown (Cl\['e]ment +Notin) +.RE +.RE +.IP \[bu] 2 +VFS +.RS 2 +.IP \[bu] 2 +Fix deadlock caused by cache cleaner and upload finishing (Nick +Craig-Wood) +.RE +.IP \[bu] 2 +Local +.RS 2 +.IP \[bu] 2 +Clean absolute paths (albertony) +.IP \[bu] 2 +Fix -L/--copy-links with filters missing directories (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Mailru +.RS 2 +.IP \[bu] 2 +Note that an app password is now needed (Nick Craig-Wood) +.IP \[bu] 2 +Allow timestamps to be before the epoch 1970-01-01 (Nick Craig-Wood) +.RE +.IP \[bu] 2 +S3 +.RS 2 +.IP \[bu] 2 +Add provider quirk \f[C]--s3-might-gzip\f[R] to fix corrupted on +transfer: sizes differ (Nick Craig-Wood) +.IP \[bu] 2 +Allow Storj to server side copy since it seems to work now (Nick +Craig-Wood) +.IP \[bu] 2 +Fix for unchecked err value in s3 listv2 (Aaron Gokaslan) +.IP \[bu] 2 +Add additional Wasabi locations (techknowlogick) +.RE +.IP \[bu] 2 +Smb +.RS 2 +.IP \[bu] 2 +Fix \f[C]Failed to sync: context canceled\f[R] at the end of syncs (Nick +Craig-Wood) +.RE +.IP \[bu] 2 +WebDAV +.RS 2 +.IP \[bu] 2 +Fix Move/Copy/DirMove when using -server-side-across-configs (Nick +Craig-Wood) +.RE .SS v1.60.0 - 2022-10-21 .PP See commits (https://github.com/rclone/rclone/compare/v1.59.0...v1.60.0) @@ -67037,10 +68579,8 @@ significant amount of metadata, which breaks the desired 1:1 mapping of files to objects. .SS Can rclone do bi-directional sync? .PP -No, not at present. -rclone only does uni-directional sync from A -> B. -It may do in the future though since it has all the primitives - it just -requires writing the algorithm to do it. +Yes, since rclone v1.58.0, bidirectional cloud +sync (https://rclone.org/bisync/) is available. .SS Can I use rclone with an HTTP proxy? .PP Yes. @@ -67073,6 +68613,17 @@ export HTTPS_PROXY=$http_proxy \f[R] .fi .PP +Note: If the proxy server requires a username and password, then use +.IP +.nf +\f[C] +export http_proxy=http://username:password\[at]proxyserver:12345 +export https_proxy=$http_proxy +export HTTP_PROXY=$http_proxy +export HTTPS_PROXY=$http_proxy +\f[R] +.fi +.PP The \f[C]NO_PROXY\f[R] allows you to disable the proxy for specific hosts. Hosts must be comma separated, and can contain domains or parts. @@ -67691,6 +69242,8 @@ andrea rota .IP \[bu] 2 nicolov .IP \[bu] 2 +Matt Joiner +.IP \[bu] 2 Dario Guzik .IP \[bu] 2 qip @@ -68534,6 +70087,50 @@ Manoj Ghosh Tom Mombourquette .IP \[bu] 2 Robert Newson +.IP \[bu] 2 +Samuel Johnson +.IP \[bu] 2 +coultonluke +.IP \[bu] 2 +Anthony Pessy +.IP \[bu] 2 +Philip Harvey +.IP \[bu] 2 +dgouju +.IP \[bu] 2 +Cl\['e]ment Notin +.IP \[bu] 2 +x3-apptech <66947598+x3-apptech@users.noreply.github.com> +.IP \[bu] 2 +Arnie97 +.IP \[bu] 2 +Roel Arents <2691308+roelarents@users.noreply.github.com> +.IP \[bu] 2 +Aaron Gokaslan +.IP \[bu] 2 +techknowlogick +.IP \[bu] 2 +rkettelerij +.IP \[bu] 2 +Kamui +.IP \[bu] 2 +asdffdsazqqq <90116442+asdffdsazqqq@users.noreply.github.com> +.IP \[bu] 2 +Nathaniel Wesley Filardo +.IP \[bu] 2 +ycdtosa +.IP \[bu] 2 +Erik Agterdenbos +.IP \[bu] 2 +Kevin Verstaen <48050031+kverstae@users.noreply.github.com> +.IP \[bu] 2 +MohammadReza +.IP \[bu] 2 +vanplus <60313789+vanplus@users.noreply.github.com> +.IP \[bu] 2 +Jack <16779171+jkpe@users.noreply.github.com> +.IP \[bu] 2 +Abdullah Saglam .SH Contact the rclone project .SS Forum .PP From 496ae8adf62b0e3a4ea304ab4ff39358ebdfbec5 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 20 Dec 2022 18:33:59 +0000 Subject: [PATCH 480/560] Start v1.62.0-DEV development --- VERSION | 2 +- docs/layouts/partials/version.html | 2 +- fs/versiontag.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 137ade0bc6194..0f4b5a76da7d2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.61.0 +v1.62.0 diff --git a/docs/layouts/partials/version.html b/docs/layouts/partials/version.html index d9a35e5f7da0c..6d2547326f35e 100644 --- a/docs/layouts/partials/version.html +++ b/docs/layouts/partials/version.html @@ -1 +1 @@ -v1.61.0 \ No newline at end of file +v1.62.0 \ No newline at end of file diff --git a/fs/versiontag.go b/fs/versiontag.go index 15da27a842b47..e18cc3bbdca3f 100644 --- a/fs/versiontag.go +++ b/fs/versiontag.go @@ -1,4 +1,4 @@ package fs // VersionTag of rclone -var VersionTag = "v1.61.0" +var VersionTag = "v1.62.0" From 5ac8cfee56b58c242c60b8fd319b8a2dd4420c9b Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Tue, 20 Dec 2022 21:05:05 +0100 Subject: [PATCH 481/560] docs: show only significant parts of version number in version introduced label --- docs/content/bisync.md | 2 +- docs/content/chunker.md | 2 +- docs/content/combine.md | 2 +- docs/content/compress.md | 2 +- docs/content/docker.md | 2 +- docs/content/fichier.md | 2 +- docs/content/filefabric.md | 2 +- docs/content/googlephotos.md | 2 +- docs/content/gui.md | 2 +- docs/content/hasher.md | 2 +- docs/content/hdfs.md | 2 +- docs/content/hidrive.md | 2 +- docs/content/internetarchive.md | 2 +- docs/content/koofr.md | 2 +- docs/content/mailru.md | 2 +- docs/content/memory.md | 2 +- docs/content/netstorage.md | 2 +- docs/content/oracleobjectstorage.md | 2 +- docs/content/premiumizeme.md | 2 +- docs/content/putio.md | 2 +- docs/content/seafile.md | 2 +- docs/content/sharefile.md | 2 +- docs/content/sia.md | 2 +- docs/content/smb.md | 2 +- docs/content/storj.md | 2 +- docs/content/sugarsync.md | 2 +- docs/content/tardigrade.md | 2 +- docs/content/uptobox.md | 2 +- docs/content/zoho.md | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/content/bisync.md b/docs/content/bisync.md index 77b46bb82b9ce..ad8beec4ae13b 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -1,7 +1,7 @@ --- title: "Bisync" description: "Bidirectional cloud sync solution in rclone" -versionIntroduced: "v1.58.0" +versionIntroduced: "v1.58" --- ## Getting started {#getting-started} diff --git a/docs/content/chunker.md b/docs/content/chunker.md index 931e9a5d82bfd..ad4354b60f887 100644 --- a/docs/content/chunker.md +++ b/docs/content/chunker.md @@ -1,7 +1,7 @@ --- title: "Chunker" description: "Split-chunking overlay remote" -versionIntroduced: "v1.50.0" +versionIntroduced: "v1.50" status: Beta --- diff --git a/docs/content/combine.md b/docs/content/combine.md index 58137a20a7db1..5fa17eb44191c 100644 --- a/docs/content/combine.md +++ b/docs/content/combine.md @@ -1,7 +1,7 @@ --- title: "Combine" description: "Combine several remotes into one" -versionIntroduced: "v1.59.0" +versionIntroduced: "v1.59" --- # {{< icon "fa fa-folder-plus" >}} Combine diff --git a/docs/content/compress.md b/docs/content/compress.md index 27343236c355a..6717354602f34 100644 --- a/docs/content/compress.md +++ b/docs/content/compress.md @@ -1,7 +1,7 @@ --- title: "Compress" description: "Compression Remote" -versionIntroduced: "v1.54.0" +versionIntroduced: "v1.54" status: Experimental --- diff --git a/docs/content/docker.md b/docs/content/docker.md index c8a1746d772a5..4c502b9a01e12 100644 --- a/docs/content/docker.md +++ b/docs/content/docker.md @@ -1,7 +1,7 @@ --- title: "Docker Volume Plugin" description: "Docker Volume Plugin" -versionIntroduced: "v1.56.0" +versionIntroduced: "v1.56" --- # Docker Volume Plugin diff --git a/docs/content/fichier.md b/docs/content/fichier.md index a71926f7620b9..e4470faeab67a 100644 --- a/docs/content/fichier.md +++ b/docs/content/fichier.md @@ -1,7 +1,7 @@ --- title: "1Fichier" description: "Rclone docs for 1Fichier" -versionIntroduced: "v1.49.0" +versionIntroduced: "v1.49" --- # {{< icon "fa fa-archive" >}} 1Fichier diff --git a/docs/content/filefabric.md b/docs/content/filefabric.md index 31469925a431f..69ee67fcdcba8 100644 --- a/docs/content/filefabric.md +++ b/docs/content/filefabric.md @@ -1,7 +1,7 @@ --- title: "Enterprise File Fabric" description: "Rclone docs for the Enterprise File Fabric backend" -versionIntroduced: "v1.54.0" +versionIntroduced: "v1.54" --- # {{< icon "fa fa-cloud" >}} Enterprise File Fabric diff --git a/docs/content/googlephotos.md b/docs/content/googlephotos.md index 0076458ae8b16..c8e5db329c79a 100644 --- a/docs/content/googlephotos.md +++ b/docs/content/googlephotos.md @@ -1,7 +1,7 @@ --- title: "Google Photos" description: "Rclone docs for Google Photos" -versionIntroduced: "v1.49.0" +versionIntroduced: "v1.49" --- # {{< icon "fa fa-images" >}} Google Photos diff --git a/docs/content/gui.md b/docs/content/gui.md index 35f80a90ae388..11a54fa2572ee 100644 --- a/docs/content/gui.md +++ b/docs/content/gui.md @@ -1,7 +1,7 @@ --- title: "GUI" description: "Web based Graphical User Interface" -versionIntroduced: "v1.49.0" +versionIntroduced: "v1.49" --- # GUI (Experimental) diff --git a/docs/content/hasher.md b/docs/content/hasher.md index b52fb3bc9553d..324f2b85f694b 100644 --- a/docs/content/hasher.md +++ b/docs/content/hasher.md @@ -1,7 +1,7 @@ --- title: "Hasher" description: "Better checksums for other remotes" -versionIntroduced: "v1.57.0" +versionIntroduced: "v1.57" status: Experimental --- diff --git a/docs/content/hdfs.md b/docs/content/hdfs.md index feee7f561813f..867e7a0964b71 100644 --- a/docs/content/hdfs.md +++ b/docs/content/hdfs.md @@ -1,7 +1,7 @@ --- title: "HDFS Remote" description: "Remote for Hadoop Distributed Filesystem" -versionIntroduced: "v1.54.0" +versionIntroduced: "v1.54" --- # {{< icon "fa fa-globe" >}} HDFS diff --git a/docs/content/hidrive.md b/docs/content/hidrive.md index 750e0890aadd9..91e8ab29b6571 100644 --- a/docs/content/hidrive.md +++ b/docs/content/hidrive.md @@ -1,7 +1,7 @@ --- title: "HiDrive" description: "Rclone docs for HiDrive" -versionIntroduced: "v1.59.0" +versionIntroduced: "v1.59" --- # {{< icon "fa fa-cloud" >}} HiDrive diff --git a/docs/content/internetarchive.md b/docs/content/internetarchive.md index e02690849cda3..71f7db35d4029 100644 --- a/docs/content/internetarchive.md +++ b/docs/content/internetarchive.md @@ -1,7 +1,7 @@ --- title: "Internet Archive" description: "Rclone docs for Internet Archive" -versionIntroduced: "v1.59.0" +versionIntroduced: "v1.59" --- # {{< icon "fa fa-archive" >}} Internet Archive diff --git a/docs/content/koofr.md b/docs/content/koofr.md index 69295385bb6a8..6fbbbcabfaefd 100644 --- a/docs/content/koofr.md +++ b/docs/content/koofr.md @@ -1,7 +1,7 @@ --- title: "Koofr" description: "Rclone docs for Koofr" -versionIntroduced: "v1.47.0" +versionIntroduced: "v1.47" --- # {{< icon "fa fa-suitcase" >}} Koofr diff --git a/docs/content/mailru.md b/docs/content/mailru.md index 0f2f106c265df..8d5b70a488b7e 100644 --- a/docs/content/mailru.md +++ b/docs/content/mailru.md @@ -1,7 +1,7 @@ --- title: "Mailru" description: "Mail.ru Cloud" -versionIntroduced: "v1.50.0" +versionIntroduced: "v1.50" --- # {{< icon "fas fa-at" >}} Mail.ru Cloud diff --git a/docs/content/memory.md b/docs/content/memory.md index 760ed9c1c7344..843fc3cbd0270 100644 --- a/docs/content/memory.md +++ b/docs/content/memory.md @@ -1,7 +1,7 @@ --- title: "Memory" description: "Rclone docs for Memory backend" -versionIntroduced: "v1.51.0" +versionIntroduced: "v1.51" --- # {{< icon "fas fa-memory" >}} Memory diff --git a/docs/content/netstorage.md b/docs/content/netstorage.md index 5e16f58902b7f..ce474bed77f5d 100644 --- a/docs/content/netstorage.md +++ b/docs/content/netstorage.md @@ -1,7 +1,7 @@ --- title: "Akamai Netstorage" description: "Rclone docs for Akamai NetStorage" -versionIntroduced: "v1.58.0" +versionIntroduced: "v1.58" --- # {{< icon "fas fa-database" >}} Akamai NetStorage diff --git a/docs/content/oracleobjectstorage.md b/docs/content/oracleobjectstorage.md index f25dcca5db8f1..29a8d8e970306 100644 --- a/docs/content/oracleobjectstorage.md +++ b/docs/content/oracleobjectstorage.md @@ -1,7 +1,7 @@ --- title: "Oracle Object Storage" description: "Rclone docs for Oracle Object Storage" -versionIntroduced: "v1.60.0" +versionIntroduced: "v1.60" --- # {{< icon "fa fa-cloud" >}} Oracle Object Storage diff --git a/docs/content/premiumizeme.md b/docs/content/premiumizeme.md index ae7f44372ead1..39d40a5cd5a06 100644 --- a/docs/content/premiumizeme.md +++ b/docs/content/premiumizeme.md @@ -1,7 +1,7 @@ --- title: "premiumize.me" description: "Rclone docs for premiumize.me" -versionIntroduced: "v1.49.0" +versionIntroduced: "v1.49" --- # {{< icon "fa fa-user" >}} premiumize.me diff --git a/docs/content/putio.md b/docs/content/putio.md index ae67f1eb61278..1c45a8d73ec57 100644 --- a/docs/content/putio.md +++ b/docs/content/putio.md @@ -1,7 +1,7 @@ --- title: "put.io" description: "Rclone docs for put.io" -versionIntroduced: "v1.49.0" +versionIntroduced: "v1.49" --- # {{< icon "fas fa-parking" >}} put.io diff --git a/docs/content/seafile.md b/docs/content/seafile.md index 55d2a178a1ae2..bfa969a5082d6 100644 --- a/docs/content/seafile.md +++ b/docs/content/seafile.md @@ -1,7 +1,7 @@ --- title: "Seafile" description: "Seafile" -versionIntroduced: "v1.52.0" +versionIntroduced: "v1.52" --- # {{< icon "fa fa-server" >}} Seafile diff --git a/docs/content/sharefile.md b/docs/content/sharefile.md index c02315872f1b8..a86b36d799b2f 100644 --- a/docs/content/sharefile.md +++ b/docs/content/sharefile.md @@ -1,7 +1,7 @@ --- title: "Citrix ShareFile" description: "Rclone docs for Citrix ShareFile" -versionIntroduced: "v1.50.0" +versionIntroduced: "v1.50" --- # {{< icon "fas fa-share-square" >}} Citrix ShareFile diff --git a/docs/content/sia.md b/docs/content/sia.md index 9bc93c5c9f5be..ae9f74a2a0693 100644 --- a/docs/content/sia.md +++ b/docs/content/sia.md @@ -1,7 +1,7 @@ --- title: "Sia" description: "Remote for Sia Decentralized Cloud" -versionIntroduced: "v1.57.0" +versionIntroduced: "v1.57" date: "2019-10-02" --- diff --git a/docs/content/smb.md b/docs/content/smb.md index 8f9769411a9f3..2cf26803b57ee 100644 --- a/docs/content/smb.md +++ b/docs/content/smb.md @@ -1,7 +1,7 @@ --- title: "SMB / CIFS" description: "Rclone docs for SMB backend" -versionIntroduced: "v1.60.0" +versionIntroduced: "v1.60" --- # {{< icon "fa fa-server" >}} SMB diff --git a/docs/content/storj.md b/docs/content/storj.md index dfafc5ba3879a..95c3e4caac8dc 100644 --- a/docs/content/storj.md +++ b/docs/content/storj.md @@ -1,7 +1,7 @@ --- title: "Storj" description: "Rclone docs for Storj" -versionIntroduced: "v1.52.0" +versionIntroduced: "v1.52" --- # {{< icon "fas fa-dove" >}} Storj diff --git a/docs/content/sugarsync.md b/docs/content/sugarsync.md index 798e251dae458..0d9a29d7341c7 100644 --- a/docs/content/sugarsync.md +++ b/docs/content/sugarsync.md @@ -1,7 +1,7 @@ --- title: "SugarSync" description: "Rclone docs for SugarSync" -versionIntroduced: "v1.51.0" +versionIntroduced: "v1.51" --- # {{< icon "fas fa-dove" >}} SugarSync diff --git a/docs/content/tardigrade.md b/docs/content/tardigrade.md index b7f8819f8d6e8..bead13e370408 100644 --- a/docs/content/tardigrade.md +++ b/docs/content/tardigrade.md @@ -1,7 +1,7 @@ --- title: "Tardigrade" description: "Rclone docs for Tardigrade" -versionIntroduced: "v1.52.0" +versionIntroduced: "v1.52" --- # {{< icon "fas fa-dove" >}} Tardigrade diff --git a/docs/content/uptobox.md b/docs/content/uptobox.md index 3e2e12e1980fe..7324058fc6e41 100644 --- a/docs/content/uptobox.md +++ b/docs/content/uptobox.md @@ -1,7 +1,7 @@ --- title: "Uptobox" description: "Rclone docs for Uptobox" -versionIntroduced: "v1.56.0" +versionIntroduced: "v1.56" --- # {{< icon "fa fa-archive" >}} Uptobox diff --git a/docs/content/zoho.md b/docs/content/zoho.md index e24c5be173e5b..18306a63b0511 100644 --- a/docs/content/zoho.md +++ b/docs/content/zoho.md @@ -1,7 +1,7 @@ --- title: "Zoho" description: "Zoho WorkDrive" -versionIntroduced: "v1.54.0" +versionIntroduced: "v1.54" --- # {{< icon "fas fa-folder" >}} Zoho Workdrive From 00e853144e0effa68988392e55852a0a9d0c4cac Mon Sep 17 00:00:00 2001 From: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> Date: Wed, 21 Dec 2022 20:40:55 +0530 Subject: [PATCH 482/560] rc: set url to the first value of rc-addr since it has been converted to an array of strings now -- fixes #6641 Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> --- cmd/rc/rc.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/rc/rc.go b/cmd/rc/rc.go index b31e340eb0836..1276fc024b364 100644 --- a/cmd/rc/rc.go +++ b/cmd/rc/rc.go @@ -156,6 +156,15 @@ func ParseOptions(options []string) (opt map[string]string) { func setAlternateFlag(flagName string, output *string) { if rcFlag := pflag.Lookup(flagName); rcFlag != nil && rcFlag.Changed { *output = rcFlag.Value.String() + if sliceValue, ok := rcFlag.Value.(pflag.SliceValue); ok { + stringSlice := sliceValue.GetSlice() + for _, value := range stringSlice { + if value != "" { + *output = value + break + } + } + } } } From d049cbb59e1bc6959ec0dfe9c869888f183a4bf4 Mon Sep 17 00:00:00 2001 From: Kaloyan Raev Date: Thu, 22 Dec 2022 15:44:10 +0200 Subject: [PATCH 483/560] s3/storj: update endpoints Storj switched to a single global s3 endpoint backed by a BGP routing. We want to stop advertizing the former regional endpoints and have the global one as the only option. --- backend/s3/s3.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index c6977b8c90e57..872e8749898d7 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -936,17 +936,11 @@ func init() { }}, }, { Name: "endpoint", - Help: "Endpoint of the Shared Gateway.", + Help: "Endpoint for Storj Gateway.", Provider: "Storj", Examples: []fs.OptionExample{{ - Value: "gateway.eu1.storjshare.io", - Help: "EU1 Shared Gateway", - }, { - Value: "gateway.us1.storjshare.io", - Help: "US1 Shared Gateway", - }, { - Value: "gateway.ap1.storjshare.io", - Help: "Asia-Pacific Shared Gateway", + Value: "gateway.storjshare.io", + Help: "Global Hosted Gateway", }}, }, { // cos endpoints: https://intl.cloud.tencent.com/document/product/436/6224 From 54c0f17f2ab40808d5ca0bbe20b8608eb37d0174 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 22 Dec 2022 14:19:38 +0000 Subject: [PATCH 484/560] azureblob: fix "409 Public access is not permitted on this storage account" This error was caused by rclone supplying an empty `x-ms-blob-public-access:` header when creating a container for private access, rather than omitting it completely. This is a valid way of specifying containers should be private, but if the storage account has the flag "Blob public access" unset then it gives "409 Public access is not permitted on this storage account". This patch fixes the problem by only supplying the header if the access is set. Fixes #6645 --- backend/azureblob/azureblob.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index af6ff9dddba83..2376f44fecfbc 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -1363,15 +1363,16 @@ func (f *Fs) makeContainer(ctx context.Context, container string) error { return nil } opt := service.CreateContainerOptions{ - // Specifies whether data in the container may be accessed publicly and the level of access - Access: &f.publicAccess, - // Optional. Specifies a user-defined name-value pair associated with the blob. //Metadata map[string]string // Optional. Specifies the encryption scope settings to set on the container. //CpkScopeInfo *CpkScopeInfo } + if f.publicAccess != "" { + // Specifies whether data in the container may be accessed publicly and the level of access + opt.Access = &f.publicAccess + } // now try to create the container return f.pacer.Call(func() (bool, error) { _, err := f.svc.CreateContainer(ctx, container, &opt) From 6257a6035cabc103abd0349f35aba4760c3f827a Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 22 Dec 2022 15:42:47 +0000 Subject: [PATCH 485/560] serve webdav: fix --baseurl handling after lib/http refactor The webdav library was confused by the Path manipulation done by lib/http when stripping the prefix. This patch adds the prefix back before calling it. Fixes #6650 --- cmd/serve/webdav/webdav.go | 3 +++ cmd/serve/webdav/webdav_test.go | 1 + 2 files changed, 4 insertions(+) diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go index 00af1f033c1cc..cdec88b4b78b3 100644 --- a/cmd/serve/webdav/webdav.go +++ b/cmd/serve/webdav/webdav.go @@ -237,6 +237,9 @@ func (w *WebDAV) ServeHTTP(rw http.ResponseWriter, r *http.Request) { w.serveDir(rw, r, remote) return } + // Add URL Prefix back to path since webdavhandler needs to + // return absolute references. + r.URL.Path = w.opt.HTTP.BaseURL + r.URL.Path w.webdavhandler.ServeHTTP(rw, r) } diff --git a/cmd/serve/webdav/webdav_test.go b/cmd/serve/webdav/webdav_test.go index fa06260757c78..2bc61a58af6bf 100644 --- a/cmd/serve/webdav/webdav_test.go +++ b/cmd/serve/webdav/webdav_test.go @@ -51,6 +51,7 @@ func TestWebDav(t *testing.T) { start := func(f fs.Fs) (configmap.Simple, func()) { opt := DefaultOpt opt.HTTP.ListenAddr = []string{testBindAddress} + opt.HTTP.BaseURL = "/prefix" opt.Auth.BasicUser = testUser opt.Auth.BasicPass = testPass opt.Template.Path = testTemplate From 448fff9a0488d1985f4a4d9f05326ff65c539f18 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 21 Dec 2022 12:32:21 +0000 Subject: [PATCH 486/560] serve restic: fix immediate exit when not using stdio In the lib/http refactor 52443c2444e3d3c4 restic: refactor to use lib/http We forgot to serve the data and wait for the server to finish. This is not tested in the unit tests as it is part of the command line handler. Fixes #6644 Fixes #6647 --- cmd/serve/restic/restic.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go index 0d4db85bd3855..0752d1ca5b5e0 100644 --- a/cmd/serve/restic/restic.go +++ b/cmd/serve/restic/restic.go @@ -173,6 +173,7 @@ with a path of ` + "`//`" + `. httpSrv.ServeConn(conn, opts) return nil } + s.Wait() return nil }) }, From 823d89af9a5655ded1aa978b5ff5c202bdae9126 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 22 Dec 2022 16:02:17 +0000 Subject: [PATCH 487/560] serve restic: don't serve via http if serving via --stdio Before this change, we started the http listener even if --stdio was supplied. This also moves the log message so the user won't see the serving via HTTP message unless they are really using that. Fixes #6646 --- cmd/serve/restic/restic.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/serve/restic/restic.go b/cmd/serve/restic/restic.go index 0752d1ca5b5e0..fbdc947e13d8d 100644 --- a/cmd/serve/restic/restic.go +++ b/cmd/serve/restic/restic.go @@ -155,7 +155,6 @@ with a path of ` + "`//`" + `. if err != nil { return err } - fs.Logf(s.f, "Serving restic REST API on %s", s.URLs()) if s.opt.Stdio { if terminal.IsTerminal(int(os.Stdout.Fd())) { return errors.New("refusing to run HTTP2 server directly on a terminal, please let restic start rclone") @@ -173,6 +172,7 @@ with a path of ` + "`//`" + `. httpSrv.ServeConn(conn, opts) return nil } + fs.Logf(s.f, "Serving restic REST API on %s", s.URLs()) s.Wait() return nil }) @@ -242,6 +242,10 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *server, err error cache: newCache(opt.CacheObjects), opt: *opt, } + // Don't bind any HTTP listeners if running with --stdio + if opt.Stdio { + opt.HTTP.ListenAddr = nil + } s.Server, err = libhttp.NewServer(ctx, libhttp.WithConfig(opt.HTTP), libhttp.WithAuth(opt.Auth), From b1b62f70d390b6bc676bdabebd299d07bab3fdfd Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 22 Dec 2022 16:55:17 +0000 Subject: [PATCH 488/560] serve webdav: fix running duplicate Serve call Before this change we were starting the server twice for webdav which is inefficient and causes problems at exit. --- cmd/serve/webdav/webdav.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go index cdec88b4b78b3..e608ae6da5631 100644 --- a/cmd/serve/webdav/webdav.go +++ b/cmd/serve/webdav/webdav.go @@ -199,8 +199,6 @@ func newWebDAV(ctx context.Context, f fs.Fs, opt *Options) (w *WebDAV, err error router.Method(method, "/*", w) } - w.Server.Serve() - return w, nil } From 638058ef91fac50248d4464179dbcf869da379f0 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 22 Dec 2022 16:52:43 +0000 Subject: [PATCH 489/560] lib/http: shutdown all servers on exit to remove unix socket Before this change only serve http was Shutting down its server which was causing other servers such as serve restic to leave behind their unix sockets. This change moves the finalisation to lib/http so all servers have it and removes it from serve http. Fixes #6648 --- cmd/serve/http/http.go | 13 ------------- lib/http/server.go | 17 +++++++++++++++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go index 78f6870f0ee4d..2ad1bff7daff1 100644 --- a/cmd/serve/http/http.go +++ b/cmd/serve/http/http.go @@ -11,14 +11,12 @@ import ( "path" "strconv" "strings" - "sync" "time" "github.com/go-chi/chi/v5/middleware" "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" - "github.com/rclone/rclone/lib/atexit" libhttp "github.com/rclone/rclone/lib/http" "github.com/rclone/rclone/lib/http/serve" "github.com/rclone/rclone/vfs" @@ -82,17 +80,6 @@ control the stats printing. log.Fatal(err) } - var finaliseOnce sync.Once - finalise := func() { - finaliseOnce.Do(func() { - if err := s.server.Shutdown(); err != nil { - log.Printf("error shutting down server: %v", err) - } - }) - } - fnHandle := atexit.Register(finalise) - defer atexit.Unregister(fnHandle) - s.server.Wait() return nil }) diff --git a/lib/http/server.go b/lib/http/server.go index 5e76e9615492c..a7c1f64b240c5 100644 --- a/lib/http/server.go +++ b/lib/http/server.go @@ -19,6 +19,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/lib/atexit" "github.com/spf13/pflag" ) @@ -145,6 +146,7 @@ type Server struct { template *TemplateConfig htmlTemplate *template.Template usingAuth bool // set if we are using auth middleware + atexitHandle atexit.FnHandle } // Option allows customizing the server @@ -390,6 +392,8 @@ func (s *Server) Serve() { // log.Printf("listening on %s", ii.url) go ii.serve(&s.wg) } + // Install an atexit handler to shutdown gracefully + s.atexitHandle = atexit.Register(func() { _ = s.Shutdown() }) } // Wait blocks while the server is serving requests @@ -402,14 +406,23 @@ func (s *Server) Router() chi.Router { return s.mux } +// Time to wait to Shutdown an HTTP server +const gracefulShutdownTime = 10 * time.Second + // Shutdown gracefully shuts down the server func (s *Server) Shutdown() error { - ctx := context.Background() + // Stop the atexit handler + if s.atexitHandle != nil { + atexit.Unregister(s.atexitHandle) + s.atexitHandle = nil + } for _, ii := range s.instances { + expiry := time.Now().Add(gracefulShutdownTime) + ctx, cancel := context.WithDeadline(context.Background(), expiry) if err := ii.httpServer.Shutdown(ctx); err != nil { log.Printf("error shutting down server: %s", err) - continue } + cancel() } s.wg.Wait() return nil From 81ce5e4961147d3342d6203fcd0f1bd7779546c8 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 23 Dec 2022 12:34:04 +0000 Subject: [PATCH 490/560] docs: correct RELEASE procedure for stable branch --- RELEASE.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 32f5d664bce0c..c2a23c277c0bd 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -74,8 +74,7 @@ Set vars First make the release branch. If this is a second point release then this will be done already. - * git branch ${BASE_TAG} ${BASE_TAG}-stable - * git co ${BASE_TAG}-stable + * git co -b ${BASE_TAG}-stable ${BASE_TAG}.0 * make startstable Now From ffeefe8a56ca09ba805816e3a036775c11eb30aa Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 23 Dec 2022 16:52:19 +0000 Subject: [PATCH 491/560] crypt: obey --ignore-checksum Before this change the crypt backend would calculate and check upload checksums regardless of the setting of --ignore-checksum. --- backend/crypt/crypt.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go index 1aece2753262d..b1c8111d0cc5f 100644 --- a/backend/crypt/crypt.go +++ b/backend/crypt/crypt.go @@ -396,6 +396,8 @@ type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .. // put implements Put or PutStream func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) { + ci := fs.GetConfig(ctx) + if f.opt.NoDataEncryption { o, err := put(ctx, in, f.newObjectInfo(src, nonce{}), options...) if err == nil && o != nil { @@ -413,6 +415,9 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [ // Find a hash the destination supports to compute a hash of // the encrypted data ht := f.Fs.Hashes().GetOne() + if ci.IgnoreChecksum { + ht = hash.None + } var hasher *hash.MultiHasher if ht != hash.None { hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht)) From 8aebf127975e5382a35e0fcee2a8401f44430cfc Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 23 Dec 2022 16:53:43 +0000 Subject: [PATCH 492/560] docs: fix unescaped HTML --- docs/content/onedrive.md | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index 236f77d7d43fe..2377b04e11455 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -721,21 +721,7 @@ Shared with me files is not supported by rclone [currently](https://github.com/r 1. Visit [https://onedrive.live.com](https://onedrive.live.com/) 2. Right click a item in `Shared`, then click `Add shortcut to My files` in the context -
      - Screenshot (Shared with me) - - ![make_shortcut](https://user-images.githubusercontent.com/60313789/206118040-7e762b3b-aa61-41a1-8649-cc18889f3572.png) -
      - + ![make_shortcut](https://user-images.githubusercontent.com/60313789/206118040-7e762b3b-aa61-41a1-8649-cc18889f3572.png "Screenshot (Shared with me)") 3. The shortcut will appear in `My files`, you can access it with rclone, it behaves like a normal folder/file. -
      - Screenshot (My Files) - - ![in_my_files](https://i.imgur.com/0S8H3li.png) -
      - -
      - Screenshot (rclone mount) - - ![rclone_mount](https://i.imgur.com/2Iq66sW.png) -
      + ![in_my_files](https://i.imgur.com/0S8H3li.png "Screenshot (My Files)") + ![rclone_mount](https://i.imgur.com/2Iq66sW.png "Screenshot (rclone mount)") From 1925ceaadebd8ab0e678a5e8782a1f45d5783be6 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 23 Dec 2022 18:26:56 +0000 Subject: [PATCH 493/560] Changelog updates from Version v1.61.1 --- docs/content/changelog.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/content/changelog.md b/docs/content/changelog.md index 2ba5494eac099..264111ef190a0 100644 --- a/docs/content/changelog.md +++ b/docs/content/changelog.md @@ -5,6 +5,27 @@ description: "Rclone Changelog" # Changelog +## v1.61.1 - 2022-12-23 + +[See commits](https://github.com/rclone/rclone/compare/v1.61.0...v1.61.1) + +* Bug Fixes + * docs: + * Show only significant parts of version number in version introduced label (albertony) + * Fix unescaped HTML (Nick Craig-Wood) + * lib/http: Shutdown all servers on exit to remove unix socket (Nick Craig-Wood) + * rc: Fix `--rc-addr` flag (which is an alternate for `--url`) (Anagh Kumar Baranwal) + * serve restic + * Don't serve via http if serving via `--stdio` (Nick Craig-Wood) + * Fix immediate exit when not using stdio (Nick Craig-Wood) + * serve webdav + * Fix `--baseurl` handling after `lib/http` refactor (Nick Craig-Wood) + * Fix running duplicate Serve call (Nick Craig-Wood) +* Azure Blob + * Fix "409 Public access is not permitted on this storage account" (Nick Craig-Wood) +* S3 + * storj: Update endpoints (Kaloyan Raev) + ## v1.61.0 - 2022-12-20 [See commits](https://github.com/rclone/rclone/compare/v1.60.0...v1.61.0) From ad5395e95352e7e1d257353cca321082b52cb51d Mon Sep 17 00:00:00 2001 From: Marks Polakovs Date: Sun, 25 Dec 2022 14:30:37 +0200 Subject: [PATCH 494/560] backend/local: fix %!w() in "failed to read directory" error --- backend/local/local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/local/local.go b/backend/local/local.go index 67784f8e6d798..25ff7fce5f08b 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -503,7 +503,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e continue } } - err = fmt.Errorf("failed to read directory %q: %w", namepath, err) + err = fmt.Errorf("failed to read directory %q: %w", namepath, fierr) fs.Errorf(dir, "%v", fierr) _ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync continue From c6c67a29ebbb3db331a096f1c4b4c5adfeca2d8c Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Mon, 26 Dec 2022 18:39:49 +0100 Subject: [PATCH 495/560] Add Marks Polakovs to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index b96d53e0a1cdd..dfcbd2ceab8f4 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -673,3 +673,4 @@ put them back in again.` >}} * vanplus <60313789+vanplus@users.noreply.github.com> * Jack <16779171+jkpe@users.noreply.github.com> * Abdullah Saglam + * Marks Polakovs From 98fa93f6d17909e9c6b9778a4dc9e67c58151f6a Mon Sep 17 00:00:00 2001 From: piyushgarg Date: Fri, 30 Dec 2022 16:52:46 +0530 Subject: [PATCH 496/560] webdav: Document Mapping/Accessing WebDAV shares on windows. Fixes #6596 Co-authored-by: Piyush --- cmd/serve/webdav/webdav.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go index e608ae6da5631..daaa537f981f8 100644 --- a/cmd/serve/webdav/webdav.go +++ b/cmd/serve/webdav/webdav.go @@ -79,6 +79,30 @@ supported hash on the backend or you can use a named hash such as "MD5" or "SHA-1". Use the [hashsum](/commands/rclone_hashsum/) command to see the full list. +### Access WebDAV on Windows +WebDAV shared folder can be mapped as a drive on Windows, however the default settings prevent it. +Windows will fail to connect to the server using insecure Basic authentication. +It will not even display any login dialog. Windows requires SSL / HTTPS connection to be used with Basic. +If you try to connect via Add Network Location Wizard you will get the following error: +"The folder you entered does not appear to be valid. Please choose another". +However, you still can connect if you set the following registry key on a client machine: +HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\WebClient\Parameters\BasicAuthLevel to 2. +The BasicAuthLevel can be set to the following values: + 0 - Basic authentication disabled + 1 - Basic authentication enabled for SSL connections only + 2 - Basic authentication enabled for SSL connections and for non-SSL connections +If required, increase the FileSizeLimitInBytes to a higher value. +Navigate to the Services interface, then restart the WebClient service. + +### Access Office applications on WebDAV +Navigate to following registry HKEY_CURRENT_USER\Software\Microsoft\Office\[14.0/15.0/16.0]\Common\Internet +Create a new DWORD BasicAuthLevel with value 2. + 0 - Basic authentication disabled + 1 - Basic authentication enabled for SSL connections only + 2 - Basic authentication enabled for SSL and for non-SSL connections + +https://learn.microsoft.com/en-us/office/troubleshoot/powerpoint/office-opens-blank-from-sharepoint + ` + libhttp.Help + libhttp.TemplateHelp + libhttp.AuthHelp + vfs.Help + proxy.Help, Annotations: map[string]string{ "versionIntroduced": "v1.39", From 1cafc12e8c2d5a5967f0eb5450db4ae2ddb8f85e Mon Sep 17 00:00:00 2001 From: Kaloyan Raev Date: Tue, 10 Jan 2023 19:40:04 +0200 Subject: [PATCH 497/560] storj: implement public link --- backend/storj/fs.go | 76 +++++++++++++++++++++++++++++++++++---- docs/content/overview.md | 2 +- fstest/fstests/fstests.go | 6 ++-- 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/backend/storj/fs.go b/backend/storj/fs.go index 9a0a5ebe27356..ed3004099ce61 100644 --- a/backend/storj/fs.go +++ b/backend/storj/fs.go @@ -23,6 +23,7 @@ import ( "golang.org/x/text/unicode/norm" "storj.io/uplink" + "storj.io/uplink/edge" ) const ( @@ -156,11 +157,12 @@ type Fs struct { // Check the interfaces are satisfied. var ( - _ fs.Fs = &Fs{} - _ fs.ListRer = &Fs{} - _ fs.PutStreamer = &Fs{} - _ fs.Mover = &Fs{} - _ fs.Copier = &Fs{} + _ fs.Fs = &Fs{} + _ fs.ListRer = &Fs{} + _ fs.PutStreamer = &Fs{} + _ fs.Mover = &Fs{} + _ fs.Copier = &Fs{} + _ fs.PublicLinker = &Fs{} ) // NewFs creates a filesystem backed by Storj. @@ -545,7 +547,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . defer func() { if err != nil { aerr := upload.Abort() - if aerr != nil { + if aerr != nil && !errors.Is(aerr, uplink.ErrUploadDone) { fs.Errorf(f, "cp input ./%s %+v: %+v", src.Remote(), options, aerr) } } @@ -560,6 +562,16 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options . _, err = io.Copy(upload, in) if err != nil { + if errors.Is(err, uplink.ErrBucketNotFound) { + // Rclone assumes the backend will create the bucket if not existing yet. + // Here we create the bucket and return a retry error for rclone to retry the upload. + _, err = f.project.EnsureBucket(ctx, bucketName) + if err != nil { + return nil, err + } + return nil, fserrors.RetryError(errors.New("bucket was not available, now created, the upload must be retried")) + } + err = fserrors.RetryError(err) fs.Errorf(f, "cp input ./%s %+v: %+v\n", src.Remote(), options, err) @@ -761,3 +773,55 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, // Return the new object return newObjectFromUplink(f, remote, newObject), nil } + +// PublicLink generates a public link to the remote path (usually readable by anyone) +func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) { + bucket, key := f.absolute(remote) + if len(bucket) == 0 { + return "", errors.New("path must be specified") + } + + // Rclone requires that a link is only generated if the remote path exists + if len(key) == 0 { + _, err := f.project.StatBucket(ctx, bucket) + if err != nil { + return "", err + } + } else { + _, err := f.project.StatObject(ctx, bucket, key) + if err != nil { + if !errors.Is(err, uplink.ErrObjectNotFound) { + return "", err + } + // No object found, check if there is such a prefix + iter := f.project.ListObjects(ctx, bucket, &uplink.ListObjectsOptions{Prefix: key + "/"}) + if iter.Err() != nil { + return "", iter.Err() + } + if !iter.Next() { + return "", err + } + } + } + + sharedPrefix := uplink.SharePrefix{Bucket: bucket, Prefix: key} + + permission := uplink.ReadOnlyPermission() + if expire.IsSet() { + permission.NotAfter = time.Now().Add(time.Duration(expire)) + } + + sharedAccess, err := f.access.Share(permission, sharedPrefix) + if err != nil { + return "", fmt.Errorf("sharing access to object failed: %w", err) + } + + creds, err := (&edge.Config{ + AuthServiceAddress: "auth.storjshare.io:7777", + }).RegisterAccess(ctx, sharedAccess, &edge.RegisterAccessOptions{Public: true}) + if err != nil { + return "", fmt.Errorf("creating public link failed: %w", err) + } + + return edge.JoinShareURL("https://link.storjshare.io", creds.AccessKeyID, bucket, key, nil) +} diff --git a/docs/content/overview.md b/docs/content/overview.md index eca35d8d49802..df19d05a973fb 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -504,7 +504,7 @@ upon backend-specific capabilities. | Sia | No | No | No | No | No | No | Yes | No | No | Yes | | SMB | No | No | Yes | Yes | No | No | Yes | No | No | Yes | | SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes | -| Storj | Yes † | Yes | Yes | No | No | Yes | Yes | No | No | No | +| Storj | Yes † | Yes | Yes | No | No | Yes | Yes | Yes | No | No | | Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | | WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No | Yes | Yes | | Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | Yes | diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 40fb398513121..9c3ea61ff5134 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -1734,8 +1734,10 @@ func Run(t *testing.T, opt *Opt) { // ensure sub remote isn't empty buf := bytes.NewBufferString("somecontent") obji := object.NewStaticObjectInfo("somefile", time.Now(), int64(buf.Len()), true, nil, nil) - _, err = subRemote.Put(ctx, buf, obji) - require.NoError(t, err) + retry(t, "Put", func() error { + _, err := subRemote.Put(ctx, buf, obji) + return err + }) link4, err := wrapPublicLinkFunc(subRemote.Features().PublicLink)(ctx, "", expiry, false) require.NoError(t, err, "Sharing root in a sub-remote should work") From 02d6d28ec4844f9b42b336173a235f1e88c3716f Mon Sep 17 00:00:00 2001 From: Ole Frost <82263101+olefrost@users.noreply.github.com> Date: Wed, 11 Jan 2023 13:42:43 +0100 Subject: [PATCH 498/560] crypt: fix for unencrypted directory names on case insensitive remotes rclone sync erroneously deleted folders renamed to a different case on crypts where directory name encryption was disabled and the underlying remote was case insensitive. Example: Renaming the folder Test to tEST before a sync to a crypt having remote=OneDrive:crypt and directory_name_encryption=false could result in the folder and all its content being deleted. The following sync would correctly create the tEST folder and upload all of the content. Additional tests have revealed other potential issues when using filename_encryption=off or directory_name_encryption=false on case insensitive remotes. The documentation has been updated to warn about potential problems when using these combinations. --- backend/crypt/crypt.go | 2 +- docs/content/crypt.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/crypt/crypt.go b/backend/crypt/crypt.go index b1c8111d0cc5f..f3eb8317da9d4 100644 --- a/backend/crypt/crypt.go +++ b/backend/crypt/crypt.go @@ -235,7 +235,7 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, // the features here are ones we could support, and they are // ANDed with the ones from wrappedFs f.features = (&fs.Features{ - CaseInsensitive: cipher.NameEncryptionMode() == NameEncryptionOff, + CaseInsensitive: !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff, DuplicateFiles: true, ReadMimeType: false, // MimeTypes not supported with crypt WriteMimeType: false, diff --git a/docs/content/crypt.md b/docs/content/crypt.md index 813aa08307e14..2b05502466b96 100644 --- a/docs/content/crypt.md +++ b/docs/content/crypt.md @@ -455,6 +455,7 @@ Properties: - "off" - Don't encrypt the file names. - Adds a ".bin" extension only. + - May cause problems on [case insensitive](/overview/#case-insensitive) [storage systems](/overview/#features) like OneDrive, Dropbox, Windows, OSX and SMB. #### --crypt-directory-name-encryption @@ -473,6 +474,7 @@ Properties: - Encrypt directory names. - "false" - Don't encrypt directory names, leave them intact. + - May cause problems on [case insensitive](/overview/#case-insensitive) [storage systems](/overview/#features) like OneDrive, Dropbox, Windows, OSX and SMB. #### --crypt-password From d66f5e8db07f4b75479c68aeb633bce3959ad413 Mon Sep 17 00:00:00 2001 From: Alex Chen Date: Thu, 12 Jan 2023 00:50:14 +0800 Subject: [PATCH 499/560] lib/oauthutil: handle fatal errors better PR #6678 --- lib/oauthutil/oauthutil.go | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go index cd9cb74d3c28d..5f3e62f231404 100644 --- a/lib/oauthutil/oauthutil.go +++ b/lib/oauthutil/oauthutil.go @@ -204,6 +204,37 @@ func (ts *TokenSource) reReadToken() (changed bool) { return changed } +type retrieveErrResponse struct { + Error string `json:"error"` +} + +// If err is nil or an error other than fatal OAuth errors, returns err itself. +// Otherwise returns a more user-friendly error. +func maybeWrapOAuthError(err error, remoteName string) (newErr error) { + newErr = err + if rErr, ok := err.(*oauth2.RetrieveError); ok { + if rErr.Response.StatusCode == 400 || rErr.Response.StatusCode == 401 { + fs.Debugf(remoteName, "got fatal oauth error: %v", rErr) + var resp retrieveErrResponse + if err = json.Unmarshal(rErr.Body, &resp); err != nil { + newErr = fmt.Errorf("(can't decode error info) - try refreshing token with \"rclone config reconnect %s:\"", remoteName) + return + } + var suggestion string + switch resp.Error { + case "invalid_client", "unauthorized_client", "unsupported_grant_type", "invalid_scope": + suggestion = fmt.Sprintf("if you're using your own client id/secret, make sure they're properly set up following the docs") + case "invalid_grant": + fallthrough + default: + suggestion = fmt.Sprintf("maybe token expired? - try refreshing with \"rclone config reconnect %s:\"", remoteName) + } + newErr = fmt.Errorf("%s: %s", resp.Error, suggestion) + } + } + return +} + // Token returns a token or an error. // Token must be safe for concurrent use by multiple goroutines. // The returned Token must not be modified. @@ -242,11 +273,15 @@ func (ts *TokenSource) Token() (*oauth2.Token, error) { if err == nil { break } + if newErr := maybeWrapOAuthError(err, ts.name); newErr != err { + err = newErr // Fatal OAuth error + break + } fs.Debugf(ts.name, "Token refresh failed try %d/%d: %v", i, maxTries, err) time.Sleep(1 * time.Second) } if err != nil { - return nil, fmt.Errorf("couldn't fetch token - maybe it has expired? - refresh with \"rclone config reconnect %s:\": %w", ts.name, err) + return nil, fmt.Errorf("couldn't fetch token: %w", err) } changed = changed || (*token != *ts.token) ts.token = token From 8ee0e2efb15bf0e4aea22c3d6a107726212575fa Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 11 Jan 2023 17:21:47 +0000 Subject: [PATCH 500/560] Add piyushgarg to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index dfcbd2ceab8f4..eaa3c0842ebec 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -674,3 +674,4 @@ put them back in again.` >}} * Jack <16779171+jkpe@users.noreply.github.com> * Abdullah Saglam * Marks Polakovs + * piyushgarg From 9d1b786a391a1b1adc2a3a58f4ea17c1c2cd0bfb Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 11 Jan 2023 17:21:47 +0000 Subject: [PATCH 501/560] Add Kaloyan Raev to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index eaa3c0842ebec..3984ed9a1ac7b 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -675,3 +675,4 @@ put them back in again.` >}} * Abdullah Saglam * Marks Polakovs * piyushgarg + * Kaloyan Raev From 8c6ff1fa7e4ef2fd5277859f22fecaf684567a85 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 11 Jan 2023 16:23:40 +0000 Subject: [PATCH 502/560] cmount: fix creating and renaming files on case insensitive backends Before this fix, we told cgofuse/WinFSP that the backend was case insensitive but didn't implement the Getpath backend function to return the normalised case of a file. Resently cgofuse started implementing case insensitive files properly but since we hadn't implemented Getpath, the file names were taking the default of all in UPPER CASE. This patch implements Getpath for cgofuse which fixes the case problems. This problem came to light when we upgraded cgofuse and WinFSP (to 1.12) which had the code to implement Getpath. Fixes #6682 --- cmd/cmount/fs.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmd/cmount/fs.go b/cmd/cmount/fs.go index c097611b754fd..a60bd833642b0 100644 --- a/cmd/cmount/fs.go +++ b/cmd/cmount/fs.go @@ -8,6 +8,7 @@ import ( "io" "os" "path" + "strings" "sync" "sync/atomic" "time" @@ -567,6 +568,21 @@ func (fsys *FS) Listxattr(path string, fill func(name string) bool) (errc int) { return -fuse.ENOSYS } +// Getpath allows a case-insensitive file system to report the correct case of +// a file path. +func (fsys *FS) Getpath(path string, fh uint64) (errc int, normalisedPath string) { + defer log.Trace(path, "Getpath fh=%d", fh)("errc=%d, normalisedPath=%q", &errc, &normalisedPath) + node, _, errc := fsys.getNode(path, fh) + if errc != 0 { + return errc, "" + } + normalisedPath = node.Path() + if !strings.HasPrefix("/", normalisedPath) { + normalisedPath = "/" + normalisedPath + } + return 0, normalisedPath +} + // Translate errors from mountlib func translateError(err error) (errc int) { if err == nil { @@ -631,6 +647,7 @@ func translateOpenFlags(inFlags int) (outFlags int) { var ( _ fuse.FileSystemInterface = (*FS)(nil) _ fuse.FileSystemOpenEx = (*FS)(nil) + _ fuse.FileSystemGetpath = (*FS)(nil) //_ fuse.FileSystemChflags = (*FS)(nil) //_ fuse.FileSystemSetcrtime = (*FS)(nil) //_ fuse.FileSystemSetchgtime = (*FS)(nil) From 71227986db2cb63e6c570bdda967e70f05f64e5a Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Thu, 12 Jan 2023 20:12:42 +0100 Subject: [PATCH 503/560] docs: remove link to nonexistent uploadfile command - fixes #6693 --- docs/content/rc.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/content/rc.md b/docs/content/rc.md index 1cb4dd55ba9f6..3e1d1c65f7e52 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -1458,8 +1458,6 @@ This takes the following parameters: - remote - a path within that remote e.g. "dir" - each part in body represents a file to be uploaded -See the [uploadfile](/commands/rclone_uploadfile/) command for more information on the above. - **Authentication is required for this call.** ### options/blocks: List all the option blocks {#options-blocks} From e2886aaddfe0ecedabb4a2a6df24c04178fdcbe5 Mon Sep 17 00:00:00 2001 From: Manoj Ghosh Date: Fri, 13 Jan 2023 04:58:52 -0800 Subject: [PATCH 504/560] oracle-object-storage: expose the storage_tier option in config --- backend/oracleobjectstorage/options.go | 16 ++++++++++++++++ docs/content/oracleobjectstorage.md | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/backend/oracleobjectstorage/options.go b/backend/oracleobjectstorage/options.go index 9e7e65fd5f380..2916157be6a46 100644 --- a/backend/oracleobjectstorage/options.go +++ b/backend/oracleobjectstorage/options.go @@ -123,6 +123,22 @@ func newOptions() []fs.Option { Value: "Default", Help: "Use the default profile", }}, + }, { + // Mapping from here: https://github.com/oracle/oci-go-sdk/blob/master/objectstorage/storage_tier.go + Name: "storage_tier", + Help: "The storage class to use when storing new objects in storage. https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/understandingstoragetiers.htm", + Default: "Standard", + Advanced: true, + Examples: []fs.OptionExample{{ + Value: "Standard", + Help: "Standard storage tier, this is the default tier", + }, { + Value: "InfrequentAccess", + Help: "InfrequentAccess storage tier", + }, { + Value: "Archive", + Help: "Archive storage tier", + }}, }, { Name: "upload_cutoff", Help: `Cutoff for switching to chunked upload. diff --git a/docs/content/oracleobjectstorage.md b/docs/content/oracleobjectstorage.md index 29a8d8e970306..8c28f28985024 100644 --- a/docs/content/oracleobjectstorage.md +++ b/docs/content/oracleobjectstorage.md @@ -290,6 +290,24 @@ Properties: Here are the Advanced options specific to oracleobjectstorage (Oracle Cloud Infrastructure Object Storage). +#### --oos-storage-tier + +The storage class to use when storing new objects in storage. https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/understandingstoragetiers.htm + +Properties: + +- Config: storage_tier +- Env Var: RCLONE_OOS_STORAGE_TIER +- Type: string +- Default: "Standard" +- Examples: + - "Standard" + - Standard storage tier, this is the default tier + - "InfrequentAccess" + - InfrequentAccess storage tier + - "Archive" + - Archive storage tier + #### --oos-upload-cutoff Cutoff for switching to chunked upload. From f08bb5bf66e7500ae62508d4cb6060673e741c1e Mon Sep 17 00:00:00 2001 From: Kaloyan Raev Date: Sun, 15 Jan 2023 18:23:49 +0200 Subject: [PATCH 505/560] storj: implement purge --- backend/storj/fs.go | 53 ++++++++++++++++++++++++++++++++++++++-- docs/content/overview.md | 11 ++++++--- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/backend/storj/fs.go b/backend/storj/fs.go index ed3004099ce61..0f941e5de15e6 100644 --- a/backend/storj/fs.go +++ b/backend/storj/fs.go @@ -162,6 +162,7 @@ var ( _ fs.PutStreamer = &Fs{} _ fs.Mover = &Fs{} _ fs.Copier = &Fs{} + _ fs.Purger = &Fs{} _ fs.PublicLinker = &Fs{} ) @@ -774,15 +775,63 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, return newObjectFromUplink(f, remote, newObject), nil } +// Purge all files in the directory specified +// +// Implement this if you have a way of deleting all the files +// quicker than just running Remove() on the result of List() +// +// Return an error if it doesn't exist +func (f *Fs) Purge(ctx context.Context, dir string) error { + bucket, directory := f.absolute(dir) + if bucket == "" { + return errors.New("can't purge from root") + } + + if directory == "" { + _, err := f.project.DeleteBucketWithObjects(ctx, bucket) + if errors.Is(err, uplink.ErrBucketNotFound) { + return fs.ErrorDirNotFound + } + return err + } + + fs.Infof(directory, "Quick delete is available only for entire bucket. Falling back to list and delete.") + objects := f.project.ListObjects(ctx, bucket, + &uplink.ListObjectsOptions{ + Prefix: directory + "/", + Recursive: true, + }, + ) + if err := objects.Err(); err != nil { + return err + } + + empty := true + for objects.Next() { + empty = false + _, err := f.project.DeleteObject(ctx, bucket, objects.Item().Key) + if err != nil { + return err + } + fs.Infof(objects.Item().Key, "Deleted") + } + + if empty { + return fs.ErrorDirNotFound + } + + return nil +} + // PublicLink generates a public link to the remote path (usually readable by anyone) func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) { bucket, key := f.absolute(remote) - if len(bucket) == 0 { + if bucket == "" { return "", errors.New("path must be specified") } // Rclone requires that a link is only generated if the remote path exists - if len(key) == 0 { + if key == "" { _, err := f.project.StatBucket(ctx, bucket) if err != nil { return "", err diff --git a/docs/content/overview.md b/docs/content/overview.md index df19d05a973fb..a9605c4802325 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -504,7 +504,7 @@ upon backend-specific capabilities. | Sia | No | No | No | No | No | No | Yes | No | No | Yes | | SMB | No | No | Yes | Yes | No | No | Yes | No | No | Yes | | SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes | -| Storj | Yes † | Yes | Yes | No | No | Yes | Yes | Yes | No | No | +| Storj | Yes ☨ | Yes | Yes | No | No | Yes | Yes | Yes | No | No | | Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | | WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No | Yes | Yes | | Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | Yes | @@ -516,9 +516,12 @@ upon backend-specific capabilities. This deletes a directory quicker than just deleting all the files in the directory. -† Note Swift and Storj implement this in order to delete -directory markers but they don't actually have a quicker way of deleting -files other than deleting them individually. +† Note Swift implements this in order to delete directory markers but +they don't actually have a quicker way of deleting files other than +deleting them individually. + +☨ Storj implements this efficiently only for entire buckets. If +purging a directory inside a bucket, files are deleted individually. ‡ StreamUpload is not supported with Nextcloud From f31ab6d17857cc5619f321f0d7146721099ba644 Mon Sep 17 00:00:00 2001 From: Fred Date: Wed, 11 Jan 2023 20:25:44 +0000 Subject: [PATCH 506/560] seafile: renew library password - fixes #6662 Passwords for encrypted libraries are kept in memory in the server and flushed after an hour. This MR fixes an issue when the library password expires after 1 hour. --- backend/seafile/renew.go | 54 +++++++++++++++++++ backend/seafile/renew_test.go | 35 ++++++++++++ backend/seafile/seafile.go | 16 ++++++ docs/content/seafile.md | 8 ++- .../init.d/seafile/docker-compose.yml | 4 +- 5 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 backend/seafile/renew.go create mode 100644 backend/seafile/renew_test.go diff --git a/backend/seafile/renew.go b/backend/seafile/renew.go new file mode 100644 index 0000000000000..a67159d52a1ad --- /dev/null +++ b/backend/seafile/renew.go @@ -0,0 +1,54 @@ +package seafile + +import ( + "sync" + "time" + + "github.com/rclone/rclone/fs" +) + +// Renew allows tokens to be renewed on expiry. +type Renew struct { + ts *time.Ticker // timer indicating when it's time to renew the token + run func() error // the callback to do the renewal + done chan interface{} // channel to end the go routine + shutdown *sync.Once +} + +// NewRenew creates a new Renew struct and starts a background process +// which renews the token whenever it expires. It uses the run() call +// to do the renewal. +func NewRenew(every time.Duration, run func() error) *Renew { + r := &Renew{ + ts: time.NewTicker(every), + run: run, + done: make(chan interface{}), + shutdown: &sync.Once{}, + } + go r.renewOnExpiry() + return r +} + +func (r *Renew) renewOnExpiry() { + for { + select { + case <-r.ts.C: + err := r.run() + if err != nil { + fs.Errorf(nil, "error while refreshing decryption token: %s", err) + } + + case <-r.done: + return + } + } +} + +// Shutdown stops the ticker and no more renewal will take place. +func (r *Renew) Shutdown() { + // closing a channel can only be done once + r.shutdown.Do(func() { + r.ts.Stop() + close(r.done) + }) +} diff --git a/backend/seafile/renew_test.go b/backend/seafile/renew_test.go new file mode 100644 index 0000000000000..4e9c4e13b8fd9 --- /dev/null +++ b/backend/seafile/renew_test.go @@ -0,0 +1,35 @@ +package seafile + +import ( + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestShouldAllowShutdownTwice(t *testing.T) { + renew := NewRenew(time.Hour, func() error { + return nil + }) + renew.Shutdown() + renew.Shutdown() +} + +func TestRenewal(t *testing.T) { + var count int64 + + wg := sync.WaitGroup{} + wg.Add(2) // run the renewal twice + renew := NewRenew(time.Millisecond, func() error { + atomic.AddInt64(&count, 1) + wg.Done() + return nil + }) + wg.Wait() + renew.Shutdown() + + // it is technically possible that a third renewal gets triggered between Wait() and Shutdown() + assert.GreaterOrEqual(t, atomic.LoadInt64(&count), int64(2)) +} diff --git a/backend/seafile/seafile.go b/backend/seafile/seafile.go index d1c2abdbcae15..58dfd79c21fad 100644 --- a/backend/seafile/seafile.go +++ b/backend/seafile/seafile.go @@ -143,6 +143,7 @@ type Fs struct { createDirMutex sync.Mutex // Protect creation of directories useOldDirectoryAPI bool // Use the old API v2 if seafile < 7 moveDirNotAvailable bool // Version < 7.0 don't have an API to move a directory + renew *Renew // Renew an encrypted library token } // ------------------------------------------------------------ @@ -268,6 +269,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e } // And remove the public link feature f.features.PublicLink = nil + + // renew the library password every 45 minutes + f.renew = NewRenew(45*time.Minute, func() error { + return f.authorizeLibrary(context.Background(), libraryID) + }) } } else { // Deactivate the cleaner feature since there's no library selected @@ -383,6 +389,15 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf return nil, fmt.Errorf("unknown state %q", config.State) } +// Shutdown the Fs +func (f *Fs) Shutdown(ctx context.Context) error { + if f.renew == nil { + return nil + } + f.renew.Shutdown() + return nil +} + // sets the AuthorizationToken up func (f *Fs) setAuthorizationToken(token string) { f.srv.SetHeader("Authorization", "Token "+token) @@ -1331,6 +1346,7 @@ var ( _ fs.PutStreamer = &Fs{} _ fs.PublicLinker = &Fs{} _ fs.UserInfoer = &Fs{} + _ fs.Shutdowner = &Fs{} _ fs.Object = &Object{} _ fs.IDer = &Object{} ) diff --git a/docs/content/seafile.md b/docs/content/seafile.md index bfa969a5082d6..57399657a6759 100644 --- a/docs/content/seafile.md +++ b/docs/content/seafile.md @@ -8,9 +8,10 @@ versionIntroduced: "v1.52" This is a backend for the [Seafile](https://www.seafile.com/) storage service: - It works with both the free community edition or the professional edition. -- Seafile versions 6.x and 7.x are all supported. +- Seafile versions 6.x, 7.x, 8.x and 9.x are all supported. - Encrypted libraries are also supported. - It supports 2FA enabled users +- Using a Library API Token is **not** supported ## Configuration @@ -256,14 +257,17 @@ that has already been shared, you will get the exact same link. ### Compatibility -It has been actively tested using the [seafile docker image](https://github.com/haiwen/seafile-docker) of these versions: +It has been actively developed using the [seafile docker image](https://github.com/haiwen/seafile-docker) of these versions: - 6.3.4 community edition - 7.0.5 community edition - 7.1.3 community edition +- 9.0.10 community edition Versions below 6.0 are not supported. Versions between 6.0 and 6.3 haven't been tested and might not work properly. +Each new version of `rclone` is automatically tested against the [latest docker image](https://hub.docker.com/r/seafileltd/seafile-mc/) of the seafile community server. + {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/seafile/seafile.go then run make backenddocs" >}} ### Standard options diff --git a/fstest/testserver/init.d/seafile/docker-compose.yml b/fstest/testserver/init.d/seafile/docker-compose.yml index 55d8e2f69c365..8e0a099cdc249 100644 --- a/fstest/testserver/init.d/seafile/docker-compose.yml +++ b/fstest/testserver/init.d/seafile/docker-compose.yml @@ -1,7 +1,7 @@ version: '2.0' services: db: - image: mariadb:10.1 + image: mariadb:10.5 environment: - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} - MYSQL_LOG_CONSOLE=true @@ -9,7 +9,7 @@ services: - ${SEAFILE_TEST_DATA}/${NAME}/seafile-mysql/db:/var/lib/mysql memcached: - image: memcached:1.5.6 + image: memcached:1.6.9 entrypoint: memcached -m 256 seafile: From 10bf8a769e0f70af21fe609a4f7d1f48fbdb8981 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 12 Jan 2023 12:21:21 +0000 Subject: [PATCH 507/560] build: update dependencies This fixes the azureblob backend so it builds again after the SDK changes. --- backend/azureblob/azureblob.go | 2 +- go.mod | 74 ++++---- go.sum | 314 +++++++++++++++++++++++++-------- 3 files changed, 275 insertions(+), 115 deletions(-) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 2376f44fecfbc..4fb141e4de752 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -2139,7 +2139,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64, rs := readSeekCloser{wrappedReader, bufferReader} options := blockblob.StageBlockOptions{ // Specify the transactional md5 for the body, to be validated by the service. - TransactionalContentMD5: transactionalMD5, + TransactionalValidation: blob.TransferValidationTypeMD5(transactionalMD5), } _, err = blb.StageBlock(ctx, blockID, &rs, &options) return o.fs.shouldRetry(ctx, err) diff --git a/go.mod b/go.mod index 8cf92794b4cc6..6206969181597 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module github.com/rclone/rclone go 1.17 require ( - bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 + bazil.org/fuse v0.0.0-20221209211307-2abb8038c751 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 - github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5 github.com/Unknwon/goconfig v1.0.0 github.com/a8m/tree v0.0.0-20210414114729-ce3525c5c2ef @@ -17,7 +17,7 @@ require ( github.com/anacrolix/log v0.13.1 github.com/artyom/mtab v1.0.0 github.com/atotto/clipboard v0.1.4 - github.com/aws/aws-sdk-go v1.44.145 + github.com/aws/aws-sdk-go v1.44.178 github.com/buengese/sgzip v0.1.1 github.com/colinmarc/hdfs/v2 v2.3.0 github.com/coreos/go-semver v0.3.0 @@ -25,23 +25,23 @@ require ( github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00 github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 github.com/gabriel-vasile/mimetype v1.4.1 - github.com/gdamore/tcell/v2 v2.5.3 - github.com/go-chi/chi/v5 v5.0.7 + github.com/gdamore/tcell/v2 v2.5.4 + github.com/go-chi/chi/v5 v5.0.8 github.com/google/uuid v1.3.0 - github.com/hanwen/go-fuse/v2 v2.1.0 + github.com/hanwen/go-fuse/v2 v2.2.0 github.com/hirochachacha/go-smb2 v1.1.0 github.com/iguanesolutions/go-systemd/v5 v5.1.0 github.com/jcmturner/gokrb5/v8 v8.4.3 github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 - github.com/klauspost/compress v1.15.12 - github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348 - github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a + github.com/klauspost/compress v1.15.14 + github.com/koofr/go-httpclient v0.0.0-20221124135700-2eb26cff5dd8 + github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-runewidth v0.0.14 github.com/mitchellh/go-homedir v1.1.0 github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 github.com/ncw/swift/v2 v2.0.1 - github.com/oracle/oci-go-sdk/v65 v65.26.1 + github.com/oracle/oci-go-sdk/v65 v65.28.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/sftp v1.13.5 github.com/pmezard/go-difflib v1.0.0 @@ -49,8 +49,8 @@ require ( github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6 github.com/rfjakob/eme v1.1.2 - github.com/rivo/uniseg v0.2.0 - github.com/shirou/gopsutil/v3 v3.22.10 + github.com/rivo/uniseg v0.4.3 + github.com/shirou/gopsutil/v3 v3.22.12 github.com/sirupsen/logrus v1.9.0 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/spf13/cobra v1.6.1 @@ -63,22 +63,22 @@ require ( github.com/yunify/qingstor-sdk-go/v3 v3.2.0 go.etcd.io/bbolt v1.3.6 goftp.io/server v0.4.1 - golang.org/x/crypto v0.3.0 - golang.org/x/net v0.4.0 - golang.org/x/oauth2 v0.2.0 + golang.org/x/crypto v0.5.0 + golang.org/x/net v0.5.0 + golang.org/x/oauth2 v0.4.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.3.0 - golang.org/x/text v0.5.0 - golang.org/x/time v0.2.0 - google.golang.org/api v0.103.0 + golang.org/x/sys v0.4.0 + golang.org/x/text v0.6.0 + golang.org/x/time v0.3.0 + google.golang.org/api v0.106.0 gopkg.in/yaml.v2 v2.4.0 - storj.io/uplink v1.9.0 + storj.io/uplink v1.10.0 ) require ( - cloud.google.com/go/compute v1.12.1 // indirect - cloud.google.com/go/compute/metadata v0.2.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + cloud.google.com/go/compute v1.14.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/calebcase/tmpfile v1.0.3 // indirect @@ -92,7 +92,7 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -104,13 +104,14 @@ require ( github.com/jcmturner/goidentity/v6 v6.0.1 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect + github.com/klauspost/cpuid/v2 v2.0.12 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/onsi/gomega v1.13.0 // indirect github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -119,30 +120,31 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect - github.com/spacemonkeygo/monkit/v3 v3.0.17 // indirect - github.com/tklauser/go-sysconf v0.3.10 // indirect - github.com/tklauser/numcpus v0.4.0 // indirect + github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect + github.com/zeebo/blake3 v0.2.3 // indirect github.com/zeebo/errs v1.3.0 // indirect go.opencensus.io v0.24.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c // indirect - google.golang.org/grpc v1.50.1 // indirect + google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect + google.golang.org/grpc v1.51.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - storj.io/common v0.0.0-20220414110316-a5cb7172d6bf // indirect - storj.io/drpc v0.0.30 // indirect + storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect + storj.io/drpc v0.0.32 // indirect ) require ( github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 + github.com/ProtonMail/go-crypto v0.0.0-20230109192245-7efeeb08f296 github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/pkg/xattr v0.4.9 golang.org/x/mobile v0.0.0-20221110043201-43a038452099 - golang.org/x/term v0.3.0 + golang.org/x/term v0.4.0 ) diff --git a/go.sum b/go.sum index e1e85c49cb0c3..b2191d4e1cdec 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05 h1:UrYe9YkT4Wpm6D+zByEyCJQzDqTPXqTDUI7bZ41i9VE= -bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05/go.mod h1:h0h5FBYpXThbvSfTqthw+0I4nmHnhTHkO5BoOHsBWqg= +bazil.org/fuse v0.0.0-20221209211307-2abb8038c751 h1:WDXfyDLJ+tg8PYC6yIkOmc/RWFrqMgxk1rLpRrlR8Ng= +bazil.org/fuse v0.0.0-20221209211307-2abb8038c751/go.mod h1:eX+feLR06AMFrTGQBzFnMMDz1vjBv2yHZBFlI9RJeaQ= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -30,6 +30,7 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= @@ -37,33 +38,46 @@ cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRY cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -71,19 +85,28 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= +cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= @@ -92,158 +115,230 @@ cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLq cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= -cloud.google.com/go/longrunning v0.1.1 h1:y50CXG4j0+qvEukslYFBCrzaXX0qpFbBzc3PchSu/LE= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -253,30 +348,45 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= @@ -284,18 +394,20 @@ dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= -github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU= -github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 h1:Oj853U9kG+RLTCQXpjvOnrv0WaZHxgmZz1TlLywgOPY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= @@ -307,8 +419,8 @@ github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5/go.mod h1:C8yoIf github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I= -github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= +github.com/ProtonMail/go-crypto v0.0.0-20230109192245-7efeeb08f296 h1:865LKksDklBvemmkvQ2TO6yArI12PChQJn9R/2S8ov0= +github.com/ProtonMail/go-crypto v0.0.0-20230109192245-7efeeb08f296/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= @@ -338,8 +450,8 @@ github.com/artyom/mtab v1.0.0 h1:r7OSVo5Jeqi8+LotZ0rT2kzfPIBp9KCpEJP8RQqGmSE= github.com/artyom/mtab v1.0.0/go.mod h1:EHpkp5OmPfS1yZX+/DFTztlJ9di5UzdDLX1/XzWPXw8= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/aws/aws-sdk-go v1.44.145 h1:KMVRrIyjBsNz3xGPuHIRnhIuKlb5h3Ii5e5jbi3cgnc= -github.com/aws/aws-sdk-go v1.44.145/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.178 h1:4igreoWPEA7xVLnOeSXLhDXTsTSPKQONZcQ3llWAJw0= +github.com/aws/aws-sdk-go v1.44.178/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -353,6 +465,7 @@ github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7N github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo= github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -411,22 +524,23 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= -github.com/gdamore/tcell/v2 v2.5.3 h1:b9XQrT6QGbgI7JvZOJXFNczOQeIYbo8BfeSMzt2sAV0= -github.com/gdamore/tcell/v2 v2.5.3/go.mod h1:wSkrPaXoiIWZqW/g7Px4xc79di6FTcpB8tvaKJ6uGBo= +github.com/gdamore/tcell/v2 v2.5.4 h1:TGU4tSjD3sCL788vFNeJnTdzpNKIw1H5dgLnJRQVv/k= +github.com/gdamore/tcell/v2 v2.5.4/go.mod h1:dZgRy5v4iMobMEcWNYBtREnDZAT9DYmfqIkrgEMxLyw= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8= -github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= +github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -439,6 +553,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -529,6 +644,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -541,8 +657,9 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= @@ -568,10 +685,8 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc= -github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= -github.com/hanwen/go-fuse/v2 v2.1.0 h1:+32ffteETaLYClUj0a3aHjZ1hOPxxaNEHiZiujuDaek= -github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= +github.com/hanwen/go-fuse/v2 v2.2.0 h1:jo5QZYmBLNcl9ovypWaQ5yXMSSV+Ch68xoC3rtZvvBM= +github.com/hanwen/go-fuse/v2 v2.2.0/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= @@ -623,20 +738,24 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b h1:tO4MX3k5bvV0Sjv5jYrxStMTJxf1m/TW24XRyHji4aU= +github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b/go.mod h1:q7yMR8BavTz/gBNtIT/uF487LMgcuEpNGKISLAjNQes= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= -github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= +github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348 h1:Lrn8srO9JDBCf2iPjqy62stl49UDwoOxZ9/NGVi+fnk= -github.com/koofr/go-httpclient v0.0.0-20200420163713-93aa7c75b348/go.mod h1:JBLy//Q5jzU3XSMxdONTD5EIj1LhTPktosxG2Bw1iho= -github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a h1:02cx9xF4W2FQ1oh8CK9dWV5BnZK2mUtcbr9xR+bZiKk= -github.com/koofr/go-koofrclient v0.0.0-20190724113126-8e5366da203a/go.mod h1:MRAz4Gsxd+OzrZ0owwrUHc0zLESL+1Y5syqK/sJxK2A= +github.com/koofr/go-httpclient v0.0.0-20221124135700-2eb26cff5dd8 h1:xcux4bzP9xUWcgCWhbiJrGW2qQUwyK8Lj9hWRCxk6hU= +github.com/koofr/go-httpclient v0.0.0-20221124135700-2eb26cff5dd8/go.mod h1:lyCsww1Zr6kDOtAPNmV8/va7lHqyeP5ImNL2cS83iYg= +github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 h1:FHVoZMOVRA+6/y4yRlbiR3WvsrOcKBd/f64H7YiWR2U= +github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6/go.mod h1:MRAz4Gsxd+OzrZ0owwrUHc0zLESL+1Y5syqK/sJxK2A= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -652,7 +771,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg= +github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -660,15 +779,14 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= -github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= -github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8= -github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI= +github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= +github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= +github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -700,20 +818,32 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/oracle/oci-go-sdk/v65 v65.26.1 h1:Ms20RSRj+CuvQmw5ET1TkmzxLBI+bmLjZ6NYANA3gkk= -github.com/oracle/oci-go-sdk/v65 v65.26.1/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE= +github.com/oracle/oci-go-sdk/v65 v65.28.0 h1:FgBWmHzT1y9H9tRQYOiX22Z8ec5p6/M5sSYRGcbWAno= +github.com/oracle/oci-go-sdk/v65 v65.28.0/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/getopt v1.1.0/go.mod h1:FxXoW1Re00sQG/+KIkuSqRL/LwQgSkv7uyac+STFsbk= @@ -769,8 +899,9 @@ github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6 h1:J832KfU2Z44Ck3XR5bvw github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6/go.mod h1:qRpxqlna6CaIq9fSRud1bDC5S7EEUEou0j8nMZ0lxO8= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= +github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -783,8 +914,9 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil/v3 v3.22.10 h1:4KMHdfBRYXGF9skjDWiL4RA2N+E8dRdodU/bOZpPoVg= -github.com/shirou/gopsutil/v3 v3.22.10/go.mod h1:QNza6r4YQoydyCfo6rH0blGfKahgibh4dQmV5xdFkQk= +github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= +github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -825,8 +957,9 @@ github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/monkit/v3 v3.0.4/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes= -github.com/spacemonkeygo/monkit/v3 v3.0.17 h1:rqIuLhRUr2UtS3WNVbPY/BwvjlwKVvSOVY5p0QVocxE= -github.com/spacemonkeygo/monkit/v3 v3.0.17/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4= +github.com/spacemonkeygo/monkit/v3 v3.0.18/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4= +github.com/spacemonkeygo/monkit/v3 v3.0.19 h1:wqBb9bpD7jXkVi4XwIp8jn1fektaVBQ+cp9SHRXgAdo= +github.com/spacemonkeygo/monkit/v3 v3.0.19/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4= github.com/spacemonkeygo/monotime v0.0.0-20180824235756-e3f48a95f98a/go.mod h1:ul4bvvnCOPZgq8w0nTkSmWVg/hauVpFS97Am1YM1XXo= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= @@ -855,10 +988,10 @@ github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf h1:Y43S3e9P1NPs/Q github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf/go.mod h1:c+cGNU1qi9bO7ZF4IRMYk+KaZTNiQ/gQrSbyMmGFq1Q= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= -github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= -github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= -github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= @@ -877,19 +1010,26 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yunify/qingstor-sdk-go/v3 v3.2.0 h1:9sB2WZMgjwSUNZhrgvaNGazVltoFUUfuS9f0uCWtTr8= github.com/yunify/qingstor-sdk-go/v3 v3.2.0/go.mod h1:KciFNuMu6F4WLk9nGwwK69sCGKLCdd9f97ac/wfumS4= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/admission/v3 v3.0.3/go.mod h1:2OWyAS5yo0Xvj2AEUosOjTUHxaY0oIIiCrXGKCYzWpo= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A= +github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= +github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs= github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= github.com/zeebo/float16 v0.1.0/go.mod h1:fssGvvXu+XS8MH57cKmyrLB/cqioYeYX/2mXCN3a5wo= github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54/go.mod h1:EI8LcOBDlSL3POyqwC1eJhOYlMBMidES+613EtmmT5w= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= @@ -924,12 +1064,13 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -969,7 +1110,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1017,6 +1160,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -1034,8 +1178,8 @@ golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1063,8 +1207,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.2.0 h1:GtQkldQ9m7yvzCL1V+LrYow3Khe0eJH0w7RbX/VbaIU= -golang.org/x/oauth2 v0.2.0/go.mod h1:Cwn6afJ8jrQwYMxQDTpISoXmXW9I6qF6vDeuuoX3Ibs= +golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= +golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1152,6 +1296,7 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1159,10 +1304,11 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220318055525-2edf467146b5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1175,15 +1321,15 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1195,14 +1341,17 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= -golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1261,7 +1410,9 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1269,7 +1420,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -1324,8 +1474,9 @@ google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= -google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0 h1:ffmW0faWCwKkpbbtvlY/K/8fUl+JKvNS5CVzRoyfCv8= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1442,8 +1593,14 @@ google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo= google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1481,8 +1638,9 @@ google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1539,9 +1697,9 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= -storj.io/common v0.0.0-20220414110316-a5cb7172d6bf h1:D5xZTDOlTTQWdAWeKKm2pFLcz1sceH+f/pVAcYB9jL8= -storj.io/common v0.0.0-20220414110316-a5cb7172d6bf/go.mod h1:LBJrpAqL4MNSrhGEwc8SJ+tIVtgfCtFEZqDy6/0j67A= -storj.io/drpc v0.0.30 h1:jqPe4T9KEu3CDBI05A2hCMgMSHLtd/E0N0yTF9QreIE= -storj.io/drpc v0.0.30/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= -storj.io/uplink v1.9.0 h1:Zg1kX1VqOQIKm0yAukteKpLuT68Be3euyNRML612ERM= -storj.io/uplink v1.9.0/go.mod h1:f6D8306j5mnRHnPDKWCiwtPM6ukyGg77to9LaAY9l6k= +storj.io/common v0.0.0-20221123115229-fed3e6651b63 h1:OuleF/3FvZe3Nnu6NdwVr+FvCXjfD4iNNdgfI2kcs3k= +storj.io/common v0.0.0-20221123115229-fed3e6651b63/go.mod h1:+gF7jbVvpjVIVHhK+EJFhfPbudX395lnPq/dKkj/Qys= +storj.io/drpc v0.0.32 h1:5p5ZwsK/VOgapaCu+oxaPVwO6UwIs+iwdMiD50+R4PI= +storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= +storj.io/uplink v1.10.0 h1:3hS0hszupHSxEoC4DsMpljaRy0uNoijEPVF6siIE28Q= +storj.io/uplink v1.10.0/go.mod h1:gJIQumB8T3tBHPRive51AVpbc+v2xe+P/goFNMSRLG4= From 559157cb582e7b50734f2037a9ba0d98e549f7d0 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 12 Jan 2023 12:36:56 +0000 Subject: [PATCH 508/560] azureblob: remove workarounds for SDK bugs after v0.6.1 update --- backend/azureblob/azureblob.go | 71 +--------------------------------- 1 file changed, 2 insertions(+), 69 deletions(-) diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 4fb141e4de752..6a2ffddd8e3cc 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -4,32 +4,6 @@ // Package azureblob provides an interface to the Microsoft Azure blob object storage system package azureblob -/* FIXME - -Note these Azure SDK bugs which are affecting the backend - -azblob UploadStream produces panic: send on closed channel if input stream has error #19612 -https://github.com/Azure/azure-sdk-for-go/issues/19612 - - FIXED by re-implementing UploadStream - -azblob: when using SharedKey credentials, can't reference some blob names with ? in #19613 -https://github.com/Azure/azure-sdk-for-go/issues/19613 - - FIXED by url encoding getBlobSVC and getBlockBlobSVC - -Azure Blob Storage paths are not URL-escaped #19475 -https://github.com/Azure/azure-sdk-for-go/issues/19475 - - FIXED by url encoding getBlobSVC and getBlockBlobSVC - -Controlling TransferManager #19579 -https://github.com/Azure/azure-sdk-for-go/issues/19579 - - FIXED by re-implementing UploadStream - -azblob: blob.StartCopyFromURL doesn't work with UTF-8 characters in the source blob #19614 -https://github.com/Azure/azure-sdk-for-go/issues/19614 - - FIXED by url encoding getBlobSVC and getBlockBlobSVC - -*/ - import ( "bytes" "context" @@ -959,18 +933,12 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { // getBlobSVC creates a blob client func (f *Fs) getBlobSVC(container, containerPath string) *blob.Client { - // FIXME the urlEncode here is a workaround for - // https://github.com/Azure/azure-sdk-for-go/issues/19613 - // https://github.com/Azure/azure-sdk-for-go/issues/19475 - return f.cntSVC(container).NewBlobClient(urlEncode(containerPath)) + return f.cntSVC(container).NewBlobClient(containerPath) } // getBlockBlobSVC creates a block blob client func (f *Fs) getBlockBlobSVC(container, containerPath string) *blockblob.Client { - // FIXME the urlEncode here is a workaround for - // https://github.com/Azure/azure-sdk-for-go/issues/19613 - // https://github.com/Azure/azure-sdk-for-go/issues/19475 - return f.cntSVC(container).NewBlockBlobClient(urlEncode(containerPath)) + return f.cntSVC(container).NewBlockBlobClient(containerPath) } // updateMetadataWithModTime adds the modTime passed in to o.meta. @@ -1903,41 +1871,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read return downloadResponse.Body, nil } -// dontEncode is the characters that do not need percent-encoding -// -// The characters that do not need percent-encoding are a subset of -// the printable ASCII characters: upper-case letters, lower-case -// letters, digits, ".", "_", "-", "/", "~", "!", "$", "'", "(", ")", -// "*", ";", "=", ":", and "@". All other byte values in a UTF-8 must -// be replaced with "%" and the two-digit hex value of the byte. -const dontEncode = (`abcdefghijklmnopqrstuvwxyz` + - `ABCDEFGHIJKLMNOPQRSTUVWXYZ` + - `0123456789` + - `._-/~!$'()*;=:@`) - -// noNeedToEncode is a bitmap of characters which don't need % encoding -var noNeedToEncode [256]bool - -func init() { - for _, c := range dontEncode { - noNeedToEncode[c] = true - } -} - -// urlEncode encodes in with % encoding -func urlEncode(in string) string { - var out bytes.Buffer - for i := 0; i < len(in); i++ { - c := in[i] - if noNeedToEncode[c] { - _ = out.WriteByte(c) - } else { - _, _ = out.WriteString(fmt.Sprintf("%%%02X", c)) - } - } - return out.String() -} - // poolWrapper wraps a pool.Pool as an azblob.TransferManager type poolWrapper struct { pool *pool.Pool From 88c0d786394248c987a7d31add790ae93ca03323 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 12 Jan 2023 16:29:16 +0000 Subject: [PATCH 509/560] build: update to fuse3 after bazil.org/fuse update --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 29bdde22be5b6..c855a51e894a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -122,7 +122,7 @@ jobs: sudo modprobe fuse sudo chmod 666 /dev/fuse sudo chown root:$USER /etc/fuse.conf - sudo apt-get install fuse libfuse-dev rpm pkg-config + sudo apt-get install fuse3 libfuse-dev rpm pkg-config if: matrix.os == 'ubuntu-latest' - name: Install Libraries on macOS From 1680c5af8f2408a66f2e1aea11ab7b15b9db70af Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 12 Jan 2023 12:16:00 +0000 Subject: [PATCH 510/560] build: update to go1.20rc3 and make go1.17 the minimum required version --- .github/workflows/build.yml | 26 +- fs/versioncheck.go | 8 +- go.mod | 2 +- go.sum | 830 ------------------------------------ 4 files changed, 18 insertions(+), 848 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c855a51e894a3..7c56e8d38ff0a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,12 +25,12 @@ jobs: strategy: fail-fast: false matrix: - job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.17', 'go1.18'] + job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.18', 'go1.19'] include: - job_name: linux os: ubuntu-latest - go: '1.19' + go: '1.20.0-rc.3' gotags: cmount build_flags: '-include "^linux/"' check: true @@ -41,14 +41,14 @@ jobs: - job_name: linux_386 os: ubuntu-latest - go: '1.19' + go: '1.20.0-rc.3' goarch: 386 gotags: cmount quicktest: true - job_name: mac_amd64 os: macos-11 - go: '1.19' + go: '1.20.0-rc.3' gotags: 'cmount' build_flags: '-include "^darwin/amd64" -cgo' quicktest: true @@ -57,14 +57,14 @@ jobs: - job_name: mac_arm64 os: macos-11 - go: '1.19' + go: '1.20.0-rc.3' gotags: 'cmount' build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib' deploy: true - job_name: windows os: windows-latest - go: '1.19' + go: '1.20.0-rc.3' gotags: cmount cgo: '0' build_flags: '-include "^windows/"' @@ -74,20 +74,20 @@ jobs: - job_name: other_os os: ubuntu-latest - go: '1.19' + go: '1.20.0-rc.3' build_flags: '-exclude "^(windows/|darwin/|linux/)"' compile_all: true deploy: true - - job_name: go1.17 + - job_name: go1.18 os: ubuntu-latest - go: '1.17' + go: '1.18' quicktest: true racequicktest: true - - job_name: go1.18 + - job_name: go1.19 os: ubuntu-latest - go: '1.18' + go: '1.19' quicktest: true racequicktest: true @@ -237,7 +237,7 @@ jobs: - name: Install Go uses: actions/setup-go@v3 with: - go-version: 1.19 + go-version: 1.20.0-rc.3 check-latest: true - name: Install govulncheck @@ -262,7 +262,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19 + go-version: 1.20.0-rc.3 - name: Go module cache uses: actions/cache@v3 diff --git a/fs/versioncheck.go b/fs/versioncheck.go index 9820eccadbc73..7a58a4ac5decd 100644 --- a/fs/versioncheck.go +++ b/fs/versioncheck.go @@ -1,8 +1,8 @@ -//go:build !go1.17 -// +build !go1.17 +//go:build !go1.18 +// +build !go1.18 package fs -// Upgrade to Go version 1.17 to compile rclone - latest stable go +// Upgrade to Go version 1.18 to compile rclone - latest stable go // compiler recommended. -func init() { Go_version_1_17_required_for_compilation() } +func init() { Go_version_1_18_required_for_compilation() } diff --git a/go.mod b/go.mod index 6206969181597..a4090a21b271c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/rclone/rclone -go 1.17 +go 1.18 require ( bazil.org/fuse v0.0.0-20221209211307-2abb8038c751 diff --git a/go.sum b/go.sum index b2191d4e1cdec..b9237905a80de 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,7 @@ bazil.org/fuse v0.0.0-20221209211307-2abb8038c751 h1:WDXfyDLJ+tg8PYC6yIkOmc/RWFrqMgxk1rLpRrlR8Ng= bazil.org/fuse v0.0.0-20221209211307-2abb8038c751/go.mod h1:eX+feLR06AMFrTGQBzFnMMDz1vjBv2yHZBFlI9RJeaQ= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= @@ -17,408 +15,48 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= -cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= -cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= -cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= -cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= -cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= -cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= -cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= -cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= -cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= -cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= -cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= -cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= -cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= -cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= -cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= -cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= -cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= -cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= -cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= -cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= -cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= -cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= -cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= -cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= -cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= -cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= -cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= -cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= -cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= -cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= -cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= -cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= -cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= -cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= -cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= -cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= -cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= -cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= -cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= -cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= -cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= -cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= -cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= -cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= -cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= -cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= -cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= -cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= -cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= -cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= -cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= -cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= -cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= -cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= -cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= -cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= -cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= -cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= -cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= -cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= -cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= -cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= -cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= -cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= -cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= -cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= -cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= -cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= -cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= -cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= -cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= -cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= -cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= -cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= -cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= -cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= -cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= -cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= -cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= -cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= -cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= -cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= -cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= -cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= -cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= -cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= -cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= -cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= -cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= -cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= -cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= -cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= -cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= -cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= -cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= -cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= -cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= -cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= -cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= -cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= -cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= -cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= -cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= -cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= -cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= -cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= -cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= -cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= -cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= -cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= -cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= -cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= -cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= -cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= -cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= -cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= -cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= -cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= -cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= -cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= -cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= -cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= -cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= -cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= -cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= -cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= -cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= -cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= -cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= -cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= -cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= -cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= -cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= -cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= -cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= -cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= -cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= -cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= -cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= -cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= -cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= -cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= -cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= -cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= -cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= -cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= -cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= -cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= -cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= -cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= -cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= -cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= -cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= -cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= -cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= -cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= -cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= -cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= -cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= -cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= -cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= -cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= -cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= -cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= -cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= -cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= -cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= -cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= -cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= -cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= -cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= -cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= -cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= -cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= -cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= -cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= -cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= -cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= -cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= -cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= -cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= -cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= -cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= -cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= -cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= -cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= -cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= -cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= -cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= -cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= -cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= -cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= -cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= -cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= -cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= -cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= -cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= -cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= -cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= -cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= -cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= -cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= -cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= -cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= -cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= -cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= -cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= -cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= -cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= -cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= -cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= -cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= -cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= -cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= -cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= -cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= -cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= -cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= -cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= -cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= -cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= -cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= -cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= -cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= -cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= -cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= -cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= -cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= -cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= -cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= -cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= -cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= -cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= -cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= -cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= -cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= -cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= -cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= -cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= -cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= -cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= -cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= -cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= -cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= -cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= -cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= -cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= -cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= -cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= -cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= -cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= -cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= -cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= -cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= -cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= -cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= -cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= -cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= -cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= -cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= -cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= -cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= -cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= -cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= -cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= -cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= -cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= -cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= -cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= -cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= -cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= -cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= -cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= -cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= -cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= -cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= -cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= -cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= -cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= -cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= -cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= -cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= -cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= -cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= -cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= -cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= -cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= -cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= -cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= -cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= -cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= -cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= -cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= -cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= -cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= -cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= -cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= -cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= -cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= -cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= -cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= -cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= -cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= -cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= -cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= -cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= -cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= -cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= -cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= -cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= -cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= -cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= -dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= -dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= -dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= -git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1 h1:Oj853U9kG+RLTCQXpjvOnrv0WaZHxgmZz1TlLywgOPY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5 h1:w/vNc+SQRYKGWBHeDrzvvNttHwZEbSAP0kmTdORl4OI= github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v0.0.0-20230109192245-7efeeb08f296 h1:865LKksDklBvemmkvQ2TO6yArI12PChQJn9R/2S8ov0= github.com/ProtonMail/go-crypto v0.0.0-20230109192245-7efeeb08f296/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= @@ -444,8 +82,6 @@ github.com/anacrolix/log v0.13.1 h1:BmVwTdxHd5VcNrLylgKwph4P4wf+5VvPgOK4yi91fTY= github.com/anacrolix/log v0.13.1/go.mod h1:D4+CvN8SnruK6zIFS/xPoRJmtvtnxs+CSfDQ+BFxZ68= github.com/anacrolix/missinggo v1.1.0/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo= github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/artyom/mtab v1.0.0 h1:r7OSVo5Jeqi8+LotZ0rT2kzfPIBp9KCpEJP8RQqGmSE= github.com/artyom/mtab v1.0.0/go.mod h1:EHpkp5OmPfS1yZX+/DFTztlJ9di5UzdDLX1/XzWPXw8= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= @@ -456,22 +92,16 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= github.com/buengese/sgzip v0.1.1 h1:ry+T8l1mlmiWEsDrH/YHZnCVWD2S3im1KLsyO+8ZmTU= github.com/buengese/sgzip v0.1.1/go.mod h1:i5ZiXGF3fhV7gL1xaRRL1nDnmpNj0X061FQzOS8VMas= -github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo= github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -479,19 +109,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/colinmarc/hdfs/v2 v2.3.0 h1:tMxOjXn6+7iPUlxAyup9Ha2hnmLe3Sv5DM2qqbSQ2VY= github.com/colinmarc/hdfs/v2 v2.3.0/go.mod h1:nsyY1uyQOomU34KVQk9Qb/lDJobN1MQ/9WS6IqcVZno= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= @@ -501,7 +122,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= -github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00 h1:xJBhC00smQpSZw3Kr0ErMUBXhUSjYoLRm2szxdbRBL0= github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00/go.mod h1:nNICngOdmNImBb/vuL+dSc0aIg3ryNATpjxThNoPw4g= @@ -509,24 +129,11 @@ github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXI github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M= github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= -github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= @@ -535,13 +142,10 @@ github.com/gdamore/tcell/v2 v2.5.4 h1:TGU4tSjD3sCL788vFNeJnTdzpNKIw1H5dgLnJRQVv/ github.com/gdamore/tcell/v2 v2.5.4/go.mod h1:dZgRy5v4iMobMEcWNYBtREnDZAT9DYmfqIkrgEMxLyw= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -553,20 +157,15 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -575,7 +174,6 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -583,8 +181,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -600,11 +196,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -620,19 +214,13 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -640,41 +228,17 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20211108044417-e9b028704de0 h1:rsq1yB2xiFLDYYaYdlGBsSkwVzsCo500wMhxvW5A/bk= -github.com/google/pprof v0.0.0-20211108044417-e9b028704de0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU= -github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= -github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= -github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -682,9 +246,6 @@ github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyC github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hanwen/go-fuse/v2 v2.2.0 h1:jo5QZYmBLNcl9ovypWaQ5yXMSSV+Ch68xoC3rtZvvBM= github.com/hanwen/go-fuse/v2 v2.2.0/go.mod h1:B1nGE/6RBFyBRC1RRnf23UpwCdyJ31eukw34oAKukAc= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -698,11 +259,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/iguanesolutions/go-systemd/v5 v5.1.0 h1:UWprhbpxjLM0vvwu4MxaBR+/KzSxgvnKpM9Q3MBhTAc= github.com/iguanesolutions/go-systemd/v5 v5.1.0/go.mod h1:XprNDEZ9zdPzEg1WrmpV1BnGorgP0WP40AGurMxeQOY= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= @@ -711,17 +269,14 @@ github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFK github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= -github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE/Tq8= github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf h1:2IYBd5TD/maMqTU2YUzp2tJL4cNaOYQ9EBullN9t9pk= github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -764,25 +319,16 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= -github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= -github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= -github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= -github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= @@ -791,7 +337,6 @@ github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWV github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= @@ -802,8 +347,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= -github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -811,42 +354,14 @@ github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 h1:nAjWYc03awJAjsozNehd github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1/go.mod h1:MLIrzg7gp/kzVBxRE1olT7CWYMCklcUWU+ekoxOD9x0= github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0= github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= -github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= -github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= -github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= -github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= -github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= -github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= -github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/oracle/oci-go-sdk/v65 v65.28.0 h1:FgBWmHzT1y9H9tRQYOiX22Z8ec5p6/M5sSYRGcbWAno= github.com/oracle/oci-go-sdk/v65 v65.28.0/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pborman/getopt v1.1.0/go.mod h1:FxXoW1Re00sQG/+KIkuSqRL/LwQgSkv7uyac+STFsbk= github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU= github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= @@ -864,7 +379,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -878,14 +392,12 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -902,43 +414,15 @@ github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Ny github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= -github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= -github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= -github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= -github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= -github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= -github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= -github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= -github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= -github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= -github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= -github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= -github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= -github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= -github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= -github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= -github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -954,19 +438,12 @@ github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= -github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= -github.com/spacemonkeygo/monkit/v3 v3.0.4/go.mod h1:JcK1pCbReQsOsMKF/POFSZCq7drXFybgGmbc27tuwes= -github.com/spacemonkeygo/monkit/v3 v3.0.18/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4= github.com/spacemonkeygo/monkit/v3 v3.0.19 h1:wqBb9bpD7jXkVi4XwIp8jn1fektaVBQ+cp9SHRXgAdo= github.com/spacemonkeygo/monkit/v3 v3.0.19/go.mod h1:kj1ViJhlyADa7DiA4xVnTuPA46lFKbM7mxQTrXCuJP4= -github.com/spacemonkeygo/monotime v0.0.0-20180824235756-e3f48a95f98a/go.mod h1:ul4bvvnCOPZgq8w0nTkSmWVg/hauVpFS97Am1YM1XXo= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -977,7 +454,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -986,16 +462,12 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf h1:Y43S3e9P1NPs/QF4R5/SdlXj2d31540hP4Gk8VKNvDg= github.com/t3rm1n4l/go-mega v0.0.0-20220725095014-c4e0c2b5debf/go.mod h1:c+cGNU1qi9bO7ZF4IRMYk+KaZTNiQ/gQrSbyMmGFq1Q= -github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= -github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k= github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= @@ -1009,72 +481,51 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yunify/qingstor-sdk-go/v3 v3.2.0 h1:9sB2WZMgjwSUNZhrgvaNGazVltoFUUfuS9f0uCWtTr8= github.com/yunify/qingstor-sdk-go/v3 v3.2.0/go.mod h1:KciFNuMu6F4WLk9nGwwK69sCGKLCdd9f97ac/wfumS4= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zeebo/admission/v3 v3.0.3/go.mod h1:2OWyAS5yo0Xvj2AEUosOjTUHxaY0oIIiCrXGKCYzWpo= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A= github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= -github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs= github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= -github.com/zeebo/float16 v0.1.0/go.mod h1:fssGvvXu+XS8MH57cKmyrLB/cqioYeYX/2mXCN3a5wo= -github.com/zeebo/incenc v0.0.0-20180505221441-0d92902eec54/go.mod h1:EI8LcOBDlSL3POyqwC1eJhOYlMBMidES+613EtmmT5w= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= goftp.io/server v0.4.1 h1:x7KG4HIxSMdK/rpYhExMinRN/aO/T9icvaG/B5e/XfY= goftp.io/server v0.4.1/go.mod h1:hFZeR656ErRt3ojMKt7H10vQ5nuWV1e0YeUTeorlR6k= -golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -1084,7 +535,6 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1095,8 +545,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20221110043201-43a038452099 h1:aIu0lKmfdgtn2uTj7JI2oN4TUrQvgB+wzTPO23bCKt8= @@ -1107,22 +555,13 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1143,73 +582,34 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220524220425-1d687d428aca/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= -golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1221,20 +621,15 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1242,14 +637,10 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1263,61 +654,28 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1327,7 +685,6 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1335,26 +692,18 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1388,7 +737,6 @@ golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -1397,33 +745,12 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1440,46 +767,9 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= -google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= -google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= -google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= -google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= -google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= -google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= -google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= google.golang.org/api v0.106.0 h1:ffmW0faWCwKkpbbtvlY/K/8fUl+JKvNS5CVzRoyfCv8= google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -1488,10 +778,6 @@ google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= -google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1514,96 +800,14 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= -google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= -google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= -google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= -google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= -google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= -google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= -google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= -google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= -google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= -google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1616,32 +820,9 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1654,8 +835,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1665,14 +844,10 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1680,11 +855,8 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1695,8 +867,6 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= storj.io/common v0.0.0-20221123115229-fed3e6651b63 h1:OuleF/3FvZe3Nnu6NdwVr+FvCXjfD4iNNdgfI2kcs3k= storj.io/common v0.0.0-20221123115229-fed3e6651b63/go.mod h1:+gF7jbVvpjVIVHhK+EJFhfPbudX395lnPq/dKkj/Qys= storj.io/drpc v0.0.32 h1:5p5ZwsK/VOgapaCu+oxaPVwO6UwIs+iwdMiD50+R4PI= From 2d1c2725e451245409f0f7a23c404d6242c07b9d Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 15 Jan 2023 16:52:39 +0000 Subject: [PATCH 511/560] webdav: fix tests after go1.20 upgrade Before this change we were sending webdav requests to the go http FileServer. In go1.20 these (rightly) started returning errors which caused the tests to fail. The test has been changed to properly mock up an About query and response so an end to end test of adding headers is possible. --- backend/webdav/webdav_internal_test.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/backend/webdav/webdav_internal_test.go b/backend/webdav/webdav_internal_test.go index 8789e8413fbf1..20dee468bade5 100644 --- a/backend/webdav/webdav_internal_test.go +++ b/backend/webdav/webdav_internal_test.go @@ -24,15 +24,23 @@ var ( // prepareServer the test server and return a function to tidy it up afterwards // with each request the headers option tests are executed func prepareServer(t *testing.T) (configmap.Simple, func()) { - // file server - fileServer := http.FileServer(http.Dir("")) - - // test the headers are there then pass on to fileServer + // test the headers are there send send a dummy response to About handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { what := fmt.Sprintf("%s %s: Header ", r.Method, r.URL.Path) assert.Equal(t, headers[1], r.Header.Get(headers[0]), what+headers[0]) assert.Equal(t, headers[3], r.Header.Get(headers[2]), what+headers[2]) - fileServer.ServeHTTP(w, r) + fmt.Fprintf(w, ` + + /remote.php/webdav/ + + + -3 + 376461895 + + HTTP/1.1 200 OK + + +`) }) // Make the test server @@ -68,7 +76,7 @@ func TestHeaders(t *testing.T) { f, tidy := prepare(t) defer tidy() - // any request will do + // send an About response since that is all the dummy server can return _, err := f.Features().About(context.Background()) require.NoError(t, err) } From ec68b7238737ab0dda2d93f9a31688ecd31be2f6 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 15 Jan 2023 18:19:41 +0000 Subject: [PATCH 512/560] lib/file: fix error message test after go1.20 upgrade --- lib/file/mkdir_windows_test.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/lib/file/mkdir_windows_test.go b/lib/file/mkdir_windows_test.go index 6b574b7921a10..29849d7f7e8fd 100644 --- a/lib/file/mkdir_windows_test.go +++ b/lib/file/mkdir_windows_test.go @@ -77,23 +77,29 @@ func unusedDrive(t *testing.T) string { return string(letter) + ":" } -func checkMkdirAll(t *testing.T, path string, valid bool, errormsg string) { +func checkMkdirAll(t *testing.T, path string, valid bool, errormsgs... string) { if valid { assert.NoError(t, MkdirAll(path, 0777)) } else { err := MkdirAll(path, 0777) assert.Error(t, err) - assert.Equal(t, errormsg, err.Error()) + ok := false + for _, msg := range errormsgs { + if err.Error() == msg { + ok = true + } + } + assert.True(t, ok, err.Error()) } } -func checkMkdirAllSubdirs(t *testing.T, path string, valid bool, errormsg string) { - checkMkdirAll(t, path, valid, errormsg) - checkMkdirAll(t, path+`\`, valid, errormsg) - checkMkdirAll(t, path+`\parent`, valid, errormsg) - checkMkdirAll(t, path+`\parent\`, valid, errormsg) - checkMkdirAll(t, path+`\parent\child`, valid, errormsg) - checkMkdirAll(t, path+`\parent\child\`, valid, errormsg) +func checkMkdirAllSubdirs(t *testing.T, path string, valid bool, errormsgs... string) { + checkMkdirAll(t, path, valid, errormsgs...) + checkMkdirAll(t, path+`\`, valid, errormsgs...) + checkMkdirAll(t, path+`\parent`, valid, errormsgs...) + checkMkdirAll(t, path+`\parent\`, valid, errormsgs...) + checkMkdirAll(t, path+`\parent\child`, valid, errormsgs...) + checkMkdirAll(t, path+`\parent\child\`, valid, errormsgs...) } // Testing paths on existing drive @@ -143,6 +149,9 @@ func TestMkdirAllOnUnusedNetworkHost(t *testing.T) { errormsg := fmt.Sprintf("mkdir %s\\: The format of the specified network name is invalid.", path) checkMkdirAllSubdirs(t, path, false, errormsg) path = `\\?\UNC\0.0.0.0\share` - errormsg = `mkdir \\?\UNC\0.0.0.0: The specified path is invalid.` - checkMkdirAllSubdirs(t, path, false, errormsg) + checkMkdirAllSubdirs(t, path, false, + `mkdir \\?\UNC\0.0.0.0: The specified path is invalid.`, // pre go1.20 + `mkdir \\?\UNC\0.0.0.0\share\: The format of the specified network name is invalid.`, + ) + } From ec20c4852314f867843df9a332c0e8d2f2c2334e Mon Sep 17 00:00:00 2001 From: IMTheNachoMan Date: Mon, 16 Jan 2023 07:40:30 -0500 Subject: [PATCH 513/560] googlephotos: fix grammar in docs (#6699) --- docs/content/googlephotos.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/googlephotos.md b/docs/content/googlephotos.md index c8e5db329c79a..69bb4a1dc5cea 100644 --- a/docs/content/googlephotos.md +++ b/docs/content/googlephotos.md @@ -121,7 +121,7 @@ files in the album. ### Layout -As Google Photos is not a general purpose cloud storage system the +As Google Photos is not a general purpose cloud storage system, the backend is laid out to help you navigate it. The directories under `media` show different ways of categorizing the @@ -471,4 +471,4 @@ Rclone cannot delete files anywhere except under `album`. ### Deleting albums -The Google Photos API does not support deleting albums - see [bug #135714733](https://issuetracker.google.com/issues/135714733). \ No newline at end of file +The Google Photos API does not support deleting albums - see [bug #135714733](https://issuetracker.google.com/issues/135714733). From ca9182d6aeea26923863a48d334021b3b0c1851c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 17 Jan 2023 11:35:12 +0000 Subject: [PATCH 514/560] Add IMTheNachoMan to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 3984ed9a1ac7b..5a2b67e7ba9ba 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -676,3 +676,4 @@ put them back in again.` >}} * Marks Polakovs * piyushgarg * Kaloyan Raev + * IMTheNachoMan From 844e8fb8bdfbe4ba707b7fd3a4ac7a2366dc95a1 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 17 Jan 2023 10:35:02 +0000 Subject: [PATCH 515/560] lib/errors: add support for unwrapping go1.20 multi errors --- lib/errors/errors.go | 12 +- lib/errors/errors_test.go | 250 +++++++++++++++++++++++++++++++------- 2 files changed, 215 insertions(+), 47 deletions(-) diff --git a/lib/errors/errors.go b/lib/errors/errors.go index ed848cbd69c73..a978909ab31c3 100644 --- a/lib/errors/errors.go +++ b/lib/errors/errors.go @@ -17,8 +17,10 @@ type WalkFunc func(error) bool // The next error in the chain is determined by the following rules: // // the return value of this method is used. -// - If the current error has a `Unwrap() error` method (golang.org/x/xerrors), +// - If the current error has a `Unwrap() error` method // the return value of this method is used. +// - If the current error has a `Unwrap() []error` method +// the return values of this method is used. // - Common errors in the Go runtime that contain an Err field will use this value. func Walk(err error, f WalkFunc) { for prev := err; err != nil; prev = err { @@ -27,6 +29,11 @@ func Walk(err error, f WalkFunc) { } switch e := err.(type) { + case multiWrapper: + for _, err = range e.Unwrap() { + Walk(err, f) + } + return case causer: err = e.Cause() case wrapper: @@ -62,3 +69,6 @@ type causer interface { type wrapper interface { Unwrap() error } +type multiWrapper interface { + Unwrap() []error +} diff --git a/lib/errors/errors_test.go b/lib/errors/errors_test.go index c208a5a2e0308..c04b6d6695b02 100644 --- a/lib/errors/errors_test.go +++ b/lib/errors/errors_test.go @@ -1,4 +1,4 @@ -package errors_test +package errors import ( "errors" @@ -6,86 +6,244 @@ import ( "testing" "github.com/stretchr/testify/assert" - - liberrors "github.com/rclone/rclone/lib/errors" ) func TestWalk(t *testing.T) { - origin := errors.New("origin") + var ( + e1 = errors.New("e1") + e2 = errors.New("e2") + e3 = errors.New("e3") + ) for _, test := range []struct { - err error - calls int - last error + err error + want []error }{ - {causerError{nil}, 1, causerError{nil}}, - {wrapperError{nil}, 1, wrapperError{nil}}, - {reflectError{nil}, 1, reflectError{nil}}, - {causerError{origin}, 2, origin}, - {wrapperError{origin}, 2, origin}, - {reflectError{origin}, 2, origin}, - {causerError{reflectError{origin}}, 3, origin}, - {wrapperError{causerError{origin}}, 3, origin}, - {reflectError{wrapperError{origin}}, 3, origin}, - {causerError{reflectError{causerError{origin}}}, 4, origin}, - {wrapperError{causerError{wrapperError{origin}}}, 4, origin}, - {reflectError{wrapperError{reflectError{origin}}}, 4, origin}, - - {stopError{nil}, 1, stopError{nil}}, - {stopError{causerError{nil}}, 1, stopError{causerError{nil}}}, - {stopError{wrapperError{nil}}, 1, stopError{wrapperError{nil}}}, - {stopError{reflectError{nil}}, 1, stopError{reflectError{nil}}}, - {causerError{stopError{origin}}, 2, stopError{origin}}, - {wrapperError{stopError{origin}}, 2, stopError{origin}}, - {reflectError{stopError{origin}}, 2, stopError{origin}}, - {causerError{reflectError{stopError{nil}}}, 3, stopError{nil}}, - {wrapperError{causerError{stopError{nil}}}, 3, stopError{nil}}, - {reflectError{wrapperError{stopError{nil}}}, 3, stopError{nil}}, + { + causerError{nil}, []error{ + causerError{nil}, + }, + }, { + wrapperError{nil}, []error{ + wrapperError{nil}, + }, + }, { + reflectError{nil}, []error{ + reflectError{nil}, + }, + }, { + causerError{e1}, []error{ + causerError{e1}, e1, + }, + }, { + wrapperError{e1}, []error{ + wrapperError{e1}, e1, + }, + }, { + reflectError{e1}, []error{ + reflectError{e1}, e1, + }, + }, { + causerError{reflectError{e1}}, []error{ + causerError{reflectError{e1}}, + reflectError{e1}, + e1, + }, + }, { + wrapperError{causerError{e1}}, []error{ + wrapperError{causerError{e1}}, + causerError{e1}, + e1, + }, + }, { + reflectError{wrapperError{e1}}, []error{ + reflectError{wrapperError{e1}}, + wrapperError{e1}, + e1, + }, + }, { + causerError{reflectError{causerError{e1}}}, []error{ + causerError{reflectError{causerError{e1}}}, + reflectError{causerError{e1}}, + causerError{e1}, + e1, + }, + }, { + wrapperError{causerError{wrapperError{e1}}}, []error{ + wrapperError{causerError{wrapperError{e1}}}, + causerError{wrapperError{e1}}, + wrapperError{e1}, + e1, + }, + }, { + reflectError{wrapperError{reflectError{e1}}}, []error{ + reflectError{wrapperError{reflectError{e1}}}, + wrapperError{reflectError{e1}}, + reflectError{e1}, + e1, + }, + }, { + stopError{nil}, []error{ + stopError{nil}, + }, + }, { + stopError{causerError{nil}}, []error{ + stopError{causerError{nil}}, + }, + }, { + stopError{wrapperError{nil}}, []error{ + stopError{wrapperError{nil}}, + }, + }, { + stopError{reflectError{nil}}, []error{ + stopError{reflectError{nil}}, + }, + }, { + causerError{stopError{e1}}, []error{ + causerError{stopError{e1}}, + stopError{e1}, + }, + }, { + wrapperError{stopError{e1}}, []error{ + wrapperError{stopError{e1}}, + stopError{e1}, + }, + }, { + reflectError{stopError{e1}}, []error{ + reflectError{stopError{e1}}, + stopError{e1}, + }, + }, { + causerError{reflectError{stopError{nil}}}, []error{ + causerError{reflectError{stopError{nil}}}, + reflectError{stopError{nil}}, + stopError{nil}, + }, + }, { + wrapperError{causerError{stopError{nil}}}, []error{ + wrapperError{causerError{stopError{nil}}}, + causerError{stopError{nil}}, + stopError{nil}, + }, + }, { + reflectError{wrapperError{stopError{nil}}}, []error{ + reflectError{wrapperError{stopError{nil}}}, + wrapperError{stopError{nil}}, + stopError{nil}, + }, + }, { + multiWrapperError{[]error{e1}}, []error{ + multiWrapperError{[]error{e1}}, + e1, + }, + }, { + multiWrapperError{[]error{}}, []error{ + multiWrapperError{[]error{}}, + }, + }, { + multiWrapperError{[]error{e1, e2, e3}}, []error{ + multiWrapperError{[]error{e1, e2, e3}}, + e1, + e2, + e3, + }, + }, { + multiWrapperError{[]error{reflectError{e1}, wrapperError{e2}, stopError{e3}}}, []error{ + multiWrapperError{[]error{reflectError{e1}, wrapperError{e2}, stopError{e3}}}, + reflectError{e1}, + e1, + wrapperError{e2}, + e2, + stopError{e3}, + }, + }, } { - var last error - calls := 0 - liberrors.Walk(test.err, func(err error) bool { - calls++ - last = err + var got []error + Walk(test.err, func(err error) bool { + got = append(got, err) _, stop := err.(stopError) return stop }) - assert.Equal(t, test.calls, calls) - assert.Equal(t, test.last, last) + assert.Equal(t, test.want, got, test.err) } } type causerError struct { err error } -type wrapperError struct { - err error -} -type reflectError struct { - Err error -} -type stopError struct { - err error -} func (e causerError) Error() string { return fmt.Sprintf("causerError(%s)", e.err) } + func (e causerError) Cause() error { return e.err } + +var ( + _ error = causerError{nil} + _ causer = causerError{nil} +) + +type wrapperError struct { + err error +} + func (e wrapperError) Unwrap() error { return e.err } + func (e wrapperError) Error() string { return fmt.Sprintf("wrapperError(%s)", e.err) } + +var ( + _ error = wrapperError{nil} + _ wrapper = wrapperError{nil} +) + +type multiWrapperError struct { + errs []error +} + +func (e multiWrapperError) Unwrap() []error { + return e.errs +} + +func (e multiWrapperError) Error() string { + return fmt.Sprintf("multiWrapperError(%s)", e.errs) +} + +var ( + _ error = multiWrapperError{nil} + _ multiWrapper = multiWrapperError{nil} +) + +type reflectError struct { + Err error +} + func (e reflectError) Error() string { return fmt.Sprintf("reflectError(%s)", e.Err) } + +var ( + _ error = reflectError{nil} +) + +type stopError struct { + err error +} + func (e stopError) Error() string { return fmt.Sprintf("stopError(%s)", e.err) } + func (e stopError) Cause() error { return e.err } + +var ( + _ error = stopError{nil} + _ causer = stopError{nil} +) From 6b17044f8e7bd8f3936c9e529c4de34db2cfc959 Mon Sep 17 00:00:00 2001 From: alankrit Date: Wed, 11 Jan 2023 07:59:51 +0000 Subject: [PATCH 516/560] fs:Added multiple ca certificate support. --- docs/content/docs.md | 4 ++-- fs/config.go | 6 +++--- fs/config/configflags/configflags.go | 2 +- fs/fshttp/http.go | 22 +++++++++++++--------- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index 535ed2486bfc1..c899b44c1f806 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -2099,9 +2099,9 @@ these options. For example this can be very useful with the HTTP or WebDAV backends. Rclone HTTP servers have their own set of configuration for SSL/TLS which you can find in their documentation. -### --ca-cert string +### --ca-cert stringArray -This loads the PEM encoded certificate authority certificate and uses +This loads the PEM encoded certificate authority certificates and uses it to verify the certificates of the servers rclone connects to. If you have generated certificates signed with a local CA then you diff --git a/fs/config.go b/fs/config.go index d496b4d88f8d6..8fc8e45d96056 100644 --- a/fs/config.go +++ b/fs/config.go @@ -120,9 +120,9 @@ type ConfigInfo struct { ProgressTerminalTitle bool Cookie bool UseMmap bool - CaCert string // Client Side CA - ClientCert string // Client Side Cert - ClientKey string // Client Side Key + CaCert []string // Client Side CA + ClientCert string // Client Side Cert + ClientKey string // Client Side Key MultiThreadCutoff SizeSuffix MultiThreadStreams int MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags) diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index abd5209d79c17..2ecf9e8c79d25 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -120,7 +120,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &ci.ProgressTerminalTitle, "progress-terminal-title", "", ci.ProgressTerminalTitle, "Show progress on the terminal title (requires -P/--progress)") flags.BoolVarP(flagSet, &ci.Cookie, "use-cookies", "", ci.Cookie, "Enable session cookiejar") flags.BoolVarP(flagSet, &ci.UseMmap, "use-mmap", "", ci.UseMmap, "Use mmap allocator (see docs)") - flags.StringVarP(flagSet, &ci.CaCert, "ca-cert", "", ci.CaCert, "CA certificate used to verify servers") + flags.StringArrayVarP(flagSet, &ci.CaCert, "ca-cert", "", ci.CaCert, "CA certificate used to verify servers") flags.StringVarP(flagSet, &ci.ClientCert, "client-cert", "", ci.ClientCert, "Client SSL certificate (PEM) for mutual TLS auth") flags.StringVarP(flagSet, &ci.ClientKey, "client-key", "", ci.ClientKey, "Client SSL private key (PEM) for mutual TLS auth") flags.FVarP(flagSet, &ci.MultiThreadCutoff, "multi-thread-cutoff", "", "Use multi-thread downloads for files above this size") diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go index b37fd460c2d6d..f17c8298ec6a6 100644 --- a/fs/fshttp/http.go +++ b/fs/fshttp/http.go @@ -72,16 +72,20 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) ht t.TLSClientConfig.Certificates = []tls.Certificate{cert} } - // Load CA cert - if ci.CaCert != "" { - caCert, err := os.ReadFile(ci.CaCert) - if err != nil { - log.Fatalf("Failed to read --ca-cert: %v", err) - } + // Load CA certs + if len(ci.CaCert) != 0 { + caCertPool := x509.NewCertPool() - ok := caCertPool.AppendCertsFromPEM(caCert) - if !ok { - log.Fatalf("Failed to add certificates from --ca-cert") + + for _, cert := range ci.CaCert { + caCert, err := os.ReadFile(cert) + if err != nil { + log.Fatalf("Failed to read --ca-cert file %q : %v", cert, err) + } + ok := caCertPool.AppendCertsFromPEM(caCert) + if !ok { + log.Fatalf("Failed to add certificates from --ca-cert file %q", cert) + } } t.TLSClientConfig.RootCAs = caCertPool } From 0272d4419283d95e392cbb91b71ec9a232640c65 Mon Sep 17 00:00:00 2001 From: albertony <12441419+albertony@users.noreply.github.com> Date: Sat, 11 Jun 2022 17:27:37 +0200 Subject: [PATCH 517/560] mount: do not treat \\?\ prefixed paths as network share paths on windows See: #6234 --- cmd/cmount/mountpoint_windows.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/cmount/mountpoint_windows.go b/cmd/cmount/mountpoint_windows.go index a1e511409b537..49f92c9c22473 100644 --- a/cmd/cmount/mountpoint_windows.go +++ b/cmd/cmount/mountpoint_windows.go @@ -18,13 +18,13 @@ import ( var isDriveRegex = regexp.MustCompile(`^[a-zA-Z]\:$`) var isDriveRootPathRegex = regexp.MustCompile(`^[a-zA-Z]\:\\$`) var isDriveOrRootPathRegex = regexp.MustCompile(`^[a-zA-Z]\:\\?$`) -var isNetworkSharePathRegex = regexp.MustCompile(`^\\\\[^\\]+\\[^\\]`) +var isNetworkSharePathRegex = regexp.MustCompile(`^\\\\[^\\\?]+\\[^\\]`) // isNetworkSharePath returns true if the given string is a valid network share path, // in the basic UNC format "\\Server\Share\Path", where the first two path components // are required ("\\Server\Share", which represents the volume). // Extended-length UNC format "\\?\UNC\Server\Share\Path" is not considered, as it is -// not supported by cgofuse/winfsp. +// not supported by cgofuse/winfsp, so returns false for any paths with prefix "\\?\". // Note: There is a UNCPath function in lib/file, but it refers to any extended-length // paths using prefix "\\?\", and not necessarily network resource UNC paths. func isNetworkSharePath(l string) bool { From 37db2abecdda3cd6edbda68934d8f7673e581542 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 20 Jan 2023 15:39:49 +0000 Subject: [PATCH 518/560] Add alankrit to contributors --- docs/content/authors.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/authors.md b/docs/content/authors.md index 5a2b67e7ba9ba..2ff0c2bcc1afc 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -677,3 +677,4 @@ put them back in again.` >}} * piyushgarg * Kaloyan Raev * IMTheNachoMan + * alankrit From 267a09001d091de33de5b7f8444bb071089e0c47 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 19 Jan 2023 15:54:10 +0000 Subject: [PATCH 519/560] mount: fix check for empty mount point on Linux #3562 --- cmd/mountlib/check_linux.go | 14 +++++++++++--- cmd/mountlib/check_other.go | 22 +--------------------- cmd/mountlib/utils.go | 22 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/cmd/mountlib/check_linux.go b/cmd/mountlib/check_linux.go index 1f241af8723bc..1c32a6d8ad426 100644 --- a/cmd/mountlib/check_linux.go +++ b/cmd/mountlib/check_linux.go @@ -33,12 +33,20 @@ func CheckMountEmpty(mountpoint string) error { if err != nil { return fmt.Errorf("cannot read %s: %w", mtabPath, err) } + foundAutofs := false for _, entry := range entries { - if entry.Dir == mountpointAbs && entry.Type != "autofs" { - return fmt.Errorf(msg, mountpointAbs) + if entry.Dir == mountpointAbs { + if entry.Type != "autofs" { + return fmt.Errorf(msg, mountpointAbs) + } + foundAutofs = true } } - return nil + // It isn't safe to list an autofs in the middle of mounting + if foundAutofs { + return nil + } + return checkMountEmpty(mountpoint) } // CheckMountReady checks whether mountpoint is mounted by rclone. diff --git a/cmd/mountlib/check_other.go b/cmd/mountlib/check_other.go index c8876632cb3d2..328f83bca580d 100644 --- a/cmd/mountlib/check_other.go +++ b/cmd/mountlib/check_other.go @@ -4,33 +4,13 @@ package mountlib import ( - "fmt" - "io" - "os" "time" - - "github.com/rclone/rclone/fs" ) // CheckMountEmpty checks if mountpoint folder is empty. // On non-Linux unixes we list directory to ensure that. func CheckMountEmpty(mountpoint string) error { - fp, err := os.Open(mountpoint) - if err != nil { - return fmt.Errorf("cannot open: %s: %w", mountpoint, err) - } - defer fs.CheckClose(fp, &err) - - _, err = fp.Readdirnames(1) - if err == io.EOF { - return nil - } - - const msg = "directory is not empty, use --allow-non-empty to mount anyway: %s" - if err == nil { - return fmt.Errorf(msg, mountpoint) - } - return fmt.Errorf(msg+": %w", mountpoint, err) + return checkMountEmpty(mountpoint) } // CheckMountReady should check if mountpoint is mounted by rclone. diff --git a/cmd/mountlib/utils.go b/cmd/mountlib/utils.go index 099e3a7677fdc..5148f818b83ff 100644 --- a/cmd/mountlib/utils.go +++ b/cmd/mountlib/utils.go @@ -2,6 +2,8 @@ package mountlib import ( "fmt" + "io" + "os" "path/filepath" "runtime" "strings" @@ -84,6 +86,26 @@ func (m *MountPoint) CheckAllowed() error { return nil } +// checkMountEmpty checks if mountpoint folder is empty by listing it. +func checkMountEmpty(mountpoint string) error { + fp, err := os.Open(mountpoint) + if err != nil { + return fmt.Errorf("cannot open: %s: %w", mountpoint, err) + } + defer fs.CheckClose(fp, &err) + + _, err = fp.Readdirnames(1) + if err == io.EOF { + return nil + } + + const msg = "%q is not empty, use --allow-non-empty to mount anyway" + if err == nil { + return fmt.Errorf(msg, mountpoint) + } + return fmt.Errorf(msg+": %w", mountpoint, err) +} + // SetVolumeName with sensible default func (m *MountPoint) SetVolumeName(vol string) { if vol == "" { From a6f6a9dcdf287eaa490c8b535ee70e85f961dbf5 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 19 Jan 2023 15:56:25 +0000 Subject: [PATCH 520/560] mount,mount2,cmount: fix --allow-non-empty #3562 Since version 3 of fuse libfuse no longer does anything when given the nonempty option and it's default is to allow mounting over non empty directories like normal mount does. Some versions of libfuse give an error when using `--allow-non-empty` which is annoying for the user. We now do this check ourselves so we no longer need to pass the option to libfuse. Fixes #3562 --- cmd/cmount/mount.go | 3 --- cmd/mount/mount.go | 3 --- cmd/mount2/mount.go | 3 --- 3 files changed, 9 deletions(-) diff --git a/cmd/cmount/mount.go b/cmd/cmount/mount.go index c9893f2f7639f..919f5d895d54f 100644 --- a/cmd/cmount/mount.go +++ b/cmd/cmount/mount.go @@ -84,9 +84,6 @@ func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib. if opt.DaemonTimeout != 0 { options = append(options, "-o", fmt.Sprintf("daemon_timeout=%d", int(opt.DaemonTimeout.Seconds()))) } - if opt.AllowNonEmpty { - options = append(options, "-o", "nonempty") - } if opt.AllowOther { options = append(options, "-o", "allow_other") } diff --git a/cmd/mount/mount.go b/cmd/mount/mount.go index 765419df4f30b..923681c7c556b 100644 --- a/cmd/mount/mount.go +++ b/cmd/mount/mount.go @@ -34,9 +34,6 @@ func mountOptions(VFS *vfs.VFS, device string, opt *mountlib.Options) (options [ if opt.AsyncRead { options = append(options, fuse.AsyncRead()) } - if opt.AllowNonEmpty { - options = append(options, fuse.AllowNonEmptyMount()) - } if opt.AllowOther { options = append(options, fuse.AllowOther()) } diff --git a/cmd/mount2/mount.go b/cmd/mount2/mount.go index cbb8905009f92..48e2f820eeaad 100644 --- a/cmd/mount2/mount.go +++ b/cmd/mount2/mount.go @@ -95,9 +95,6 @@ func mountOptions(fsys *FS, f fs.Fs, opt *mountlib.Options) (mountOpts *fuse.Mou } var opts []string // FIXME doesn't work opts = append(opts, fmt.Sprintf("max_readahead=%d", maxReadAhead)) - if fsys.opt.AllowNonEmpty { - opts = append(opts, "nonempty") - } if fsys.opt.AllowOther { opts = append(opts, "allow_other") } From 7fa21949a73603ea53a4e457dcb15440781c6e06 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sat, 21 Jan 2023 22:45:06 +0100 Subject: [PATCH 521/560] realdebrid remote --- .github/workflows/build.yml | 85 +- .../workflows/build_publish_docker_image.yml | 9 +- .../build_publish_release_docker_image.yml | 21 +- Makefile | 9 +- backend/all/all.go | 1 + backend/realdebrid/api/types.go | 97 ++ backend/realdebrid/realdebrid.go | 1526 +++++++++++++++++ 7 files changed, 1680 insertions(+), 68 deletions(-) create mode 100644 backend/realdebrid/api/types.go create mode 100644 backend/realdebrid/realdebrid.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c56e8d38ff0a..784a236ccf9c8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,12 +25,12 @@ jobs: strategy: fail-fast: false matrix: - job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.18', 'go1.19'] + job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.16', 'go1.17'] include: - job_name: linux os: ubuntu-latest - go: '1.20.0-rc.3' + go: '1.18.x' gotags: cmount build_flags: '-include "^linux/"' check: true @@ -39,16 +39,9 @@ jobs: librclonetest: true deploy: true - - job_name: linux_386 - os: ubuntu-latest - go: '1.20.0-rc.3' - goarch: 386 - gotags: cmount - quicktest: true - - job_name: mac_amd64 os: macos-11 - go: '1.20.0-rc.3' + go: '1.18.x' gotags: 'cmount' build_flags: '-include "^darwin/amd64" -cgo' quicktest: true @@ -57,14 +50,14 @@ jobs: - job_name: mac_arm64 os: macos-11 - go: '1.20.0-rc.3' + go: '1.18.x' gotags: 'cmount' build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib' deploy: true - job_name: windows os: windows-latest - go: '1.20.0-rc.3' + go: '1.18.x' gotags: cmount cgo: '0' build_flags: '-include "^windows/"' @@ -74,20 +67,20 @@ jobs: - job_name: other_os os: ubuntu-latest - go: '1.20.0-rc.3' + go: '1.18.x' build_flags: '-exclude "^(windows/|darwin/|linux/)"' compile_all: true deploy: true - - job_name: go1.18 + - job_name: go1.16 os: ubuntu-latest - go: '1.18' + go: '1.16.x' quicktest: true racequicktest: true - - job_name: go1.19 + - job_name: go1.17 os: ubuntu-latest - go: '1.19' + go: '1.17.x' quicktest: true racequicktest: true @@ -97,13 +90,14 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 0 - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v2 with: + stable: 'false' go-version: ${{ matrix.go }} check-latest: true @@ -122,7 +116,7 @@ jobs: sudo modprobe fuse sudo chmod 666 /dev/fuse sudo chown root:$USER /etc/fuse.conf - sudo apt-get install fuse3 libfuse-dev rpm pkg-config + sudo apt-get install fuse libfuse-dev rpm pkg-config if: matrix.os == 'ubuntu-latest' - name: Install Libraries on macOS @@ -161,7 +155,7 @@ jobs: env - name: Go module cache - uses: actions/cache@v3 + uses: actions/cache@v2 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} @@ -225,27 +219,14 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v2 - name: Code quality test - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v2 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version version: latest - # Run govulncheck on the latest go version, the one we build binaries with - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: 1.20.0-rc.3 - check-latest: true - - - name: Install govulncheck - run: go install golang.org/x/vuln/cmd/govulncheck@latest - - - name: Scan for vulnerabilities - run: govulncheck ./... - android: if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }} timeout-minutes: 30 @@ -254,18 +235,22 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 0 # Upgrade together with NDK version - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v1 with: - go-version: 1.20.0-rc.3 + go-version: 1.18.x + + # Upgrade together with Go version. Using a GitHub-provided version saves around 2 minutes. + - name: Force NDK version + run: echo "y" | sudo ${ANDROID_HOME}/tools/bin/sdkmanager --install "ndk;23.1.7779620" | grep -v = || true - name: Go module cache - uses: actions/cache@v3 + uses: actions/cache@v2 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} @@ -286,29 +271,27 @@ jobs: go install golang.org/x/mobile/cmd/gobind@latest go install golang.org/x/mobile/cmd/gomobile@latest env PATH=$PATH:~/go/bin gomobile init - echo "RCLONE_NDK_VERSION=21" >> $GITHUB_ENV - name: arm-v7a gomobile build - run: env PATH=$PATH:~/go/bin gomobile bind -androidapi ${RCLONE_NDK_VERSION} -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile + run: env PATH=$PATH:~/go/bin gomobile bind -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile - name: arm-v7a Set environment variables shell: bash run: | - echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV + echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi16-clang)" >> $GITHUB_ENV echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV echo 'GOOS=android' >> $GITHUB_ENV echo 'GOARCH=arm' >> $GITHUB_ENV echo 'GOARM=7' >> $GITHUB_ENV echo 'CGO_ENABLED=1' >> $GITHUB_ENV echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV - - name: arm-v7a build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv7a . + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-armv7a . - name: arm64-v8a Set environment variables shell: bash run: | - echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV + echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang)" >> $GITHUB_ENV echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV echo 'GOOS=android' >> $GITHUB_ENV echo 'GOARCH=arm64' >> $GITHUB_ENV @@ -316,12 +299,12 @@ jobs: echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV - name: arm64-v8a build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv8a . + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-armv8a . - name: x86 Set environment variables shell: bash run: | - echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV + echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android16-clang)" >> $GITHUB_ENV echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV echo 'GOOS=android' >> $GITHUB_ENV echo 'GOARCH=386' >> $GITHUB_ENV @@ -329,12 +312,12 @@ jobs: echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV - name: x86 build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x86 . + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-16-x86 . - name: x64 Set environment variables shell: bash run: | - echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV + echo "CC=$(echo $ANDROID_HOME/ndk/23.1.7779620/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android21-clang)" >> $GITHUB_ENV echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV echo 'GOOS=android' >> $GITHUB_ENV echo 'GOARCH=amd64' >> $GITHUB_ENV @@ -342,7 +325,7 @@ jobs: echo 'CGO_LDFLAGS=-fuse-ld=lld -s -w' >> $GITHUB_ENV - name: x64 build - run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x64 . + run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-21-x64 . - name: Upload artifacts run: | diff --git a/.github/workflows/build_publish_docker_image.yml b/.github/workflows/build_publish_docker_image.yml index af52509732534..e1da28a53ce9e 100644 --- a/.github/workflows/build_publish_docker_image.yml +++ b/.github/workflows/build_publish_docker_image.yml @@ -4,22 +4,25 @@ on: push: branches: - master + paths-ignore: + - '**/README.md' + - '**/.github/**' jobs: build: - if: github.repository == 'rclone/rclone' + if: github.repository == 'rclone/rclone' || 1 == 1 runs-on: ubuntu-latest name: Build image job steps: - name: Checkout master - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 0 - name: Build and publish image uses: ilteoood/docker_buildx@1.1.0 with: tag: beta - imageName: rclone/rclone + imageName: rclone_rd platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 publish: true dockerHubUser: ${{ secrets.DOCKER_HUB_USER }} diff --git a/.github/workflows/build_publish_release_docker_image.yml b/.github/workflows/build_publish_release_docker_image.yml index 6fe9213c78761..9d89545dcd710 100644 --- a/.github/workflows/build_publish_release_docker_image.yml +++ b/.github/workflows/build_publish_release_docker_image.yml @@ -3,15 +3,20 @@ name: Docker release build on: release: types: [published] + workflow_dispatch: + inputs: + manual: + required: true + default: true jobs: build: - if: github.repository == 'rclone/rclone' + if: github.repository == 'rclone/rclone' || 1 == 1 runs-on: ubuntu-latest name: Build image job steps: - name: Checkout master - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 0 - name: Get actual patch version @@ -27,33 +32,33 @@ jobs: uses: ilteoood/docker_buildx@1.1.0 with: tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }} - imageName: rclone/rclone + imageName: itstoggle/rclone_rd platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 publish: true dockerHubUser: ${{ secrets.DOCKER_HUB_USER }} dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }} build_docker_volume_plugin: - if: github.repository == 'rclone/rclone' + if: github.repository == 'rclone/rclone' || 1 == 1 needs: build runs-on: ubuntu-latest name: Build docker plugin job steps: - name: Checkout master - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 0 - name: Build and publish docker plugin shell: bash run: | VER=${GITHUB_REF#refs/tags/} - PLUGIN_USER=rclone + PLUGIN_USER=itstoggle docker login --username ${{ secrets.DOCKER_HUB_USER }} \ --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}" for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do export PLUGIN_USER PLUGIN_ARCH make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-} - make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v} + make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-master done make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest - make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v} + make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=master diff --git a/Makefile b/Makefile index 7a9f4a660914a..a5ad7f1425495 100644 --- a/Makefile +++ b/Makefile @@ -81,9 +81,6 @@ quicktest: racequicktest: RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./... -compiletest: - RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -run XXX ./... - # Do source code quality checks check: rclone @echo "-- START CODE QUALITY REPORT -------------------------------" @@ -265,12 +262,12 @@ winzip: zip -9 rclone-$(TAG).zip rclone.exe # docker volume plugin -PLUGIN_USER ?= rclone +PLUGIN_USER ?= itstoggle PLUGIN_TAG ?= latest PLUGIN_BASE_TAG ?= latest PLUGIN_ARCH ?= amd64 -PLUGIN_IMAGE := $(PLUGIN_USER)/docker-volume-rclone:$(PLUGIN_TAG) -PLUGIN_BASE := $(PLUGIN_USER)/rclone:$(PLUGIN_BASE_TAG) +PLUGIN_IMAGE := $(PLUGIN_USER)/docker-volume-rclone_rd:$(PLUGIN_TAG) +PLUGIN_BASE := $(PLUGIN_USER)/rclone_rd:$(PLUGIN_BASE_TAG) PLUGIN_BUILD_DIR := ./build/docker-plugin PLUGIN_CONTRIB_DIR := ./contrib/docker-plugin/managed diff --git a/backend/all/all.go b/backend/all/all.go index 08786222a9732..007eed5dcff8b 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -39,6 +39,7 @@ import ( _ "github.com/rclone/rclone/backend/premiumizeme" _ "github.com/rclone/rclone/backend/putio" _ "github.com/rclone/rclone/backend/qingstor" + _ "github.com/rclone/rclone/backend/realdebrid" _ "github.com/rclone/rclone/backend/s3" _ "github.com/rclone/rclone/backend/seafile" _ "github.com/rclone/rclone/backend/sftp" diff --git a/backend/realdebrid/api/types.go b/backend/realdebrid/api/types.go new file mode 100644 index 0000000000000..4e31ec2d3cfc2 --- /dev/null +++ b/backend/realdebrid/api/types.go @@ -0,0 +1,97 @@ +// Package api contains definitions for using the premiumize.me API +package api + +import "fmt" + +// Response is returned by all messages and embedded in the +// structures below +type Response struct { + Message string `json:"message,omitempty"` + Status string `json:"status"` +} + +// Error satisfies the error interface +func (e *Response) Error() string { + return fmt.Sprintf("%s: %s", e.Status, e.Message) +} + +// AsErr checks the status and returns an err if bad or nil if good +func (e *Response) AsErr() error { + if e.Status != "success" { + return e + } + return nil +} + +// Item Types +const ( + ItemTypeFolder = "folder" + ItemTypeFile = "file" +) + +// Item refers to a file or folder +type Item struct { + Breadcrumbs []Breadcrumb `` + CreatedAt int64 `` + ID string `json:"id,omitempty"` + ParentID string `` + Link string `json:"download,omitempty"` + OriginalLink string `json:"link,omitempty"` + Name string `json:"filename,omitempty"` + Size int64 `json:"filesize,omitempty"` + Status string `json:"status,omitempty"` + StreamLink string `` + Type string `json:"type,omitempty"` + TranscodeStatus string `` + IP string `` + MimeType string `json:"mimeType,omitempty"` + Ended string `json:"added,omitempty"` + Generated string `json:"generated,omitempty"` + Links []string `json:"links,omitempty"` + Files []File `json:"files,omitempty"` + TorrentHash string `json:"hash,omitempty"` +} + +type File struct { + ID int64 `json:"id,omitempty"` + Selected int64 `json:"selected,omitempty"` +} + +// Breadcrumb is part the breadcrumb trail for a file or folder. It +// is returned as part of folder/list if required +type Breadcrumb struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + ParentID string `json:"parent_id,omitempty"` +} + +// FolderListResponse is the response to folder/list +type FolderListResponse struct { + Response + Content []Item `` //`json:"content"` + Name string `json:"filename,omitempty"` //`json:"name,omitempty"` + ParentID string `` //`json:"parent_id,omitempty"` + FolderID string `` //`json:"folder_id,omitempty"` +} + +// FolderCreateResponse is the response to folder/create +type FolderCreateResponse struct { + Response + ID string `json:"id,omitempty"` +} + +// FolderUploadinfoResponse is the response to folder/uploadinfo +type FolderUploadinfoResponse struct { + Response + Token string `json:"token,omitempty"` + URL string `json:"url,omitempty"` +} + +// AccountInfoResponse is the response to account/info +type AccountInfoResponse struct { + Response + CustomerID string `json:"customer_id,omitempty"` + LimitUsed float64 `json:"limit_used,omitempty"` // fraction 0..1 of download traffic limit + PremiumUntil int64 `json:"premium_until,omitempty"` + SpaceUsed float64 `json:"space_used,omitempty"` +} diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go new file mode 100644 index 0000000000000..3648adf42a37c --- /dev/null +++ b/backend/realdebrid/realdebrid.go @@ -0,0 +1,1526 @@ +// Package realdebrid provides an interface to the real-debrid.com +// object storage system. +package realdebrid + +/* +Run of rclone info +stringNeedsEscaping = []rune{ + 0x00, 0x0A, 0x0D, 0x22, 0x2F, 0x5C, 0xBF, 0xFE + 0x00, 0x0A, 0x0D, '"', '/', '\\', 0xBF, 0xFE +} +maxFileLength = 255 +canWriteUnnormalized = true +canReadUnnormalized = true +canReadRenormalized = false +canStream = true +*/ + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "regexp" + "strconv" + "strings" + "time" + + "github.com/rclone/rclone/backend/realdebrid/api" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/fserrors" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/dircache" + "github.com/rclone/rclone/lib/encoder" + "github.com/rclone/rclone/lib/oauthutil" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/rest" + "golang.org/x/oauth2" +) + +const ( + rcloneClientID = "X245A4XAIBGVM" + rcloneEncryptedClientSecret = "B5YIvQoRIhcpAYs8HYeyjb9gK-ftmZEbqdh_gNfc4RgO9Q" + minSleep = 10 * time.Millisecond + maxSleep = 2 * time.Second + decayConstant = 2 // bigger for slower decay, exponential + rootID = "0" // ID of root folder is always this + rootURL = "https://api.real-debrid.com/rest/1.0" +) + +// Globals +var ( + // Description of how to auth for this app + oauthConfig = &oauth2.Config{ + Scopes: nil, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://api.real-debrid.com/oauth/v2/auth", + TokenURL: "https://api.real-debrid.com/oauth/v2/token", + }, + ClientID: rcloneClientID, + ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), + RedirectURL: oauthutil.RedirectURL, + } +) + +//Global lists of recieved content. +//Realdebrid content is provided in pages with 100 items per page. +//To limit api calls all pages are stored here and are only updated on changes in the total length +var cached []api.Item +var torrents []api.Item +var broken_torrents []string +var lastcheck int64 = time.Now().Unix() +var lastFileMod int64 = 0 +var interval int64 = 15 * 60 +var structure = make(map[string]string) +var levels = make(map[string][]string) +var folders = make(map[string][]api.Item) +var regex_folders = make(map[string]string) +var id2name = make(map[string]string) +var move_chars = " -> " +var regx_chars = " == " + +// Register with Fs +func init() { + fs.Register(&fs.RegInfo{ + Name: "realdebrid", + Description: "real-debrid.com", + NewFs: NewFs, + Options: []fs.Option{{ + Name: "api_key", + Help: `please provide your RealDebrid API key.`, + Default: "", + }, { + Name: "sort_file", + Help: `please provide the full path to a file that should be used for sorting`, + Advanced: true, + Default: "./sorting.txt", + }, { + Name: config.ConfigEncoding, + Help: config.ConfigEncodingHelp, + Advanced: true, + // Encode invalid UTF-8 bytes as json doesn't handle them properly. + Default: (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeDoubleQuote | + encoder.EncodeInvalidUtf8), + }}, + }) +} + +// Options defines the configuration for this backend +type Options struct { + SortFile string `config:"sort_file"` + APIKey string `config:"api_key"` + Enc encoder.MultiEncoder `config:"encoding"` +} + +// Fs represents a remote cloud storage system +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + features *fs.Features // optional features + srv *rest.Client // the connection to the server + dirCache *dircache.DirCache // Map of directory path to directory id + pacer *fs.Pacer // pacer for API calls + tokenRenewer *oauthutil.Renew // renew the token on expiry +} + +// Object describes a file +type Object struct { + fs *Fs // what this object is part of + remote string // The remote path + hasMetaData bool // metadata is present and correct + size int64 // size of the object + modTime time.Time // modification time of the object + id string // ID of the object + ParentID string // ID of parent directory + mimeType string // Mime type of object + url string // URL to download file + TorrentHash string // Torrent Hash +} + +// ------------------------------------------------------------ + +// Name of the remote (as passed into NewFs) +func (f *Fs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *Fs) Root() string { + return f.root +} + +// String converts this Fs to a string +func (f *Fs) String() string { + return fmt.Sprintf("realdebrid root '%s'", f.root) +} + +// Features returns the optional features of this Fs +func (f *Fs) Features() *fs.Features { + return f.features +} + +// parsePath parses a realdebrid 'url' +func parsePath(path string) (root string) { + root = strings.Trim(path, "/") + return +} + +// retryErrorCodes is a slice of error codes that we will retry +var retryErrorCodes = []int{ + 429, // Too Many Requests. + 500, // Internal Server Error + 502, // Bad Gateway + 504, // Gateway Timeout + 509, // Bandwidth Limit Exceeded +} + +// shouldRetry returns a boolean as to whether this resp and err +// deserve to be retried. It returns the err as a convenience +func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { + if fserrors.ContextError(ctx, &err) { + return false, err + } + return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err +} + +// readMetaDataForPath reads the metadata from the path +func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOnly bool, filesOnly bool) (info *api.Item, err error) { + // defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err) + leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false) + if err != nil { + if err == fs.ErrorDirNotFound { + return nil, fs.ErrorObjectNotFound + } + return nil, err + } + if len(directoryID) > 0 && directoryID != "0" { + if last := directoryID[len(directoryID)-1]; last != '/' { + directoryID = directoryID + "/" + } + } + lcLeaf := strings.ToLower(leaf) + _, found, err := f.listAll(ctx, directoryID, directoriesOnly, filesOnly, func(item *api.Item) bool { + if strings.ToLower(item.Name) == lcLeaf { + info = item + return true + } + return false + }) + if err != nil { + return nil, err + } + if !found { + return nil, fs.ErrorObjectNotFound + } + return info, nil +} + +// errorHandler parses a non 2xx error response into an error +func errorHandler(resp *http.Response) error { + body, err := rest.ReadBody(resp) + if err != nil { + body = nil + } + var e = api.Response{ + Message: string(body), + Status: fmt.Sprintf("%s (%d)", resp.Status, resp.StatusCode), + } + if body != nil { + _ = json.Unmarshal(body, &e) + } + return &e +} + +// Return a url.Values with the api key in +func (f *Fs) baseParams() url.Values { + params := url.Values{} + if f.opt.APIKey != "" { + params.Add("auth_token", f.opt.APIKey) + } + return params +} + +// NewFs constructs an Fs from the path, container:path +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse config into Options struct + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + root = parsePath(root) + + var client *http.Client + var ts *oauthutil.TokenSource + if opt.APIKey == "" { + client, ts, err = oauthutil.NewClient(ctx, name, m, oauthConfig) + if err != nil { + return nil, fmt.Errorf("failed to configure realdebrid: %w", err) + } + } else { + client = fshttp.NewClient(ctx) + } + + f := &Fs{ + name: name, + root: root, + opt: *opt, + srv: rest.NewClient(client).SetRoot(rootURL), + pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), + } + f.features = (&fs.Features{ + CaseInsensitive: true, + CanHaveEmptyDirectories: true, + ReadMimeType: true, + }).Fill(ctx, f) + f.srv.SetErrorHandler(errorHandler) + + // Renew the token in the background + if ts != nil { + f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { + _, err := f.About(ctx) + return err + }) + } + + // Get rootID + f.dirCache = dircache.New(root, rootID, f) + + // Find the current root + err = f.dirCache.FindRoot(ctx, false) + if err != nil { + // Assume it is a file + newRoot, remote := dircache.SplitPath(root) + tempF := *f + tempF.dirCache = dircache.New(newRoot, rootID, &tempF) + tempF.root = newRoot + // Make new Fs which is the parent + err = tempF.dirCache.FindRoot(ctx, false) + if err != nil { + // No root so return old f + return f, nil + } + _, err := tempF.newObjectWithInfo(ctx, remote, nil) + if err != nil { + if err == fs.ErrorObjectNotFound { + // File doesn't exist so return old f + return f, nil + } + return nil, err + } + f.features.Fill(ctx, &tempF) + // XXX: update the old f here instead of returning tempF, since + // `features` were already filled with functions having *f as a receiver. + // See https://github.com/rclone/rclone/issues/2182 + f.dirCache = tempF.dirCache + f.root = tempF.root + // return an error with an fs which points to the parent + return f, fs.ErrorIsFile + } + return f, nil +} + +// Return an Object from a path +// +// If it can't be found it returns the error fs.ErrorObjectNotFound. +func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) { + o := &Object{ + fs: f, + remote: remote, + } + var err error + if info != nil { + // Set info + err = o.setMetaData(info) + } else { + err = o.readMetaData(ctx) // reads info and meta, returning an error + } + if err != nil { + return nil, err + } + return o, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound. +func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + return f.newObjectWithInfo(ctx, remote, nil) +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { + // Find the leaf in pathID + var newDirID string + newDirID, found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool { + if strings.EqualFold(item.Name, leaf) { + pathIDOut = item.ID + return true + } + return false + }) + // Update the Root directory ID to its actual value + if pathID == rootID { + f.dirCache.SetRootIDAlias(newDirID) + } + return pathIDOut, found, err +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, err error) { + // Open the file + if len(dirID) > 0 { + if first := dirID[0]; first != '/' { + dirID = "/" + dirID + } + if last := dirID[len(dirID)-1]; last != '/' { + dirID = dirID + "/" + } + } else { + dirID = "/" + } + file, err := os.OpenFile(f.opt.SortFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + fmt.Println(err) + return "", err + } + defer file.Close() + _, err = file.WriteString(dirID + leaf + "\n") + if err != nil { + fmt.Println(err) + return dirID + leaf, err + } + return dirID + leaf, nil +} + +// Redownload a dead torrent +func (f *Fs) redownloadTorrent(ctx context.Context, torrent api.Item) (redownloaded_torrent api.Item) { + fmt.Println("Redownloading dead torrent: " + torrent.Name) + //Get dead torrent file and hash info + var method = "GET" + var path = "/torrents/info/" + torrent.ID + var opts = rest.Opts{ + Method: method, + Path: path, + Parameters: f.baseParams(), + } + _, _ = f.srv.CallJSON(ctx, &opts, nil, &torrent) + var selected_files []int64 + var dead_torrent_id = torrent.ID + for _, file := range torrent.Files { + if file.Selected == 1 { + selected_files = append(selected_files, file.ID) + } + } + var selected_files_str = strings.Trim(strings.Join(strings.Fields(fmt.Sprint(selected_files)), ","), "[]") + //Delete old download links + for _, link := range torrent.Links { + for i, cachedfile := range cached { + if cachedfile.OriginalLink == link { + path = "/downloads/delete/" + cachedfile.ID + opts = rest.Opts{ + Method: "DELETE", + Path: path, + Parameters: f.baseParams(), + } + var resp *http.Response + var result api.Response + var retries = 0 + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) + for resp.StatusCode == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) + retries += 1 + } + cached[i].OriginalLink = "this-is-not-a-link" + } + } + } + //Add torrent again + path = "/torrents/addMagnet" + method = "POST" + opts = rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "magnet": {"magnet:?xt=urn:btih:" + torrent.TorrentHash}, + }, + Parameters: f.baseParams(), + } + _, _ = f.srv.CallJSON(ctx, &opts, nil, &torrent) + method = "GET" + path = "/torrents/info/" + torrent.ID + opts = rest.Opts{ + Method: method, + Path: path, + Parameters: f.baseParams(), + } + _, _ = f.srv.CallJSON(ctx, &opts, nil, &torrent) + var tries = 0 + for torrent.Status != "waiting_files_selection" && tries < 5 { + time.Sleep(time.Duration(1) * time.Second) + _, _ = f.srv.CallJSON(ctx, &opts, nil, &torrent) + tries += 1 + } + //Select the same files again + path = "/torrents/selectFiles/" + torrent.ID + method = "POST" + opts = rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "files": {selected_files_str}, + }, + Parameters: f.baseParams(), + } + _, _ = f.srv.CallJSON(ctx, &opts, nil, &torrent) + //Delete the old torrent + path = "/torrents/delete/" + dead_torrent_id + method = "DELETE" + opts = rest.Opts{ + Method: method, + Path: path, + Parameters: f.baseParams(), + } + _, _ = f.srv.CallJSON(ctx, &opts, nil, &torrent) + torrent.Status = "downloaded" + lastcheck = time.Now().Unix() - interval + for i, TorrentID := range broken_torrents { + if dead_torrent_id == TorrentID { + broken_torrents[i] = broken_torrents[len(broken_torrents)-1] + broken_torrents = broken_torrents[:len(broken_torrents)-1] + } + } + return torrent +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *Fs) folder_exists(dirID string) bool { + if dirID == "/" { + return false + } + if _, ok := folders[dirID]; ok { + return true + } + return false +} + +// list the objects into the function supplied +// +// If directories is set it only sends directories +// User function to process a File item from listAll +// +// Should return true to finish processing +type listAllFn func(*api.Item) bool + +// Lists the directory required calling the user function on each item found +// +// If the user fn ever returns true then it early exits with found = true +// +// It returns a newDirID which is what the system returned as the directory ID +func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (newDirID string, found bool, err error) { + path := "/downloads" + method := "GET" + var partialresult []api.Item + var result []api.Item + var resp *http.Response + + if (dirID == rootID) || !(f.folder_exists(dirID)) { + + //create folder structure + // + // Open the sorting file + + file, err := os.Open(f.opt.SortFile) + if err != nil { + fmt.Println(err) + } + defer file.Close() + + fileInfo, _ := file.Stat() + fileModTime := fileInfo.ModTime().Unix() + if fileModTime > lastFileMod { + + // Reset saved folder structure + fmt.Println("reading updated sorting file") + structure = make(map[string]string) + levels = make(map[string][]string) + folders = make(map[string][]api.Item) + regex_folders = make(map[string]string) + + // Read the file line by line + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if strings.Contains(scanner.Text(), "#") { + continue + } else if strings.Contains(scanner.Text(), move_chars) { + // Split the line by " -> " + parts := strings.Split(scanner.Text(), move_chars) + // Add the key-value pair to the map + structure[parts[0]] = parts[1] + } else if strings.Contains(scanner.Text(), regx_chars) { + // Split the line by " -> " + parts := strings.Split(scanner.Text(), regx_chars) + // Add the key-value pair to the map + regex_folders[parts[0]] = parts[1] + } else { + structure[scanner.Text()] = scanner.Text() + } + } + + // Iterate through the map + for _, newLocation := range structure { + + // Split the new location by "/" + locationParts := strings.Split(newLocation, "/") + + // Create a new variable to store the full path + var location string + + // Iterate through each level of the new location + for _, locationPart := range locationParts { + // Append the full path to the corresponding level + var skip bool + skip = false + for _, val := range levels[location] { + if val == locationPart { + skip = true + break + } + } + if skip { + location = location + locationPart + "/" + continue + } + + levels[location] = append(levels[location], locationPart) + + // Append the current location part to the full path + location = location + locationPart + "/" + } + } + + } + + //create regex definitions + + var regex_defs = make(map[string]*regexp.Regexp) + for folder, regex_def := range regex_folders { + r, _ := regexp.Compile(regex_def) + regex_defs[folder] = r + } + + //update global cached list + opts := rest.Opts{ + Method: method, + Path: path, + Parameters: f.baseParams(), + } + opts.Parameters.Set("includebreadcrumbs", "false") + opts.Parameters.Set("limit", "1") + var newcached []api.Item + var totalcount int + var printed = false + totalcount = 2 + for len(newcached) < totalcount { + partialresult = nil + resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + var retries = 0 + for resp.StatusCode == 429 && retries <= 5 { + partialresult = nil + time.Sleep(time.Duration(2) * time.Second) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + retries += 1 + } + if err == nil { + totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) + if err == nil { + if totalcount != len(cached) || time.Now().Unix()-lastcheck > interval { + if time.Now().Unix()-lastcheck > interval && !printed { + fmt.Println("Updating links and torrents") + printed = true + } + newcached = append(newcached, partialresult...) + opts.Parameters.Set("offset", strconv.Itoa(len(newcached))) + opts.Parameters.Set("limit", "2500") + } else { + newcached = cached + } + } else { + break + } + } else { + break + } + } + //fmt.Printf("Done.\n") + //fmt.Printf("Updating RealDebrid Torrents ... ") + cached = newcached + //get torrents + path = "/torrents" + opts = rest.Opts{ + Method: method, + Path: path, + Parameters: f.baseParams(), + } + opts.Parameters.Set("limit", "1") + var newtorrents []api.Item + totalcount = 2 + for len(newtorrents) < totalcount { + partialresult = nil + resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + var retries = 0 + for resp.StatusCode == 429 && retries <= 5 { + partialresult = nil + time.Sleep(time.Duration(2) * time.Second) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + retries += 1 + } + if err == nil { + totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) + if err == nil { + if totalcount != len(torrents) || time.Now().Unix()-lastcheck > interval { + newtorrents = append(newtorrents, partialresult...) + opts.Parameters.Set("offset", strconv.Itoa(len(newtorrents))) + opts.Parameters.Set("limit", "2500") + } else { + newtorrents = torrents + } + } else { + break + } + } else { + break + } + } + lastcheck = time.Now().Unix() + //fmt.Printf("Done.\n") + torrents = newtorrents + + //Iterate through built file and torrent list: + // + //build system folder/file structure + var broken = false + for i, torrent := range torrents { + //handle dead torrents + broken = false + for _, TorrentID := range broken_torrents { + if torrent.ID == TorrentID { + broken = true + } + } + if torrent.Status == "dead" || broken { + torrents[i] = f.redownloadTorrent(ctx, torrent) + } + //if folders is not up to date: + if fileModTime > lastFileMod { + if i == 1 { //len(torrents)-1 { + lastFileMod = fileModTime + } + } else { + continue + } + //add torrents to their cahnged folders, create new ones in standard locations + var parentdir string + var dir string + if _, ok := structure["/"+torrent.Name]; ok { + parentdir = strings.Join(strings.Split(structure["/"+torrent.Name], "/")[:len(strings.Split(structure["/"+torrent.Name], "/"))-1], "/") + "/" + dir = structure["/"+torrent.Name] + } else { + //handle regex defined folers + parentdir = "/default/" + for folder, r := range regex_defs { + match := r.MatchString(torrent.Name) + if match { + parentdir = folder + "/" + break + } + } + dir = parentdir + torrent.Name + } + locationParts := strings.Split(dir, "/") + var location string + for _, locationPart := range locationParts { + // Append the full path to the corresponding level + var skip bool + skip = false + for _, val := range levels[location] { + if val == locationPart { + skip = true + break + } + } + if skip { + location = location + locationPart + "/" + continue + } + levels[location] = append(levels[location], locationPart) + location = location + locationPart + "/" + } + var folder api.Item + folder.ID = dir + if _, ok := structure["/"+torrent.Name]; ok { + folder.Name = strings.Split(structure["/"+torrent.Name], "/")[len(strings.Split(structure["/"+torrent.Name], "/"))-1] + } else { + folder.Name = torrent.Name + } + folder.Type = "folder" + folder.Generated = torrent.Generated + folder.Ended = torrent.Ended + // var skip = false + // for _, val := range folders[parentdir] { + // if val.Name == folder.Name { + // skip = true + // break + // } + // } + // if !skip { + folders[parentdir] = append(folders[parentdir], folder) + // } + id2name[torrent.ID] = torrent.Name + //iterate through files + var broken = false + for _, link := range torrent.Links { + var ItemFile api.Item + for _, cachedfile := range cached { + if cachedfile.OriginalLink == link { + ItemFile = cachedfile + break + } + } + if ItemFile.Link == "" { + //fmt.Printf("Creating new unrestricted direct link for: '%s'\n", torrent.Name) + path = "/unrestrict/link" + method = "POST" + opts := rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "link": {link}, + }, + Parameters: f.baseParams(), + } + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp.StatusCode == 503 { + broken = true + break + } + var retries = 0 + for resp.StatusCode == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + retries += 1 + } + } + ItemFile.ParentID = torrent.ID + ItemFile.TorrentHash = torrent.TorrentHash + ItemFile.Generated = torrent.Generated + ItemFile.Type = "file" + var fileparentdir = dir + if _, ok := structure["/"+torrent.Name+"/"+ItemFile.Name]; ok { + fileparentdir = strings.Join(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[:len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1], "/") + "/" + ItemFile.Name = strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1] + } + // var skip = false + // for _, val := range folders[fileparentdir] { + // if val.Name == ItemFile.Name { + // skip = true + // break + // } + // } + // if !skip { + folders[fileparentdir] = append(folders[fileparentdir], ItemFile) + // } + } + if broken { + torrents[i] = f.redownloadTorrent(ctx, torrent) + torrent = torrents[i] + for _, link := range torrent.Links { + var ItemFile api.Item + //fmt.Printf("Creating new unrestricted direct link for: '%s'\n", torrent.Name) + path = "/unrestrict/link" + method = "POST" + opts := rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "link": {link}, + }, + Parameters: f.baseParams(), + } + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + var retries = 0 + for resp.StatusCode == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + retries += 1 + } + ItemFile.ParentID = torrent.ID + ItemFile.TorrentHash = torrent.TorrentHash + ItemFile.Generated = torrent.Generated + ItemFile.Type = "file" + var fileparentdir = dir + if _, ok := structure["/"+torrent.Name+"/"+ItemFile.Name]; ok { + fileparentdir = strings.Join(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[:len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1], "/") + "/" + ItemFile.Name = strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1] + } + // var skip = false + // for _, val := range folders[fileparentdir] { + // if val.Name == ItemFile.Name { + // skip = true + // break + // } + // } + // if !skip { + folders[fileparentdir] = append(folders[fileparentdir], ItemFile) + // } + } + } + if i > 0 { + break + } + } + for key, level := range levels { + for _, foldername := range level { + var skip = false + for _, folder := range folders[key] { + if foldername == folder.Name { + skip = true + break + } + } + if skip { + continue + } + var folder api.Item + folder.ID = key + foldername + "/" + folder.Name = foldername + folder.Type = "folder" + folder.Generated = "1969-04-20T16:20:00.000Z" + folders[key] = append(folders[key], folder) + //folders[folder.ID] = []api.Item{} + } + } + //return root folder structure if dirID == rootID + if dirID == rootID { + result = append(result, folders["/"]...) + } + } + + if f.folder_exists(dirID) { + //return dirID folder structure + result = append(result, folders[dirID]...) + } else if dirID != rootID { + //handle edge cases + if f.folder_exists(dirID + "/") { + result = append(result, folders[dirID+"/"]...) + } else if f.folder_exists(dirID[:len(dirID)-1]) { + result = append(result, folders[dirID[:len(dirID)-1]]...) + } else { + var parentdirID = strings.Join(strings.Split(dirID, "/")[:len(strings.Split(dirID, "/"))-1], "/") + if last := parentdirID[len(parentdirID)-1]; last != '/' { + parentdirID = parentdirID + "/" + } + if first := parentdirID[0]; first != '/' { + parentdirID = "/" + parentdirID + } + result = append(result, folders[parentdirID]...) + } + + } + + if err != nil { + return newDirID, found, fmt.Errorf("couldn't list files: %w", err) + } + for i := range result { + item := &result[i] + layout := "2006-01-02T15:04:05.000Z" + if item.Generated != "" { + t, _ := time.Parse(layout, item.Generated) + item.CreatedAt = t.Unix() + } else if item.Ended != "" { + t, _ := time.Parse(layout, item.Ended) + item.CreatedAt = t.Unix() + } + if item.Type == api.ItemTypeFolder { + if filesOnly { + continue + } + } else if item.Type == api.ItemTypeFile { + if directoriesOnly { + continue + } + } else { + fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type) + continue + } + item.Name = f.opt.Enc.ToStandardName(item.Name) + if fn(item) { + found = true + break + } + } + return +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. +func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + //fmt.Println("Listing Items ... ") + directoryID, err := f.dirCache.FindDir(ctx, dir, false) + if err != nil { + return nil, err + } + var iErr error + _, _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool { + remote := path.Join(dir, info.Name) + if info.Type == api.ItemTypeFolder { + // cache the directory ID for later lookups + f.dirCache.Put(remote, info.ID) + d := fs.NewDir(remote, time.Unix(info.CreatedAt, 0)).SetID(info.ID) + var skip = false + for _, entry := range entries { + if d.Remote() == entry.Remote() { + skip = true + break + } + } + if !skip { + entries = append(entries, d) + } + } else if info.Type == api.ItemTypeFile { + o, err := f.newObjectWithInfo(ctx, remote, info) + if err != nil { + iErr = err + return true + } + var skip = false + for _, entry := range entries { + if o.Remote() == entry.Remote() { + skip = true + break + } + } + if !skip { + entries = append(entries, o) + } + } + return false + }) + if err != nil { + return nil, err + } + if iErr != nil { + return nil, iErr + } + //fmt.Println("Done Listing Items.") + return entries, nil +} + +// Creates from the parameters passed in a half finished Object which +// must have setMetaData called on it +// +// Returns the object, leaf, directoryID and error +// +// Used to create new objects +func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { + // Create the directory for the object if it doesn't exist + leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true) + if err != nil { + return + } + // Temporary Object under construction + o = &Object{ + fs: f, + remote: remote, + } + return o, leaf, directoryID, nil +} + +// Put the object +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil) + switch err { + case nil: + return existingObj, existingObj.Update(ctx, in, src, options...) + case fs.ErrorObjectNotFound: + // Not found so create it + return f.PutUnchecked(ctx, in, src, options...) + default: + return nil, err + } +} + +// PutUnchecked the object into the container +// +// This will produce an error if the object already exists +// +// Copy the reader in to the new object which is returned +// +// The new object may have been created if an error is returned +func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + remote := src.Remote() + size := src.Size() + modTime := src.ModTime(ctx) + + o, _, _, err := f.createObject(ctx, remote, modTime, size) + if err != nil { + return nil, err + } + return o, o.Update(ctx, in, src, options...) +} + +// Mkdir creates the container if it doesn't exist +func (f *Fs) Mkdir(ctx context.Context, dir string) error { + _, err := f.dirCache.FindDir(ctx, dir, true) + if err != nil { + return err + } + return nil +} + +// purgeCheck removes the root directory, if check is set then it +// refuses to do so if it has anything in +func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { + //fmt.Printf("Purging torrent: '%s'\n", rootID) + root := path.Join(f.root, dir) + if root == "" { + return errors.New("can't purge root directory") + } + dc := f.dirCache + rootID, err := dc.FindDir(ctx, dir, false) + if err != nil { + return err + } + path := "/torrents/delete/" + rootID + opts := rest.Opts{ + Method: "DELETE", + Path: path, + Parameters: f.baseParams(), + } + var resp *http.Response + var result api.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) + return shouldRetry(ctx, resp, err) + }) + f.dirCache.FlushDir(dir) + return nil +} + +// Rmdir deletes the root folder +// +// Returns an error if it isn't empty +func (f *Fs) Rmdir(ctx context.Context, dir string) error { + //fmt.Printf("Rmdir: '%s'\n", dir) + return f.purgeCheck(ctx, dir, true) +} + +// Precision return the precision of this Fs +func (f *Fs) Precision() time.Duration { + return fs.ModTimeNotSupported +} + +// Purge deletes all the files in the directory +// +// Optional interface: Only implement this if you have a way of +// deleting all the files quicker than just running Remove() on the +// result of List() +func (f *Fs) Purge(ctx context.Context, dir string) error { + //fmt.Printf("Purge: '%s'\n", dir) + return f.purgeCheck(ctx, dir, false) +} + +// move a file or folder +// +// This is complicated by the fact that there is an API to move files +// between directories and a separate one to rename them. We try to +// call the minimum number of API calls. +func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDirectoryID, newDirectoryID string) (err error) { + for _, torrent := range torrents { + if torrent.ID == oldDirectoryID { + oldDirectoryID = "/" + torrent.Name + "/" + break + } else if !isFile && torrent.Name == oldLeaf { + oldDirectoryID = "/" + break + } + } + if newDirectoryID == "0" { + newDirectoryID = "/" + } + // Open the file + file, err := os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + fmt.Println(err) + return err + } + defer file.Close() + if last := newDirectoryID[len(newDirectoryID)-1]; last != '/' { + newDirectoryID = newDirectoryID + "/" + } + if first := oldDirectoryID[0]; first != '/' { + oldDirectoryID = "/" + oldDirectoryID + } + searchString := move_chars + oldDirectoryID + oldLeaf + retryString := oldDirectoryID + oldLeaf + move_chars + newDirectoryID + newLeaf + newString := move_chars + newDirectoryID + newLeaf + scanner := bufio.NewScanner(file) + var lines []string + var replaced = false + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, searchString) { + line = strings.Replace(line, searchString, newString, -1) + replaced = true + } else if strings.Contains(line, retryString) { + replaced = true + } + lines = append(lines, line) + } + if err := scanner.Err(); err != nil { + fmt.Println(err) + return err + } + file.Truncate(0) + file.Seek(0, 0) + if !replaced { + var line = oldDirectoryID + oldLeaf + move_chars + newDirectoryID + newLeaf + lines = append(lines, line) + } + for _, line := range lines { + _, err := file.WriteString(line + "\n") + if err != nil { + fmt.Println(err) + return err + } + } + return nil +} + +// Move src to this remote using server-side move operations. +// +// This is stored with the remote path given +// +// It returns the destination Object and a possible error +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantMove +func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + srcObj, ok := src.(*Object) + if !ok { + fs.Debugf(src, "Can't move - not same remote type") + return nil, fs.ErrorCantMove + } + + // Create temporary object + dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) + if err != nil { + return nil, err + } + + // Do the move + err = f.move(ctx, true, srcObj.id, path.Base(srcObj.remote), leaf, srcObj.ParentID, directoryID) + if err != nil { + return nil, err + } + + err = dstObj.readMetaData(ctx) + if err != nil { + return nil, err + } + return dstObj, nil +} + +// DirMove moves src, srcRemote to this remote at dstRemote +// using server-side move operations. +// +// Will only be called if src.Fs().Name() == f.Name() +// +// If it isn't possible then return fs.ErrorCantDirMove +// +// If destination exists then return fs.ErrorDirExists +func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { + srcFs, ok := src.(*Fs) + if !ok { + fs.Debugf(srcFs, "Can't move directory - not same remote type") + return fs.ErrorCantDirMove + } + + srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) + if err != nil { + return err + } + + // Do the move + err = f.move(ctx, false, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID) + if err != nil { + return err + } + srcFs.dirCache.FlushDir(srcRemote) + return nil +} + +// PublicLink adds a "readable by anyone with link" permission on the given file or folder. +func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) { + _, err := f.dirCache.FindDir(ctx, remote, false) + if err == nil { + return "", fs.ErrorCantShareDirectories + } + o, err := f.NewObject(ctx, remote) + if err != nil { + return "", err + } + return o.(*Object).url, nil +} + +// About gets quota information +func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { + return usage, nil +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *Fs) DirCacheFlush() { + f.dirCache.ResetRoot() +} + +// Hashes returns the supported hash sets. +func (f *Fs) Hashes() hash.Set { + return hash.Set(hash.None) +} + +// ------------------------------------------------------------ + +// Fs returns the parent Fs +func (o *Object) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *Object) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *Object) Remote() string { + return o.remote +} + +// Hash returns the SHA-1 of an object returning a lowercase hex string +func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// Size returns the size of an object in bytes +func (o *Object) Size() int64 { + err := o.readMetaData(context.TODO()) + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return 0 + } + return o.size +} + +// setMetaData sets the metadata from info +func (o *Object) setMetaData(info *api.Item) (err error) { + if info.Type != "file" { + return fmt.Errorf("%q is %q: %w", o.remote, info.Type, fs.ErrorNotAFile) + } + o.hasMetaData = true + o.size = info.Size + o.modTime = time.Unix(info.CreatedAt, 0) + o.id = info.ID + o.mimeType = info.MimeType + o.url = info.Link + o.ParentID = info.ParentID + o.TorrentHash = info.TorrentHash + return nil +} + +// readMetaData gets the metadata if it hasn't already been fetched +// +// it also sets the info +func (o *Object) readMetaData(ctx context.Context) (err error) { + if o.hasMetaData { + return nil + } + info, err := o.fs.readMetaDataForPath(ctx, o.remote, false, true) + if err != nil { + return err + } + return o.setMetaData(info) +} + +// ModTime returns the modification time of the object +// +// +// It attempts to read the objects mtime and if that isn't present the +// LastModified returned in the http headers +func (o *Object) ModTime(ctx context.Context) time.Time { + err := o.readMetaData(ctx) + if err != nil { + fs.Logf(o, "Failed to read metadata: %v", err) + return time.Now() + } + return o.modTime +} + +// SetModTime sets the modification time of the local fs object +func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { + return fs.ErrorCantSetModTime +} + +// Storable returns a boolean showing whether this object storable +func (o *Object) Storable() bool { + return true +} + +// Open an object for read +func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + if o.url == "" { + return nil, errors.New("can't download - no URL") + } + fs.FixRangeOption(options, o.size) + var resp *http.Response + var err_code = 0 + opts := rest.Opts{ + Path: "", + RootURL: o.url, + Method: "GET", + Options: options, + } + err = o.fs.pacer.Call(func() (bool, error) { + resp, err = o.fs.srv.Call(ctx, &opts) + if resp != nil { + err_code = resp.StatusCode + } + return shouldRetry(ctx, resp, err) + }) + if err != nil { + if err_code == 503 { + for _, TorrentID := range broken_torrents { + if o.ParentID == TorrentID { + return nil, err + } + } + fmt.Println("Error opening file: '" + o.url + "'.") + fmt.Println("This link seems to be broken. Torrent will be re-downloaded on next refresh.") + broken_torrents = append(broken_torrents, o.ParentID) + } + return nil, err + } + return resp.Body, err +} + +// Update the object with the contents of the io.Reader, modTime and size +// +// If existing is set then it updates the object rather than creating a new one +// +// The new object may have been created if an error is returned +func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { + return nil +} + +// Remove an object by ID +func (f *Fs) remove(ctx context.Context, id ...string) (err error) { + //fmt.Printf("Removing direct link id: '%s'\n", id[0]) + //if f.opt.RootFolderID == "torrents" { + // fmt.Printf("Removing torrent id: '%s'\n", id[1]) + //} + path := "/downloads/delete/" + id[0] + opts := rest.Opts{ + Method: "DELETE", + Path: path, + Parameters: f.baseParams(), + } + var resp *http.Response + var result api.Response + var retries = 0 + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) + for resp.StatusCode == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) + retries += 1 + } + path = "/torrents/delete/" + id[1] + opts = rest.Opts{ + Method: "DELETE", + Path: path, + Parameters: f.baseParams(), + } + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) + if resp.StatusCode == 429 { + time.Sleep(time.Duration(2) * time.Second) + _, _ = f.srv.CallJSON(ctx, &opts, nil, &result) + } + lastcheck = time.Now().Unix() - interval + return nil +} + +// Remove an object +func (o *Object) Remove(ctx context.Context) error { + //fmt.Printf("Removing: '%s'\n", o.remote) + err := o.readMetaData(ctx) + if err != nil { + return fmt.Errorf("Remove: Failed to read metadata: %w", err) + } + if o.ParentID != "" { + return o.fs.remove(ctx, o.id, o.ParentID) + } else { + return o.fs.remove(ctx, o.id) + } +} + +// MimeType of an Object if known, "" otherwise +func (o *Object) MimeType(ctx context.Context) string { + return o.mimeType +} + +// ID returns the ID of the Object if known, or "" if not +func (o *Object) ID() string { + return o.id +} + +// Check the interfaces are satisfied +var ( + _ fs.Fs = (*Fs)(nil) + _ fs.Purger = (*Fs)(nil) + _ fs.Mover = (*Fs)(nil) + _ fs.DirMover = (*Fs)(nil) + _ fs.DirCacheFlusher = (*Fs)(nil) + _ fs.Abouter = (*Fs)(nil) + _ fs.PublicLinker = (*Fs)(nil) + _ fs.Object = (*Object)(nil) + _ fs.MimeTyper = (*Object)(nil) + _ fs.IDer = (*Object)(nil) +) From e84232987919e0723daf43ccc554546bf96d4fce Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sat, 21 Jan 2023 22:53:01 +0100 Subject: [PATCH 522/560] sorting file --- .vscode/launch.json | 15 +++++++++++++++ sorting.txt | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 .vscode/launch.json create mode 100644 sorting.txt diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000000000..4eb5cdc5404e6 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Launch", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}", + "env": {}, + "args": ["tree","realdebrid:"],//["moveto","realdebrid:/shows/some shit/Family.Guy.S19E01.1080p.HULU.WEB-DL.DDP5.1.H.264-CtrlHD.mkv","realdebrid:/shows/some other shit/Family.Guy.S19E01.1080p.HULU.WEB-DL.DDP5.1.H.264-CtrlHD.mkv"],//["move","realdebrid:/shows/Family.Guy.S19.1080p.HULU.WEBRip.DDP5.1.x264-CtrlHD[rartv]","realdebrid:/shows/some shit"],//["moveto","realdebrid:/shows/Our.Universe/IVE RENAMED YOU BITCH/Our.Universe.S01E01.Chasing.Starlight.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-SMURF.mkv","realdebrid:/new folder/someshit.mkv"], //["tree","realdebrid:"],// + "console": "integratedTerminal" + } + ] +} \ No newline at end of file diff --git a/sorting.txt b/sorting.txt new file mode 100644 index 0000000000000..d337b486bae74 --- /dev/null +++ b/sorting.txt @@ -0,0 +1,33 @@ +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~ rclone_rd sorting file ~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# - write comment lines using "#" +# +# - write regex definitions using: folder + " == " + regex definition. You can edit the exising ones or create new ones. +# Order matters for regex folders, first match will be final destination. Make sure there are no trailing space characters. +# torrents that dont match any regex definition end up in a folder named "default". +# Example: /movies == (?i)(19|20)([0-9]{2} ?\.?) +# +# - create new directories using "/foldername" +# Example: /shit +# +# - write move/renaming changes using: "actual torrent title" + "/" + "actual file name" + " -> " + "destination" +# You do not need to create the directories you are moving stuff to, this will be done automatically. +# Example: /Our.Universe.S01.1080p.[rartv] -> /shows/Our Universe/Season 1 +# +# - never leave an empty line between lines, always end with a newline as last character +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~ manual and regex folders: ~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +/shows == (?i)(S[0-9]{2}|SEASONS?.[0-9]|COMPLETE|[^457a-z\W\s]-[0-9]+) +/movies == (?i)(19|20)([0-9]{2} ?\.?) +/default +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~ recorded/manual changes to the structure: ~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# From 9bc32c05d606d139397806a0abb874d34e62c568 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sun, 22 Jan 2023 10:24:42 +0100 Subject: [PATCH 523/560] create sorting file if it doesnt exist --- backend/realdebrid/realdebrid.go | 104 +++++++++--------- ...istro_window.bat - Verkn\303\274pfung.lnk" | Bin 0 -> 1059 bytes 2 files changed, 52 insertions(+), 52 deletions(-) create mode 100644 "open_distro_window.bat - Verkn\303\274pfung.lnk" diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 3648adf42a37c..bad39a22860c5 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -73,9 +73,7 @@ var ( } ) -//Global lists of recieved content. -//Realdebrid content is provided in pages with 100 items per page. -//To limit api calls all pages are stored here and are only updated on changes in the total length +//Global variables var cached []api.Item var torrents []api.Item var broken_torrents []string @@ -89,6 +87,40 @@ var regex_folders = make(map[string]string) var id2name = make(map[string]string) var move_chars = " -> " var regx_chars = " == " +var default_sorting = `# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~ rclone_rd sorting file ~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# - write comment lines using "#" +# +# - write regex definitions using: folder + " == " + regex definition. You can edit the exising ones or create new ones. +# Order matters for regex folders, first match will be final destination. Make sure there are no trailing space characters. +# torrents that dont match any regex definition end up in a folder named "default". +# Example: /movies == (?i)(19|20)([0-9]{2} ?\.?) +# +# - create new directories using "/foldername" +# Example: /shit +# +# - write move/renaming changes using: "actual torrent title" + "/" + "actual file name" + " -> " + "destination" +# You do not need to create the directories you are moving stuff to, this will be done automatically. +# Example: /Our.Universe.S01.1080p.[rartv] -> /shows/Our Universe/Season 1 +# +# - never leave an empty line between lines, always end with a newline as last character +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~ manual and regex folders: ~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +/shows == (?i)(S[0-9]{2}|SEASONS?.[0-9]|COMPLETE|[^457a-z\W\s]-[0-9]+) +/movies == (?i)(19|20)([0-9]{2} ?\.?) +/default +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~ recorded/manual changes to the structure: ~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +` // Register with Fs func init() { @@ -546,13 +578,26 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Open the sorting file file, err := os.Open(f.opt.SortFile) - if err != nil { + if os.IsNotExist(err) { + fmt.Println("creating default sorting file") + file, err = os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + fmt.Println(err) + } + defer file.Close() + + if _, err := file.WriteString(default_sorting); err != nil { + fmt.Println(err) + } + } else if err != nil { fmt.Println(err) + } else { + defer file.Close() } - defer file.Close() fileInfo, _ := file.Stat() fileModTime := fileInfo.ModTime().Unix() + if fileModTime > lastFileMod { // Reset saved folder structure @@ -781,16 +826,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi folder.Type = "folder" folder.Generated = torrent.Generated folder.Ended = torrent.Ended - // var skip = false - // for _, val := range folders[parentdir] { - // if val.Name == folder.Name { - // skip = true - // break - // } - // } - // if !skip { folders[parentdir] = append(folders[parentdir], folder) - // } id2name[torrent.ID] = torrent.Name //iterate through files var broken = false @@ -835,16 +871,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi fileparentdir = strings.Join(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[:len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1], "/") + "/" ItemFile.Name = strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1] } - // var skip = false - // for _, val := range folders[fileparentdir] { - // if val.Name == ItemFile.Name { - // skip = true - // break - // } - // } - // if !skip { folders[fileparentdir] = append(folders[fileparentdir], ItemFile) - // } } if broken { torrents[i] = f.redownloadTorrent(ctx, torrent) @@ -878,16 +905,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi fileparentdir = strings.Join(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[:len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1], "/") + "/" ItemFile.Name = strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1] } - // var skip = false - // for _, val := range folders[fileparentdir] { - // if val.Name == ItemFile.Name { - // skip = true - // break - // } - // } - // if !skip { folders[fileparentdir] = append(folders[fileparentdir], ItemFile) - // } } } if i > 0 { @@ -999,32 +1017,14 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e // cache the directory ID for later lookups f.dirCache.Put(remote, info.ID) d := fs.NewDir(remote, time.Unix(info.CreatedAt, 0)).SetID(info.ID) - var skip = false - for _, entry := range entries { - if d.Remote() == entry.Remote() { - skip = true - break - } - } - if !skip { - entries = append(entries, d) - } + entries = append(entries, d) } else if info.Type == api.ItemTypeFile { o, err := f.newObjectWithInfo(ctx, remote, info) if err != nil { iErr = err return true } - var skip = false - for _, entry := range entries { - if o.Remote() == entry.Remote() { - skip = true - break - } - } - if !skip { - entries = append(entries, o) - } + entries = append(entries, o) } return false }) diff --git "a/open_distro_window.bat - Verkn\303\274pfung.lnk" "b/open_distro_window.bat - Verkn\303\274pfung.lnk" new file mode 100644 index 0000000000000000000000000000000000000000..d4223800811975be04175144d6828b2ed1f160dd GIT binary patch literal 1059 zcmb7DU1*YF6nH+rVMF}B1JwwGE%-U*%^okAxI~{Zi4;q%0=@2Ac4h3P!TwbBKPozl2*{?kY4Lc zBqMQuIF=qwCH$FKJe)$a$W1ghuO5Z6MIj-+`JgnDTLv{JboQjzzxFo|xp^XZCtm$IJe*kC>*?j;2 literal 0 HcmV?d00001 From 58c65e2d6a108ea7b88766583a1c2661f5e1bc9d Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sun, 22 Jan 2023 11:23:07 +0100 Subject: [PATCH 524/560] small fixes for folder creation in root dir --- .gitignore | 1 + backend/realdebrid/realdebrid.go | 25 ++++++++++++++----------- sorting.txt | 18 ++++++++++-------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index ff61403f32e3d..236363b218a3e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ fuzz-build.zip *.rej Thumbs.db __pycache__ +sorting.txt diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index bad39a22860c5..211001c194beb 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -87,11 +87,10 @@ var regex_folders = make(map[string]string) var id2name = make(map[string]string) var move_chars = " -> " var regx_chars = " == " -var default_sorting = `# -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +var default_sorting = `# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~~~~~ rclone_rd sorting file ~~~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# + # - write comment lines using "#" # # - write regex definitions using: folder + " == " + regex definition. You can edit the exising ones or create new ones. @@ -105,21 +104,20 @@ var default_sorting = `# # - write move/renaming changes using: "actual torrent title" + "/" + "actual file name" + " -> " + "destination" # You do not need to create the directories you are moving stuff to, this will be done automatically. # Example: /Our.Universe.S01.1080p.[rartv] -> /shows/Our Universe/Season 1 -# -# - never leave an empty line between lines, always end with a newline as last character -# + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~~~ manual and regex folders: ~~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# + /shows == (?i)(S[0-9]{2}|SEASONS?.[0-9]|COMPLETE|[^457a-z\W\s]-[0-9]+) /movies == (?i)(19|20)([0-9]{2} ?\.?) /default -# + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~ recorded/manual changes to the structure: ~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# + + ` // Register with Fs @@ -414,8 +412,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin // CreateDir makes a directory with pathID as parent and name leaf func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, err error) { - // Open the file - if len(dirID) > 0 { + if len(dirID) > 0 && dirID != "0" { if first := dirID[0]; first != '/' { dirID = "/" + dirID } @@ -612,6 +609,8 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi for scanner.Scan() { if strings.Contains(scanner.Text(), "#") { continue + } else if len(scanner.Text()) == 0 || scanner.Text() == "\n" || scanner.Text() == "\n\r" { + continue } else if strings.Contains(scanner.Text(), move_chars) { // Split the line by " -> " parts := strings.Split(scanner.Text(), move_chars) @@ -945,8 +944,10 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } else if dirID != rootID { //handle edge cases if f.folder_exists(dirID + "/") { + fmt.Println("changed regested dirID: " + dirID + " to: " + dirID + "/ to find content.") result = append(result, folders[dirID+"/"]...) } else if f.folder_exists(dirID[:len(dirID)-1]) { + fmt.Println("changed regested dirID: " + dirID + " to: " + dirID[:len(dirID)-1] + " to find content.") result = append(result, folders[dirID[:len(dirID)-1]]...) } else { var parentdirID = strings.Join(strings.Split(dirID, "/")[:len(strings.Split(dirID, "/"))-1], "/") @@ -956,6 +957,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if first := parentdirID[0]; first != '/' { parentdirID = "/" + parentdirID } + fmt.Println("changed regested dirID: " + dirID + " to: " + parentdirID + " to find content.") result = append(result, folders[parentdirID]...) } @@ -1407,6 +1409,7 @@ func (o *Object) Storable() bool { // Open an object for read func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { + fmt.Println("opening file: " + o.id + " with parent: " + o.ParentID + " using link: " + o.url) if o.url == "" { return nil, errors.New("can't download - no URL") } diff --git a/sorting.txt b/sorting.txt index d337b486bae74..8a6f30177c478 100644 --- a/sorting.txt +++ b/sorting.txt @@ -1,8 +1,7 @@ -# # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~~~~~ rclone_rd sorting file ~~~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# + # - write comment lines using "#" # # - write regex definitions using: folder + " == " + regex definition. You can edit the exising ones or create new ones. @@ -16,18 +15,21 @@ # - write move/renaming changes using: "actual torrent title" + "/" + "actual file name" + " -> " + "destination" # You do not need to create the directories you are moving stuff to, this will be done automatically. # Example: /Our.Universe.S01.1080p.[rartv] -> /shows/Our Universe/Season 1 -# -# - never leave an empty line between lines, always end with a newline as last character -# + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~~~ manual and regex folders: ~~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# + /shows == (?i)(S[0-9]{2}|SEASONS?.[0-9]|COMPLETE|[^457a-z\W\s]-[0-9]+) /movies == (?i)(19|20)([0-9]{2} ?\.?) /default -# + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~ recorded/manual changes to the structure: ~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# + +/Family.Guy.S19.1080p.HULU.WEBRip.DDP5.1.x264-CtrlHD[rartv] -> /shows/another rename +/shows/Neuer Ordner +/shows/Neuer Ordner -> /shows/fuck +/Neuer Ordner +/Our.Universe.S01.1080p.NF.WEBRip.DDP5.1.Atmos.x264-SMURF[rartv] -> /shows/fuck/shit From bdb8ed962519cfbc3c79469f9873a6134eb5fb94 Mon Sep 17 00:00:00 2001 From: itsToggle <71379623+itsToggle@users.noreply.github.com> Date: Sun, 22 Jan 2023 11:27:39 +0100 Subject: [PATCH 525/560] clean sorting file --- sorting.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sorting.txt b/sorting.txt index 8a6f30177c478..81b805dd2b3a2 100644 --- a/sorting.txt +++ b/sorting.txt @@ -28,8 +28,3 @@ # ~~~ recorded/manual changes to the structure: ~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -/Family.Guy.S19.1080p.HULU.WEBRip.DDP5.1.x264-CtrlHD[rartv] -> /shows/another rename -/shows/Neuer Ordner -/shows/Neuer Ordner -> /shows/fuck -/Neuer Ordner -/Our.Universe.S01.1080p.NF.WEBRip.DDP5.1.Atmos.x264-SMURF[rartv] -> /shows/fuck/shit From 82ce4868343a3ba60aa4068442682deac91fffae Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sun, 22 Jan 2023 14:45:36 +0100 Subject: [PATCH 526/560] fix renaming causing directory to be empty --- .vscode/launch.json | 14 ++++++++++++-- backend/realdebrid/realdebrid.go | 8 ++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 4eb5cdc5404e6..e07fcb6267651 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -2,13 +2,23 @@ "version": "0.2.0", "configurations": [ { - "name": "Launch", + "name": "Move", "type": "go", "request": "launch", "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["tree","realdebrid:"],//["moveto","realdebrid:/shows/some shit/Family.Guy.S19E01.1080p.HULU.WEB-DL.DDP5.1.H.264-CtrlHD.mkv","realdebrid:/shows/some other shit/Family.Guy.S19E01.1080p.HULU.WEB-DL.DDP5.1.H.264-CtrlHD.mkv"],//["move","realdebrid:/shows/Family.Guy.S19.1080p.HULU.WEBRip.DDP5.1.x264-CtrlHD[rartv]","realdebrid:/shows/some shit"],//["moveto","realdebrid:/shows/Our.Universe/IVE RENAMED YOU BITCH/Our.Universe.S01E01.Chasing.Starlight.1080p.NF.WEB-DL.DDP5.1.Atmos.H.264-SMURF.mkv","realdebrid:/new folder/someshit.mkv"], //["tree","realdebrid:"],// + "args": ["moveto","realdebrid:/shows/Our.Universe.S01.1080p.NF.WEBRip.DDP5.1.Atmos.x264-SMURF[rartv]","realdebrid:/shows/fuck/shit"], + "console": "integratedTerminal" + }, + { + "name": "List", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}", + "env": {}, + "args": ["tree","realdebrid:"], "console": "integratedTerminal" } ] diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 211001c194beb..b70ba06bfd25e 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -1285,6 +1285,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string if err != nil { return err } + + // List the files to again to make them visible in the new location instantly + if last := dstDirectoryID[len(dstDirectoryID)-1]; last != '/' { + dstDirectoryID = dstDirectoryID + "/" + } + + f.List(ctx, dstDirectoryID+dstLeaf) + srcFs.dirCache.FlushDir(srcRemote) return nil } From e000e3b7a10e94bd7c9a6ea7d7a7978ad24d5605 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Wed, 22 Feb 2023 15:54:48 +0100 Subject: [PATCH 527/560] file oriented artificial sorting --- .vscode/launch.json | 12 +- backend/realdebrid/api/types.go | 1 + backend/realdebrid/realdebrid.go | 581 ++++++++++++++++--------------- sorting.txt | 2 +- 4 files changed, 316 insertions(+), 280 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index e07fcb6267651..8f5fbab6549b3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["moveto","realdebrid:/shows/Our.Universe.S01.1080p.NF.WEBRip.DDP5.1.Atmos.x264-SMURF[rartv]","realdebrid:/shows/fuck/shit"], + "args": ["moveto","realdebrid:/default/kafferep-cletus[PRiME].mkv","realdebrid:/default/Cast Away 2002/"], "console": "integratedTerminal" }, { @@ -20,6 +20,16 @@ "env": {}, "args": ["tree","realdebrid:"], "console": "integratedTerminal" + }, + { + "name": "Mkdir", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}", + "env": {}, + "args": ["mkdir","realdebrid:/new"], + "console": "integratedTerminal" } ] } \ No newline at end of file diff --git a/backend/realdebrid/api/types.go b/backend/realdebrid/api/types.go index 4e31ec2d3cfc2..753ff0e0271c8 100644 --- a/backend/realdebrid/api/types.go +++ b/backend/realdebrid/api/types.go @@ -38,6 +38,7 @@ type Item struct { Link string `json:"download,omitempty"` OriginalLink string `json:"link,omitempty"` Name string `json:"filename,omitempty"` + DefaultLocation string `` Size int64 `json:"filesize,omitempty"` Status string `json:"status,omitempty"` StreamLink string `` diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index b70ba06bfd25e..e0d27ae545261 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -54,7 +54,7 @@ const ( minSleep = 10 * time.Millisecond maxSleep = 2 * time.Second decayConstant = 2 // bigger for slower decay, exponential - rootID = "0" // ID of root folder is always this + rootID = "/" // ID of root folder is always this rootURL = "https://api.real-debrid.com/rest/1.0" ) @@ -80,11 +80,11 @@ var broken_torrents []string var lastcheck int64 = time.Now().Unix() var lastFileMod int64 = 0 var interval int64 = 15 * 60 -var structure = make(map[string]string) -var levels = make(map[string][]string) +var mapping = make(map[string]string) +var sorting_file = make(map[string]string) var folders = make(map[string][]api.Item) var regex_folders = make(map[string]string) -var id2name = make(map[string]string) +var regex_defs = make(map[string]*regexp.Regexp) var move_chars = " -> " var regx_chars = " == " var default_sorting = `# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -106,7 +106,7 @@ var default_sorting = `# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Example: /Our.Universe.S01.1080p.[rartv] -> /shows/Our Universe/Season 1 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# ~~~~~~~~~~~ manual and regex folders: ~~~~~~~~~~~ +# ~~~~~~~~~ top level and regex folders: ~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /shows == (?i)(S[0-9]{2}|SEASONS?.[0-9]|COMPLETE|[^457a-z\W\s]-[0-9]+) @@ -412,7 +412,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin // CreateDir makes a directory with pathID as parent and name leaf func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, err error) { - if len(dirID) > 0 && dirID != "0" { + if len(dirID) > 0 && dirID != "/" { if first := dirID[0]; first != '/' { dirID = "/" + dirID } @@ -420,7 +420,11 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e dirID = dirID + "/" } } else { - dirID = "/" + err := fmt.Errorf("cant create directories in root directory. this is reserved for regex folders") + return "", err + } + if last := leaf[len(leaf)-1]; last != '/' { + leaf = leaf + "/" } file, err := os.OpenFile(f.opt.SortFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { @@ -568,42 +572,61 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi var result []api.Item var resp *http.Response - if (dirID == rootID) || !(f.folder_exists(dirID)) { - - //create folder structure - // - // Open the sorting file - - file, err := os.Open(f.opt.SortFile) - if os.IsNotExist(err) { - fmt.Println("creating default sorting file") - file, err = os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) - if err != nil { - fmt.Println(err) + if _, ok := folders[dirID]; !ok { + if _, ok := folders[dirID+"/"]; ok { + dirID = dirID + "/" + } else if _, ok := folders["/"+dirID+"/"]; ok { + dirID = "/" + dirID + "/" + } else if len(dirID) > 1 { + if last := dirID[len(dirID)-1]; last == '/' { + if _, ok := folders[dirID[:len(dirID)-1]]; ok { + dirID = dirID[:len(dirID)-1] + } } - defer file.Close() + } + } + // Open the sorting file - if _, err := file.WriteString(default_sorting); err != nil { - fmt.Println(err) - } - } else if err != nil { + file, err := os.Open(f.opt.SortFile) + if os.IsNotExist(err) { + fs.LogPrint(fs.LogLevelNotice, "Creating sorting file.") + file, err = os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + fmt.Println(err) + } + defer file.Close() + + if _, err := file.WriteString(default_sorting); err != nil { fmt.Println(err) - } else { - defer file.Close() } + } else if err != nil { + fmt.Println(err) + } else { + defer file.Close() + } - fileInfo, _ := file.Stat() - fileModTime := fileInfo.ModTime().Unix() + var updated = false + fileInfo, _ := file.Stat() + fileModTime := fileInfo.ModTime().Unix() + + if fileModTime > lastFileMod { + updated = true + } - if fileModTime > lastFileMod { + if (dirID == rootID) || !(f.folder_exists(dirID)) || updated { + // Create folder structure + // + if updated { // Reset saved folder structure - fmt.Println("reading updated sorting file") - structure = make(map[string]string) - levels = make(map[string][]string) + fs.LogPrint(fs.LogLevelInfo, "Reading updated sorting file.") folders = make(map[string][]api.Item) regex_folders = make(map[string]string) - + regex_defs = make(map[string]*regexp.Regexp) + mapping = make(map[string]string) + sorting_file = make(map[string]string) + // Flush the directory cache + f.dirCache.Flush() // Read the file line by line scanner := bufio.NewScanner(file) for scanner.Scan() { @@ -615,57 +638,27 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Split the line by " -> " parts := strings.Split(scanner.Text(), move_chars) // Add the key-value pair to the map - structure[parts[0]] = parts[1] + mapping[parts[0]] = parts[1] } else if strings.Contains(scanner.Text(), regx_chars) { // Split the line by " -> " parts := strings.Split(scanner.Text(), regx_chars) // Add the key-value pair to the map regex_folders[parts[0]] = parts[1] } else { - structure[scanner.Text()] = scanner.Text() + mapping[scanner.Text()] = scanner.Text() } } - // Iterate through the map - for _, newLocation := range structure { - - // Split the new location by "/" - locationParts := strings.Split(newLocation, "/") - - // Create a new variable to store the full path - var location string - - // Iterate through each level of the new location - for _, locationPart := range locationParts { - // Append the full path to the corresponding level - var skip bool - skip = false - for _, val := range levels[location] { - if val == locationPart { - skip = true - break - } - } - if skip { - location = location + locationPart + "/" - continue - } - - levels[location] = append(levels[location], locationPart) - - // Append the current location part to the full path - location = location + locationPart + "/" - } + //create regex definitions + for folder, regex_def := range regex_folders { + r, _ := regexp.Compile(regex_def) + regex_defs[folder] = r } - } - - //create regex definitions + for key, value := range mapping { + sorting_file[key] = value + } - var regex_defs = make(map[string]*regexp.Regexp) - for folder, regex_def := range regex_folders { - r, _ := regexp.Compile(regex_def) - regex_defs[folder] = r } //update global cached list @@ -679,6 +672,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi var newcached []api.Item var totalcount int var printed = false + totalcount = 2 for len(newcached) < totalcount { partialresult = nil @@ -695,12 +689,13 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if err == nil { if totalcount != len(cached) || time.Now().Unix()-lastcheck > interval { if time.Now().Unix()-lastcheck > interval && !printed { - fmt.Println("Updating links and torrents") + fs.LogPrint(fs.LogLevelInfo, "Updating all links and torrents.") printed = true } newcached = append(newcached, partialresult...) opts.Parameters.Set("offset", strconv.Itoa(len(newcached))) opts.Parameters.Set("limit", "2500") + updated = true } else { newcached = cached } @@ -741,6 +736,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi newtorrents = append(newtorrents, partialresult...) opts.Parameters.Set("offset", strconv.Itoa(len(newtorrents))) opts.Parameters.Set("limit", "2500") + updated = true } else { newtorrents = torrents } @@ -751,218 +747,196 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi break } } + // Set everything as being up to date lastcheck = time.Now().Unix() + lastFileMod = fileModTime //fmt.Printf("Done.\n") torrents = newtorrents - //Iterate through built file and torrent list: - // - //build system folder/file structure + // Iterate through built file and torrent list: var broken = false - for i, torrent := range torrents { - //handle dead torrents - broken = false - for _, TorrentID := range broken_torrents { - if torrent.ID == TorrentID { - broken = true + if updated { + for i := range torrents { + //handle dead torrents + broken = false + for _, TorrentID := range broken_torrents { + if torrents[i].ID == TorrentID { + broken = true + } } - } - if torrent.Status == "dead" || broken { - torrents[i] = f.redownloadTorrent(ctx, torrent) - } - //if folders is not up to date: - if fileModTime > lastFileMod { - if i == 1 { //len(torrents)-1 { - lastFileMod = fileModTime + if torrents[i].Status == "dead" || broken { + torrents[i] = f.redownloadTorrent(ctx, torrents[i]) } - } else { - continue - } - //add torrents to their cahnged folders, create new ones in standard locations - var parentdir string - var dir string - if _, ok := structure["/"+torrent.Name]; ok { - parentdir = strings.Join(strings.Split(structure["/"+torrent.Name], "/")[:len(strings.Split(structure["/"+torrent.Name], "/"))-1], "/") + "/" - dir = structure["/"+torrent.Name] - } else { - //handle regex defined folers - parentdir = "/default/" + //set default torrents[i] location + torrents[i].DefaultLocation = "/default/" for folder, r := range regex_defs { - match := r.MatchString(torrent.Name) + match := r.MatchString(torrents[i].Name) if match { - parentdir = folder + "/" + torrents[i].DefaultLocation = folder + "/" break } } - dir = parentdir + torrent.Name - } - locationParts := strings.Split(dir, "/") - var location string - for _, locationPart := range locationParts { - // Append the full path to the corresponding level - var skip bool - skip = false - for _, val := range levels[location] { - if val == locationPart { - skip = true - break - } - } - if skip { - location = location + locationPart + "/" - continue - } - levels[location] = append(levels[location], locationPart) - location = location + locationPart + "/" - } - var folder api.Item - folder.ID = dir - if _, ok := structure["/"+torrent.Name]; ok { - folder.Name = strings.Split(structure["/"+torrent.Name], "/")[len(strings.Split(structure["/"+torrent.Name], "/"))-1] - } else { - folder.Name = torrent.Name - } - folder.Type = "folder" - folder.Generated = torrent.Generated - folder.Ended = torrent.Ended - folders[parentdir] = append(folders[parentdir], folder) - id2name[torrent.ID] = torrent.Name - //iterate through files - var broken = false - for _, link := range torrent.Links { - var ItemFile api.Item - for _, cachedfile := range cached { - if cachedfile.OriginalLink == link { - ItemFile = cachedfile - break + //iterate through files + var broken = false + for _, link := range torrents[i].Links { + if link == "" { + continue } - } - if ItemFile.Link == "" { - //fmt.Printf("Creating new unrestricted direct link for: '%s'\n", torrent.Name) - path = "/unrestrict/link" - method = "POST" - opts := rest.Opts{ - Method: method, - Path: path, - MultipartParams: url.Values{ - "link": {link}, - }, - Parameters: f.baseParams(), + var ItemFile api.Item + for _, cachedfile := range cached { + if cachedfile.OriginalLink == link { + ItemFile = cachedfile + break + } } - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp.StatusCode == 503 { - broken = true - break + if ItemFile.Name == "" { + continue } - var retries = 0 - for resp.StatusCode == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) + if ItemFile.Link == "" { + fs.LogPrint(fs.LogLevelDebug, "Creating new unrestricted direct link for torrent: "+torrents[i].Name) + path = "/unrestrict/link" + method = "POST" + opts := rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "link": {link}, + }, + Parameters: f.baseParams(), + } resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - retries += 1 + if resp.StatusCode == 503 { + broken = true + break + } + var retries = 0 + for resp.StatusCode == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + retries += 1 + } } - } - ItemFile.ParentID = torrent.ID - ItemFile.TorrentHash = torrent.TorrentHash - ItemFile.Generated = torrent.Generated - ItemFile.Type = "file" - var fileparentdir = dir - if _, ok := structure["/"+torrent.Name+"/"+ItemFile.Name]; ok { - fileparentdir = strings.Join(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[:len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1], "/") + "/" - ItemFile.Name = strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1] - } - folders[fileparentdir] = append(folders[fileparentdir], ItemFile) - } - if broken { - torrents[i] = f.redownloadTorrent(ctx, torrent) - torrent = torrents[i] - for _, link := range torrent.Links { - var ItemFile api.Item - //fmt.Printf("Creating new unrestricted direct link for: '%s'\n", torrent.Name) - path = "/unrestrict/link" - method = "POST" - opts := rest.Opts{ - Method: method, - Path: path, - MultipartParams: url.Values{ - "link": {link}, - }, - Parameters: f.baseParams(), + mapping_id := "/" + torrents[i].Name + "/" + ItemFile.Name + ItemFile.DefaultLocation = torrents[i].DefaultLocation + torrents[i].Name + "/" + ItemFile.ParentID = torrents[i].ID + ItemFile.TorrentHash = torrents[i].TorrentHash + ItemFile.Generated = torrents[i].Generated + ItemFile.Type = "file" + if _, ok := mapping[mapping_id]; !ok { + if _, ok := mapping["/"+torrents[i].Name+"/"]; ok { + mapping[mapping_id] = mapping["/"+torrents[i].Name+"/"] + } else { + mapping[mapping_id] = ItemFile.DefaultLocation + } + } else { + ItemFile.Name = strings.Split(mapping[mapping_id], "/")[len(strings.Split(mapping[mapping_id], "/"))-1] + if ItemFile.Name == "" { + continue + } + mapping[mapping_id] = strings.Join(strings.Split(mapping[mapping_id], "/")[:len(strings.Split(mapping[mapping_id], "/"))-1], "/") + "/" } - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - var retries = 0 - for resp.StatusCode == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) + folders[mapping[mapping_id]] = append(folders[mapping[mapping_id]], ItemFile) + } + if broken { + torrents[i] = f.redownloadTorrent(ctx, torrents[i]) + for _, link := range torrents[i].Links { + var ItemFile api.Item + fs.LogPrint(fs.LogLevelDebug, "Creating new unrestricted direct link for torrent: "+torrents[i].Name) + path = "/unrestrict/link" + method = "POST" + opts := rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "link": {link}, + }, + Parameters: f.baseParams(), + } resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - retries += 1 - } - ItemFile.ParentID = torrent.ID - ItemFile.TorrentHash = torrent.TorrentHash - ItemFile.Generated = torrent.Generated - ItemFile.Type = "file" - var fileparentdir = dir - if _, ok := structure["/"+torrent.Name+"/"+ItemFile.Name]; ok { - fileparentdir = strings.Join(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[:len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1], "/") + "/" - ItemFile.Name = strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/")[len(strings.Split(structure["/"+torrent.Name+"/"+ItemFile.Name], "/"))-1] + var retries = 0 + for resp.StatusCode == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + retries += 1 + } + if ItemFile.Name == "" { + continue + } + mapping_id := "/" + torrents[i].Name + "/" + ItemFile.Name + ItemFile.DefaultLocation = torrents[i].DefaultLocation + torrents[i].Name + "/" + ItemFile.ParentID = torrents[i].ID + ItemFile.TorrentHash = torrents[i].TorrentHash + ItemFile.Generated = torrents[i].Generated + ItemFile.Type = "file" + if _, ok := mapping[mapping_id]; !ok { + if _, ok := mapping["/"+torrents[i].Name+"/"]; ok { + mapping[mapping_id] = mapping["/"+torrents[i].Name+"/"] + } else { + mapping[mapping_id] = ItemFile.DefaultLocation + } + } else { + ItemFile.Name = strings.Split(mapping[mapping_id], "/")[len(strings.Split(mapping[mapping_id], "/"))-1] + if ItemFile.Name == "" { + continue + } + mapping[mapping_id] = strings.Join(strings.Split(mapping[mapping_id], "/")[:len(strings.Split(mapping[mapping_id], "/"))-1], "/") + "/" + } + folders[mapping[mapping_id]] = append(folders[mapping[mapping_id]], ItemFile) } - folders[fileparentdir] = append(folders[fileparentdir], ItemFile) } } - if i > 0 { - break - } - } - for key, level := range levels { - for _, foldername := range level { - var skip = false - for _, folder := range folders[key] { - if foldername == folder.Name { + + // Iterate through the map + for _, newLocation := range mapping { + + // Split the new location by "/" + locationParts := strings.Split(newLocation, "/") + + // Create a new variable to store the full path + var location string + + // Iterate through each level of the new location + for _, locationPart := range locationParts { + // Append the full path to the corresponding level + var skip bool + skip = false + for _, val := range folders[location] { + if val.Name == locationPart { + skip = true + break + } + } + if len(location) > 0 { + if last := location[len(location)-1]; last != '/' { + skip = true + } + } else { skip = true - break } + if skip { + location = location + locationPart + "/" + continue + } + + // Create the missing folders + var ItemFolder api.Item + ItemFolder.Name = locationPart + ItemFolder.ID = location + locationPart + ItemFolder.Type = "folder" + if locationPart != "" { + folders[location] = append(folders[location], ItemFolder) + } + // Append the current location part to the full path + location = location + locationPart + "/" } - if skip { - continue - } - var folder api.Item - folder.ID = key + foldername + "/" - folder.Name = foldername - folder.Type = "folder" - folder.Generated = "1969-04-20T16:20:00.000Z" - folders[key] = append(folders[key], folder) - //folders[folder.ID] = []api.Item{} } - } - //return root folder structure if dirID == rootID - if dirID == rootID { - result = append(result, folders["/"]...) - } - } - if f.folder_exists(dirID) { - //return dirID folder structure - result = append(result, folders[dirID]...) - } else if dirID != rootID { - //handle edge cases - if f.folder_exists(dirID + "/") { - fmt.Println("changed regested dirID: " + dirID + " to: " + dirID + "/ to find content.") - result = append(result, folders[dirID+"/"]...) - } else if f.folder_exists(dirID[:len(dirID)-1]) { - fmt.Println("changed regested dirID: " + dirID + " to: " + dirID[:len(dirID)-1] + " to find content.") - result = append(result, folders[dirID[:len(dirID)-1]]...) - } else { - var parentdirID = strings.Join(strings.Split(dirID, "/")[:len(strings.Split(dirID, "/"))-1], "/") - if last := parentdirID[len(parentdirID)-1]; last != '/' { - parentdirID = parentdirID + "/" - } - if first := parentdirID[0]; first != '/' { - parentdirID = "/" + parentdirID - } - fmt.Println("changed regested dirID: " + dirID + " to: " + parentdirID + " to find content.") - result = append(result, folders[parentdirID]...) } } + result = append(result, folders[dirID]...) + if err != nil { return newDirID, found, fmt.Errorf("couldn't list files: %w", err) } @@ -1160,10 +1134,8 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // move a file or folder // -// This is complicated by the fact that there is an API to move files -// between directories and a separate one to rename them. We try to -// call the minimum number of API calls. func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDirectoryID, newDirectoryID string) (err error) { + // Handle IDs for _, torrent := range torrents { if torrent.ID == oldDirectoryID { oldDirectoryID = "/" + torrent.Name + "/" @@ -1173,8 +1145,14 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir break } } - if newDirectoryID == "0" { - newDirectoryID = "/" + // Handle Files + if isFile && len(id) == 13 { + for _, file := range cached { + if file.ID == id { + oldLeaf = file.Name + break + } + } } // Open the file file, err := os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) @@ -1186,23 +1164,39 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir if last := newDirectoryID[len(newDirectoryID)-1]; last != '/' { newDirectoryID = newDirectoryID + "/" } + if first := newDirectoryID[0]; first != '/' { + newDirectoryID = "/" + newDirectoryID + } + if last := oldDirectoryID[len(oldDirectoryID)-1]; last != '/' { + oldDirectoryID = oldDirectoryID + "/" + } if first := oldDirectoryID[0]; first != '/' { oldDirectoryID = "/" + oldDirectoryID } - searchString := move_chars + oldDirectoryID + oldLeaf - retryString := oldDirectoryID + oldLeaf + move_chars + newDirectoryID + newLeaf - newString := move_chars + newDirectoryID + newLeaf + if !isFile { + if last := newLeaf[len(newLeaf)-1]; last != '/' { + newLeaf = newLeaf + "/" + } + if last := oldLeaf[len(oldLeaf)-1]; last != '/' { + oldLeaf = oldLeaf + "/" + } + } + // Get all files that should be moved. + var affected_items []string + for key, value := range sorting_file { + if strings.Contains(value, oldDirectoryID+oldLeaf) { + affected_items = append(affected_items, key) + } + } + if len(affected_items) == 0 { + affected_items = append(affected_items, oldDirectoryID+oldLeaf) + } scanner := bufio.NewScanner(file) var lines []string + var new_lines = make(map[string]string) var replaced = false for scanner.Scan() { line := scanner.Text() - if strings.Contains(line, searchString) { - line = strings.Replace(line, searchString, newString, -1) - replaced = true - } else if strings.Contains(line, retryString) { - replaced = true - } lines = append(lines, line) } if err := scanner.Err(); err != nil { @@ -1211,12 +1205,32 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir } file.Truncate(0) file.Seek(0, 0) - if !replaced { - var line = oldDirectoryID + oldLeaf + move_chars + newDirectoryID + newLeaf - lines = append(lines, line) + // move all affected items + for _, affected_item := range affected_items { + for _, line := range lines { + if strings.Contains(line, affected_item+move_chars) && !isFile { + new_dst := strings.Replace(strings.Split(line, move_chars)[len(strings.Split(line, move_chars))-1], oldDirectoryID+oldLeaf, newDirectoryID+newLeaf, -1) + new_lines[line] = affected_item + move_chars + new_dst + replaced = true + } else if strings.Contains(line, affected_item+move_chars) && isFile { + new_lines[line] = affected_item + move_chars + newDirectoryID + newLeaf + replaced = true + } else if line == affected_item { + new_lines[line] = newDirectoryID + newLeaf + replaced = true + } + } + if !replaced { + var line = affected_item + move_chars + newDirectoryID + newLeaf + lines = append(lines, line) + } } for _, line := range lines { - _, err := file.WriteString(line + "\n") + write := line + if _, ok := new_lines[line]; ok { + write = new_lines[line] + } + _, err := file.WriteString(write + "\n") if err != nil { fmt.Println(err) return err @@ -1290,6 +1304,9 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string if last := dstDirectoryID[len(dstDirectoryID)-1]; last != '/' { dstDirectoryID = dstDirectoryID + "/" } + if first := dstDirectoryID[0]; first != '/' { + dstDirectoryID = "/" + dstDirectoryID + } f.List(ctx, dstDirectoryID+dstLeaf) @@ -1417,7 +1434,6 @@ func (o *Object) Storable() bool { // Open an object for read func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { - fmt.Println("opening file: " + o.id + " with parent: " + o.ParentID + " using link: " + o.url) if o.url == "" { return nil, errors.New("can't download - no URL") } @@ -1438,14 +1454,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read return shouldRetry(ctx, resp, err) }) if err != nil { - if err_code == 503 { + if err_code == 503 || err_code == 404 { for _, TorrentID := range broken_torrents { if o.ParentID == TorrentID { return nil, err } } - fmt.Println("Error opening file: '" + o.url + "'.") - fmt.Println("This link seems to be broken. Torrent will be re-downloaded on next refresh.") + err = fmt.Errorf("Error opening file: '" + o.url + "'. This link seems to be broken. Torrent will be re-downloaded on next refresh.") broken_torrents = append(broken_torrents, o.ParentID) } return nil, err @@ -1477,10 +1492,17 @@ func (f *Fs) remove(ctx context.Context, id ...string) (err error) { var resp *http.Response var result api.Response var retries = 0 + err_code := 0 resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) - for resp.StatusCode == 429 && retries <= 5 { + if resp != nil { + err_code = resp.StatusCode + } + for err_code == 429 && retries <= 5 { time.Sleep(time.Duration(2) * time.Second) resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) + if resp != nil { + err_code = resp.StatusCode + } retries += 1 } path = "/torrents/delete/" + id[1] @@ -1490,7 +1512,10 @@ func (f *Fs) remove(ctx context.Context, id ...string) (err error) { Parameters: f.baseParams(), } resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) - if resp.StatusCode == 429 { + if resp != nil { + err_code = resp.StatusCode + } + if err_code == 429 { time.Sleep(time.Duration(2) * time.Second) _, _ = f.srv.CallJSON(ctx, &opts, nil, &result) } diff --git a/sorting.txt b/sorting.txt index 81b805dd2b3a2..ea7c039acd44f 100644 --- a/sorting.txt +++ b/sorting.txt @@ -17,7 +17,7 @@ # Example: /Our.Universe.S01.1080p.[rartv] -> /shows/Our Universe/Season 1 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# ~~~~~~~~~~~ manual and regex folders: ~~~~~~~~~~~ +# ~~~~~~~~~ top level and regex folders: ~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /shows == (?i)(S[0-9]{2}|SEASONS?.[0-9]|COMPLETE|[^457a-z\W\s]-[0-9]+) From a33d93dfb03a47c15741ef6b373fd91bf8aa3e40 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Wed, 22 Feb 2023 20:03:54 +0100 Subject: [PATCH 528/560] fix crash on line 832, no reponse status code erros --- backend/realdebrid/realdebrid.go | 64 +++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index e0d27ae545261..4e8e4a759e63f 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -442,7 +442,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e // Redownload a dead torrent func (f *Fs) redownloadTorrent(ctx context.Context, torrent api.Item) (redownloaded_torrent api.Item) { - fmt.Println("Redownloading dead torrent: " + torrent.Name) + fs.LogPrint(fs.LogLevelNotice, "Redownloading dead torrent: "+torrent.Name) //Get dead torrent file and hash info var method = "GET" var path = "/torrents/info/" + torrent.ID @@ -473,10 +473,17 @@ func (f *Fs) redownloadTorrent(ctx context.Context, torrent api.Item) (redownloa var resp *http.Response var result api.Response var retries = 0 + var err_code = 0 resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) - for resp.StatusCode == 429 && retries <= 5 { + if resp != nil { + err_code = resp.StatusCode + } + for err_code == 429 && retries <= 5 { time.Sleep(time.Duration(2) * time.Second) resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) + if resp != nil { + err_code = resp.StatusCode + } retries += 1 } cached[i].OriginalLink = "this-is-not-a-link" @@ -672,16 +679,22 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi var newcached []api.Item var totalcount int var printed = false - + var err_code = 429 totalcount = 2 for len(newcached) < totalcount { partialresult = nil resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) var retries = 0 - for resp.StatusCode == 429 && retries <= 5 { + if resp != nil { + err_code = resp.StatusCode + } + for err_code == 429 && retries <= 5 { partialresult = nil time.Sleep(time.Duration(2) * time.Second) resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + if resp != nil { + err_code = resp.StatusCode + } retries += 1 } if err == nil { @@ -719,14 +732,21 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi opts.Parameters.Set("limit", "1") var newtorrents []api.Item totalcount = 2 + err_code = 429 for len(newtorrents) < totalcount { partialresult = nil resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + if resp != nil { + err_code = resp.StatusCode + } var retries = 0 - for resp.StatusCode == 429 && retries <= 5 { + for err_code == 429 && retries <= 5 { partialresult = nil time.Sleep(time.Duration(2) * time.Second) resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + if resp != nil { + err_code = resp.StatusCode + } retries += 1 } if err == nil { @@ -779,6 +799,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi //iterate through files var broken = false for _, link := range torrents[i].Links { + err_code = 0 if link == "" { continue } @@ -805,14 +826,20 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi Parameters: f.baseParams(), } resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp.StatusCode == 503 { + if resp != nil { + err_code = resp.StatusCode + } + if err_code == 503 || err_code == 404 { broken = true break } var retries = 0 - for resp.StatusCode == 429 && retries <= 5 { + for err_code == 429 && retries <= 5 { time.Sleep(time.Duration(2) * time.Second) resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } retries += 1 } } @@ -829,7 +856,12 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi mapping[mapping_id] = ItemFile.DefaultLocation } } else { - ItemFile.Name = strings.Split(mapping[mapping_id], "/")[len(strings.Split(mapping[mapping_id], "/"))-1] + if len(mapping[mapping_id]) > 0 { + split := strings.Split(mapping[mapping_id], "/") + if len(split) > 0 { + ItemFile.Name = split[len(strings.Split(mapping[mapping_id], "/"))-1] + } + } if ItemFile.Name == "" { continue } @@ -840,6 +872,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if broken { torrents[i] = f.redownloadTorrent(ctx, torrents[i]) for _, link := range torrents[i].Links { + err_code = 0 var ItemFile api.Item fs.LogPrint(fs.LogLevelDebug, "Creating new unrestricted direct link for torrent: "+torrents[i].Name) path = "/unrestrict/link" @@ -853,10 +886,16 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi Parameters: f.baseParams(), } resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } var retries = 0 - for resp.StatusCode == 429 && retries <= 5 { + for err_code == 429 && retries <= 5 { time.Sleep(time.Duration(2) * time.Second) resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } retries += 1 } if ItemFile.Name == "" { @@ -875,7 +914,12 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi mapping[mapping_id] = ItemFile.DefaultLocation } } else { - ItemFile.Name = strings.Split(mapping[mapping_id], "/")[len(strings.Split(mapping[mapping_id], "/"))-1] + if len(mapping[mapping_id]) > 0 { + split := strings.Split(mapping[mapping_id], "/") + if len(split) > 0 { + ItemFile.Name = split[len(strings.Split(mapping[mapping_id], "/"))-1] + } + } if ItemFile.Name == "" { continue } From 271afc6dc08b154d8a9bc96fefd930bc6aecb654 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Fri, 24 Feb 2023 12:48:54 +0100 Subject: [PATCH 529/560] single file deletion --- .vscode/launch.json | 16 ++- backend/realdebrid/api/types.go | 1 + backend/realdebrid/realdebrid.go | 203 +++++++++++++++++++++++-------- 3 files changed, 168 insertions(+), 52 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 8f5fbab6549b3..241d8ca8afeee 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["moveto","realdebrid:/default/kafferep-cletus[PRiME].mkv","realdebrid:/default/Cast Away 2002/"], + "args": ["moveto","realdebrid:/default/kafferep-cletus[PRiME].mkv/kafferep-cletus[PRiME].mkv","realdebrid:/default/Cast Away 2002/kafferep-cletus[PRiME].mkv"], "console": "integratedTerminal" }, { @@ -18,7 +18,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["tree","realdebrid:"], + "args": ["tree","realdebrid:/default/"], "console": "integratedTerminal" }, { @@ -30,6 +30,16 @@ "env": {}, "args": ["mkdir","realdebrid:/new"], "console": "integratedTerminal" - } + }, + { + "name": "Remove", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}", + "env": {}, + "args": ["delete","realdebrid:/default/Scrubs/Scrubs S01E01 My First Day (c) [1080p x265 10bit Joy].mkv"], + "console": "integratedTerminal" + }, ] } \ No newline at end of file diff --git a/backend/realdebrid/api/types.go b/backend/realdebrid/api/types.go index 753ff0e0271c8..ce6397188e42c 100644 --- a/backend/realdebrid/api/types.go +++ b/backend/realdebrid/api/types.go @@ -39,6 +39,7 @@ type Item struct { OriginalLink string `json:"link,omitempty"` Name string `json:"filename,omitempty"` DefaultLocation string `` + MappingID string `` Size int64 `json:"filesize,omitempty"` Status string `json:"status,omitempty"` StreamLink string `` diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 4e8e4a759e63f..665631bc67d1f 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -29,6 +29,7 @@ import ( "regexp" "strconv" "strings" + "sync" "time" "github.com/rclone/rclone/backend/realdebrid/api" @@ -80,11 +81,13 @@ var broken_torrents []string var lastcheck int64 = time.Now().Unix() var lastFileMod int64 = 0 var interval int64 = 15 * 60 +var sequential = &sync.RWMutex{} var mapping = make(map[string]string) var sorting_file = make(map[string]string) var folders = make(map[string][]api.Item) var regex_folders = make(map[string]string) var regex_defs = make(map[string]*regexp.Regexp) +var trash_indicator = ".trashed" var move_chars = " -> " var regx_chars = " == " var default_sorting = `# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -132,7 +135,7 @@ func init() { Default: "", }, { Name: "sort_file", - Help: `please provide the full path to a file that should be used for sorting`, + Help: `please provide the full path to a file (file does not need to exist) that should be used for sorting`, Advanced: true, Default: "./sorting.txt", }, { @@ -179,6 +182,7 @@ type Object struct { mimeType string // Mime type of object url string // URL to download file TorrentHash string // Torrent Hash + MappingID string // Internal Mapping ID used for sorting } // ------------------------------------------------------------ @@ -573,6 +577,7 @@ type listAllFn func(*api.Item) bool // // It returns a newDirID which is what the system returned as the directory ID func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (newDirID string, found bool, err error) { + path := "/downloads" method := "GET" var partialresult []api.Item @@ -596,7 +601,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi file, err := os.Open(f.opt.SortFile) if os.IsNotExist(err) { - fs.LogPrint(fs.LogLevelNotice, "Creating sorting file.") + fs.LogPrint(fs.LogLevelWarning, "no sorting file found. creating new empty sorting file.") file, err = os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) if err != nil { fmt.Println(err) @@ -625,8 +630,9 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Create folder structure // if updated { + // Reset saved folder structure - fs.LogPrint(fs.LogLevelInfo, "Reading updated sorting file.") + fs.LogPrint(fs.LogLevelInfo, "reading updated sorting file.") folders = make(map[string][]api.Item) regex_folders = make(map[string]string) regex_defs = make(map[string]*regexp.Regexp) @@ -810,11 +816,8 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi break } } - if ItemFile.Name == "" { - continue - } if ItemFile.Link == "" { - fs.LogPrint(fs.LogLevelDebug, "Creating new unrestricted direct link for torrent: "+torrents[i].Name) + fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) path = "/unrestrict/link" method = "POST" opts := rest.Opts{ @@ -843,7 +846,11 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi retries += 1 } } + if ItemFile.Name == "" { + continue + } mapping_id := "/" + torrents[i].Name + "/" + ItemFile.Name + ItemFile.MappingID = mapping_id ItemFile.DefaultLocation = torrents[i].DefaultLocation + torrents[i].Name + "/" ItemFile.ParentID = torrents[i].ID ItemFile.TorrentHash = torrents[i].TorrentHash @@ -862,7 +869,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi ItemFile.Name = split[len(strings.Split(mapping[mapping_id], "/"))-1] } } - if ItemFile.Name == "" { + if ItemFile.Name == "" || strings.HasSuffix(ItemFile.Name, trash_indicator) { continue } mapping[mapping_id] = strings.Join(strings.Split(mapping[mapping_id], "/")[:len(strings.Split(mapping[mapping_id], "/"))-1], "/") + "/" @@ -874,7 +881,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi for _, link := range torrents[i].Links { err_code = 0 var ItemFile api.Item - fs.LogPrint(fs.LogLevelDebug, "Creating new unrestricted direct link for torrent: "+torrents[i].Name) + fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) path = "/unrestrict/link" method = "POST" opts := rest.Opts{ @@ -902,6 +909,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi continue } mapping_id := "/" + torrents[i].Name + "/" + ItemFile.Name + ItemFile.MappingID = mapping_id ItemFile.DefaultLocation = torrents[i].DefaultLocation + torrents[i].Name + "/" ItemFile.ParentID = torrents[i].ID ItemFile.TorrentHash = torrents[i].TorrentHash @@ -920,7 +928,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi ItemFile.Name = split[len(strings.Split(mapping[mapping_id], "/"))-1] } } - if ItemFile.Name == "" { + if ItemFile.Name == "" || strings.HasSuffix(ItemFile.Name, trash_indicator) { continue } mapping[mapping_id] = strings.Join(strings.Split(mapping[mapping_id], "/")[:len(strings.Split(mapping[mapping_id], "/"))-1], "/") + "/" @@ -933,6 +941,10 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Iterate through the map for _, newLocation := range mapping { + if strings.HasSuffix(newLocation, trash_indicator) { + continue + } + // Split the new location by "/" locationParts := strings.Split(newLocation, "/") @@ -1127,7 +1139,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error { // purgeCheck removes the root directory, if check is set then it // refuses to do so if it has anything in func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { - //fmt.Printf("Purging torrent: '%s'\n", rootID) root := path.Join(f.root, dir) if root == "" { return errors.New("can't purge root directory") @@ -1137,18 +1148,24 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { if err != nil { return err } - path := "/torrents/delete/" + rootID - opts := rest.Opts{ - Method: "DELETE", - Path: path, - Parameters: f.baseParams(), + + // if rootID is a torrent ID + if len(rootID) == 13 && rootID == strings.ToUpper(rootID) { + fs.LogPrint(fs.LogLevelDebug, "removing realdebrid torrent id: "+rootID) + path := "/torrents/delete/" + rootID + opts := rest.Opts{ + Method: "DELETE", + Path: path, + Parameters: f.baseParams(), + } + var resp *http.Response + var result api.Response + err = f.pacer.Call(func() (bool, error) { + resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) + return shouldRetry(ctx, resp, err) + }) + lastcheck = time.Now().Unix() - interval } - var resp *http.Response - var result api.Response - err = f.pacer.Call(func() (bool, error) { - resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) - return shouldRetry(ctx, resp, err) - }) f.dirCache.FlushDir(dir) return nil } @@ -1293,6 +1310,8 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir // // If it isn't possible then return fs.ErrorCantMove func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + sequential.Lock() + defer sequential.Unlock() srcObj, ok := src.(*Object) if !ok { fs.Debugf(src, "Can't move - not same remote type") @@ -1312,7 +1331,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, } err = dstObj.readMetaData(ctx) - if err != nil { + if err != nil && !strings.HasSuffix(remote, trash_indicator) { return nil, err } return dstObj, nil @@ -1327,6 +1346,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, // // If destination exists then return fs.ErrorDirExists func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { + sequential.Lock() + defer sequential.Unlock() srcFs, ok := src.(*Fs) if !ok { fs.Debugf(srcFs, "Can't move directory - not same remote type") @@ -1435,6 +1456,7 @@ func (o *Object) setMetaData(info *api.Item) (err error) { o.url = info.Link o.ParentID = info.ParentID o.TorrentHash = info.TorrentHash + o.MappingID = info.MappingID return nil } @@ -1521,40 +1543,127 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return nil } -// Remove an object by ID -func (f *Fs) remove(ctx context.Context, id ...string) (err error) { - //fmt.Printf("Removing direct link id: '%s'\n", id[0]) - //if f.opt.RootFolderID == "torrents" { - // fmt.Printf("Removing torrent id: '%s'\n", id[1]) - //} - path := "/downloads/delete/" + id[0] - opts := rest.Opts{ - Method: "DELETE", - Path: path, - Parameters: f.baseParams(), - } +// Remove an object by ID (always a file) +func (f *Fs) remove(ctx context.Context, o *Object) (err error) { var resp *http.Response var result api.Response var retries = 0 - err_code := 0 - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) - if resp != nil { - err_code = resp.StatusCode + var oldDirectoryID = "" + var torrent_files = 0 + // Get parent torrent item + for i := range torrents { + if torrents[i].ID == o.ParentID { + oldDirectoryID = "/" + torrents[i].Name + "/" + for _, link := range torrents[i].Links { + if link != "" { + torrent_files += 1 + } + } + break + } } - for err_code == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) + + // lock + sequential.Lock() + defer sequential.Unlock() + + // Get all trashed files + var affected_items []string + for key, value := range sorting_file { + if strings.Contains(key, oldDirectoryID) { + if strings.HasSuffix(value, trash_indicator) { + affected_items = append(affected_items, key) + } + } + } + if len(affected_items) == 0 { + affected_items = append(affected_items, o.MappingID) + } + + // if not all files are trashed + if len(affected_items) < torrent_files { + // move file to trash + fs.LogPrint(fs.LogLevelDebug, "moving file: "+o.MappingID+" to internal trash") + sequential.Unlock() + f.Move(ctx, o, o.remote+trash_indicator) + + // delete the link on realdebrid + fs.LogPrint(fs.LogLevelDebug, "removing realdebrid link id: "+o.id) + path := "/downloads/delete/" + o.id + opts := rest.Opts{ + Method: "DELETE", + Path: path, + Parameters: f.baseParams(), + } + err_code := 0 resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) if resp != nil { err_code = resp.StatusCode } - retries += 1 + for err_code == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) + if resp != nil { + err_code = resp.StatusCode + } + retries += 1 + } + return nil } - path = "/torrents/delete/" + id[1] - opts = rest.Opts{ + + // if all files are trashed + fs.LogPrint(fs.LogLevelDebug, "all files of torrent: "+o.ParentID+" are in internal trash") + + // read sort file + file, err := os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + fmt.Println(err) + return err + } + defer file.Close() + + // delete mappings and torrent if all files are trashed + scanner := bufio.NewScanner(file) + var lines []string + var new_lines = make(map[string]string) + for scanner.Scan() { + line := scanner.Text() + lines = append(lines, line) + } + if err := scanner.Err(); err != nil { + fmt.Println(err) + return err + } + file.Truncate(0) + file.Seek(0, 0) + + // move all affected items + for _, affected_item := range affected_items { + for _, line := range lines { + if strings.Contains(line, affected_item+move_chars) { + new_lines[line] = "" + } + } + } + for _, line := range lines { + write := line + if _, ok := new_lines[line]; ok { + continue + } + _, err := file.WriteString(write + "\n") + if err != nil { + fmt.Println(err) + return err + } + } + fs.LogPrint(fs.LogLevelDebug, "removing realdebrid torrent id: "+o.ParentID) + path := "/torrents/delete/" + o.ParentID + opts := rest.Opts{ Method: "DELETE", Path: path, Parameters: f.baseParams(), } + err_code := 0 resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) if resp != nil { err_code = resp.StatusCode @@ -1574,11 +1683,7 @@ func (o *Object) Remove(ctx context.Context) error { if err != nil { return fmt.Errorf("Remove: Failed to read metadata: %w", err) } - if o.ParentID != "" { - return o.fs.remove(ctx, o.id, o.ParentID) - } else { - return o.fs.remove(ctx, o.id) - } + return o.fs.remove(ctx, o) } // MimeType of an Object if known, "" otherwise From 6d15ac67081ec071d2aa6d81d1b67cc4f660bde9 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Fri, 24 Feb 2023 15:24:47 +0100 Subject: [PATCH 530/560] unrestrict links on individual directory access --- .vscode/launch.json | 2 +- backend/realdebrid/realdebrid.go | 171 +++++++++++++++---------------- 2 files changed, 81 insertions(+), 92 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 241d8ca8afeee..a125173fa2967 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -18,7 +18,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["tree","realdebrid:/default/"], + "args": ["tree","realdebrid:/movies/"], "console": "integratedTerminal" }, { diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 665631bc67d1f..3f7624c560425 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -803,7 +803,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } } //iterate through files - var broken = false for _, link := range torrents[i].Links { err_code = 0 if link == "" { @@ -816,36 +815,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi break } } - if ItemFile.Link == "" { - fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) - path = "/unrestrict/link" - method = "POST" - opts := rest.Opts{ - Method: method, - Path: path, - MultipartParams: url.Values{ - "link": {link}, - }, - Parameters: f.baseParams(), - } - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - if err_code == 503 || err_code == 404 { - broken = true - break - } - var retries = 0 - for err_code == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - retries += 1 - } - } if ItemFile.Name == "" { continue } @@ -876,66 +845,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } folders[mapping[mapping_id]] = append(folders[mapping[mapping_id]], ItemFile) } - if broken { - torrents[i] = f.redownloadTorrent(ctx, torrents[i]) - for _, link := range torrents[i].Links { - err_code = 0 - var ItemFile api.Item - fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) - path = "/unrestrict/link" - method = "POST" - opts := rest.Opts{ - Method: method, - Path: path, - MultipartParams: url.Values{ - "link": {link}, - }, - Parameters: f.baseParams(), - } - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - var retries = 0 - for err_code == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - retries += 1 - } - if ItemFile.Name == "" { - continue - } - mapping_id := "/" + torrents[i].Name + "/" + ItemFile.Name - ItemFile.MappingID = mapping_id - ItemFile.DefaultLocation = torrents[i].DefaultLocation + torrents[i].Name + "/" - ItemFile.ParentID = torrents[i].ID - ItemFile.TorrentHash = torrents[i].TorrentHash - ItemFile.Generated = torrents[i].Generated - ItemFile.Type = "file" - if _, ok := mapping[mapping_id]; !ok { - if _, ok := mapping["/"+torrents[i].Name+"/"]; ok { - mapping[mapping_id] = mapping["/"+torrents[i].Name+"/"] - } else { - mapping[mapping_id] = ItemFile.DefaultLocation - } - } else { - if len(mapping[mapping_id]) > 0 { - split := strings.Split(mapping[mapping_id], "/") - if len(split) > 0 { - ItemFile.Name = split[len(strings.Split(mapping[mapping_id], "/"))-1] - } - } - if ItemFile.Name == "" || strings.HasSuffix(ItemFile.Name, trash_indicator) { - continue - } - mapping[mapping_id] = strings.Join(strings.Split(mapping[mapping_id], "/")[:len(strings.Split(mapping[mapping_id], "/"))-1], "/") + "/" - } - folders[mapping[mapping_id]] = append(folders[mapping[mapping_id]], ItemFile) - } - } + } // Iterate through the map @@ -991,6 +901,85 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } + for i := range folders[dirID] { + broken := false + err_code := 0 + if folders[dirID][i].Link == "" && folders[dirID][i].Type == "file" { + fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for file: "+folders[dirID][i].Name) + path = "/unrestrict/link" + method = "POST" + opts := rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "link": {folders[dirID][i].OriginalLink}, + }, + Parameters: f.baseParams(), + } + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &folders[dirID][i]) + if resp != nil { + err_code = resp.StatusCode + } + if err_code == 503 || err_code == 404 { + broken = true + break + } + var retries = 0 + for err_code == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &folders[dirID][i]) + if resp != nil { + err_code = resp.StatusCode + } + retries += 1 + } + } + if broken { + for j := range torrents { + if torrents[j].ID == folders[dirID][i].ParentID { + var old_links []string + old_links = append(old_links, torrents[j].Links...) + torrents[j] = f.redownloadTorrent(ctx, torrents[j]) + for k, link := range torrents[j].Links { + for l := range folders[dirID] { + if folders[dirID][l].OriginalLink == old_links[k] && old_links[k] != "" { + var ItemFile api.Item + err_code = 0 + fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+folders[dirID][l].Name) + path = "/unrestrict/link" + method = "POST" + opts := rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "link": {link}, + }, + Parameters: f.baseParams(), + } + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } + var retries = 0 + for err_code == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } + retries += 1 + } + folders[dirID][l].Link = ItemFile.Link + folders[dirID][l].OriginalLink = ItemFile.OriginalLink + folders[dirID][l].ParentID = ItemFile.ParentID + } + } + } + } + } + } + } + result = append(result, folders[dirID]...) if err != nil { From b2bb571e79fb538f65abf97f45d83948e64df116 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Fri, 24 Feb 2023 17:11:51 +0100 Subject: [PATCH 531/560] revert latest changes --- .vscode/launch.json | 2 +- backend/realdebrid/realdebrid.go | 218 ++++++++++++++++--------------- 2 files changed, 114 insertions(+), 106 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index a125173fa2967..241d8ca8afeee 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -18,7 +18,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["tree","realdebrid:/movies/"], + "args": ["tree","realdebrid:/default/"], "console": "integratedTerminal" }, { diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 3f7624c560425..5e2d8926fd03c 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -597,8 +597,8 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } } } - // Open the sorting file + // Open the sorting file file, err := os.Open(f.opt.SortFile) if os.IsNotExist(err) { fs.LogPrint(fs.LogLevelWarning, "no sorting file found. creating new empty sorting file.") @@ -628,7 +628,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if (dirID == rootID) || !(f.folder_exists(dirID)) || updated { // Create folder structure - // if updated { // Reset saved folder structure @@ -781,39 +780,128 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Iterate through built file and torrent list: var broken = false - if updated { - for i := range torrents { - //handle dead torrents - broken = false - for _, TorrentID := range broken_torrents { - if torrents[i].ID == TorrentID { + for i := range torrents { + //handle dead torrents + broken = false + for _, TorrentID := range broken_torrents { + if torrents[i].ID == TorrentID { + broken = true + } + } + if torrents[i].Status == "dead" || broken { + torrents[i] = f.redownloadTorrent(ctx, torrents[i]) + } + //set default torrents[i] location + torrents[i].DefaultLocation = "/default/" + for folder, r := range regex_defs { + match := r.MatchString(torrents[i].Name) + if match { + torrents[i].DefaultLocation = folder + "/" + break + } + } + //iterate through files + var broken = false + for _, link := range torrents[i].Links { + err_code = 0 + if link == "" { + continue + } + var ItemFile api.Item + for _, cachedfile := range cached { + if cachedfile.OriginalLink == link { + ItemFile = cachedfile + break + } + } + if ItemFile.Link == "" { + fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) + path = "/unrestrict/link" + method = "POST" + opts := rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "link": {link}, + }, + Parameters: f.baseParams(), + } + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } + if err_code == 503 || err_code == 404 { broken = true + break + } + var retries = 0 + for err_code == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } + retries += 1 } } - if torrents[i].Status == "dead" || broken { - torrents[i] = f.redownloadTorrent(ctx, torrents[i]) + if ItemFile.Name == "" { + continue } - //set default torrents[i] location - torrents[i].DefaultLocation = "/default/" - for folder, r := range regex_defs { - match := r.MatchString(torrents[i].Name) - if match { - torrents[i].DefaultLocation = folder + "/" - break + mapping_id := "/" + torrents[i].Name + "/" + ItemFile.Name + ItemFile.MappingID = mapping_id + ItemFile.DefaultLocation = torrents[i].DefaultLocation + torrents[i].Name + "/" + ItemFile.ParentID = torrents[i].ID + ItemFile.TorrentHash = torrents[i].TorrentHash + ItemFile.Generated = torrents[i].Generated + ItemFile.Type = "file" + if _, ok := mapping[mapping_id]; !ok { + if _, ok := mapping["/"+torrents[i].Name+"/"]; ok { + mapping[mapping_id] = mapping["/"+torrents[i].Name+"/"] + } else { + mapping[mapping_id] = ItemFile.DefaultLocation + } + } else { + if len(mapping[mapping_id]) > 0 { + split := strings.Split(mapping[mapping_id], "/") + if len(split) > 0 { + ItemFile.Name = split[len(strings.Split(mapping[mapping_id], "/"))-1] + } + } + if ItemFile.Name == "" || strings.HasSuffix(ItemFile.Name, trash_indicator) { + continue } + mapping[mapping_id] = strings.Join(strings.Split(mapping[mapping_id], "/")[:len(strings.Split(mapping[mapping_id], "/"))-1], "/") + "/" } - //iterate through files + folders[mapping[mapping_id]] = append(folders[mapping[mapping_id]], ItemFile) + } + if broken { + torrents[i] = f.redownloadTorrent(ctx, torrents[i]) for _, link := range torrents[i].Links { err_code = 0 - if link == "" { - continue - } var ItemFile api.Item - for _, cachedfile := range cached { - if cachedfile.OriginalLink == link { - ItemFile = cachedfile - break + fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) + path = "/unrestrict/link" + method = "POST" + opts := rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "link": {link}, + }, + Parameters: f.baseParams(), + } + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } + var retries = 0 + for err_code == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode } + retries += 1 } if ItemFile.Name == "" { continue @@ -845,7 +933,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } folders[mapping[mapping_id]] = append(folders[mapping[mapping_id]], ItemFile) } - } // Iterate through the map @@ -901,85 +988,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } - for i := range folders[dirID] { - broken := false - err_code := 0 - if folders[dirID][i].Link == "" && folders[dirID][i].Type == "file" { - fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for file: "+folders[dirID][i].Name) - path = "/unrestrict/link" - method = "POST" - opts := rest.Opts{ - Method: method, - Path: path, - MultipartParams: url.Values{ - "link": {folders[dirID][i].OriginalLink}, - }, - Parameters: f.baseParams(), - } - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &folders[dirID][i]) - if resp != nil { - err_code = resp.StatusCode - } - if err_code == 503 || err_code == 404 { - broken = true - break - } - var retries = 0 - for err_code == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &folders[dirID][i]) - if resp != nil { - err_code = resp.StatusCode - } - retries += 1 - } - } - if broken { - for j := range torrents { - if torrents[j].ID == folders[dirID][i].ParentID { - var old_links []string - old_links = append(old_links, torrents[j].Links...) - torrents[j] = f.redownloadTorrent(ctx, torrents[j]) - for k, link := range torrents[j].Links { - for l := range folders[dirID] { - if folders[dirID][l].OriginalLink == old_links[k] && old_links[k] != "" { - var ItemFile api.Item - err_code = 0 - fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+folders[dirID][l].Name) - path = "/unrestrict/link" - method = "POST" - opts := rest.Opts{ - Method: method, - Path: path, - MultipartParams: url.Values{ - "link": {link}, - }, - Parameters: f.baseParams(), - } - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - var retries = 0 - for err_code == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - retries += 1 - } - folders[dirID][l].Link = ItemFile.Link - folders[dirID][l].OriginalLink = ItemFile.OriginalLink - folders[dirID][l].ParentID = ItemFile.ParentID - } - } - } - } - } - } - } - result = append(result, folders[dirID]...) if err != nil { From b6595240077f3317b94e3f480710d674b546e639 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sat, 25 Feb 2023 15:17:57 +0100 Subject: [PATCH 532/560] more precise mutex locks/unlocks --- backend/realdebrid/realdebrid.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 5e2d8926fd03c..610d46db946d3 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -81,7 +81,7 @@ var broken_torrents []string var lastcheck int64 = time.Now().Unix() var lastFileMod int64 = 0 var interval int64 = 15 * 60 -var sequential = &sync.RWMutex{} +var sequential_maps = &sync.RWMutex{} var mapping = make(map[string]string) var sorting_file = make(map[string]string) var folders = make(map[string][]api.Item) @@ -629,7 +629,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Create folder structure if updated { - + sequential_maps.Lock() // Reset saved folder structure fs.LogPrint(fs.LogLevelInfo, "reading updated sorting file.") folders = make(map[string][]api.Item) @@ -670,7 +670,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi for key, value := range mapping { sorting_file[key] = value } - + sequential_maps.Unlock() } //update global cached list @@ -1307,26 +1307,28 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir // // If it isn't possible then return fs.ErrorCantMove func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { - sequential.Lock() - defer sequential.Unlock() + sequential_maps.Lock() srcObj, ok := src.(*Object) if !ok { fs.Debugf(src, "Can't move - not same remote type") + sequential_maps.Unlock() return nil, fs.ErrorCantMove } // Create temporary object dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) if err != nil { + sequential_maps.Unlock() return nil, err } // Do the move err = f.move(ctx, true, srcObj.id, path.Base(srcObj.remote), leaf, srcObj.ParentID, directoryID) if err != nil { + sequential_maps.Unlock() return nil, err } - + sequential_maps.Unlock() err = dstObj.readMetaData(ctx) if err != nil && !strings.HasSuffix(remote, trash_indicator) { return nil, err @@ -1343,22 +1345,24 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, // // If destination exists then return fs.ErrorDirExists func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { - sequential.Lock() - defer sequential.Unlock() + sequential_maps.Lock() srcFs, ok := src.(*Fs) if !ok { fs.Debugf(srcFs, "Can't move directory - not same remote type") + sequential_maps.Unlock() return fs.ErrorCantDirMove } srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) if err != nil { + sequential_maps.Unlock() return err } // Do the move err = f.move(ctx, false, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID) if err != nil { + sequential_maps.Unlock() return err } @@ -1369,7 +1373,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string if first := dstDirectoryID[0]; first != '/' { dstDirectoryID = "/" + dstDirectoryID } - + sequential_maps.Unlock() f.List(ctx, dstDirectoryID+dstLeaf) srcFs.dirCache.FlushDir(srcRemote) @@ -1561,8 +1565,8 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { } // lock - sequential.Lock() - defer sequential.Unlock() + sequential_maps.Lock() + defer sequential_maps.Unlock() // Get all trashed files var affected_items []string @@ -1581,7 +1585,7 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { if len(affected_items) < torrent_files { // move file to trash fs.LogPrint(fs.LogLevelDebug, "moving file: "+o.MappingID+" to internal trash") - sequential.Unlock() + sequential_maps.Unlock() f.Move(ctx, o, o.remote+trash_indicator) // delete the link on realdebrid From 1c01365e50d212bdd2623f1de144f9e753e118fb Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sat, 25 Feb 2023 16:30:34 +0100 Subject: [PATCH 533/560] use sync.Map instead of mutex --- backend/realdebrid/realdebrid.go | 374 ++++++++++++++++--------------- sorting.txt | 2 + 2 files changed, 197 insertions(+), 179 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 610d46db946d3..ef482c483f22d 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -81,12 +81,11 @@ var broken_torrents []string var lastcheck int64 = time.Now().Unix() var lastFileMod int64 = 0 var interval int64 = 15 * 60 -var sequential_maps = &sync.RWMutex{} -var mapping = make(map[string]string) -var sorting_file = make(map[string]string) -var folders = make(map[string][]api.Item) -var regex_folders = make(map[string]string) -var regex_defs = make(map[string]*regexp.Regexp) +var mapping sync.Map +var sorting_file sync.Map +var folders sync.Map +var regex_folders sync.Map +var regex_defs sync.Map var trash_indicator = ".trashed" var move_chars = " -> " var regx_chars = " == " @@ -557,7 +556,7 @@ func (f *Fs) folder_exists(dirID string) bool { if dirID == "/" { return false } - if _, ok := folders[dirID]; ok { + if _, ok := folders.Load(dirID); ok { return true } return false @@ -584,21 +583,21 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi var result []api.Item var resp *http.Response - if _, ok := folders[dirID]; !ok { - if _, ok := folders[dirID+"/"]; ok { + if _, ok := folders.Load(dirID); !ok { + if _, ok := folders.Load(dirID + "/"); ok { dirID = dirID + "/" - } else if _, ok := folders["/"+dirID+"/"]; ok { + } else if _, ok := folders.Load("/" + dirID + "/"); ok { dirID = "/" + dirID + "/" } else if len(dirID) > 1 { if last := dirID[len(dirID)-1]; last == '/' { - if _, ok := folders[dirID[:len(dirID)-1]]; ok { + if _, ok := folders.Load(dirID[:len(dirID)-1]); ok { dirID = dirID[:len(dirID)-1] } } } } - // Open the sorting file + file, err := os.Open(f.opt.SortFile) if os.IsNotExist(err) { fs.LogPrint(fs.LogLevelWarning, "no sorting file found. creating new empty sorting file.") @@ -628,15 +627,16 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if (dirID == rootID) || !(f.folder_exists(dirID)) || updated { // Create folder structure + // if updated { - sequential_maps.Lock() + // Reset saved folder structure fs.LogPrint(fs.LogLevelInfo, "reading updated sorting file.") - folders = make(map[string][]api.Item) - regex_folders = make(map[string]string) - regex_defs = make(map[string]*regexp.Regexp) - mapping = make(map[string]string) - sorting_file = make(map[string]string) + folders = sync.Map{} + regex_folders = sync.Map{} + regex_defs = sync.Map{} + mapping = sync.Map{} + sorting_file = sync.Map{} // Flush the directory cache f.dirCache.Flush() // Read the file line by line @@ -650,27 +650,29 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Split the line by " -> " parts := strings.Split(scanner.Text(), move_chars) // Add the key-value pair to the map - mapping[parts[0]] = parts[1] + mapping.Store(parts[0], parts[1]) } else if strings.Contains(scanner.Text(), regx_chars) { // Split the line by " -> " parts := strings.Split(scanner.Text(), regx_chars) // Add the key-value pair to the map - regex_folders[parts[0]] = parts[1] + mapping.Store(parts[0], parts[1]) } else { - mapping[scanner.Text()] = scanner.Text() + mapping.Store(scanner.Text(), scanner.Text()) } } //create regex definitions - for folder, regex_def := range regex_folders { - r, _ := regexp.Compile(regex_def) - regex_defs[folder] = r - } + regex_folders.Range(func(folder, regex_def interface{}) bool { + r, _ := regexp.Compile(regex_def.(string)) + regex_defs.Store(folder, r) + return true + }) + + mapping.Range(func(key, value interface{}) bool { + sorting_file.Store(key, value) + return true + }) - for key, value := range mapping { - sorting_file[key] = value - } - sequential_maps.Unlock() } //update global cached list @@ -780,128 +782,71 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Iterate through built file and torrent list: var broken = false - for i := range torrents { - //handle dead torrents - broken = false - for _, TorrentID := range broken_torrents { - if torrents[i].ID == TorrentID { - broken = true - } - } - if torrents[i].Status == "dead" || broken { - torrents[i] = f.redownloadTorrent(ctx, torrents[i]) - } - //set default torrents[i] location - torrents[i].DefaultLocation = "/default/" - for folder, r := range regex_defs { - match := r.MatchString(torrents[i].Name) - if match { - torrents[i].DefaultLocation = folder + "/" - break - } - } - //iterate through files - var broken = false - for _, link := range torrents[i].Links { - err_code = 0 - if link == "" { - continue - } - var ItemFile api.Item - for _, cachedfile := range cached { - if cachedfile.OriginalLink == link { - ItemFile = cachedfile - break - } - } - if ItemFile.Link == "" { - fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) - path = "/unrestrict/link" - method = "POST" - opts := rest.Opts{ - Method: method, - Path: path, - MultipartParams: url.Values{ - "link": {link}, - }, - Parameters: f.baseParams(), - } - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - if err_code == 503 || err_code == 404 { + if updated { + for i := range torrents { + //handle dead torrents + broken = false + for _, TorrentID := range broken_torrents { + if torrents[i].ID == TorrentID { broken = true - break - } - var retries = 0 - for err_code == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - retries += 1 } } - if ItemFile.Name == "" { - continue + if torrents[i].Status == "dead" || broken { + torrents[i] = f.redownloadTorrent(ctx, torrents[i]) } - mapping_id := "/" + torrents[i].Name + "/" + ItemFile.Name - ItemFile.MappingID = mapping_id - ItemFile.DefaultLocation = torrents[i].DefaultLocation + torrents[i].Name + "/" - ItemFile.ParentID = torrents[i].ID - ItemFile.TorrentHash = torrents[i].TorrentHash - ItemFile.Generated = torrents[i].Generated - ItemFile.Type = "file" - if _, ok := mapping[mapping_id]; !ok { - if _, ok := mapping["/"+torrents[i].Name+"/"]; ok { - mapping[mapping_id] = mapping["/"+torrents[i].Name+"/"] - } else { - mapping[mapping_id] = ItemFile.DefaultLocation - } - } else { - if len(mapping[mapping_id]) > 0 { - split := strings.Split(mapping[mapping_id], "/") - if len(split) > 0 { - ItemFile.Name = split[len(strings.Split(mapping[mapping_id], "/"))-1] - } - } - if ItemFile.Name == "" || strings.HasSuffix(ItemFile.Name, trash_indicator) { - continue + //set default torrents[i] location + torrents[i].DefaultLocation = "/default/" + regex_defs.Range(func(folder, r interface{}) bool { + match := r.(*regexp.Regexp).MatchString(torrents[i].Name) + if match { + torrents[i].DefaultLocation = folder.(string) + "/" + return false } - mapping[mapping_id] = strings.Join(strings.Split(mapping[mapping_id], "/")[:len(strings.Split(mapping[mapping_id], "/"))-1], "/") + "/" - } - folders[mapping[mapping_id]] = append(folders[mapping[mapping_id]], ItemFile) - } - if broken { - torrents[i] = f.redownloadTorrent(ctx, torrents[i]) + return true + }) + //iterate through files + var broken = false for _, link := range torrents[i].Links { err_code = 0 - var ItemFile api.Item - fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) - path = "/unrestrict/link" - method = "POST" - opts := rest.Opts{ - Method: method, - Path: path, - MultipartParams: url.Values{ - "link": {link}, - }, - Parameters: f.baseParams(), + if link == "" { + continue } - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode + var ItemFile api.Item + for _, cachedfile := range cached { + if cachedfile.OriginalLink == link { + ItemFile = cachedfile + break + } } - var retries = 0 - for err_code == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) + if ItemFile.Link == "" { + fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) + path = "/unrestrict/link" + method = "POST" + opts := rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "link": {link}, + }, + Parameters: f.baseParams(), + } resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) if resp != nil { err_code = resp.StatusCode } - retries += 1 + if err_code == 503 || err_code == 404 { + broken = true + break + } + var retries = 0 + for err_code == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } + retries += 1 + } } if ItemFile.Name == "" { continue @@ -913,37 +858,113 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi ItemFile.TorrentHash = torrents[i].TorrentHash ItemFile.Generated = torrents[i].Generated ItemFile.Type = "file" - if _, ok := mapping[mapping_id]; !ok { - if _, ok := mapping["/"+torrents[i].Name+"/"]; ok { - mapping[mapping_id] = mapping["/"+torrents[i].Name+"/"] + if _, ok := mapping.Load(mapping_id); !ok { + if _, ok := mapping.Load("/" + torrents[i].Name + "/"); ok { + value, _ := mapping.Load("/" + torrents[i].Name + "/") + mapping.Store(mapping_id, value) } else { - mapping[mapping_id] = ItemFile.DefaultLocation + mapping.Store(mapping_id, ItemFile.DefaultLocation) } } else { - if len(mapping[mapping_id]) > 0 { - split := strings.Split(mapping[mapping_id], "/") + if value, _ := mapping.Load(mapping_id); len(value.(string)) > 0 { + value, _ := mapping.Load(mapping_id) + split := strings.Split(value.(string), "/") if len(split) > 0 { - ItemFile.Name = split[len(strings.Split(mapping[mapping_id], "/"))-1] + ItemFile.Name = split[len(strings.Split(value.(string), "/"))-1] } } if ItemFile.Name == "" || strings.HasSuffix(ItemFile.Name, trash_indicator) { continue } - mapping[mapping_id] = strings.Join(strings.Split(mapping[mapping_id], "/")[:len(strings.Split(mapping[mapping_id], "/"))-1], "/") + "/" + value, _ := mapping.Load(mapping_id) + mapping.Store(mapping_id, strings.Join(strings.Split(value.(string), "/")[:len(strings.Split(value.(string), "/"))-1], "/")+"/") + } + value, _ := mapping.Load(mapping_id) + list, ok := folders.Load(value) + if !ok { + list = []api.Item{} + } + folders.Store(value, append(list.([]api.Item), ItemFile)) + } + if broken { + torrents[i] = f.redownloadTorrent(ctx, torrents[i]) + for _, link := range torrents[i].Links { + err_code = 0 + var ItemFile api.Item + fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) + path = "/unrestrict/link" + method = "POST" + opts := rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "link": {link}, + }, + Parameters: f.baseParams(), + } + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } + var retries = 0 + for err_code == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } + retries += 1 + } + if ItemFile.Name == "" { + continue + } + mapping_id := "/" + torrents[i].Name + "/" + ItemFile.Name + ItemFile.MappingID = mapping_id + ItemFile.DefaultLocation = torrents[i].DefaultLocation + torrents[i].Name + "/" + ItemFile.ParentID = torrents[i].ID + ItemFile.TorrentHash = torrents[i].TorrentHash + ItemFile.Generated = torrents[i].Generated + ItemFile.Type = "file" + if _, ok := mapping.Load(mapping_id); !ok { + if _, ok := mapping.Load("/" + torrents[i].Name + "/"); ok { + value, _ := mapping.Load("/" + torrents[i].Name + "/") + mapping.Store(mapping_id, value) + } else { + mapping.Store(mapping_id, ItemFile.DefaultLocation) + } + } else { + if value, _ := mapping.Load(mapping_id); len(value.(string)) > 0 { + value, _ := mapping.Load(mapping_id) + split := strings.Split(value.(string), "/") + if len(split) > 0 { + ItemFile.Name = split[len(strings.Split(value.(string), "/"))-1] + } + } + if ItemFile.Name == "" || strings.HasSuffix(ItemFile.Name, trash_indicator) { + continue + } + value, _ := mapping.Load(mapping_id) + mapping.Store(mapping_id, strings.Join(strings.Split(value.(string), "/")[:len(strings.Split(value.(string), "/"))-1], "/")+"/") + } + value, _ := mapping.Load(mapping_id) + list, ok := folders.Load(value) + if !ok { + list = []api.Item{} + } + folders.Store(value, append(list.([]api.Item), ItemFile)) } - folders[mapping[mapping_id]] = append(folders[mapping[mapping_id]], ItemFile) } } // Iterate through the map - for _, newLocation := range mapping { + mapping.Range(func(_, newLocation interface{}) bool { - if strings.HasSuffix(newLocation, trash_indicator) { - continue + if strings.HasSuffix(newLocation.(string), trash_indicator) { + return true } // Split the new location by "/" - locationParts := strings.Split(newLocation, "/") + locationParts := strings.Split(newLocation.(string), "/") // Create a new variable to store the full path var location string @@ -953,7 +974,11 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Append the full path to the corresponding level var skip bool skip = false - for _, val := range folders[location] { + list, ok := folders.Load(location) + if !ok { + list = []api.Item{} + } + for _, val := range list.([]api.Item) { if val.Name == locationPart { skip = true break @@ -977,18 +1002,20 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi ItemFolder.ID = location + locationPart ItemFolder.Type = "folder" if locationPart != "" { - folders[location] = append(folders[location], ItemFolder) + folders.Store(location, append(list.([]api.Item), ItemFolder)) } // Append the current location part to the full path location = location + locationPart + "/" } - } + return true + }) } } - result = append(result, folders[dirID]...) + value, _ := folders.Load(dirID) + result = append(result, value.([]api.Item)...) if err != nil { return newDirID, found, fmt.Errorf("couldn't list files: %w", err) @@ -1241,11 +1268,12 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir } // Get all files that should be moved. var affected_items []string - for key, value := range sorting_file { - if strings.Contains(value, oldDirectoryID+oldLeaf) { - affected_items = append(affected_items, key) + sorting_file.Range(func(key, value interface{}) bool { + if strings.Contains(value.(string), oldDirectoryID+oldLeaf) { + affected_items = append(affected_items, key.(string)) } - } + return true + }) if len(affected_items) == 0 { affected_items = append(affected_items, oldDirectoryID+oldLeaf) } @@ -1307,28 +1335,24 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir // // If it isn't possible then return fs.ErrorCantMove func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { - sequential_maps.Lock() srcObj, ok := src.(*Object) if !ok { fs.Debugf(src, "Can't move - not same remote type") - sequential_maps.Unlock() return nil, fs.ErrorCantMove } // Create temporary object dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) if err != nil { - sequential_maps.Unlock() return nil, err } // Do the move err = f.move(ctx, true, srcObj.id, path.Base(srcObj.remote), leaf, srcObj.ParentID, directoryID) if err != nil { - sequential_maps.Unlock() return nil, err } - sequential_maps.Unlock() + err = dstObj.readMetaData(ctx) if err != nil && !strings.HasSuffix(remote, trash_indicator) { return nil, err @@ -1345,24 +1369,20 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, // // If destination exists then return fs.ErrorDirExists func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { - sequential_maps.Lock() srcFs, ok := src.(*Fs) if !ok { fs.Debugf(srcFs, "Can't move directory - not same remote type") - sequential_maps.Unlock() return fs.ErrorCantDirMove } srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) if err != nil { - sequential_maps.Unlock() return err } // Do the move err = f.move(ctx, false, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID) if err != nil { - sequential_maps.Unlock() return err } @@ -1373,7 +1393,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string if first := dstDirectoryID[0]; first != '/' { dstDirectoryID = "/" + dstDirectoryID } - sequential_maps.Unlock() + f.List(ctx, dstDirectoryID+dstLeaf) srcFs.dirCache.FlushDir(srcRemote) @@ -1564,19 +1584,16 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { } } - // lock - sequential_maps.Lock() - defer sequential_maps.Unlock() - // Get all trashed files var affected_items []string - for key, value := range sorting_file { - if strings.Contains(key, oldDirectoryID) { - if strings.HasSuffix(value, trash_indicator) { - affected_items = append(affected_items, key) + sorting_file.Range(func(key, value interface{}) bool { + if strings.Contains(key.(string), oldDirectoryID) { + if strings.HasSuffix(value.(string), trash_indicator) { + affected_items = append(affected_items, key.(string)) } } - } + return true + }) if len(affected_items) == 0 { affected_items = append(affected_items, o.MappingID) } @@ -1585,7 +1602,6 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { if len(affected_items) < torrent_files { // move file to trash fs.LogPrint(fs.LogLevelDebug, "moving file: "+o.MappingID+" to internal trash") - sequential_maps.Unlock() f.Move(ctx, o, o.remote+trash_indicator) // delete the link on realdebrid diff --git a/sorting.txt b/sorting.txt index ea7c039acd44f..0bc28f6f29660 100644 --- a/sorting.txt +++ b/sorting.txt @@ -28,3 +28,5 @@ # ~~~ recorded/manual changes to the structure: ~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +/default/Cast Away 2002/ +/kafferep-cletus[PRiME].mkv/kafferep-cletus[PRiME].mkv -> /default/Cast Away 2002/kafferep-cletus[PRiME].mkv From 43558e1d9c786a656d6aaf75473d79c6701aa286 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sat, 25 Feb 2023 16:35:18 +0100 Subject: [PATCH 534/560] fix regex folders --- backend/realdebrid/realdebrid.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index ef482c483f22d..a942d52f707a4 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -652,10 +652,10 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Add the key-value pair to the map mapping.Store(parts[0], parts[1]) } else if strings.Contains(scanner.Text(), regx_chars) { - // Split the line by " -> " + // Split the line by " == " parts := strings.Split(scanner.Text(), regx_chars) // Add the key-value pair to the map - mapping.Store(parts[0], parts[1]) + regex_folders.Store(parts[0], parts[1]) } else { mapping.Store(scanner.Text(), scanner.Text()) } From 5d4015990a8827c3e6a3a46cb42befa30e07a270 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sat, 25 Feb 2023 16:52:28 +0100 Subject: [PATCH 535/560] clear sync.maps w/ creating race conditions --- backend/realdebrid/realdebrid.go | 22 +++++++++++++++------- sorting.txt | 2 -- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index a942d52f707a4..4649719447870 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -562,6 +562,14 @@ func (f *Fs) folder_exists(dirID string) bool { return false } +// Clean sync.map without creating race conditions +func eraseSyncMap(m sync.Map) { + m.Range(func(key interface{}, value interface{}) bool { + m.Delete(key) + return true + }) +} + // list the objects into the function supplied // // If directories is set it only sends directories @@ -631,12 +639,12 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if updated { // Reset saved folder structure - fs.LogPrint(fs.LogLevelInfo, "reading updated sorting file.") - folders = sync.Map{} - regex_folders = sync.Map{} - regex_defs = sync.Map{} - mapping = sync.Map{} - sorting_file = sync.Map{} + fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file.") + eraseSyncMap(folders) + eraseSyncMap(regex_folders) + eraseSyncMap(regex_defs) + eraseSyncMap(mapping) + eraseSyncMap(sorting_file) // Flush the directory cache f.dirCache.Flush() // Read the file line by line @@ -709,7 +717,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if err == nil { if totalcount != len(cached) || time.Now().Unix()-lastcheck > interval { if time.Now().Unix()-lastcheck > interval && !printed { - fs.LogPrint(fs.LogLevelInfo, "Updating all links and torrents.") + fs.LogPrint(fs.LogLevelDebug, "updating all links and torrents") printed = true } newcached = append(newcached, partialresult...) diff --git a/sorting.txt b/sorting.txt index 0bc28f6f29660..ea7c039acd44f 100644 --- a/sorting.txt +++ b/sorting.txt @@ -28,5 +28,3 @@ # ~~~ recorded/manual changes to the structure: ~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -/default/Cast Away 2002/ -/kafferep-cletus[PRiME].mkv/kafferep-cletus[PRiME].mkv -> /default/Cast Away 2002/kafferep-cletus[PRiME].mkv From 9e64db15cce05777ac14a8a46a64c237ab7b0715 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sat, 25 Feb 2023 17:02:41 +0100 Subject: [PATCH 536/560] check if sortfile was updated w/o opening it --- backend/realdebrid/realdebrid.go | 41 ++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 4649719447870..0be2cc1df7fe0 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -604,30 +604,19 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } } } - // Open the sorting file - - file, err := os.Open(f.opt.SortFile) + // Check the sorting file for updates + fileModTime := time.Now().Unix() + fileInfo, err := os.Stat(f.opt.SortFile) if os.IsNotExist(err) { - fs.LogPrint(fs.LogLevelWarning, "no sorting file found. creating new empty sorting file.") - file, err = os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) - if err != nil { - fmt.Println(err) - } - defer file.Close() - - if _, err := file.WriteString(default_sorting); err != nil { - fmt.Println(err) - } + // file does not exist } else if err != nil { - fmt.Println(err) + // error occurred while checking file info } else { - defer file.Close() + // file exists, get modification time + fileModTime = fileInfo.ModTime().Unix() } var updated = false - fileInfo, _ := file.Stat() - fileModTime := fileInfo.ModTime().Unix() - if fileModTime > lastFileMod { updated = true } @@ -637,7 +626,23 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Create folder structure // if updated { + file, err := os.Open(f.opt.SortFile) + if os.IsNotExist(err) { + fs.LogPrint(fs.LogLevelWarning, "no sorting file found. creating new empty sorting file.") + file, err = os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + fmt.Println(err) + } + defer file.Close() + if _, err := file.WriteString(default_sorting); err != nil { + fmt.Println(err) + } + } else if err != nil { + fmt.Println(err) + } else { + defer file.Close() + } // Reset saved folder structure fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file.") eraseSyncMap(folders) From f7d77f74828c27ae6f3b80d3afa45cbad8969fd4 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sat, 25 Feb 2023 17:20:44 +0100 Subject: [PATCH 537/560] fix small first run issue --- .gitignore | 2 ++ backend/realdebrid/realdebrid.go | 1 + 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 236363b218a3e..38c92688ca525 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ fuzz-build.zip Thumbs.db __pycache__ sorting.txt +rclone-as-beta-linux-arm64 +rclone-as-beta.exe diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 0be2cc1df7fe0..f11e900abfe23 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -638,6 +638,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if _, err := file.WriteString(default_sorting); err != nil { fmt.Println(err) } + file.Sync() // flush file contents to disk } else if err != nil { fmt.Println(err) } else { From 6a636c777db0fcf96ab84041191d114c7c2a895e Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sat, 25 Feb 2023 17:31:02 +0100 Subject: [PATCH 538/560] fix first startup issues --- .gitignore | 2 ++ backend/realdebrid/realdebrid.go | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 38c92688ca525..6c31c2dc00543 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ __pycache__ sorting.txt rclone-as-beta-linux-arm64 rclone-as-beta.exe +z-rclone-as-beta-linux-arm64 +z-rclone-as-beta.exe diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index f11e900abfe23..dc67c1d800db1 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -624,8 +624,9 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if (dirID == rootID) || !(f.folder_exists(dirID)) || updated { // Create folder structure - // if updated { + + // read the file, create if missing file, err := os.Open(f.opt.SortFile) if os.IsNotExist(err) { fs.LogPrint(fs.LogLevelWarning, "no sorting file found. creating new empty sorting file.") @@ -644,6 +645,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } else { defer file.Close() } + // Reset saved folder structure fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file.") eraseSyncMap(folders) @@ -651,9 +653,14 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi eraseSyncMap(regex_defs) eraseSyncMap(mapping) eraseSyncMap(sorting_file) + // Flush the directory cache f.dirCache.Flush() + // Read the file line by line + if _, err := file.Seek(0, io.SeekStart); err != nil { + fmt.Println(err) + } scanner := bufio.NewScanner(file) for scanner.Scan() { if strings.Contains(scanner.Text(), "#") { From a67e87e87b0d5067091b9e7e3c85a4bd90b8d6b6 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sat, 25 Feb 2023 18:50:10 +0100 Subject: [PATCH 539/560] mutex lock move functions --- .gitignore | 1 + backend/realdebrid/realdebrid.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 6c31c2dc00543..2a7950c61d00a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ rclone-as-beta-linux-arm64 rclone-as-beta.exe z-rclone-as-beta-linux-arm64 z-rclone-as-beta.exe +z-rclone-as-beta-linux diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index dc67c1d800db1..d0fa4e60cd086 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -81,6 +81,7 @@ var broken_torrents []string var lastcheck int64 = time.Now().Unix() var lastFileMod int64 = 0 var interval int64 = 15 * 60 +var mutex = &sync.Mutex{} var mapping sync.Map var sorting_file sync.Map var folders sync.Map @@ -1356,6 +1357,8 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir // // If it isn't possible then return fs.ErrorCantMove func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + mutex.Lock() + defer mutex.Unlock() srcObj, ok := src.(*Object) if !ok { fs.Debugf(src, "Can't move - not same remote type") @@ -1390,6 +1393,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, // // If destination exists then return fs.ErrorDirExists func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { + mutex.Lock() + defer mutex.Unlock() srcFs, ok := src.(*Fs) if !ok { fs.Debugf(srcFs, "Can't move directory - not same remote type") From 99179793df660296c4302ce0922f89c9aeb15ea3 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sat, 25 Feb 2023 19:20:44 +0100 Subject: [PATCH 540/560] fix creating empty folders --- backend/realdebrid/realdebrid.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index d0fa4e60cd086..36ccb7187d0fc 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -1037,6 +1037,9 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } value, _ := folders.Load(dirID) + if value == nil { + value = []api.Item{} + } result = append(result, value.([]api.Item)...) if err != nil { From 77b37479a50d93c01b710608b64bd7b50579d38c Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sun, 26 Feb 2023 12:11:50 +0100 Subject: [PATCH 541/560] switch to xsync, fix parallel mv/rn --- .vscode/launch.json | 2 +- backend/realdebrid/realdebrid.go | 264 ++++++++++++++++--------------- go.mod | 1 + go.sum | 2 + sorting.txt | 7 +- 5 files changed, 142 insertions(+), 134 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 241d8ca8afeee..0f01f4b3d6de9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["moveto","realdebrid:/default/kafferep-cletus[PRiME].mkv/kafferep-cletus[PRiME].mkv","realdebrid:/default/Cast Away 2002/kafferep-cletus[PRiME].mkv"], + "args": ["moveto","realdebrid:/default/Scrubs/","realdebrid:/default/Test/","--disable","DirMove"], "console": "integratedTerminal" }, { diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 36ccb7187d0fc..7b7ae89fc28f1 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -32,6 +32,7 @@ import ( "sync" "time" + "github.com/puzpuzpuz/xsync" "github.com/rclone/rclone/backend/realdebrid/api" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" @@ -82,11 +83,11 @@ var lastcheck int64 = time.Now().Unix() var lastFileMod int64 = 0 var interval int64 = 15 * 60 var mutex = &sync.Mutex{} -var mapping sync.Map -var sorting_file sync.Map -var folders sync.Map -var regex_folders sync.Map -var regex_defs sync.Map +var force_update = false +var mapping = xsync.NewMap() +var sorting_file = xsync.NewMap() +var folders = xsync.NewMap() +var regex_defs = xsync.NewMap() var trash_indicator = ".trashed" var move_chars = " -> " var regx_chars = " == " @@ -96,7 +97,7 @@ var default_sorting = `# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # - write comment lines using "#" # -# - write regex definitions using: folder + " == " + regex definition. You can edit the exising ones or create new ones. +# - write regex definitions using: "/foldername" + " == " + regex definition. You can edit the exising ones or create new ones. # Order matters for regex folders, first match will be final destination. Make sure there are no trailing space characters. # torrents that dont match any regex definition end up in a folder named "default". # Example: /movies == (?i)(19|20)([0-9]{2} ?\.?) @@ -104,9 +105,10 @@ var default_sorting = `# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # - create new directories using "/foldername" # Example: /shit # -# - write move/renaming changes using: "actual torrent title" + "/" + "actual file name" + " -> " + "destination" +# - write move/renaming changes using: "/" + "actual torrent title" + "/" + "actual file name" + " -> " + "destination" # You do not need to create the directories you are moving stuff to, this will be done automatically. -# Example: /Our.Universe.S01.1080p.[rartv] -> /shows/Our Universe/Season 1 +# Example: /some.show.S01/ -> /shows/some.show/season 1/ +# Example: /some.show.S01/some.show.S01E01.mkv -> /shows/some.show/season 1/episode 1.mkv # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~ top level and regex folders: ~~~~~~~~~~ @@ -564,8 +566,8 @@ func (f *Fs) folder_exists(dirID string) bool { } // Clean sync.map without creating race conditions -func eraseSyncMap(m sync.Map) { - m.Range(func(key interface{}, value interface{}) bool { +func eraseSyncMap(m *xsync.Map) { + m.Range(func(key string, value interface{}) bool { m.Delete(key) return true }) @@ -607,18 +609,19 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } // Check the sorting file for updates fileModTime := time.Now().Unix() - fileInfo, err := os.Stat(f.opt.SortFile) - if os.IsNotExist(err) { - // file does not exist - } else if err != nil { - // error occurred while checking file info - } else { - // file exists, get modification time - fileModTime = fileInfo.ModTime().Unix() + if !force_update { + fileInfo, err := os.Stat(f.opt.SortFile) + if os.IsNotExist(err) { + // file does not exist + } else if err != nil { + // error occurred while checking file info + } else { + // file exists, get modification time + fileModTime = fileInfo.ModTime().Unix() + } } - var updated = false - if fileModTime > lastFileMod { + if fileModTime > lastFileMod || force_update { updated = true } @@ -650,13 +653,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Reset saved folder structure fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file.") eraseSyncMap(folders) - eraseSyncMap(regex_folders) - eraseSyncMap(regex_defs) - eraseSyncMap(mapping) - eraseSyncMap(sorting_file) - - // Flush the directory cache - f.dirCache.Flush() // Read the file line by line if _, err := file.Seek(0, io.SeekStart); err != nil { @@ -664,146 +660,152 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } scanner := bufio.NewScanner(file) for scanner.Scan() { - if strings.Contains(scanner.Text(), "#") { + if strings.HasPrefix(scanner.Text(), "#") { continue } else if len(scanner.Text()) == 0 || scanner.Text() == "\n" || scanner.Text() == "\n\r" { continue } else if strings.Contains(scanner.Text(), move_chars) { // Split the line by " -> " parts := strings.Split(scanner.Text(), move_chars) - // Add the key-value pair to the map - mapping.Store(parts[0], parts[1]) + // Add the key-value pair to the map if not present or changed + oldValue, ok := mapping.Load(parts[0]) + if !ok || oldValue.(string) != parts[1] { + mapping.Store(parts[0], parts[1]) + } } else if strings.Contains(scanner.Text(), regx_chars) { // Split the line by " == " parts := strings.Split(scanner.Text(), regx_chars) // Add the key-value pair to the map - regex_folders.Store(parts[0], parts[1]) + oldValue, ok := regex_defs.Load(parts[0]) + if !ok || oldValue.(*regexp.Regexp).String() != parts[1] { + r, _ := regexp.Compile(parts[1]) + regex_defs.Store(parts[0], r) + } } else { - mapping.Store(scanner.Text(), scanner.Text()) + oldValue, ok := mapping.Load(scanner.Text()) + if !ok || oldValue.(string) != scanner.Text() { + mapping.Store(scanner.Text(), scanner.Text()) + } } } - //create regex definitions - regex_folders.Range(func(folder, regex_def interface{}) bool { - r, _ := regexp.Compile(regex_def.(string)) - regex_defs.Store(folder, r) - return true - }) - - mapping.Range(func(key, value interface{}) bool { + mapping.Range(func(key string, value interface{}) bool { sorting_file.Store(key, value) return true }) } - - //update global cached list - opts := rest.Opts{ - Method: method, - Path: path, - Parameters: f.baseParams(), - } - opts.Parameters.Set("includebreadcrumbs", "false") - opts.Parameters.Set("limit", "1") - var newcached []api.Item - var totalcount int - var printed = false - var err_code = 429 - totalcount = 2 - for len(newcached) < totalcount { - partialresult = nil - resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) - var retries = 0 - if resp != nil { - err_code = resp.StatusCode + if !force_update { + //update global cached list + opts := rest.Opts{ + Method: method, + Path: path, + Parameters: f.baseParams(), } - for err_code == 429 && retries <= 5 { + opts.Parameters.Set("includebreadcrumbs", "false") + opts.Parameters.Set("limit", "1") + var newcached []api.Item + var totalcount int + var printed = false + var err_code = 429 + totalcount = 2 + for len(newcached) < totalcount { partialresult = nil - time.Sleep(time.Duration(2) * time.Second) resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + var retries = 0 if resp != nil { err_code = resp.StatusCode } - retries += 1 - } - if err == nil { - totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) + for err_code == 429 && retries <= 5 { + partialresult = nil + time.Sleep(time.Duration(2) * time.Second) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + if resp != nil { + err_code = resp.StatusCode + } + retries += 1 + } if err == nil { - if totalcount != len(cached) || time.Now().Unix()-lastcheck > interval { - if time.Now().Unix()-lastcheck > interval && !printed { - fs.LogPrint(fs.LogLevelDebug, "updating all links and torrents") - printed = true + totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) + if err == nil { + if totalcount != len(cached) || time.Now().Unix()-lastcheck > interval { + if time.Now().Unix()-lastcheck > interval && !printed { + fs.LogPrint(fs.LogLevelDebug, "updating all links and torrents") + printed = true + } + newcached = append(newcached, partialresult...) + opts.Parameters.Set("offset", strconv.Itoa(len(newcached))) + opts.Parameters.Set("limit", "2500") + updated = true + } else { + newcached = cached } - newcached = append(newcached, partialresult...) - opts.Parameters.Set("offset", strconv.Itoa(len(newcached))) - opts.Parameters.Set("limit", "2500") - updated = true } else { - newcached = cached + break } } else { break } - } else { - break } - } - //fmt.Printf("Done.\n") - //fmt.Printf("Updating RealDebrid Torrents ... ") - cached = newcached - //get torrents - path = "/torrents" - opts = rest.Opts{ - Method: method, - Path: path, - Parameters: f.baseParams(), - } - opts.Parameters.Set("limit", "1") - var newtorrents []api.Item - totalcount = 2 - err_code = 429 - for len(newtorrents) < totalcount { - partialresult = nil - resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) - if resp != nil { - err_code = resp.StatusCode + //fmt.Printf("Done.\n") + //fmt.Printf("Updating RealDebrid Torrents ... ") + cached = newcached + //get torrents + path = "/torrents" + opts = rest.Opts{ + Method: method, + Path: path, + Parameters: f.baseParams(), } - var retries = 0 - for err_code == 429 && retries <= 5 { + opts.Parameters.Set("limit", "1") + var newtorrents []api.Item + totalcount = 2 + err_code = 429 + for len(newtorrents) < totalcount { partialresult = nil - time.Sleep(time.Duration(2) * time.Second) resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) if resp != nil { err_code = resp.StatusCode } - retries += 1 - } - if err == nil { - totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) + var retries = 0 + for err_code == 429 && retries <= 5 { + partialresult = nil + time.Sleep(time.Duration(2) * time.Second) + resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + if resp != nil { + err_code = resp.StatusCode + } + retries += 1 + } if err == nil { - if totalcount != len(torrents) || time.Now().Unix()-lastcheck > interval { - newtorrents = append(newtorrents, partialresult...) - opts.Parameters.Set("offset", strconv.Itoa(len(newtorrents))) - opts.Parameters.Set("limit", "2500") - updated = true + totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) + if err == nil { + if totalcount != len(torrents) || time.Now().Unix()-lastcheck > interval { + newtorrents = append(newtorrents, partialresult...) + opts.Parameters.Set("offset", strconv.Itoa(len(newtorrents))) + opts.Parameters.Set("limit", "2500") + updated = true + } else { + newtorrents = torrents + } } else { - newtorrents = torrents + break } } else { break } - } else { - break } + // Set everything as being up to date + lastcheck = time.Now().Unix() + lastFileMod = fileModTime + //fmt.Printf("Done.\n") + torrents = newtorrents } - // Set everything as being up to date - lastcheck = time.Now().Unix() - lastFileMod = fileModTime - //fmt.Printf("Done.\n") - torrents = newtorrents - + // set force update to false + force_update = false // Iterate through built file and torrent list: var broken = false + var err_code = 0 if updated { for i := range torrents { //handle dead torrents @@ -818,10 +820,10 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } //set default torrents[i] location torrents[i].DefaultLocation = "/default/" - regex_defs.Range(func(folder, r interface{}) bool { + regex_defs.Range(func(folder string, r interface{}) bool { match := r.(*regexp.Regexp).MatchString(torrents[i].Name) if match { - torrents[i].DefaultLocation = folder.(string) + "/" + torrents[i].DefaultLocation = folder + "/" return false } return true @@ -902,11 +904,11 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi mapping.Store(mapping_id, strings.Join(strings.Split(value.(string), "/")[:len(strings.Split(value.(string), "/"))-1], "/")+"/") } value, _ := mapping.Load(mapping_id) - list, ok := folders.Load(value) + list, ok := folders.Load(value.(string)) if !ok { list = []api.Item{} } - folders.Store(value, append(list.([]api.Item), ItemFile)) + folders.Store(value.(string), append(list.([]api.Item), ItemFile)) } if broken { torrents[i] = f.redownloadTorrent(ctx, torrents[i]) @@ -969,17 +971,17 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi mapping.Store(mapping_id, strings.Join(strings.Split(value.(string), "/")[:len(strings.Split(value.(string), "/"))-1], "/")+"/") } value, _ := mapping.Load(mapping_id) - list, ok := folders.Load(value) + list, ok := folders.Load(value.(string)) if !ok { list = []api.Item{} } - folders.Store(value, append(list.([]api.Item), ItemFile)) + folders.Store(value.(string), append(list.([]api.Item), ItemFile)) } } } // Iterate through the map - mapping.Range(func(_, newLocation interface{}) bool { + mapping.Range(func(_ string, newLocation interface{}) bool { if strings.HasSuffix(newLocation.(string), trash_indicator) { return true @@ -1293,9 +1295,9 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir } // Get all files that should be moved. var affected_items []string - sorting_file.Range(func(key, value interface{}) bool { + sorting_file.Range(func(key string, value interface{}) bool { if strings.Contains(value.(string), oldDirectoryID+oldLeaf) { - affected_items = append(affected_items, key.(string)) + affected_items = append(affected_items, key) } return true }) @@ -1362,6 +1364,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { mutex.Lock() defer mutex.Unlock() + force_update = true srcObj, ok := src.(*Object) if !ok { fs.Debugf(src, "Can't move - not same remote type") @@ -1398,6 +1401,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { mutex.Lock() defer mutex.Unlock() + force_update = true srcFs, ok := src.(*Fs) if !ok { fs.Debugf(srcFs, "Can't move directory - not same remote type") @@ -1615,10 +1619,10 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { // Get all trashed files var affected_items []string - sorting_file.Range(func(key, value interface{}) bool { - if strings.Contains(key.(string), oldDirectoryID) { + sorting_file.Range(func(key string, value interface{}) bool { + if strings.Contains(key, oldDirectoryID) { if strings.HasSuffix(value.(string), trash_indicator) { - affected_items = append(affected_items, key.(string)) + affected_items = append(affected_items, key) } } return true diff --git a/go.mod b/go.mod index a4090a21b271c..9a8158979fa8e 100644 --- a/go.mod +++ b/go.mod @@ -145,6 +145,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/pkg/xattr v0.4.9 + github.com/puzpuzpuz/xsync v1.5.2 golang.org/x/mobile v0.0.0-20221110043201-43a038452099 golang.org/x/term v0.4.0 ) diff --git a/go.sum b/go.sum index b9237905a80de..8bbaf5165f97e 100644 --- a/go.sum +++ b/go.sum @@ -407,6 +407,8 @@ github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8= github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU= +github.com/puzpuzpuz/xsync v1.5.2 h1:yRAP4wqSOZG+/4pxJ08fPTwrfL0IzE/LKQ/cw509qGY= +github.com/puzpuzpuz/xsync v1.5.2/go.mod h1:K98BYhX3k1dQ2M63t1YNVDanbwUPmBCAhNmVrrxfiGg= github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6 h1:J832KfU2Z44Ck3XR5bvw2UxShP0QnjueruNQ6dTYH+g= github.com/rclone/ftp v0.0.0-20221014110213-e44dedbc76c6/go.mod h1:qRpxqlna6CaIq9fSRud1bDC5S7EEUEou0j8nMZ0lxO8= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= diff --git a/sorting.txt b/sorting.txt index ea7c039acd44f..c1f3499beb836 100644 --- a/sorting.txt +++ b/sorting.txt @@ -4,7 +4,7 @@ # - write comment lines using "#" # -# - write regex definitions using: folder + " == " + regex definition. You can edit the exising ones or create new ones. +# - write regex definitions using: "/foldername" + " == " + regex definition. You can edit the exising ones or create new ones. # Order matters for regex folders, first match will be final destination. Make sure there are no trailing space characters. # torrents that dont match any regex definition end up in a folder named "default". # Example: /movies == (?i)(19|20)([0-9]{2} ?\.?) @@ -12,9 +12,10 @@ # - create new directories using "/foldername" # Example: /shit # -# - write move/renaming changes using: "actual torrent title" + "/" + "actual file name" + " -> " + "destination" +# - write move/renaming changes using: "/" + "actual torrent title" + "/" + "actual file name" + " -> " + "destination" # You do not need to create the directories you are moving stuff to, this will be done automatically. -# Example: /Our.Universe.S01.1080p.[rartv] -> /shows/Our Universe/Season 1 +# Example: /some.show.S01/ -> /shows/some.show/season 1/ +# Example: /some.show.S01/some.show.S01E01.mkv -> /shows/some.show/season 1/episode 1.mkv # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~ top level and regex folders: ~~~~~~~~~~ From 6d41992b7cc0c67b4833dff3d6fe21101df484b7 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sun, 26 Feb 2023 13:36:53 +0100 Subject: [PATCH 542/560] truly rebuild mapping if its not called by the move function --- backend/realdebrid/realdebrid.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 7b7ae89fc28f1..69b5c667c086f 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -653,6 +653,11 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Reset saved folder structure fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file.") eraseSyncMap(folders) + if !force_update { + eraseSyncMap(mapping) + eraseSyncMap(regex_defs) + eraseSyncMap(sorting_file) + } // Read the file line by line if _, err := file.Seek(0, io.SeekStart); err != nil { From c23978ceae81fbfc3e32e77aece282029a1de845 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sun, 26 Feb 2023 15:03:32 +0100 Subject: [PATCH 543/560] restore regex folder returning first match --- backend/realdebrid/realdebrid.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 69b5c667c086f..cf18e15a3ccdb 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -826,10 +826,12 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi //set default torrents[i] location torrents[i].DefaultLocation = "/default/" regex_defs.Range(func(folder string, r interface{}) bool { - match := r.(*regexp.Regexp).MatchString(torrents[i].Name) - if match { - torrents[i].DefaultLocation = folder + "/" - return false + if torrents[i].DefaultLocation == "/default/" { + match := r.(*regexp.Regexp).MatchString(torrents[i].Name) + if match { + torrents[i].DefaultLocation = folder + "/" + return false // break out of loop + } } return true }) From 62b2f5d57816555650ee430dfbde5b987a51551f Mon Sep 17 00:00:00 2001 From: itsToggle Date: Tue, 28 Feb 2023 09:44:22 +0100 Subject: [PATCH 544/560] small renaming, deleting fixes, fixed crash on 914 --- .vscode/launch.json | 2 +- backend/realdebrid/realdebrid.go | 90 +++++++++++++++++--------------- 2 files changed, 50 insertions(+), 42 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 0f01f4b3d6de9..611214c57535c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["moveto","realdebrid:/default/Scrubs/","realdebrid:/default/Test/","--disable","DirMove"], + "args": ["moveto","realdebrid:/default/PISS","realdebrid:/default/MULTI",],//"--disable","DirMove"], "console": "integratedTerminal" }, { diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index cf18e15a3ccdb..723c7b4ac4ba3 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -653,11 +653,9 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Reset saved folder structure fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file.") eraseSyncMap(folders) - if !force_update { - eraseSyncMap(mapping) - eraseSyncMap(regex_defs) - eraseSyncMap(sorting_file) - } + eraseSyncMap(mapping) + eraseSyncMap(regex_defs) + eraseSyncMap(sorting_file) // Read the file line by line if _, err := file.Seek(0, io.SeekStart); err != nil { @@ -911,11 +909,13 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi mapping.Store(mapping_id, strings.Join(strings.Split(value.(string), "/")[:len(strings.Split(value.(string), "/"))-1], "/")+"/") } value, _ := mapping.Load(mapping_id) - list, ok := folders.Load(value.(string)) - if !ok { - list = []api.Item{} + if value != nil { + list, ok := folders.Load(value.(string)) + if !ok { + list = []api.Item{} + } + folders.Store(value.(string), append(list.([]api.Item), ItemFile)) } - folders.Store(value.(string), append(list.([]api.Item), ItemFile)) } if broken { torrents[i] = f.redownloadTorrent(ctx, torrents[i]) @@ -978,11 +978,13 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi mapping.Store(mapping_id, strings.Join(strings.Split(value.(string), "/")[:len(strings.Split(value.(string), "/"))-1], "/")+"/") } value, _ := mapping.Load(mapping_id) - list, ok := folders.Load(value.(string)) - if !ok { - list = []api.Item{} + if value != nil { + list, ok := folders.Load(value.(string)) + if !ok { + list = []api.Item{} + } + folders.Store(value.(string), append(list.([]api.Item), ItemFile)) } - folders.Store(value.(string), append(list.([]api.Item), ItemFile)) } } } @@ -1254,7 +1256,10 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // move a file or folder // func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDirectoryID, newDirectoryID string) (err error) { + mutex.Lock() + defer mutex.Unlock() // Handle IDs + oldDirectoryID_o := oldDirectoryID for _, torrent := range torrents { if torrent.ID == oldDirectoryID { oldDirectoryID = "/" + torrent.Name + "/" @@ -1292,6 +1297,12 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir if first := oldDirectoryID[0]; first != '/' { oldDirectoryID = "/" + oldDirectoryID } + if last := oldDirectoryID_o[len(oldDirectoryID_o)-1]; last != '/' { + oldDirectoryID_o = oldDirectoryID_o + "/" + } + if first := oldDirectoryID_o[0]; first != '/' { + oldDirectoryID_o = "/" + oldDirectoryID_o + } if !isFile { if last := newLeaf[len(newLeaf)-1]; last != '/' { newLeaf = newLeaf + "/" @@ -1329,7 +1340,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir for _, affected_item := range affected_items { for _, line := range lines { if strings.Contains(line, affected_item+move_chars) && !isFile { - new_dst := strings.Replace(strings.Split(line, move_chars)[len(strings.Split(line, move_chars))-1], oldDirectoryID+oldLeaf, newDirectoryID+newLeaf, -1) + new_dst := strings.Replace(strings.Split(line, move_chars)[len(strings.Split(line, move_chars))-1], oldDirectoryID_o+oldLeaf, newDirectoryID+newLeaf, -1) new_lines[line] = affected_item + move_chars + new_dst replaced = true } else if strings.Contains(line, affected_item+move_chars) && isFile { @@ -1369,8 +1380,6 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir // // If it isn't possible then return fs.ErrorCantMove func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { - mutex.Lock() - defer mutex.Unlock() force_update = true srcObj, ok := src.(*Object) if !ok { @@ -1406,8 +1415,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, // // If destination exists then return fs.ErrorDirExists func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { - mutex.Lock() - defer mutex.Unlock() force_update = true srcFs, ok := src.(*Fs) if !ok { @@ -1608,9 +1615,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op func (f *Fs) remove(ctx context.Context, o *Object) (err error) { var resp *http.Response var result api.Response - var retries = 0 var oldDirectoryID = "" var torrent_files = 0 + // Get parent torrent item for i := range torrents { if torrents[i].ID == o.ParentID { @@ -1644,33 +1651,34 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { fs.LogPrint(fs.LogLevelDebug, "moving file: "+o.MappingID+" to internal trash") f.Move(ctx, o, o.remote+trash_indicator) - // delete the link on realdebrid - fs.LogPrint(fs.LogLevelDebug, "removing realdebrid link id: "+o.id) - path := "/downloads/delete/" + o.id - opts := rest.Opts{ - Method: "DELETE", - Path: path, - Parameters: f.baseParams(), - } - err_code := 0 - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) - if resp != nil { - err_code = resp.StatusCode - } - for err_code == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) - if resp != nil { - err_code = resp.StatusCode - } - retries += 1 - } + // // delete the link on realdebrid + // fs.LogPrint(fs.LogLevelDebug, "removing realdebrid link id: "+o.id) + // path := "/downloads/delete/" + o.id + // opts := rest.Opts{ + // Method: "DELETE", + // Path: path, + // Parameters: f.baseParams(), + // } + // err_code := 0 + // resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) + // if resp != nil { + // err_code = resp.StatusCode + // } + // for err_code == 429 && retries <= 5 { + // time.Sleep(time.Duration(2) * time.Second) + // resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) + // if resp != nil { + // err_code = resp.StatusCode + // } + // retries += 1 + // } return nil } // if all files are trashed fs.LogPrint(fs.LogLevelDebug, "all files of torrent: "+o.ParentID+" are in internal trash") - + mutex.Lock() + defer mutex.Unlock() // read sort file file, err := os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) if err != nil { From 592b16349025dff3e39e1043a762f33c2a2967bf Mon Sep 17 00:00:00 2001 From: itsToggle Date: Tue, 28 Feb 2023 14:46:56 +0100 Subject: [PATCH 545/560] fix regex random order --- .vscode/launch.json | 2 +- backend/realdebrid/realdebrid.go | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 611214c57535c..b2b0fb6d9c211 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -18,7 +18,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["tree","realdebrid:/default/"], + "args": ["tree","realdebrid:/movies/"], "console": "integratedTerminal" }, { diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 723c7b4ac4ba3..7eca5276527cc 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -75,6 +75,11 @@ var ( } ) +type RegexValuePair struct { + Regex *regexp.Regexp + Value string +} + //Global variables var cached []api.Item var torrents []api.Item @@ -87,7 +92,7 @@ var force_update = false var mapping = xsync.NewMap() var sorting_file = xsync.NewMap() var folders = xsync.NewMap() -var regex_defs = xsync.NewMap() +var regex_defs = []RegexValuePair{} var trash_indicator = ".trashed" var move_chars = " -> " var regx_chars = " == " @@ -654,8 +659,8 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file.") eraseSyncMap(folders) eraseSyncMap(mapping) - eraseSyncMap(regex_defs) eraseSyncMap(sorting_file) + regex_defs = []RegexValuePair{} // Read the file line by line if _, err := file.Seek(0, io.SeekStart); err != nil { @@ -679,11 +684,9 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Split the line by " == " parts := strings.Split(scanner.Text(), regx_chars) // Add the key-value pair to the map - oldValue, ok := regex_defs.Load(parts[0]) - if !ok || oldValue.(*regexp.Regexp).String() != parts[1] { - r, _ := regexp.Compile(parts[1]) - regex_defs.Store(parts[0], r) - } + r, _ := regexp.Compile(parts[1]) + regex_defs = append(regex_defs, RegexValuePair{r, parts[0]}) + } else { oldValue, ok := mapping.Load(scanner.Text()) if !ok || oldValue.(string) != scanner.Text() { @@ -823,16 +826,13 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } //set default torrents[i] location torrents[i].DefaultLocation = "/default/" - regex_defs.Range(func(folder string, r interface{}) bool { - if torrents[i].DefaultLocation == "/default/" { - match := r.(*regexp.Regexp).MatchString(torrents[i].Name) - if match { - torrents[i].DefaultLocation = folder + "/" - return false // break out of loop - } + for _, pair := range regex_defs { + match := pair.Regex.MatchString(torrents[i].Name) + if match { + torrents[i].DefaultLocation = pair.Value + "/" + break } - return true - }) + } //iterate through files var broken = false for _, link := range torrents[i].Links { From cba02c6daaf0d4afd7ceb1dc4ed3e5853e8d0569 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Tue, 28 Feb 2023 17:35:10 +0100 Subject: [PATCH 546/560] speed up moving/renaming actions --- .vscode/launch.json | 6 +- backend/realdebrid/realdebrid.go | 105 ++++++++++++++++++++++++++----- 2 files changed, 91 insertions(+), 20 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index b2b0fb6d9c211..06207f20220ab 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["moveto","realdebrid:/default/PISS","realdebrid:/default/MULTI",],//"--disable","DirMove"], + "args": ["moveto","realdebrid:/default/MULTI","realdebrid:/default/Scrubs","--disable","DirMove"], "console": "integratedTerminal" }, { @@ -18,7 +18,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["tree","realdebrid:/movies/"], + "args": ["tree","realdebrid:/default/"], "console": "integratedTerminal" }, { @@ -38,7 +38,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["delete","realdebrid:/default/Scrubs/Scrubs S01E01 My First Day (c) [1080p x265 10bit Joy].mkv"], + "args": ["delete","realdebrid:/default/some.show/season 1"], "console": "integratedTerminal" }, ] diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 7eca5276527cc..c5fabd9d58041 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -87,7 +87,8 @@ var broken_torrents []string var lastcheck int64 = time.Now().Unix() var lastFileMod int64 = 0 var interval int64 = 15 * 60 -var mutex = &sync.Mutex{} +var file_mutex = &sync.Mutex{} +var move_mutex = &sync.Mutex{} var force_update = false var mapping = xsync.NewMap() var sorting_file = xsync.NewMap() @@ -423,6 +424,8 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin // CreateDir makes a directory with pathID as parent and name leaf func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, err error) { + file_mutex.Lock() + defer file_mutex.Unlock() if len(dirID) > 0 && dirID != "/" { if first := dirID[0]; first != '/' { dirID = "/" + dirID @@ -632,9 +635,10 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if (dirID == rootID) || !(f.folder_exists(dirID)) || updated { + var force_updated_items []string // Create folder structure if updated { - + file_mutex.Lock() // read the file, create if missing file, err := os.Open(f.opt.SortFile) if os.IsNotExist(err) { @@ -657,10 +661,12 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Reset saved folder structure fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file.") - eraseSyncMap(folders) - eraseSyncMap(mapping) - eraseSyncMap(sorting_file) regex_defs = []RegexValuePair{} + if !force_update { + eraseSyncMap(folders) + eraseSyncMap(mapping) + eraseSyncMap(sorting_file) + } // Read the file line by line if _, err := file.Seek(0, io.SeekStart); err != nil { @@ -679,6 +685,10 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi oldValue, ok := mapping.Load(parts[0]) if !ok || oldValue.(string) != parts[1] { mapping.Store(parts[0], parts[1]) + if force_update { + force_updated_items = append(force_updated_items, parts[0]) + sorting_file.Store(parts[0], parts[1]) + } } } else if strings.Contains(scanner.Text(), regx_chars) { // Split the line by " == " @@ -691,14 +701,25 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi oldValue, ok := mapping.Load(scanner.Text()) if !ok || oldValue.(string) != scanner.Text() { mapping.Store(scanner.Text(), scanner.Text()) + if force_update { + sorting_file.Store(scanner.Text(), scanner.Text()) + force_updated_items = append(force_updated_items, scanner.Text()) + } } } } - mapping.Range(func(key string, value interface{}) bool { - sorting_file.Store(key, value) - return true - }) + file_mutex.Unlock() + + if !force_update { + mapping.Range(func(key string, value interface{}) bool { + oldValue, ok := sorting_file.Load(key) + if !ok || oldValue != value { + sorting_file.Store(key, value) + } + return true + }) + } } if !force_update { @@ -807,13 +828,24 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi //fmt.Printf("Done.\n") torrents = newtorrents } - // set force update to false - force_update = false + // Iterate through built file and torrent list: var broken = false var err_code = 0 if updated { for i := range torrents { + //if force update, skip unaffected torrents + if force_update { + var skip = true + for _, force_updated_item := range force_updated_items { + if strings.Contains(force_updated_item, torrents[i].Name) { + skip = false + } + } + if skip { + continue + } + } //handle dead torrents broken = false for _, TorrentID := range broken_torrents { @@ -914,6 +946,16 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if !ok { list = []api.Item{} } + skip := false + for _, existing_item := range list.([]api.Item) { + if existing_item.Name == ItemFile.Name { + skip = true + break + } + } + if skip { + continue + } folders.Store(value.(string), append(list.([]api.Item), ItemFile)) } } @@ -983,6 +1025,16 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if !ok { list = []api.Item{} } + skip := false + for _, existing_item := range list.([]api.Item) { + if existing_item.Name == ItemFile.Name { + skip = true + break + } + } + if skip { + continue + } folders.Store(value.(string), append(list.([]api.Item), ItemFile)) } } @@ -990,7 +1042,19 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } // Iterate through the map - mapping.Range(func(_ string, newLocation interface{}) bool { + mapping.Range(func(key string, newLocation interface{}) bool { + + if force_update { + skip := true + for _, force_updated_item := range force_updated_items { + if strings.Contains(key, force_updated_item) { + skip = false + } + } + if skip { + return true + } + } if strings.HasSuffix(newLocation.(string), trash_indicator) { return true @@ -1003,7 +1067,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi var location string // Iterate through each level of the new location - for _, locationPart := range locationParts { + for _, locationPart := range locationParts[:len(locationParts)-1] { // Append the full path to the corresponding level var skip bool skip = false @@ -1045,6 +1109,9 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } + // set force update to false + force_update = false + } value, _ := folders.Load(dirID) @@ -1256,8 +1323,8 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // move a file or folder // func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDirectoryID, newDirectoryID string) (err error) { - mutex.Lock() - defer mutex.Unlock() + file_mutex.Lock() + defer file_mutex.Unlock() // Handle IDs oldDirectoryID_o := oldDirectoryID for _, torrent := range torrents { @@ -1380,6 +1447,8 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir // // If it isn't possible then return fs.ErrorCantMove func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + move_mutex.Lock() + defer move_mutex.Unlock() force_update = true srcObj, ok := src.(*Object) if !ok { @@ -1415,6 +1484,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, // // If destination exists then return fs.ErrorDirExists func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { + move_mutex.Lock() + defer move_mutex.Unlock() force_update = true srcFs, ok := src.(*Fs) if !ok { @@ -1677,8 +1748,8 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { // if all files are trashed fs.LogPrint(fs.LogLevelDebug, "all files of torrent: "+o.ParentID+" are in internal trash") - mutex.Lock() - defer mutex.Unlock() + file_mutex.Lock() + defer file_mutex.Unlock() // read sort file file, err := os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) if err != nil { From e131020d8400a3c8339ec543a0af20c8e109fea0 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Thu, 10 Aug 2023 12:22:18 +0200 Subject: [PATCH 547/560] simplify mutex & improve action speed --- .vscode/launch.json | 2 +- backend/realdebrid/realdebrid.go | 296 +++++++++++++++---------------- 2 files changed, 149 insertions(+), 149 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 06207f20220ab..c0342169ead16 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["moveto","realdebrid:/default/MULTI","realdebrid:/default/Scrubs","--disable","DirMove"], + "args": ["moveto","realdebrid:shows/Westworld.S01-S04.1080p.WEBRip.AAC.6CH.x265.HEVC-PSA/","realdebrid:default/fuck","--disable","DirMove"], "console": "integratedTerminal" }, { diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index c5fabd9d58041..e06085b70b61b 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "io" + "math" "net/http" "net/url" "os" @@ -87,9 +88,7 @@ var broken_torrents []string var lastcheck int64 = time.Now().Unix() var lastFileMod int64 = 0 var interval int64 = 15 * 60 -var file_mutex = &sync.Mutex{} -var move_mutex = &sync.Mutex{} -var force_update = false +var file_mutex = &sync.RWMutex{} var mapping = xsync.NewMap() var sorting_file = xsync.NewMap() var folders = xsync.NewMap() @@ -424,8 +423,10 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin // CreateDir makes a directory with pathID as parent and name leaf func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, err error) { + // fs.LogPrint(fs.LogLevelDebug, "CreateDir locking file_mutex") file_mutex.Lock() defer file_mutex.Unlock() + // defer fs.LogPrint(fs.LogLevelDebug, "CreateDir unlocking file_mutex") if len(dirID) > 0 && dirID != "/" { if first := dirID[0]; first != '/' { dirID = "/" + dirID @@ -617,7 +618,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } // Check the sorting file for updates fileModTime := time.Now().Unix() - if !force_update { + if math.Abs(float64(fileModTime-lastFileMod)) >= 5 { fileInfo, err := os.Stat(f.opt.SortFile) if os.IsNotExist(err) { // file does not exist @@ -629,16 +630,17 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } } var updated = false - if fileModTime > lastFileMod || force_update { + if fileModTime > lastFileMod { updated = true } - if (dirID == rootID) || !(f.folder_exists(dirID)) || updated { - - var force_updated_items []string + if ((dirID == rootID) || !(f.folder_exists(dirID)) || updated) && math.Abs(float64(fileModTime-lastFileMod)) >= 5 { // Create folder structure if updated { - file_mutex.Lock() + // fs.LogPrint(fs.LogLevelDebug, "ListAll Rlocking file_mutex") + file_mutex.RLock() + defer file_mutex.RUnlock() + // defer fs.LogPrint(fs.LogLevelDebug, "ListAll Runlocking file_mutex") // read the file, create if missing file, err := os.Open(f.opt.SortFile) if os.IsNotExist(err) { @@ -662,11 +664,9 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Reset saved folder structure fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file.") regex_defs = []RegexValuePair{} - if !force_update { - eraseSyncMap(folders) - eraseSyncMap(mapping) - eraseSyncMap(sorting_file) - } + eraseSyncMap(folders) + eraseSyncMap(mapping) + eraseSyncMap(sorting_file) // Read the file line by line if _, err := file.Seek(0, io.SeekStart); err != nil { @@ -685,10 +685,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi oldValue, ok := mapping.Load(parts[0]) if !ok || oldValue.(string) != parts[1] { mapping.Store(parts[0], parts[1]) - if force_update { - force_updated_items = append(force_updated_items, parts[0]) - sorting_file.Store(parts[0], parts[1]) - } } } else if strings.Contains(scanner.Text(), regx_chars) { // Split the line by " == " @@ -701,151 +697,132 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi oldValue, ok := mapping.Load(scanner.Text()) if !ok || oldValue.(string) != scanner.Text() { mapping.Store(scanner.Text(), scanner.Text()) - if force_update { - sorting_file.Store(scanner.Text(), scanner.Text()) - force_updated_items = append(force_updated_items, scanner.Text()) - } } } } - file_mutex.Unlock() - - if !force_update { - mapping.Range(func(key string, value interface{}) bool { - oldValue, ok := sorting_file.Load(key) - if !ok || oldValue != value { - sorting_file.Store(key, value) - } - return true - }) - } + mapping.Range(func(key string, value interface{}) bool { + oldValue, ok := sorting_file.Load(key) + if !ok || oldValue != value { + sorting_file.Store(key, value) + } + return true + }) } - if !force_update { - //update global cached list - opts := rest.Opts{ - Method: method, - Path: path, - Parameters: f.baseParams(), + //update global cached list + opts := rest.Opts{ + Method: method, + Path: path, + Parameters: f.baseParams(), + } + opts.Parameters.Set("includebreadcrumbs", "false") + opts.Parameters.Set("limit", "1") + var newcached []api.Item + var totalcount int + var printed = false + var err_code = 429 + totalcount = 2 + for len(newcached) < totalcount { + partialresult = nil + resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + var retries = 0 + if resp != nil { + err_code = resp.StatusCode } - opts.Parameters.Set("includebreadcrumbs", "false") - opts.Parameters.Set("limit", "1") - var newcached []api.Item - var totalcount int - var printed = false - var err_code = 429 - totalcount = 2 - for len(newcached) < totalcount { + for err_code == 429 && retries <= 5 { partialresult = nil + time.Sleep(time.Duration(2) * time.Second) resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) - var retries = 0 if resp != nil { err_code = resp.StatusCode } - for err_code == 429 && retries <= 5 { - partialresult = nil - time.Sleep(time.Duration(2) * time.Second) - resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) - if resp != nil { - err_code = resp.StatusCode - } - retries += 1 - } + retries += 1 + } + if err == nil { + totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) if err == nil { - totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) - if err == nil { - if totalcount != len(cached) || time.Now().Unix()-lastcheck > interval { - if time.Now().Unix()-lastcheck > interval && !printed { - fs.LogPrint(fs.LogLevelDebug, "updating all links and torrents") - printed = true - } - newcached = append(newcached, partialresult...) - opts.Parameters.Set("offset", strconv.Itoa(len(newcached))) - opts.Parameters.Set("limit", "2500") - updated = true - } else { - newcached = cached + if totalcount != len(cached) || time.Now().Unix()-lastcheck > interval { + if time.Now().Unix()-lastcheck > interval && !printed { + fs.LogPrint(fs.LogLevelDebug, "updating all links and torrents") + printed = true } + newcached = append(newcached, partialresult...) + opts.Parameters.Set("offset", strconv.Itoa(len(newcached))) + opts.Parameters.Set("limit", "2500") + // fs.LogPrint(fs.LogLevelDebug, "Setting updated to true") + updated = true } else { - break + newcached = cached } } else { break } + } else { + break } - //fmt.Printf("Done.\n") - //fmt.Printf("Updating RealDebrid Torrents ... ") - cached = newcached - //get torrents - path = "/torrents" - opts = rest.Opts{ - Method: method, - Path: path, - Parameters: f.baseParams(), + } + //fmt.Printf("Done.\n") + //fmt.Printf("Updating RealDebrid Torrents ... ") + cached = newcached + //get torrents + path = "/torrents" + opts = rest.Opts{ + Method: method, + Path: path, + Parameters: f.baseParams(), + } + opts.Parameters.Set("limit", "1") + var newtorrents []api.Item + totalcount = 2 + err_code = 429 + for len(newtorrents) < totalcount { + partialresult = nil + resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) + if resp != nil { + err_code = resp.StatusCode } - opts.Parameters.Set("limit", "1") - var newtorrents []api.Item - totalcount = 2 - err_code = 429 - for len(newtorrents) < totalcount { + var retries = 0 + for err_code == 429 && retries <= 5 { partialresult = nil + time.Sleep(time.Duration(2) * time.Second) resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) if resp != nil { err_code = resp.StatusCode } - var retries = 0 - for err_code == 429 && retries <= 5 { - partialresult = nil - time.Sleep(time.Duration(2) * time.Second) - resp, err = f.srv.CallJSON(ctx, &opts, nil, &partialresult) - if resp != nil { - err_code = resp.StatusCode - } - retries += 1 - } + retries += 1 + } + if err == nil { + totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) if err == nil { - totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) - if err == nil { - if totalcount != len(torrents) || time.Now().Unix()-lastcheck > interval { - newtorrents = append(newtorrents, partialresult...) - opts.Parameters.Set("offset", strconv.Itoa(len(newtorrents))) - opts.Parameters.Set("limit", "2500") - updated = true - } else { - newtorrents = torrents - } + if totalcount != len(torrents) || time.Now().Unix()-lastcheck > interval { + newtorrents = append(newtorrents, partialresult...) + opts.Parameters.Set("offset", strconv.Itoa(len(newtorrents))) + opts.Parameters.Set("limit", "2500") + // fs.LogPrint(fs.LogLevelDebug, "Setting updated to true") + updated = true } else { - break + newtorrents = torrents } } else { break } + } else { + break } - // Set everything as being up to date - lastcheck = time.Now().Unix() - lastFileMod = fileModTime - //fmt.Printf("Done.\n") - torrents = newtorrents } + // Set everything as being up to date + lastcheck = time.Now().Unix() + lastFileMod = fileModTime + //fmt.Printf("Done.\n") + torrents = newtorrents + // Iterate through built file and torrent list: var broken = false - var err_code = 0 + err_code = 0 if updated { for i := range torrents { - //if force update, skip unaffected torrents - if force_update { - var skip = true - for _, force_updated_item := range force_updated_items { - if strings.Contains(force_updated_item, torrents[i].Name) { - skip = false - } - } - if skip { - continue - } - } //handle dead torrents broken = false for _, TorrentID := range broken_torrents { @@ -1044,18 +1021,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // Iterate through the map mapping.Range(func(key string, newLocation interface{}) bool { - if force_update { - skip := true - for _, force_updated_item := range force_updated_items { - if strings.Contains(key, force_updated_item) { - skip = false - } - } - if skip { - return true - } - } - if strings.HasSuffix(newLocation.(string), trash_indicator) { return true } @@ -1109,9 +1074,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } - // set force update to false - force_update = false - } value, _ := folders.Load(dirID) @@ -1279,7 +1241,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { // if rootID is a torrent ID if len(rootID) == 13 && rootID == strings.ToUpper(rootID) { fs.LogPrint(fs.LogLevelDebug, "removing realdebrid torrent id: "+rootID) - path := "/torrents/delete/" + rootID + path := "/torrents/delete/DISABLED" + rootID opts := rest.Opts{ Method: "DELETE", Path: path, @@ -1323,8 +1285,10 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // move a file or folder // func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDirectoryID, newDirectoryID string) (err error) { + // fs.LogPrint(fs.LogLevelDebug, "move locking file_mutex") file_mutex.Lock() defer file_mutex.Unlock() + // defer fs.LogPrint(fs.LogLevelDebug, "move unlocking file_mutex") // Handle IDs oldDirectoryID_o := oldDirectoryID for _, torrent := range torrents { @@ -1386,7 +1350,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir } return true }) - if len(affected_items) == 0 { + if len(affected_items) == 0 || !isFile { affected_items = append(affected_items, oldDirectoryID+oldLeaf) } scanner := bufio.NewScanner(file) @@ -1447,9 +1411,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir // // If it isn't possible then return fs.ErrorCantMove func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { - move_mutex.Lock() - defer move_mutex.Unlock() - force_update = true + srcObj, ok := src.(*Object) if !ok { fs.Debugf(src, "Can't move - not same remote type") @@ -1468,7 +1430,46 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, return nil, err } + // Remove the old item from the folder structure + oldDir, _ := mapping.LoadAndDelete(srcObj.MappingID) + var newItems []api.Item + items, _ := folders.Load(oldDir.(string)) + for i := range items.([]api.Item) { + if items.([]api.Item)[i].ID != srcObj.id { + newItems = append(newItems, items.([]api.Item)[i]) + } + } + folders.Store(oldDir.(string), newItems) + + // Add the new item to the folder structure + if last := directoryID[len(directoryID)-1]; last != '/' { + directoryID = directoryID + "/" + } + if first := directoryID[0]; first != '/' { + directoryID = "/" + directoryID + } + mapping.Store(srcObj.MappingID, directoryID) + newItem := api.Item{} + newItem.Name = strings.Split(remote, "/")[len(strings.Split(remote, "/"))-1] + newItem.Size = srcObj.size + newItem.ID = srcObj.id + newItem.MimeType = srcObj.mimeType + newItem.Link = srcObj.url + newItem.ParentID = srcObj.ParentID + newItem.TorrentHash = srcObj.TorrentHash + newItem.MappingID = srcObj.MappingID + newItem.Type = "file" + newItem.Generated = srcObj.modTime.Format("2006-01-02T15:04:05.000Z") + items, _ = folders.Load(directoryID) + if items == nil { + items = []api.Item{} + } + items = items.([]api.Item) + items = append(items.([]api.Item), newItem) + folders.Store(directoryID, items) + err = dstObj.readMetaData(ctx) + if err != nil && !strings.HasSuffix(remote, trash_indicator) { return nil, err } @@ -1484,9 +1485,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, // // If destination exists then return fs.ErrorDirExists func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { - move_mutex.Lock() - defer move_mutex.Unlock() - force_update = true srcFs, ok := src.(*Fs) if !ok { fs.Debugf(srcFs, "Can't move directory - not same remote type") @@ -1748,8 +1746,10 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { // if all files are trashed fs.LogPrint(fs.LogLevelDebug, "all files of torrent: "+o.ParentID+" are in internal trash") + // fs.LogPrint(fs.LogLevelDebug, "remove locking file_mutex") file_mutex.Lock() defer file_mutex.Unlock() + // defer fs.LogPrint(fs.LogLevelDebug, "remove unlocking file_mutex") // read sort file file, err := os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) if err != nil { @@ -1793,7 +1793,7 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { } } fs.LogPrint(fs.LogLevelDebug, "removing realdebrid torrent id: "+o.ParentID) - path := "/torrents/delete/" + o.ParentID + path := "/torrents/delete/DISABLED" + o.ParentID opts := rest.Opts{ Method: "DELETE", Path: path, From 4c3015289b60a3f0e19d92ebfbe037adaa476b32 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sun, 27 Aug 2023 13:57:58 +0200 Subject: [PATCH 548/560] unrestrict only when opening files --- .vscode/launch.json | 2 +- backend/realdebrid/realdebrid.go | 254 +++++++++++++------------------ 2 files changed, 106 insertions(+), 150 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index c0342169ead16..a31a11690a292 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -18,7 +18,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["tree","realdebrid:/default/"], + "args": ["tree","realdebrid:"], "console": "integratedTerminal" }, { diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index e06085b70b61b..c1614597e35d4 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -85,10 +85,12 @@ type RegexValuePair struct { var cached []api.Item var torrents []api.Item var broken_torrents []string -var lastcheck int64 = time.Now().Unix() +var lastcheck int64 = 0 var lastFileMod int64 = 0 var interval int64 = 15 * 60 var file_mutex = &sync.RWMutex{} +var move_mutex = &sync.RWMutex{} +var moving = false var mapping = xsync.NewMap() var sorting_file = xsync.NewMap() var folders = xsync.NewMap() @@ -179,17 +181,18 @@ type Fs struct { // Object describes a file type Object struct { - fs *Fs // what this object is part of - remote string // The remote path - hasMetaData bool // metadata is present and correct - size int64 // size of the object - modTime time.Time // modification time of the object - id string // ID of the object - ParentID string // ID of parent directory - mimeType string // Mime type of object - url string // URL to download file - TorrentHash string // Torrent Hash - MappingID string // Internal Mapping ID used for sorting + fs *Fs // what this object is part of + remote string // The remote path + hasMetaData bool // metadata is present and correct + size int64 // size of the object + modTime time.Time // modification time of the object + id string // ID of the object + ParentID string // ID of parent directory + mimeType string // Mime type of object + url string // URL to download file + TorrentHash string // Torrent Hash + MappingID string // Internal Mapping ID used for sorting + OriginalLink string //Internal original Link } // ------------------------------------------------------------ @@ -634,12 +637,11 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi updated = true } - if ((dirID == rootID) || !(f.folder_exists(dirID)) || updated) && math.Abs(float64(fileModTime-lastFileMod)) >= 5 { + if ((dirID == rootID) || !(f.folder_exists(dirID)) || updated) && !moving { // Create folder structure if updated { // fs.LogPrint(fs.LogLevelDebug, "ListAll Rlocking file_mutex") file_mutex.RLock() - defer file_mutex.RUnlock() // defer fs.LogPrint(fs.LogLevelDebug, "ListAll Runlocking file_mutex") // read the file, create if missing file, err := os.Open(f.opt.SortFile) @@ -701,6 +703,8 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } } + file_mutex.RUnlock() + mapping.Range(func(key string, value interface{}) bool { oldValue, ok := sorting_file.Load(key) if !ok || oldValue != value { @@ -843,59 +847,23 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } } //iterate through files - var broken = false for _, link := range torrents[i].Links { err_code = 0 if link == "" { continue } var ItemFile api.Item - for _, cachedfile := range cached { - if cachedfile.OriginalLink == link { - ItemFile = cachedfile - break - } - } - if ItemFile.Link == "" { - fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) - path = "/unrestrict/link" - method = "POST" - opts := rest.Opts{ - Method: method, - Path: path, - MultipartParams: url.Values{ - "link": {link}, - }, - Parameters: f.baseParams(), - } - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - if err_code == 503 || err_code == 404 { - broken = true - break - } - var retries = 0 - for err_code == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - retries += 1 - } - } - if ItemFile.Name == "" { - continue - } - mapping_id := "/" + torrents[i].Name + "/" + ItemFile.Name + ItemFile.Name = strings.Split(link, "/")[len(strings.Split(link, "/"))-1] + ItemFile.ID = ItemFile.Name + mapping_id := "/" + torrents[i].Name + "/" + ItemFile.ID ItemFile.MappingID = mapping_id ItemFile.DefaultLocation = torrents[i].DefaultLocation + torrents[i].Name + "/" ItemFile.ParentID = torrents[i].ID ItemFile.TorrentHash = torrents[i].TorrentHash ItemFile.Generated = torrents[i].Generated ItemFile.Type = "file" + ItemFile.Link = link + ItemFile.OriginalLink = link if _, ok := mapping.Load(mapping_id); !ok { if _, ok := mapping.Load("/" + torrents[i].Name + "/"); ok { value, _ := mapping.Load("/" + torrents[i].Name + "/") @@ -936,86 +904,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi folders.Store(value.(string), append(list.([]api.Item), ItemFile)) } } - if broken { - torrents[i] = f.redownloadTorrent(ctx, torrents[i]) - for _, link := range torrents[i].Links { - err_code = 0 - var ItemFile api.Item - fs.LogPrint(fs.LogLevelDebug, "creating new unrestricted direct link for torrent: "+torrents[i].Name) - path = "/unrestrict/link" - method = "POST" - opts := rest.Opts{ - Method: method, - Path: path, - MultipartParams: url.Values{ - "link": {link}, - }, - Parameters: f.baseParams(), - } - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - var retries = 0 - for err_code == 429 && retries <= 5 { - time.Sleep(time.Duration(2) * time.Second) - resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) - if resp != nil { - err_code = resp.StatusCode - } - retries += 1 - } - if ItemFile.Name == "" { - continue - } - mapping_id := "/" + torrents[i].Name + "/" + ItemFile.Name - ItemFile.MappingID = mapping_id - ItemFile.DefaultLocation = torrents[i].DefaultLocation + torrents[i].Name + "/" - ItemFile.ParentID = torrents[i].ID - ItemFile.TorrentHash = torrents[i].TorrentHash - ItemFile.Generated = torrents[i].Generated - ItemFile.Type = "file" - if _, ok := mapping.Load(mapping_id); !ok { - if _, ok := mapping.Load("/" + torrents[i].Name + "/"); ok { - value, _ := mapping.Load("/" + torrents[i].Name + "/") - mapping.Store(mapping_id, value) - } else { - mapping.Store(mapping_id, ItemFile.DefaultLocation) - } - } else { - if value, _ := mapping.Load(mapping_id); len(value.(string)) > 0 { - value, _ := mapping.Load(mapping_id) - split := strings.Split(value.(string), "/") - if len(split) > 0 { - ItemFile.Name = split[len(strings.Split(value.(string), "/"))-1] - } - } - if ItemFile.Name == "" || strings.HasSuffix(ItemFile.Name, trash_indicator) { - continue - } - value, _ := mapping.Load(mapping_id) - mapping.Store(mapping_id, strings.Join(strings.Split(value.(string), "/")[:len(strings.Split(value.(string), "/"))-1], "/")+"/") - } - value, _ := mapping.Load(mapping_id) - if value != nil { - list, ok := folders.Load(value.(string)) - if !ok { - list = []api.Item{} - } - skip := false - for _, existing_item := range list.([]api.Item) { - if existing_item.Name == ItemFile.Name { - skip = true - break - } - } - if skip { - continue - } - folders.Store(value.(string), append(list.([]api.Item), ItemFile)) - } - } - } } // Iterate through the map @@ -1080,12 +968,67 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if value == nil { value = []api.Item{} } + result = append(result, value.([]api.Item)...) if err != nil { return newDirID, found, fmt.Errorf("couldn't list files: %w", err) } for i := range result { + // Turn temporary restricted items into unretricted items + if result[i].Type == api.ItemTypeFile { + expired := true + for _, cachedfile := range cached { + if cachedfile.OriginalLink == result[i].OriginalLink { + result[i].Name = cachedfile.Name + result[i].Link = cachedfile.Link + // result[i].ID = cachedfile.ID + expired = false + break + } + } + if expired { + fs.LogPrint(fs.LogLevelDebug, fmt.Sprintf("creating new link for file %s from torrent hash %s", result[i].Name, result[i].TorrentHash)) + var ItemFile api.Item + broken := false + path = "/unrestrict/link" + method = "POST" + opts := rest.Opts{ + Method: method, + Path: path, + MultipartParams: url.Values{ + "link": {result[i].OriginalLink}, + }, + Parameters: f.baseParams(), + } + err_code := 0 + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } + if err_code == 503 || err_code == 404 { + broken = true + } + if !broken { + var retries = 0 + for err_code == 429 && retries <= 5 { + time.Sleep(time.Duration(2) * time.Second) + resp, _ = f.srv.CallJSON(ctx, &opts, nil, &ItemFile) + if resp != nil { + err_code = resp.StatusCode + } + retries += 1 + } + if ItemFile.Link != "" && ItemFile.Name != "" { + result[i].Name = ItemFile.Name + result[i].Link = ItemFile.Link + // result[i].ID = ItemFile.ID + } + } else { + continue + } + } + } item := &result[i] layout := "2006-01-02T15:04:05.000Z" if item.Generated != "" { @@ -1285,10 +1228,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error { // move a file or folder // func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDirectoryID, newDirectoryID string) (err error) { - // fs.LogPrint(fs.LogLevelDebug, "move locking file_mutex") - file_mutex.Lock() - defer file_mutex.Unlock() - // defer fs.LogPrint(fs.LogLevelDebug, "move unlocking file_mutex") + // Handle IDs oldDirectoryID_o := oldDirectoryID for _, torrent := range torrents { @@ -1300,15 +1240,16 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir break } } - // Handle Files + // // Handle Files if isFile && len(id) == 13 { - for _, file := range cached { - if file.ID == id { - oldLeaf = file.Name - break - } - } + oldLeaf = id } + + // There was a change, so update the file requiring a lock + // fs.LogPrint(fs.LogLevelDebug, "move locking file_mutex") + file_mutex.Lock() + defer file_mutex.Unlock() + // defer fs.LogPrint(fs.LogLevelDebug, "move unlocking file_mutex") // Open the file file, err := os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) if err != nil { @@ -1316,6 +1257,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir return err } defer file.Close() + if last := newDirectoryID[len(newDirectoryID)-1]; last != '/' { newDirectoryID = newDirectoryID + "/" } @@ -1412,6 +1354,11 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir // If it isn't possible then return fs.ErrorCantMove func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { + move_mutex.Lock() + defer move_mutex.Unlock() + moving = true + defer func() { moving = false }() + srcObj, ok := src.(*Object) if !ok { fs.Debugf(src, "Can't move - not same remote type") @@ -1433,6 +1380,10 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, // Remove the old item from the folder structure oldDir, _ := mapping.LoadAndDelete(srcObj.MappingID) var newItems []api.Item + piss := folders.Size() + if piss == 123123123 { + return nil, nil + } items, _ := folders.Load(oldDir.(string)) for i := range items.([]api.Item) { if items.([]api.Item)[i].ID != srcObj.id { @@ -1450,11 +1401,12 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, } mapping.Store(srcObj.MappingID, directoryID) newItem := api.Item{} - newItem.Name = strings.Split(remote, "/")[len(strings.Split(remote, "/"))-1] + newItem.Name = srcObj.id // strings.Split(remote, "/")[len(strings.Split(remote, "/"))-1] newItem.Size = srcObj.size newItem.ID = srcObj.id newItem.MimeType = srcObj.mimeType newItem.Link = srcObj.url + newItem.OriginalLink = srcObj.OriginalLink newItem.ParentID = srcObj.ParentID newItem.TorrentHash = srcObj.TorrentHash newItem.MappingID = srcObj.MappingID @@ -1485,6 +1437,9 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, // // If destination exists then return fs.ErrorDirExists func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { + move_mutex.Lock() + defer move_mutex.Unlock() + srcFs, ok := src.(*Fs) if !ok { fs.Debugf(srcFs, "Can't move directory - not same remote type") @@ -1594,6 +1549,7 @@ func (o *Object) setMetaData(info *api.Item) (err error) { o.ParentID = info.ParentID o.TorrentHash = info.TorrentHash o.MappingID = info.MappingID + o.OriginalLink = info.OriginalLink return nil } @@ -1663,8 +1619,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read return nil, err } } - err = fmt.Errorf("Error opening file: '" + o.url + "'. This link seems to be broken. Torrent will be re-downloaded on next refresh.") - broken_torrents = append(broken_torrents, o.ParentID) + // err = fmt.Errorf("Error opening file: '" + o.url + "'. This link seems to be broken. Torrent will be re-downloaded.") + // broken_torrents = append(broken_torrents, o.ParentID) } return nil, err } From 66ebbda9d41f7955c87b85dc901266c845e26aaa Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sun, 27 Aug 2023 15:15:01 +0200 Subject: [PATCH 549/560] fixing no metadata found errors --- backend/realdebrid/realdebrid.go | 34 +++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index c1614597e35d4..e65c3e121788e 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -646,7 +646,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi // read the file, create if missing file, err := os.Open(f.opt.SortFile) if os.IsNotExist(err) { - fs.LogPrint(fs.LogLevelWarning, "no sorting file found. creating new empty sorting file.") + fs.LogPrint(fs.LogLevelWarning, "no sorting file found - creating new empty sorting file") file, err = os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) if err != nil { fmt.Println(err) @@ -664,7 +664,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } // Reset saved folder structure - fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file.") + fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file") regex_defs = []RegexValuePair{} eraseSyncMap(folders) eraseSyncMap(mapping) @@ -982,7 +982,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if cachedfile.OriginalLink == result[i].OriginalLink { result[i].Name = cachedfile.Name result[i].Link = cachedfile.Link - // result[i].ID = cachedfile.ID + result[i].Size = cachedfile.Size expired = false break } @@ -1022,11 +1022,21 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if ItemFile.Link != "" && ItemFile.Name != "" { result[i].Name = ItemFile.Name result[i].Link = ItemFile.Link - // result[i].ID = ItemFile.ID + result[i].Size = ItemFile.Size } } else { continue } + + } + if _, ok := sorting_file.Load(result[i].MappingID); ok { + if value, _ := sorting_file.Load(result[i].MappingID); len(value.(string)) > 0 { + value, _ := sorting_file.Load(result[i].MappingID) + split := strings.Split(value.(string), "/") + if len(split) > 0 { + result[i].Name = split[len(strings.Split(value.(string), "/"))-1] + } + } } } item := &result[i] @@ -1284,6 +1294,9 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir oldLeaf = oldLeaf + "/" } } + + // fs.LogPrint(fs.LogLevelDebug, fmt.Sprintf("moving oldLeaf %s from oldDirectoryID %s to newLeaf %s newDirectoryID %s", oldLeaf, oldDirectoryID, newLeaf, newDirectoryID)) + // Get all files that should be moved. var affected_items []string sorting_file.Range(func(key string, value interface{}) bool { @@ -1366,7 +1379,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, } // Create temporary object - dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) + _, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) if err != nil { return nil, err } @@ -1380,10 +1393,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, // Remove the old item from the folder structure oldDir, _ := mapping.LoadAndDelete(srcObj.MappingID) var newItems []api.Item - piss := folders.Size() - if piss == 123123123 { - return nil, nil - } items, _ := folders.Load(oldDir.(string)) for i := range items.([]api.Item) { if items.([]api.Item)[i].ID != srcObj.id { @@ -1401,7 +1410,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, } mapping.Store(srcObj.MappingID, directoryID) newItem := api.Item{} - newItem.Name = srcObj.id // strings.Split(remote, "/")[len(strings.Split(remote, "/"))-1] + newItem.Name = strings.Split(remote, "/")[len(strings.Split(remote, "/"))-1] newItem.Size = srcObj.size newItem.ID = srcObj.id newItem.MimeType = srcObj.mimeType @@ -1420,12 +1429,13 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, items = append(items.([]api.Item), newItem) folders.Store(directoryID, items) - err = dstObj.readMetaData(ctx) + // moving = false + // err = dstObj.readMetaData(ctx) if err != nil && !strings.HasSuffix(remote, trash_indicator) { return nil, err } - return dstObj, nil + return srcObj, nil } // DirMove moves src, srcRemote to this remote at dstRemote From dd4ffeff47a715593ca4ad1e5bf714dcbf344959 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sun, 27 Aug 2023 15:23:24 +0200 Subject: [PATCH 550/560] adjusted sorting file comment message --- backend/realdebrid/realdebrid.go | 4 ++-- sorting.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index e65c3e121788e..9233cd93eda8d 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -112,10 +112,10 @@ var default_sorting = `# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # - create new directories using "/foldername" # Example: /shit # -# - write move/renaming changes using: "/" + "actual torrent title" + "/" + "actual file name" + " -> " + "destination" +# - write move/renaming changes using: "/" + "actual torrent title" + "/" + "file ID" + " -> " + "destination" # You do not need to create the directories you are moving stuff to, this will be done automatically. # Example: /some.show.S01/ -> /shows/some.show/season 1/ -# Example: /some.show.S01/some.show.S01E01.mkv -> /shows/some.show/season 1/episode 1.mkv +# Example: /some.show.S01/ABCDEFGHIJKL -> /shows/some.show/season 1/episode 1.mkv # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~ top level and regex folders: ~~~~~~~~~~ diff --git a/sorting.txt b/sorting.txt index c1f3499beb836..a4368b228b91e 100644 --- a/sorting.txt +++ b/sorting.txt @@ -12,10 +12,10 @@ # - create new directories using "/foldername" # Example: /shit # -# - write move/renaming changes using: "/" + "actual torrent title" + "/" + "actual file name" + " -> " + "destination" +# - write move/renaming changes using: "/" + "actual torrent title" + "/" + "file ID" + " -> " + "destination" # You do not need to create the directories you are moving stuff to, this will be done automatically. # Example: /some.show.S01/ -> /shows/some.show/season 1/ -# Example: /some.show.S01/some.show.S01E01.mkv -> /shows/some.show/season 1/episode 1.mkv +# Example: /some.show.S01/ABCDEFGHIJKL -> /shows/some.show/season 1/episode 1.mkv # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~ top level and regex folders: ~~~~~~~~~~ From 2a8711a35bd182045c61e2af519c852d615d2235 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Sun, 27 Aug 2023 16:05:31 +0200 Subject: [PATCH 551/560] re-enable deleting --- backend/realdebrid/realdebrid.go | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 9233cd93eda8d..518bca0073163 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -146,7 +146,7 @@ func init() { Name: "sort_file", Help: `please provide the full path to a file (file does not need to exist) that should be used for sorting`, Advanced: true, - Default: "./sorting.txt", + Default: path.Join(path.Dir(config.GetConfigPath()), "sorting.txt"), }, { Name: config.ConfigEncoding, Help: config.ConfigEncodingHelp, @@ -1685,28 +1685,6 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { // move file to trash fs.LogPrint(fs.LogLevelDebug, "moving file: "+o.MappingID+" to internal trash") f.Move(ctx, o, o.remote+trash_indicator) - - // // delete the link on realdebrid - // fs.LogPrint(fs.LogLevelDebug, "removing realdebrid link id: "+o.id) - // path := "/downloads/delete/" + o.id - // opts := rest.Opts{ - // Method: "DELETE", - // Path: path, - // Parameters: f.baseParams(), - // } - // err_code := 0 - // resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) - // if resp != nil { - // err_code = resp.StatusCode - // } - // for err_code == 429 && retries <= 5 { - // time.Sleep(time.Duration(2) * time.Second) - // resp, _ = f.srv.CallJSON(ctx, &opts, nil, &result) - // if resp != nil { - // err_code = resp.StatusCode - // } - // retries += 1 - // } return nil } @@ -1759,7 +1737,7 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { } } fs.LogPrint(fs.LogLevelDebug, "removing realdebrid torrent id: "+o.ParentID) - path := "/torrents/delete/DISABLED" + o.ParentID + path := "/torrents/delete/" + o.ParentID opts := rest.Opts{ Method: "DELETE", Path: path, From 1a8b0dfc095262013c7285ea7ddf0a875143f582 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Mon, 28 Aug 2023 08:50:11 +0200 Subject: [PATCH 552/560] fix some torrents not redownloading, timezone stuff --- backend/realdebrid/realdebrid.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 518bca0073163..3fcf5849c3127 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -834,7 +834,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi broken = true } } - if torrents[i].Status == "dead" || broken { + if torrents[i].Status == "dead" || torrents[i].Status == "error" || broken { torrents[i] = f.redownloadTorrent(ctx, torrents[i]) } //set default torrents[i] location @@ -974,6 +974,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if err != nil { return newDirID, found, fmt.Errorf("couldn't list files: %w", err) } + loc, _ := time.LoadLocation("Europe/Paris") for i := range result { // Turn temporary restricted items into unretricted items if result[i].Type == api.ItemTypeFile { @@ -1025,7 +1026,12 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi result[i].Size = ItemFile.Size } } else { - continue + for k, torrent := range torrents { + if torrent.ID == result[i].ParentID { + torrents[k] = f.redownloadTorrent(ctx, torrents[k]) + break + } + } } } @@ -1042,10 +1048,10 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi item := &result[i] layout := "2006-01-02T15:04:05.000Z" if item.Generated != "" { - t, _ := time.Parse(layout, item.Generated) + t, _ := time.ParseInLocation(layout, item.Ended, loc) item.CreatedAt = t.Unix() } else if item.Ended != "" { - t, _ := time.Parse(layout, item.Ended) + t, _ := time.ParseInLocation(layout, item.Ended, loc) item.CreatedAt = t.Unix() } if item.Type == api.ItemTypeFolder { From 3f75f4408fd89b1332253cdb7c08eb3053005d9c Mon Sep 17 00:00:00 2001 From: itsToggle Date: Mon, 28 Aug 2023 09:44:08 +0200 Subject: [PATCH 553/560] small improvements when opening broken files --- backend/realdebrid/realdebrid.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 3fcf5849c3127..af705a87d89cd 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -1626,6 +1626,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read if resp != nil { err_code = resp.StatusCode } + if err_code == 503 || err_code == 404 { + return false, err + } return shouldRetry(ctx, resp, err) }) if err != nil { @@ -1635,8 +1638,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read return nil, err } } - // err = fmt.Errorf("Error opening file: '" + o.url + "'. This link seems to be broken. Torrent will be re-downloaded.") - // broken_torrents = append(broken_torrents, o.ParentID) + err = fmt.Errorf("error opening file: '" + o.url + "' this link seems to be broken - torrent will be re-downloaded") + broken_torrents = append(broken_torrents, o.ParentID) } return nil, err } From f996b5e36c29ffad811b656725a7d671ba45ccbc Mon Sep 17 00:00:00 2001 From: itsToggle Date: Mon, 28 Aug 2023 10:22:15 +0200 Subject: [PATCH 554/560] correct file date --- backend/realdebrid/realdebrid.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index af705a87d89cd..e0b0d6fac4c8f 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -861,6 +861,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi ItemFile.ParentID = torrents[i].ID ItemFile.TorrentHash = torrents[i].TorrentHash ItemFile.Generated = torrents[i].Generated + ItemFile.Ended = torrents[i].Ended ItemFile.Type = "file" ItemFile.Link = link ItemFile.OriginalLink = link From 869702c7dd46a0a152d397d800a374ce07330e25 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Mon, 28 Aug 2023 11:32:46 +0200 Subject: [PATCH 555/560] fix removing torrents --- .vscode/launch.json | 2 +- backend/realdebrid/realdebrid.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index a31a11690a292..313d2bfe20825 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -38,7 +38,7 @@ "mode": "auto", "program": "${fileDirname}", "env": {}, - "args": ["delete","realdebrid:/default/some.show/season 1"], + "args": ["delete","realdebrid:/shows/Archer Season 3 [2160p AI x265 S98 Joy]"], "console": "integratedTerminal" }, ] diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index e0b0d6fac4c8f..66436a0d6de8e 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -1676,6 +1676,16 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { } } + // Check if o.MappingID exists in sorting_file and update its value + value, exists := sorting_file.Load(o.MappingID) + if exists { + // Append trash_indicator to the existing value + sorting_file.Store(o.MappingID, value.(string)+trash_indicator) + } else { + // Add o.MappingID as a key with the value o.MappingID + trash_indicator + sorting_file.Store(o.MappingID, o.MappingID+trash_indicator) + } + // Get all trashed files var affected_items []string sorting_file.Range(func(key string, value interface{}) bool { @@ -1686,9 +1696,6 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { } return true }) - if len(affected_items) == 0 { - affected_items = append(affected_items, o.MappingID) - } // if not all files are trashed if len(affected_items) < torrent_files { From 4a18c72497d0dd9093df48f44db3b394bf73fee2 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Tue, 29 Aug 2023 12:47:52 +0200 Subject: [PATCH 556/560] fix redownloading dead torrent loops --- backend/realdebrid/realdebrid.go | 69 ++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 66436a0d6de8e..e27082f301bba 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -85,6 +85,7 @@ type RegexValuePair struct { var cached []api.Item var torrents []api.Item var broken_torrents []string +var broken_links []string var lastcheck int64 = 0 var lastFileMod int64 = 0 var interval int64 = 15 * 60 @@ -460,7 +461,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e // Redownload a dead torrent func (f *Fs) redownloadTorrent(ctx context.Context, torrent api.Item) (redownloaded_torrent api.Item) { - fs.LogPrint(fs.LogLevelNotice, "Redownloading dead torrent: "+torrent.Name) + fs.LogPrint(fs.LogLevelNotice, "redownloading dead torrent: "+torrent.Name) //Get dead torrent file and hash info var method = "GET" var path = "/torrents/info/" + torrent.ID @@ -990,7 +991,26 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } } if expired { - fs.LogPrint(fs.LogLevelDebug, fmt.Sprintf("creating new link for file %s from torrent hash %s", result[i].Name, result[i].TorrentHash)) + count := 0 + for _, link := range broken_links { + if link == result[i].OriginalLink { + count += 1 + } + } + if count > 1 { + // a redownload of the torrent did not fix the issue + fs.LogPrint(fs.LogLevelDebug, fmt.Sprintf("redownloading did not fix file: %s from torrent hash: %s", result[i].Name, result[i].TorrentHash)) + location := result[i].DefaultLocation + result[i].Name + if value, ok := mapping.Load(result[i].MappingID); ok { + if len(value.(string)) > 0 { + location = value.(string) + } + } + expired_object, _ := f.newObjectWithInfo(ctx, location, &result[i]) + f.remove(ctx, expired_object.(*Object)) + continue + } + fs.LogPrint(fs.LogLevelDebug, fmt.Sprintf("creating new link for file: %s from torrent hash: %s", result[i].Name, result[i].TorrentHash)) var ItemFile api.Item broken := false path = "/unrestrict/link" @@ -1008,7 +1028,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if resp != nil { err_code = resp.StatusCode } - if err_code == 503 || err_code == 404 { + if err_code == 503 || err_code == 404 || resp == nil { broken = true } if !broken { @@ -1027,12 +1047,29 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi result[i].Size = ItemFile.Size } } else { + broken_links = append(broken_links, result[i].OriginalLink) + count := 0 + for _, link := range broken_links { + if link == result[i].OriginalLink { + count += 1 + } + } + if count > 1 { + // a redownload of the torrent did not fix the issue + fs.LogPrint(fs.LogLevelDebug, fmt.Sprintf("redownloading did not fix file: %s from torrent hash: %s", result[i].Name, result[i].TorrentHash)) + location := "broken" + result[i].Name = "broken" + expired_object, _ := f.newObjectWithInfo(ctx, location, &result[i]) + f.remove(ctx, expired_object.(*Object)) + continue + } for k, torrent := range torrents { if torrent.ID == result[i].ParentID { torrents[k] = f.redownloadTorrent(ctx, torrents[k]) break } } + } } @@ -1073,6 +1110,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi break } } + return } @@ -1632,17 +1670,22 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } return shouldRetry(ctx, resp, err) }) - if err != nil { - if err_code == 503 || err_code == 404 { - for _, TorrentID := range broken_torrents { - if o.ParentID == TorrentID { - return nil, err - } + + if err != nil || err_code == 404 || err_code == 503 { + if err_code == 404 || (err != nil && strings.Contains(err.Error(), "404")) { + err_code = 404 + } else if err_code == 503 || (err != nil && strings.Contains(err.Error(), "503")) { + err_code = 503 + } else { + return nil, fmt.Errorf("error: (unknown) accessing url: %s", o.url) + } + for _, TorrentID := range broken_torrents { + if o.ParentID == TorrentID { + return nil, err } - err = fmt.Errorf("error opening file: '" + o.url + "' this link seems to be broken - torrent will be re-downloaded") - broken_torrents = append(broken_torrents, o.ParentID) } - return nil, err + broken_torrents = append(broken_torrents, o.ParentID) + return nil, fmt.Errorf("error: %d accessing url: %s", err_code, o.url) } return resp.Body, err } @@ -1698,7 +1741,7 @@ func (f *Fs) remove(ctx context.Context, o *Object) (err error) { }) // if not all files are trashed - if len(affected_items) < torrent_files { + if len(affected_items) < torrent_files || o.remote == "broken" { // move file to trash fs.LogPrint(fs.LogLevelDebug, "moving file: "+o.MappingID+" to internal trash") f.Move(ctx, o, o.remote+trash_indicator) From e6581a6a4ce8eb725ddb3c8b7c28ec438e824a34 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Tue, 29 Aug 2023 13:36:49 +0200 Subject: [PATCH 557/560] fix changes to the realdebrid website not reflecting immediatly --- backend/realdebrid/realdebrid.go | 159 +++++++++++++++---------------- 1 file changed, 79 insertions(+), 80 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index e27082f301bba..49552bfcad2ad 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -639,82 +639,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } if ((dirID == rootID) || !(f.folder_exists(dirID)) || updated) && !moving { - // Create folder structure - if updated { - // fs.LogPrint(fs.LogLevelDebug, "ListAll Rlocking file_mutex") - file_mutex.RLock() - // defer fs.LogPrint(fs.LogLevelDebug, "ListAll Runlocking file_mutex") - // read the file, create if missing - file, err := os.Open(f.opt.SortFile) - if os.IsNotExist(err) { - fs.LogPrint(fs.LogLevelWarning, "no sorting file found - creating new empty sorting file") - file, err = os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) - if err != nil { - fmt.Println(err) - } - defer file.Close() - - if _, err := file.WriteString(default_sorting); err != nil { - fmt.Println(err) - } - file.Sync() // flush file contents to disk - } else if err != nil { - fmt.Println(err) - } else { - defer file.Close() - } - - // Reset saved folder structure - fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file") - regex_defs = []RegexValuePair{} - eraseSyncMap(folders) - eraseSyncMap(mapping) - eraseSyncMap(sorting_file) - - // Read the file line by line - if _, err := file.Seek(0, io.SeekStart); err != nil { - fmt.Println(err) - } - scanner := bufio.NewScanner(file) - for scanner.Scan() { - if strings.HasPrefix(scanner.Text(), "#") { - continue - } else if len(scanner.Text()) == 0 || scanner.Text() == "\n" || scanner.Text() == "\n\r" { - continue - } else if strings.Contains(scanner.Text(), move_chars) { - // Split the line by " -> " - parts := strings.Split(scanner.Text(), move_chars) - // Add the key-value pair to the map if not present or changed - oldValue, ok := mapping.Load(parts[0]) - if !ok || oldValue.(string) != parts[1] { - mapping.Store(parts[0], parts[1]) - } - } else if strings.Contains(scanner.Text(), regx_chars) { - // Split the line by " == " - parts := strings.Split(scanner.Text(), regx_chars) - // Add the key-value pair to the map - r, _ := regexp.Compile(parts[1]) - regex_defs = append(regex_defs, RegexValuePair{r, parts[0]}) - - } else { - oldValue, ok := mapping.Load(scanner.Text()) - if !ok || oldValue.(string) != scanner.Text() { - mapping.Store(scanner.Text(), scanner.Text()) - } - } - } - - file_mutex.RUnlock() - - mapping.Range(func(key string, value interface{}) bool { - oldValue, ok := sorting_file.Load(key) - if !ok || oldValue != value { - sorting_file.Store(key, value) - } - return true - }) - - } //update global cached list opts := rest.Opts{ Method: method, @@ -748,7 +672,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) if err == nil { if totalcount != len(cached) || time.Now().Unix()-lastcheck > interval { - if time.Now().Unix()-lastcheck > interval && !printed { + if !printed { fs.LogPrint(fs.LogLevelDebug, "updating all links and torrents") printed = true } @@ -823,10 +747,85 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi //fmt.Printf("Done.\n") torrents = newtorrents - // Iterate through built file and torrent list: - var broken = false - err_code = 0 + // Create folder structure if updated { + // fs.LogPrint(fs.LogLevelDebug, "ListAll Rlocking file_mutex") + file_mutex.RLock() + // defer fs.LogPrint(fs.LogLevelDebug, "ListAll Runlocking file_mutex") + // read the file, create if missing + file, err := os.Open(f.opt.SortFile) + if os.IsNotExist(err) { + fs.LogPrint(fs.LogLevelWarning, "no sorting file found - creating new empty sorting file") + file, err = os.OpenFile(f.opt.SortFile, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + fmt.Println(err) + } + defer file.Close() + + if _, err := file.WriteString(default_sorting); err != nil { + fmt.Println(err) + } + file.Sync() // flush file contents to disk + } else if err != nil { + fmt.Println(err) + } else { + defer file.Close() + } + + // Reset saved folder structure + fs.LogPrint(fs.LogLevelDebug, "reading updated sorting file") + regex_defs = []RegexValuePair{} + eraseSyncMap(folders) + eraseSyncMap(mapping) + eraseSyncMap(sorting_file) + + // Read the file line by line + if _, err := file.Seek(0, io.SeekStart); err != nil { + fmt.Println(err) + } + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "#") { + continue + } else if len(scanner.Text()) == 0 || scanner.Text() == "\n" || scanner.Text() == "\n\r" { + continue + } else if strings.Contains(scanner.Text(), move_chars) { + // Split the line by " -> " + parts := strings.Split(scanner.Text(), move_chars) + // Add the key-value pair to the map if not present or changed + oldValue, ok := mapping.Load(parts[0]) + if !ok || oldValue.(string) != parts[1] { + mapping.Store(parts[0], parts[1]) + } + } else if strings.Contains(scanner.Text(), regx_chars) { + // Split the line by " == " + parts := strings.Split(scanner.Text(), regx_chars) + // Add the key-value pair to the map + r, _ := regexp.Compile(parts[1]) + regex_defs = append(regex_defs, RegexValuePair{r, parts[0]}) + + } else { + oldValue, ok := mapping.Load(scanner.Text()) + if !ok || oldValue.(string) != scanner.Text() { + mapping.Store(scanner.Text(), scanner.Text()) + } + } + } + + file_mutex.RUnlock() + + mapping.Range(func(key string, value interface{}) bool { + oldValue, ok := sorting_file.Load(key) + if !ok || oldValue != value { + sorting_file.Store(key, value) + } + return true + }) + + // Iterate through built file and torrent list: + var broken = false + err_code = 0 + for i := range torrents { //handle dead torrents broken = false From a0a532d56a45cf47e73179892d626c256a283ea3 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Tue, 29 Aug 2023 13:39:01 +0200 Subject: [PATCH 558/560] add updating message --- backend/realdebrid/realdebrid.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 49552bfcad2ad..82a8e0f630c5a 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -725,6 +725,10 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi totalcount, err = strconv.Atoi(resp.Header["X-Total-Count"][0]) if err == nil { if totalcount != len(torrents) || time.Now().Unix()-lastcheck > interval { + if !printed { + fs.LogPrint(fs.LogLevelDebug, "updating all links and torrents") + printed = true + } newtorrents = append(newtorrents, partialresult...) opts.Parameters.Set("offset", strconv.Itoa(len(newtorrents))) opts.Parameters.Set("limit", "2500") From b79464c6c345e66823a43b4deea732982b2bc402 Mon Sep 17 00:00:00 2001 From: itsToggle Date: Tue, 29 Aug 2023 15:19:43 +0200 Subject: [PATCH 559/560] improve complete refresh timer --- backend/realdebrid/realdebrid.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 82a8e0f630c5a..8a86edae335bb 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -746,7 +746,9 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi } // Set everything as being up to date - lastcheck = time.Now().Unix() + if time.Now().Unix()-lastcheck > interval { + lastcheck = time.Now().Unix() + } lastFileMod = fileModTime //fmt.Printf("Done.\n") torrents = newtorrents From 5b2e10bbab19c46165ae3d42b9a30443ec8edecf Mon Sep 17 00:00:00 2001 From: itsToggle Date: Wed, 30 Aug 2023 09:08:03 +0200 Subject: [PATCH 560/560] better RD error messaging --- backend/realdebrid/realdebrid.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/realdebrid/realdebrid.go b/backend/realdebrid/realdebrid.go index 8a86edae335bb..1dd5a5cdec986 100644 --- a/backend/realdebrid/realdebrid.go +++ b/backend/realdebrid/realdebrid.go @@ -981,7 +981,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi if err != nil { return newDirID, found, fmt.Errorf("couldn't list files: %w", err) } - loc, _ := time.LoadLocation("Europe/Paris") + loc := time.FixedZone("Europe/Paris", 2*60*60) for i := range result { // Turn temporary restricted items into unretricted items if result[i].Type == api.ItemTypeFile { @@ -1670,9 +1670,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read if resp != nil { err_code = resp.StatusCode } - if err_code == 503 || err_code == 404 { - return false, err - } return shouldRetry(ctx, resp, err) }) @@ -1689,8 +1686,14 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read return nil, err } } - broken_torrents = append(broken_torrents, o.ParentID) - return nil, fmt.Errorf("error: %d accessing url: %s", err_code, o.url) + msg := "" + if err != nil && strings.Contains(err.Error(), "
      ") { + msg = strings.Split(err.Error(), "
      ")[len(strings.Split(err.Error(), "
      "))-1] + msg = strings.Split(msg, "
      ")[0] + msg = "(" + msg + ") " + } + // broken_torrents = append(broken_torrents, o.ParentID) + return nil, fmt.Errorf("error: %d %saccessing url: %s", err_code, msg, o.url) } return resp.Body, err }