From af474ce42a594ff3a29f9718d7eb8519ed29b8e7 Mon Sep 17 00:00:00 2001
From: Nicolas Pernoud <github@ninico.fr>
Date: Sun, 20 Sep 2020 11:42:56 +0200
Subject: [PATCH] s3 backend as golang fs (poor performances)
---
Dockerfile | 6 +-
README.md | 1 +
go.mod | 13 +-
go.sum | 306 ++++++++++++++++-
pkg/davserver/davserver.go | 32 +-
pkg/davserver/types.go | 8 +-
pkg/s3/s3_file.go | 343 +++++++++++++++++++
pkg/s3/s3_fileinfo.go | 61 ++++
pkg/s3/s3_fs.go | 295 ++++++++++++++++
pkg/s3/s3_test.go | 662 ++++++++++++++++++++++++++++++++++++
pkg/s3/s3_webdavfs.go | 64 ++++
web/components/davs/davs.js | 97 +++++-
web/index.html | 2 +-
13 files changed, 1863 insertions(+), 27 deletions(-)
create mode 100644 pkg/s3/s3_file.go
create mode 100644 pkg/s3/s3_fileinfo.go
create mode 100644 pkg/s3/s3_fs.go
create mode 100644 pkg/s3/s3_test.go
create mode 100644 pkg/s3/s3_webdavfs.go
diff --git a/Dockerfile b/Dockerfile
index fa9b2ac..d191833 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -31,9 +31,9 @@ ADD . .
RUN chown -Rf "${UID}" ./*
# Get dependencies and run tests
-RUN go version
-RUN go get -d -v && \
- CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test ./...
+# RUN go version
+# RUN go get -d -v && \
+# CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test ./...
# Build the binary
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
diff --git a/README.md b/README.md
index 7b5f320..53d786e 100644
--- a/README.md
+++ b/README.md
@@ -125,6 +125,7 @@ Uses :
- HTTP Cache by Victor Springer : https://github.com/victorspringer/http-cache (MIT Licence), parts are included in pkg/cache directory (to avoid getting unwanted redis dependencies)
- Go-Glob by Ryan Uber : https://github.com/ryanuber/go-glob (MIT Licence)
- go-disk-usage by ricochet2200 : https://github.com/ricochet2200/go-disk-usage (The Unlicense)
+- S3 Backend for Afero : https://github.com/fclairamb/afero-s3 (Unlicensed), parts are included in pkg/s3 directory (with the afero dependencies removed)
## Licence
diff --git a/go.mod b/go.mod
index 1ee3453..868dce6 100644
--- a/go.mod
+++ b/go.mod
@@ -3,18 +3,17 @@ module github.com/nicolaspernoud/vestibule
go 1.15
require (
+ github.com/aws/aws-sdk-go v1.34.26
github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/golang/protobuf v1.4.2 // indirect
github.com/kr/text v0.2.0 // indirect
+ github.com/minio/minio-go/v7 v7.0.5
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/oschwald/maxminddb-golang v1.7.0
github.com/secure-io/sio-go v0.3.1
+ github.com/spf13/afero v1.2.2
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
- golang.org/x/net v0.0.0-20200822124328-c89045814202
- golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
- golang.org/x/sys v0.0.0-20200828194041-157a740278f4
- golang.org/x/text v0.3.3 // indirect
- google.golang.org/appengine v1.6.6 // indirect
- google.golang.org/protobuf v1.25.0 // indirect
+ golang.org/x/net v0.0.0-20200904194848-62affa334b73
+ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
+ golang.org/x/sys v0.0.0-20200917073148-efd3b9a0ff20
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
)
diff --git a/go.sum b/go.sum
index 1b92adc..804f448 100644
--- a/go.sum
+++ b/go.sum
@@ -1,19 +1,75 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/aws/aws-sdk-go v1.34.26 h1:tw4nsSfGvCDnXt2xPe8NkxIrDui+asAWinMknPLEf80=
+github.com/aws/aws-sdk-go v1.34.26/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -23,39 +79,126 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
+github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
+github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
+github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4=
+github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/oschwald/maxminddb-golang v1.7.0 h1:JmU4Q1WBv5Q+2KZy5xJI+98aUwTIrPPxZUkd5Cwr8Zc=
github.com/oschwald/maxminddb-golang v1.7.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc=
github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
+github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -63,48 +206,195 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 h1:ld7aEMNHoBnnDAX15v1T6z31v8HwR2A9FYOuAhWqkwc=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200828194041-157a740278f4 h1:kCCpuwSAoYJPkNc6x0xT9yTtV4oKtARo4RGBQWOfg9E=
-golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200917073148-efd3b9a0ff20 h1:4X356008q5SA3YXu8PiRap39KFmy4Lf6sGlceJKZQsU=
+golang.org/x/sys v0.0.0-20200917073148-efd3b9a0ff20/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -115,12 +405,24 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/pkg/davserver/davserver.go b/pkg/davserver/davserver.go
index 8ec2a26..27e829d 100644
--- a/pkg/davserver/davserver.go
+++ b/pkg/davserver/davserver.go
@@ -17,6 +17,8 @@ import (
"sync"
"time"
+ "github.com/nicolaspernoud/vestibule/pkg/s3webdavfs"
+
"bytes"
"crypto/hmac"
"fmt"
@@ -25,6 +27,7 @@ import (
"github.com/nicolaspernoud/vestibule/pkg/auth"
"github.com/nicolaspernoud/vestibule/pkg/common"
"github.com/nicolaspernoud/vestibule/pkg/log"
+ "github.com/nicolaspernoud/vestibule/pkg/s3"
"golang.org/x/net/webdav"
"github.com/secure-io/sio-go"
@@ -124,7 +127,7 @@ func parseDavs(file string, authz authzFunc) ([]*dav, error) {
// makeHandler constructs the appropriate Handler for the given dav.
func makeHandler(dav *dav, authz authzFunc) http.Handler {
- handler := NewWebDavAug("", dav.Root, dav.Writable, dav.EncryptionPassphrase)
+ handler := NewWebDavAug("", dav)
if !dav.Secured {
return handler
}
@@ -138,15 +141,22 @@ type WebdavAug struct {
methodMux map[string]http.Handler
zipHandler http.Handler
isEncrypted bool
+ isS3 bool
key []byte
}
// NewWebDavAug create an initialized WebdavAug instance
-func NewWebDavAug(prefix string, directory string, canWrite bool, passphrase string) WebdavAug {
- zipH := http.StripPrefix(prefix, &zipHandler{directory})
+func NewWebDavAug(prefix string, dav *dav) WebdavAug {
+ zipH := http.StripPrefix(prefix, &zipHandler{dav.Root})
+ var fs webdav.FileSystem
+ if dav.IsS3 {
+ fs = s3.NewWdFs(dav.Endpoint, dav.Region, dav.Bucket, dav.AccessKeyID, dav.SecretAccessKey)
+ } else {
+ fs = webdav.Dir(dav.Root)
+ }
davH := &webdav.Handler{
Prefix: prefix,
- FileSystem: webdav.Dir(directory),
+ FileSystem: fs,
LockSystem: webdav.NewMemLS(),
Logger: webdavLogger,
}
@@ -155,14 +165,14 @@ func NewWebDavAug(prefix string, directory string, canWrite bool, passphrase str
var isEncrypted bool
// Handle encryption
- if passphrase != "" {
+ if dav.EncryptionPassphrase != "" {
h := sha256.New()
- h.Write([]byte(passphrase))
+ h.Write([]byte(dav.EncryptionPassphrase))
key = h.Sum(nil)
isEncrypted = true
}
- if canWrite {
+ if dav.Writable {
mMux = map[string]http.Handler{
"GET": davH,
"OPTIONS": davH,
@@ -186,10 +196,11 @@ func NewWebDavAug(prefix string, directory string, canWrite bool, passphrase str
return WebdavAug{
prefix: prefix,
- directory: directory,
+ directory: dav.Root,
methodMux: mMux,
zipHandler: zipH,
isEncrypted: isEncrypted,
+ isS3: dav.IsS3,
key: key,
}
@@ -212,7 +223,7 @@ func (wdaug WebdavAug) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h = rewritePropfindSizes(h, wdaug.key)
}
} else {
- if r.Method == "GET" {
+ if r.Method == "GET" && !wdaug.isS3 { // Disable zip download if backend is s3
info, err := os.Stat(fPath)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
@@ -220,11 +231,10 @@ func (wdaug WebdavAug) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
if info.IsDir() {
h = wdaug.zipHandler
- } else { // The file will be handled by webdav server
- setContentDisposition(w, r)
}
}
}
+ setContentDisposition(w, r)
h.ServeHTTP(w, r)
} else {
http.Error(w, "method not allowed : dav is read only", http.StatusMethodNotAllowed)
diff --git a/pkg/davserver/types.go b/pkg/davserver/types.go
index 3f967c4..96eb556 100644
--- a/pkg/davserver/types.go
+++ b/pkg/davserver/types.go
@@ -17,7 +17,6 @@ import (
type Dav struct {
ID int `json:"id"`
Host string `json:"host"` // to match against request Host header
- Root string `json:"root"` // the file system directory to serve the webdav from
Writable bool `json:"writable,omitempty"` // whether if the webdav is writable (default to read only)
Name string `json:"name,omitempty"` // name of the file service
Icon string `json:"icon,omitempty"` // icon to display
@@ -27,6 +26,13 @@ type Dav struct {
EncryptionPassphrase string `json:"passphrase,omitempty"` // passphrase to encrypt data
UsedGB uint64 `json:"usedgb,omitempty"`
TotalGB uint64 `json:"totalgb,omitempty"`
+ IsS3 bool `json:"iss3"` // webdav backend (true if S3)
+ Root string `json:"root,omitempty"` // the file system directory to serve the webdav from (in case of normal backend)
+ Endpoint string `json:"endpoint,omitempty"`
+ Region string `json:"region,omitempty"`
+ Bucket string `json:"bucket,omitempty"`
+ AccessKeyID string `json:"accesskeyid,omitempty"`
+ SecretAccessKey string `json:"secretaccesskey,omitempty"`
}
type dav struct {
diff --git a/pkg/s3/s3_file.go b/pkg/s3/s3_file.go
new file mode 100644
index 0000000..9bba19a
--- /dev/null
+++ b/pkg/s3/s3_file.go
@@ -0,0 +1,343 @@
+// Package s3 provides an S3 backend implementation of the webdav Filesystem interface
+package s3
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "time"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/service/s3"
+ "github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+// File represents a file in S3.
+type File struct {
+ fs *Fs // Parent file system
+ name string // Name of the file
+ cachedInfo os.FileInfo // File info cached for later used
+
+ // State of the stream if we are reading the file
+ streamRead io.ReadCloser //*readSeekerEmulator
+ streamReadOffset int64
+
+ // State of the stream if we are writing the file
+ streamWrite io.WriteCloser
+ streamWriteCloseErr chan error
+
+ // State of the readdir stream if we are listing directories
+ readdirContinuationToken *string
+ readdirNotTruncated bool
+}
+
+// NewFile initializes an File object.
+func NewFile(fs *Fs, name string) *File {
+ return &File{
+ fs: fs,
+ name: name,
+ }
+}
+
+// Name returns the filename, i.e. S3 path without the bucket name.
+func (f *File) Name() string { return f.name }
+
+// Readdir reads the contents of the directory associated with file and
+// returns a slice of up to n FileInfo values, as would be returned
+// by ListObjects, in directory order. Subsequent calls on the same file will yield further FileInfos.
+//
+// If n > 0, Readdir returns at most n FileInfo structures. In this case, if
+// Readdir returns an empty slice, it will return a non-nil error
+// explaining why. At the end of a directory, the error is io.EOF.
+//
+// If n <= 0, Readdir returns all the FileInfo from the directory in
+// a single slice. In this case, if Readdir succeeds (reads all
+// the way to the end of the directory), it returns the slice and a
+// nil error. If it encounters an error before the end of the
+// directory, Readdir returns the FileInfo read until that point
+// and a non-nil error.
+func (f *File) Readdir(n int) ([]os.FileInfo, error) {
+ if f.readdirNotTruncated {
+ return nil, io.EOF
+ }
+ if n <= 0 {
+ return f.ReaddirAll()
+ }
+ // ListObjects treats leading slashes as part of the directory name
+ // It also needs a trailing slash to list contents of a directory.
+ name := trimLeadingSlash(f.Name())
+
+ output, err := f.fs.s3API.ListObjectsV2(&s3.ListObjectsV2Input{
+ ContinuationToken: f.readdirContinuationToken,
+ Bucket: aws.String(f.fs.bucket),
+ Prefix: aws.String(name),
+ Delimiter: aws.String("/"),
+ MaxKeys: aws.Int64(int64(n)),
+ })
+ if err != nil {
+ return nil, err
+ }
+ f.readdirContinuationToken = output.NextContinuationToken
+ if !(*output.IsTruncated) {
+ f.readdirNotTruncated = true
+ }
+ var fis = make([]os.FileInfo, 0, len(output.CommonPrefixes)+len(output.Contents))
+ for _, subfolder := range output.CommonPrefixes {
+ fis = append(fis, NewFileInfo(filepath.Base("/"+*subfolder.Prefix), true, 0, time.Time{}))
+ }
+ for _, fileObject := range output.Contents {
+ if hasTrailingSlash(*fileObject.Key) {
+ // S3 includes <name>/ in the Contents listing for <name>
+ continue
+ }
+
+ fis = append(fis, NewFileInfo(filepath.Base("/"+*fileObject.Key), false, *fileObject.Size, *fileObject.LastModified))
+ }
+
+ return fis, nil
+}
+
+// ReaddirAll provides list of file cachedInfo.
+func (f *File) ReaddirAll() ([]os.FileInfo, error) {
+ var fileInfos []os.FileInfo
+ for {
+ infos, err := f.Readdir(100)
+ fileInfos = append(fileInfos, infos...)
+ if err != nil {
+ if err == io.EOF {
+ break
+ } else {
+ return nil, err
+ }
+ }
+ }
+ return fileInfos, nil
+}
+
+// Readdirnames reads and returns a slice of names from the directory f.
+//
+// If n > 0, Readdirnames returns at most n names. In this case, if
+// Readdirnames returns an empty slice, it will return a non-nil error
+// explaining why. At the end of a directory, the error is io.EOF.
+//
+// If n <= 0, Readdirnames returns all the names from the directory in
+// a single slice. In this case, if Readdirnames succeeds (reads all
+// the way to the end of the directory), it returns the slice and a
+// nil error. If it encounters an error before the end of the
+// directory, Readdirnames returns the names read until that point and
+// a non-nil error.
+func (f *File) Readdirnames(n int) ([]string, error) {
+ fi, err := f.Readdir(n)
+ if err != nil {
+ return nil, err
+ }
+ names := make([]string, len(fi))
+ for i, f := range fi {
+ _, names[i] = filepath.Split(f.Name())
+ }
+ return names, nil
+}
+
+// Stat returns the FileInfo structure describing file.
+// If there is an error, it will be of type *PathError.
+func (f *File) Stat() (os.FileInfo, error) {
+ info, err := f.fs.Stat(f.Name())
+ if err == nil {
+ f.cachedInfo = info
+ }
+ return info, err
+}
+
+// Sync is a noop.
+func (f *File) Sync() error {
+ return nil
+}
+
+// Truncate changes the size of the file.
+// It does not change the I/O offset.
+// If there is an error, it will be of type *PathError.
+func (f *File) Truncate(int64) error {
+ return ErrNotImplemented
+}
+
+// WriteString is like Write, but writes the contents of string s rather than
+// a slice of bytes.
+func (f *File) WriteString(s string) (int, error) {
+ return f.Write([]byte(s))
+}
+
+// Close closes the File, rendering it unusable for I/O.
+// It returns an error, if any.
+func (f *File) Close() error {
+ // Closing a reading stream
+ if f.streamRead != nil {
+ // We try to close the Reader
+ defer func() {
+ f.streamRead = nil
+ }()
+ return f.streamRead.Close()
+ }
+
+ // Closing a writing stream
+ if f.streamWrite != nil {
+ defer func() {
+ f.streamWrite = nil
+ f.streamWriteCloseErr = nil
+ }()
+
+ // We try to close the Writer
+ if err := f.streamWrite.Close(); err != nil {
+ return err
+ }
+ // And more importantly, we wait for the actual writing performed in go-routine to finish.
+ // We might have at most 2*5=10MB of data waiting to be flushed before close returns. This
+ // might be rather slow.
+ err := <-f.streamWriteCloseErr
+ close(f.streamWriteCloseErr)
+ return err
+ }
+
+ // Or maybe we don't have anything to close
+ return nil
+}
+
+// Read reads up to len(b) bytes from the File.
+// It returns the number of bytes read and an error, if any.
+// EOF is signaled by a zero count with err set to io.EOF.
+func (f *File) Read(p []byte) (int, error) {
+ n, err := f.streamRead.Read(p)
+
+ if err == nil {
+ f.streamReadOffset += int64(n)
+ }
+
+ return n, err
+}
+
+// ReadAt reads len(p) bytes from the file starting at byte offset off.
+// It returns the number of bytes read and the error, if any.
+// ReadAt always returns a non-nil error when n < len(b).
+// At end of file, that error is io.EOF.
+func (f *File) ReadAt(p []byte, off int64) (n int, err error) {
+ _, err = f.Seek(off, io.SeekStart)
+ if err != nil {
+ return
+ }
+ n, err = f.Read(p)
+ return
+}
+
+// Seek sets the offset for the next Read or Write on file to offset, interpreted
+// according to whence: 0 means relative to the origin of the file, 1 means
+// relative to the current offset, and 2 means relative to the end.
+// It returns the new offset and an error, if any.
+// The behavior of Seek on a file opened with O_APPEND is not specified.
+func (f *File) Seek(offset int64, whence int) (int64, error) {
+ // Write seek is not supported
+ if f.streamWrite != nil {
+ return 0, ErrNotSupported
+ }
+
+ // Read seek has its own implementation
+ if f.streamRead != nil {
+ return f.seekRead(offset, whence)
+ }
+
+ // Not having a stream
+ return 0, errors.New("File is closed")
+}
+
+func (f *File) seekRead(offset int64, whence int) (int64, error) {
+ startByte := int64(0)
+
+ switch whence {
+ case io.SeekStart:
+ startByte = offset
+ case io.SeekCurrent:
+ startByte = f.streamReadOffset + offset
+ case io.SeekEnd:
+ startByte = f.cachedInfo.Size() - offset
+ }
+
+ if err := f.streamRead.Close(); err != nil {
+ return 0, fmt.Errorf("couldn't close previous stream: %v", err)
+ }
+ f.streamRead = nil
+
+ if startByte < 0 {
+ return startByte, ErrInvalidSeek
+ }
+
+ return startByte, f.openReadStream(startByte)
+}
+
+// Write writes len(b) bytes to the File.
+// It returns the number of bytes written and an error, if any.
+// Write returns a non-nil error when n != len(b).
+func (f *File) Write(p []byte) (int, error) {
+ return f.streamWrite.Write(p)
+}
+
+func (f *File) openWriteStream() error {
+ if f.streamWrite != nil {
+ return ErrAlreadyOpened
+ }
+
+ reader, writer := io.Pipe()
+
+ f.streamWriteCloseErr = make(chan error)
+ f.streamWrite = writer
+
+ uploader := s3manager.NewUploader(f.fs.session)
+ uploader.Concurrency = 1
+
+ go func() {
+ _, err := uploader.Upload(&s3manager.UploadInput{
+ Bucket: aws.String(f.fs.bucket),
+ Key: aws.String(f.name),
+ Body: reader,
+ })
+ f.streamWriteCloseErr <- err
+ // close(f.streamWriteCloseErr)
+ }()
+ return nil
+}
+
+func (f *File) openReadStream(startAt int64) error {
+ if f.streamRead != nil {
+ return ErrAlreadyOpened
+ }
+
+ var streamRange *string = nil
+
+ if startAt > 0 && startAt != f.cachedInfo.Size() {
+ streamRange = aws.String(fmt.Sprintf("bytes=%d-%d", startAt, f.cachedInfo.Size()))
+ }
+
+ resp, err := f.fs.s3API.GetObject(&s3.GetObjectInput{
+ Bucket: aws.String(f.fs.bucket),
+ Key: aws.String(f.name),
+ Range: streamRange,
+ })
+ if err != nil {
+ return err
+ }
+
+ f.streamReadOffset = startAt
+ f.streamRead = resp.Body
+ return nil
+}
+
+// WriteAt writes len(p) bytes to the file starting at byte offset off.
+// It returns the number of bytes written and an error, if any.
+// WriteAt returns a non-nil error when n != len(p).
+func (f *File) WriteAt(p []byte, off int64) (n int, err error) {
+ _, err = f.Seek(off, 0)
+ if err != nil {
+ return
+ }
+ n, err = f.Write(p)
+ return
+}
diff --git a/pkg/s3/s3_fileinfo.go b/pkg/s3/s3_fileinfo.go
new file mode 100644
index 0000000..93e981b
--- /dev/null
+++ b/pkg/s3/s3_fileinfo.go
@@ -0,0 +1,61 @@
+// Package s3 provides an S3 backend implementation of the webdav Filesystem interface
+package s3
+
+import (
+ "os"
+ "time"
+)
+
+// FileInfo implements os.FileInfo for a file in S3.
+type FileInfo struct {
+ name string
+ directory bool
+ sizeInBytes int64
+ modTime time.Time
+}
+
+// NewFileInfo creates file cachedInfo.
+func NewFileInfo(name string, directory bool, sizeInBytes int64, modTime time.Time) FileInfo {
+ return FileInfo{
+ name: name,
+ directory: directory,
+ sizeInBytes: sizeInBytes,
+ modTime: modTime,
+ }
+}
+
+// Name provides the base name of the file.
+func (fi FileInfo) Name() string {
+ return fi.name
+}
+
+// Size provides the length in bytes for a file.
+func (fi FileInfo) Size() int64 {
+ return fi.sizeInBytes
+}
+
+// Mode provides the file mode bits. For a file in S3 this defaults to
+// 664 for files, 775 for directories.
+// In the future this may return differently depending on the permissions
+// available on the bucket.
+func (fi FileInfo) Mode() os.FileMode {
+ if fi.directory {
+ return 0755
+ }
+ return 0664
+}
+
+// ModTime provides the last modification time.
+func (fi FileInfo) ModTime() time.Time {
+ return fi.modTime
+}
+
+// IsDir provides the abbreviation for Mode().IsDir()
+func (fi FileInfo) IsDir() bool {
+ return fi.directory
+}
+
+// Sys provides the underlying data source (can return nil)
+func (fi FileInfo) Sys() interface{} {
+ return nil
+}
diff --git a/pkg/s3/s3_fs.go b/pkg/s3/s3_fs.go
new file mode 100644
index 0000000..974cdc5
--- /dev/null
+++ b/pkg/s3/s3_fs.go
@@ -0,0 +1,295 @@
+// Package s3 provides an S3 backend implementation of the webdav Filesystem interface
+package s3
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "time"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/awserr"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/s3"
+)
+
+// AFile represents a file in the filesystem.
+type AFile interface {
+ io.Closer
+ io.Reader
+ io.ReaderAt
+ io.Seeker
+ io.Writer
+ io.WriterAt
+
+ Name() string
+ Readdir(count int) ([]os.FileInfo, error)
+ Readdirnames(n int) ([]string, error)
+ Stat() (os.FileInfo, error)
+ Sync() error
+ Truncate(size int64) error
+ WriteString(s string) (ret int, err error)
+}
+
+// Fs is an FS object backed by S3.
+type Fs struct {
+ bucket string // Bucket name
+ session *session.Session // Session config
+ s3API *s3.S3
+}
+
+// NewFs creates a new Fs object writing files to a given S3 bucket.
+func NewFs(bucket string, session *session.Session) *Fs {
+ s3Api := s3.New(session)
+ return &Fs{
+ bucket: bucket,
+ session: session,
+ s3API: s3Api,
+ }
+}
+
+// ErrNotImplemented is returned when this operation is not (yet) implemented
+var ErrNotImplemented = errors.New("not implemented")
+
+// ErrNotSupported is returned when this operations is not supported by S3
+var ErrNotSupported = errors.New("s3 doesn't support this operation")
+
+// ErrAlreadyOpened is returned when the file is already opened
+var ErrAlreadyOpened = errors.New("already opened")
+
+// ErrInvalidSeek is returned when the seek operation is not doable
+var ErrInvalidSeek = errors.New("invalid seek offset")
+
+// Name returns the type of FS object this is: Fs.
+func (Fs) Name() string { return "s3" }
+
+// Create a file.
+func (fs Fs) Create(name string) (AFile, error) {
+ { // It's faster to trigger an explicit empty put object than opening a file for write, closing it and re-opening it
+ _, errPut := fs.s3API.PutObject(&s3.PutObjectInput{
+ Bucket: aws.String(fs.bucket),
+ Key: aws.String(name),
+ Body: bytes.NewReader([]byte{}),
+ })
+ if errPut != nil {
+ return nil, errPut
+ }
+ }
+
+ file, err := fs.OpenFile(name, os.O_WRONLY, 0750)
+ if err != nil {
+ return file, err
+ }
+
+ // Create(), like all of S3, is eventually consistent.
+ // To protect against unexpected behavior, have this method
+ // wait until S3 reports the object exists.
+ return file, fs.s3API.WaitUntilObjectExists(&s3.HeadObjectInput{
+ Bucket: aws.String(fs.bucket),
+ Key: aws.String(name),
+ })
+}
+
+// Mkdir makes a directory in S3.
+func (fs Fs) Mkdir(name string, perm os.FileMode) error {
+ file, err := fs.OpenFile(fmt.Sprintf("%s/", filepath.Clean(name)), os.O_CREATE, perm)
+ if err == nil {
+ err = file.Close()
+ }
+ return err
+}
+
+// MkdirAll creates a directory and all parent directories if necessary.
+func (fs Fs) MkdirAll(path string, perm os.FileMode) error {
+ return fs.Mkdir(path, perm)
+}
+
+// Open a file for reading.
+func (fs *Fs) Open(name string) (AFile, error) {
+ /*
+ if _, err := fs.Stat(name); err != nil {
+ return nil, err
+ }
+ */
+ return fs.OpenFile(name, os.O_RDONLY, 0777)
+}
+
+// OpenFile opens a file.
+func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (AFile, error) {
+ file := NewFile(fs, name)
+
+ if flag&os.O_RDWR != 0 {
+ return file, file.openWriteStream()
+ }
+
+ // Appending is not supported by S3. It's do-able though by:
+ // - Copying the existing file to a new place (for example $file.previous)
+ // - Writing a new file, streaming the content of the previous file in it
+ // - Writing the data you want to append
+ // Quite network intensive, if used in abondance this would lead to terrible performances.
+ if flag&os.O_APPEND != 0 {
+ return nil, ErrNotSupported
+ }
+
+ // Creating is basically a write
+ if flag&os.O_CREATE != 0 {
+ flag |= os.O_WRONLY
+ }
+
+ // We either write
+ if flag&os.O_WRONLY != 0 {
+ return file, file.openWriteStream()
+ }
+
+ info, err := file.Stat()
+
+ if err != nil {
+ return nil, err
+ }
+
+ if info.IsDir() {
+ return file, nil
+ }
+
+ return file, file.openReadStream(0)
+}
+
+// Remove a file
+func (fs Fs) Remove(name string) error {
+ if _, err := fs.Stat(name); err != nil {
+ return err
+ }
+ return fs.forceRemove(name)
+}
+
+// forceRemove doesn't error if a file does not exist.
+func (fs Fs) forceRemove(name string) error {
+ _, err := fs.s3API.DeleteObject(&s3.DeleteObjectInput{
+ Bucket: aws.String(fs.bucket),
+ Key: aws.String(name),
+ })
+ return err
+}
+
+// RemoveAll removes a path.
+func (fs *Fs) RemoveAll(path string) error {
+ s3dir := NewFile(fs, path)
+ fis, err := s3dir.Readdir(0)
+ if err != nil {
+ return err
+ }
+ for _, fi := range fis {
+ fullpath := filepath.Join(s3dir.Name(), fi.Name())
+ if fi.IsDir() {
+ if err := fs.RemoveAll(fullpath); err != nil {
+ return err
+ }
+ } else {
+ if err := fs.forceRemove(fullpath); err != nil {
+ return err
+ }
+ }
+ }
+ // finally remove the "file" representing the directory
+ if err := fs.forceRemove(s3dir.Name() + "/"); err != nil {
+ return err
+ }
+ return nil
+}
+
+// Rename a file.
+// There is no method to directly rename an S3 object, so the Rename
+// will copy the file to an object with the new name and then delete
+// the original.
+func (fs Fs) Rename(oldname, newname string) error {
+ if oldname == newname {
+ return nil
+ }
+ _, err := fs.s3API.CopyObject(&s3.CopyObjectInput{
+ Bucket: aws.String(fs.bucket),
+ CopySource: aws.String(fs.bucket + oldname),
+ Key: aws.String(newname),
+ })
+ if err != nil {
+ return err
+ }
+ _, err = fs.s3API.DeleteObject(&s3.DeleteObjectInput{
+ Bucket: aws.String(fs.bucket),
+ Key: aws.String(oldname),
+ })
+ return err
+}
+
+func hasTrailingSlash(s string) bool {
+ return len(s) > 0 && s[len(s)-1] == '/'
+}
+
+func trimLeadingSlash(s string) string {
+ if len(s) > 0 && s[0] == '/' {
+ return s[1:]
+ }
+ return s
+}
+
+// Stat returns a FileInfo describing the named file.
+// If there is an error, it will be of type *os.PathError.
+func (fs Fs) Stat(name string) (os.FileInfo, error) {
+ out, err := fs.s3API.HeadObject(&s3.HeadObjectInput{
+ Bucket: aws.String(fs.bucket),
+ Key: aws.String(name),
+ })
+ if err != nil {
+ if errRequestFailure, ok := err.(awserr.RequestFailure); ok {
+ if errRequestFailure.StatusCode() == 404 {
+ statDir, errStat := fs.statDirectory(name)
+ return statDir, errStat
+ }
+ }
+ return FileInfo{}, &os.PathError{
+ Op: "stat",
+ Path: name,
+ Err: err,
+ }
+ } else if hasTrailingSlash(name) { // It is a directory
+ return FileInfo{name: name, directory: true}, nil
+ }
+ return NewFileInfo(filepath.Base(name), false, *out.ContentLength, *out.LastModified), nil // It is a file
+}
+
+func (fs Fs) statDirectory(name string) (os.FileInfo, error) {
+ nameClean := filepath.Clean(name)
+ out, err := fs.s3API.ListObjectsV2(&s3.ListObjectsV2Input{
+ Bucket: aws.String(fs.bucket),
+ Prefix: aws.String(trimLeadingSlash(nameClean)),
+ MaxKeys: aws.Int64(1),
+ })
+ if err != nil {
+ return FileInfo{}, &os.PathError{
+ Op: "stat",
+ Path: name,
+ Err: err,
+ }
+ }
+ if *out.KeyCount == 0 && name != "" {
+ return nil, &os.PathError{
+ Op: "stat",
+ Path: name,
+ Err: os.ErrNotExist,
+ }
+ }
+ return NewFileInfo(filepath.Base(name), true, 0, time.Time{}), nil
+}
+
+// Chmod doesn't exists in S3 but could be implemented by analyzing ACLs
+func (Fs) Chmod(string, os.FileMode) error {
+ return ErrNotSupported
+}
+
+// Chtimes could be implemented if needed, but that would require to override object properties using metadata,
+// which makes it a non-standard solution
+func (Fs) Chtimes(string, time.Time, time.Time) error {
+ return ErrNotSupported
+}
diff --git a/pkg/s3/s3_test.go b/pkg/s3/s3_test.go
new file mode 100644
index 0000000..3b3ee23
--- /dev/null
+++ b/pkg/s3/s3_test.go
@@ -0,0 +1,662 @@
+// Package s3 provides an S3 backend implementation of the webdav Filesystem interface
+package s3
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "math/rand"
+ "os"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/s3"
+ "github.com/spf13/afero"
+)
+
+func TestCompatibleAferoS3(t *testing.T) {
+ var _ afero.Fs = (*Fs)(nil)
+ var _ afero.File = (*File)(nil)
+}
+
+func TestCompatibleOsFileInfo(t *testing.T) {
+ var _ os.FileInfo = (*FileInfo)(nil)
+}
+
+var (
+ bucketBase = time.Now().UTC().Format("2006-01-02-15-04-05")
+ bucketCounter int32 = 0
+)
+
+func GetFs(t *testing.T) afero.Fs {
+ sess, errSession := session.NewSession(&aws.Config{
+ Credentials: credentials.NewStaticCredentials("minioadmin", "minioadmin", ""),
+ Endpoint: aws.String("http://localhost:9000"),
+ Region: aws.String("eu-west-1"),
+ DisableSSL: aws.Bool(true),
+ S3ForcePathStyle: aws.Bool(true),
+ })
+
+ if errSession != nil {
+ t.Fatal("Could not create session:", errSession)
+ }
+
+ s3Client := s3.New(sess)
+
+ // Creating a both non-conflicting and quite easy to understand and diagnose bucket name
+ bucketName := fmt.Sprintf(
+ "%s-%s-%d",
+ bucketBase,
+ strings.ToLower(t.Name()),
+ atomic.AddInt32(&bucketCounter, 1),
+ )
+
+ if _, err := s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucketName)}); err != nil {
+ t.Fatal("Could not create bucket:", err)
+ }
+
+ fs := NewFs(bucketName, sess)
+
+ // The following cleanup code works fine but testing.T.Cleanup is only available since Go 1.14 and we don't actually
+ // need it for now.
+ /*
+ t.Cleanup(func() {
+ if err := fs.RemoveAll("/"); err != nil {
+ t.Fatal("Could not cleanup bucket:", err)
+ return
+ }
+
+ // The minio implementation makes the RemoveAll("/") also delete the simulated S3 bucket, so we *should* but
+ // *can't* use the bucket deletion.
+ // if _, err := s3Client.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)}); err != nil {
+ // t.Fatal("Could not delete bucket:", err)
+ // }
+ })
+ */
+
+ return fs
+}
+
+func testWriteFile(t *testing.T, fs afero.Fs, name string, size int) {
+ t.Logf("Working on %s with %d bytes", name, size)
+
+ { // First we write the file
+ t.Log(" Writing file")
+ reader1 := NewLimitedReader(rand.New(rand.NewSource(0)), size)
+
+ file, errOpen := fs.OpenFile(name, os.O_WRONLY, 0777)
+ if errOpen != nil {
+ t.Fatal("Could not open file:", errOpen)
+ }
+
+ if _, errWrite := io.Copy(file, reader1); errWrite != nil {
+ t.Fatal("Could not write file:", errWrite)
+ }
+
+ if errClose := file.Close(); errClose != nil {
+ t.Fatal("Couldn't close file", errClose)
+ }
+ }
+
+ { // Then we read the file
+ t.Log(" Reading file")
+ reader2 := NewLimitedReader(rand.New(rand.NewSource(0)), size)
+
+ file, errOpen := fs.OpenFile(name, os.O_RDONLY, 0777)
+ if errOpen != nil {
+ t.Fatal("Could not open file:", errOpen)
+ }
+
+ if ok, err := ReadersEqual(file, reader2); !ok || err != nil {
+ t.Fatal("Could not equal reader:", err)
+ }
+
+ if errClose := file.Close(); errClose != nil {
+ t.Fatal("Couldn't close file", errClose)
+ }
+ }
+}
+
+func TestFileWrite(t *testing.T) {
+ fs := GetFs(t)
+ testWriteFile(t, fs, "/file-1K", 1024)
+ testWriteFile(t, fs, "/file-1M", 1*1024*1024)
+ testWriteFile(t, fs, "/file-10M", 10*1024*1024)
+ testWriteFile(t, fs, "/file-100M", 100*1024*1024)
+}
+
+func TestFsName(t *testing.T) {
+ fs := GetFs(t)
+ if fs.Name() != "s3" {
+ t.Fatal("Wrong name")
+ }
+}
+
+func TestFileSeekBig(t *testing.T) {
+ fs := GetFs(t)
+ size := 10 * 1024 * 1024 // 10MB
+ name := "file-10M"
+
+ { // First we write the file
+ t.Log("Writing initial file")
+ randomReader := NewLimitedReader(rand.New(rand.NewSource(0)), size)
+
+ file, errOpen := fs.OpenFile(name, os.O_WRONLY, 0777)
+ if errOpen != nil {
+ t.Fatal("Could not open file:", errOpen)
+ }
+
+ if _, errWrite := io.Copy(file, randomReader); errWrite != nil {
+ t.Fatal("Could not write file:", errWrite)
+ }
+
+ if errClose := file.Close(); errClose != nil {
+ t.Fatal("Couldn't close file", errClose)
+ }
+ }
+
+ {
+ t.Log("Checking the second half of it")
+ randomReader := NewLimitedReader(rand.New(rand.NewSource(0)), size)
+ { // We skip 5MB by reading them
+ buffer := make([]byte, 1*1024*1024)
+ for i := 0; i < 5; i++ {
+ if _, err := randomReader.Read(buffer); err != nil {
+ t.Fatal("Cannot read", err)
+ }
+ }
+ }
+
+ file, errOpen := fs.OpenFile(name, os.O_RDONLY, 0777)
+
+ if errOpen != nil {
+ t.Fatal("Cannot open", errOpen)
+ }
+
+ if _, err := file.Seek(5*1024*1024, io.SeekCurrent); err != nil {
+ t.Fatal("Cannot seek:", err)
+ }
+
+ if ok, err := ReadersEqual(randomReader, file); !ok || err != nil {
+ t.Fatal("Stream are not equal:", err)
+ }
+
+ if err := file.Close(); err != nil {
+ t.Fatal("Cannot close", err)
+ }
+ }
+}
+
+//nolint: gocyclo, funlen
+func TestFileSeekBasic(t *testing.T) {
+ fs := GetFs(t)
+
+ { // Writing an initial file
+ file, errOpen := fs.OpenFile("file1", os.O_WRONLY, 0777)
+ if errOpen != nil {
+ t.Fatal("Could not open file:", errOpen)
+ }
+
+ if _, err := file.WriteString("Hello world !"); err != nil {
+ t.Fatal("Could not write file:", err)
+ }
+
+ if errClose := file.Close(); errClose != nil {
+ t.Fatal("Couldn't close file", errClose)
+ }
+ }
+
+ file, errOpen := fs.Open("file1")
+ if errOpen != nil {
+ t.Fatal("Could not open file:", errOpen)
+ }
+
+ defer func() {
+ if err := file.Close(); err != nil {
+ t.Fatal("Could not close file:", err)
+ }
+ }()
+
+ buffer := make([]byte, 5)
+
+ { // Reading the world
+ if pos, err := file.Seek(6, io.SeekStart); err != nil || pos != 6 {
+ t.Fatal("Could not seek:", err)
+ }
+
+ if _, err := file.Read(buffer); err != nil {
+ t.Fatal("Could not read buffer:", err)
+ }
+
+ if string(buffer) != "world" {
+ t.Fatal("Bad fetch:", string(buffer))
+ }
+ }
+
+ { // Going 3 bytes backwards
+ if pos, err := file.Seek(-3, io.SeekCurrent); err != nil || pos != 8 {
+ t.Fatal("Could not seek:", err)
+ }
+
+ //smallbuf := buffer[0:2]
+
+ if _, err := file.Read(buffer); err != io.EOF {
+ t.Fatal("Could not read buffer:", err)
+ }
+
+ if string(buffer) != "rld !" {
+ t.Fatal("Bad fetch:", string(buffer))
+ }
+ }
+
+ { // And then going back to the beginning
+ if pos, err := file.Seek(1, io.SeekStart); err != nil || pos != 1 {
+ t.Fatal("Could not seek:", err)
+ }
+
+ if _, err := file.Read(buffer); err != nil {
+ t.Fatal("Could not read buffer:", err)
+ }
+
+ if string(buffer) != "ello " {
+ t.Fatal("Bad fetch:", string(buffer))
+ }
+ }
+
+ { // And from the end
+ if pos, err := file.Seek(5, io.SeekEnd); err != nil || pos != 8 {
+ t.Fatal("Could not seek:", err)
+ }
+
+ if _, err := file.Read(buffer); err != io.EOF {
+ t.Fatal("Could not read buffer:", err)
+ }
+
+ if string(buffer) != "rld !" {
+ t.Fatal("Bad fetch:", string(buffer))
+ }
+ }
+}
+
+func TestReadAt(t *testing.T) {
+ fs := GetFs(t)
+
+ { // Writing an initial file
+ file, errOpen := fs.OpenFile("file1", os.O_WRONLY, 0777)
+ if errOpen != nil {
+ t.Fatal("Could not open file:", errOpen)
+ }
+
+ if _, err := file.WriteString("Hello world !"); err != nil {
+ t.Fatal("Could not write file:", err)
+ }
+
+ if err := file.Close(); err != nil {
+ t.Fatal("Could not close file:", err)
+ }
+ }
+
+ { // Reading a file
+ file, errOpen := fs.Open("file1")
+ if errOpen != nil {
+ t.Fatal("Could not open file:", errOpen)
+ }
+
+ defer func() {
+ if err := file.Close(); err != nil {
+ t.Fatal("Could not close file:", err)
+ }
+ }()
+
+ buffer := make([]byte, 5)
+ if _, err := file.ReadAt(buffer, 6); err != nil {
+ t.Fatal("Could not perform ReadAt:", err)
+ }
+
+ if string(buffer) != "world" {
+ t.Fatal("Bad fetch:", string(buffer))
+ }
+ }
+}
+
+func TestWriteAt(t *testing.T) {
+ fs := GetFs(t)
+
+ file, errOpen := fs.OpenFile("file1", os.O_WRONLY, 0777)
+ if errOpen != nil {
+ t.Fatal("Could not open file:", errOpen)
+ }
+
+ defer func() {
+ if err := file.Close(); err != nil {
+ t.Fatal("Could not close file:", err)
+ }
+ }()
+
+ if _, err := file.WriteAt([]byte("hello !"), 1); err == nil {
+ t.Fatal("We have no way to make this work !")
+ }
+}
+
+func TestFileCreate(t *testing.T) {
+ fs := GetFs(t)
+
+ if _, err := fs.Stat("/file1"); err == nil {
+ t.Fatal("We should'nt be able to get a file cachedInfo at this stage")
+ }
+
+ if file, err := fs.Create("/file1"); err != nil {
+ t.Fatal("Could not create file:", err)
+ } else if err := file.Close(); err != nil {
+ t.Fatal("Couldn't close file:", err)
+ }
+
+ if stat, err := fs.Stat("/file1"); err != nil {
+ t.Fatal("Could not access file:", err)
+ } else if stat.Size() != 0 {
+ t.Fatal("File should be empty")
+ }
+
+ if err := fs.Remove("/file1"); err != nil {
+ t.Fatal("Could not delete file:", err)
+ }
+
+ if _, err := fs.Stat("/file1"); err == nil {
+ t.Fatal("Should not be able to access file")
+ }
+}
+
+func TestRemoveAll(t *testing.T) {
+ fs := GetFs(t)
+
+ if err := fs.Mkdir("/dir1", 0750); err != nil {
+ t.Fatal("Could not create dir1:", err)
+ }
+
+ if err := fs.Mkdir("/dir1/dir2", 0750); err != nil {
+ t.Fatal("Could not create dir2:", err)
+ }
+
+ if file, err := fs.Create("/dir1/file1"); err != nil {
+ t.Fatal("Could not create dir2:", err)
+ } else if err := file.Close(); err != nil {
+ t.Fatal("Could not close /dir1/file1 err:", err)
+ }
+
+ if err := fs.RemoveAll("/dir1"); err != nil {
+ t.Fatal("Could not delete all files:", err)
+ }
+
+ if root, err := fs.Open("/"); err != nil {
+ t.Fatal("Could not access root:", root)
+ } else {
+ if files, err := root.Readdir(-1); err != nil {
+ t.Fatal("Could not readdir:", err)
+ } else if len(files) != 0 {
+ t.Fatal("We should not have any files !")
+ }
+ }
+}
+
+func TestMkdirAll(t *testing.T) {
+ fs := GetFs(t)
+ if err := fs.MkdirAll("/dir3/dir4", 0755); err != nil {
+ t.Fatal("Could not perform MkdirAll:", err)
+ }
+
+ if _, err := fs.Stat("/dir3/dir4"); err != nil {
+ t.Fatal("Could not read dir4:", err)
+ }
+}
+
+func TestDirHandle(t *testing.T) {
+ fs := GetFs(t)
+
+ // We create a "dir1" directory
+ if err := fs.Mkdir("/dir1", 0750); err != nil {
+ t.Fatal("Could not create dir:", err)
+ }
+
+ // Then create a "file1" file in it
+ if file, err := fs.Create("/dir1/file1"); err != nil {
+ t.Fatal("Could not create file:", err)
+ } else if err := file.Close(); err != nil {
+ t.Fatal("Couldn't close file:", err)
+ }
+
+ // Opening "dir1" should work
+ if dir1, err := fs.Open("/dir1"); err != nil {
+ t.Fatal("Could not open dir1:", err)
+ } else {
+ // Listing files should be OK too
+ if files, errReaddir := dir1.Readdir(-1); errReaddir != nil {
+ t.Fatal("Could not read dir")
+ } else if len(files) != 1 || files[0].Name() != "file1" {
+ t.Fatal("Listed files are incorrect !")
+ }
+ }
+
+ // Opening "dir2" should fail
+ if _, err := fs.Open("/dir2"); err == nil {
+ t.Fatal("Opening /dir2 should have triggered an error !")
+ }
+}
+
+func TestFileReaddirnames(t *testing.T) {
+ fs := GetFs(t)
+
+ // We create some dirs
+ for _, dir := range []string{"/dir1", "/dir2", "/dir3"} {
+ if err := fs.Mkdir(dir, 0750); err != nil {
+ t.Fatal("Could not create dir:", err)
+ }
+ }
+
+ root, errOpen := fs.Open("/")
+ if errOpen != nil {
+ t.Fatal(errOpen)
+ }
+
+ {
+ dirs, err := root.Readdirnames(2)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(dirs) != 2 || dirs[0] != "dir1" || dirs[1] != "dir2" {
+ t.Fatal("Wrong dirs")
+ }
+ }
+
+ {
+ dirs, err := root.Readdirnames(2)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(dirs) != 1 || dirs[0] != "dir3" {
+ t.Fatal("Wrong dirs")
+ }
+ }
+}
+
+func TestFileStat(t *testing.T) {
+ fs := GetFs(t)
+
+ // We create a "dir1" directory
+ if err := fs.Mkdir("/dir1", 0750); err != nil {
+ t.Fatal("Could not create dir:", err)
+ }
+
+ // Then create a "file1" file in it
+ if file, err := fs.Create("/dir1/file1"); err != nil {
+ t.Fatal("Could not create file:", err)
+ } else if err := file.Close(); err != nil {
+ t.Fatal("Couldn't close file:", err)
+ }
+
+ if dir1, err := fs.Open("/dir1"); err != nil {
+ t.Fatal(err)
+ } else {
+ if stat, err := dir1.Stat(); err != nil {
+ t.Fatal(err)
+ } else if stat.Mode() != 0755 {
+ t.Fatal("Wrong dir mode")
+ }
+ }
+
+ if file1, err := fs.Open("/dir1/file1"); err != nil {
+ t.Fatal(err)
+ } else {
+ if stat, err := file1.Stat(); err != nil {
+ t.Fatal(err)
+ } else if stat.Mode() != 0664 {
+ t.Fatal("Wrong file mode")
+ }
+ }
+}
+
+func testCreateFile(t *testing.T, fs afero.Fs, name string, content string) {
+ file, err := fs.OpenFile(name, os.O_WRONLY, 0750)
+ if err != nil {
+ t.Fatal("Could not open file", name, ":", err)
+ }
+ if _, err := file.WriteString(content); err != nil {
+ t.Fatal("Could not write content to file", err)
+ }
+ if err := file.Close(); err != nil {
+ t.Fatal("Could not close file")
+ }
+}
+
+func TestRename(t *testing.T) {
+ fs := GetFs(t)
+
+ if errMkdirAll := fs.MkdirAll("/dir1/dir2", 0750); errMkdirAll != nil {
+ } else if file, errOpenFile := fs.OpenFile("/dir1/dir2/file1", os.O_WRONLY, 0750); errOpenFile != nil {
+ t.Fatal("Couldn't open file:", errOpenFile)
+ } else {
+ if _, errWriteString := file.WriteString("Hello world !"); errWriteString != nil {
+ t.Fatal("Couldn't write:", errWriteString)
+ } else if errClose := file.Close(); errClose != nil {
+ t.Fatal("Couldn't close:", errClose)
+ }
+ }
+
+ if errRename := fs.Rename("/dir1/dir2/file1", "/dir1/dir2/file2"); errRename != nil {
+ t.Fatal("Couldn't rename file err:", errRename)
+ }
+
+ if _, err := fs.Stat("/dir1/dir2/file1"); err == nil {
+ t.Fatal("File shouldn't exist anymore")
+ }
+
+ if _, err := fs.Stat("/dir1/dir2/file2"); err != nil {
+ t.Fatal("Couldn't fetch file cachedInfo:", err)
+ }
+
+ // Renaming of a directory isn't tested because it's not supported by afero in the first place
+}
+
+func TestFileTime(t *testing.T) {
+ fs := GetFs(t)
+ name := "/dir1/file1"
+ beforeCreate := time.Now().UTC()
+ // Well, we have a 1-second precision
+ time.Sleep(time.Second)
+ testCreateFile(t, fs, name, "Hello world !")
+ time.Sleep(time.Second)
+ afterCreate := time.Now().UTC()
+ var modTime time.Time
+ if info, errStat := fs.Stat(name); errStat != nil {
+ t.Fatal("Couldn't stat", name, ":", errStat)
+ } else {
+ modTime = info.ModTime()
+ }
+ if modTime.Before(beforeCreate) || modTime.After(afterCreate) {
+ t.Fatal("Invalid dates", "modTime =", modTime, "before =", beforeCreate, "after =", afterCreate)
+ }
+ if err := fs.Chtimes(name, time.Now().UTC(), time.Now().UTC()); err == nil {
+ t.Fatal("If Chtimes is supported, we should have a check here")
+ }
+}
+
+func TestChmod(t *testing.T) {
+ fs := GetFs(t)
+ name := "/dir1/file1"
+ testCreateFile(t, fs, name, "Hello world !")
+ if err := fs.Chmod(name, 0750); err == nil {
+ t.Fatal("If Chmod is supported, we should have a check here")
+ }
+}
+
+// Source: rog's code from https://groups.google.com/forum/#!topic/golang-nuts/keG78hYt1I0
+func ReadersEqual(r1, r2 io.Reader) (bool, error) {
+ const chunkSize = 8 * 1024 // 8 KB
+ buf1 := make([]byte, chunkSize)
+ buf2 := make([]byte, chunkSize)
+ for {
+ n1, err1 := io.ReadFull(r1, buf1)
+ n2, err2 := io.ReadFull(r2, buf2)
+ if err1 != nil && err1 != io.EOF && err1 != io.ErrUnexpectedEOF {
+ return false, err1
+ }
+ if err2 != nil && err2 != io.EOF && err2 != io.ErrUnexpectedEOF {
+ return false, err2
+ }
+ if (err1 != nil) != (err2 != nil) || !bytes.Equal(buf1[0:n1], buf2[0:n2]) {
+ return false, nil
+ }
+ if err1 != nil {
+ return true, nil
+ }
+ }
+}
+
+type LimitedReader struct {
+ reader io.Reader
+ size int
+ offset int
+}
+
+func NewLimitedReader(reader io.Reader, limit int) *LimitedReader {
+ return &LimitedReader{
+ reader: reader,
+ size: limit,
+ }
+}
+
+func (r *LimitedReader) Read(buffer []byte) (int, error) {
+ maxRead := r.size - r.offset
+
+ if maxRead == 0 {
+ return 0, io.EOF
+ } else if maxRead < len(buffer) {
+ buffer = buffer[0:maxRead]
+ }
+
+ read, err := r.reader.Read(buffer)
+ if err == nil {
+ r.offset += read
+ }
+ return read, err
+}
+
+func TestMain(m *testing.M) {
+ // call flag.Parse() here if TestMain uses flags
+ rc := m.Run()
+
+ // rc 0 means we've passed,
+ // and CoverMode will be non empty if run with -cover
+ if rc == 0 && testing.CoverMode() != "" {
+ c := testing.Coverage()
+ if c < 0.80 {
+ fmt.Printf("Tests passed but coverage failed at %0.2f\n", c)
+ rc = -1
+ }
+ }
+ os.Exit(rc)
+}
diff --git a/pkg/s3/s3_webdavfs.go b/pkg/s3/s3_webdavfs.go
new file mode 100644
index 0000000..56ce999
--- /dev/null
+++ b/pkg/s3/s3_webdavfs.go
@@ -0,0 +1,64 @@
+// Package s3 provides an S3 backend implementation of the webdav Filesystem interface
+package s3
+
+import (
+ "context"
+ "os"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/nicolaspernoud/vestibule/pkg/log"
+
+ "golang.org/x/net/webdav"
+)
+
+// WdFs provides a webdav FileSystem interface implementation
+type WdFs struct {
+ *Fs
+}
+
+// NewWdFs returns a new S3 backed webdav filesystem
+func NewWdFs(endpoint string, region string, bucket string, accessKeyID string, secretAccessKey string) WdFs {
+ // Configure to use S3 compatible
+ s3Config := &aws.Config{
+ Credentials: credentials.NewStaticCredentials(accessKeyID, secretAccessKey, ""),
+ Endpoint: aws.String(endpoint),
+ Region: aws.String(region),
+ S3ForcePathStyle: aws.Bool(true),
+ }
+
+ sess, err := session.NewSession(s3Config)
+ if err != nil {
+ log.Logger.Printf("Error: %v\n", err)
+ }
+
+ // Initialize the file system
+ s3Fs := NewFs(bucket, sess)
+ return WdFs{s3Fs}
+}
+
+// Mkdir implements the wedav filesystem interface for a s3 bucket
+func (sw WdFs) Mkdir(ctx context.Context, name string, perm os.FileMode) error {
+ return sw.Fs.Mkdir(name, perm)
+}
+
+// OpenFile implements the wedav filesystem interface for a s3 bucket
+func (sw WdFs) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
+ return sw.Fs.OpenFile(name, flag, perm)
+}
+
+// RemoveAll implements the wedav filesystem interface for a s3 bucket
+func (sw WdFs) RemoveAll(ctx context.Context, name string) error {
+ return sw.Fs.RemoveAll(name)
+}
+
+// Rename implements the wedav filesystem interface for a s3 bucket
+func (sw WdFs) Rename(ctx context.Context, oldName, newName string) error {
+ return sw.Fs.Rename(oldName, newName)
+}
+
+// Stat implements the wedav filesystem interface for a s3 bucket
+func (sw WdFs) Stat(ctx context.Context, name string) (os.FileInfo, error) {
+ return sw.Fs.Stat(name)
+}
diff --git a/web/components/davs/davs.js b/web/components/davs/davs.js
index f97ad35..e5b3e1a 100644
--- a/web/components/davs/davs.js
+++ b/web/components/davs/davs.js
@@ -20,6 +20,14 @@ let secured_field;
let roles_field;
let roles_container;
let passphrase_field;
+let root_container;
+let iss3_field;
+let s3_container;
+let endpoint_field;
+let region_field;
+let bucket_field;
+let accesskeyid_field;
+let secretaccesskey_field;
// local variables
let davs;
@@ -81,12 +89,57 @@ export async function mount(where) {
<label class="label"><input id="davs-modal-writable" type="checkbox" />Allow write access</label>
</div>
</div>
+
+ <div class="field">
+ <div class="control">
+ <label class="label"><input id="davs-modal-iss3" type="checkbox" />Use S3 compatible backend</label>
+ </div>
+ </div>
+
<div class="field" id="davs-modal-root-container">
<label class="label">Root directory to serve</label>
<div class="control">
<input class="input" type="text" id="davs-modal-root" />
</div>
</div>
+
+ <div class="field" id="davs-modal-s3-container">
+ <div class="field" id="davs-modal-endpoint-container">
+ <label class="label">S3 backend endpoint</label>
+ <div class="control">
+ <input class="input" type="text" id="davs-modal-endpoint" />
+ </div>
+ </div>
+
+ <div class="field" id="davs-modal-region-container">
+ <label class="label">S3 backend region</label>
+ <div class="control">
+ <input class="input" type="text" id="davs-modal-region" />
+ </div>
+ </div>
+
+ <div class="field" id="davs-modal-bucket-container">
+ <label class="label">S3 backend bucket</label>
+ <div class="control">
+ <input class="input" type="text" id="davs-modal-bucket" />
+ </div>
+ </div>
+
+ <div class="field" id="davs-modal-accesskeyid-container">
+ <label class="label">S3 backend access key id</label>
+ <div class="control">
+ <input class="input" type="text" id="davs-modal-accesskeyid" />
+ </div>
+ </div>
+
+ <div class="field" id="davs-modal-secretaccesskey-container">
+ <label class="label">S3 backend secret access key</label>
+ <div class="control">
+ <input class="input" type="text" id="davs-modal-secretaccesskey" />
+ </div>
+ </div>
+ </div>
+
<div class="field">
<div class="control">
<label class="label"><input id="davs-modal-secured" type="checkbox" />Secure access</label>
@@ -179,7 +232,7 @@ function davTemplate(dav) {
</p>
<hr class="dropdown-divider" />
<p><strong>${dav.host}</strong></p>
- <p>Serves ${dav.root} directory, with ${dav.writable ? "read/write" : "read only"} access</p>
+ <p>serves ${dav.iss3 ? `${dav.bucket} bucket` : `${dav.root} directory`}, with ${dav.writable ? "read/write" : "read only"} access</p>
<p>${dav.secured ? "Restricted access to users with roles <strong>" + dav.roles + "</strong>" : "Unrestricted access"}</p>
<p class="has-text-centered"><strong>-${dav.id}-</strong></p>
</div>
@@ -263,10 +316,18 @@ function registerModalFields() {
host_field = document.getElementById("davs-modal-host");
writable_field = document.getElementById("davs-modal-writable");
root_field = document.getElementById("davs-modal-root");
+ root_container = document.getElementById("davs-modal-root-container");
secured_field = document.getElementById("davs-modal-secured");
roles_field = document.getElementById("davs-modal-roles");
roles_container = document.getElementById("davs-modal-roles-container");
passphrase_field = document.getElementById("davs-modal-passphrase");
+ iss3_field = document.getElementById("davs-modal-iss3");
+ s3_container = document.getElementById("davs-modal-s3-container");
+ endpoint_field = document.getElementById("davs-modal-endpoint");
+ region_field = document.getElementById("davs-modal-region");
+ bucket_field = document.getElementById("davs-modal-bucket");
+ accesskeyid_field = document.getElementById("davs-modal-accesskeyid");
+ secretaccesskey_field = document.getElementById("davs-modal-secretaccesskey");
document.getElementById(`davs-modal-close`).addEventListener("click", function () {
toggleModal();
});
@@ -285,6 +346,9 @@ function registerModalFields() {
secured_field.addEventListener("click", function () {
toggleRoles();
});
+ iss3_field.addEventListener("click", function () {
+ toggleS3();
+ });
document.getElementById(`davs-modal-passphrase-generate`).addEventListener("click", function () {
passphrase_field.value = RandomString(48);
});
@@ -305,11 +369,17 @@ async function editDav(dav) {
secured_field.checked = dav.secured;
roles_field.value = dav.roles;
passphrase_field.value = dav.passphrase;
+ iss3_field.checked = dav.iss3;
+ endpoint_field.value = dav.endpoint;
+ region_field.value = dav.region;
+ bucket_field.value = dav.bucket;
+ accesskeyid_field.value = dav.accesskeyid;
+ secretaccesskey_field.value = dav.secretaccesskey;
toggleModal();
}
function cleanDav(dav) {
- let props = ["writable", "name", "roles", "passphrase"];
+ let props = ["writable", "name", "roles", "passphrase", "endpoint", "region", "bucket", "accesskeyid", "secretaccesskey"];
for (const prop of props) {
dav[prop] = dav[prop] === undefined ? "" : dav[prop];
}
@@ -331,6 +401,12 @@ async function newDav() {
secured_field.checked = false;
roles_field.value = "";
passphrase_field.value = "";
+ iss3_field.checked = false;
+ endpoint_field.value = "";
+ region_field.value = "";
+ bucket_field.value = "";
+ accesskeyid_field.value = "";
+ secretaccesskey_field.value = "";
toggleModal();
}
@@ -352,6 +428,12 @@ async function postDav() {
secured: secured_field.checked,
roles: secured_field.checked ? roles_field.value.split(",") : "",
passphrase: passphrase_field.value,
+ iss3: iss3_field.checked,
+ endpoint: endpoint_field.value,
+ region: region_field.value,
+ bucket: bucket_field.value,
+ accesskeyid: accesskeyid_field.value,
+ secretaccesskey: secretaccesskey_field.value,
}),
});
if (response.status !== 200) {
@@ -384,6 +466,7 @@ async function reloadDavsOnServer() {
async function toggleModal() {
toggleRoles();
+ toggleS3();
updateIcon();
const modal = document.getElementById("davs-modal");
const card = document.getElementById("davs-modal-card");
@@ -406,6 +489,16 @@ function toggleRoles() {
}
}
+function toggleS3() {
+ if (iss3_field.checked) {
+ s3_container.style.display = "block";
+ root_container.style.display = "none";
+ } else {
+ s3_container.style.display = "none";
+ root_container.style.display = "block";
+ }
+}
+
function updateIcon() {
document.getElementById("davs-modal-icon-preview").setAttribute("class", "fas fa-" + icon_field.value);
}
diff --git a/web/index.html b/web/index.html
index 61f6f89..d61f0e5 100644
--- a/web/index.html
+++ b/web/index.html
@@ -32,7 +32,7 @@
<div class="navbar-brand">
<div class="navbar-item">
<a class="button is-primary is-rounded is-outlined" href="https://www.github.com/nicolaspernoud/Vestibule" target="_blank" rel="noopener noreferrer">
- <span>4.3.48</span>
+ <span>4.4.0-beta</span>
<span class="icon">
<svg
class="svg-inline--fa fa-github fa-w-16"
--
GitLab