Merge pull request #306 from tus/v1

Next major release v1.0.0

.travis.yml

@@ -1,13 +1,12 @@
 language: go
 go:
-- 1.5
-- 1.6
-- 1.7
-- 1.8
-- 1.9
-- "1.10"
-- 1.11
 - 1.12
+- 1.13
+env:
+- GO111MODULE=on
+os:
+- linux
+- windows
 sudo: required
 addons:
   apt:
@@ -18,15 +17,12 @@ cache:
   directories:
   - $HOME/.gimme
   - "$HOME/google-cloud-sdk/"
-env:
-  global:
-  - GO15VENDOREXPERIMENT=1
 install:
 - true
 script:
-- ./.scripts/test_all.sh
+- ./scripts/test_all.sh
 before_deploy:
-- if [[ "$TRAVIS_TAG" != "" ]]; then ./.scripts/build_all.sh; fi
+- if [[ "$TRAVIS_TAG" != "" ]]; then ./scripts/build_all.sh; fi
 deploy:
 - provider: releases
   api_key:
@@ -36,11 +32,13 @@ deploy:
   skip_cleanup: true
   on:
     tags: true
-    go: 1.12
+    go: 1.13
     repo: tus/tusd
+    os: linux
 - provider: script
-  script: .scripts/deploy_kube.sh
+  script: scripts/deploy_kube.sh
   on:
     branch: master
-    go: 1.12
+    go: 1.13
     repo: tus/tusd
+    os: linux

README.md

@@ -21,6 +21,9 @@ nearly any other cloud provider could easily be added to tusd.
 
 **Protocol version:** 1.0.0
 
+This branch contains tusd v1. If you are looking for the previous major release, after which
+breaking changes have been introduced, please look at the [0.14.0 tag](https://github.com/tus/tusd/tree/0.14.0).
+
 ## Getting started
 
 ### Download pre-builts binaries (recommended)
@@ -31,7 +34,9 @@ Windows in various formats of the
 
 ### Compile from source
 
-The only requirement for building tusd is [Go](http://golang.org/doc/install) 1.5 or newer.
+The only requirement for building tusd is [Go](http://golang.org/doc/install).
+Currently only Go 1.12 and 1.13 is tested and supported and in the future only the two latest
+major releases will be supported.
 If you meet this criteria, you can clone the git repository, install the remaining
 dependencies and build the binary:
 
@@ -52,7 +57,7 @@ snippet demonstrates how to start a tusd process which accepts tus uploads at
 `http://localhost:1080/files/` (notice the trailing slash) and stores them locally in the `./data` directory:
 
 ```
-$ tusd -dir=./data
+$ tusd -upload-dir./data
 [tusd] Using './data' as directory storage.
 [tusd] Using 0.00MB as maximum size.
 [tusd] Using 0.0.0.0:1080 as address to listen.
@@ -200,20 +205,20 @@ func main() {
 
 ```
 
-Please consult the [online documentation](https://godoc.org/github.com/tus/tusd)
+Please consult the [online documentation](https://godoc.org/github.com/tus/tusd/pkg)
 for more details about tusd's APIs and its sub-packages.
 
 ## Implementing own storages
 
 The tusd server is built to be as flexible as possible and to allow the use
 of different upload storage mechanisms. By default the tusd binary includes
-[`filestore`](https://godoc.org/github.com/tus/tusd/filestore) which will save every upload
+[`filestore`](https://godoc.org/github.com/tus/tusd/pkg/filestore) which will save every upload
 to a specific directory on disk.
 
 If you have different requirements, you can build your own storage backend
 which will save the files to S3, a remote FTP server or similar. Doing so
-is as simple as implementing the [`tusd.DataStore`](https://godoc.org/github.com/tus/tusd/#DataStore)
-interface and using the new struct in the [configuration object](https://godoc.org/github.com/tus/tusd/#Config).
+is as simple as implementing the [`tusd.DataStore`](https://godoc.org/github.com/tus/tusd/pkg/#DataStore)
+interface and using the new struct in the [configuration object](https://godoc.org/github.com/tus/tusd/pkg/#Config).
 Please consult the documentation about detailed information about the
 required methods.
 
@@ -222,19 +227,18 @@ required methods.
 This repository does not only contain the HTTP server's code but also other
 useful tools:
 
-* [**s3store**](https://godoc.org/github.com/tus/tusd/s3store): A storage backend using AWS S3
-* [**filestore**](https://godoc.org/github.com/tus/tusd/filestore): A storage backend using the local file system
-* [**gcsstore**](https://godoc.org/github.com/tus/tusd/gcsstore): A storage backend using Google cloud storage
-* [**memorylocker**](https://godoc.org/github.com/tus/tusd/memorylocker): An in-memory locker for handling concurrent uploads
-* [**consullocker**](https://godoc.org/github.com/tus/tusd/consullocker): A locker using the distributed Consul service
-* [**etcd3locker**](https://godoc.org/github.com/tus/tusd/etcd3locker): A locker using the distributed KV etcd3 store
-* [**limitedstore**](https://godoc.org/github.com/tus/tusd/limitedstore): A storage wrapper limiting the total used space for uploads
+* [**s3store**](https://godoc.org/github.com/tus/tusd/pkg/s3store): A storage backend using AWS S3
+* [**filestore**](https://godoc.org/github.com/tus/tusd/pkg/filestore): A storage backend using the local file system
+* [**gcsstore**](https://godoc.org/github.com/tus/tusd/pkg/gcsstore): A storage backend using Google cloud storage
+* [**memorylocker**](https://godoc.org/github.com/tus/tusd/pkg/memorylocker): An in-memory locker for handling concurrent uploads
+* [**filelocker**](https://godoc.org/github.com/tus/tusd/pkg/filelocker): A disk-based locker for handling concurrent uploads
 
 ### 3rd-Party tusd Packages
 
 The following packages are supported by 3rd-party maintainers outside of this repository. Please file issues respective to the packages in their respective repositories.
 
 * [**tusd-dynamo-locker**](https://github.com/chen-anders/tusd-dynamo-locker): A locker using AWS DynamoDB store
+* [**tusd-etcd3-locker**](https://github.com/tus/tusd-etcd3-locker): A locker using the distributed KV etcd3 store
 
 ## Running the testsuite
 
@@ -249,7 +253,7 @@ go test -v ./...
 
 ### How can I access tusd using HTTPS?
 
-The tusd binary, once executed, listens on the provided port for only non-encrypted HTTP requests and *does not accept* HTTPS connections. This decision has been made to limit the functionality inside this repository which has to be developed, tested and maintained. If you want to send requests to tusd in a secure fashion - what we absolutely encourage, we recommend you to utilize a reverse proxy in front of tusd which accepts incoming HTTPS connections and forwards them to tusd using plain HTTP. More information about this topic, including sample configurations for Nginx and Apache, can be found in [issue #86](https://github.com/tus/tusd/issues/86#issuecomment-269569077) and in the [Apache example configuration](/docs/apache2.conf).
+The tusd binary, once executed, listens on the provided port for only non-encrypted HTTP requests and *does not accept* HTTPS connections. This decision has been made to limit the functionality inside this repository which has to be developed, tested and maintained. If you want to send requests to tusd in a secure fashion - what we absolutely encourage, we recommend you to utilize a reverse proxy in front of tusd which accepts incoming HTTPS connections and forwards them to tusd using plain HTTP. More information about this topic, including sample configurations for Nginx and Apache, can be found in [issue #86](https://github.com/tus/tusd/issues/86#issuecomment-269569077) and in the [Apache example configuration](/examples/apache2.conf).
 
 ### Can I run tusd behind a reverse proxy?
 
@@ -261,7 +265,7 @@ Yes, it is absolutely possible to do so. Firstly, you should execute the tusd bi
 
 - *Forward hostname and scheme.* If the proxy rewrites the request URL, the tusd server does not know the original URL which was used to reach the proxy. This behavior can lead to situations, where tusd returns a redirect to a URL which can not be reached by the client. To avoid this confusion, you can explicitly tell tusd which hostname and scheme to use by supplying the `X-Forwarded-Host` and `X-Forwarded-Proto` headers.
 
-Explicit examples for the above points can be found in the [Nginx configuration](/docs/nginx.conf) which is used to power the [master.tus.io](https://master.tus.io) instace.
+Explicit examples for the above points can be found in the [Nginx configuration](/examples/nginx.conf) which is used to power the [master.tus.io](https://master.tus.io) instace.
 
 ### Can I run custom verification/authentication checks before an upload begins?
 

appveyor.yml

@@ -2,25 +2,13 @@ clone_folder: c:\projects\go\src\github.com\tus\tusd
 
 environment:
   GOPATH: c:\projects\go
-  GO15VENDOREXPERIMENT: 1
-
-install:
-  - git submodule update --init --recursive
+  GO111MODULE: on
 
 build_script:
   - set PATH=%GOPATH%\bin;%PATH%
   - go env
   - go version
-  - go get ./s3store
-  - go get ./consullocker
-  - go get ./prometheuscollector
-  - go get github.com/hashicorp/consul
 
 test_script:
-  - go test .
-  - go test ./filestore
-  - go test ./limitedstore
-  - go test ./memorylocker
-  - go test ./s3store
-  - go vet ./prometheuscollector
-  - go test ./gcsstore
+  - go test ./pkg/...
+  - go vet ./pkg/...

cmd/tusd/cli/composer.go

@@ -2,25 +2,27 @@ package cli
 
 import (
 	"os"
+	"path/filepath"
+	"strings"
 
-	"github.com/tus/tusd"
-	"github.com/tus/tusd/filestore"
-	"github.com/tus/tusd/gcsstore"
-	"github.com/tus/tusd/limitedstore"
-	"github.com/tus/tusd/memorylocker"
-	"github.com/tus/tusd/s3store"
+	"github.com/tus/tusd/pkg/filelocker"
+	"github.com/tus/tusd/pkg/filestore"
+	"github.com/tus/tusd/pkg/gcsstore"
+	"github.com/tus/tusd/pkg/handler"
+	"github.com/tus/tusd/pkg/memorylocker"
+	"github.com/tus/tusd/pkg/s3store"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 )
 
-var Composer *tusd.StoreComposer
+var Composer *handler.StoreComposer
 
 func CreateComposer() {
 	// Attempt to use S3 as a backend if the -s3-bucket option has been supplied.
 	// If not, we default to storing them locally on disk.
-	Composer = tusd.NewStoreComposer()
+	Composer = handler.NewStoreComposer()
 	if Flags.S3Bucket != "" {
 		s3Config := aws.NewConfig()
 
@@ -41,6 +43,11 @@ func CreateComposer() {
 		locker := memorylocker.New()
 		locker.UseIn(Composer)
 	} else if Flags.GCSBucket != "" {
+		if Flags.GCSObjectPrefix != "" && strings.Contains(Flags.GCSObjectPrefix, "_") {
+			stderr.Fatalf("gcs-object-prefix value (%s) can't contain underscore. "+
+				"Please remove underscore from the value", Flags.GCSObjectPrefix)
+		}
+
 		// Derivce credentials from service account file path passed in
 		// GCS_SERVICE_ACCOUNT_FILE environment variable.
 		gcsSAF := os.Getenv("GCS_SERVICE_ACCOUNT_FILE")
@@ -62,7 +69,10 @@ func CreateComposer() {
 		locker := memorylocker.New()
 		locker.UseIn(Composer)
 	} else {
-		dir := Flags.UploadDir
+		dir, err := filepath.Abs(Flags.UploadDir)
+		if err != nil {
+			stderr.Fatalf("Unable to make absolute path: %s", err)
+		}
 
 		stdout.Printf("Using '%s' as directory storage.\n", dir)
 		if err := os.MkdirAll(dir, os.FileMode(0774)); err != nil {
@@ -71,19 +81,9 @@ func CreateComposer() {
 
 		store := filestore.New(dir)
 		store.UseIn(Composer)
-	}
 
-	storeSize := Flags.StoreSize
-	maxSize := Flags.MaxSize
-
-	if storeSize > 0 {
-		limitedstore.New(storeSize, Composer.Core, Composer.Terminater).UseIn(Composer)
-		stdout.Printf("Using %.2fMB as storage size.\n", float64(storeSize)/1024/1024)
-
-		// We need to ensure that a single upload can fit into the storage size
-		if maxSize > storeSize || maxSize == 0 {
-			Flags.MaxSize = storeSize
-		}
+		locker := filelocker.New(dir)
+		locker.UseIn(Composer)
 	}
 
 	stdout.Printf("Using %.2fMB as maximum size.\n", float64(Flags.MaxSize)/1024/1024)

cmd/tusd/cli/flags.go

@@ -14,7 +14,6 @@ var Flags struct {
 	HttpSock            string
 	MaxSize             int64
 	UploadDir           string
-	StoreSize           int64
 	Basepath            string
 	Timeout             int64
 	S3Bucket            string
@@ -35,18 +34,14 @@ var Flags struct {
 	MetricsPath         string
 	BehindProxy         bool
 	VerboseOutput       bool
-
-	FileHooksInstalled bool
-	HttpHooksInstalled bool
 }
 
 func ParseFlags() {
 	flag.StringVar(&Flags.HttpHost, "host", "0.0.0.0", "Host to bind HTTP server to")
 	flag.StringVar(&Flags.HttpPort, "port", "1080", "Port to bind HTTP server to")
 	flag.StringVar(&Flags.HttpSock, "unix-sock", "", "If set, will listen to a UNIX socket at this location instead of a TCP socket")
 	flag.Int64Var(&Flags.MaxSize, "max-size", 0, "Maximum size of a single upload in bytes")
-	flag.StringVar(&Flags.UploadDir, "dir", "./data", "Directory to store uploads in")
-	flag.Int64Var(&Flags.StoreSize, "store-size", 0, "Size of space allowed for storage")
+	flag.StringVar(&Flags.UploadDir, "upload-dir", "./data", "Directory to store uploads in")
 	flag.StringVar(&Flags.Basepath, "base-path", "/files/", "Basepath of the HTTP server")
 	flag.Int64Var(&Flags.Timeout, "timeout", 30*1000, "Read timeout for connections in milliseconds.  A zero value means that reads will not timeout")
 	flag.StringVar(&Flags.S3Bucket, "s3-bucket", "", "Use AWS S3 with this bucket as storage backend (requires the AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION environment variables to be set)")
@@ -72,27 +67,6 @@ func ParseFlags() {
 
 	if Flags.FileHooksDir != "" {
 		Flags.FileHooksDir, _ = filepath.Abs(Flags.FileHooksDir)
-		Flags.FileHooksInstalled = true
-
-		stdout.Printf("Using '%s' for hooks", Flags.FileHooksDir)
-	}
-
-	if Flags.HttpHooksEndpoint != "" {
-		Flags.HttpHooksInstalled = true
-
-		stdout.Printf("Using '%s' as the endpoint for hooks", Flags.HttpHooksEndpoint)
-	}
-
-	if Flags.UploadDir == "" && Flags.S3Bucket == "" {
-		stderr.Fatalf("Either an upload directory (using -dir) or an AWS S3 Bucket " +
-			"(using -s3-bucket) must be specified to start tusd but " +
-			"neither flag was provided. Please consult `tusd -help` for " +
-			"more information on these options.")
-	}
-
-	if Flags.GCSObjectPrefix != "" && strings.Contains(Flags.GCSObjectPrefix, "_") {
-		stderr.Fatalf("gcs-object-prefix value (%s) can't contain underscore. "+
-			"Please remove underscore from the value", Flags.GCSObjectPrefix)
 	}
 }
 
@@ -114,11 +88,4 @@ func SetEnabledHooks() {
 	if len(Flags.EnabledHooks) == 0 {
 		Flags.EnabledHooks = hooks.AvailableHooks
 	}
-
-	var enabledHooksString []string
-	for _, h := range Flags.EnabledHooks {
-		enabledHooksString = append(enabledHooksString, string(h))
-	}
-
-	stdout.Printf("Enabled hook events: %s", strings.Join(enabledHooksString, ", "))
 }