add pre-get hook

.github/workflows/main.yml

@@ -5,8 +5,9 @@ name: Test
 jobs:
   test:
     strategy:
+      fail-fast: false
       matrix:
-        go-version: [1.12.x, 1.13.x]
+        go-version: [1.16.x, 1.17.x]
         platform: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:
@@ -26,14 +27,14 @@ jobs:
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')
     steps:
-    - name: Install Go 1.13.1
+    - name: Install Go 1.17.2
       uses: actions/setup-go@v2
       with:
-        go-version: '1.13.1'
+        go-version: '1.17.2'
     - name: Checkout code
-      uses: actions/checkout@v2        
+      uses: actions/checkout@v2
     - name: Build TUSD
-      if: startsWith(github.ref, 'refs/tags/')    
+      if: startsWith(github.ref, 'refs/tags/')
       env:
         GO111MODULE: on
       run: ./scripts/build_all.sh
@@ -43,7 +44,7 @@ jobs:
       with:
         files: tusd_*.*
       env:
-        GITHUB_TOKEN: ${{ secrets.GH_RELEASE_TOKEN }} 
+        GITHUB_TOKEN: ${{ secrets.GH_RELEASE_TOKEN }}
     - name: Deploy to heroku
       uses: akhileshns/[email protected]
       with:
@@ -53,8 +54,8 @@ jobs:
     - uses: azure/docker-login@v1
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}        
+        password: ${{ secrets.DOCKER_PASSWORD }}
     - name: Build and push docker image
       run: |
         docker build -t tusproject/tusd:$GITHUB_SHA .
-        docker push tusproject/tusd:$GITHUB_SHA           
+        docker push tusproject/tusd:$GITHUB_SHA

.gitignore

@@ -3,3 +3,5 @@ cover.out
 data/
 node_modules/
 .DS_Store
+./tusd
+cmd/tusd/tusd

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13-alpine AS builder
+FROM golang:1.16-alpine AS builder
 
 # Copy in the git repo from the build context
 COPY . /go/src/github.com/tus/tusd/

cmd/tusd/cli/composer.go

@@ -1,10 +1,12 @@
 package cli
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/tus/tusd/pkg/azurestore"
 	"github.com/tus/tusd/pkg/filelocker"
 	"github.com/tus/tusd/pkg/filestore"
 	"github.com/tus/tusd/pkg/gcsstore"
@@ -91,6 +93,49 @@ func CreateComposer() {
 
 		locker := memorylocker.New()
 		locker.UseIn(Composer)
+	} else if Flags.AzStorage != "" {
+
+		accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
+		if accountName == "" {
+			stderr.Fatalf("No service account name for Azure BlockBlob Storage using the AZURE_STORAGE_ACCOUNT environment variable.\n")
+		}
+
+		accountKey := os.Getenv("AZURE_STORAGE_KEY")
+		if accountKey == "" {
+			stderr.Fatalf("No service account key for Azure BlockBlob Storage using the AZURE_STORAGE_KEY environment variable.\n")
+		}
+
+		azureEndpoint := Flags.AzEndpoint
+		// Enables support for using Azurite as a storage emulator without messing with proxies and stuff
+		// e.g. http://127.0.0.1:10000/devstoreaccount1
+		if azureEndpoint == "" {
+			azureEndpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
+			stdout.Printf("Custom Azure Endpoint not specified in flag variable azure-endpoint.\n"+
+				"Using endpoint %s\n", azureEndpoint)
+		} else {
+			stdout.Printf("Using Azure endpoint %s\n", azureEndpoint)
+		}
+
+		azConfig := &azurestore.AzConfig{
+			AccountName:         accountName,
+			AccountKey:          accountKey,
+			ContainerName:       Flags.AzStorage,
+			ContainerAccessType: Flags.AzContainerAccessType,
+			BlobAccessTier:      Flags.AzBlobAccessTier,
+			Endpoint:            azureEndpoint,
+		}
+
+		azService, err := azurestore.NewAzureService(azConfig)
+		if err != nil {
+			stderr.Fatalf(err.Error())
+		}
+
+		store := azurestore.New(azService)
+		store.ObjectPrefix = Flags.AzObjectPrefix
+		store.Container = Flags.AzStorage
+
+		store.UseIn(Composer)
+
 	} else {
 		dir, err := filepath.Abs(Flags.UploadDir)
 		if err != nil {

cmd/tusd/cli/flags.go

@@ -29,6 +29,11 @@ var Flags struct {
 	S3DisableSSL            bool
 	GCSBucket               string
 	GCSObjectPrefix         string
+	AzStorage               string
+	AzContainerAccessType   string
+	AzBlobAccessTier        string
+	AzObjectPrefix          string
+	AzEndpoint              string
 	EnabledHooksString      string
 	FileHooksDir            string
 	HttpHooksEndpoint       string
@@ -69,8 +74,13 @@ func ParseFlags() {
 	flag.BoolVar(&Flags.S3DisableContentHashes, "s3-disable-content-hashes", false, "Disable the calculation of MD5 and SHA256 hashes for the content that gets uploaded to S3 for minimized CPU usage (experimental and may be removed in the future)")
 	flag.BoolVar(&Flags.S3DisableSSL, "s3-disable-ssl", false, "Disable SSL and only use HTTP for communication with S3 (experimental and may be removed in the future)")
 	flag.StringVar(&Flags.GCSBucket, "gcs-bucket", "", "Use Google Cloud Storage with this bucket as storage backend (requires the GCS_SERVICE_ACCOUNT_FILE environment variable to be set)")
-	flag.StringVar(&Flags.GCSObjectPrefix, "gcs-object-prefix", "", "Prefix for GCS object names (can't contain underscore character)")
-	flag.StringVar(&Flags.EnabledHooksString, "hooks-enabled-events", "pre-create,post-create,post-receive,post-terminate,post-finish", "Comma separated list of enabled hook events (e.g. post-create,post-finish). Leave empty to enable default events")
+	flag.StringVar(&Flags.GCSObjectPrefix, "gcs-object-prefix", "", "Prefix for GCS object names")
+	flag.StringVar(&Flags.AzStorage, "azure-storage", "", "Use Azure BlockBlob Storage with this container name as a storage backend (requires the AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY environment variable to be set)")
+	flag.StringVar(&Flags.AzContainerAccessType, "azure-container-access-type", "", "Access type when creating a new container if it does not exist (possible values: blob, container, '')")
+	flag.StringVar(&Flags.AzBlobAccessTier, "azure-blob-access-tier", "", "Blob access tier when uploading new files (possible values: archive, cool, hot, '')")
+	flag.StringVar(&Flags.AzObjectPrefix, "azure-object-prefix", "", "Prefix for Azure object names")
+	flag.StringVar(&Flags.AzEndpoint, "azure-endpoint", "", "Custom Endpoint to use for Azure BlockBlob Storage (requires azure-storage to be pass)")
+	flag.StringVar(&Flags.EnabledHooksString, "hooks-enabled-events", "pre-get,pre-create,post-create,post-receive,post-terminate,post-finish", "Comma separated list of enabled hook events (e.g. post-create,post-finish). Leave empty to enable default events")
 	flag.StringVar(&Flags.FileHooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
 	flag.StringVar(&Flags.HttpHooksEndpoint, "hooks-http", "", "An HTTP endpoint to which hook events will be sent to")
 	flag.StringVar(&Flags.HttpHooksForwardHeaders, "hooks-http-forward-headers", "", "List of HTTP request headers to be forwarded from the client request to the hook endpoint")

cmd/tusd/cli/hooks.go

@@ -43,13 +43,18 @@ func preFinishCallback(info handler.HookEvent) error {
 	return hookCallback(hooks.HookPreFinish, info)
 }
 
+func preGetCallback(info handler.HookEvent) error {
+	return hookCallback(hooks.HookPreGet, info)
+}
+
 func SetupHookMetrics() {
 	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostFinish)).Add(0)
 	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostTerminate)).Add(0)
 	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostReceive)).Add(0)
 	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPostCreate)).Add(0)
 	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPreCreate)).Add(0)
 	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPreFinish)).Add(0)
+	MetricsHookErrorsTotal.WithLabelValues(string(hooks.HookPreGet)).Add(0)
 }
 
 func SetupPreHooks(config *handler.Config) error {
@@ -99,6 +104,7 @@ func SetupPreHooks(config *handler.Config) error {
 
 	config.PreUploadCreateCallback = preCreateCallback
 	config.PreFinishResponseCallback = preFinishCallback
+	config.PreGetCallback = preGetCallback
 
 	return nil
 }
@@ -140,6 +146,8 @@ func invokeHookSync(typ hooks.HookType, info handler.HookEvent, captureOutput bo
 		logEv(stdout, "UploadFinished", "id", id, "size", strconv.FormatInt(size, 10))
 	case hooks.HookPostTerminate:
 		logEv(stdout, "UploadTerminated", "id", id)
+	case hooks.HookPreGet:
+		logEv(stdout, "PreGet", "id", id)
 	}
 
 	if hookHandler == nil {

cmd/tusd/cli/hooks/hooks.go

@@ -18,9 +18,10 @@ const (
 	HookPostCreate    HookType = "post-create"
 	HookPreCreate     HookType = "pre-create"
 	HookPreFinish     HookType = "pre-finish"
+	HookPreGet        HookType = "pre-get"
 )
 
-var AvailableHooks []HookType = []HookType{HookPreCreate, HookPostCreate, HookPostReceive, HookPostTerminate, HookPostFinish, HookPreFinish}
+var AvailableHooks []HookType = []HookType{HookPreCreate, HookPostCreate, HookPostReceive, HookPostTerminate, HookPostFinish, HookPreFinish, HookPreGet}
 
 type hookDataStore struct {
 	handler.DataStore

cmd/tusd/cli/hooks/plugin.go

@@ -14,6 +14,7 @@ type PluginHookHandler interface {
 	PostFinish(info handler.HookEvent) error
 	PostTerminate(info handler.HookEvent) error
 	PreFinish(info handler.HookEvent) error
+	PreGet(info handler.HookEvent) error
 }
 
 type PluginHook struct {
@@ -57,6 +58,8 @@ func (h PluginHook) InvokeHook(typ HookType, info handler.HookEvent, captureOutp
 		err = h.handler.PreCreate(info)
 	case HookPreFinish:
 		err = h.handler.PreFinish(info)
+	case HookPreGet:
+		err = h.handler.PreGet(info)
 	default:
 		err = fmt.Errorf("hooks: unknown hook named %s", typ)
 	}

cmd/tusd/cli/log.go

@@ -7,8 +7,8 @@ import (
 	"github.com/tus/tusd/pkg/handler"
 )
 
-var stdout = log.New(os.Stdout, "[tusd] ", log.Ldate|log.Ltime)
-var stderr = log.New(os.Stderr, "[tusd] ", log.Ldate|log.Ltime)
+var stdout = log.New(os.Stdout, "[tusd] ", log.LstdFlags|log.Lmicroseconds)
+var stderr = log.New(os.Stderr, "[tusd] ", log.LstdFlags|log.Lmicroseconds)
 
 func logEv(logOutput *log.Logger, eventName string, details ...string) {
 	handler.LogEvent(logOutput, eventName, details...)

docs/hooks.md

@@ -13,7 +13,7 @@ If not otherwise noted, all hooks are invoked in a *non-blocking* way, meaning t
 
 ## Blocking Hooks
 
-On the other hand, there are a few *blocking* hooks, such as caused by the `pre-create` and `pre-finish` events. Because their exit code will dictate whether tusd will accept the current incoming request, tusd will wait until the hook process has exited. Therefore, in order to keep the response times low, one should avoid to make time-consuming operations inside the processes for blocking hooks.
+On the other hand, there are a few *blocking* hooks, such as caused by the `pre-create`, `pre-get` and `pre-finish` events. Because their exit code will dictate whether tusd will accept the current incoming request, tusd will wait until the hook process has exited. Therefore, in order to keep the response times low, one should avoid to make time-consuming operations inside the processes for blocking hooks.
 
 ### Blocking File Hooks
 
@@ -39,6 +39,12 @@ This event will be triggered after an upload is fully finished but before a resp
 This is a blocking hook, as such it can be used to validate or post-process an uploaded file.
 A non-zero exit code or HTTP response greater than `400` will return a HTTP 500 error to the client.
 
+### pre-get
+
+This event will be triggered before an upload read returned to the client.
+This is a blocking hook, as such it can be used to validate access limits.
+A non-zero exit code or HTTP response greater than `400` will return a HTTP 500 error to the client.
+
 ### post-finish
 
 This event will be triggered after an upload is fully finished, meaning that all chunks have been transfered and saved in the storage. After this point, no further modifications, except possible deletion, can be made to the upload entity and it may be desirable to use the file for further processing or notify other applications of the completions of this upload.

examples/hooks/pre-get

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+filename=$(cat /dev/stdin | jq .MetaData.filename)
+if [[ $filename != "public" ]; then
+  echo "Error: access unauthorized"
+  exit 1
+fi

go.mod

@@ -1,9 +1,13 @@
 module github.com/tus/tusd
 
-go 1.12
+// Specify the Go version needed for the Heroku deployment
+// See https://github.com/heroku/heroku-buildpack-go#go-module-specifics
+// +heroku goVersion go1.16
+go 1.16
 
 require (
 	cloud.google.com/go v0.40.0
+	github.com/Azure/azure-storage-blob-go v0.13.0
 	github.com/aws/aws-sdk-go v1.20.1
 	github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40
 	github.com/golang/mock v1.3.1