Initial version

.gitignore

@@ -13,3 +13,5 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+
+.idea/

README.md

@@ -0,0 +1,11 @@
+# s3-trash
+
+**DANGER**: **this removes all the objects from an S3 bucket. There's no way to recover deleted objects from S3 once this completes**.
+
+**USE AT YOUR OWN RISK**
+
+This is a small tool that lists all the objects versions in an S3 bucket and deletes them.
+Listing and deleting happens in parallel, thanks to Go concurrency model.
+
+Deletes happen in bulk operations of at most 1,000 deletes per call.
+Multiple connections can be opened to S3 to maximize speed.

deleter.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"sync"
+
+	"github.com/aws/aws-sdk-go/aws"
+	awss3 "github.com/aws/aws-sdk-go/service/s3"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	MaxBulkOpSize = 1000
+)
+
+func doDelete(s3 *awss3.S3, params *awss3.DeleteObjectsInput, status *Status) {
+	_, err := s3.DeleteObjects(params)
+	if err != nil {
+		if status.IncrementErrors() == 1 {
+			log.WithError(err).Error("Failed to delete objects")
+		}
+	} else {
+		lastObject := params.Delete.Objects[len(params.Delete.Objects)-1]
+		status.Update(MaxBulkOpSize, *lastObject.Key)
+	}
+}
+
+func DeleteObjects(bucketName string, s3 *awss3.S3, status *Status, objChan ObjectChannel, wg *sync.WaitGroup) {
+	buffer := [MaxBulkOpSize]*awss3.ObjectIdentifier{}
+	current := 0
+	for obj := range objChan {
+
+		buffer[current] = obj
+		current++
+		if current < MaxBulkOpSize {
+			continue
+		}
+
+		current = 0
+		p := awss3.DeleteObjectsInput{
+			Bucket: aws.String(bucketName),
+			Delete: &awss3.Delete{
+				Objects: buffer[:],
+			},
+		}
+		doDelete(s3, &p, status)
+	}
+
+	if current != 0 {
+		p := awss3.DeleteObjectsInput{
+			Bucket: aws.String(bucketName),
+			Delete: &awss3.Delete{
+				Objects: buffer[:current],
+			},
+		}
+		doDelete(s3, &p, status)
+	}
+	wg.Done()
+}

go.mod

@@ -0,0 +1,10 @@
+module github.com/j-vizcaino/s3-trash
+
+go 1.13
+
+require (
+	github.com/aws/aws-sdk-go v1.28.11
+	github.com/sirupsen/logrus v1.4.2
+	github.com/spf13/cobra v0.0.5
+	go.uber.org/atomic v1.5.1
+)

go.sum

@@ -0,0 +1,56 @@
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/aws/aws-sdk-go v1.28.11 h1:L2G5qI91s51cUP3hJli4mXRIZZ3alZHcwHWOJdMclKk=
+github.com/aws/aws-sdk-go v1.28.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
+go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

lister.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	awss3 "github.com/aws/aws-sdk-go/service/s3"
+	log "github.com/sirupsen/logrus"
+)
+
+func listBucket(bucketName string, s3 *awss3.S3, out ObjectChannel) error {
+	params := awss3.ListObjectVersionsInput{}
+	params.SetBucket(bucketName)
+
+
+	log.WithField("bucket", bucketName).Info("Listing object versions in bucket")
+
+	return s3.ListObjectVersionsPages(&params, func(res *awss3.ListObjectVersionsOutput, lastPage bool) bool {
+		for _, marker := range res.DeleteMarkers {
+			out <- &awss3.ObjectIdentifier{Key: marker.Key, VersionId: marker.VersionId}
+		}
+		return true
+	})
+}

main.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"os"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	awss3 "github.com/aws/aws-sdk-go/service/s3"
+	"github.com/spf13/cobra"
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	awsRegion string
+	connectionCount int
+)
+
+type ObjectChannel chan *awss3.ObjectIdentifier
+
+func runTrash(_ *cobra.Command, args []string) {
+	bucketName := args[0]
+
+	sess := session.Must(session.NewSession())
+
+	s3 := awss3.New(sess, aws.NewConfig().WithRegion(awsRegion))
+	status := Status{}
+	doneChan := make(chan bool)
+
+	objChan := make(ObjectChannel, MaxBulkOpSize*connectionCount)
+
+	wg := sync.WaitGroup{}
+
+	for i := 0; i < connectionCount; i++ {
+		wg.Add(1)
+		go DeleteObjects(bucketName, s3, &status, objChan, &wg)
+	}
+
+	go status.Display(time.Second, doneChan)
+
+	err := listBucket(bucketName, s3, objChan)
+
+	if err != nil {
+		log.WithError(err).Error("Failed to list object versions")
+		os.Exit(1)
+	}
+	close(objChan)
+	wg.Wait()
+
+}
+
+func main() {
+	log.SetFormatter(&log.TextFormatter{DisableTimestamp: true})
+
+	cmd := &cobra.Command{
+		Use: "s3-trash BUCKET_NAME",
+		Short: "Really empty an S3 bucket (including objects' versions)",
+		Run: runTrash,
+		Args: cobra.ExactArgs(1),
+	}
+	flags := cmd.Flags()
+	flags.StringVar(&awsRegion, "region", "us-east-1", "AWS region")
+	flags.IntVar(&connectionCount, "connections", 32, "Number of concurrent connections to S3")
+
+	if err := cmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

status.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"time"
+
+	"go.uber.org/atomic"
+	log "github.com/sirupsen/logrus"
+)
+
+type Status struct {
+	deletedCount   atomic.Uint64
+	errorCount     atomic.Uint64
+	lastDeletedKey atomic.String
+}
+
+func (s *Status) Update(addDeleted int, lastKey string) {
+	if addDeleted > 0 {
+		s.deletedCount.Add(uint64(addDeleted))
+	}
+	s.lastDeletedKey.Store(lastKey)
+}
+
+func (s *Status) IncrementErrors() uint64 {
+	return s.errorCount.Inc()
+}
+
+func (s *Status) Display(period time.Duration, done <-chan bool) {
+	ticker := time.NewTicker(period)
+	keepGoing := true
+
+	log.WithFields(log.Fields{
+		"deleted": s.deletedCount.Load(),
+		"errors": s.errorCount.Load(),
+		"last_object": s.lastDeletedKey.Load(),
+	}).Info("Status")
+
+	var oldDelete, oldErr uint64
+
+	for keepGoing {
+		select {
+		case <-ticker.C:
+			newDelete, newErr := s.deletedCount.Load(), s.errorCount.Load()
+			if newDelete == oldDelete && newErr == oldErr {
+				continue
+			}
+			log.WithFields(log.Fields{
+				"deleted":     newDelete,
+				"errors":      newErr,
+				"last_object": s.lastDeletedKey.Load(),
+			}).Info("Status")
+			oldDelete, oldErr = newDelete, newErr
+
+		case <-done:
+			ticker.Stop()
+			keepGoing = false
+		}
+	}
+	log.WithFields(log.Fields{
+		"deleted": s.deletedCount.Load(),
+		"errors": s.errorCount.Load(),
+		"last_object": s.lastDeletedKey.Load(),
+	}).Info("Done")
+}