diff --git a/pkg/fs/fs-bucket.go b/pkg/fs/fs-bucket.go
index e7ff8d358..1a598e82a 100644
--- a/pkg/fs/fs-bucket.go
+++ b/pkg/fs/fs-bucket.go
@@ -36,14 +36,11 @@ func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
 	}
 	bucket = fs.denormalizeBucket(bucket)
 	bucketDir := filepath.Join(fs.path, bucket)
-	// Check if bucket exists.
-	if _, e := os.Stat(bucketDir); e != nil {
+	if e := os.Remove(bucketDir); e != nil {
+		// Error if there was no bucket in the first place.
 		if os.IsNotExist(e) {
 			return probe.NewError(BucketNotFound{Bucket: bucket})
 		}
-		return probe.NewError(e)
-	}
-	if e := os.Remove(bucketDir); e != nil {
 		// On windows the string is slightly different, handle it here.
 		if strings.Contains(e.Error(), "directory is not empty") {
 			return probe.NewError(BucketNotEmpty{Bucket: bucket})