Add GetObjectNInfo to object layer (#6449)
The new call combines GetObjectInfo and GetObject, and returns an object with a ReadCloser interface. Also adds a number of end-to-end encryption tests at the handler level.
This commit is contained in:
Aditya Manthramurthy
committed by
Harshavardhana
Harshavardhana
parent
7d0645fb3a
commit
36e51d0cee
@@ -22,7 +22,9 @@ import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
var hasServerSideEncryptionHeaderTests = []struct {
|
||||
@@ -325,7 +327,7 @@ func TestParseSSECopyCustomerRequest(t *testing.T) {
|
||||
request.Header = headers
|
||||
globalIsSSL = test.useTLS
|
||||
|
||||
_, err := ParseSSECopyCustomerRequest(request, test.metadata)
|
||||
_, err := ParseSSECopyCustomerRequest(request.Header, test.metadata)
|
||||
if err != test.err {
|
||||
t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
|
||||
}
|
||||
@@ -557,7 +559,7 @@ var decryptObjectInfoTests = []struct {
|
||||
|
||||
func TestDecryptObjectInfo(t *testing.T) {
|
||||
for i, test := range decryptObjectInfoTests {
|
||||
if encrypted, err := DecryptObjectInfo(&test.info, test.headers); err != test.expErr {
|
||||
if encrypted, err := DecryptObjectInfo(test.info, test.headers); err != test.expErr {
|
||||
t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
|
||||
} else if enc := crypto.IsEncrypted(test.info.UserDefined); encrypted && enc != encrypted {
|
||||
t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)
|
||||
@@ -566,3 +568,285 @@ func TestDecryptObjectInfo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDecryptedRange(t *testing.T) {
|
||||
var (
|
||||
pkgSz = int64(64) * humanize.KiByte
|
||||
minPartSz = int64(5) * humanize.MiByte
|
||||
maxPartSz = int64(5) * humanize.GiByte
|
||||
|
||||
getEncSize = func(s int64) int64 {
|
||||
v, _ := sio.EncryptedSize(uint64(s))
|
||||
return int64(v)
|
||||
}
|
||||
udMap = func(isMulti bool) map[string]string {
|
||||
m := map[string]string{
|
||||
crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256,
|
||||
crypto.SSEMultipart: "1",
|
||||
}
|
||||
if !isMulti {
|
||||
delete(m, crypto.SSEMultipart)
|
||||
}
|
||||
return m
|
||||
}
|
||||
)
|
||||
|
||||
// Single part object tests
|
||||
var (
|
||||
mkSPObj = func(s int64) ObjectInfo {
|
||||
return ObjectInfo{
|
||||
Size: getEncSize(s),
|
||||
UserDefined: udMap(false),
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
testSP := []struct {
|
||||
decSz int64
|
||||
oi ObjectInfo
|
||||
}{
|
||||
{0, mkSPObj(0)},
|
||||
{1, mkSPObj(1)},
|
||||
{pkgSz - 1, mkSPObj(pkgSz - 1)},
|
||||
{pkgSz, mkSPObj(pkgSz)},
|
||||
{2*pkgSz - 1, mkSPObj(2*pkgSz - 1)},
|
||||
{minPartSz, mkSPObj(minPartSz)},
|
||||
{maxPartSz, mkSPObj(maxPartSz)},
|
||||
}
|
||||
|
||||
for i, test := range testSP {
|
||||
{
|
||||
// nil range
|
||||
o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(nil)
|
||||
if err != nil {
|
||||
t.Errorf("Case %d: unexpected err: %v", i, err)
|
||||
}
|
||||
if skip != 0 || sn != 0 || ps != 0 || o != 0 || l != getEncSize(test.decSz) {
|
||||
t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
|
||||
}
|
||||
}
|
||||
|
||||
if test.decSz >= 10 {
|
||||
// first 10 bytes
|
||||
o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, 0, 9})
|
||||
if err != nil {
|
||||
t.Errorf("Case %d: unexpected err: %v", i, err)
|
||||
}
|
||||
var rLen int64 = pkgSz + 32
|
||||
if test.decSz < pkgSz {
|
||||
rLen = test.decSz + 32
|
||||
}
|
||||
if skip != 0 || sn != 0 || ps != 0 || o != 0 || l != rLen {
|
||||
t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
|
||||
}
|
||||
}
|
||||
|
||||
kb32 := int64(32) * humanize.KiByte
|
||||
if test.decSz >= (64+32)*humanize.KiByte {
|
||||
// Skip the first 32Kib, and read the next 64Kib
|
||||
o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, kb32, 3*kb32 - 1})
|
||||
if err != nil {
|
||||
t.Errorf("Case %d: unexpected err: %v", i, err)
|
||||
}
|
||||
var rLen int64 = (pkgSz + 32) * 2
|
||||
if test.decSz < 2*pkgSz {
|
||||
rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32)
|
||||
}
|
||||
if skip != kb32 || sn != 0 || ps != 0 || o != 0 || l != rLen {
|
||||
t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
|
||||
}
|
||||
}
|
||||
|
||||
if test.decSz >= (64*2+32)*humanize.KiByte {
|
||||
// Skip the first 96Kib and read the next 64Kib
|
||||
o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, 3 * kb32, 5*kb32 - 1})
|
||||
if err != nil {
|
||||
t.Errorf("Case %d: unexpected err: %v", i, err)
|
||||
}
|
||||
var rLen int64 = (pkgSz + 32) * 2
|
||||
if test.decSz-pkgSz < 2*pkgSz {
|
||||
rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32*2)
|
||||
}
|
||||
if skip != kb32 || sn != 1 || ps != 0 || o != pkgSz+32 || l != rLen {
|
||||
t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Multipart object tests
|
||||
var (
|
||||
// make a multipart object-info given part sizes
|
||||
mkMPObj = func(sizes []int64) ObjectInfo {
|
||||
r := make([]objectPartInfo, len(sizes))
|
||||
sum := int64(0)
|
||||
for i, s := range sizes {
|
||||
r[i].Number = i
|
||||
r[i].Size = int64(getEncSize(s))
|
||||
sum += r[i].Size
|
||||
}
|
||||
return ObjectInfo{
|
||||
Size: sum,
|
||||
UserDefined: udMap(true),
|
||||
Parts: r,
|
||||
}
|
||||
}
|
||||
// Simple useful utilities
|
||||
repeat = func(k int64, n int) []int64 {
|
||||
a := []int64{}
|
||||
for i := 0; i < n; i++ {
|
||||
a = append(a, k)
|
||||
}
|
||||
return a
|
||||
}
|
||||
lsum = func(s []int64) int64 {
|
||||
sum := int64(0)
|
||||
for _, i := range s {
|
||||
if i < 0 {
|
||||
return -1
|
||||
}
|
||||
sum += i
|
||||
}
|
||||
return sum
|
||||
}
|
||||
esum = func(oi ObjectInfo) int64 {
|
||||
sum := int64(0)
|
||||
for _, i := range oi.Parts {
|
||||
sum += i.Size
|
||||
}
|
||||
return sum
|
||||
}
|
||||
)
|
||||
|
||||
s1 := []int64{5487701, 5487799, 3}
|
||||
s2 := repeat(5487701, 5)
|
||||
s3 := repeat(maxPartSz, 10000)
|
||||
testMPs := []struct {
|
||||
decSizes []int64
|
||||
oi ObjectInfo
|
||||
}{
|
||||
{s1, mkMPObj(s1)},
|
||||
{s2, mkMPObj(s2)},
|
||||
{s3, mkMPObj(s3)},
|
||||
}
|
||||
|
||||
// This function is a reference (re-)implementation of
|
||||
// decrypted range computation, written solely for the purpose
|
||||
// of the unit tests.
|
||||
//
|
||||
// `s` gives the decrypted part sizes, and the other
|
||||
// parameters describe the desired read segment. When
|
||||
// `isFromEnd` is true, `skipLen` argument is ignored.
|
||||
decryptedRangeRef := func(s []int64, skipLen, readLen int64, isFromEnd bool) (o, l, skip int64, sn uint32, ps int) {
|
||||
oSize := lsum(s)
|
||||
if isFromEnd {
|
||||
skipLen = oSize - readLen
|
||||
}
|
||||
if skipLen < 0 || readLen < 0 || oSize < 0 || skipLen+readLen > oSize {
|
||||
t.Fatalf("Impossible read specified: %d %d %d", skipLen, readLen, oSize)
|
||||
}
|
||||
|
||||
var cumulativeSum, cumulativeEncSum int64
|
||||
toRead := readLen
|
||||
readStart := false
|
||||
for i, v := range s {
|
||||
partOffset := int64(0)
|
||||
partDarePkgOffset := int64(0)
|
||||
if !readStart && cumulativeSum+v > skipLen {
|
||||
// Read starts at the current part
|
||||
readStart = true
|
||||
|
||||
partOffset = skipLen - cumulativeSum
|
||||
|
||||
// All return values except `l` are
|
||||
// calculated here.
|
||||
sn = uint32(partOffset / pkgSz)
|
||||
skip = partOffset % pkgSz
|
||||
ps = i
|
||||
o = cumulativeEncSum + int64(sn)*(pkgSz+32)
|
||||
|
||||
partDarePkgOffset = partOffset - skip
|
||||
}
|
||||
if readStart {
|
||||
currentPartBytes := v - partOffset
|
||||
currentPartDareBytes := v - partDarePkgOffset
|
||||
if currentPartBytes < toRead {
|
||||
toRead -= currentPartBytes
|
||||
l += getEncSize(currentPartDareBytes)
|
||||
} else {
|
||||
// current part has the last
|
||||
// byte required
|
||||
lbPartOffset := partOffset + toRead - 1
|
||||
|
||||
// round up the lbPartOffset
|
||||
// to the end of the
|
||||
// corresponding DARE package
|
||||
lbPkgEndOffset := lbPartOffset - (lbPartOffset % pkgSz) + pkgSz
|
||||
if lbPkgEndOffset > v {
|
||||
lbPkgEndOffset = v
|
||||
}
|
||||
bytesToDrop := v - lbPkgEndOffset
|
||||
|
||||
// Last segment to update `l`
|
||||
l += getEncSize(currentPartDareBytes - bytesToDrop)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
cumulativeSum += v
|
||||
cumulativeEncSum += getEncSize(v)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
for i, test := range testMPs {
|
||||
{
|
||||
// nil range
|
||||
o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(nil)
|
||||
if err != nil {
|
||||
t.Errorf("Case %d: unexpected err: %v", i, err)
|
||||
}
|
||||
if o != 0 || l != esum(test.oi) || skip != 0 || sn != 0 || ps != 0 {
|
||||
t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
|
||||
}
|
||||
}
|
||||
|
||||
// Skip 1Mib and read 1Mib (in the decrypted object)
|
||||
//
|
||||
// The check below ensures the object is large enough
|
||||
// for the read.
|
||||
if lsum(test.decSizes) >= 2*humanize.MiByte {
|
||||
skipLen, readLen := int64(1)*humanize.MiByte, int64(1)*humanize.MiByte
|
||||
o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, skipLen, skipLen + readLen - 1})
|
||||
if err != nil {
|
||||
t.Errorf("Case %d: unexpected err: %v", i, err)
|
||||
}
|
||||
|
||||
oRef, lRef, skipRef, snRef, psRef := decryptedRangeRef(test.decSizes, skipLen, readLen, false)
|
||||
if o != oRef || l != lRef || skip != skipRef || sn != snRef || ps != psRef {
|
||||
t.Errorf("Case %d: test failed: %d %d %d %d %d (Ref: %d %d %d %d %d)",
|
||||
i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef)
|
||||
}
|
||||
}
|
||||
|
||||
// Read the last 6Mib+1 bytes of the (decrypted)
|
||||
// object
|
||||
//
|
||||
// The check below ensures the object is large enough
|
||||
// for the read.
|
||||
readLen := int64(6)*humanize.MiByte + 1
|
||||
if lsum(test.decSizes) >= readLen {
|
||||
o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{true, -readLen, -1})
|
||||
if err != nil {
|
||||
t.Errorf("Case %d: unexpected err: %v", i, err)
|
||||
}
|
||||
|
||||
oRef, lRef, skipRef, snRef, psRef := decryptedRangeRef(test.decSizes, 0, readLen, true)
|
||||
if o != oRef || l != lRef || skip != skipRef || sn != snRef || ps != psRef {
|
||||
t.Errorf("Case %d: test failed: %d %d %d %d %d (Ref: %d %d %d %d %d)",
|
||||
i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user