Add GET ranges (#155)

Add option that will request random ranges with GET.

README.md

@@ -285,6 +285,10 @@ Throughput, split into 299 x 1s:
 The `GET` operations will contain the time until the first byte was received.
 This can be accessed using the `--analyze.v` parameter.
 
+It is possible to test speed of partial file requests using the `--range` option.
+This will start reading each object at a random offset and read a random number of bytes.
+Using this produces output similar to `--obj.randsize` - and they can even be combined. 
+
 ## PUT
 
 Benchmarking put operations will upload objects of size `--obj.size` until `--duration` time has elapsed.

cli/get.go

@@ -36,6 +36,10 @@ var (
 			Value: "10MiB",
 			Usage: "Size of each generated object. Can be a number or 10KiB/MiB/GiB. All sizes are base 2 binary.",
 		},
+		cli.BoolFlag{
+			Name:  "range",
+			Usage: "Do ranged get operations. Will request with random offset and length.",
+		},
 	}
 )
 
@@ -71,6 +75,7 @@ func mainGet(ctx *cli.Context) error {
 			Location:    "",
 			PutOpts:     putOpts(ctx),
 		},
+		RandomRanges:  ctx.Bool("range"),
 		CreateObjects: ctx.Int("objects"),
 		GetOpts:       minio.GetObjectOptions{ServerSideEncryption: sse},
 	}

pkg/bench/get.go

@@ -35,6 +35,7 @@ import (
 // Get benchmarks download speed.
 type Get struct {
 	CreateObjects int
+	RandomRanges  bool
 	Collector     *Collector
 	objects       generator.Objects
 
@@ -182,6 +183,14 @@ func (g *Get) Start(ctx context.Context, wait chan struct{}) (Operations, error)
 					ObjPerOp: 1,
 					Endpoint: client.EndpointURL().String(),
 				}
+				if g.RandomRanges && op.Size > 2 {
+					// Randomize length similar to --obj.randsize
+					size := generator.GetExpRandSize(rng, op.Size-2)
+					start := rng.Int63n(op.Size - size)
+					end := start + size
+					op.Size = end - start + 1
+					opts.SetRange(start, end)
+				}
 				op.Start = time.Now()
 				var err error
 				opts.VersionID = obj.VersionID
@@ -202,8 +211,8 @@ func (g *Get) Start(ctx context.Context, wait chan struct{}) (Operations, error)
 				}
 				op.FirstByte = fbr.t
 				op.End = time.Now()
-				if n != obj.Size && op.Err == "" {
-					op.Err = fmt.Sprint("unexpected download size. want:", obj.Size, ", got:", n)
+				if n != op.Size && op.Err == "" {
+					op.Err = fmt.Sprint("unexpected download size. want:", op.Size, ", got:", n)
 					g.Error(op.Err)
 				}
 				rcv <- op

pkg/generator/generator.go

@@ -20,6 +20,7 @@ package generator
 import (
 	"errors"
 	"io"
+	"math"
 	"math/rand"
 	"runtime"
 )
@@ -170,3 +171,24 @@ func randASCIIBytes(dst []byte, rng *rand.Rand) {
 		rnd *= 2654435761
 	}
 }
+
+// GetExpRandSize will return an exponential random size from 1 to and including max.
+// Minimum size: 127 bytes, max scale is 256 times smaller than max size.
+func GetExpRandSize(rng *rand.Rand, max int64) int64 {
+	if max < 10 {
+		if max == 0 {
+			return 0
+		}
+		return 1 + rng.Int63n(max)
+	}
+	logSizeMax := math.Log2(float64(max - 1))
+	logSizeMin := math.Max(7, logSizeMax-8)
+	lsDelta := logSizeMax - logSizeMin
+	random := rng.Float64()
+	logSize := random * lsDelta
+	if logSize > 1 {
+		return 1 + int64(math.Pow(2, logSize+logSizeMin))
+	}
+	// For lowest part, do linear
+	return 1 + int64(random*math.Pow(2, logSizeMin+1))
+}

pkg/generator/options.go

@@ -19,7 +19,6 @@ package generator
 
 import (
 	"errors"
-	"math"
 	"math/rand"
 )
 
@@ -44,17 +43,7 @@ func (o Options) getSize(rng *rand.Rand) int64 {
 	if !o.randSize {
 		return o.totalSize
 	}
-	logSizeMax := math.Log2(float64(o.totalSize - 1))
-	// Minimum size: 127 bytes, max scale is 256 times smaller than max size.
-	logSizeMin := math.Max(7, logSizeMax-8)
-	lsDelta := logSizeMax - logSizeMin
-	random := rng.Float64()
-	logSize := random * lsDelta
-	if logSize > 1 {
-		return 1 + int64(math.Pow(2, logSize+logSizeMin))
-	}
-	// For lowest part, do linear
-	return 1 + int64(random*math.Pow(2, logSizeMin+1))
+	return GetExpRandSize(rng, o.totalSize)
 }
 
 func defaultOptions() Options {