HD-4220: adapt get range

Author: fferrandis

backend.go

@@ -372,7 +372,6 @@ func (backend *Backend) ChunkInfo(fragRangeLen int, pieceSize int) ChunkInfo {
 	if nrChunks*chunkSize != fragRangeLen {
 		nrChunks++
 	}
-
 	return ChunkInfo{
 		ChunkSize: chunkSize,
 		NrChunk:   nrChunks,
@@ -392,7 +391,7 @@ func (backend *Backend) LinearizeMatrix(frags []ValidatedFragment, pieceSize int
 	/* Fragments are sorted beforehand with the index of the first chunk.
 	   All chunks of a fragments share the same index. */
 	fragsIndex := make([]int, len(frags))
-	for i := 0; i < len(frags); i++ {
+	for i := range frags {
 		fragsIndex[i] = i
 	}
 
@@ -591,41 +590,65 @@ type RangeMatrix struct {
  *     p4 [-[*]- -]
  *
  */
-func (backend *Backend) GetRangeMatrix(startIncl, endIncl, pieceSize, fragSize int) *RangeMatrix {
-	chunkSize := pieceSize + backend.headerSize
-	groupSize := pieceSize * backend.K
+func (backend *Backend) GetRangeMatrix(startIncl, endIncl, cellDataSize, fragSize int) *RangeMatrix {
+	nrColumns := backend.K
+	cellSize := cellDataSize + backend.headerSize
+	lineSize := cellDataSize * nrColumns
 
 	/* At this point we don't know what is the true payload size, but we
 	   can at least check that it doesn't exceed the maximum payload that
 	   this configuration can handle. */
-	nrChunkByFrag := fragSize / chunkSize
-	dataLenPerFrag := fragSize - nrChunkByFrag*backend.headerSize
-	maxDataLen := dataLenPerFrag * backend.K
-	if startIncl >= maxDataLen || endIncl >= maxDataLen || startIncl > endIncl {
+	nrLines := fragSize / cellSize
+
+	/* Inside a fragment (ie, inside a column), what is the
+	   stored amount of data? */
+	fragDataSize := fragSize - nrLines*backend.headerSize
+
+	maxDataSize := fragDataSize * nrColumns
+
+	if startIncl >= maxDataSize || endIncl >= maxDataSize || startIncl > endIncl {
 		return nil
 	}
 
-	pieceStartIncl := startIncl / pieceSize
-	pieceEndIncl := endIncl / pieceSize
+	/* convert cells indices to (x,y) indices */
+
+	/* Considering our (x,y) matrix as a single row, what
+	   are the indices of the start and end of the range we are interested in? */
+	idxStart := startIncl / cellDataSize
+	idxEnd := endIncl / cellDataSize
+
+	/* as we have the indices of the first cell and the last cell,
+	 * we can derive the indices of the first line and the last line
+	 * Based on this first line and last line, we can deduce
+	 * the amount of data to read in each fragment.
+	 */
+	lineStart := idxStart / nrColumns
+	lineEnd := idxEnd / nrColumns
+
+	/*
+	 * as we have the indices of the first cell and the last cell,
+	 * we can compute the first column (e.g the first fragment)
+	 * where to start the read
+	 */
+	columnStart := idxStart % nrColumns
 
-	groupStartIncl := pieceStartIncl / backend.K
-	groupEndIncl := pieceEndIncl / backend.K
+	nrCellsToRead := (idxEnd + 1 - idxStart)
+	dataOffset := idxStart * cellDataSize
 
-	fragFirstIncl := pieceStartIncl % backend.K
-	fragCount := (pieceEndIncl + 1 - pieceStartIncl)
-	dataOffset := pieceStartIncl * pieceSize
+	totalLines := (maxDataSize + lineSize - 1) / lineSize
+	isLastStripe := (lineEnd == totalLines-1)
 
 	/* When wrapping around, we read the full groups. */
-	if fragFirstIncl+fragCount > backend.K {
-		fragFirstIncl = 0
-		fragCount = backend.K
-		dataOffset = groupStartIncl * groupSize
+	if columnStart+nrCellsToRead > nrColumns || isLastStripe {
+		columnStart = 0
+		nrCellsToRead = nrColumns
+		dataOffset = lineStart * lineSize
 	}
 
 	/* For each fragment, this is the minimum range to read -- including
 	   the header -- to decode or repair the data. */
-	inFragRangeStartIncl := groupStartIncl * chunkSize
-	inFragRangeEndExcl := (groupEndIncl + 1) * chunkSize
+	inFragRangeStartIncl := lineStart * cellSize
+	inFragRangeEndExcl := (lineEnd + 1) * cellSize
 
 	/* The output buffer only contains the data necessary to read the range,
 	   and the requested range must be adjusted to be relative
@@ -637,13 +660,13 @@ func (backend *Backend) GetRangeMatrix(startIncl, endIncl, pieceSize, fragSize i
 	linearizedRangeStartIncl := startIncl - dataOffset
 
 	/* Decoding always works on a group boundary. */
-	decodedRangeStartIncl := startIncl - groupStartIncl*groupSize
+	decodedRangeStartIncl := startIncl - lineStart*lineSize
 
 	return &RangeMatrix{
 		ReqStartIncl:             startIncl,
 		ReqEndIncl:               endIncl,
-		FragFirstIncl:            fragFirstIncl,
-		FragCount:                fragCount,
+		FragFirstIncl:            columnStart,
+		FragCount:                nrCellsToRead,
 		InFragRangeStartIncl:     inFragRangeStartIncl,
 		InFragRangeEndExcl:       inFragRangeEndExcl,
 		DecodedRangeStartIncl:    decodedRangeStartIncl,

backend_test.go

@@ -802,17 +802,107 @@ func TestEncodeM(t *testing.T) {
 	_ = backend.Close()
 }
 
-func TestLinearizeMatrix(t *testing.T) {
-	assert := assert.New(t)
+func TestTwoLinearizeMatrix(t *testing.T) {
+	backend, err := InitBackend(Params{Name: "isa_l_rs_vand", K: 2, M: 1, W: 8, HD: 5})
+	require.NoError(t, err)
+	defer func() {
+		_ = backend.Close()
+	}()
+	currentChunkSize := 512
+	dataSize := currentChunkSize*2 + 10
+	startIncl := dataSize - 3
+	endIncl := dataSize - 1
+
+	data := make([]byte, dataSize)
+	for i := range dataSize {
+		data[i] = byte('A' + i%26)
+	}
+	bm := NewBufferMatrix(currentChunkSize, len(data), backend.K)
+	_, err = io.Copy(bm, bytes.NewReader(data))
+	require.NoError(t, err)
+	bm.Finish()
+	encoded, err := backend.EncodeMatrixWithBufferMatrix(bm, currentChunkSize)
+	require.NoError(t, err)
+	defer encoded.Free()
 
+	rangeM := backend.GetRangeMatrix(startIncl, endIncl, currentChunkSize, len(encoded.Data[0]))
+	require.NotNil(t, rangeM)
+
+	/* Decode the matrix as if it was requested and
+	   checks that the result matches the payload on the requested range. */
+	frags := make([][]byte, 0)
+	for i := 0; i < rangeM.FragCount; i++ {
+		fragIdx := (rangeM.FragFirstIncl + i) % backend.K
+		buffer := encoded.Data[fragIdx][rangeM.InFragRangeStartIncl:rangeM.InFragRangeEndExcl]
+		frags = append(frags, buffer)
+	}
+
+	decoded, err := backend.LinearizeMatrix(frags, currentChunkSize)
+	require.NoError(t, err)
+	defer decoded.Free()
+
+	expected := data[startIncl:endIncl]
+
+	linearizedRangeEndExcl := rangeM.LinearizedRangeStartIncl + (endIncl - startIncl)
+	found := decoded.Data[rangeM.LinearizedRangeStartIncl:linearizedRangeEndExcl]
+
+	require.True(t, bytes.Equal(expected, found))
+}
+
+func TestOneLinearizeMatrix(t *testing.T) {
+	backend, err := InitBackend(Params{Name: "isa_l_rs_vand", K: 4, M: 2, W: 8, HD: 5})
+	require.NoError(t, err)
+	defer func() {
+		_ = backend.Close()
+	}()
+	dataSize := 105623
+	startIncl := 59441
+	endIncl := 64149
+	data := make([]byte, dataSize)
+	for i := range dataSize {
+		data[i] = byte('A' + i%26)
+	}
+	bm := NewBufferMatrix(DefaultChunkSize, len(data), backend.K)
+	_, err = io.Copy(bm, bytes.NewReader(data))
+	require.NoError(t, err)
+	bm.Finish()
+	encoded, err := backend.EncodeMatrixWithBufferMatrix(bm, DefaultChunkSize)
+	require.NoError(t, err)
+	defer encoded.Free()
+
+	rangeM := backend.GetRangeMatrix(startIncl, endIncl, DefaultChunkSize, len(encoded.Data[0]))
+	require.NotNil(t, rangeM)
+
+	/* Decode the matrix as if it was requested and
+	   checks that the result matches the payload on the requested range. */
+	frags := make([][]byte, 0)
+	for i := 0; i < rangeM.FragCount; i++ {
+		fragIdx := (rangeM.FragFirstIncl + i) % backend.K
+		buffer := encoded.Data[fragIdx][rangeM.InFragRangeStartIncl:rangeM.InFragRangeEndExcl]
+		frags = append(frags, buffer)
+	}
+
+	decoded, err := backend.LinearizeMatrix(frags, DefaultChunkSize)
+	require.NoError(t, err)
+	defer decoded.Free()
+
+	expected := data[startIncl : endIncl+1]
+
+	linearizedRangeEndExcl := rangeM.LinearizedRangeStartIncl + (endIncl - startIncl) + 1
+	found := decoded.Data[rangeM.LinearizedRangeStartIncl:linearizedRangeEndExcl]
+	require.True(t, bytes.Equal(expected, found))
+}
+
+func TestLinearizeMatrix(t *testing.T) {
 	pieceSize := DefaultChunkSize
 	k := 4
 	m := 1
 
 	backend, err := InitBackend(Params{Name: "isa_l_rs_vand", K: k, M: m, W: 8, HD: m})
-	if err != nil {
-		t.Fatalf("cannot init backend: (%v)", err)
-	}
+	require.NoError(t, err)
+	defer func() {
+		_ = backend.Close()
+	}()
 
 	rangeValues := func(values []reflect.Value, rng *rand.Rand) {
 		dataSize := 1 + rng.Intn(7*1024*1024)
@@ -848,7 +938,7 @@ func TestLinearizeMatrix(t *testing.T) {
 
 		fragSize := len(encoded.Data[0])
 		rangeM := backend.GetRangeMatrix(startIncl, endIncl, pieceSize, fragSize)
-		assert.NotNil(rangeM)
+		require.NotNil(t, rangeM)
 
 		/* Decode the matrix as if it was requested and
 		   checks that the result matches the payload on the requested range. */
@@ -860,7 +950,7 @@ func TestLinearizeMatrix(t *testing.T) {
 		}
 
 		decoded, err := backend.LinearizeMatrix(frags, pieceSize)
-		assert.Nil(err)
+		require.NoError(t, err)
 		defer decoded.Free()
 
 		expected := data[startIncl : endIncl+1]
@@ -874,9 +964,8 @@ func TestLinearizeMatrix(t *testing.T) {
 		Values: rangeValues,
 	}
 
-	if err := quick.Check(checkRange, &config); err != nil {
-		t.Error(err)
-	}
+	require.NoError(t, quick.Check(checkRange, &config))
+
 }
 
 func TestDecodeMatrix(t *testing.T) {