Merge pull request #229 from 0xAozora/v4

Support for Append

Author: pierrec

bench_test.go

@@ -85,14 +85,15 @@ func mustLoadFile(f string) []byte {
 }
 
 var (
-	pg1661    = mustLoadFile("testdata/pg1661.txt.gz")
-	digits    = mustLoadFile("testdata/e.txt.gz")
-	twain     = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt.gz")
-	random    = mustLoadFile("testdata/random.data.gz")
-	pg1661LZ4 = mustLoadFile("testdata/pg1661.txt.lz4")
-	digitsLZ4 = mustLoadFile("testdata/e.txt.lz4")
-	twainLZ4  = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt.lz4")
-	randomLZ4 = mustLoadFile("testdata/random.data.lz4")
+	pg1661            = mustLoadFile("testdata/pg1661.txt.gz")
+	digits            = mustLoadFile("testdata/e.txt.gz")
+	twain             = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt.gz")
+	random            = mustLoadFile("testdata/random.data.gz")
+	pg1661LZ4         = mustLoadFile("testdata/pg1661.txt.lz4")
+	digitsLZ4         = mustLoadFile("testdata/e.txt.lz4")
+	twainLZ4          = mustLoadFile("testdata/Mark.Twain-Tom.Sawyer.txt.lz4")
+	randomLZ4         = mustLoadFile("testdata/random.data.lz4")
+	randomAppendedLZ4 = mustLoadFile("testdata/random_appended.data.lz4")
 )
 
 func benchmarkUncompress(b *testing.B, compressed []byte) {

internal/lz4errors/errors.go

@@ -16,4 +16,5 @@ const (
 	ErrOptionInvalidBlockSize        Error = "lz4: invalid block size"
 	ErrOptionNotApplicable           Error = "lz4: option not applicable"
 	ErrWriterNotClosed               Error = "lz4: writer not closed"
+	ErrEndOfStream                   Error = "lz4: end of stream reached"
 )

internal/lz4stream/block.go

@@ -143,7 +143,7 @@ func (b *Blocks) initR(f *Frame, num int, src io.Reader) (chan []byte, error) {
 		c <- nil // signal the collection loop that we are done
 		<-c      // wait for the collect loop to complete
 		if f.isLegacy() && cum == cumx {
-			err = io.EOF
+			err = lz4errors.ErrEndOfStream
 		}
 		b.closeR(err)
 		close(data)
@@ -290,11 +290,11 @@ func (b *FrameDataBlock) Read(f *Frame, src io.Reader, cum uint32) (uint32, erro
 			// Only works in non concurrent mode, for concurrent mode
 			// it is handled separately.
 			// Linux kernel format appends the total uncompressed size at the end.
-			return 0, io.EOF
+			return 0, lz4errors.ErrEndOfStream
 		}
 	} else if x == 0 {
 		// Marker for end of stream.
-		return 0, io.EOF
+		return 0, lz4errors.ErrEndOfStream
 	}
 	b.Size = DataBlockSize(x)
 

reader.go

@@ -117,6 +117,7 @@ func (r *Reader) Read(buf []byte) (n int, err error) {
 	for len(buf) > 0 {
 		var bn int
 		if r.idx == 0 {
+		read:
 			if r.isNotConcurrent() {
 				bn, err = r.read(buf)
 			} else {
@@ -129,6 +130,21 @@ func (r *Reader) Read(buf []byte) (n int, err error) {
 			}
 			switch err {
 			case nil:
+			case lz4errors.ErrEndOfStream:
+
+				// Read Checksum.
+				err = r.frame.CloseR(r.src)
+				if err != nil {
+					return
+				}
+
+				//Check for new stream.
+				r.Reset(r.src)
+				if err = r.init(); r.state.next(err) {
+					return
+				}
+
+				goto read
 			case io.EOF:
 				if er := r.frame.CloseR(r.src); er != nil {
 					err = er
@@ -235,6 +251,7 @@ func (r *Reader) WriteTo(w io.Writer) (n int64, err error) {
 	for {
 		var bn int
 		var dst []byte
+	read:
 		if r.isNotConcurrent() {
 			bn, err = r.read(data)
 			dst = data[:bn]
@@ -249,6 +266,26 @@ func (r *Reader) WriteTo(w io.Writer) (n int64, err error) {
 		}
 		switch err {
 		case nil:
+		case lz4errors.ErrEndOfStream:
+
+			// Read Checksum.
+			err = r.frame.CloseR(r.src)
+			if err != nil {
+				return
+			}
+
+			//Check for new stream.
+			r.Reset(r.src)
+			if err = r.init(); r.state.next(err) {
+
+				if err == io.EOF {
+					err = nil
+				}
+
+				return
+			}
+
+			goto read
 		case io.EOF:
 			err = r.frame.CloseR(r.src)
 			return

reader_test.go

@@ -155,6 +155,21 @@ func TestReader_Reset(t *testing.T) {
 	}
 }
 
+func TestReader_Append(t *testing.T) {
+	data := randomAppendedLZ4
+
+	src := bytes.NewReader(data)
+	zr := lz4.NewReader(src)
+
+	b, err := io.ReadAll(zr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(b, random) {
+		t.Fatal("result does not match original")
+	}
+}
+
 type brokenWriter int
 
 func (w *brokenWriter) Write(p []byte) (n int, err error) {