Better handling for ZeroBlobs including sparse files

convex-core/src/main/java/convex/core/crypto/bc/BCKeyPair.java

@@ -38,7 +38,7 @@ public static BCKeyPair create(Blob seed) {
 		long n=seed.count();
 		if (seed.count() != SEED_LENGTH) throw new IllegalArgumentException("32 bytes private key material expected as seed but got: "+n);
 
-		Ed25519PrivateKeyParameters priv=new Ed25519PrivateKeyParameters(Blobs.zeroBasedArray(seed), 0);
+		Ed25519PrivateKeyParameters priv=new Ed25519PrivateKeyParameters(Blobs.ensureZeroBasedArray(seed), 0);
 		byte[] publicBytes=priv.generatePublicKey().getEncoded();
 		AccountKey publicKey=AccountKey.wrap(publicBytes);
 

convex-core/src/main/java/convex/core/data/Blob.java

@@ -28,7 +28,8 @@ public class Blob extends AArrayBlob {
 	public static final Blob NULL_ENCODING = Blob.wrap(new byte[] {Tag.NULL});
 
 	public static final int CHUNK_LENGTH = 4096;
-
+
+	public static final Blob EMPTY_CHUNK = wrap(new byte[CHUNK_LENGTH]);
 
 	private Blob(byte[] bytes, int offset, int length) {
 		super(bytes, offset, length);

convex-core/src/main/java/convex/core/data/BlobTree.java

@@ -363,7 +363,7 @@ public int estimatedEncodingSize() {
 	 */
 	@Override
 	public ABlob append(ABlob d) {
-		BlobTree acc=this;
+		BlobTree acc=this; // accumulator for appended BlobTree
 		long off=0; // offset into d
 		long dlen=d.count();
 
@@ -395,7 +395,7 @@ public ABlob append(ABlob d) {
 				}
 			}
 
-			// Next level takes a following child with up to as memy bytes as acc
+			// Next level takes a following child with up to as many bytes as acc
 			long take=Math.min(acc.count(), dlen-off);
 			BlobTree nextLevel=BlobTree.createWithChildren(new ABlob[] {acc,d.slice(off,off+take)});
 			acc=nextLevel;

convex-core/src/main/java/convex/core/data/Blobs.java

@@ -4,6 +4,7 @@
 
 import org.bouncycastle.util.Arrays;
 
+import convex.core.data.impl.ZeroBlob;
 import convex.core.data.util.BlobBuilder;
 import convex.core.exceptions.BadFormatException;
 import convex.core.lang.RT;
@@ -126,7 +127,7 @@ public static <T extends ABlob> T read(Blob source, int pos) throws BadFormatExc
 	 * Create a Blob entirely filled with a given value
 	 * @param value Byte value to fill with (low 8 bits used)
 	 * @param length Length of Blob to create
-	 * @return BlobTree filled with given value
+	 * @return Blob filled with given value
 	 */
 	public static ABlob createFilled(int value, long length) {
 		byte fillByte=(byte)value;
@@ -146,6 +147,16 @@ public static ABlob createFilled(int value, long length) {
 		children[n-1]=lastSize==subSize?fullChild:Blobs.createFilled(fillByte, lastSize);
 		return BlobTree.createWithChildren(children);
 	}
+
+	/**
+	 * Create a Blob entirely filled with zeros
+	 * @param length Length of Blob to create
+	 * @return Blob filled with zeros
+	 */
+	public static ABlob createZero(long length) {
+		if (length<=Blob.CHUNK_LENGTH) return Blob.EMPTY_CHUNK.slice(0,length);
+		return ZeroBlob.create(length);
+	}
 
 	public static Blob empty() {
 		return Blob.EMPTY;
@@ -154,11 +165,12 @@ public static Blob empty() {
 	/** 
 	 * Gets a zero-based array containing the contents of the given Blob.
 	 * MAY use current internal array if possible.
+	 * WARNING: may return underlying array, should never be mutated
 	 * 
 	 * @param b Blob to get array for
 	 * @return byte array containing the blob contents starting at offset zero
 	 */
-	public static byte[] zeroBasedArray(AArrayBlob b) {
+	public static byte[] ensureZeroBasedArray(AArrayBlob b) {
 		if (b.getInternalOffset()==0) {
 			return b.getInternalArray();
 		} else {

convex-core/src/main/java/convex/core/data/impl/ZeroBlob.java

@@ -2,6 +2,7 @@
 
 import org.bouncycastle.util.Arrays;
 
+import convex.core.data.ABlob;
 import convex.core.data.ADerivedBlob;
 
 /**
@@ -14,13 +15,22 @@ public class ZeroBlob extends ADerivedBlob {
 	protected ZeroBlob(long count) {
 		super(count);
 	}
+
+	@Override
+	public ABlob append(ABlob other) {
+		if (other instanceof ZeroBlob) {
+			// fast path for appending zeros :-)
+			return create(count+((ZeroBlob)other).count);
+		}
+		return super.append(other);
+	}
 
 	@Override
 	public ZeroBlob sliceImpl(long start, long end) {
 		return new ZeroBlob(end-start);
 	}
 
-	public static ZeroBlob create(int count) {
+	public static ZeroBlob create(long count) {
 		if (count<0) throw new IllegalArgumentException("Negative Count");
 		if (count==0) return EMPTY;
 		return new ZeroBlob(count);

convex-core/src/main/java/convex/dlfs/impl/DLFileChannel.java

@@ -15,6 +15,7 @@
 import convex.core.data.ACell;
 import convex.core.data.AVector;
 import convex.core.data.Blob;
+import convex.core.data.Blobs;
 import convex.dlfs.DLFSNode;
 import convex.dlfs.DLFileSystem;
 import convex.dlfs.DLPath;
@@ -101,21 +102,23 @@ public int read(ByteBuffer dst) throws IOException {
 	public int write(ByteBuffer src) throws IOException {
 		synchronized(this) {
 			checkOpen();
+			long pos=position;
 			AVector<ACell> node=getNode();
 			ABlob data = DLFSNode.getData(node);
 			if (data==null) throw new NoSuchFileException(path.toString());
 
-			if (data.count()<position) {
-				// TODO: is this sane?
-				position=data.count();
+			if (data.count()<pos) {
+				// extend file with zeros to start at new position
+				// ZeroBlob implementation makes this relatively cheap
+				data=data.append(Blobs.createZero(pos-data.count()));
 			}
 
 			Blob b=Blob.fromByteBuffer(src);
 			long n=b.count();
-			ABlob newData=data.replaceSlice(position,b);
+			ABlob newData=data.replaceSlice(pos,b);
 
 			// position after replaced slice
-			position=position+n;
+			position=pos+n;
 
 			if (newData!=data) {
 				AVector<ACell> newNode=node.assoc(DLFSNode.POS_DATA, newData);

convex-core/src/test/java/convex/core/data/BlobsTest.java

@@ -49,7 +49,13 @@ public void testCompare() {
 
 	@Test
 	public void testEqualsCases() {
+		// LongBlob is a Blob, but a similar Address isn't
+		assertEquals(Blob.fromHex("0000000000000001"),LongBlob.create(1));
 		assertNotEquals(Blob.fromHex("0000000000000001"),Address.create(1));
+
+		// Two empty Bloblikes are different
+		assertNotEquals(Strings.EMPTY,Blob.EMPTY);
+
 		assertNotEquals(Address.create(1),Blob.fromHex("0000000000000001"));
 	}
 
@@ -484,6 +490,8 @@ public void testZeroBlobs() {
 
 		assertSame(Blob.EMPTY,z0.toFlatBlob());
 
+		assertTrue(z3.append(z4) instanceof ZeroBlob);
+
 		doBlobTests(z0);
 		doBlobTests(z1);
 		doBlobTests(z2);

convex-sodium/src/main/java/convex/core/crypto/sodium/SodiumKeyPair.java

@@ -64,7 +64,7 @@ public ASignature sign(AArrayBlob hash) {
 		byte[] signature=new byte[Ed25519Signature.SIGNATURE_LENGTH];
 		if (SodiumProvider.SODIUM_SIGN.cryptoSignDetached(
 				signature,
-				Blobs.zeroBasedArray(hash),
+				Blobs.ensureZeroBasedArray(hash),
 				mlength,
 				secretKeyBytes)) {;
 				return Ed25519Signature.wrap(signature);