Skip to content

Commit

Permalink
Fix for DLFS upload of very large Blobs
Browse files Browse the repository at this point in the history
  • Loading branch information
mikera committed Apr 18, 2024
1 parent ca04cf9 commit 58d119a
Show file tree
Hide file tree
Showing 9 changed files with 32 additions and 24 deletions.
4 changes: 2 additions & 2 deletions convex-core/src/main/java/convex/core/data/AArrayBlob.java
Original file line number Diff line number Diff line change
Expand Up @@ -247,8 +247,8 @@ public boolean equals(ABlob o) {
}

@Override
public boolean equalsBytes(byte[] bytes, int byteOffset) {
return Utils.arrayEquals(store, offset, bytes, byteOffset, length);
public boolean equalsBytes(byte[] bytes, long byteOffset) {
return Utils.arrayEquals(store, offset, bytes, Utils.checkedInt(byteOffset), length);
}

public boolean equalsBytes(ABlob k) {
Expand Down
15 changes: 12 additions & 3 deletions convex-core/src/main/java/convex/core/data/ABlob.java
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ public boolean equals(ACell o) {
* @param offset Offset into byte array from which to start comparison
* @return true if exactly equal, false otherwise
*/
public abstract boolean equalsBytes(byte[] bytes, int offset);
public abstract boolean equalsBytes(byte[] bytes, long offset);

/**
* Compares this Blob to another Blob, in lexicographic order sorting by first
Expand Down Expand Up @@ -327,8 +327,17 @@ public int read(long offset, ByteBuffer dest) {
*/
public ABlob replaceSlice(long position, ABlob b) {
long end=Math.min(position+b.count(),count());
ABlob rest=slice(end,count());
return slice(0,position).append(b).append(rest);
ABlob head=slice(0,position);
if (head==null)throw new IllegalArgumentException("Invalid "+position+ " in blob of size "+count());
ABlob tail=slice(end,count());
ABlob firstPart= head.append(b);
ABlob result= firstPart.append(tail);

if (result.count()<0) {
head.append(b);
throw new Error("This is bad!!");
}
return result;
}


Expand Down
5 changes: 5 additions & 0 deletions convex-core/src/main/java/convex/core/data/ALongBlob.java
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,11 @@ public final boolean equalsBytes(ABlob b) {
if (b.count()!=LENGTH) return false;
return value==b.longValue();
}

@Override
public boolean equalsBytes(byte[] bytes, long byteOffset) {
return value==Utils.readLong(bytes, Utils.checkedInt(byteOffset),8);
}

@Override
public abstract byte getTag();
Expand Down
5 changes: 1 addition & 4 deletions convex-core/src/main/java/convex/core/data/Address.java
Original file line number Diff line number Diff line change
Expand Up @@ -235,10 +235,7 @@ protected void updateDigest(MessageDigest digest) {
toFlatBlob().updateDigest(digest);
}

@Override
public boolean equalsBytes(byte[] bytes, int byteOffset) {
return value==Utils.readLong(bytes, byteOffset,8);
}


public static final int MAX_ENCODING_LENGTH = 1+Format.MAX_VLC_COUNT_LENGTH;

Expand Down
13 changes: 7 additions & 6 deletions convex-core/src/main/java/convex/core/data/BlobTree.java
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ public ABlob slice(long start, long end) {
if (end >count) return null;
long length=end-start;;
if (length<0) return null;
if (start==end) return Blob.EMPTY;

if ((start == 0L) && (length == this.count)) return this;

Expand Down Expand Up @@ -258,7 +259,7 @@ protected void updateDigest(MessageDigest digest) {

@Override
public byte byteAtUnchecked(long i) {
int childLength = childLength();
long childLength = childLength();
int ci = (int) (i >> (shift + Blobs.CHUNK_SHIFT));
return getChild(ci).byteAtUnchecked(i - ci * childLength);
}
Expand All @@ -267,8 +268,8 @@ public byte byteAtUnchecked(long i) {
* Gets the length in bytes of each full child of this BlobTree
* @return
*/
private int childLength() {
return 1 << (shift + Blobs.CHUNK_SHIFT);
private long childLength() {
return 1L << (shift + Blobs.CHUNK_SHIFT);
}

@Override
Expand All @@ -288,8 +289,8 @@ public boolean equals(BlobTree b) {
}

@Override
public boolean equalsBytes(byte[] bytes, int byteOffset) {
int clen=childLength();
public boolean equalsBytes(byte[] bytes, long byteOffset) {
long clen=childLength();
for (int i=0; i<children.length; i++) {
if (!(getChild(i).equalsBytes(bytes, byteOffset+i*clen))) return false;
}
Expand Down Expand Up @@ -453,7 +454,7 @@ public void validate() throws InvalidDataException {
super.validate();
int n = children.length;
if ((n < 2) | (n > FANOUT)) throw new InvalidDataException("Illegal number of BlobTree children: " + n, this);
int clen = childLength();
long clen = childLength();
long total = 0;

// We need to validate and check the lengths of all child notes. Note that only the last child can
Expand Down
5 changes: 0 additions & 5 deletions convex-core/src/main/java/convex/core/data/LongBlob.java
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,6 @@ public byte getTag() {
return Tag.BLOB;
}

@Override
public boolean equalsBytes(byte[] bytes, int byteOffset) {
return value==Utils.readLong(bytes, byteOffset,8);
}

@Override
public boolean isCanonical() {
return false;
Expand Down
5 changes: 3 additions & 2 deletions convex-core/src/main/java/convex/dlfs/impl/DLFileChannel.java
Original file line number Diff line number Diff line change
Expand Up @@ -114,12 +114,13 @@ public int write(ByteBuffer src) throws IOException {
long n=b.count();
ABlob newData=data.replaceSlice(position,b);

// position after replaced slice
position=position+n;

if (newData!=data) {
AVector<ACell> newNode=node.assoc(DLFSNode.POS_DATA, newData);
updateNode(newNode);
}
// position after replaced slice
position=position+n;

return (int)n;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public void run() {
frame.setTitle(getTitle());
frame.setIconImage(Toolkit.getDefaultToolkit()
.getImage(MainGUI.class.getResource("/images/Convex.png")));
frame.setBounds(50, 50, 1000, 800);
frame.setBounds(50, 50, 1200, 920);

Toolkit.closeIfFirstFrame(frame);

Expand Down
2 changes: 1 addition & 1 deletion convex-gui/src/main/java/convex/gui/dlfs/PreviewPanel.java
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,12 @@ public void setPath(Path path) {
sb.append("DLFS Directory accessible\n");
}
sb.append("Entries: "+DLFSNode.getDirectoryEntries(node).count()+"\n");
sb.append("Node Hash: "+node.getHash()+"\n");
} else {
ABlob data=DLFSNode.getData(node);
sb.append("File Size: "+data.count()+"\n");
sb.append("Data Hash: "+data.getHash()+"\n");
}
sb.append("Node Hash: "+node.getHash()+"\n");
} else {
sb.append("Not a DLFS file: "+Utils.getClassName(path));
}
Expand Down

0 comments on commit 58d119a

Please sign in to comment.