Skip to content

Commit f24698d

Browse files
AndreyVMarkelovAndreyVMarkelov
authored
Merge pull request #599 from dropbox/add_content_hasher_utility
Add content hasher utility
2 parents e6f7fef + bb0d316 commit f24698d

4 files changed

Lines changed: 279 additions & 18 deletions

File tree

‎core/api/core.api‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -827,6 +827,14 @@ public class com/dropbox/core/util/CountingOutputStream : java/io/OutputStream {
827827
public fun write ([BII)V
828828
}
829829

830+
public final class com/dropbox/core/util/DropboxContentHasher {
831+
public static final field BLOCK_SIZE I
832+
public static fun hash (Ljava/io/InputStream;)[B
833+
public static fun hash ([B)[B
834+
public static fun hashHex (Ljava/io/InputStream;)Ljava/lang/String;
835+
public static fun hashHex ([B)Ljava/lang/String;
836+
}
837+
830838
public abstract class com/dropbox/core/util/DumpWriter {
831839
public fun <init> ()V
832840
public abstract fun f (Ljava/lang/String;)Lcom/dropbox/core/util/DumpWriter;

‎core/src/main/java/com/dropbox/core/util/DropboxContentHasher.java‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
package com.dropbox.core.util;
2+
3+
import java.io.ByteArrayInputStream;
4+
import java.io.IOException;
5+
import java.io.InputStream;
6+
import java.security.MessageDigest;
7+
import java.security.NoSuchAlgorithmException;
8+
import java.util.Objects;
9+
10+
/**
11+
* Utility for calculating Dropbox content hashes.
12+
*
13+
* <p>The Dropbox content hash is calculated by splitting the input into
14+
* 4 MiB blocks, calculating the SHA-256 hash of each block, concatenating
15+
* those block hashes, and calculating the SHA-256 hash of the resulting
16+
* sequence.</p>
17+
*/
18+
public final class DropboxContentHasher {
19+
public static final int BLOCK_SIZE = 4 * 1024 * 1024;
20+
21+
private static final String SHA_256 = "SHA-256";
22+
private static final char[] HEX_DIGITS = "0123456789abcdef".toCharArray();
23+
24+
private DropboxContentHasher() {
25+
}
26+
27+
/**
28+
* Calculates the Dropbox content hash for an input stream.
29+
*
30+
* <p>This method does not close the supplied stream.</p>
31+
*
32+
* @param input stream to read
33+
* @return raw 32-byte Dropbox content hash
34+
* @throws IOException if the stream cannot be read
35+
*/
36+
public static byte[] hash(InputStream input) throws IOException {
37+
Objects.requireNonNull(input, "input");
38+
39+
MessageDigest overallDigest = newSha256Digest();
40+
MessageDigest blockDigest = newSha256Digest();
41+
byte[] buffer = new byte[BLOCK_SIZE];
42+
43+
while (true) {
44+
int blockLength = readBlock(input, buffer);
45+
if (blockLength == 0) {
46+
break;
47+
}
48+
49+
blockDigest.reset();
50+
blockDigest.update(buffer, 0, blockLength);
51+
overallDigest.update(blockDigest.digest());
52+
}
53+
54+
return overallDigest.digest();
55+
}
56+
57+
/**
58+
* Calculates the lowercase hexadecimal Dropbox content hash for an input stream.
59+
*
60+
* <p>This method does not close the supplied stream.</p>
61+
*
62+
* @param input stream to read
63+
* @return lowercase hexadecimal Dropbox content hash
64+
* @throws IOException if the stream cannot be read
65+
*/
66+
public static String hashHex(InputStream input) throws IOException {
67+
return toHex(hash(input));
68+
}
69+
70+
/**
71+
* Calculates the Dropbox content hash for a byte array.
72+
*
73+
* @param input bytes to hash
74+
* @return raw 32-byte Dropbox content hash
75+
*/
76+
public static byte[] hash(byte[] input) {
77+
Objects.requireNonNull(input, "input");
78+
79+
try {
80+
return hash(new ByteArrayInputStream(input));
81+
} catch (IOException ex) {
82+
// ByteArrayInputStream does not throw IOException while reading.
83+
throw new AssertionError("Unexpected ByteArrayInputStream failure", ex);
84+
}
85+
}
86+
87+
/**
88+
* Calculates the lowercase hexadecimal Dropbox content hash for a byte array.
89+
*
90+
* @param input bytes to hash
91+
* @return lowercase hexadecimal Dropbox content hash
92+
*/
93+
public static String hashHex(byte[] input) {
94+
return toHex(hash(input));
95+
}
96+
97+
private static int readBlock(InputStream input, byte[] buffer) throws IOException {
98+
int offset = 0;
99+
100+
while (offset < buffer.length) {
101+
int bytesRead = input.read(buffer, offset, buffer.length - offset);
102+
103+
if (bytesRead < 0) {
104+
break;
105+
}
106+
107+
if (bytesRead == 0) {
108+
int value = input.read();
109+
if (value < 0) {
110+
break;
111+
}
112+
113+
buffer[offset++] = (byte) value;
114+
} else {
115+
offset += bytesRead;
116+
}
117+
}
118+
119+
return offset;
120+
}
121+
122+
private static MessageDigest newSha256Digest() {
123+
try {
124+
return MessageDigest.getInstance(SHA_256);
125+
} catch (NoSuchAlgorithmException ex) {
126+
throw new IllegalStateException("SHA-256 is not available", ex);
127+
}
128+
}
129+
130+
private static String toHex(byte[] bytes) {
131+
char[] result = new char[bytes.length * 2];
132+
133+
for (int i = 0; i < bytes.length; i++) {
134+
int value = bytes[i] & 0xff;
135+
result[i * 2] = HEX_DIGITS[value >>> 4];
136+
result[i * 2 + 1] = HEX_DIGITS[value & 0x0f];
137+
}
138+
139+
return new String(result);
140+
}
141+
}

‎core/src/test/java/com/dropbox/core/util/DropboxContentHasherTest.java‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
package com.dropbox.core.util;
2+
3+
import org.testng.annotations.Test;
4+
5+
import java.io.ByteArrayInputStream;
6+
import java.io.FilterInputStream;
7+
import java.io.IOException;
8+
import java.io.InputStream;
9+
import java.util.Random;
10+
11+
import static org.testng.Assert.assertEquals;
12+
import static org.testng.Assert.assertFalse;
13+
14+
public class DropboxContentHasherTest {
15+
16+
@Test
17+
public void testEmpty() {
18+
assertEquals(
19+
DropboxContentHasher.hashHex(new byte[0]),
20+
"e3b0c44298fc1c149afbf4c8996fb924"
21+
+ "27ae41e4649b934ca495991b7852b855");
22+
}
23+
24+
@Test
25+
public void testByteArrayAndStreamMatch() throws Exception {
26+
byte[] data = randomBytes(12345);
27+
28+
assertEquals(
29+
DropboxContentHasher.hashHex(data),
30+
DropboxContentHasher.hashHex(new ByteArrayInputStream(data))
31+
);
32+
}
33+
34+
@Test
35+
public void testExactlyOneBlock() throws Exception {
36+
byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE);
37+
38+
assertEquals(
39+
DropboxContentHasher.hashHex(data),
40+
DropboxContentHasher.hashHex(new ByteArrayInputStream(data))
41+
);
42+
}
43+
44+
@Test
45+
public void testBlockPlusOneByte() throws Exception {
46+
byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE + 1);
47+
48+
assertEquals(
49+
DropboxContentHasher.hashHex(data),
50+
DropboxContentHasher.hashHex(new ByteArrayInputStream(data))
51+
);
52+
}
53+
54+
@Test
55+
public void testMultipleBlocks() throws Exception {
56+
byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE * 3 + 123);
57+
58+
assertEquals(
59+
DropboxContentHasher.hashHex(data),
60+
DropboxContentHasher.hashHex(new ByteArrayInputStream(data))
61+
);
62+
}
63+
64+
@Test
65+
public void testPartialReads() throws Exception {
66+
byte[] data = randomBytes(DropboxContentHasher.BLOCK_SIZE + 123);
67+
68+
InputStream partial = new FilterInputStream(
69+
new ByteArrayInputStream(data)
70+
) {
71+
@Override
72+
public int read(byte[] b, int off, int len) throws IOException {
73+
return super.read(b, off, Math.min(len, 37));
74+
}
75+
};
76+
77+
assertEquals(
78+
DropboxContentHasher.hashHex(data),
79+
DropboxContentHasher.hashHex(partial)
80+
);
81+
}
82+
83+
@Test
84+
public void testDoesNotCloseStream() throws Exception {
85+
CloseTrackingInputStream in =
86+
new CloseTrackingInputStream(
87+
new ByteArrayInputStream(randomBytes(1024)));
88+
89+
DropboxContentHasher.hashHex(in);
90+
91+
assertFalse(in.closed);
92+
}
93+
94+
private static byte[] randomBytes(int size) {
95+
byte[] data = new byte[size];
96+
new Random(12345).nextBytes(data);
97+
return data;
98+
}
99+
100+
private static final class CloseTrackingInputStream
101+
extends FilterInputStream {
102+
103+
boolean closed;
104+
105+
CloseTrackingInputStream(InputStream in) {
106+
super(in);
107+
}
108+
109+
@Override
110+
public void close() throws IOException {
111+
closed = true;
112+
super.close();
113+
}
114+
}
115+
}

‎core/src/test/java/com/dropbox/core/v2/DbxClientV2IT.java‎

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,25 @@
11
package com.dropbox.core.v2;
22

3-
import static com.google.common.truth.Truth.assertThat;
4-
import static com.google.common.truth.Truth.assertWithMessage;
5-
6-
import java.io.ByteArrayInputStream;
7-
import java.io.ByteArrayOutputStream;
8-
import java.util.Date;
9-
import java.util.List;
10-
import java.util.Locale;
11-
12-
import com.dropbox.core.util.IOUtil.ProgressListener;
13-
import org.testng.annotations.Test;
14-
153
import com.dropbox.core.BadRequestException;
164
import com.dropbox.core.DbxApiException;
175
import com.dropbox.core.DbxDownloader;
186
import com.dropbox.core.ITUtil;
7+
import com.dropbox.core.util.DropboxContentHasher;
8+
import com.dropbox.core.util.IOUtil.ProgressListener;
199
import com.dropbox.core.v2.fileproperties.PropertyGroup;
20-
import com.dropbox.core.v2.files.FileMetadata;
21-
import com.dropbox.core.v2.files.GetMetadataError;
22-
import com.dropbox.core.v2.files.GetMetadataErrorException;
23-
import com.dropbox.core.v2.files.LookupError;
24-
import com.dropbox.core.v2.files.Metadata;
25-
import com.dropbox.core.v2.files.WriteMode;
10+
import com.dropbox.core.v2.files.*;
2611
import com.dropbox.core.v2.users.BasicAccount;
2712
import com.dropbox.core.v2.users.FullAccount;
13+
import org.testng.annotations.Test;
14+
15+
import java.io.ByteArrayInputStream;
16+
import java.io.ByteArrayOutputStream;
17+
import java.util.Date;
18+
import java.util.List;
19+
import java.util.Locale;
20+
21+
import static com.google.common.truth.Truth.assertThat;
22+
import static com.google.common.truth.Truth.assertWithMessage;
2823

2924
public class DbxClientV2IT {
3025
@Test
@@ -99,6 +94,7 @@ private void testUploadAndDownload(DbxClientV2 client, boolean trackProgress) th
9994
assertThat(metadata.getName()).isEqualTo(filename);
10095
assertThat(metadata.getPathLower()).isEqualTo(path.toLowerCase());
10196
assertThat(metadata.getSize()).isEqualTo(contents.length);
97+
assertThat(metadata.getContentHash()).isEqualTo(DropboxContentHasher.hashHex(contents));
10298

10399
Metadata actual = client.files().getMetadata(path);
104100

@@ -119,6 +115,7 @@ private void testUploadAndDownload(DbxClientV2 client, boolean trackProgress) th
119115
assertFileMetadataEquivalent(actualResult, metadata);
120116
assertThat(actualContents).isEqualTo(contents);
121117
assertThat(downloader.getContentType()).isEqualTo("application/octet-stream");
118+
assertThat(DropboxContentHasher.hashHex(actualContents)).isEqualTo(metadata.getContentHash());
122119
} catch (AssertionError e) {
123120
// so subsequent tests don't fail due to file not being cleaned up
124121
client.files().deleteV2(path).getMetadata();

0 commit comments

Comments
 (0)