From bd9ebe75a0aa14d4c7b3e7e4dcaa873bcf20cd4f Mon Sep 17 00:00:00 2001 From: akwizgran Date: Thu, 6 Dec 2018 17:22:34 +0000 Subject: [PATCH] Add initial stream hashing implementation. --- .../bramble/api/sync/MessageFactory.java | 3 + .../bramble/api/sync/tree/StreamHasher.java | 23 ++++++ .../bramble/api/sync/tree/TreeHasher.java | 3 + .../bramble/sync/MessageFactoryImpl.java | 19 ++++- .../bramble/sync/tree/HashTree.java | 16 ++++ .../bramble/sync/tree/HashTreeImpl.java | 46 +++++++++++ .../bramble/sync/tree/StreamHasherImpl.java | 76 +++++++++++++++++++ .../bramble/sync/tree/TreeHasherImpl.java | 4 + 8 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 bramble-api/src/main/java/org/briarproject/bramble/api/sync/tree/StreamHasher.java create mode 100644 bramble-core/src/main/java/org/briarproject/bramble/sync/tree/HashTree.java create mode 100644 bramble-core/src/main/java/org/briarproject/bramble/sync/tree/HashTreeImpl.java create mode 100644 bramble-core/src/main/java/org/briarproject/bramble/sync/tree/StreamHasherImpl.java diff --git a/bramble-api/src/main/java/org/briarproject/bramble/api/sync/MessageFactory.java b/bramble-api/src/main/java/org/briarproject/bramble/api/sync/MessageFactory.java index cab854381..b7891e5cc 100644 --- a/bramble-api/src/main/java/org/briarproject/bramble/api/sync/MessageFactory.java +++ b/bramble-api/src/main/java/org/briarproject/bramble/api/sync/MessageFactory.java @@ -1,6 +1,7 @@ package org.briarproject.bramble.api.sync; import org.briarproject.bramble.api.nullsafety.NotNullByDefault; +import org.briarproject.bramble.api.sync.tree.TreeHash; @NotNullByDefault public interface MessageFactory { @@ -10,4 +11,6 @@ public interface MessageFactory { Message createMessage(byte[] raw); byte[] getRawMessage(Message m); + + MessageId getMessageId(GroupId g, long timestamp, TreeHash rootHash); } diff --git a/bramble-api/src/main/java/org/briarproject/bramble/api/sync/tree/StreamHasher.java b/bramble-api/src/main/java/org/briarproject/bramble/api/sync/tree/StreamHasher.java new file mode 100644 index 000000000..9e72cb4ff --- /dev/null +++ b/bramble-api/src/main/java/org/briarproject/bramble/api/sync/tree/StreamHasher.java @@ -0,0 +1,23 @@ +package org.briarproject.bramble.api.sync.tree; + +import org.briarproject.bramble.api.db.DbException; +import org.briarproject.bramble.api.io.BlockSink; +import org.briarproject.bramble.api.io.HashingId; +import org.briarproject.bramble.api.nullsafety.NotNullByDefault; +import org.briarproject.bramble.api.sync.GroupId; +import org.briarproject.bramble.api.sync.MessageId; + +import java.io.IOException; +import java.io.InputStream; + +@NotNullByDefault +public interface StreamHasher { + + /** + * Reads the given input stream, divides the data into blocks, stores + * the blocks and the resulting hash tree using the given block sink and + * temporary ID, and returns the message ID. + */ + MessageId hash(InputStream in, BlockSink sink, HashingId h, GroupId g, + long timestamp) throws IOException, DbException; +} diff --git a/bramble-api/src/main/java/org/briarproject/bramble/api/sync/tree/TreeHasher.java b/bramble-api/src/main/java/org/briarproject/bramble/api/sync/tree/TreeHasher.java index a52d555b0..e2fae2a34 100644 --- a/bramble-api/src/main/java/org/briarproject/bramble/api/sync/tree/TreeHasher.java +++ b/bramble-api/src/main/java/org/briarproject/bramble/api/sync/tree/TreeHasher.java @@ -1,5 +1,8 @@ package org.briarproject.bramble.api.sync.tree; +import org.briarproject.bramble.api.nullsafety.NotNullByDefault; + +@NotNullByDefault public interface TreeHasher { LeafNode hashBlock(int blockNumber, byte[] data); diff --git a/bramble-core/src/main/java/org/briarproject/bramble/sync/MessageFactoryImpl.java b/bramble-core/src/main/java/org/briarproject/bramble/sync/MessageFactoryImpl.java index f979abab0..3854c80b1 100644 --- a/bramble-core/src/main/java/org/briarproject/bramble/sync/MessageFactoryImpl.java +++ b/bramble-core/src/main/java/org/briarproject/bramble/sync/MessageFactoryImpl.java @@ -7,6 +7,7 @@ import org.briarproject.bramble.api.sync.GroupId; import org.briarproject.bramble.api.sync.Message; import org.briarproject.bramble.api.sync.MessageFactory; import org.briarproject.bramble.api.sync.MessageId; +import org.briarproject.bramble.api.sync.tree.TreeHash; import org.briarproject.bramble.util.ByteUtils; import javax.annotation.concurrent.Immutable; @@ -39,13 +40,19 @@ class MessageFactoryImpl implements MessageFactory { if (body.length == 0) throw new IllegalArgumentException(); if (body.length > MAX_MESSAGE_BODY_LENGTH) throw new IllegalArgumentException(); - MessageId id = getMessageId(g, timestamp, body); + MessageId id = getMessageIdFromBody(g, timestamp, body); return new Message(id, g, timestamp, body); } - private MessageId getMessageId(GroupId g, long timestamp, byte[] body) { + private MessageId getMessageIdFromBody(GroupId g, long timestamp, + byte[] body) { // There's only one block, so the root hash is the hash of the block byte[] rootHash = crypto.hash(BLOCK_LABEL, FORMAT_VERSION_BYTES, body); + return getMessageIdFromRootHash(g, timestamp, rootHash); + } + + private MessageId getMessageIdFromRootHash(GroupId g, long timestamp, + byte[] rootHash) { byte[] timeBytes = new byte[INT_64_BYTES]; ByteUtils.writeUint64(timestamp, timeBytes, 0); byte[] idHash = crypto.hash(ID_LABEL, FORMAT_VERSION_BYTES, @@ -65,7 +72,7 @@ class MessageFactoryImpl implements MessageFactory { long timestamp = ByteUtils.readUint64(raw, UniqueId.LENGTH); byte[] body = new byte[raw.length - MESSAGE_HEADER_LENGTH]; System.arraycopy(raw, MESSAGE_HEADER_LENGTH, body, 0, body.length); - MessageId id = getMessageId(g, timestamp, body); + MessageId id = getMessageIdFromBody(g, timestamp, body); return new Message(id, g, timestamp, body); } @@ -78,4 +85,10 @@ class MessageFactoryImpl implements MessageFactory { System.arraycopy(body, 0, raw, MESSAGE_HEADER_LENGTH, body.length); return raw; } + + @Override + public MessageId getMessageId(GroupId g, long timestamp, + TreeHash rootHash) { + return getMessageIdFromRootHash(g, timestamp, rootHash.getBytes()); + } } diff --git a/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/HashTree.java b/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/HashTree.java new file mode 100644 index 000000000..fe7f60fb5 --- /dev/null +++ b/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/HashTree.java @@ -0,0 +1,16 @@ +package org.briarproject.bramble.sync.tree; + +import org.briarproject.bramble.api.nullsafety.NotNullByDefault; +import org.briarproject.bramble.api.sync.tree.LeafNode; +import org.briarproject.bramble.api.sync.tree.TreeNode; + +import javax.annotation.concurrent.NotThreadSafe; + +@NotThreadSafe +@NotNullByDefault +interface HashTree { + + void addLeaf(LeafNode leaf); + + TreeNode getRoot(); +} diff --git a/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/HashTreeImpl.java b/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/HashTreeImpl.java new file mode 100644 index 000000000..32e5e2570 --- /dev/null +++ b/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/HashTreeImpl.java @@ -0,0 +1,46 @@ +package org.briarproject.bramble.sync.tree; + +import org.briarproject.bramble.api.nullsafety.NotNullByDefault; +import org.briarproject.bramble.api.sync.tree.LeafNode; +import org.briarproject.bramble.api.sync.tree.TreeHasher; +import org.briarproject.bramble.api.sync.tree.TreeNode; + +import java.util.Deque; +import java.util.LinkedList; + +import javax.inject.Inject; + +@NotNullByDefault +class HashTreeImpl implements HashTree { + + private final TreeHasher treeHasher; + private final Deque nodes = new LinkedList<>(); + + @Inject + HashTreeImpl(TreeHasher treeHasher) { + this.treeHasher = treeHasher; + } + + @Override + public void addLeaf(LeafNode leaf) { + TreeNode add = leaf; + int height = leaf.getHeight(); + TreeNode last = nodes.peekLast(); + while (last != null && last.getHeight() == height) { + add = treeHasher.mergeTrees(last, add); + height = add.getHeight(); + nodes.removeLast(); + last = nodes.peekLast(); + } + nodes.addLast(add); + } + + @Override + public TreeNode getRoot() { + TreeNode root = nodes.removeLast(); + while (!nodes.isEmpty()) { + root = treeHasher.mergeTrees(nodes.removeLast(), root); + } + return root; + } +} diff --git a/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/StreamHasherImpl.java b/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/StreamHasherImpl.java new file mode 100644 index 000000000..d30545bee --- /dev/null +++ b/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/StreamHasherImpl.java @@ -0,0 +1,76 @@ +package org.briarproject.bramble.sync.tree; + +import org.briarproject.bramble.api.db.DbException; +import org.briarproject.bramble.api.io.BlockSink; +import org.briarproject.bramble.api.io.HashingId; +import org.briarproject.bramble.api.nullsafety.NotNullByDefault; +import org.briarproject.bramble.api.sync.GroupId; +import org.briarproject.bramble.api.sync.MessageFactory; +import org.briarproject.bramble.api.sync.MessageId; +import org.briarproject.bramble.api.sync.tree.LeafNode; +import org.briarproject.bramble.api.sync.tree.StreamHasher; +import org.briarproject.bramble.api.sync.tree.TreeHash; +import org.briarproject.bramble.api.sync.tree.TreeHasher; + +import java.io.IOException; +import java.io.InputStream; + +import javax.annotation.concurrent.Immutable; +import javax.inject.Inject; +import javax.inject.Provider; + +import static java.util.Arrays.copyOfRange; +import static org.briarproject.bramble.api.sync.SyncConstants.MAX_BLOCK_LENGTH; + +@Immutable +@NotNullByDefault +class StreamHasherImpl implements StreamHasher { + + private final TreeHasher treeHasher; + private final MessageFactory messageFactory; + private final Provider hashTreeProvider; + + @Inject + StreamHasherImpl(TreeHasher treeHasher, MessageFactory messageFactory, + Provider hashTreeProvider) { + this.treeHasher = treeHasher; + this.messageFactory = messageFactory; + this.hashTreeProvider = hashTreeProvider; + } + + @Override + public MessageId hash(InputStream in, BlockSink sink, HashingId h, + GroupId g, long timestamp) throws IOException, DbException { + HashTree hashTree = hashTreeProvider.get(); + byte[] block = new byte[MAX_BLOCK_LENGTH]; + int read; + for (int blockNumber = 0; (read = read(in, block)) > 0; blockNumber++) { + byte[] data; + if (read == block.length) data = block; + else data = copyOfRange(block, 0, read); + sink.putBlock(h, blockNumber, data); + LeafNode leaf = treeHasher.hashBlock(blockNumber, data); + hashTree.addLeaf(leaf); + } + // TODO: Set paths on block sink + TreeHash rootHash = hashTree.getRoot().getHash(); + MessageId m = messageFactory.getMessageId(g, timestamp, rootHash); + sink.setMessageId(h, m); + return m; + } + + /** + * Reads a block from the given input stream and returns the number of + * bytes read, or 0 if no bytes were read before reaching the end of the + * stream. + */ + private int read(InputStream in, byte[] block) throws IOException { + int offset = 0; + while (offset < block.length) { + int read = in.read(block, offset, block.length - offset); + if (read == -1) return offset; + offset += read; + } + return offset; + } +} diff --git a/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/TreeHasherImpl.java b/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/TreeHasherImpl.java index 3d15c2d77..90fcf0f67 100644 --- a/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/TreeHasherImpl.java +++ b/bramble-core/src/main/java/org/briarproject/bramble/sync/tree/TreeHasherImpl.java @@ -1,18 +1,22 @@ package org.briarproject.bramble.sync.tree; import org.briarproject.bramble.api.crypto.CryptoComponent; +import org.briarproject.bramble.api.nullsafety.NotNullByDefault; import org.briarproject.bramble.api.sync.tree.LeafNode; import org.briarproject.bramble.api.sync.tree.ParentNode; import org.briarproject.bramble.api.sync.tree.TreeHash; import org.briarproject.bramble.api.sync.tree.TreeHasher; import org.briarproject.bramble.api.sync.tree.TreeNode; +import javax.annotation.concurrent.Immutable; import javax.inject.Inject; import static org.briarproject.bramble.api.sync.Message.FORMAT_VERSION; import static org.briarproject.bramble.api.sync.MessageId.BLOCK_LABEL; import static org.briarproject.bramble.api.sync.MessageId.TREE_LABEL; +@Immutable +@NotNullByDefault class TreeHasherImpl implements TreeHasher { private static final byte[] FORMAT_VERSION_BYTES =