Merge branch '2226-defer-marking-messages-and-acks-as-sent' into 'master'

Defer marking messages and acks as sent

Closes #2296

See merge request briar/briar!1635

bramble-api/src/main/java/org/briarproject/bramble/api/db/DatabaseComponent.java

@@ -33,11 +33,18 @@ import java.util.List;
 import java.util.Map;
 
 import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
 
 /**
  * Encapsulates the database implementation and exposes high-level operations
  * to other components.
+ * <p>
+ * With the exception of the {@link #open(SecretKey, MigrationListener)} and
+ * {@link #close()} methods, which must not be called concurrently, the
+ * database can be accessed from any thread. See {@link TransactionManager}
+ * for locking behaviour.
  */
+@ThreadSafe
 @NotNullByDefault
 public interface DatabaseComponent extends TransactionManager {
 
@@ -193,26 +200,15 @@ public interface DatabaseComponent extends TransactionManager {
 			throws DbException;
 
 	/**
-	 * Returns a batch of messages for the given contact, with a total length
-	 * less than or equal to the given length, for transmission over a
-	 * transport with the given maximum latency. Returns null if there are no
-	 * sendable messages that fit in the given length.
+	 * Returns a batch of messages for the given contact, for transmission over
+	 * a transport with the given maximum latency. The total length of the
+	 * messages, including record headers, will be no more than the given
+	 * capacity. Returns null if there are no sendable messages that would fit
+	 * in the given capacity.
 	 */
 	@Nullable
 	Collection<Message> generateBatch(Transaction txn, ContactId c,
-			int maxLength, long maxLatency) throws DbException;
-
-	/**
-	 * Returns a batch of messages for the given contact containing the
-	 * messages with the given IDs, for transmission over a transport with
-	 * the given maximum latency.
-	 * <p/>
-	 * If any of the given messages are not in the database or are not visible
-	 * to the contact, they are omitted from the batch without throwing an
-	 * exception.
-	 */
-	Collection<Message> generateBatch(Transaction txn, ContactId c,
-			Collection<MessageId> ids, long maxLatency) throws DbException;
+			long capacity, long maxLatency) throws DbException;
 
 	/**
 	 * Returns an offer for the given contact for transmission over a
@@ -232,15 +228,16 @@ public interface DatabaseComponent extends TransactionManager {
 			throws DbException;
 
 	/**
-	 * Returns a batch of messages for the given contact, with a total length
-	 * less than or equal to the given length, for transmission over a
-	 * transport with the given maximum latency. Only messages that have been
-	 * requested by the contact are returned. Returns null if there are no
-	 * sendable messages that fit in the given length.
+	 * Returns a batch of messages for the given contact, for transmission over
+	 * a transport with the given maximum latency. Only messages that have been
+	 * requested by the contact are returned. The total length of the messages,
+	 * including record headers, will be no more than the given capacity.
+	 * Returns null if there are no sendable messages that have been requested
+	 * by the contact and would fit in the given capacity.
 	 */
 	@Nullable
 	Collection<Message> generateRequestedBatch(Transaction txn, ContactId c,
-			int maxLength, long maxLatency) throws DbException;
+			long capacity, long maxLatency) throws DbException;
 
 	/**
 	 * Returns the contact with the given ID.
@@ -344,6 +341,30 @@ public interface DatabaseComponent extends TransactionManager {
 	Collection<MessageId> getMessageIds(Transaction txn, GroupId g,
 			Metadata query) throws DbException;
 
+	/**
+	 * Returns the IDs of some messages received from the given contact that
+	 * need to be acknowledged, up to the given number of messages.
+	 * <p/>
+	 * Read-only.
+	 */
+	Collection<MessageId> getMessagesToAck(Transaction txn, ContactId c,
+			int maxMessages) throws DbException;
+
+	/**
+	 * Returns the IDs of some messages that are eligible to be sent to the
+	 * given contact over a transport with the given maximum latency. The total
+	 * length of the messages including record headers will be no more than the
+	 * given capacity.
+	 * <p/>
+	 * Unlike {@link #getUnackedMessagesToSend(Transaction, ContactId)} this
+	 * method does not return messages that have already been sent unless they
+	 * are due for retransmission.
+	 * <p/>
+	 * Read-only.
+	 */
+	Collection<MessageId> getMessagesToSend(Transaction txn, ContactId c,
+			long capacity, long maxLatency) throws DbException;
+
 	/**
 	 * Returns the IDs of any messages that need to be validated.
 	 * <p/>
@@ -460,15 +481,30 @@ public interface DatabaseComponent extends TransactionManager {
 	MessageStatus getMessageStatus(Transaction txn, ContactId c, MessageId m)
 			throws DbException;
 
+	/**
+	 * Returns the message with the given ID for transmission to the given
+	 * contact over a transport with the given maximum latency. Returns null
+	 * if the message is no longer visible to the contact.
+	 *
+	 * @param markAsSent True if the message should be marked as sent.
+	 * If false it can be marked as sent by calling
+	 * {@link #setMessagesSent(Transaction, ContactId, Collection, long)}.
+	 */
+	@Nullable
+	Message getMessageToSend(Transaction txn, ContactId c, MessageId m,
+			long maxLatency, boolean markAsSent) throws DbException;
+
 	/**
 	 * Returns the IDs of all messages that are eligible to be sent to the
-	 * given contact, together with their raw lengths. This may include
-	 * messages that have already been sent and are not yet due for
-	 * retransmission.
+	 * given contact.
+	 * <p>
+	 * Unlike {@link #getMessagesToSend(Transaction, ContactId, long, long)}
+	 * this method may return messages that have already been sent and are
+	 * not yet due for retransmission.
 	 * <p/>
 	 * Read-only.
 	 */
-	Map<MessageId, Integer> getUnackedMessagesToSend(Transaction txn,
+	Collection<MessageId> getUnackedMessagesToSend(Transaction txn,
 			ContactId c) throws DbException;
 
 	/**
@@ -648,6 +684,13 @@ public interface DatabaseComponent extends TransactionManager {
 	void removeTransportKeys(Transaction txn, TransportId t, KeySetId k)
 			throws DbException;
 
+	/**
+	 * Records an ack for the given messages as having been sent to the given
+	 * contact.
+	 */
+	void setAckSent(Transaction txn, ContactId c, Collection<MessageId> acked)
+			throws DbException;
+
 	/**
 	 * Sets the cleanup timer duration for the given message. This does not
 	 * start the message's cleanup timer.
@@ -694,6 +737,13 @@ public interface DatabaseComponent extends TransactionManager {
 	void setMessageState(Transaction txn, MessageId m, MessageState state)
 			throws DbException;
 
+	/**
+	 * Records the given messages as having been sent to the given contact
+	 * over a transport with the given maximum latency.
+	 */
+	void setMessagesSent(Transaction txn, ContactId c,
+			Collection<MessageId> sent, long maxLatency) throws DbException;
+
 	/**
 	 * Adds dependencies for a message
 	 */

bramble-api/src/main/java/org/briarproject/bramble/api/db/DatabaseExecutor.java

@@ -18,6 +18,10 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME;
  * submitted, tasks are not run concurrently, and submitting a task will never
  * block. Tasks must not run indefinitely. Tasks submitted during shutdown are
  * discarded.
+ * <p>
+ * It is not mandatory to use this executor for database tasks. The database
+ * can be accessed from any thread, but this executor's guarantee that tasks
+ * are run in the order they're submitted may be useful in some cases.
  */
 @Qualifier
 @Target({FIELD, METHOD, PARAMETER})

bramble-api/src/main/java/org/briarproject/bramble/api/db/Transaction.java

@@ -45,6 +45,9 @@ public class Transaction {
 	/**
 	 * Attaches an event to be broadcast when the transaction has been
 	 * committed. The event will be broadcast on the {@link EventExecutor}.
+	 * Events and {@link #attach(Runnable) tasks} are submitted to the
+	 * {@link EventExecutor} in the order they were attached to the
+	 * transaction.
 	 */
 	public void attach(Event e) {
 		if (actions == null) actions = new ArrayList<>();
@@ -54,6 +57,9 @@ public class Transaction {
 	/**
 	 * Attaches a task to be executed when the transaction has been
 	 * committed. The task will be run on the {@link EventExecutor}.
+	 * {@link #attach(Event) Events} and tasks are submitted to the
+	 * {@link EventExecutor} in the order they were attached to the
+	 * transaction.
 	 */
 	public void attach(Runnable r) {
 		if (actions == null) actions = new ArrayList<>();

bramble-api/src/main/java/org/briarproject/bramble/api/db/TransactionManager.java

@@ -1,51 +1,95 @@
 package org.briarproject.bramble.api.db;
 
+import org.briarproject.bramble.api.event.EventExecutor;
 import org.briarproject.bramble.api.nullsafety.NotNullByDefault;
 
 import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
 
+/**
+ * An interface for managing database transactions.
+ * <p>
+ * Read-only transactions may access the database concurrently. Read-write
+ * transactions access the database exclusively, so starting a read-only or
+ * read-write transaction will block until there are no read-write
+ * transactions in progress.
+ * <p>
+ * Failing to {@link #endTransaction(Transaction) end} a transaction will
+ * prevent other callers from accessing the database, so it is recommended to
+ * use the {@link #transaction(boolean, DbRunnable)},
+ * {@link #transactionWithResult(boolean, DbCallable)} and
+ * {@link #transactionWithNullableResult(boolean, NullableDbCallable)} methods
+ * where possible, which handle committing or aborting the transaction on the
+ * caller's behalf.
+ * <p>
+ * Transactions are not reentrant, i.e. it is not permitted to start a
+ * transaction on a thread that already has a transaction in progress.
+ */
+@ThreadSafe
 @NotNullByDefault
 public interface TransactionManager {
 
 	/**
-	 * Starts a new transaction and returns an object representing it.
-	 * <p/>
-	 * This method acquires locks, so it must not be called while holding a
-	 * lock.
+	 * Starts a new transaction and returns an object representing it. This
+	 * method acquires the database lock, which is held until
+	 * {@link #endTransaction(Transaction)} is called.
 	 *
-	 * @param readOnly true if the transaction will only be used for reading.
+	 * @param readOnly True if the transaction will only be used for reading,
+	 * in which case the database lock can be shared with other read-only
+	 * transactions.
 	 */
 	Transaction startTransaction(boolean readOnly) throws DbException;
 
 	/**
 	 * Commits a transaction to the database.
+	 * {@link #endTransaction(Transaction)} must be called to release the
+	 * database lock.
 	 */
 	void commitTransaction(Transaction txn) throws DbException;
 
 	/**
-	 * Ends a transaction. If the transaction has not been committed,
-	 * it will be aborted. If the transaction has been committed,
-	 * any events attached to the transaction are broadcast.
-	 * The database lock will be released in either case.
+	 * Ends a transaction. If the transaction has not been committed by
+	 * calling {@link #commitTransaction(Transaction)}, it is aborted and the
+	 * database lock is released.
+	 * <p>
+	 * If the transaction has been committed, any
+	 * {@link Transaction#attach events} attached to the transaction are
+	 * broadcast and any {@link Transaction#attach(Runnable) tasks} attached
+	 * to the transaction are submitted to the {@link EventExecutor}. The
+	 * database lock is then released.
 	 */
 	void endTransaction(Transaction txn);
 
 	/**
-	 * Runs the given task within a transaction.
+	 * Runs the given task within a transaction. The database lock is held
+	 * while running the task.
+	 *
+	 * @param readOnly True if the transaction will only be used for reading,
+	 * in which case the database lock can be shared with other read-only
+	 * transactions.
 	 */
 	<E extends Exception> void transaction(boolean readOnly,
 			DbRunnable<E> task) throws DbException, E;
 
 	/**
 	 * Runs the given task within a transaction and returns the result of the
-	 * task.
+	 * task. The database lock is held while running the task.
+	 *
+	 * @param readOnly True if the transaction will only be used for reading,
+	 * in which case the database lock can be shared with other read-only
+	 * transactions.
 	 */
 	<R, E extends Exception> R transactionWithResult(boolean readOnly,
 			DbCallable<R, E> task) throws DbException, E;
 
 	/**
 	 * Runs the given task within a transaction and returns the result of the
-	 * task, which may be null.
+	 * task, which may be null. The database lock is held while running the
+	 * task.
+	 *
+	 * @param readOnly True if the transaction will only be used for reading,
+	 * in which case the database lock can be shared with other read-only
+	 * transactions.
 	 */
 	@Nullable
 	<R, E extends Exception> R transactionWithNullableResult(boolean readOnly,

bramble-api/src/main/java/org/briarproject/bramble/api/mailbox/MailboxConstants.java

@@ -0,0 +1,23 @@
+package org.briarproject.bramble.api.mailbox;
+
+import static org.briarproject.bramble.api.transport.TransportConstants.MAX_FRAME_LENGTH;
+import static org.briarproject.bramble.api.transport.TransportConstants.MAX_PAYLOAD_LENGTH;
+import static org.briarproject.bramble.api.transport.TransportConstants.STREAM_HEADER_LENGTH;
+import static org.briarproject.bramble.api.transport.TransportConstants.TAG_LENGTH;
+
+public interface MailboxConstants {
+
+	/**
+	 * The maximum length of a file that can be uploaded to or downloaded from
+	 * a mailbox.
+	 */
+	int MAX_FILE_BYTES = 1024 * 1024;
+
+	/**
+	 * The maximum length of the plaintext payload of a file, such that the
+	 * ciphertext is no more than {@link #MAX_FILE_BYTES}.
+	 */
+	int MAX_FILE_PAYLOAD_BYTES =
+			(MAX_FILE_BYTES - TAG_LENGTH - STREAM_HEADER_LENGTH)
+					/ MAX_FRAME_LENGTH * MAX_PAYLOAD_LENGTH;
+}

bramble-api/src/main/java/org/briarproject/bramble/api/record/RecordWriter.java

@@ -12,4 +12,6 @@ public interface RecordWriter {
 	void flush() throws IOException;
 
 	void close() throws IOException;
+
+	long getBytesWritten();
 }

bramble-api/src/main/java/org/briarproject/bramble/api/sync/DeferredSendHandler.java

@@ -0,0 +1,15 @@
+package org.briarproject.bramble.api.sync;
+
+import java.util.Collection;
+
+/**
+ * An interface for holding the IDs of messages sent and acked during an
+ * outgoing {@link SyncSession} so they can be recorded in the DB as sent
+ * or acked at some later time.
+ */
+public interface DeferredSendHandler {
+
+	void onAckSent(Collection<MessageId> acked);
+
+	void onMessageSent(MessageId sent);
+}

bramble-api/src/main/java/org/briarproject/bramble/api/sync/SyncRecordWriter.java

@@ -20,4 +20,6 @@ public interface SyncRecordWriter {
 	void writePriority(Priority p) throws IOException;
 
 	void flush() throws IOException;
+
+	long getBytesWritten();
 }