Separate the sync layer from its clients. #112

briar-core/src/org/briarproject/db/Database.java

@@ -6,14 +6,16 @@ import org.briarproject.api.TransportProperties;
 import org.briarproject.api.contact.Contact;
 import org.briarproject.api.contact.ContactId;
 import org.briarproject.api.db.DbException;
+import org.briarproject.api.db.Metadata;
 import org.briarproject.api.identity.Author;
 import org.briarproject.api.identity.AuthorId;
 import org.briarproject.api.identity.LocalAuthor;
+import org.briarproject.api.sync.ClientId;
 import org.briarproject.api.sync.Group;
 import org.briarproject.api.sync.GroupId;
 import org.briarproject.api.sync.Message;
-import org.briarproject.api.sync.MessageHeader;
 import org.briarproject.api.sync.MessageId;
+import org.briarproject.api.sync.MessageStatus;
 import org.briarproject.api.sync.SubscriptionAck;
 import org.briarproject.api.sync.SubscriptionUpdate;
 import org.briarproject.api.sync.TransportAck;
@@ -85,6 +87,13 @@ interface Database<T> {
 	ContactId addContact(T txn, Author remote, AuthorId local)
 			throws DbException;
 
+	/**
+	 * Adds a group to the given contact's subscriptions.
+	 * <p>
+	 * Locking: write.
+	 */
+	void addGroup(T txn, ContactId c, Group g) throws DbException;
+
 	/**
 	 * Subscribes to a group, or returns false if the user already has the
 	 * maximum number of subscriptions.
@@ -265,29 +274,12 @@ interface Database<T> {
 	Group getGroup(T txn, GroupId g) throws DbException;
 
 	/**
-	 * Returns all groups to which the user subscribes, excluding inboxes.
+	 * Returns all groups to which the user subscribes.
 	 * <p>
 	 * Locking: read.
 	 */
 	Collection<Group> getGroups(T txn) throws DbException;
 
-	/**
-	 * Returns the ID of the inbox group for the given contact, or null if no
-	 * inbox group has been set.
-	 * <p>
-	 * Locking: read.
-	 */
-	GroupId getInboxGroupId(T txn, ContactId c) throws DbException;
-
-	/**
-	 * Returns the headers of all messages in the inbox group for the given
-	 * contact, or null if no inbox group has been set.
-	 * <p>
-	 * Locking: read.
-	 */
-	Collection<MessageHeader> getInboxMessageHeaders(T txn, ContactId c)
-			throws DbException;
-
 	/**
 	 * Returns the local pseudonym with the given ID.
 	 * <p>
@@ -319,19 +311,37 @@ interface Database<T> {
 			throws DbException;
 
 	/**
-	 * Returns the body of the message identified by the given ID.
+	 * Returns the metadata for all messages in the given group.
 	 * <p>
 	 * Locking: read.
 	 */
-	byte[] getMessageBody(T txn, MessageId m) throws DbException;
+	Map<MessageId, Metadata> getMessageMetadata(T txn, GroupId g)
+			throws DbException;
 
 	/**
-	 * Returns the headers of all messages in the given group.
+	 * Returns the metadata for the given message.
 	 * <p>
 	 * Locking: read.
 	 */
-	Collection<MessageHeader> getMessageHeaders(T txn, GroupId g)
-			throws DbException;
+	Metadata getMessageMetadata(T txn, MessageId m) throws DbException;
+
+	/**
+	 * Returns the status of all messages in the given group with respect
+	 * to the given contact.
+	 * <p>
+	 * Locking: read
+	 */
+	Collection<MessageStatus> getMessageStatus(T txn, ContactId c, GroupId g)
+		throws DbException;
+
+	/**
+	 * Returns the status of the given message with respect to the given
+	 * contact.
+	 * <p>
+	 * Locking: read
+	 */
+	MessageStatus getMessageStatus(T txn, ContactId c, MessageId m)
+		throws DbException;
 
 	/**
 	 * Returns the IDs of some messages received from the given contact that
@@ -370,28 +380,21 @@ interface Database<T> {
 			int maxMessages) throws DbException;
 
 	/**
-	 * Returns the parent of the given message, or null if either the message
-	 * has no parent, or the parent is absent from the database, or the parent
-	 * belongs to a different group.
+	 * Returns the IDs of any messages that need to be validated by the given
+	 * client.
 	 * <p>
 	 * Locking: read.
 	 */
-	MessageId getParent(T txn, MessageId m) throws DbException;
+	Collection<MessageId> getMessagesToValidate(T txn, ClientId c)
+		throws DbException;
 
 	/**
-	 * Returns the message identified by the given ID, in serialised form.
+	 * Returns the message with the given ID, in serialised form.
 	 * <p>
 	 * Locking: read.
 	 */
 	byte[] getRawMessage(T txn, MessageId m) throws DbException;
 
-	/**
-	 * Returns true if the given message is marked as read.
-	 * <p>
-	 * Locking: read.
-	 */
-	boolean getReadFlag(T txn, MessageId m) throws DbException;
-
 	/**
 	 * Returns all remote properties for the given transport.
 	 * <p>
@@ -475,13 +478,6 @@ interface Database<T> {
 	Collection<TransportUpdate> getTransportUpdates(T txn, ContactId c,
 			int maxLatency) throws DbException;
 
-	/**
-	 * Returns the number of unread messages in each subscribed group.
-	 * <p>
-	 * Locking: read.
-	 */
-	Map<GroupId, Integer> getUnreadMessageCounts(T txn) throws DbException;
-
 	/**
 	 * Returns the IDs of all contacts to which the given group is visible.
 	 * <p>
@@ -525,6 +521,15 @@ interface Database<T> {
 	void mergeLocalProperties(T txn, TransportId t, TransportProperties p)
 			throws DbException;
 
+	/*
+	 * Merges the given metadata with the existing metadata for the given
+	 * message.
+	 * <p>
+	 * Locking: write.
+	 */
+	void mergeMessageMetadata(T txn, MessageId m, Metadata meta)
+			throws DbException;
+
 	/**
 	 * Merges the given settings with the existing settings in the given
 	 * namespace.
@@ -624,6 +629,10 @@ interface Database<T> {
 	 */
 	void resetExpiryTime(T txn, ContactId c, MessageId m) throws DbException;
 
+	/** Marks the given message as valid or invalid. */
+	void setMessageValidity(T txn, MessageId m, boolean valid)
+			throws DbException;
+
 	/**
 	 * Sets the reordering window for the given contact and transport in the
 	 * given rotation period.
@@ -634,30 +643,15 @@ interface Database<T> {
 			long rotationPeriod, long base, byte[] bitmap) throws DbException;
 
 	/**
-	 * Updates the groups to which the given contact subscribes and returns
-	 * true, unless an update with an equal or higher version number has
-	 * already been received from the contact.
+	 * Updates the given contact's subscriptions and returns true, unless an
+	 * update with an equal or higher version number has already been received
+	 * from the contact.
 	 * <p>
 	 * Locking: write.
 	 */
 	boolean setGroups(T txn, ContactId c, Collection<Group> groups,
 			long version) throws DbException;
 
-	/**
-	 * Makes a group visible to the given contact, adds it to the contact's
-	 * subscriptions, and sets it as the inbox group for the contact.
-	 * <p>
-	 * Locking: write.
-	 */
-	void setInboxGroup(T txn, ContactId c, Group g) throws DbException;
-
-	/**
-	 * Marks a message as read or unread.
-	 * <p>
-	 * Locking: write.
-	 */
-	void setReadFlag(T txn, MessageId m, boolean read) throws DbException;
-
 	/**
 	 * Sets the remote transport properties for the given contact, replacing
 	 * any existing properties.

briar-core/src/org/briarproject/db/DatabaseComponentImpl.java

@@ -9,6 +9,7 @@ import org.briarproject.api.db.ContactExistsException;
 import org.briarproject.api.db.DatabaseComponent;
 import org.briarproject.api.db.DbException;
 import org.briarproject.api.db.LocalAuthorExistsException;
+import org.briarproject.api.db.Metadata;
 import org.briarproject.api.db.NoSuchContactException;
 import org.briarproject.api.db.NoSuchLocalAuthorException;
 import org.briarproject.api.db.NoSuchMessageException;
@@ -25,6 +26,7 @@ import org.briarproject.api.event.MessageAddedEvent;
 import org.briarproject.api.event.MessageRequestedEvent;
 import org.briarproject.api.event.MessageToAckEvent;
 import org.briarproject.api.event.MessageToRequestEvent;
+import org.briarproject.api.event.MessageValidatedEvent;
 import org.briarproject.api.event.MessagesAckedEvent;
 import org.briarproject.api.event.MessagesSentEvent;
 import org.briarproject.api.event.RemoteSubscriptionsUpdatedEvent;
@@ -39,11 +41,12 @@ import org.briarproject.api.identity.AuthorId;
 import org.briarproject.api.identity.LocalAuthor;
 import org.briarproject.api.lifecycle.ShutdownManager;
 import org.briarproject.api.sync.Ack;
+import org.briarproject.api.sync.ClientId;
 import org.briarproject.api.sync.Group;
 import org.briarproject.api.sync.GroupId;
 import org.briarproject.api.sync.Message;
-import org.briarproject.api.sync.MessageHeader;
 import org.briarproject.api.sync.MessageId;
+import org.briarproject.api.sync.MessageStatus;
 import org.briarproject.api.sync.Offer;
 import org.briarproject.api.sync.Request;
 import org.briarproject.api.sync.SubscriptionAck;
@@ -165,6 +168,22 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 		return c;
 	}
 
+	public void addGroup(ContactId c, Group g) throws DbException {
+		lock.writeLock().lock();
+		try {
+			T txn = db.startTransaction();
+			try {
+				db.addGroup(txn, c, g);
+				db.commitTransaction(txn);
+			} catch (DbException e) {
+				db.abortTransaction(txn);
+				throw e;
+			}
+		} finally {
+			lock.writeLock().unlock();
+		}
+	}
+
 	public boolean addGroup(Group g) throws DbException {
 		boolean added = false;
 		lock.writeLock().lock();
@@ -204,15 +223,19 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 		eventBus.broadcast(new LocalAuthorAddedEvent(a.getId()));
 	}
 
-	public void addLocalMessage(Message m) throws DbException {
+	public void addLocalMessage(Message m, ClientId c, Metadata meta)
+			throws DbException {
 		boolean duplicate, subscribed;
 		lock.writeLock().lock();
 		try {
 			T txn = db.startTransaction();
 			try {
 				duplicate = db.containsMessage(txn, m.getId());
-				subscribed = db.containsGroup(txn, m.getGroup().getId());
-				if (!duplicate && subscribed) addMessage(txn, m, null);
+				subscribed = db.containsGroup(txn, m.getGroupId());
+				if (!duplicate && subscribed) {
+					addMessage(txn, m, null);
+					db.mergeMessageMetadata(txn, m.getId(), meta);
+				}
 				db.commitTransaction(txn);
 			} catch (DbException e) {
 				db.abortTransaction(txn);
@@ -223,26 +246,21 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 		}
 		if (!duplicate && subscribed) {
 			eventBus.broadcast(new MessageAddedEvent(m, null));
+			eventBus.broadcast(new MessageValidatedEvent(m, c, true, true));
 		}
 	}
 
 	/**
-	 * Stores a message, initialises its status with respect to each contact,
-	 * and marks it as read if it was locally generated.
+	 * Stores a message and initialises its status with respect to each contact.
 	 * <p>
 	 * Locking: write.
 	 * @param sender null for a locally generated message.
 	 */
 	private void addMessage(T txn, Message m, ContactId sender)
 			throws DbException {
-		if (sender == null) {
-			db.addMessage(txn, m, true);
-			db.setReadFlag(txn, m.getId(), true);
-		} else {
-			db.addMessage(txn, m, false);
-		}
-		Group g = m.getGroup();
-		Collection<ContactId> visibility = db.getVisibility(txn, g.getId());
+		db.addMessage(txn, m, sender == null);
+		GroupId g = m.getGroupId();
+		Collection<ContactId> visibility = db.getVisibility(txn, g);
 		visibility = new HashSet<ContactId>(visibility);
 		for (ContactId c : db.getContactIds(txn)) {
 			if (visibility.contains(c)) {
@@ -595,46 +613,6 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 		}
 	}
 
-	public GroupId getInboxGroupId(ContactId c) throws DbException {
-		lock.readLock().lock();
-		try {
-			T txn = db.startTransaction();
-			try {
-				if (!db.containsContact(txn, c))
-					throw new NoSuchContactException();
-				GroupId inbox = db.getInboxGroupId(txn, c);
-				db.commitTransaction(txn);
-				return inbox;
-			} catch (DbException e) {
-				db.abortTransaction(txn);
-				throw e;
-			}
-		} finally {
-			lock.readLock().unlock();
-		}
-	}
-
-	public Collection<MessageHeader> getInboxMessageHeaders(ContactId c)
-			throws DbException {
-		lock.readLock().lock();
-		try {
-			T txn = db.startTransaction();
-			try {
-				if (!db.containsContact(txn, c))
-					throw new NoSuchContactException();
-				Collection<MessageHeader> headers =
-						db.getInboxMessageHeaders(txn, c);
-				db.commitTransaction(txn);
-				return headers;
-			} catch (DbException e) {
-				db.abortTransaction(txn);
-				throw e;
-			}
-		} finally {
-			lock.readLock().unlock();
-		}
-	}
-
 	public LocalAuthor getLocalAuthor(AuthorId a) throws DbException {
 		lock.readLock().lock();
 		try {
@@ -710,16 +688,15 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 		}
 	}
 
-	public byte[] getMessageBody(MessageId m) throws DbException {
+	public Collection<MessageId> getMessagesToValidate(ClientId c)
+			throws DbException {
 		lock.readLock().lock();
 		try {
 			T txn = db.startTransaction();
 			try {
-				if (!db.containsMessage(txn, m))
-					throw new NoSuchMessageException();
-				byte[] body = db.getMessageBody(txn, m);
+				Collection<MessageId> ids = db.getMessagesToValidate(txn, c);
 				db.commitTransaction(txn);
-				return body;
+				return ids;
 			} catch (DbException e) {
 				db.abortTransaction(txn);
 				throw e;
@@ -729,7 +706,26 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 		}
 	}
 
-	public Collection<MessageHeader> getMessageHeaders(GroupId g)
+	public byte[] getRawMessage(MessageId m) throws DbException {
+		lock.readLock().lock();
+		try {
+			T txn = db.startTransaction();
+			try {
+				if (!db.containsMessage(txn, m))
+					throw new NoSuchMessageException();
+				byte[] raw = db.getRawMessage(txn, m);
+				db.commitTransaction(txn);
+				return raw;
+			} catch (DbException e) {
+				db.abortTransaction(txn);
+				throw e;
+			}
+		} finally {
+			lock.readLock().unlock();
+		}
+	}
+
+	public Map<MessageId, Metadata> getMessageMetadata(GroupId g)
 			throws DbException {
 		lock.readLock().lock();
 		try {
@@ -737,10 +733,10 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 			try {
 				if (!db.containsGroup(txn, g))
 					throw new NoSuchSubscriptionException();
-				Collection<MessageHeader> headers =
-						db.getMessageHeaders(txn, g);
+				Map<MessageId, Metadata> metadata =
+						db.getMessageMetadata(txn, g);
 				db.commitTransaction(txn);
-				return headers;
+				return metadata;
 			} catch (DbException e) {
 				db.abortTransaction(txn);
 				throw e;
@@ -750,16 +746,61 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 		}
 	}
 
-	public boolean getReadFlag(MessageId m) throws DbException {
+	public Metadata getMessageMetadata(MessageId m) throws DbException {
 		lock.readLock().lock();
 		try {
 			T txn = db.startTransaction();
 			try {
 				if (!db.containsMessage(txn, m))
 					throw new NoSuchMessageException();
-				boolean read = db.getReadFlag(txn, m);
+				Metadata metadata = db.getMessageMetadata(txn, m);
 				db.commitTransaction(txn);
-				return read;
+				return metadata;
+			} catch (DbException e) {
+				db.abortTransaction(txn);
+				throw e;
+			}
+		} finally {
+			lock.readLock().unlock();
+		}
+	}
+
+	public Collection<MessageStatus> getMessageStatus(ContactId c, GroupId g)
+			throws DbException {
+		lock.readLock().lock();
+		try {
+			T txn = db.startTransaction();
+			try {
+				if (!db.containsContact(txn, c))
+					throw new NoSuchContactException();
+				if (!db.containsGroup(txn, g))
+					throw new NoSuchSubscriptionException();
+				Collection<MessageStatus> statuses =
+						db.getMessageStatus(txn, c, g);
+				db.commitTransaction(txn);
+				return statuses;
+			} catch (DbException e) {
+				db.abortTransaction(txn);
+				throw e;
+			}
+		} finally {
+			lock.readLock().unlock();
+		}
+	}
+
+	public MessageStatus getMessageStatus(ContactId c, MessageId m)
+			throws DbException {
+		lock.readLock().lock();
+		try {
+			T txn = db.startTransaction();
+			try {
+				if (!db.containsContact(txn, c))
+					throw new NoSuchContactException();
+				if (!db.containsMessage(txn, m))
+					throw new NoSuchMessageException();
+				MessageStatus status = db.getMessageStatus(txn, c, m);
+				db.commitTransaction(txn);
+				return status;
 			} catch (DbException e) {
 				db.abortTransaction(txn);
 				throw e;
@@ -862,23 +903,6 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 		}
 	}
 
-	public Map<GroupId, Integer> getUnreadMessageCounts() throws DbException {
-		lock.readLock().lock();
-		try {
-			T txn = db.startTransaction();
-			try {
-				Map<GroupId, Integer> counts = db.getUnreadMessageCounts(txn);
-				db.commitTransaction(txn);
-				return counts;
-			} catch (DbException e) {
-				db.abortTransaction(txn);
-				throw e;
-			}
-		} finally {
-			lock.readLock().unlock();
-		}
-	}
-
 	public Collection<ContactId> getVisibility(GroupId g) throws DbException {
 		lock.readLock().lock();
 		try {
@@ -943,6 +967,25 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 		if (changed) eventBus.broadcast(new LocalTransportsUpdatedEvent());
 	}
 
+	public void mergeMessageMetadata(MessageId m, Metadata meta)
+			throws DbException {
+		lock.writeLock().lock();
+		try {
+			T txn = db.startTransaction();
+			try {
+				if (!db.containsMessage(txn, m))
+					throw new NoSuchMessageException();
+				db.mergeMessageMetadata(txn, m, meta);
+				db.commitTransaction(txn);
+			} catch (DbException e) {
+				db.abortTransaction(txn);
+				throw e;
+			}
+		} finally {
+			lock.writeLock().unlock();
+		}
+	}
+
 	public void mergeSettings(Settings s, String namespace) throws DbException {
 		boolean changed = false;
 		lock.writeLock().lock();
@@ -998,7 +1041,7 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 				if (!db.containsContact(txn, c))
 					throw new NoSuchContactException();
 				duplicate = db.containsMessage(txn, m.getId());
-				visible = db.containsVisibleGroup(txn, c, m.getGroup().getId());
+				visible = db.containsVisibleGroup(txn, c, m.getGroupId());
 				if (visible) {
 					if (!duplicate) addMessage(txn, m, c);
 					db.raiseAckFlag(txn, c, m.getId());
@@ -1012,9 +1055,8 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 			lock.writeLock().unlock();
 		}
 		if (visible) {
-			if (!duplicate) {
+			if (!duplicate)
 				eventBus.broadcast(new MessageAddedEvent(m, c));
-			}
 			eventBus.broadcast(new MessageToAckEvent(c));
 		}
 	}
@@ -1170,8 +1212,6 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 			try {
 				if (!db.containsContact(txn, c))
 					throw new NoSuchContactException();
-				GroupId g = db.getInboxGroupId(txn, c);
-				if (g != null) db.removeGroup(txn, g);
 				db.removeContact(txn, c);
 				db.commitTransaction(txn);
 			} catch (DbException e) {
@@ -1216,10 +1256,6 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 				if (!db.containsLocalAuthor(txn, a))
 					throw new NoSuchLocalAuthorException();
 				affected = db.getContacts(txn, a);
-				for (ContactId c : affected) {
-					GroupId g = db.getInboxGroupId(txn, c);
-					if (g != null) db.removeGroup(txn, g);
-				}
 				db.removeLocalAuthor(txn, a);
 				db.commitTransaction(txn);
 			} catch (DbException e) {
@@ -1253,32 +1289,15 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 		eventBus.broadcast(new TransportRemovedEvent(t));
 	}
 
-	public void setInboxGroup(ContactId c, Group g) throws DbException {
+	public void setMessageValidity(Message m, ClientId c, boolean valid)
+			throws DbException {
 		lock.writeLock().lock();
 		try {
 			T txn = db.startTransaction();
 			try {
-				if (!db.containsContact(txn, c))
-					throw new NoSuchContactException();
-				db.setInboxGroup(txn, c, g);
-				db.commitTransaction(txn);
-			} catch (DbException e) {
-				db.abortTransaction(txn);
-				throw e;
-			}
-		} finally {
-			lock.writeLock().unlock();
-		}
-	}
-
-	public void setReadFlag(MessageId m, boolean read) throws DbException {
-		lock.writeLock().lock();
-		try {
-			T txn = db.startTransaction();
-			try {
-				if (!db.containsMessage(txn, m))
+				if (!db.containsMessage(txn, m.getId()))
 					throw new NoSuchMessageException();
-				db.setReadFlag(txn, m, read);
+				db.setMessageValidity(txn, m.getId(), valid);
 				db.commitTransaction(txn);
 			} catch (DbException e) {
 				db.abortTransaction(txn);
@@ -1287,6 +1306,7 @@ class DatabaseComponentImpl<T> implements DatabaseComponent {
 		} finally {
 			lock.writeLock().unlock();
 		}
+		eventBus.broadcast(new MessageValidatedEvent(m, c, false, valid));
 	}
 
 	public void setRemoteProperties(ContactId c,

briar-core/src/org/briarproject/db/DatabaseConstants.java

@@ -8,34 +8,4 @@ interface DatabaseConstants {
 	 * limit is reached, additional offers will not be stored.
 	 */
 	int MAX_OFFERED_MESSAGES = 1000;
-
-	// FIXME: These should be configurable
-
-	/**
-	 * The minimum amount of space in bytes that should be kept free on the
-	 * device where the database is stored. Whenever less than this much space
-	 * is free, old messages will be expired from the database.
-	 */
-	long MIN_FREE_SPACE = 50 * 1024 * 1024; // 50 MiB
-
-	/**
-	 * The minimum amount of space in bytes that must be kept free on the device
-	 * where the database is stored. If less than this much space is free and
-	 * there are no more messages to expire, an Error will be thrown.
-	 */
-	long CRITICAL_FREE_SPACE = 10 * 1024 * 1024; // 10 MiB
-
-	/**
-	 * The amount of free space will be checked whenever this many transactions
-	 * have been started since the last check.
-	 * <p>
-	 * FIXME: Increase this after implementing BTPv2 (smaller packets)?
-	 */
-	int MAX_TRANSACTIONS_BETWEEN_SPACE_CHECKS = 10;
-
-	/**
-	 * Up to this many bytes of messages will be expired from the database each
-	 * time it is necessary to expire messages.
-	 */
-	int BYTES_PER_SWEEP = 10 * 1024 * 1024; // 10 MiB
 }

briar-core/src/org/briarproject/db/JdbcDatabase.java

@@ -8,15 +8,16 @@ import org.briarproject.api.contact.ContactId;
 import org.briarproject.api.crypto.SecretKey;
 import org.briarproject.api.db.DbClosedException;
 import org.briarproject.api.db.DbException;
+import org.briarproject.api.db.Metadata;
 import org.briarproject.api.identity.Author;
 import org.briarproject.api.identity.AuthorId;
 import org.briarproject.api.identity.LocalAuthor;
+import org.briarproject.api.sync.ClientId;
 import org.briarproject.api.sync.Group;
 import org.briarproject.api.sync.GroupId;
 import org.briarproject.api.sync.Message;
-import org.briarproject.api.sync.MessageHeader;
-import org.briarproject.api.sync.MessageHeader.State;
 import org.briarproject.api.sync.MessageId;
+import org.briarproject.api.sync.MessageStatus;
 import org.briarproject.api.sync.SubscriptionAck;
 import org.briarproject.api.sync.SubscriptionUpdate;
 import org.briarproject.api.sync.TransportAck;
@@ -48,13 +49,9 @@ import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.logging.Logger;
 
-import static java.sql.Types.BINARY;
-import static java.sql.Types.VARCHAR;
 import static java.util.logging.Level.WARNING;
-import static org.briarproject.api.identity.Author.Status.ANONYMOUS;
-import static org.briarproject.api.identity.Author.Status.UNKNOWN;
-import static org.briarproject.api.identity.Author.Status.VERIFIED;
-import static org.briarproject.api.sync.MessagingConstants.MAX_SUBSCRIPTIONS;
+import static org.briarproject.api.db.Metadata.REMOVE;
+import static org.briarproject.api.sync.SyncConstants.MAX_SUBSCRIPTIONS;
 import static org.briarproject.db.ExponentialBackoff.calculateExpiry;
 
 /**
@@ -63,8 +60,12 @@ import static org.briarproject.db.ExponentialBackoff.calculateExpiry;
  */
 abstract class JdbcDatabase implements Database<Connection> {
 
-	private static final int SCHEMA_VERSION = 12;
-	private static final int MIN_SCHEMA_VERSION = 12;
+	private static final int SCHEMA_VERSION = 14;
+	private static final int MIN_SCHEMA_VERSION = 14;
+
+	private static final int VALIDATION_UNKNOWN = 0;
+	private static final int VALIDATION_INVALID = 1;
+	private static final int VALIDATION_VALID = 2;
 
 	private static final String CREATE_SETTINGS =
 			"CREATE TABLE settings"
@@ -98,8 +99,8 @@ abstract class JdbcDatabase implements Database<Connection> {
 	private static final String CREATE_GROUPS =
 			"CREATE TABLE groups"
 					+ " (groupId HASH NOT NULL,"
-					+ " name VARCHAR NOT NULL,"
-					+ " salt BINARY NOT NULL,"
+					+ " clientId HASH NOT NULL,"
+					+ " descriptor BINARY NOT NULL,"
 					+ " visibleToAll BOOLEAN NOT NULL,"
 					+ " PRIMARY KEY (groupId))";
 
@@ -107,7 +108,6 @@ abstract class JdbcDatabase implements Database<Connection> {
 			"CREATE TABLE groupVisibilities"
 					+ " (contactId INT NOT NULL,"
 					+ " groupId HASH NOT NULL,"
-					+ " inbox BOOLEAN NOT NULL,"
 					+ " PRIMARY KEY (contactId, groupId),"
 					+ " FOREIGN KEY (contactId)"
 					+ " REFERENCES contacts (contactId)"
@@ -120,8 +120,8 @@ abstract class JdbcDatabase implements Database<Connection> {
 			"CREATE TABLE contactGroups"
 					+ " (contactId INT NOT NULL,"
 					+ " groupId HASH NOT NULL," // Not a foreign key
-					+ " name VARCHAR NOT NULL,"
-					+ " salt BINARY NOT NULL,"
+					+ " clientId HASH NOT NULL,"
+					+ " descriptor BINARY NOT NULL,"
 					+ " PRIMARY KEY (contactId, groupId),"
 					+ " FOREIGN KEY (contactId)"
 					+ " REFERENCES contacts (contactId)"
@@ -144,26 +144,26 @@ abstract class JdbcDatabase implements Database<Connection> {
 	private static final String CREATE_MESSAGES =
 			"CREATE TABLE messages"
 					+ " (messageId HASH NOT NULL,"
-					+ " parentId HASH," // Null for the first msg in a thread
 					+ " groupId HASH NOT NULL,"
-					+ " authorId HASH," // Null for private/anon messages
-					+ " authorName VARCHAR," // Null for private/anon messages
-					+ " authorKey VARCHAR," // Null for private/anon messages
-					+ " contentType VARCHAR NOT NULL,"
 					+ " timestamp BIGINT NOT NULL,"
-					+ " length INT NOT NULL,"
-					+ " bodyStart INT NOT NULL,"
-					+ " bodyLength INT NOT NULL,"
-					+ " raw BLOB NOT NULL,"
 					+ " local BOOLEAN NOT NULL,"
-					+ " read BOOLEAN NOT NULL,"
+					+ " valid INT NOT NULL,"
+					+ " length INT NOT NULL,"
+					+ " raw BLOB NOT NULL,"
 					+ " PRIMARY KEY (messageId),"
 					+ " FOREIGN KEY (groupId)"
 					+ " REFERENCES groups (groupId)"
 					+ " ON DELETE CASCADE)";
 
-	private static final String INDEX_MESSAGES_BY_TIMESTAMP =
-			"CREATE INDEX messagesByTimestamp ON messages (timestamp)";
+	private static final String CREATE_MESSAGE_METADATA =
+			"CREATE TABLE messageMetadata"
+					+ " (messageId HASH NOT NULL,"
+					+ " key VARCHAR NOT NULL,"
+					+ " value BINARY NOT NULL,"
+					+ " PRIMARY KEY (messageId, key),"
+					+ " FOREIGN KEY (messageId)"
+					+ " REFERENCES messages (messageId)"
+					+ " ON DELETE CASCADE)";
 
 	private static final String CREATE_OFFERS =
 			"CREATE TABLE offers"
@@ -191,12 +191,6 @@ abstract class JdbcDatabase implements Database<Connection> {
 					+ " REFERENCES contacts (contactId)"
 					+ " ON DELETE CASCADE)";
 
-	private static final String INDEX_STATUSES_BY_MESSAGE =
-			"CREATE INDEX statusesByMessage ON statuses (messageId)";
-
-	private static final String INDEX_STATUSES_BY_CONTACT =
-			"CREATE INDEX statusesByContact ON statuses (contactId)";
-
 	private static final String CREATE_TRANSPORTS =
 			"CREATE TABLE transports"
 					+ " (transportId VARCHAR NOT NULL,"
@@ -393,11 +387,9 @@ abstract class JdbcDatabase implements Database<Connection> {
 			s.executeUpdate(insertTypeNames(CREATE_CONTACT_GROUPS));
 			s.executeUpdate(insertTypeNames(CREATE_GROUP_VERSIONS));
 			s.executeUpdate(insertTypeNames(CREATE_MESSAGES));
-			s.executeUpdate(INDEX_MESSAGES_BY_TIMESTAMP);
+			s.executeUpdate(insertTypeNames(CREATE_MESSAGE_METADATA));
 			s.executeUpdate(insertTypeNames(CREATE_OFFERS));
 			s.executeUpdate(insertTypeNames(CREATE_STATUSES));
-			s.executeUpdate(INDEX_STATUSES_BY_MESSAGE);
-			s.executeUpdate(INDEX_STATUSES_BY_CONTACT);
 			s.executeUpdate(insertTypeNames(CREATE_TRANSPORTS));
 			s.executeUpdate(insertTypeNames(CREATE_TRANSPORT_CONFIGS));
 			s.executeUpdate(insertTypeNames(CREATE_TRANSPORT_PROPS));
@@ -596,9 +588,8 @@ abstract class JdbcDatabase implements Database<Connection> {
 			rs.close();
 			ps.close();
 			if (!ids.isEmpty()) {
-				sql = "INSERT INTO groupVisibilities"
-						+ " (contactId, groupId, inbox)"
-						+ " VALUES (?, ?, FALSE)";
+				sql = "INSERT INTO groupVisibilities (contactId, groupId)"
+						+ " VALUES (?, ?)";
 				ps = txn.prepareStatement(sql);
 				ps.setInt(1, c.getInt());
 				for (byte[] id : ids) {
@@ -656,6 +647,40 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
+	public void addGroup(Connection txn, ContactId c, Group g)
+			throws DbException {
+		PreparedStatement ps = null;
+		ResultSet rs = null;
+		try {
+			String sql = "SELECT NULL FROM contactGroups"
+					+ " WHERE contactId = ? AND groupId = ?";
+			ps = txn.prepareStatement(sql);
+			ps.setInt(1, c.getInt());
+			ps.setBytes(2, g.getId().getBytes());
+			rs = ps.executeQuery();
+			boolean found = rs.next();
+			if (rs.next()) throw new DbStateException();
+			rs.close();
+			ps.close();
+			if (found) return;
+			sql = "INSERT INTO contactGroups"
+					+ " (contactId, groupId, clientId, descriptor)"
+					+ " VALUES (?, ?, ?, ?)";
+			ps = txn.prepareStatement(sql);
+			ps.setInt(1, c.getInt());
+			ps.setBytes(2, g.getId().getBytes());
+			ps.setBytes(3, g.getClientId().getBytes());
+			ps.setBytes(4, g.getDescriptor());
+			int affected = ps.executeUpdate();
+			if (affected != 1) throw new DbStateException();
+			ps.close();
+		} catch (SQLException e) {
+			tryToClose(rs);
+			tryToClose(ps);
+			throw new DbException(e);
+		}
+	}
+
 	public boolean addGroup(Connection txn, Group g) throws DbException {
 		PreparedStatement ps = null;
 		ResultSet rs = null;
@@ -671,12 +696,12 @@ abstract class JdbcDatabase implements Database<Connection> {
 			if (count > MAX_SUBSCRIPTIONS) throw new DbStateException();
 			if (count == MAX_SUBSCRIPTIONS) return false;
 			sql = "INSERT INTO groups"
-					+ " (groupId, name, salt, visibleToAll)"
+					+ " (groupId, clientId, descriptor, visibleToAll)"
 					+ " VALUES (?, ?, ?, FALSE)";
 			ps = txn.prepareStatement(sql);
 			ps.setBytes(1, g.getId().getBytes());
-			ps.setString(2, g.getName());
-			ps.setBytes(3, g.getSalt());
+			ps.setBytes(2, g.getClientId().getBytes());
+			ps.setBytes(3, g.getDescriptor());
 			int affected = ps.executeUpdate();
 			if (affected != 1) throw new DbStateException();
 			ps.close();
@@ -714,34 +739,18 @@ abstract class JdbcDatabase implements Database<Connection> {
 			throws DbException {
 		PreparedStatement ps = null;
 		try {
-			String sql = "INSERT INTO messages (messageId, parentId, groupId,"
-					+ " authorId, authorName, authorKey, contentType,"
-					+ " timestamp, length, bodyStart, bodyLength, raw,"
-					+ " local, read)"
-					+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, FALSE)";
+			String sql = "INSERT INTO messages (messageId, groupId, timestamp,"
+					+ " local, valid, length, raw)"
+					+ " VALUES (?, ?, ?, ?, ?, ?, ?)";
 			ps = txn.prepareStatement(sql);
 			ps.setBytes(1, m.getId().getBytes());
-			if (m.getParent() == null) ps.setNull(2, BINARY);
-			else ps.setBytes(2, m.getParent().getBytes());
-			ps.setBytes(3, m.getGroup().getId().getBytes());
-			Author a = m.getAuthor();
-			if (a == null) {
-				ps.setNull(4, BINARY);
-				ps.setNull(5, VARCHAR);
-				ps.setNull(6, BINARY);
-			} else {
-				ps.setBytes(4, a.getId().getBytes());
-				ps.setString(5, a.getName());
-				ps.setBytes(6, a.getPublicKey());
-			}
-			ps.setString(7, m.getContentType());
-			ps.setLong(8, m.getTimestamp());
-			byte[] raw = m.getSerialised();
-			ps.setInt(9, raw.length);
-			ps.setInt(10, m.getBodyStart());
-			ps.setInt(11, m.getBodyLength());
-			ps.setBytes(12, raw);
-			ps.setBoolean(13, local);
+			ps.setBytes(2, m.getGroupId().getBytes());
+			ps.setLong(3, m.getTimestamp());
+			ps.setBoolean(4, local);
+			ps.setInt(5, local ? VALIDATION_VALID : VALIDATION_UNKNOWN);
+			byte[] raw = m.getRaw();
+			ps.setInt(6, raw.length);
+			ps.setBytes(7, raw);
 			int affected = ps.executeUpdate();
 			if (affected != 1) throw new DbStateException();
 			ps.close();
@@ -924,9 +933,8 @@ abstract class JdbcDatabase implements Database<Connection> {
 			throws DbException {
 		PreparedStatement ps = null;
 		try {
-			String sql = "INSERT INTO groupVisibilities"
-					+ " (contactId, groupId, inbox)"
-					+ " VALUES (?, ?, FALSE)";
+			String sql = "INSERT INTO groupVisibilities (contactId, groupId)"
+					+ " VALUES (?, ?)";
 			ps = txn.prepareStatement(sql);
 			ps.setInt(1, c.getInt());
 			ps.setBytes(2, g.getBytes());
@@ -1152,7 +1160,8 @@ abstract class JdbcDatabase implements Database<Connection> {
 		PreparedStatement ps = null;
 		ResultSet rs = null;
 		try {
-			String sql = "SELECT DISTINCT cg.groupId, cg.name, cg.salt"
+			String sql = "SELECT DISTINCT"
+					+ " cg.groupId, cg.clientId, cg.descriptor"
 					+ " FROM contactGroups AS cg"
 					+ " LEFT OUTER JOIN groups AS g"
 					+ " ON cg.groupId = g.groupId"
@@ -1165,9 +1174,9 @@ abstract class JdbcDatabase implements Database<Connection> {
 			while (rs.next()) {
 				GroupId id = new GroupId(rs.getBytes(1));
 				if (!ids.add(id)) throw new DbStateException();
-				String name = rs.getString(2);
-				byte[] salt = rs.getBytes(3);
-				groups.add(new Group(id, name, salt));
+				ClientId clientId = new ClientId(rs.getBytes(2));
+				byte[] descriptor = rs.getBytes(3);
+				groups.add(new Group(id, clientId, descriptor));
 			}
 			rs.close();
 			ps.close();
@@ -1281,16 +1290,17 @@ abstract class JdbcDatabase implements Database<Connection> {
 		PreparedStatement ps = null;
 		ResultSet rs = null;
 		try {
-			String sql = "SELECT name, salt FROM groups WHERE groupId = ?";
+			String sql = "SELECT clientId, descriptor FROM groups"
+					+ " WHERE groupId = ?";
 			ps = txn.prepareStatement(sql);
 			ps.setBytes(1, g.getBytes());
 			rs = ps.executeQuery();
 			if (!rs.next()) throw new DbStateException();
-			String name = rs.getString(1);
-			byte[] salt = rs.getBytes(2);
+			ClientId clientId = new ClientId(rs.getBytes(1));
+			byte[] descriptor = rs.getBytes(2);
 			rs.close();
 			ps.close();
-			return new Group(g, name, salt);
+			return new Group(g, clientId, descriptor);
 		} catch (SQLException e) {
 			tryToClose(rs);
 			tryToClose(ps);
@@ -1302,20 +1312,15 @@ abstract class JdbcDatabase implements Database<Connection> {
 		PreparedStatement ps = null;
 		ResultSet rs = null;
 		try {
-			String sql = "SELECT DISTINCT g.groupId, name, salt"
-					+ " FROM groups AS g"
-					+ " LEFT OUTER JOIN groupVisibilities AS gv"
-					+ " ON g.groupId = gv.groupId"
-					+ " WHERE gv.inbox IS NULL OR gv.inbox = FALSE"
-					+ " GROUP BY g.groupId";
+			String sql = "SELECT groupId, clientId, descriptor FROM groups";
 			ps = txn.prepareStatement(sql);
 			rs = ps.executeQuery();
 			List<Group> groups = new ArrayList<Group>();
 			while (rs.next()) {
 				GroupId id = new GroupId(rs.getBytes(1));
-				String name = rs.getString(2);
-				byte[] salt = rs.getBytes(3);
-				groups.add(new Group(id, name, salt));
+				ClientId clientId = new ClientId(rs.getBytes(2));
+				byte[] descriptor = rs.getBytes(3);
+				groups.add(new Group(id, clientId, descriptor));
 			}
 			rs.close();
 			ps.close();
@@ -1327,103 +1332,6 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
-	public GroupId getInboxGroupId(Connection txn, ContactId c)
-			throws DbException {
-		PreparedStatement ps = null;
-		ResultSet rs = null;
-		try {
-			String sql = "SELECT groupId FROM groupVisibilities"
-					+ " WHERE contactId = ?"
-					+ " AND inbox = TRUE";
-			ps = txn.prepareStatement(sql);
-			ps.setInt(1, c.getInt());
-			rs = ps.executeQuery();
-			GroupId inbox = null;
-			if (rs.next()) inbox = new GroupId(rs.getBytes(1));
-			if (rs.next()) throw new DbStateException();
-			rs.close();
-			ps.close();
-			return inbox;
-		} catch (SQLException e) {
-			tryToClose(rs);
-			tryToClose(ps);
-			throw new DbException(e);
-		}
-	}
-
-	public Collection<MessageHeader> getInboxMessageHeaders(Connection txn,
-			ContactId c) throws DbException {
-		PreparedStatement ps = null;
-		ResultSet rs = null;
-		try {
-			// Get the local and remote authors
-			String sql = "SELECT la.authorId, la.name, la.publicKey,"
-					+ " c.authorId, c.name, c.publicKey"
-					+ " FROM localAuthors AS la"
-					+ " JOIN contacts AS c"
-					+ " ON la.authorId = c.localAuthorId"
-					+ " WHERE contactId = ?";
-			ps = txn.prepareStatement(sql);
-			ps.setInt(1, c.getInt());
-			rs = ps.executeQuery();
-			if (!rs.next()) throw new DbException();
-			AuthorId localId = new AuthorId(rs.getBytes(1));
-			String localName = rs.getString(2);
-			byte[] localKey = rs.getBytes(3);
-			Author localAuthor = new Author(localId, localName, localKey);
-			AuthorId remoteId = new AuthorId(rs.getBytes(4));
-			String remoteName = rs.getString(5);
-			byte[] remoteKey = rs.getBytes(6);
-			Author remoteAuthor = new Author(remoteId, remoteName, remoteKey);
-			if (rs.next()) throw new DbException();
-			// Get the message headers
-			sql = "SELECT m.messageId, parentId, m.groupId, contentType,"
-					+ " timestamp, local, read, seen, s.txCount"
-					+ " FROM messages AS m"
-					+ " JOIN groups AS g"
-					+ " ON m.groupId = g.groupId"
-					+ " JOIN groupVisibilities AS gv"
-					+ " ON m.groupId = gv.groupId"
-					+ " JOIN statuses AS s"
-					+ " ON m.messageId = s.messageId"
-					+ " AND gv.contactId = s.contactId"
-					+ " WHERE gv.contactId = ?"
-					+ " AND inbox = TRUE";
-			ps = txn.prepareStatement(sql);
-			ps.setInt(1, c.getInt());
-			rs = ps.executeQuery();
-			List<MessageHeader> headers = new ArrayList<MessageHeader>();
-			while (rs.next()) {
-				MessageId id = new MessageId(rs.getBytes(1));
-				byte[] b = rs.getBytes(2);
-				MessageId parent = b == null ? null : new MessageId(b);
-				GroupId groupId = new GroupId(rs.getBytes(3));
-				String contentType = rs.getString(4);
-				long timestamp = rs.getLong(5);
-				boolean local = rs.getBoolean(6);
-				boolean read = rs.getBoolean(7);
-				boolean seen = rs.getBoolean(8);
-				Author author = local ? localAuthor : remoteAuthor;
-
-				// initialize message status
-				State status;
-				if (seen) status = State.DELIVERED;
-				else if (rs.getInt(9) > 0) status = State.SENT;
-				else status = State.STORED;
-
-				headers.add(new MessageHeader(id, parent, groupId, author,
-						VERIFIED, contentType, timestamp, local, read, status));
-			}
-			rs.close();
-			ps.close();
-			return Collections.unmodifiableList(headers);
-		} catch (SQLException e) {
-			tryToClose(rs);
-			tryToClose(ps);
-			throw new DbException(e);
-		}
-	}
-
 	public LocalAuthor getLocalAuthor(Connection txn, AuthorId a)
 			throws DbException {
 		PreparedStatement ps = null;
@@ -1538,25 +1446,35 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
-	public byte[] getMessageBody(Connection txn, MessageId m)
-			throws DbException {
+	public Map<MessageId, Metadata> getMessageMetadata(Connection txn,
+			GroupId g) throws DbException {
 		PreparedStatement ps = null;
 		ResultSet rs = null;
 		try {
-			String sql = "SELECT bodyStart, bodyLength, raw FROM messages"
-					+ " WHERE messageId = ?";
+			String sql = "SELECT m.messageId, key, value"
+					+ " FROM messages AS m"
+					+ " JOIN messageMetadata AS md"
+					+ " ON m.messageId = md.messageId"
+					+ " WHERE groupId = ?"
+					+ " ORDER BY m.messageId";
 			ps = txn.prepareStatement(sql);
-			ps.setBytes(1, m.getBytes());
+			ps.setBytes(1, g.getBytes());
 			rs = ps.executeQuery();
-			if (!rs.next()) throw new DbStateException();
-			int bodyStart = rs.getInt(1);
-			int bodyLength = rs.getInt(2);
-			// Bytes are indexed from 1 rather than 0
-			byte[] body = rs.getBlob(3).getBytes(bodyStart + 1, bodyLength);
-			if (rs.next()) throw new DbStateException();
+			Map<MessageId, Metadata> all = new HashMap<MessageId, Metadata>();
+			Metadata metadata = null;
+			MessageId lastMessageId = null;
+			while (rs.next()) {
+				MessageId messageId = new MessageId(rs.getBytes(1));
+				if (!messageId.equals(lastMessageId)) {
+					metadata = new Metadata();
+					all.put(messageId, metadata);
+					lastMessageId = messageId;
+				}
+				metadata.put(rs.getString(2), rs.getBytes(3));
+			}
 			rs.close();
 			ps.close();
-			return body;
+			return Collections.unmodifiableMap(all);
 		} catch (SQLException e) {
 			tryToClose(rs);
 			tryToClose(ps);
@@ -1564,60 +1482,81 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
-	/**
-	 * This method is used to get group messages.
-	 * The message status won't be used.
-	 */
-	public Collection<MessageHeader> getMessageHeaders(Connection txn,
-			GroupId g) throws DbException {
+	public Metadata getMessageMetadata(Connection txn, MessageId m)
+			throws DbException {
 		PreparedStatement ps = null;
 		ResultSet rs = null;
 		try {
-			String sql = "SELECT messageId, parentId, m.authorId, authorName,"
-					+ " authorKey, contentType, timestamp, local, read,"
-					+ " la.authorId IS NOT NULL, c.authorId IS NOT NULL"
+			String sql = "SELECT key, value"
+					+ " FROM messageMetadata"
+					+ " WHERE messageId = ?";
+			ps = txn.prepareStatement(sql);
+			ps.setBytes(1, m.getBytes());
+			rs = ps.executeQuery();
+			Metadata metadata = new Metadata();
+			while (rs.next()) metadata.put(rs.getString(1), rs.getBytes(2));
+			rs.close();
+			ps.close();
+			return metadata;
+		} catch (SQLException e) {
+			tryToClose(rs);
+			tryToClose(ps);
+			throw new DbException(e);
+		}
+	}
+
+	public Collection<MessageStatus> getMessageStatus(Connection txn,
+			ContactId c, GroupId g) throws DbException {
+		PreparedStatement ps = null;
+		ResultSet rs = null;
+		try {
+			String sql = "SELECT m.messageId, txCount > 0, seen"
 					+ " FROM messages AS m"
-					+ " LEFT OUTER JOIN localAuthors AS la"
-					+ " ON m.authorId = la.authorId"
-					+ " LEFT OUTER JOIN contacts AS c"
-					+ " ON m.authorId = c.authorId"
-					+ " WHERE groupId = ?";
+					+ " JOIN statuses AS s"
+					+ " ON m.messageId = s.messageId"
+					+ " WHERE groupId = ?"
+					+ " AND contactId = ?";
 			ps = txn.prepareStatement(sql);
 			ps.setBytes(1, g.getBytes());
+			ps.setInt(2, c.getInt());
 			rs = ps.executeQuery();
-			List<MessageHeader> headers = new ArrayList<MessageHeader>();
+			List<MessageStatus> statuses = new ArrayList<MessageStatus>();
 			while (rs.next()) {
-				MessageId id = new MessageId(rs.getBytes(1));
-				byte[] b = rs.getBytes(2);
-				MessageId parent = b == null ? null : new MessageId(b);
-				Author author;
-				b = rs.getBytes(3);
-				if (b == null) {
-					author = null;
-				} else {
-					AuthorId authorId = new AuthorId(b);
-					String authorName = rs.getString(4);
-					byte[] authorKey = rs.getBytes(5);
-					author = new Author(authorId, authorName, authorKey);
-				}
-				String contentType = rs.getString(6);
-				long timestamp = rs.getLong(7);
-				boolean local = rs.getBoolean(8);
-				boolean read = rs.getBoolean(9);
-				boolean isSelf = rs.getBoolean(10);
-				boolean isContact = rs.getBoolean(11);
-
-				Author.Status status;
-				if (author == null) status = ANONYMOUS;
-				else if (isSelf || isContact) status = VERIFIED;
-				else status = UNKNOWN;
-
-				headers.add(new MessageHeader(id, parent, g, author, status,
-						contentType, timestamp, local, read, State.STORED));
+				MessageId messageId = new MessageId(rs.getBytes(1));
+				boolean sent = rs.getBoolean(2);
+				boolean seen = rs.getBoolean(3);
+				statuses.add(new MessageStatus(messageId, c, sent, seen));
 			}
 			rs.close();
 			ps.close();
-			return Collections.unmodifiableList(headers);
+			return Collections.unmodifiableList(statuses);
+		} catch (SQLException e) {
+			tryToClose(rs);
+			tryToClose(ps);
+			throw new DbException(e);
+		}
+	}
+
+	public MessageStatus getMessageStatus(Connection txn,
+			ContactId c, MessageId m) throws DbException {
+		PreparedStatement ps = null;
+		ResultSet rs = null;
+		try {
+			String sql = "SELECT txCount > 0, seen"
+					+ " FROM statuses"
+					+ " WHERE messageId = ?"
+					+ " AND contactId = ?";
+			ps = txn.prepareStatement(sql);
+			ps.setBytes(1, m.getBytes());
+			ps.setInt(2, c.getInt());
+			rs = ps.executeQuery();
+			if (!rs.next()) throw new DbStateException();
+			boolean sent = rs.getBoolean(1);
+			boolean seen = rs.getBoolean(2);
+			if (rs.next()) throw new DbStateException();
+			rs.close();
+			ps.close();
+			return new MessageStatus(m, c, sent, seen);
 		} catch (SQLException e) {
 			tryToClose(rs);
 			tryToClose(ps);
@@ -1665,13 +1604,15 @@ abstract class JdbcDatabase implements Database<Connection> {
 					+ " ON m.messageId = s.messageId"
 					+ " AND cg.contactId = s.contactId"
 					+ " WHERE cg.contactId = ?"
+					+ " AND valid = ?"
 					+ " AND seen = FALSE AND requested = FALSE"
 					+ " AND s.expiry < ?"
 					+ " ORDER BY timestamp DESC LIMIT ?";
 			ps = txn.prepareStatement(sql);
 			ps.setInt(1, c.getInt());
-			ps.setLong(2, now);
-			ps.setInt(3, maxMessages);
+			ps.setInt(2, VALIDATION_VALID);
+			ps.setLong(3, now);
+			ps.setInt(4, maxMessages);
 			rs = ps.executeQuery();
 			List<MessageId> ids = new ArrayList<MessageId>();
 			while (rs.next()) ids.add(new MessageId(rs.getBytes(1)));
@@ -1725,12 +1666,14 @@ abstract class JdbcDatabase implements Database<Connection> {
 					+ " ON m.messageId = s.messageId"
 					+ " AND cg.contactId = s.contactId"
 					+ " WHERE cg.contactId = ?"
+					+ " AND valid = ?"
 					+ " AND seen = FALSE"
 					+ " AND s.expiry < ?"
 					+ " ORDER BY timestamp DESC";
 			ps = txn.prepareStatement(sql);
 			ps.setInt(1, c.getInt());
-			ps.setLong(2, now);
+			ps.setInt(2, VALIDATION_VALID);
+			ps.setLong(3, now);
 			rs = ps.executeQuery();
 			List<MessageId> ids = new ArrayList<MessageId>();
 			int total = 0;
@@ -1750,26 +1693,23 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
-	public MessageId getParent(Connection txn, MessageId m) throws DbException {
+	public Collection<MessageId> getMessagesToValidate(Connection txn,
+			ClientId c) throws DbException {
 		PreparedStatement ps = null;
 		ResultSet rs = null;
 		try {
-			String sql = "SELECT m1.parentId FROM messages AS m1"
-					+ " JOIN messages AS m2"
-					+ " ON m1.parentId = m2.messageId"
-					+ " AND m1.groupId = m2.groupId"
-					+ " WHERE m1.messageId = ?";
+			String sql = "SELECT messageId FROM messages AS m"
+					+ " JOIN groups AS g ON m.groupId = g.groupId"
+					+ " WHERE valid = ? AND clientId = ?";
 			ps = txn.prepareStatement(sql);
-			ps.setBytes(1, m.getBytes());
+			ps.setInt(1, VALIDATION_UNKNOWN);
+			ps.setBytes(2, c.getBytes());
 			rs = ps.executeQuery();
-			MessageId parent = null;
-			if (rs.next()) {
-				parent = new MessageId(rs.getBytes(1));
-				if (rs.next()) throw new DbStateException();
-			}
+			List<MessageId> ids = new ArrayList<MessageId>();
+			while (rs.next()) ids.add(new MessageId(rs.getBytes(1)));
 			rs.close();
 			ps.close();
-			return parent;
+			return Collections.unmodifiableList(ids);
 		} catch (SQLException e) {
 			tryToClose(rs);
 			tryToClose(ps);
@@ -1782,14 +1722,12 @@ abstract class JdbcDatabase implements Database<Connection> {
 		PreparedStatement ps = null;
 		ResultSet rs = null;
 		try {
-			String sql = "SELECT length, raw FROM messages WHERE messageId = ?";
+			String sql = "SELECT raw FROM messages WHERE messageId = ?";
 			ps = txn.prepareStatement(sql);
 			ps.setBytes(1, m.getBytes());
 			rs = ps.executeQuery();
 			if (!rs.next()) throw new DbStateException();
-			int length = rs.getInt(1);
-			byte[] raw = rs.getBlob(2).getBytes(1, length);
-			if (raw.length != length) throw new DbStateException();
+			byte[] raw = rs.getBytes(1);
 			if (rs.next()) throw new DbStateException();
 			rs.close();
 			ps.close();
@@ -1801,27 +1739,6 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
-	public boolean getReadFlag(Connection txn, MessageId m) throws DbException {
-		PreparedStatement ps = null;
-		ResultSet rs = null;
-		try {
-			String sql = "SELECT read FROM messages WHERE messageId = ?";
-			ps = txn.prepareStatement(sql);
-			ps.setBytes(1, m.getBytes());
-			rs = ps.executeQuery();
-			if (!rs.next()) throw new DbStateException();
-			boolean read = rs.getBoolean(1);
-			if (rs.next()) throw new DbStateException();
-			rs.close();
-			ps.close();
-			return read;
-		} catch (SQLException e) {
-			tryToClose(rs);
-			tryToClose(ps);
-			throw new DbException(e);
-		}
-	}
-
 	public Map<ContactId, TransportProperties> getRemoteProperties(
 			Connection txn, TransportId t) throws DbException {
 		PreparedStatement ps = null;
@@ -1874,12 +1791,14 @@ abstract class JdbcDatabase implements Database<Connection> {
 					+ " ON m.messageId = s.messageId"
 					+ " AND cg.contactId = s.contactId"
 					+ " WHERE cg.contactId = ?"
+					+ " AND valid = ?"
 					+ " AND seen = FALSE AND requested = TRUE"
 					+ " AND s.expiry < ?"
 					+ " ORDER BY timestamp DESC";
 			ps = txn.prepareStatement(sql);
 			ps.setInt(1, c.getInt());
-			ps.setLong(2, now);
+			ps.setInt(2, VALIDATION_VALID);
+			ps.setLong(3, now);
 			rs = ps.executeQuery();
 			List<MessageId> ids = new ArrayList<MessageId>();
 			int total = 0;
@@ -1993,7 +1912,8 @@ abstract class JdbcDatabase implements Database<Connection> {
 		PreparedStatement ps = null;
 		ResultSet rs = null;
 		try {
-			String sql = "SELECT g.groupId, name, salt, localVersion, txCount"
+			String sql = "SELECT g.groupId, clientId, descriptor,"
+					+ " localVersion, txCount"
 					+ " FROM groups AS g"
 					+ " JOIN groupVisibilities AS gvis"
 					+ " ON g.groupId = gvis.groupId"
@@ -2013,9 +1933,9 @@ abstract class JdbcDatabase implements Database<Connection> {
 			while (rs.next()) {
 				GroupId id = new GroupId(rs.getBytes(1));
 				if (!ids.add(id)) throw new DbStateException();
-				String name = rs.getString(2);
-				byte[] salt = rs.getBytes(3);
-				groups.add(new Group(id, name, salt));
+				ClientId clientId = new ClientId(rs.getBytes(2));
+				byte[] descriptor = rs.getBytes(3);
+				groups.add(new Group(id, clientId, descriptor));
 				version = rs.getLong(4);
 				txCount = rs.getInt(5);
 			}
@@ -2233,32 +2153,6 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
-	public Map<GroupId, Integer> getUnreadMessageCounts(Connection txn)
-			throws DbException {
-		PreparedStatement ps = null;
-		ResultSet rs = null;
-		try {
-			String sql = "SELECT groupId, COUNT(*)"
-					+ " FROM messages AS m"
-					+ " WHERE read = FALSE"
-					+ " GROUP BY groupId";
-			ps = txn.prepareStatement(sql);
-			rs = ps.executeQuery();
-			Map<GroupId, Integer> counts = new HashMap<GroupId, Integer>();
-			while (rs.next()) {
-				GroupId groupId = new GroupId(rs.getBytes(1));
-				counts.put(groupId, rs.getInt(2));
-			}
-			rs.close();
-			ps.close();
-			return Collections.unmodifiableMap(counts);
-		} catch (SQLException e) {
-			tryToClose(rs);
-			tryToClose(ps);
-			throw new DbException(e);
-		}
-	}
-
 	public Collection<ContactId> getVisibility(Connection txn, GroupId g)
 			throws DbException {
 		PreparedStatement ps = null;
@@ -2419,11 +2313,87 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
+	public void mergeMessageMetadata(Connection txn, MessageId m, Metadata meta)
+			throws DbException {
+		PreparedStatement ps = null;
+		try {
+			// Determine which keys are being removed
+			List<String> removed = new ArrayList<String>();
+			Map<String, byte[]> retained = new HashMap<String, byte[]>();
+			for (Entry<String, byte[]> e : meta.entrySet()) {
+				if (e.getValue() == REMOVE) removed.add(e.getKey());
+				else retained.put(e.getKey(), e.getValue());
+			}
+			// Delete any keys that are being removed
+			if (!removed.isEmpty()) {
+				String sql = "DELETE FROM messageMetadata"
+						+ " WHERE messageId = ? AND key = ?";
+				ps = txn.prepareStatement(sql);
+				ps.setBytes(1, m.getBytes());
+				for (String key : removed) {
+					ps.setString(2, key);
+					ps.addBatch();
+				}
+				int[] batchAffected = ps.executeBatch();
+				if (batchAffected.length != removed.size())
+					throw new DbStateException();
+				for (int i = 0; i < batchAffected.length; i++) {
+					if (batchAffected[i] < 0) throw new DbStateException();
+					if (batchAffected[i] > 1) throw new DbStateException();
+				}
+				ps.close();
+			}
+			if (retained.isEmpty()) return;
+			// Update any keys that already exist
+			String sql = "UPDATE messageMetadata SET value = ?"
+					+ " WHERE messageId = ? AND key = ?";
+			ps = txn.prepareStatement(sql);
+			ps.setBytes(2, m.getBytes());
+			for (Entry<String, byte[]> e : retained.entrySet()) {
+				ps.setBytes(1, e.getValue());
+				ps.setString(3, e.getKey());
+				ps.addBatch();
+			}
+			int[] batchAffected = ps.executeBatch();
+			if (batchAffected.length != retained.size())
+				throw new DbStateException();
+			for (int i = 0; i < batchAffected.length; i++) {
+				if (batchAffected[i] < 0) throw new DbStateException();
+				if (batchAffected[i] > 1) throw new DbStateException();
+			}
+			// Insert any keys that don't already exist
+			sql = "INSERT INTO messageMetadata (messageId, key, value)"
+					+ " VALUES (?, ?, ?)";
+			ps = txn.prepareStatement(sql);
+			ps.setBytes(1, m.getBytes());
+			int updateIndex = 0, inserted = 0;
+			for (Entry<String, byte[]> e : retained.entrySet()) {
+				if (batchAffected[updateIndex] == 0) {
+					ps.setString(2, e.getKey());
+					ps.setBytes(3, e.getValue());
+					ps.addBatch();
+					inserted++;
+				}
+				updateIndex++;
+			}
+			batchAffected = ps.executeBatch();
+			if (batchAffected.length != inserted) throw new DbStateException();
+			for (int i = 0; i < batchAffected.length; i++) {
+				if (batchAffected[i] != 1) throw new DbStateException();
+			}
+			ps.close();
+		} catch (SQLException e) {
+			tryToClose(ps);
+			throw new DbException(e);
+		}
+	}
+
 	public void mergeSettings(Connection txn, Settings s, String namespace) throws DbException {
 		PreparedStatement ps = null;
 		try {
 			// Update any settings that already exist
-			String sql = "UPDATE settings SET value = ? WHERE key = ? AND namespace = ?";
+			String sql = "UPDATE settings SET value = ?"
+					+ " WHERE key = ? AND namespace = ?";
 			ps = txn.prepareStatement(sql);
 			for (Entry<String, String> e : s.entrySet()) {
 				ps.setString(1, e.getValue());
@@ -2438,7 +2408,8 @@ abstract class JdbcDatabase implements Database<Connection> {
 				if (batchAffected[i] > 1) throw new DbStateException();
 			}
 			// Insert any settings that don't already exist
-			sql = "INSERT INTO settings (key, value, namespace) VALUES (?, ?, ?)";
+			sql = "INSERT INTO settings (key, value, namespace)"
+					+ " VALUES (?, ?, ?)";
 			ps = txn.prepareStatement(sql);
 			int updateIndex = 0, inserted = 0;
 			for (Entry<String, String> e : s.entrySet()) {
@@ -2714,6 +2685,23 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
+	public void setMessageValidity(Connection txn, MessageId m, boolean valid)
+		throws DbException {
+		PreparedStatement ps = null;
+		try {
+			String sql = "UPDATE messages SET valid = ? WHERE messageId = ?";
+			ps = txn.prepareStatement(sql);
+			ps.setInt(1, valid ? VALIDATION_VALID : VALIDATION_INVALID);
+			ps.setBytes(2, m.getBytes());
+			int affected = ps.executeUpdate();
+			if (affected < 0) throw new DbStateException();
+			ps.close();
+		} catch (SQLException e) {
+			tryToClose(ps);
+			throw new DbException(e);
+		}
+	}
+
 	public void setReorderingWindow(Connection txn, ContactId c, TransportId t,
 			long rotationPeriod, long base, byte[] bitmap) throws DbException {
 		PreparedStatement ps = null;
@@ -2798,14 +2786,14 @@ abstract class JdbcDatabase implements Database<Connection> {
 			// Store the new subscriptions, if any
 			if (groups.isEmpty()) return true;
 			sql = "INSERT INTO contactGroups"
-					+ " (contactId, groupId, name, salt)"
+					+ " (contactId, groupId, clientId, descriptor)"
 					+ " VALUES (?, ?, ?, ?)";
 			ps = txn.prepareStatement(sql);
 			ps.setInt(1, c.getInt());
 			for (Group g : groups) {
 				ps.setBytes(2, g.getId().getBytes());
-				ps.setString(3, g.getName());
-				ps.setBytes(4, g.getSalt());
+				ps.setBytes(3, g.getClientId().getBytes());
+				ps.setBytes(4, g.getDescriptor());
 				ps.addBatch();
 			}
 			int[] batchAffected = ps.executeBatch();
@@ -2823,66 +2811,6 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
-	public void setInboxGroup(Connection txn, ContactId c, Group g)
-			throws DbException {
-		PreparedStatement ps = null;
-		try {
-			// Unset any existing inbox group for the contact
-			String sql = "UPDATE groupVisibilities"
-					+ " SET inbox = FALSE"
-					+ " WHERE contactId = ?"
-					+ " AND inbox = TRUE";
-			ps = txn.prepareStatement(sql);
-			ps.setInt(1, c.getInt());
-			ps.executeUpdate();
-			int affected = ps.executeUpdate();
-			if (affected < 0 || affected > 1) throw new DbStateException();
-			ps.close();
-			// Make the group visible to the contact and set it as the inbox
-			sql = "INSERT INTO groupVisibilities"
-					+ " (contactId, groupId, inbox)"
-					+ " VALUES (?, ?, TRUE)";
-			ps = txn.prepareStatement(sql);
-			ps.setInt(1, c.getInt());
-			ps.setBytes(2, g.getId().getBytes());
-			affected = ps.executeUpdate();
-			if (affected != 1) throw new DbStateException();
-			ps.close();
-			// Add the group to the contact's subscriptions
-			sql = "INSERT INTO contactGroups"
-					+ " (contactId, groupId, name, salt)"
-					+ " VALUES (?, ?, ?, ?)";
-			ps = txn.prepareStatement(sql);
-			ps.setInt(1, c.getInt());
-			ps.setBytes(2, g.getId().getBytes());
-			ps.setString(3, g.getName());
-			ps.setBytes(4, g.getSalt());
-			affected = ps.executeUpdate();
-			if (affected != 1) throw new DbStateException();
-			ps.close();
-		} catch (SQLException e) {
-			tryToClose(ps);
-			throw new DbException(e);
-		}
-	}
-
-	public void setReadFlag(Connection txn, MessageId m, boolean read)
-			throws DbException {
-		PreparedStatement ps = null;
-		try {
-			String sql = "UPDATE messages SET read = ? WHERE messageId = ?";
-			ps = txn.prepareStatement(sql);
-			ps.setBoolean(1, read);
-			ps.setBytes(2, m.getBytes());
-			int affected = ps.executeUpdate();
-			if (affected < 0 || affected > 1) throw new DbStateException();
-			ps.close();
-		} catch (SQLException e) {
-			tryToClose(ps);
-			throw new DbException(e);
-		}
-	}
-
 	public void setRemoteProperties(Connection txn, ContactId c,
 			Map<TransportId, TransportProperties> p) throws DbException {
 		PreparedStatement ps = null;