Merge branch 'compact-at-startup' into 'master'

Compact the database at startup

See merge request briar/briar!955

bramble-api/src/main/java/org/briarproject/bramble/api/StringMap.java

@@ -38,4 +38,18 @@ public abstract class StringMap extends Hashtable<String, String> {
 	public void putInt(String key, int value) {
 		put(key, String.valueOf(value));
 	}
+
+	public long getLong(String key, long defaultValue) {
+		String s = get(key);
+		if (s == null) return defaultValue;
+		try {
+			return Long.valueOf(s);
+		} catch (NumberFormatException e) {
+			return defaultValue;
+		}
+	}
+
+	public void putLong(String key, long value) {
+		put(key, String.valueOf(value));
+	}
 }

bramble-api/src/main/java/org/briarproject/bramble/api/db/MigrationListener.java

@@ -6,6 +6,10 @@ public interface MigrationListener {
 	 * This is called when a migration is started while opening the database.
 	 * It will be called once for each migration being applied.
 	 */
-	void onMigrationRun();
+	void onDatabaseMigration();
 
+	/**
+	 * This is called when compaction is started while opening the database.
+	 */
+	void onDatabaseCompaction();
 }

bramble-api/src/main/java/org/briarproject/bramble/api/lifecycle/LifecycleManager.java

@@ -34,7 +34,8 @@ public interface LifecycleManager {
 	 */
 	enum LifecycleState {
 
-		STARTING, MIGRATING_DATABASE, STARTING_SERVICES, RUNNING, STOPPING;
+		STARTING, MIGRATING_DATABASE, COMPACTING_DATABASE, STARTING_SERVICES,
+		RUNNING, STOPPING;
 
 		public boolean isAfter(LifecycleState state) {
 			return ordinal() > state.ordinal();

bramble-api/src/test/java/org/briarproject/bramble/test/SettableClock.java

@@ -0,0 +1,24 @@
+package org.briarproject.bramble.test;
+
+import org.briarproject.bramble.api.system.Clock;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public class SettableClock implements Clock {
+
+	private final AtomicLong time;
+
+	public SettableClock(AtomicLong time) {
+		this.time = time;
+	}
+
+	@Override
+	public long currentTimeMillis() {
+		return time.get();
+	}
+
+	@Override
+	public void sleep(long milliseconds) throws InterruptedException {
+		Thread.sleep(milliseconds);
+	}
+}

bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseConstants.java

@@ -2,6 +2,8 @@ package org.briarproject.bramble.db;
 
 import org.briarproject.bramble.api.settings.Settings;
 
+import static java.util.concurrent.TimeUnit.DAYS;
+
 interface DatabaseConstants {
 
 	/**
@@ -23,4 +25,16 @@ interface DatabaseConstants {
 	 */
 	String SCHEMA_VERSION_KEY = "schemaVersion";
 
+	/**
+	 * The {@link Settings} key under which the time of the last database
+	 * compaction is stored.
+	 */
+	String LAST_COMPACTED_KEY = "lastCompacted";
+
+	/**
+	 * The maximum time between database compactions in milliseconds. When the
+	 * database is opened it will be compacted if more than this amount of time
+	 * has passed since the last compaction.
+	 */
+	long MAX_COMPACTION_INTERVAL_MS = DAYS.toMillis(30);
 }

bramble-core/src/main/java/org/briarproject/bramble/db/H2Database.java

@@ -13,6 +13,7 @@ import java.io.File;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.Properties;
 
 import javax.annotation.Nullable;
@@ -106,4 +107,22 @@ class H2Database extends JdbcDatabase {
 	String getUrl() {
 		return url;
 	}
+
+	@Override
+	protected void compactAndClose() throws DbException {
+		Connection c = null;
+		Statement s = null;
+		try {
+			c = createConnection();
+			closeAllConnections();
+			s = c.createStatement();
+			s.execute("SHUTDOWN COMPACT");
+			s.close();
+			c.close();
+		} catch (SQLException e) {
+			tryToClose(s);
+			tryToClose(c);
+			throw new DbException(e);
+		}
+	}
 }

bramble-core/src/main/java/org/briarproject/bramble/db/HyperSqlDatabase.java

@@ -61,14 +61,18 @@ class HyperSqlDatabase extends JdbcDatabase {
 
 	@Override
 	public void close() throws DbException {
+		Connection c = null;
+		Statement s = null;
 		try {
 			super.closeAllConnections();
-			Connection c = createConnection();
-			Statement s = c.createStatement();
+			c = createConnection();
+			s = c.createStatement();
 			s.executeQuery("SHUTDOWN");
 			s.close();
 			c.close();
 		} catch (SQLException e) {
+			tryToClose(s);
+			tryToClose(c);
 			throw new DbException(e);
 		}
 	}
@@ -104,4 +108,22 @@ class HyperSqlDatabase extends JdbcDatabase {
 		String hex = StringUtils.toHexString(key.getBytes());
 		return DriverManager.getConnection(url + ";crypt_key=" + hex);
 	}
+
+	@Override
+	protected void compactAndClose() throws DbException {
+		Connection c = null;
+		Statement s = null;
+		try {
+			super.closeAllConnections();
+			c = createConnection();
+			s = c.createStatement();
+			s.executeQuery("SHUTDOWN COMPACT");
+			s.close();
+			c.close();
+		} catch (SQLException e) {
+			tryToClose(s);
+			tryToClose(c);
+			throw new DbException(e);
+		}
+	}
 }

bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java

@@ -67,9 +67,13 @@ import static org.briarproject.bramble.api.sync.ValidationManager.State.DELIVERE
 import static org.briarproject.bramble.api.sync.ValidationManager.State.PENDING;
 import static org.briarproject.bramble.api.sync.ValidationManager.State.UNKNOWN;
 import static org.briarproject.bramble.db.DatabaseConstants.DB_SETTINGS_NAMESPACE;
+import static org.briarproject.bramble.db.DatabaseConstants.LAST_COMPACTED_KEY;
+import static org.briarproject.bramble.db.DatabaseConstants.MAX_COMPACTION_INTERVAL_MS;
 import static org.briarproject.bramble.db.DatabaseConstants.SCHEMA_VERSION_KEY;
 import static org.briarproject.bramble.db.ExponentialBackoff.calculateExpiry;
+import static org.briarproject.bramble.util.LogUtils.logDuration;
 import static org.briarproject.bramble.util.LogUtils.logException;
+import static org.briarproject.bramble.util.LogUtils.now;
 
 /**
  * A generic database implementation that can be used with any JDBC-compatible
@@ -317,9 +321,10 @@ abstract class JdbcDatabase implements Database<Connection> {
 	private int openConnections = 0; // Locking: connectionsLock
 	private boolean closed = false; // Locking: connectionsLock
 
-	@Nullable
 	protected abstract Connection createConnection() throws SQLException;
 
+	protected abstract void compactAndClose() throws DbException;
+
 	private final Lock connectionsLock = new ReentrantLock();
 	private final Condition connectionsChanged = connectionsLock.newCondition();
 
@@ -344,13 +349,16 @@ abstract class JdbcDatabase implements Database<Connection> {
 			throw new DbException(e);
 		}
 		// Open the database and create the tables and indexes if necessary
+		boolean compact;
 		Connection txn = startTransaction();
 		try {
 			if (reopen) {
-				checkSchemaVersion(txn, listener);
+				Settings s = getSettings(txn, DB_SETTINGS_NAMESPACE);
+				compact = migrateSchema(txn, s, listener) || isCompactionDue(s);
 			} else {
 				createTables(txn);
-				storeSchemaVersion(txn, CODE_SCHEMA_VERSION);
+				initialiseSettings(txn);
+				compact = false;
 			}
 			createIndexes(txn);
 			commitTransaction(txn);
@@ -358,6 +366,25 @@ abstract class JdbcDatabase implements Database<Connection> {
 			abortTransaction(txn);
 			throw e;
 		}
+		// Compact the database if necessary
+		if (compact) {
+			if (listener != null) listener.onDatabaseCompaction();
+			long start = now();
+			compactAndClose();
+			logDuration(LOG, "Compacting database", start);
+			// Allow the next transaction to reopen the DB
+			synchronized (connectionsLock) {
+				closed = false;
+			}
+			txn = startTransaction();
+			try {
+				storeLastCompacted(txn);
+				commitTransaction(txn);
+			} catch (DbException e) {
+				abortTransaction(txn);
+				throw e;
+			}
+		}
 	}
 
 	/**
@@ -365,17 +392,18 @@ abstract class JdbcDatabase implements Database<Connection> {
 	 * version used by the current code and applies any suitable migrations to
 	 * the data if necessary.
 	 *
+	 * @return true if any migrations were applied, false if the schema was
+	 * already current
 	 * @throws DataTooNewException if the data uses a newer schema than the
 	 * current code
 	 * @throws DataTooOldException if the data uses an older schema than the
 	 * current code and cannot be migrated
 	 */
-	private void checkSchemaVersion(Connection txn,
+	private boolean migrateSchema(Connection txn, Settings s,
 			@Nullable MigrationListener listener) throws DbException {
-		Settings s = getSettings(txn, DB_SETTINGS_NAMESPACE);
 		int dataSchemaVersion = s.getInt(SCHEMA_VERSION_KEY, -1);
 		if (dataSchemaVersion == -1) throw new DbException();
-		if (dataSchemaVersion == CODE_SCHEMA_VERSION) return;
+		if (dataSchemaVersion == CODE_SCHEMA_VERSION) return false;
 		if (CODE_SCHEMA_VERSION < dataSchemaVersion)
 			throw new DataTooNewException();
 		// Apply any suitable migrations in order
@@ -384,7 +412,7 @@ abstract class JdbcDatabase implements Database<Connection> {
 			if (start == dataSchemaVersion) {
 				if (LOG.isLoggable(INFO))
 					LOG.info("Migrating from schema " + start + " to " + end);
-				if (listener != null) listener.onMigrationRun();
+				if (listener != null) listener.onDatabaseMigration();
 				// Apply the migration
 				m.migrate(txn);
 				// Store the new schema version
@@ -394,6 +422,7 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 		if (dataSchemaVersion != CODE_SCHEMA_VERSION)
 			throw new DataTooOldException();
+		return true;
 	}
 
 	// Package access for testing
@@ -401,6 +430,14 @@ abstract class JdbcDatabase implements Database<Connection> {
 		return Arrays.asList(new Migration38_39(), new Migration39_40());
 	}
 
+	private boolean isCompactionDue(Settings s) {
+		long lastCompacted = s.getLong(LAST_COMPACTED_KEY, 0);
+		long elapsed = clock.currentTimeMillis() - lastCompacted;
+		if (LOG.isLoggable(INFO))
+			LOG.info(elapsed + " ms since last compaction");
+		return elapsed > MAX_COMPACTION_INTERVAL_MS;
+	}
+
 	private void storeSchemaVersion(Connection txn, int version)
 			throws DbException {
 		Settings s = new Settings();
@@ -408,6 +445,19 @@ abstract class JdbcDatabase implements Database<Connection> {
 		mergeSettings(txn, s, DB_SETTINGS_NAMESPACE);
 	}
 
+	private void storeLastCompacted(Connection txn) throws DbException {
+		Settings s = new Settings();
+		s.putLong(LAST_COMPACTED_KEY, clock.currentTimeMillis());
+		mergeSettings(txn, s, DB_SETTINGS_NAMESPACE);
+	}
+
+	private void initialiseSettings(Connection txn) throws DbException {
+		Settings s = new Settings();
+		s.putInt(SCHEMA_VERSION_KEY, CODE_SCHEMA_VERSION);
+		s.putLong(LAST_COMPACTED_KEY, clock.currentTimeMillis());
+		mergeSettings(txn, s, DB_SETTINGS_NAMESPACE);
+	}
+
 	private void tryToClose(@Nullable ResultSet rs) {
 		try {
 			if (rs != null) rs.close();
@@ -416,7 +466,7 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
-	private void tryToClose(@Nullable Statement s) {
+	protected void tryToClose(@Nullable Statement s) {
 		try {
 			if (s != null) s.close();
 		} catch (SQLException e) {
@@ -424,6 +474,14 @@ abstract class JdbcDatabase implements Database<Connection> {
 		}
 	}
 
+	protected void tryToClose(@Nullable Connection c) {
+		try {
+			if (c != null) c.close();
+		} catch (SQLException e) {
+			logException(LOG, WARNING, e);
+		}
+	}
+
 	private void createTables(Connection txn) throws DbException {
 		Statement s = null;
 		try {
@@ -489,7 +547,6 @@ abstract class JdbcDatabase implements Database<Connection> {
 			if (txn == null) {
 				// Open a new connection
 				txn = createConnection();
-				if (txn == null) throw new DbException();
 				txn.setAutoCommit(false);
 				connectionsLock.lock();
 				try {

bramble-core/src/main/java/org/briarproject/bramble/lifecycle/LifecycleManagerImpl.java

@@ -29,6 +29,7 @@ import javax.inject.Inject;
 import static java.util.logging.Level.FINE;
 import static java.util.logging.Level.INFO;
 import static java.util.logging.Level.WARNING;
+import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.COMPACTING_DATABASE;
 import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.MIGRATING_DATABASE;
 import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.RUNNING;
 import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.STARTING;
@@ -159,11 +160,17 @@ class LifecycleManagerImpl implements LifecycleManager, MigrationListener {
 	}
 
 	@Override
-	public void onMigrationRun() {
+	public void onDatabaseMigration() {
 		state = MIGRATING_DATABASE;
 		eventBus.broadcast(new LifecycleEvent(MIGRATING_DATABASE));
 	}
 
+	@Override
+	public void onDatabaseCompaction() {
+		state = COMPACTING_DATABASE;
+		eventBus.broadcast(new LifecycleEvent(COMPACTING_DATABASE));
+	}
+
 	@Override
 	public void stopServices() {
 		try {

bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java

@@ -26,11 +26,10 @@ import org.briarproject.bramble.api.transport.KeySetId;
 import org.briarproject.bramble.api.transport.OutgoingKeys;
 import org.briarproject.bramble.api.transport.TransportKeys;
 import org.briarproject.bramble.system.SystemClock;
-import org.briarproject.bramble.test.ArrayClock;
 import org.briarproject.bramble.test.BrambleTestCase;
+import org.briarproject.bramble.test.SettableClock;
 import org.briarproject.bramble.test.TestDatabaseConfig;
 import org.briarproject.bramble.test.TestMessageFactory;
-import org.briarproject.bramble.test.TestUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -46,6 +45,7 @@ import java.util.Map.Entry;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
@@ -61,6 +61,9 @@ import static org.briarproject.bramble.api.sync.ValidationManager.State.DELIVERE
 import static org.briarproject.bramble.api.sync.ValidationManager.State.INVALID;
 import static org.briarproject.bramble.api.sync.ValidationManager.State.PENDING;
 import static org.briarproject.bramble.api.sync.ValidationManager.State.UNKNOWN;
+import static org.briarproject.bramble.db.DatabaseConstants.DB_SETTINGS_NAMESPACE;
+import static org.briarproject.bramble.db.DatabaseConstants.LAST_COMPACTED_KEY;
+import static org.briarproject.bramble.db.DatabaseConstants.MAX_COMPACTION_INTERVAL_MS;
 import static org.briarproject.bramble.test.TestUtils.deleteTestDirectory;
 import static org.briarproject.bramble.test.TestUtils.getAuthor;
 import static org.briarproject.bramble.test.TestUtils.getClientId;
@@ -1818,10 +1821,9 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
 	@Test
 	public void testMessageRetransmission() throws Exception {
 		long now = System.currentTimeMillis();
-		long steps[] = {now, now, now + MAX_LATENCY * 2 - 1,
-				now + MAX_LATENCY * 2};
+		AtomicLong time = new AtomicLong(now);
 		Database<Connection> db =
-				open(false, new TestMessageFactory(), new ArrayClock(steps));
+				open(false, new TestMessageFactory(), new SettableClock(time));
 		Connection txn = db.startTransaction();
 
 		// Add a contact, a shared group and a shared message
@@ -1847,11 +1849,13 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
 
 		// Time: now + MAX_LATENCY * 2 - 1
 		// The message should not yet be sendable
+		time.set(now + MAX_LATENCY * 2 - 1);
 		ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE, MAX_LATENCY);
 		assertTrue(ids.isEmpty());
 
 		// Time: now + MAX_LATENCY * 2
 		// The message should have expired and should now be sendable
+		time.set(now + MAX_LATENCY * 2);
 		ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE, MAX_LATENCY);
 		assertEquals(singletonList(messageId), ids);
 
@@ -1859,13 +1863,12 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
 		db.close();
 	}
 
-
 	@Test
 	public void testFasterMessageRetransmission() throws Exception {
 		long now = System.currentTimeMillis();
-		long steps[] = {now, now, now, now, now + 1};
+		AtomicLong time = new AtomicLong(now);
 		Database<Connection> db =
-				open(false, new TestMessageFactory(), new ArrayClock(steps));
+				open(false, new TestMessageFactory(), new SettableClock(time));
 		Connection txn = db.startTransaction();
 
 		// Add a contact, a shared group and a shared message
@@ -1903,6 +1906,7 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
 		// Time: now + 1
 		// The message should no longer be sendable via the faster transport,
 		// as the ETA is now equal
+		time.set(now + 1);
 		ids = db.getMessagesToSend(txn, contactId, ONE_MEGABYTE,
 				MAX_LATENCY - 1);
 		assertTrue(ids.isEmpty());
@@ -1911,6 +1915,45 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
 		db.close();
 	}
 
+	@Test
+	public void testCompactionTime() throws Exception {
+		MessageFactory messageFactory = new TestMessageFactory();
+		long now = System.currentTimeMillis();
+		AtomicLong time = new AtomicLong(now);
+		Clock clock = new SettableClock(time);
+
+		// Time: now
+		// The last compaction time should be initialised to the current time
+		Database<Connection> db = open(false, messageFactory, clock);
+		Connection txn = db.startTransaction();
+		Settings s = db.getSettings(txn, DB_SETTINGS_NAMESPACE);
+		assertEquals(now, s.getLong(LAST_COMPACTED_KEY, 0));
+		db.commitTransaction(txn);
+		db.close();
+
+		// Time: now + MAX_COMPACTION_INTERVAL_MS
+		// The DB should not be compacted, so the last compaction time should
+		// not be updated
+		time.set(now + MAX_COMPACTION_INTERVAL_MS);
+		db = open(true, messageFactory, clock);
+		txn = db.startTransaction();
+		s = db.getSettings(txn, DB_SETTINGS_NAMESPACE);
+		assertEquals(now, s.getLong(LAST_COMPACTED_KEY, 0));
+		db.commitTransaction(txn);
+		db.close();
+
+		// Time: now + MAX_COMPACTION_INTERVAL_MS + 1
+		// The DB should be compacted, so the last compaction time should be
+		// updated
+		time.set(now + MAX_COMPACTION_INTERVAL_MS + 1);
+		db = open(true, messageFactory, clock);
+		txn = db.startTransaction();
+		s = db.getSettings(txn, DB_SETTINGS_NAMESPACE);
+		assertEquals(now + MAX_COMPACTION_INTERVAL_MS + 1,
+				s.getLong(LAST_COMPACTED_KEY, 0));
+		db.commitTransaction(txn);
+		db.close();
+	}
 
 	private Database<Connection> open(boolean resume) throws Exception {
 		return open(resume, new TestMessageFactory(), new SystemClock());
@@ -1921,7 +1964,7 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase {
 		Database<Connection> db =
 				createDatabase(new TestDatabaseConfig(testDir, MAX_SIZE),
 						messageFactory, clock);
-		if (!resume) TestUtils.deleteTestDirectory(testDir);
+		if (!resume) deleteTestDirectory(testDir);
 		db.open(key, null);
 		return db;
 	}

briar-android/src/main/java/org/briarproject/briar/android/login/OpenDatabaseActivity.java

@@ -20,6 +20,7 @@ import org.briarproject.briar.android.navdrawer.NavDrawerActivity;
 import javax.annotation.ParametersAreNonnullByDefault;
 import javax.inject.Inject;
 
+import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.COMPACTING_DATABASE;
 import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.MIGRATING_DATABASE;
 import static org.briarproject.bramble.api.lifecycle.LifecycleManager.LifecycleState.STARTING_SERVICES;
 
@@ -34,7 +35,7 @@ public class OpenDatabaseActivity extends BriarActivity
 
 	private TextView textView;
 	private ImageView imageView;
-	private boolean showingMigration = false;
+	private boolean showingMigration = false, showingCompaction = false;
 
 	@Override
 	public void onCreate(@Nullable Bundle state) {
@@ -57,6 +58,7 @@ public class OpenDatabaseActivity extends BriarActivity
 			finishAndStartApp();
 		} else {
 			if (state == MIGRATING_DATABASE) showMigration();
+			else if (state == COMPACTING_DATABASE) showCompaction();
 			eventBus.addListener(this);
 		}
 	}
@@ -75,6 +77,8 @@ public class OpenDatabaseActivity extends BriarActivity
 				runOnUiThreadUnlessDestroyed(this::finishAndStartApp);
 			else if (state == MIGRATING_DATABASE)
 				runOnUiThreadUnlessDestroyed(this::showMigration);
+			else if (state == COMPACTING_DATABASE)
+				runOnUiThreadUnlessDestroyed(this::showCompaction);
 		}
 	}
 
@@ -85,6 +89,13 @@ public class OpenDatabaseActivity extends BriarActivity
 		showingMigration = true;
 	}
 
+	private void showCompaction() {
+		if (showingCompaction) return;
+		textView.setText(R.string.startup_compact_database);
+		imageView.setImageResource(R.drawable.startup_migration);
+		showingCompaction = true;
+	}
+
 	private void finishAndStartApp() {
 		startActivity(new Intent(this, NavDrawerActivity.class));
 		supportFinishAfterTransition();

briar-android/src/main/res/values/strings.xml

@@ -54,6 +54,7 @@
 	<string name="download_briar_button">Download Briar 1.0</string>
 	<string name="startup_open_database">Decrypting Database…</string>
 	<string name="startup_migrate_database">Upgrading Database…</string>
+	<string name="startup_compact_database">Compacting Database…</string>
 
 	<!-- Navigation Drawer -->
 	<string name="nav_drawer_open_description">Open the navigation drawer</string>