Add tests to compare benchmarks.

bramble-core/src/test/java/org/briarproject/bramble/db/H2DatabasePerformanceSelfComparisonTest.java

@@ -0,0 +1,23 @@
+package org.briarproject.bramble.db;
+
+import org.briarproject.bramble.api.db.DatabaseConfig;
+import org.briarproject.bramble.api.system.Clock;
+import org.junit.Ignore;
+
+import java.sql.Connection;
+
+@Ignore
+public class H2DatabasePerformanceSelfComparisonTest
+		extends JdbcDatabasePerformanceComparisonTest {
+
+	@Override
+	Database<Connection> createDatabase(boolean conditionA,
+			DatabaseConfig databaseConfig, Clock clock) {
+		return new H2Database(databaseConfig, clock);
+	}
+
+	@Override
+	protected String getTestName() {
+		return H2DatabasePerformanceSelfComparisonTest.class.getSimpleName();
+	}
+}

bramble-core/src/test/java/org/briarproject/bramble/db/H2DatabasePerformanceTest.java

@@ -5,7 +5,8 @@ import org.briarproject.bramble.api.system.Clock;
 import org.junit.Ignore;
 
 @Ignore
-public class H2DatabasePerformanceTest extends JdbcDatabasePerformanceTest {
+public class H2DatabasePerformanceTest
+		extends JdbcSingleDatabasePerformanceTest {
 
 	@Override
 	protected String getTestName() {

bramble-core/src/test/java/org/briarproject/bramble/db/H2HyperSqlDatabasePerformanceComparisonTest.java

@@ -0,0 +1,25 @@
+package org.briarproject.bramble.db;
+
+import org.briarproject.bramble.api.db.DatabaseConfig;
+import org.briarproject.bramble.api.system.Clock;
+import org.junit.Ignore;
+
+import java.sql.Connection;
+
+@Ignore
+public class H2HyperSqlDatabasePerformanceComparisonTest
+		extends JdbcDatabasePerformanceComparisonTest {
+
+	@Override
+	Database<Connection> createDatabase(boolean conditionA,
+			DatabaseConfig databaseConfig, Clock clock) {
+		if (conditionA) return new H2Database(databaseConfig, clock);
+		else return new HyperSqlDatabase(databaseConfig, clock);
+	}
+
+	@Override
+	protected String getTestName() {
+		return H2HyperSqlDatabasePerformanceComparisonTest.class
+				.getSimpleName();
+	}
+}

bramble-core/src/test/java/org/briarproject/bramble/db/HyperSqlDatabasePerformanceTest.java

@@ -6,7 +6,7 @@ import org.junit.Ignore;
 
 @Ignore
 public class HyperSqlDatabasePerformanceTest
-		extends JdbcDatabasePerformanceTest {
+		extends JdbcSingleDatabasePerformanceTest {
 
 	@Override
 	protected String getTestName() {

bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabasePerformanceComparisonTest.java

@@ -0,0 +1,89 @@
+package org.briarproject.bramble.db;
+
+import org.briarproject.bramble.api.db.DatabaseConfig;
+import org.briarproject.bramble.api.db.DbException;
+import org.briarproject.bramble.api.system.Clock;
+import org.briarproject.bramble.system.SystemClock;
+import org.briarproject.bramble.test.TestDatabaseConfig;
+import org.briarproject.bramble.test.UTest;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.briarproject.bramble.test.TestUtils.deleteTestDirectory;
+import static org.briarproject.bramble.test.TestUtils.getMean;
+import static org.briarproject.bramble.test.TestUtils.getMedian;
+import static org.briarproject.bramble.test.TestUtils.getStandardDeviation;
+import static org.briarproject.bramble.test.UTest.Z_CRITICAL_0_01;
+
+public abstract class JdbcDatabasePerformanceComparisonTest
+		extends JdbcDatabasePerformanceTest {
+
+	/**
+	 * How many blocks of each condition to compare.
+	 */
+	private static final int COMPARISON_BLOCKS = 10;
+
+	abstract Database<Connection> createDatabase(boolean conditionA,
+			DatabaseConfig databaseConfig, Clock clock);
+
+	@Override
+	protected void benchmark(String name,
+			BenchmarkTask<Database<Connection>> task) throws Exception {
+		List<Double> aDurations = new ArrayList<>();
+		List<Double> bDurations = new ArrayList<>();
+		boolean aFirst = true;
+		for (int i = 0; i < COMPARISON_BLOCKS; i++) {
+			// Alternate between running the A and B benchmarks first
+			if (aFirst) {
+				aDurations.addAll(benchmark(true, task).durations);
+				bDurations.addAll(benchmark(false, task).durations);
+			} else {
+				bDurations.addAll(benchmark(false, task).durations);
+				aDurations.addAll(benchmark(true, task).durations);
+			}
+			aFirst = !aFirst;
+		}
+		// Compare the results using a small P value, which increases our
+		// chance of getting an inconclusive result, making this a conservative
+		// test for performance differences
+		UTest.Result comparison = UTest.test(aDurations, bDurations,
+				Z_CRITICAL_0_01);
+		writeResult(name, aDurations, bDurations, comparison);
+	}
+
+	private SteadyStateResult benchmark(boolean conditionA,
+			BenchmarkTask<Database<Connection>> task) throws Exception {
+		deleteTestDirectory(testDir);
+		Database<Connection> db = openDatabase(conditionA);
+		populateDatabase(db);
+		db.close();
+		db = openDatabase(conditionA);
+		// Measure blocks of iterations until we reach a steady state
+		SteadyStateResult result = measureSteadyState(db, task);
+		db.close();
+		return result;
+	}
+
+	private Database<Connection> openDatabase(boolean conditionA)
+			throws DbException {
+		Database<Connection> db = createDatabase(conditionA,
+				new TestDatabaseConfig(testDir, MAX_SIZE), new SystemClock());
+		db.open();
+		return db;
+	}
+
+	private void writeResult(String name, List<Double> aDurations,
+			List<Double> bDurations, UTest.Result comparison)
+			throws IOException {
+		String result = String.format("%s\t%,d\t%,d\t%,d\t%,d\t%,d\t%,d\t%s",
+				name, (long) getMean(aDurations), (long) getMedian(aDurations),
+				(long) getStandardDeviation(aDurations),
+				(long) getMean(bDurations), (long) getMedian(bDurations),
+				(long) getStandardDeviation(bDurations),
+				comparison.name());
+		writeResult(result);
+	}
+}

bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabasePerformanceTest.java

@@ -2,7 +2,6 @@ package org.briarproject.bramble.db;
 
 import org.briarproject.bramble.api.contact.Contact;
 import org.briarproject.bramble.api.contact.ContactId;
-import org.briarproject.bramble.api.db.DatabaseConfig;
 import org.briarproject.bramble.api.db.DbException;
 import org.briarproject.bramble.api.db.Metadata;
 import org.briarproject.bramble.api.identity.AuthorId;
@@ -13,10 +12,7 @@ import org.briarproject.bramble.api.sync.GroupId;
 import org.briarproject.bramble.api.sync.Message;
 import org.briarproject.bramble.api.sync.MessageId;
 import org.briarproject.bramble.api.sync.ValidationManager.State;
-import org.briarproject.bramble.api.system.Clock;
-import org.briarproject.bramble.system.SystemClock;
 import org.briarproject.bramble.test.BrambleTestCase;
-import org.briarproject.bramble.test.TestDatabaseConfig;
 import org.briarproject.bramble.test.UTest;
 import org.junit.After;
 import org.junit.Before;
@@ -24,6 +20,7 @@ import org.junit.Test;
 
 import java.io.File;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.PrintWriter;
 import java.sql.Connection;
 import java.util.ArrayList;
@@ -39,12 +36,9 @@ import static org.briarproject.bramble.test.TestUtils.deleteTestDirectory;
 import static org.briarproject.bramble.test.TestUtils.getAuthor;
 import static org.briarproject.bramble.test.TestUtils.getGroup;
 import static org.briarproject.bramble.test.TestUtils.getLocalAuthor;
-import static org.briarproject.bramble.test.TestUtils.getMean;
-import static org.briarproject.bramble.test.TestUtils.getMedian;
 import static org.briarproject.bramble.test.TestUtils.getMessage;
 import static org.briarproject.bramble.test.TestUtils.getRandomBytes;
 import static org.briarproject.bramble.test.TestUtils.getRandomId;
-import static org.briarproject.bramble.test.TestUtils.getStandardDeviation;
 import static org.briarproject.bramble.test.TestUtils.getTestDirectory;
 import static org.briarproject.bramble.test.UTest.Result.INCONCLUSIVE;
 import static org.briarproject.bramble.test.UTest.Z_CRITICAL_0_1;
@@ -54,7 +48,7 @@ import static org.junit.Assert.assertTrue;
 public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
 
 	private static final int ONE_MEGABYTE = 1024 * 1024;
-	private static final int MAX_SIZE = 100 * ONE_MEGABYTE;
+	static final int MAX_SIZE = 100 * ONE_MEGABYTE;
 
 	/**
 	 * How many contacts to simulate.
@@ -90,12 +84,19 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
 	private static final int OFFERED_MESSAGES_PER_CONTACT = 100;
 
 	/**
-	 * How many times to run each benchmark while measuring.
+	 * How many benchmark iterations to run in each block.
 	 */
-	private static final int MEASUREMENT_ITERATIONS = 100;
+	private static final int ITERATIONS_PER_BLOCK = 10;
 
-	private final File testDir = getTestDirectory();
+	/**
-	private final Random random = new Random();
+	 * How many blocks must be similar before we conclude a steady state has
+	 * been reached.
+	 */
+	private static final int STEADY_STATE_BLOCKS = 5;
+
+	protected final File testDir = getTestDirectory();
+	private final File resultsFile = new File(getTestName() + ".tsv");
+	protected final Random random = new Random();
 
 	private LocalAuthor localAuthor;
 	private List<ClientId> clientIds;
@@ -108,8 +109,8 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
 
 	protected abstract String getTestName();
 
-	protected abstract JdbcDatabase createDatabase(DatabaseConfig config,
+	protected abstract void benchmark(String name,
-			Clock clock);
+			BenchmarkTask<Database<Connection>> task) throws Exception;
 
 	@Before
 	public void setUp() {
@@ -522,48 +523,7 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
 		return list.get(random.nextInt(list.size()));
 	}
 
-	private void benchmark(String name,
+	void populateDatabase(Database<Connection> db) throws DbException {
-			BenchmarkTask<Database<Connection>> task) throws Exception {
-		deleteTestDirectory(testDir);
-		Database<Connection> db = openDatabase();
-		populateDatabase(db);
-		db.close();
-		db = openDatabase();
-		// Measure the first iteration
-		long start = System.nanoTime();
-		task.run(db);
-		long firstDuration = System.nanoTime() - start;
-		// Measure blocks of iterations until we reach a steady state
-		List<Double> oldDurations = measureBlock(db, task);
-		List<Double> durations = measureBlock(db, task);
-		int blocks = 2;
-		while (UTest.test(oldDurations, durations, Z_CRITICAL_0_1)
-				!= INCONCLUSIVE) {
-			oldDurations = durations;
-			durations = measureBlock(db, task);
-			blocks++;
-		}
-		db.close();
-		String result = String.format("%s\t%d\t%,d\t%,d\t%,d\t%,d", name,
-				blocks, firstDuration, (long) getMean(durations),
-				(long) getMedian(durations),
-				(long) getStandardDeviation(durations));
-		System.out.println(result);
-		File resultsFile = new File(getTestName() + ".tsv");
-		PrintWriter out =
-				new PrintWriter(new FileOutputStream(resultsFile, true), true);
-		out.println(new Date() + "\t" + result);
-		out.close();
-	}
-
-	private Database<Connection> openDatabase() throws DbException {
-		Database<Connection> db = createDatabase(
-				new TestDatabaseConfig(testDir, MAX_SIZE), new SystemClock());
-		db.open();
-		return db;
-	}
-
-	private void populateDatabase(Database<Connection> db) throws DbException {
 		localAuthor = getLocalAuthor();
 		clientIds = new ArrayList<>();
 		contacts = new ArrayList<>();
@@ -640,17 +600,6 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
 		db.commitTransaction(txn);
 	}
 
-	private List<Double> measureBlock(Database<Connection> db,
-			BenchmarkTask<Database<Connection>> task) throws Exception {
-		List<Double> durations = new ArrayList<>(MEASUREMENT_ITERATIONS);
-		for (int i = 0; i < MEASUREMENT_ITERATIONS; i++) {
-			long start = System.nanoTime();
-			task.run(db);
-			durations.add((double) (System.nanoTime() - start));
-		}
-		return durations;
-	}
-
 	private ClientId getClientId() {
 		return new ClientId(getRandomString(CLIENT_ID_LENGTH));
 	}
@@ -664,4 +613,56 @@ public abstract class JdbcDatabasePerformanceTest extends BrambleTestCase {
 		}
 		return meta;
 	}
+
+	long measureOne(Database<Connection> db,
+			BenchmarkTask<Database<Connection>> task) throws Exception {
+		long start = System.nanoTime();
+		task.run(db);
+		return System.nanoTime() - start;
+	}
+
+	private List<Double> measureBlock(Database<Connection> db,
+			BenchmarkTask<Database<Connection>> task) throws Exception {
+		List<Double> durations = new ArrayList<>(ITERATIONS_PER_BLOCK);
+		for (int i = 0; i < ITERATIONS_PER_BLOCK; i++)
+			durations.add((double) measureOne(db, task));
+		return durations;
+	}
+
+	SteadyStateResult measureSteadyState(Database<Connection> db,
+			BenchmarkTask<Database<Connection>> task) throws Exception {
+		List<Double> durations = measureBlock(db, task);
+		int blocks = 1, steadyBlocks = 1;
+		while (steadyBlocks < STEADY_STATE_BLOCKS) {
+			List<Double> prev = durations;
+			durations = measureBlock(db, task);
+			// Compare to the previous block with a large P value, which
+			// decreases our chance of getting an inconclusive result, making
+			// this a conservative test for steady state
+			if (UTest.test(prev, durations, Z_CRITICAL_0_1) == INCONCLUSIVE)
+				steadyBlocks++;
+			else steadyBlocks = 1;
+			blocks++;
+		}
+		return new SteadyStateResult(blocks, durations);
+	}
+
+	void writeResult(String result) throws IOException {
+		System.out.println(result);
+		PrintWriter out =
+				new PrintWriter(new FileOutputStream(resultsFile, true), true);
+		out.println(new Date() + "\t" + result);
+		out.close();
+	}
+
+	static class SteadyStateResult {
+
+		final int blocks;
+		final List<Double> durations;
+
+		SteadyStateResult(int blocks, List<Double> durations) {
+			this.blocks = blocks;
+			this.durations = durations;
+		}
+	}
 }

bramble-core/src/test/java/org/briarproject/bramble/db/JdbcSingleDatabasePerformanceTest.java

@@ -0,0 +1,55 @@
+package org.briarproject.bramble.db;
+
+import org.briarproject.bramble.api.db.DatabaseConfig;
+import org.briarproject.bramble.api.db.DbException;
+import org.briarproject.bramble.api.system.Clock;
+import org.briarproject.bramble.system.SystemClock;
+import org.briarproject.bramble.test.TestDatabaseConfig;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.util.List;
+
+import static org.briarproject.bramble.test.TestUtils.deleteTestDirectory;
+import static org.briarproject.bramble.test.TestUtils.getMean;
+import static org.briarproject.bramble.test.TestUtils.getMedian;
+import static org.briarproject.bramble.test.TestUtils.getStandardDeviation;
+
+public abstract class JdbcSingleDatabasePerformanceTest
+		extends JdbcDatabasePerformanceTest {
+
+	abstract Database<Connection> createDatabase(DatabaseConfig databaseConfig,
+			Clock clock);
+
+	@Override
+	protected void benchmark(String name,
+			BenchmarkTask<Database<Connection>> task) throws Exception {
+		deleteTestDirectory(testDir);
+		Database<Connection> db = openDatabase();
+		populateDatabase(db);
+		db.close();
+		db = openDatabase();
+		// Measure the first iteration
+		long firstDuration = measureOne(db, task);
+		// Measure blocks of iterations until we reach a steady state
+		SteadyStateResult result = measureSteadyState(db, task);
+		db.close();
+		writeResult(name, result.blocks, firstDuration, result.durations);
+	}
+
+	private Database<Connection> openDatabase() throws DbException {
+		Database<Connection> db = createDatabase(
+				new TestDatabaseConfig(testDir, MAX_SIZE), new SystemClock());
+		db.open();
+		return db;
+	}
+
+	private void writeResult(String name, int blocks, long firstDuration,
+			List<Double> durations) throws IOException {
+		String result = String.format("%s\t%d\t%,d\t%,d\t%,d\t%,d", name,
+				blocks, firstDuration, (long) getMean(durations),
+				(long) getMedian(durations),
+				(long) getStandardDeviation(durations));
+		writeResult(result);
+	}
+}