From e01e0b57c335490a70e0eee4a6f96ed646b78655 Mon Sep 17 00:00:00 2001 From: akwizgran Date: Tue, 12 Feb 2019 14:33:12 +0000 Subject: [PATCH] Measure performance of DB lookups for parent blocks. --- .../bramble/api/db/DatabaseComponent.java | 3 + .../org/briarproject/bramble/db/Database.java | 3 + .../bramble/db/DatabaseComponentImpl.java | 7 ++ .../briarproject/bramble/db/JdbcDatabase.java | 99 +++++++++++++++++++ .../bramble/db/JdbcDatabaseTest.java | 13 +++ .../briar/android/AndroidComponent.java | 3 + .../android/navdrawer/NavDrawerActivity.java | 1 + .../navdrawer/NavDrawerController.java | 1 + .../navdrawer/NavDrawerControllerImpl.java | 31 +++++- 9 files changed, 159 insertions(+), 2 deletions(-) diff --git a/bramble-api/src/main/java/org/briarproject/bramble/api/db/DatabaseComponent.java b/bramble-api/src/main/java/org/briarproject/bramble/api/db/DatabaseComponent.java index 9ec8b5c7b..3f64699a3 100644 --- a/bramble-api/src/main/java/org/briarproject/bramble/api/db/DatabaseComponent.java +++ b/bramble-api/src/main/java/org/briarproject/bramble/api/db/DatabaseComponent.java @@ -25,6 +25,7 @@ import org.briarproject.bramble.api.transport.KeySetId; import org.briarproject.bramble.api.transport.TransportKeys; import java.util.Collection; +import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -150,6 +151,8 @@ public interface DatabaseComponent { boolean containsLocalAuthor(Transaction txn, AuthorId local) throws DbException; + int countFakes(Transaction txn, List ids) throws DbException; + /** * Deletes the message with the given ID. Unlike * {@link #removeMessage(Transaction, MessageId)}, the message ID, diff --git a/bramble-core/src/main/java/org/briarproject/bramble/db/Database.java b/bramble-core/src/main/java/org/briarproject/bramble/db/Database.java index 6bb428578..1b5c34cde 100644 --- a/bramble-core/src/main/java/org/briarproject/bramble/db/Database.java +++ b/bramble-core/src/main/java/org/briarproject/bramble/db/Database.java @@ -28,6 +28,7 @@ import org.briarproject.bramble.api.transport.KeySetId; import org.briarproject.bramble.api.transport.TransportKeys; import java.util.Collection; +import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -188,6 +189,8 @@ interface Database { boolean containsVisibleMessage(T txn, ContactId c, MessageId m) throws DbException; + int countFakes(T txn, List ids) throws DbException; + /** * Returns the number of messages offered by the given contact. *

diff --git a/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseComponentImpl.java b/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseComponentImpl.java index 97e7670c9..ea58e65f4 100644 --- a/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseComponentImpl.java +++ b/bramble-core/src/main/java/org/briarproject/bramble/db/DatabaseComponentImpl.java @@ -313,6 +313,13 @@ class DatabaseComponentImpl implements DatabaseComponent { return db.containsLocalAuthor(txn, local); } + @Override + public int countFakes(Transaction transaction, List ids) + throws DbException { + T txn = unbox(transaction); + return db.countFakes(txn, ids); + } + @Override public void deleteMessage(Transaction transaction, MessageId m) throws DbException { diff --git a/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java b/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java index 458a55bea..a04a5cff9 100644 --- a/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java +++ b/bramble-core/src/main/java/org/briarproject/bramble/db/JdbcDatabase.java @@ -1,5 +1,6 @@ package org.briarproject.bramble.db; +import org.briarproject.bramble.api.UniqueId; import org.briarproject.bramble.api.contact.Contact; import org.briarproject.bramble.api.contact.ContactId; import org.briarproject.bramble.api.crypto.SecretKey; @@ -47,6 +48,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Random; import java.util.Set; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; @@ -92,6 +94,8 @@ abstract class JdbcDatabase implements Database { private static final int OFFSET_CURR = 0; private static final int OFFSET_NEXT = 1; + private static final int NUM_FAKES = 32 * 1024; + private static final String CREATE_SETTINGS = "CREATE TABLE settings" + " (namespace _STRING NOT NULL," @@ -285,6 +289,20 @@ abstract class JdbcDatabase implements Database { + " REFERENCES contacts (contactId)" + " ON DELETE CASCADE)"; + // In H2 the MEMORY keyword creates a persistent table with in-memory + // indexes, which must be dropped and recreated when reopening the database. + // In HyperSQL it creates a persistent table that's entirely cached in + // memory, which is the default. Dropping and recreating indexes is + // unnecessary. + private static final String CREATE_FAKES = + "CREATE MEMORY TABLE fakes (fakeId _HASH NOT NULL)"; + + private static final String DROP_INDEX_FAKES_BY_FAKE_ID = + "DROP INDEX IF EXISTS fakesByFakeId"; + + private static final String INDEX_FAKES_BY_FAKE_ID = + "CREATE INDEX fakesByFakeId ON fakes (fakeId)"; + private static final String INDEX_CONTACTS_BY_AUTHOR_ID = "CREATE INDEX IF NOT EXISTS contactsByAuthorId" + " ON contacts (authorId)"; @@ -376,6 +394,16 @@ abstract class JdbcDatabase implements Database { txn = startTransaction(); try { storeLastCompacted(txn); + createInMemoryIndexes(txn); + commitTransaction(txn); + } catch (DbException e) { + abortTransaction(txn); + throw e; + } + } else { + txn = startTransaction(); + try { + createInMemoryIndexes(txn); commitTransaction(txn); } catch (DbException e) { abortTransaction(txn); @@ -461,6 +489,7 @@ abstract class JdbcDatabase implements Database { private void createTables(Connection txn) throws DbException { Statement s = null; + PreparedStatement ps = null; try { s = txn.createStatement(); s.executeUpdate(dbTypes.replaceTypes(CREATE_SETTINGS)); @@ -477,9 +506,29 @@ abstract class JdbcDatabase implements Database { s.executeUpdate(dbTypes.replaceTypes(CREATE_TRANSPORTS)); s.executeUpdate(dbTypes.replaceTypes(CREATE_OUTGOING_KEYS)); s.executeUpdate(dbTypes.replaceTypes(CREATE_INCOMING_KEYS)); + s.executeUpdate(dbTypes.replaceTypes(CREATE_FAKES)); s.close(); + Random random = new Random(); + long start = now(); + ps = txn.prepareStatement("INSERT INTO fakes (fakeId) VALUES (?)"); + int inserted = 0; + while (inserted < NUM_FAKES) { + int batchSize = Math.min(1024, NUM_FAKES - inserted); + for (int i = 0; i < batchSize; i++) { + byte[] id = new byte[UniqueId.LENGTH]; + random.nextBytes(id); + ps.setBytes(1, id); + ps.addBatch(); + } + ps.executeBatch(); + inserted += batchSize; + } + ps.close(); + LOG.info("Inserting " + NUM_FAKES + " fakes took " + + (now() - start) + " ms"); } catch (SQLException e) { tryToClose(s, LOG, WARNING); + tryToClose(ps, LOG, WARNING); throw new DbException(e); } } @@ -501,6 +550,21 @@ abstract class JdbcDatabase implements Database { } } + private void createInMemoryIndexes(Connection txn) throws DbException { + Statement s = null; + try { + long start = now(); + s = txn.createStatement(); + s.executeUpdate(dbTypes.replaceTypes(DROP_INDEX_FAKES_BY_FAKE_ID)); + s.executeUpdate(dbTypes.replaceTypes(INDEX_FAKES_BY_FAKE_ID)); + s.close(); + LOG.info("Indexing fakes took " + (now() - start) + " ms"); + } catch (SQLException e) { + tryToClose(s, LOG, WARNING); + throw new DbException(e); + } + } + @Override public Connection startTransaction() throws DbException { Connection txn; @@ -1153,6 +1217,41 @@ abstract class JdbcDatabase implements Database { } } + @Override + public int countFakes(Connection txn, List ids) + throws DbException { + if (ids.isEmpty()) return 0; + PreparedStatement ps = null; + ResultSet rs = null; + try { + long start = now(); + StringBuilder sb = new StringBuilder(); + sb.append("SELECT COUNT (fakeId) FROM fakes WHERE fakeId IN (?"); + for (int i = 1; i < ids.size(); i++) { + sb.append(",?"); + } + sb.append(')'); + String sql = sb.toString(); + ps = txn.prepareStatement(sql); + for (int i = 0; i < ids.size(); i++) { + ps.setBytes(i + 1, ids.get(i)); + } + rs = ps.executeQuery(); + if (!rs.next()) throw new DbException(); + int count = rs.getInt(1); + if (rs.next()) throw new DbException(); + rs.close(); + ps.close(); + LOG.info("Counting " + ids.size() + " fakes took " + + (now() - start) + " ms, found " + count); + return count; + } catch (SQLException e) { + tryToClose(rs, LOG, WARNING); + tryToClose(ps, LOG, WARNING); + throw new DbException(e); + } + } + @Override public int countOfferedMessages(Connection txn, ContactId c) throws DbException { diff --git a/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java index dcfc96fba..6bdb6b7ab 100644 --- a/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java +++ b/bramble-core/src/test/java/org/briarproject/bramble/db/JdbcDatabaseTest.java @@ -131,6 +131,19 @@ public abstract class JdbcDatabaseTest extends BrambleTestCase { assertTrue(testDir.mkdirs()); } + @Test + public void testFakes() throws Exception { + List ids = new ArrayList<>(1024); + for (int i = 0; i < 1024; i++) { + ids.add(getRandomId()); + } + Database db = open(false); + Connection txn = db.startTransaction(); + assertEquals(0, db.countFakes(txn, ids)); + db.commitTransaction(txn); + db.close(); + } + @Test public void testPersistence() throws Exception { // Store some records diff --git a/briar-android/src/main/java/org/briarproject/briar/android/AndroidComponent.java b/briar-android/src/main/java/org/briarproject/briar/android/AndroidComponent.java index 90a76e6da..88f48b8b5 100644 --- a/briar-android/src/main/java/org/briarproject/briar/android/AndroidComponent.java +++ b/briar-android/src/main/java/org/briarproject/briar/android/AndroidComponent.java @@ -12,6 +12,7 @@ import org.briarproject.bramble.api.contact.ContactExchangeTask; import org.briarproject.bramble.api.contact.ContactManager; import org.briarproject.bramble.api.crypto.CryptoExecutor; import org.briarproject.bramble.api.crypto.PasswordStrengthEstimator; +import org.briarproject.bramble.api.db.DatabaseComponent; import org.briarproject.bramble.api.db.DatabaseExecutor; import org.briarproject.bramble.api.event.EventBus; import org.briarproject.bramble.api.identity.IdentityManager; @@ -82,6 +83,8 @@ public interface AndroidComponent @DatabaseExecutor Executor databaseExecutor(); + DatabaseComponent databaseComponent(); + MessageTracker messageTracker(); LifecycleManager lifecycleManager(); diff --git a/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerActivity.java b/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerActivity.java index 67d6d8195..03e0696de 100644 --- a/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerActivity.java +++ b/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerActivity.java @@ -223,6 +223,7 @@ public class NavDrawerActivity extends BriarActivity implements @Override public boolean onNavigationItemSelected(@NonNull MenuItem item) { + controller.countFakes(); drawerLayout.closeDrawer(START); clearBackStack(); if (item.getItemId() == R.id.nav_btn_lock) { diff --git a/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerController.java b/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerController.java index d2d883e90..d19ad32bf 100644 --- a/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerController.java +++ b/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerController.java @@ -21,4 +21,5 @@ public interface NavDrawerController extends ActivityLifecycleController { void shouldAskForDozeWhitelisting(Context ctx, ResultHandler handler); + void countFakes(); } diff --git a/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerControllerImpl.java b/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerControllerImpl.java index 2f851569c..7532ac965 100644 --- a/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerControllerImpl.java +++ b/briar-android/src/main/java/org/briarproject/briar/android/navdrawer/NavDrawerControllerImpl.java @@ -3,6 +3,8 @@ package org.briarproject.briar.android.navdrawer; import android.app.Activity; import android.content.Context; +import org.briarproject.bramble.api.UniqueId; +import org.briarproject.bramble.api.db.DatabaseComponent; import org.briarproject.bramble.api.db.DatabaseExecutor; import org.briarproject.bramble.api.db.DbException; import org.briarproject.bramble.api.event.Event; @@ -21,6 +23,9 @@ import org.briarproject.bramble.api.settings.SettingsManager; import org.briarproject.briar.android.controller.DbControllerImpl; import org.briarproject.briar.android.controller.handler.ResultHandler; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; import java.util.concurrent.Executor; import java.util.logging.Logger; @@ -49,6 +54,7 @@ public class NavDrawerControllerImpl extends DbControllerImpl private static final String EXPIRY_DATE_WARNING = "expiryDateWarning"; private static final String EXPIRY_SHOW_UPDATE = "expiryShowUpdate"; + private final DatabaseComponent db; private final PluginManager pluginManager; private final SettingsManager settingsManager; private final EventBus eventBus; @@ -57,9 +63,11 @@ public class NavDrawerControllerImpl extends DbControllerImpl @Inject NavDrawerControllerImpl(@DatabaseExecutor Executor dbExecutor, - LifecycleManager lifecycleManager, PluginManager pluginManager, - SettingsManager settingsManager, EventBus eventBus) { + DatabaseComponent db, LifecycleManager lifecycleManager, + PluginManager pluginManager, SettingsManager settingsManager, + EventBus eventBus) { super(dbExecutor, lifecycleManager); + this.db = db; this.pluginManager = pluginManager; this.settingsManager = settingsManager; this.eventBus = eventBus; @@ -185,6 +193,25 @@ public class NavDrawerControllerImpl extends DbControllerImpl }); } + @Override + public void countFakes() { + int numFakes = 1024; + Random random = new Random(); + List ids = new ArrayList<>(numFakes); + for (int i = 0; i < numFakes; i++) { + byte[] id = new byte[UniqueId.LENGTH]; + random.nextBytes(id); + ids.add(id); + } + runOnDbThread(() -> { + try { + db.transaction(true, txn -> db.countFakes(txn, ids)); + } catch (DbException e) { + logException(LOG, WARNING, e); + } + }); + } + @Override public boolean isTransportRunning(TransportId transportId) { Plugin plugin = pluginManager.getPlugin(transportId);