Merge branch '573-hidden-service-for-crash-reports-and-feedback' into 'master'

Server-side code for accepting crash reports and feedback

* Moved some shared code for copying InputStreams to OutputStreams into a utility class
* Modified the  dev report sender to send one report per connection
  * Easier to handle on the server side
  * If the connection fails after sending any reports, they don't need to be resent
  * Tor will reuse the circuit, so it's cheap
* Added server-side code for accepting dev reports
  * We need to protect the server's resources from DoS attacks
  * Reports can't be larger than 1 MB
  * Connections are limited to an average rate of one per minute
  * The rate limiter uses a token bucket to allow bursts of up to 1,000 connections
  * If the rate limit is exceeded, connection attempts will fail - clients will retry next time they sign in
  * The limits can be raised when we move to a bigger server (and when we have some users)


See merge request !288

briar-android/src/org/briarproject/android/util/AndroidUtils.java

@@ -9,7 +9,7 @@ import android.support.design.widget.TextInputLayout;
 import android.text.format.DateUtils;
 
 import org.briarproject.R;
-import org.briarproject.util.FileUtils;
+import org.briarproject.util.IoUtils;
 import org.briarproject.util.StringUtils;
 
 import java.io.File;
@@ -89,7 +89,7 @@ public class AndroidUtils {
 		if (children != null) {
 			for (File child : children) {
 				if (!child.getName().equals("lib"))
-					FileUtils.deleteFileOrDir(child);
+					IoUtils.deleteFileOrDir(child);
 			}
 		}
 	}

briar-android/src/org/briarproject/plugins/tor/TorPlugin.java

@@ -34,8 +34,10 @@ import org.briarproject.api.properties.TransportProperties;
 import org.briarproject.api.reporting.DevReporter;
 import org.briarproject.api.settings.Settings;
 import org.briarproject.api.system.LocationUtils;
+import org.briarproject.util.IoUtils;
 import org.briarproject.util.StringUtils;
 
+import java.io.Closeable;
 import java.io.EOFException;
 import java.io.File;
 import java.io.FileInputStream;
@@ -243,17 +245,17 @@ class TorPlugin implements DuplexPlugin, EventHandler, EventListener {
 			// Unzip the Tor binary to the filesystem
 			in = getTorInputStream();
 			out = new FileOutputStream(torFile);
-			copy(in, out);
+			IoUtils.copy(in, out);
 			// Make the Tor binary executable
 			if (!torFile.setExecutable(true, true)) throw new IOException();
 			// Unzip the GeoIP database to the filesystem
 			in = getGeoIpInputStream();
 			out = new FileOutputStream(geoIpFile);
-			copy(in, out);
+			IoUtils.copy(in, out);
 			// Copy the config file to the filesystem
 			in = getConfigInputStream();
 			out = new FileOutputStream(configFile);
-			copy(in, out);
+			IoUtils.copy(in, out);
 			doneFile.createNewFile();
 		} catch (IOException e) {
 			tryToClose(in);
@@ -284,28 +286,9 @@ class TorPlugin implements DuplexPlugin, EventHandler, EventListener {
 		return appContext.getResources().getAssets().open("torrc");
 	}
 
-	private void copy(InputStream in, OutputStream out) throws IOException {
-		byte[] buf = new byte[4096];
-		while (true) {
-			int read = in.read(buf);
-			if (read == -1) break;
-			out.write(buf, 0, read);
-		}
-		in.close();
-		out.close();
-	}
-
-	private void tryToClose(InputStream in) {
+	private void tryToClose(Closeable c) {
 		try {
-			if (in != null) in.close();
-		} catch (IOException e) {
-			if (LOG.isLoggable(WARNING)) LOG.log(WARNING, e.toString(), e);
-		}
-	}
-
-	private void tryToClose(OutputStream out) {
-		try {
-			if (out != null) out.close();
+			if (c != null) c.close();
 		} catch (IOException e) {
 			if (LOG.isLoggable(WARNING)) LOG.log(WARNING, e.toString(), e);
 		}

briar-api/src/org/briarproject/api/crypto/CryptoComponent.java

@@ -3,7 +3,6 @@ package org.briarproject.api.crypto;
 import org.briarproject.api.TransportId;
 import org.briarproject.api.transport.TransportKeys;
 
-import java.io.IOException;
 import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
 
@@ -163,5 +162,11 @@ public interface CryptoComponent {
 	/**
 	 * Encrypts the given plaintext to the given public key.
 	 */
-	String encryptToKey(PublicKey publicKey, byte[] plaintext);
+	byte[] encryptToKey(PublicKey publicKey, byte[] plaintext);
+
+	/**
+	 * Encodes the given data as a hex string divided into lines of the given
+	 * length. The line terminator is CRLF.
+	 */
+	String asciiArmour(byte[] b, int lineLength);
 }

briar-core/src/org/briarproject/crypto/CryptoComponentImpl.java

@@ -120,20 +120,24 @@ class CryptoComponentImpl implements CryptoComponent {
 		messageEncrypter = new MessageEncrypter(secureRandom);
 	}
 
+	@Override
 	public SecretKey generateSecretKey() {
 		byte[] b = new byte[SecretKey.LENGTH];
 		secureRandom.nextBytes(b);
 		return new SecretKey(b);
 	}
 
+	@Override
 	public MessageDigest getMessageDigest() {
 		return new DigestWrapper(new Blake2sDigest());
 	}
 
+	@Override
 	public PseudoRandom getPseudoRandom(int seed1, int seed2) {
 		return new PseudoRandomImpl(seed1, seed2);
 	}
 
+	@Override
 	public SecureRandom getSecureRandom() {
 		return secureRandom;
 	}
@@ -157,10 +161,12 @@ class CryptoComponentImpl implements CryptoComponent {
 		return secret;
 	}
 
+	@Override
 	public Signature getSignature() {
 		return new SignatureImpl(secureRandom);
 	}
 
+	@Override
 	public KeyPair generateAgreementKeyPair() {
 		AsymmetricCipherKeyPair keyPair =
 				agreementKeyPairGenerator.generateKeyPair();
@@ -176,10 +182,12 @@ class CryptoComponentImpl implements CryptoComponent {
 		return new KeyPair(publicKey, privateKey);
 	}
 
+	@Override
 	public KeyParser getAgreementKeyParser() {
 		return agreementKeyParser;
 	}
 
+	@Override
 	public KeyPair generateSignatureKeyPair() {
 		AsymmetricCipherKeyPair keyPair =
 				signatureKeyPairGenerator.generateKeyPair();
@@ -195,14 +203,17 @@ class CryptoComponentImpl implements CryptoComponent {
 		return new KeyPair(publicKey, privateKey);
 	}
 
+	@Override
 	public KeyParser getSignatureKeyParser() {
 		return signatureKeyParser;
 	}
 
+	@Override
 	public KeyParser getMessageKeyParser() {
 		return messageEncrypter.getKeyParser();
 	}
 
+	@Override
 	public int generateBTInvitationCode() {
 		int codeBytes = (CODE_BITS + 7) / 8;
 		byte[] random = new byte[codeBytes];
@@ -210,21 +221,25 @@ class CryptoComponentImpl implements CryptoComponent {
 		return ByteUtils.readUint(random, CODE_BITS);
 	}
 
+	@Override
 	public int deriveBTConfirmationCode(SecretKey master, boolean alice) {
 		byte[] b = macKdf(master, alice ? BT_A_CONFIRM : BT_B_CONFIRM);
 		return ByteUtils.readUint(b, CODE_BITS);
 	}
 
+	@Override
 	public SecretKey deriveHeaderKey(SecretKey master,
 			boolean alice) {
 		return new SecretKey(macKdf(master, alice ? A_INVITE : B_INVITE));
 	}
 
+	@Override
 	public byte[] deriveSignatureNonce(SecretKey master,
 			boolean alice) {
 		return macKdf(master, alice ? A_SIG_NONCE : B_SIG_NONCE);
 	}
 
+	@Override
 	public byte[] deriveKeyCommitment(byte[] publicKey) {
 		byte[] hash = hash(COMMIT, publicKey);
 		// The output is the first COMMIT_LENGTH bytes of the hash
@@ -233,6 +248,7 @@ class CryptoComponentImpl implements CryptoComponent {
 		return commitment;
 	}
 
+	@Override
 	public SecretKey deriveSharedSecret(byte[] theirPublicKey,
 			KeyPair ourKeyPair, boolean alice) throws GeneralSecurityException {
 		PrivateKey ourPriv = ourKeyPair.getPrivate();
@@ -249,6 +265,7 @@ class CryptoComponentImpl implements CryptoComponent {
 		return new SecretKey(hash(SHARED_SECRET, raw, alicePub, bobPub));
 	}
 
+	@Override
 	public byte[] deriveConfirmationRecord(SecretKey sharedSecret,
 			byte[] theirPayload, byte[] ourPayload, byte[] theirPublicKey,
 			KeyPair ourKeyPair, boolean alice, boolean aliceRecord) {
@@ -271,16 +288,19 @@ class CryptoComponentImpl implements CryptoComponent {
 			return macKdf(ck, bobPayload, bobPub, alicePayload, alicePub);
 	}
 
+	@Override
 	public SecretKey deriveMasterSecret(SecretKey sharedSecret) {
 		return new SecretKey(macKdf(sharedSecret, MASTER_KEY));
 	}
 
+	@Override
 	public SecretKey deriveMasterSecret(byte[] theirPublicKey,
 			KeyPair ourKeyPair, boolean alice) throws GeneralSecurityException {
 		return deriveMasterSecret(deriveSharedSecret(
 				theirPublicKey,ourKeyPair, alice));
 	}
 
+	@Override
 	public TransportKeys deriveTransportKeys(TransportId t,
 			SecretKey master, long rotationPeriod, boolean alice) {
 		// Keys for the previous period are derived from the master secret
@@ -308,6 +328,7 @@ class CryptoComponentImpl implements CryptoComponent {
 		return new TransportKeys(t, inPrev, inCurr, inNext, outCurr);
 	}
 
+	@Override
 	public TransportKeys rotateTransportKeys(TransportKeys k,
 			long rotationPeriod) {
 		if (k.getRotationPeriod() >= rotationPeriod) return k;
@@ -350,6 +371,7 @@ class CryptoComponentImpl implements CryptoComponent {
 		return new SecretKey(macKdf(master, alice ? A_HEADER : B_HEADER, id));
 	}
 
+	@Override
 	public void encodeTag(byte[] tag, SecretKey tagKey, long streamNumber) {
 		if (tag.length < TAG_LENGTH) throw new IllegalArgumentException();
 		if (streamNumber < 0 || streamNumber > MAX_32_BIT_UNSIGNED)
@@ -369,6 +391,7 @@ class CryptoComponentImpl implements CryptoComponent {
 		System.arraycopy(mac, 0, tag, 0, TAG_LENGTH);
 	}
 
+	@Override
 	public byte[] hash(byte[]... inputs) {
 		MessageDigest digest = getMessageDigest();
 		byte[] length = new byte[INT_32_BYTES];
@@ -380,6 +403,7 @@ class CryptoComponentImpl implements CryptoComponent {
 		return digest.digest();
 	}
 
+	@Override
 	public byte[] encryptWithPassword(byte[] input, String password) {
 		AuthenticatedCipher cipher = new XSalsa20Poly1305AuthenticatedCipher();
 		int macBytes = cipher.getMacBytes();
@@ -411,6 +435,7 @@ class CryptoComponentImpl implements CryptoComponent {
 		}
 	}
 
+	@Override
 	public byte[] decryptWithPassword(byte[] input, String password) {
 		AuthenticatedCipher cipher = new XSalsa20Poly1305AuthenticatedCipher();
 		int macBytes = cipher.getMacBytes();
@@ -445,15 +470,20 @@ class CryptoComponentImpl implements CryptoComponent {
 		}
 	}
 
-	public String encryptToKey(PublicKey publicKey, byte[] plaintext) {
+	@Override
+	public byte[] encryptToKey(PublicKey publicKey, byte[] plaintext) {
 		try {
-			byte[] ciphertext = messageEncrypter.encrypt(publicKey, plaintext);
-			return AsciiArmour.wrap(ciphertext, 70);
+			return messageEncrypter.encrypt(publicKey, plaintext);
 		} catch (CryptoException e) {
 			throw new RuntimeException(e);
 		}
 	}
 
+	@Override
+	public String asciiArmour(byte[] b, int lineLength) {
+		return AsciiArmour.wrap(b, lineLength);
+	}
+
 	// Key derivation function based on a pseudo-random function - see
 	// NIST SP 800-108, section 5.1
 	private byte[] macKdf(SecretKey key, byte[]... inputs) {

briar-core/src/org/briarproject/crypto/MessageEncrypter.java

@@ -212,11 +212,12 @@ public class MessageEncrypter {
 	}
 
 	private static String readFully(InputStream in) throws IOException {
+		String newline = System.getProperty("line.separator");
 		StringBuilder stringBuilder = new StringBuilder();
 		Scanner scanner = new Scanner(in);
 		while (scanner.hasNextLine()) {
 			stringBuilder.append(scanner.nextLine());
-			stringBuilder.append(System.lineSeparator());
+			stringBuilder.append(newline);
 		}
 		scanner.close();
 		in.close();

briar-core/src/org/briarproject/reporting/DevReportServer.java

@@ -0,0 +1,145 @@
+package org.briarproject.reporting;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.concurrent.Semaphore;
+
+public class DevReportServer {
+
+	private static final String FILE_PREFIX = "report-";
+	private static final String FILE_SUFFIX = ".enc";
+	private static final int MAX_REPORT_LENGTH = 1024 * 1024;
+	private static final int MIN_REQUEST_INTERVAL_MS = 60 * 1000; // 1 minute
+	private static final int MAX_TOKENS = 1000;
+	private static final int SOCKET_TIMEOUT_MS = 60 * 1000; // 1 minute
+
+	private final InetSocketAddress listenAddress;
+	private final File reportDir;
+
+	private DevReportServer(InetSocketAddress listenAddress, File reportDir) {
+		this.listenAddress = listenAddress;
+		this.reportDir = reportDir;
+	}
+
+	private void listen() throws IOException {
+		ServerSocket ss = new ServerSocket();
+		ss.bind(listenAddress);
+		TokenBucket bucket = new TokenBucket();
+		bucket.start();
+		try {
+			while (true) {
+				Socket s = ss.accept();
+				System.out.println("Incoming connection");
+				bucket.waitForToken();
+				new ReportSaver(s).start();
+			}
+		} catch (InterruptedException e) {
+			System.err.println("Interrupted while listening");
+		} finally {
+			ss.close();
+		}
+	}
+
+	public static void main(String[] args) throws Exception {
+		if (args.length != 3) {
+			System.err.println("Usage:");
+			System.err.println("DevReportServer <addr> <port> <report_dir>");
+			System.exit(1);
+		}
+		int port = Integer.parseInt(args[1]);
+		InetSocketAddress listenAddress = new InetSocketAddress(args[0], port);
+		File reportDir = new File(args[2]);
+		System.out.println("Listening on " + listenAddress);
+		System.out.println("Saving reports to " + reportDir);
+		new DevReportServer(listenAddress, reportDir).listen();
+	}
+
+	private static class TokenBucket extends Thread {
+
+		private final Semaphore semaphore = new Semaphore(MAX_TOKENS);
+
+		private TokenBucket() {
+			setDaemon(true);
+		}
+
+		private void waitForToken() throws InterruptedException {
+			// Wait for a token to become available and remove it
+			semaphore.acquire();
+		}
+
+		@Override
+		public void run() {
+			try {
+				while (true) {
+					// If the bucket isn't full, add a token
+					if (semaphore.availablePermits() < MAX_TOKENS) {
+						System.out.println("Adding token to bucket");
+						semaphore.release();
+					}
+					Thread.sleep(MIN_REQUEST_INTERVAL_MS);
+				}
+			} catch (InterruptedException e) {
+				System.err.println("Interrupted while sleeping");
+			}
+		}
+	}
+
+	private class ReportSaver extends Thread {
+
+		private final Socket socket;
+
+		private ReportSaver(Socket socket) {
+			this.socket = socket;
+			setDaemon(true);
+		}
+
+		@Override
+		public void run() {
+			InputStream in = null;
+			File reportFile = null;
+			OutputStream out = null;
+			try {
+				socket.setSoTimeout(SOCKET_TIMEOUT_MS);
+				in = socket.getInputStream();
+				reportDir.mkdirs();
+				reportFile = File.createTempFile(FILE_PREFIX, FILE_SUFFIX,
+						reportDir);
+				out = new FileOutputStream(reportFile);
+				System.out.println("Saving report to " + reportFile);
+				byte[] b = new byte[4096];
+				int length = 0;
+				while (true) {
+					int read = in.read(b);
+					if (read == -1) break;
+					if (length + read > MAX_REPORT_LENGTH)
+						throw new IOException("Report is too long");
+					out.write(b, 0, read);
+					length += read;
+				}
+				out.flush();
+				System.out.println("Saved " + length + " bytes");
+			} catch (IOException e) {
+				e.printStackTrace();
+				if (reportFile != null) reportFile.delete();
+			} finally {
+				tryToClose(in);
+				tryToClose(out);
+			}
+		}
+
+		private void tryToClose(Closeable c) {
+			try {
+				if (c != null) c.close();
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+}

briar-core/src/org/briarproject/reporting/DevReporterImpl.java

@@ -1,7 +1,5 @@
 package org.briarproject.reporting;
 
-import com.google.common.io.Files;
-
 import net.sourceforge.jsocks.socks.Socks5Proxy;
 import net.sourceforge.jsocks.socks.SocksException;
 import net.sourceforge.jsocks.socks.SocksSocket;
@@ -10,19 +8,21 @@ import org.briarproject.api.crypto.CryptoComponent;
 import org.briarproject.api.reporting.DevConfig;
 import org.briarproject.api.reporting.DevReporter;
 import org.briarproject.util.StringUtils;
+import org.h2.util.IOUtils;
 
+import java.io.Closeable;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.net.Socket;
 import java.net.SocketException;
 import java.net.UnknownHostException;
-import java.nio.charset.Charset;
-import java.util.List;
 import java.util.logging.Logger;
 
 import static java.util.logging.Level.WARNING;
@@ -33,7 +33,7 @@ class DevReporterImpl implements DevReporter {
 			Logger.getLogger(DevReporterImpl.class.getName());
 
 	private static final int SOCKET_TIMEOUT = 30 * 1000; // 30 seconds
-	private static final String CRLF = "\r\n";
+	private static final int LINE_LENGTH = 70;
 
 	private CryptoComponent crypto;
 	private DevConfig devConfig;
@@ -55,16 +55,17 @@ class DevReporterImpl implements DevReporter {
 	@Override
 	public void encryptReportToFile(File reportDir, String filename,
 			String report) throws FileNotFoundException {
-		String encryptedReport =
-				crypto.encryptToKey(devConfig.getDevPublicKey(),
-						StringUtils.toUtf8(report));
+		byte[] plaintext = StringUtils.toUtf8(report);
+		byte[] ciphertext = crypto.encryptToKey(devConfig.getDevPublicKey(),
+				plaintext);
+		String armoured = crypto.asciiArmour(ciphertext, LINE_LENGTH);
 
 		File f = new File(reportDir, filename);
 		PrintWriter writer = null;
 		try {
 			writer = new PrintWriter(
 					new OutputStreamWriter(new FileOutputStream(f)));
-			writer.append(encryptedReport);
+			writer.append(armoured);
 			writer.flush();
 		} finally {
 			if (writer != null)
@@ -78,41 +79,31 @@ class DevReporterImpl implements DevReporter {
 		if (reports == null || reports.length == 0)
 			return; // No reports to send
 
-		LOG.info("Connecting to developers");
-		Socket s;
-		try {
-			s = connectToDevelopers(socksPort);
-		} catch (IOException e) {
-			if (LOG.isLoggable(WARNING))
-				LOG.log(WARNING, "Could not connect to developers", e);
-			return;
-		}
-
 		LOG.info("Sending reports to developers");
-		OutputStream output;
-		PrintWriter writer = null;
-		try {
-			output = s.getOutputStream();
-			writer = new PrintWriter(
-					new OutputStreamWriter(output, "UTF-8"), true);
-			for (File f : reports) {
-				List<String> encryptedReport = Files.readLines(f,
-						Charset.forName("UTF-8"));
-				writer.append(f.getName()).append(CRLF);
-				for (String line : encryptedReport) {
-					writer.append(line).append(CRLF);
-				}
-				writer.append(CRLF);
-				writer.flush();
+		for (File f : reports) {
+			OutputStream out = null;
+			InputStream in = null;
+			try {
+				Socket s = connectToDevelopers(socksPort);
+				out = s.getOutputStream();
+				in = new FileInputStream(f);
+				IOUtils.copy(in, out);
 				f.delete();
+			} catch (IOException e) {
+				LOG.log(WARNING, "Failed to send reports", e);
+				tryToClose(out);
+				tryToClose(in);
+				return;
 			}
-			LOG.info("Reports sent");
+		}
+		LOG.info("Reports sent");
+	}
+
+	private void tryToClose(Closeable c) {
+		try {
+			if (c != null) c.close();
 		} catch (IOException e) {
-			if (LOG.isLoggable(WARNING))
-				LOG.log(WARNING, "Connection to developers failed", e);
-		} finally {
-			if (writer != null)
-				writer.close();
+			if (LOG.isLoggable(WARNING)) LOG.log(WARNING, e.toString(), e);
 		}
 	}
 }

briar-core/src/org/briarproject/util/FileUtils.java

@@ -1,17 +0,0 @@
-package org.briarproject.util;
-
-import java.io.File;
-
-public class FileUtils {
-
-	public static void deleteFileOrDir(File f) {
-		if (f.isFile()) {
-			f.delete();
-		} else if (f.isDirectory()) {
-			File[] children = f.listFiles();
-			if (children != null)
-				for (File child : children) deleteFileOrDir(child);
-			f.delete();
-		}
-	}
-}

briar-core/src/org/briarproject/util/IoUtils.java

@@ -0,0 +1,40 @@
+package org.briarproject.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public class IoUtils {
+
+	public static void deleteFileOrDir(File f) {
+		if (f.isFile()) {
+			f.delete();
+		} else if (f.isDirectory()) {
+			File[] children = f.listFiles();
+			if (children != null)
+				for (File child : children) deleteFileOrDir(child);
+			f.delete();
+		}
+	}
+
+	public static void copy(InputStream in, OutputStream out)
+			throws IOException {
+		byte[] buf = new byte[4096];
+		try {
+			try {
+				while (true) {
+					int read = in.read(buf);
+					if (read == -1) break;
+					out.write(buf, 0, read);
+				}
+				out.flush();
+			} finally {
+				in.close();
+			}
+		} finally {
+			out.close();
+		}
+	}
+
+}

briar-tests/src/org/briarproject/TestUtils.java

@@ -2,7 +2,7 @@ package org.briarproject;
 
 import org.briarproject.api.UniqueId;
 import org.briarproject.api.crypto.SecretKey;
-import org.briarproject.util.FileUtils;
+import org.briarproject.util.IoUtils;
 
 import java.io.File;
 import java.util.Random;
@@ -20,7 +20,7 @@ public class TestUtils {
 	}
 
 	public static void deleteTestDirectory(File testDir) {
-		FileUtils.deleteFileOrDir(testDir);
+		IoUtils.deleteFileOrDir(testDir);
 		testDir.getParentFile().delete(); // Delete if empty
 	}