Crop camera preview before looking for QR code.

briar-android/src/main/java/org/briarproject/briar/android/keyagreement/CameraView.java

@@ -6,7 +6,6 @@ import android.hardware.Camera.AutoFocusCallback;
 import android.hardware.Camera.CameraInfo;
 import android.hardware.Camera.Parameters;
 import android.hardware.Camera.Size;
-import android.os.Build;
 import android.support.annotation.Nullable;
 import android.support.annotation.UiThread;
 import android.util.AttributeSet;
@@ -22,6 +21,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.logging.Logger;
 
+import static android.hardware.Camera.CameraInfo.CAMERA_FACING_BACK;
 import static android.hardware.Camera.CameraInfo.CAMERA_FACING_FRONT;
 import static android.hardware.Camera.Parameters.FLASH_MODE_OFF;
 import static android.hardware.Camera.Parameters.FOCUS_MODE_AUTO;
@@ -32,6 +32,7 @@ import static android.hardware.Camera.Parameters.FOCUS_MODE_FIXED;
 import static android.hardware.Camera.Parameters.FOCUS_MODE_MACRO;
 import static android.hardware.Camera.Parameters.SCENE_MODE_AUTO;
 import static android.hardware.Camera.Parameters.SCENE_MODE_BARCODE;
+import static android.os.Build.VERSION.SDK_INT;
 import static java.util.logging.Level.INFO;
 import static java.util.logging.Level.WARNING;
 
@@ -41,7 +42,12 @@ import static java.util.logging.Level.WARNING;
 public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 		AutoFocusCallback, View.OnClickListener {
 
+	// Heuristic for the ideal preview size - small previews don't have enough
+	// detail, large previews are slow to decode
+	private static final int IDEAL_PIXELS = 500 * 1000;
+
 	private static final int AUTO_FOCUS_RETRY_DELAY = 5000; // Milliseconds
+
 	private static final Logger LOG =
 			Logger.getLogger(CameraView.class.getName());
 
@@ -49,6 +55,7 @@ public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 
 	@Nullable
 	private Camera camera = null;
+	private int cameraIndex = 0;
 	private PreviewConsumer previewConsumer = null;
 	private Surface surface = null;
 	private int displayOrientation = 0, surfaceWidth = 0, surfaceHeight = 0;
@@ -89,15 +96,32 @@ public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 	}
 
 	@UiThread
-	public void start() throws CameraException {
+	public void start(int rotationDegrees) throws CameraException {
 		LOG.info("Opening camera");
 		try {
-			camera = Camera.open();
+			int cameras = Camera.getNumberOfCameras();
+			if (cameras == 0) throw new CameraException("No camera");
+			// Try to find a back-facing camera
+			for (int i = 0; i < cameras; i++) {
+				CameraInfo info = new CameraInfo();
+				Camera.getCameraInfo(i, info);
+				if (info.facing == CAMERA_FACING_BACK) {
+					LOG.info("Using back-facing camera");
+					camera = Camera.open(i);
+					cameraIndex = i;
+					break;
+				}
+			}
+			// If we can't find a back-facing camera, use a front-facing one
+			if (camera == null) {
+				LOG.info("Using front-facing camera");
+				camera = Camera.open(0);
+				cameraIndex = 0;
+			}
 		} catch (RuntimeException e) {
 			throw new CameraException(e);
 		}
-		if (camera == null) throw new CameraException("No back-facing camera");
-		setDisplayOrientation(0);
+		setDisplayOrientation(rotationDegrees);
 		// Use barcode scene mode if it's available
 		Parameters params = camera.getParameters();
 		params = setSceneMode(camera, params);
@@ -163,7 +187,7 @@ public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 	private void startConsumer() throws CameraException {
 		if (camera == null) throw new CameraException("Camera is null");
 		startAutoFocus();
-		previewConsumer.start(camera);
+		previewConsumer.start(camera, cameraIndex);
 	}
 
 	@UiThread
@@ -202,10 +226,11 @@ public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 	@UiThread
 	private void setDisplayOrientation(int rotationDegrees)
 			throws CameraException {
+		if (camera == null) throw new CameraException("Camera is null");
 		int orientation;
 		CameraInfo info = new CameraInfo();
 		try {
-			Camera.getCameraInfo(0, info);
+			Camera.getCameraInfo(cameraIndex, info);
 		} catch (RuntimeException e) {
 			throw new CameraException(e);
 		}
@@ -215,9 +240,11 @@ public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 		} else {
 			orientation = (info.orientation - rotationDegrees + 360) % 360;
 		}
-		if (LOG.isLoggable(INFO))
-			LOG.info("Display orientation " + orientation + " degrees");
-		if (camera == null) throw new CameraException("Camera is null");
+		if (LOG.isLoggable(INFO)) {
+			LOG.info("Screen rotation " + rotationDegrees
+					+ " degrees, camera orientation " + orientation
+					+ " degrees");
+		}
 		try {
 			camera.setDisplayOrientation(orientation);
 		} catch (RuntimeException e) {
@@ -285,8 +312,7 @@ public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 
 	@UiThread
 	private void setVideoStabilisation(Parameters params) {
-		if (Build.VERSION.SDK_INT >= 15 &&
-				params.isVideoStabilizationSupported()) {
+		if (SDK_INT >= 15 && params.isVideoStabilizationSupported()) {
 			params.setVideoStabilization(true);
 		}
 	}
@@ -313,6 +339,8 @@ public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 	@UiThread
 	private void setPreviewSize(Parameters params) {
 		if (surfaceWidth == 0 || surfaceHeight == 0) return;
+		// Choose a preview size that's close to the aspect ratio of the
+		// surface and close to the ideal size for decoding
 		float idealRatio = (float) surfaceWidth / surfaceHeight;
 		boolean rotatePreview = displayOrientation % 180 == 90;
 		List<Size> sizes = params.getSupportedPreviewSizes();
@@ -323,11 +351,12 @@ public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 			int height = rotatePreview ? size.width : size.height;
 			float ratio = (float) width / height;
 			float stretch = Math.max(ratio / idealRatio, idealRatio / ratio);
-			int pixels = width * height;
-			float score = pixels / stretch;
+			float pixels = width * height;
+			float zoom = Math.max(pixels / IDEAL_PIXELS, IDEAL_PIXELS / pixels);
+			float score = 1 / (stretch * zoom);
 			if (LOG.isLoggable(INFO)) {
 				LOG.info("Size " + size.width + "x" + size.height
-						+ ", stretch " + stretch + ", pixels " + pixels
+						+ ", stretch " + stretch + ", zoom " + zoom
 						+ ", score " + score);
 			}
 			if (bestSize == null || score > bestScore) {
@@ -358,7 +387,7 @@ public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 			} catch (RuntimeException e) {
 				throw new CameraException(e);
 			}
-			if (Build.VERSION.SDK_INT >= 15) {
+			if (SDK_INT >= 15) {
 				LOG.info("Video stabilisation enabled: "
 						+ params.getVideoStabilization());
 			}
@@ -389,8 +418,7 @@ public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 			surface.release();
 		}
 		surface = holder.getSurface();
-		// Start the preview when the camera and the surface are both ready
-		if (camera != null && !previewStarted) startPreview(holder);
+		// We'll start the preview when surfaceChanged() is called
 	}
 
 	@Override
@@ -416,7 +444,7 @@ public class CameraView extends SurfaceView implements SurfaceHolder.Callback,
 		surfaceWidth = w;
 		surfaceHeight = h;
 		if (camera == null) return; // We are stopped
-		stopPreview();
+		if (previewStarted) stopPreview();
 		try {
 			Parameters params = camera.getParameters();
 			setPreviewSize(params);

briar-android/src/main/java/org/briarproject/briar/android/keyagreement/PreviewConsumer.java

@@ -10,7 +10,7 @@ import org.briarproject.bramble.api.nullsafety.NotNullByDefault;
 interface PreviewConsumer {
 
 	@UiThread
-	void start(Camera camera);
+	void start(Camera camera, int cameraIndex);
 
 	@UiThread
 	void stop();

briar-android/src/main/java/org/briarproject/briar/android/keyagreement/QrCodeDecoder.java

@@ -1,6 +1,7 @@
 package org.briarproject.briar.android.keyagreement;
 
 import android.hardware.Camera;
+import android.hardware.Camera.CameraInfo;
 import android.hardware.Camera.PreviewCallback;
 import android.hardware.Camera.Size;
 import android.os.AsyncTask;
@@ -36,20 +37,23 @@ class QrCodeDecoder implements PreviewConsumer, PreviewCallback {
 	private final ResultCallback callback;
 
 	private Camera camera = null;
+	private int cameraIndex = 0;
 
 	QrCodeDecoder(ResultCallback callback) {
 		this.callback = callback;
 	}
 
 	@Override
-	public void start(Camera camera) {
+	public void start(Camera camera, int cameraIndex) {
 		this.camera = camera;
+		this.cameraIndex = cameraIndex;
 		askForPreviewFrame();
 	}
 
 	@Override
 	public void stop() {
 		camera = null;
+		cameraIndex = 0;
 	}
 
 	@UiThread
@@ -61,45 +65,64 @@ class QrCodeDecoder implements PreviewConsumer, PreviewCallback {
 	@Override
 	public void onPreviewFrame(byte[] data, Camera camera) {
 		if (camera == this.camera) {
+			LOG.info("Got preview frame");
 			try {
 				Size size = camera.getParameters().getPreviewSize();
-				new DecoderTask(data, size.width, size.height).execute();
+				// The preview should be in NV21 format: width * height bytes of
+				// Y followed by width * height / 2 bytes of interleaved U and V
+				if (data.length == size.width * size.height * 3 / 2) {
+					CameraInfo info = new CameraInfo();
+					Camera.getCameraInfo(cameraIndex, info);
+					new DecoderTask(data, size.width, size.height,
+							info.orientation).execute();
+				} else {
+					// Camera parameters have changed - ask for a new preview
+					LOG.info("Preview size does not match camera parameters");
+					askForPreviewFrame();
+				}
 			} catch (RuntimeException e) {
 				LOG.log(WARNING, "Error getting camera parameters.", e);
 			}
+		} else {
+			LOG.info("Camera has changed, ignoring preview frame");
 		}
 	}
 
 	private class DecoderTask extends AsyncTask<Void, Void, Void> {
 
 		private final byte[] data;
-		private final int width, height;
+		private final int width, height, orientation;
 
-		private DecoderTask(byte[] data, int width, int height) {
+		private DecoderTask(byte[] data, int width, int height,
+				int orientation) {
 			this.data = data;
 			this.width = width;
 			this.height = height;
+			this.orientation = orientation;
 		}
 
 		@Override
 		protected Void doInBackground(Void... params) {
 			long now = System.currentTimeMillis();
-			LuminanceSource src = new PlanarYUVLuminanceSource(data, width,
-					height, 0, 0, width, height, false);
-			BinaryBitmap bitmap = new BinaryBitmap(new HybridBinarizer(src));
+			BinaryBitmap bitmap = binarize(data, width, height, orientation);
 			Result result = null;
 			try {
 				result = reader.decode(bitmap);
+				long duration = System.currentTimeMillis() - now;
+				if (LOG.isLoggable(INFO))
+					LOG.info("Decoding barcode took " + duration + " ms");
 			} catch (ReaderException e) {
-				return null; // No barcode found
+				// No barcode found
+				long duration = System.currentTimeMillis() - now;
+				if (LOG.isLoggable(INFO))
+					LOG.info("No barcode found after " + duration + " ms");
+				return null;
 			} catch (RuntimeException e) {
-				return null; // Preview data did not match width and height
+				LOG.warning("Invalid preview frame");
+				return null;
 			} finally {
 				reader.reset();
 			}
-			long duration = System.currentTimeMillis() - now;
-			if (LOG.isLoggable(INFO))
-				LOG.info("Decoding barcode took " + duration + " ms");
 			callback.handleResult(result);
 			return null;
 		}
@@ -110,6 +133,19 @@ class QrCodeDecoder implements PreviewConsumer, PreviewCallback {
 		}
 	}
 
+	private static BinaryBitmap binarize(byte[] data, int width, int height,
+			int orientation) {
+		// Crop to a square at the top (portrait) or left (landscape) of the
+		// screen - this will be faster to decode and should include
+		// everything visible in the viewfinder
+		int crop = Math.min(width, height);
+		int left = orientation >= 180 ? width - crop : 0;
+		int top = orientation >= 180 ? height - crop : 0;
+		LuminanceSource src = new PlanarYUVLuminanceSource(data, width,
+				height, left, top, crop, crop, false);
+		return new BinaryBitmap(new HybridBinarizer(src));
+	}
+
 	@NotNullByDefault
 	interface ResultCallback {
 

briar-android/src/main/java/org/briarproject/briar/android/keyagreement/QrCodeUtils.java

@@ -32,21 +32,24 @@ class QrCodeUtils {
 			// Generate QR code
 			BitMatrix encoded = new QRCodeWriter().encode(input, QR_CODE,
 					smallestDimen, smallestDimen);
-			// Convert QR code to Bitmap
-			int width = encoded.getWidth();
-			int height = encoded.getHeight();
-			int[] pixels = new int[width * height];
-			for (int x = 0; x < width; x++) {
-				for (int y = 0; y < height; y++) {
-					pixels[y * width + x] = encoded.get(x, y) ? BLACK : WHITE;
-				}
-			}
-			Bitmap qr = Bitmap.createBitmap(width, height, ARGB_8888);
-			qr.setPixels(pixels, 0, width, 0, 0, width, height);
-			return qr;
+			return renderQrCode(encoded);
 		} catch (WriterException e) {
 			if (LOG.isLoggable(WARNING)) LOG.log(WARNING, e.toString(), e);
 			return null;
 		}
 	}
+
+	private static Bitmap renderQrCode(BitMatrix matrix) {
+		int width = matrix.getWidth();
+		int height = matrix.getHeight();
+		int[] pixels = new int[width * height];
+		for (int x = 0; x < width; x++) {
+			for (int y = 0; y < height; y++) {
+				pixels[y * width + x] = matrix.get(x, y) ? BLACK : WHITE;
+			}
+		}
+		Bitmap qr = Bitmap.createBitmap(width, height, ARGB_8888);
+		qr.setPixels(pixels, 0, width, 0, 0, width, height);
+		return qr;
+	}
 }

briar-android/src/main/java/org/briarproject/briar/android/keyagreement/ShowQrCodeFragment.java

@@ -11,7 +11,9 @@ import android.os.Bundle;
 import android.support.annotation.UiThread;
 import android.util.Base64;
 import android.util.DisplayMetrics;
+import android.view.Display;
 import android.view.LayoutInflater;
+import android.view.Surface;
 import android.view.View;
 import android.view.ViewGroup;
 import android.view.animation.AlphaAnimation;
@@ -144,7 +146,7 @@ public class ShowQrCodeFragment extends BaseEventFragment
 	public void onStart() {
 		super.onStart();
 		try {
-			cameraView.start();
+			cameraView.start(getScreenRotationDegrees());
 		} catch (CameraException e) {
 			logCameraExceptionAndFinish(e);
 		}
@@ -164,6 +166,22 @@ public class ShowQrCodeFragment extends BaseEventFragment
 		}
 	}
 
+	private int getScreenRotationDegrees() {
+		Display d = getActivity().getWindowManager().getDefaultDisplay();
+		switch (d.getRotation()) {
+			case Surface.ROTATION_0:
+				return 0;
+			case Surface.ROTATION_90:
+				return 90;
+			case Surface.ROTATION_180:
+				return 180;
+			case Surface.ROTATION_270:
+				return 270;
+			default:
+				throw new AssertionError();
+		}
+	}
+
 	@Override
 	public void onStop() {
 		super.onStop();
@@ -219,10 +237,13 @@ public class ShowQrCodeFragment extends BaseEventFragment
 			if (LOG.isLoggable(INFO))
 				LOG.info("Remote payload is " + encoded.length + " bytes");
 			Payload remotePayload = payloadParser.parse(encoded);
+			cameraView.stop();
 			cameraView.setVisibility(INVISIBLE);
 			statusView.setVisibility(VISIBLE);
 			status.setText(R.string.connecting_to_device);
 			task.connectAndRunProtocol(remotePayload);
+		} catch (CameraException e) {
+			logCameraExceptionAndFinish(e);
 		} catch (IOException | IllegalArgumentException e) {
 			// TODO show failure
 			Toast.makeText(getActivity(), R.string.qr_code_invalid,