Merge branch '569-convert-html-to-plain-text-safely-and-readably' into 'master'

Clean HTML from RSS feeds with Jsoup and Show Link Warning

This adds HTML cleaning with the Jsoup library based on a whitelist. The resulting HTML is then used to create a `Spannable` in Android. This spannable is traversed and URLs are replaced by a custom dialog fragment that shows the following warning.

![device-2016-09-07-154936](/uploads/87a9800bde2676b0f2ddf5a894aa04eb/device-2016-09-07-154936.png)

Closes #569

See merge request !311

briar-android/res/layout/fragment_blog_post.xml

@@ -10,7 +10,7 @@
 		android:descendantFocusability="beforeDescendants"
 		android:focusable="true"
 		android:focusableInTouchMode="true">
-		<!-- Above Focusability attributes prevent automatic scroll-down,
+		<!-- Above focusability attributes prevent automatic scroll-down,
 		     because body text is selectable -->
 
 		<include

briar-android/res/layout/fragment_link_dialog.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="utf-8"?>
+<LinearLayout
+	xmlns:android="http://schemas.android.com/apk/res/android"
+	xmlns:tools="http://schemas.android.com/tools"
+	android:layout_width="match_parent"
+	android:layout_height="match_parent"
+	android:orientation="vertical">
+
+	<LinearLayout
+		android:layout_width="wrap_content"
+		android:layout_height="wrap_content"
+		android:orientation="vertical"
+		android:padding="@dimen/margin_large">
+
+		<TextView
+			android:layout_width="match_parent"
+			android:layout_height="wrap_content"
+			android:text="@string/link_warning_title"
+			android:textColor="@color/briar_primary"
+			android:textSize="@dimen/text_size_large"
+			android:textStyle="bold"/>
+
+		<TextView
+			android:layout_width="match_parent"
+			android:layout_height="wrap_content"
+			android:layout_marginTop="@dimen/margin_large"
+			android:text="@string/link_warning_intro"
+			android:textColor="@color/briar_primary"
+			android:textSize="@dimen/text_size_medium"/>
+
+		<TextView
+			android:id="@+id/urlView"
+			android:layout_width="match_parent"
+			android:layout_height="wrap_content"
+			android:layout_marginTop="@dimen/margin_large"
+			android:textIsSelectable="true"
+			android:typeface="monospace"
+			tools:text="http://very.bad.site.com"/>
+
+		<TextView
+			android:layout_width="match_parent"
+			android:layout_height="wrap_content"
+			android:layout_marginTop="@dimen/margin_large"
+			android:text="@string/link_warning_text"
+			android:textColor="@color/briar_primary"
+			android:textSize="@dimen/text_size_medium"/>
+
+	</LinearLayout>
+
+	<LinearLayout
+		android:layout_width="match_parent"
+		android:layout_height="wrap_content"
+		android:orientation="horizontal">
+
+		<Button
+			android:id="@+id/cancelButton"
+			style="@style/BriarButtonFlat.Positive"
+			android:layout_width="wrap_content"
+			android:layout_height="wrap_content"
+			android:layout_weight="0.5"
+			android:text="@string/cancel"/>
+
+		<Button
+			android:id="@+id/openButton"
+			style="@style/BriarButtonFlat.Negative"
+			android:layout_width="wrap_content"
+			android:layout_height="wrap_content"
+			android:layout_weight="0.5"
+			android:text="@string/link_warning_open_link"/>
+
+	</LinearLayout>
+
+</LinearLayout>

briar-android/res/values/strings.xml

@@ -308,6 +308,12 @@
 	<string name="feedback_settings_title">Feedback</string>
 	<string name="send_feedback">Send feedback</string>
 
+	<!-- Link Warning -->
+	<string name="link_warning_title">Link Warning</string>
+	<string name="link_warning_intro">You are about to open the following link with an external app.</string>
+	<string name="link_warning_text">This can be used to identify you. Think about whether you trust the person that sent you this link and consider opening it with Orfox.</string>
+	<string name="link_warning_open_link">Open Link</string>
+
 	<!-- Multiple Identities -->
 	<string name="anonymous">Anonymous</string>
 	<string name="new_identity_title">New Identity</string>

briar-android/src/org/briarproject/android/blogs/BlogPostViewHolder.java

@@ -8,6 +8,7 @@ import android.support.v4.app.ActivityCompat;
 import android.support.v4.app.ActivityOptionsCompat;
 import android.support.v4.view.ViewCompat;
 import android.support.v7.widget.RecyclerView;
+import android.text.Spanned;
 import android.view.LayoutInflater;
 import android.view.View;
 import android.view.View.OnClickListener;
@@ -30,6 +31,8 @@ import static org.briarproject.android.BriarActivity.GROUP_ID;
 import static org.briarproject.android.blogs.BasePostPagerFragment.POST_ID;
 import static org.briarproject.android.util.AndroidUtils.TEASER_LENGTH;
 import static org.briarproject.android.util.AndroidUtils.getTeaser;
+import static org.briarproject.android.util.AndroidUtils.getSpanned;
+import static org.briarproject.android.util.AndroidUtils.makeLinksClickable;
 import static org.briarproject.api.blogs.MessageType.POST;
 
 @UiThread
@@ -108,15 +111,17 @@ class BlogPostViewHolder extends RecyclerView.ViewHolder {
 		}
 
 		// post body
-		CharSequence bodyText = item.getBody();
+		Spanned bodyText = getSpanned(item.getBody());
 		if (listener == null) {
+			body.setText(bodyText);
 			body.setTextIsSelectable(true);
+			makeLinksClickable(body);
 		} else {
 			body.setTextIsSelectable(false);
 			if (item.getBody().length() > TEASER_LENGTH)
-				bodyText = getTeaser(ctx, item.getBody());
+				bodyText = getTeaser(ctx, bodyText);
+			body.setText(bodyText);
 		}
-		body.setText(bodyText);
 
 		// reblog button
 		reblogButton.setOnClickListener(new OnClickListener() {

briar-android/src/org/briarproject/android/util/AndroidUtils.java

@@ -6,14 +6,24 @@ import android.content.Context;
 import android.os.Build;
 import android.provider.Settings;
 import android.support.design.widget.TextInputLayout;
+import android.support.v4.app.FragmentManager;
 import android.support.v4.content.ContextCompat;
+import android.support.v7.app.AppCompatActivity;
+import android.text.Html;
 import android.text.Spannable;
 import android.text.SpannableString;
 import android.text.SpannableStringBuilder;
+import android.text.Spanned;
 import android.text.format.DateUtils;
+import android.text.method.LinkMovementMethod;
+import android.text.style.ClickableSpan;
 import android.text.style.ForegroundColorSpan;
+import android.text.style.URLSpan;
+import android.view.View;
+import android.widget.TextView;
 
 import org.briarproject.R;
+import org.briarproject.android.widget.LinkDialogFragment;
 import org.briarproject.util.IoUtils;
 import org.briarproject.util.StringUtils;
 
@@ -36,7 +46,7 @@ import static android.text.format.DateUtils.WEEK_IN_MILLIS;
 public class AndroidUtils {
 
 	public static final long MIN_RESOLUTION = MINUTE_IN_MILLIS;
-	public static final int TEASER_LENGTH = 240;
+	public static final int TEASER_LENGTH = 320;
 
 	// Fake Bluetooth address returned by BluetoothAdapter on API 23 and later
 	private static final String FAKE_BLUETOOTH_ADDRESS = "02:00:00:00:00:00";
@@ -121,13 +131,13 @@ public class AndroidUtils {
 						MIN_RESOLUTION, flags).toString();
 	}
 
-	public static SpannableStringBuilder getTeaser(Context ctx, String body) {
+	public static SpannableStringBuilder getTeaser(Context ctx, Spanned body) {
 		if (body.length() < TEASER_LENGTH)
 			throw new IllegalArgumentException(
 					"String is shorter than TEASER_LENGTH");
 
 		SpannableStringBuilder builder =
-				new SpannableStringBuilder(body.substring(0, TEASER_LENGTH));
+				new SpannableStringBuilder(body.subSequence(0, TEASER_LENGTH));
 		String ellipsis = ctx.getString(R.string.ellipsis);
 		builder.append(ellipsis).append(" ");
 
@@ -142,4 +152,31 @@ public class AndroidUtils {
 		return builder;
 	}
 
+	public static Spanned getSpanned(String s) {
+		return Html.fromHtml(s);
+	}
+
+	public static void makeLinksClickable(TextView v) {
+		SpannableStringBuilder ssb = new SpannableStringBuilder(v.getText());
+		URLSpan[] spans = ssb.getSpans(0, ssb.length(), URLSpan.class);
+		for (URLSpan span : spans) {
+			int start = ssb.getSpanStart(span);
+			int end = ssb.getSpanEnd(span);
+			final String url = span.getURL();
+			ssb.removeSpan(span);
+			ClickableSpan cSpan = new ClickableSpan() {
+				@Override
+				public void onClick(View v2) {
+					LinkDialogFragment f = LinkDialogFragment.newInstance(url);
+					FragmentManager fm = ((AppCompatActivity) v2.getContext())
+							.getSupportFragmentManager();
+					f.show(fm, f.getUniqueTag());
+				}
+			};
+			ssb.setSpan(cSpan, start, end, 0);
+		}
+		v.setText(ssb);
+		v.setMovementMethod(ArticleMovementMethod.getInstance());
+	}
+
 }

briar-android/src/org/briarproject/android/util/ArticleMovementMethod.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2006 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.briarproject.android.util;
+
+import android.text.Layout;
+import android.text.Spannable;
+import android.text.method.ArrowKeyMovementMethod;
+import android.text.method.MovementMethod;
+import android.text.style.ClickableSpan;
+import android.view.MotionEvent;
+import android.widget.TextView;
+
+public class ArticleMovementMethod extends ArrowKeyMovementMethod {
+
+	private static ArticleMovementMethod sInstance;
+
+	public static MovementMethod getInstance() {
+		if (sInstance == null) {
+			sInstance = new ArticleMovementMethod();
+		}
+		return sInstance;
+	}
+
+	@Override
+	public boolean onTouchEvent(TextView widget, Spannable buffer,
+			MotionEvent event) {
+		int action = event.getAction();
+
+		if (action == MotionEvent.ACTION_UP) {
+			int x = (int) event.getX();
+			int y = (int) event.getY();
+
+			x -= widget.getTotalPaddingLeft();
+			y -= widget.getTotalPaddingTop();
+
+			x += widget.getScrollX();
+			y += widget.getScrollY();
+
+			Layout layout = widget.getLayout();
+			int line = layout.getLineForVertical(y);
+			int off = layout.getOffsetForHorizontal(line, x);
+
+			ClickableSpan[] link =
+					buffer.getSpans(off, off, ClickableSpan.class);
+
+			if (link.length != 0) {
+				link[0].onClick(widget);
+			}
+		}
+		return super.onTouchEvent(widget, buffer, event);
+	}
+
+}

briar-android/src/org/briarproject/android/widget/LinkDialogFragment.java

@@ -0,0 +1,88 @@
+package org.briarproject.android.widget;
+
+
+import android.content.Intent;
+import android.content.pm.PackageManager;
+import android.net.Uri;
+import android.os.Bundle;
+import android.support.v4.app.DialogFragment;
+import android.view.LayoutInflater;
+import android.view.View;
+import android.view.View.OnClickListener;
+import android.view.ViewGroup;
+import android.widget.Button;
+import android.widget.TextView;
+
+import org.briarproject.R;
+
+import java.util.List;
+
+public class LinkDialogFragment extends DialogFragment {
+
+	private static final String TAG = LinkDialogFragment.class.getName();
+
+	private String url;
+
+	public static LinkDialogFragment newInstance(String url) {
+		LinkDialogFragment f = new LinkDialogFragment();
+
+		Bundle args = new Bundle();
+		args.putString("url", url);
+		f.setArguments(args);
+
+		return f;
+	}
+
+	@Override
+	public void onCreate(Bundle savedInstanceState) {
+		super.onCreate(savedInstanceState);
+
+		url = getArguments().getString("url");
+
+		setStyle(STYLE_NO_TITLE, R.style.BriarDialogTheme);
+	}
+
+	@Override
+	public View onCreateView(LayoutInflater inflater, ViewGroup container,
+			Bundle savedInstanceState) {
+
+		View v = inflater.inflate(R.layout.fragment_link_dialog, container,
+				false);
+
+		TextView urlView = (TextView) v.findViewById(R.id.urlView);
+		urlView.setText(url);
+
+		// prepare normal intent or intent chooser
+		Intent i = new Intent(Intent.ACTION_VIEW, Uri.parse(url));
+		PackageManager packageManager = getContext().getPackageManager();
+		List activities = packageManager.queryIntentActivities(i,
+				PackageManager.MATCH_DEFAULT_ONLY);
+		boolean choice = activities.size() > 1;
+		final Intent intent = choice ? Intent.createChooser(i,
+				getString(R.string.link_warning_open_link)) : i;
+
+		Button openButton = (Button) v.findViewById(R.id.openButton);
+		openButton.setOnClickListener(new OnClickListener() {
+			@Override
+			public void onClick(View v) {
+				startActivity(intent);
+				getDialog().dismiss();
+			}
+		});
+
+		Button cancelButton = (Button) v.findViewById(R.id.cancelButton);
+		cancelButton.setOnClickListener(new OnClickListener() {
+			@Override
+			public void onClick(View v) {
+				getDialog().cancel();
+			}
+		});
+
+		return v;
+	}
+
+	public String getUniqueTag() {
+		return TAG;
+	}
+
+}

briar-core/build.gradle

@@ -13,6 +13,7 @@ dependencies {
 	compile 'org.jdom:jdom2:2.0.6'
 	compile 'org.slf4j:slf4j-api:1.7.21'
 	compile 'com.squareup.okhttp3:okhttp:3.3.1'
+	compile 'org.jsoup:jsoup:1.9.2'
 }
 
 dependencyVerification {
@@ -25,6 +26,7 @@ dependencyVerification {
 			'com.squareup.okhttp3:okhttp:a47f4efa166551cd5acc04f1071d82dafbf05638c21f9ca13068bc6633e3bff6',
 			'com.rometools:rome-utils:2be18a1edc601c31fe49c2000bb5484dd75182309270c2a2561d71888d81587a',
 			'com.squareup.okio:okio:5cfea5afe6c6e441a4dbf6053a07a733b1249d1009382eb44ac2255ccedd0c15',
+			'org.jsoup:jsoup:9c1885f1b182256e06f1e30b8451caed0c0dee96299d6348f968d18b54d0a46a',
 	]
 }
 

briar-core/src/org/briarproject/feed/FeedManagerImpl.java

@@ -42,6 +42,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.security.GeneralSecurityException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Date;
@@ -66,6 +67,9 @@ import static org.briarproject.api.feed.FeedConstants.FETCH_DELAY_INITIAL;
 import static org.briarproject.api.feed.FeedConstants.FETCH_INTERVAL;
 import static org.briarproject.api.feed.FeedConstants.FETCH_UNIT;
 import static org.briarproject.api.feed.FeedConstants.KEY_FEEDS;
+import static org.briarproject.util.HtmlUtils.article;
+import static org.briarproject.util.HtmlUtils.clean;
+import static org.briarproject.util.HtmlUtils.stripAll;
 
 class FeedManagerImpl implements FeedManager, Client, EventListener {
 
@@ -337,13 +341,13 @@ class FeedManagerImpl implements FeedManager, Client, EventListener {
 
 		SyndFeed f = getSyndFeed(getFeedInputStream(feed.getUrl()));
 		title = StringUtils.isNullOrEmpty(f.getTitle()) ? null : f.getTitle();
-		if (title != null) title = stripHTML(title);
+		if (title != null) title = clean(title, stripAll);
 		description = StringUtils.isNullOrEmpty(f.getDescription()) ? null :
 				f.getDescription();
-		if (description != null) description = stripHTML(description);
+		if (description != null) description = clean(description, stripAll);
 		author =
 				StringUtils.isNullOrEmpty(f.getAuthor()) ? null : f.getAuthor();
-		if (author != null) author = stripHTML(author);
+		if (author != null) author = clean(author, stripAll);
 
 		if (f.getEntries().size() == 0)
 			throw new FeedException("Feed has no entries");
@@ -418,23 +422,23 @@ class FeedManagerImpl implements FeedManager, Client, EventListener {
 		// build post body
 		StringBuilder b = new StringBuilder();
 		if (feed.getTitle() != null) {
-			// HTML in feed title was already stripped
+			b.append("<h3>").append(feed.getTitle()).append("</h3>");
-			b.append(feed.getTitle()).append("\n\n");
 		}
 		if (!StringUtils.isNullOrEmpty(entry.getTitle())) {
-			b.append(stripHTML(entry.getTitle())).append("\n\n");
+			b.append("<h1>").append(entry.getTitle()).append("</h1>");
 		}
 		for (SyndContent content : entry.getContents()) {
-			// extract content and do a very simple HTML tag stripping
 			if (content.getValue() != null)
-				b.append(stripHTML(content.getValue()));
+				b.append(content.getValue());
 		}
 		if (entry.getContents().size() == 0) {
-			if (entry.getDescription().getValue() != null)
+			if (entry.getDescription() != null &&
-				b.append(stripHTML(entry.getDescription().getValue()));
+					entry.getDescription().getValue() != null)
+				b.append(entry.getDescription().getValue());
 		}
+		b.append("<p>");
 		if (!StringUtils.isNullOrEmpty(entry.getAuthor())) {
-			b.append("\n\n-- ").append(stripHTML(entry.getAuthor()));
+			b.append("-- ").append(entry.getAuthor());
 		}
 		if (entry.getPublishedDate() != null) {
 			b.append(" (").append(entry.getPublishedDate().toString())
@@ -443,8 +447,11 @@ class FeedManagerImpl implements FeedManager, Client, EventListener {
 			b.append(" (").append(entry.getUpdatedDate().toString())
 					.append(")");
 		}
-		if (!StringUtils.isNullOrEmpty(entry.getLink())) {
+		b.append("</p>");
-			b.append("\n\n").append(stripHTML(entry.getLink()));
+		String link = entry.getLink();
+		if (!StringUtils.isNullOrEmpty(link)) {
+			b.append("<a href=\"").append(link).append("\">").append(link)
+					.append("</a>");
 		}
 
 		// get other information for post
@@ -476,14 +483,12 @@ class FeedManagerImpl implements FeedManager, Client, EventListener {
 		}
 	}
 
-	private String stripHTML(String s) {
-		s = s.replaceAll("<script.*?>(?s).*?</script>", "");
-		return StringUtils.trim(s.replaceAll("<(?s).*?>", ""));
-	}
-
 	private String getPostBody(String text) {
-		if (text.length() <= MAX_BLOG_POST_BODY_LENGTH) return text;
+		text = clean(text, article);
-		else return text.substring(0, MAX_BLOG_POST_BODY_LENGTH);
+		byte[] textBytes = StringUtils.toUtf8(text);
+		if (textBytes.length <= MAX_BLOG_POST_BODY_LENGTH)
+			return text;
+		return StringUtils.fromUtf8(textBytes, 0, MAX_BLOG_POST_BODY_LENGTH);
 	}
 
 	/**

briar-core/src/org/briarproject/util/HtmlUtils.java

@@ -0,0 +1,16 @@
+package org.briarproject.util;
+
+import org.jsoup.Jsoup;
+import org.jsoup.safety.Whitelist;
+
+public class HtmlUtils {
+
+	public static Whitelist stripAll = Whitelist.none();
+	public static Whitelist article =
+			Whitelist.basic().addTags("h1", "h2", "h3", "h4", "h5", "h6");
+
+	public static String clean(String s, Whitelist list) {
+		return Jsoup.clean(s, list);
+	}
+
+}

briar-core/src/org/briarproject/util/PrivacyUtils.java

@@ -1,6 +1,5 @@
 package org.briarproject.util;
 
-import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;