Kaladaran/briar
0
0
Fork 0
mirror of https://code.briarproject.org/briar/briar.git synced 2026-02-14 03:39:05 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch '486-convert-rss-feed-entries-into-briar-blog-posts' into 'master'

Browse Source 
Post new RSS entries into the user's personal blog

Closes #486

See merge request !250
This commit is contained in:
akwizgran
2016-08-03 09:36:55 +00:00
parent 85c9b48196 6454acdaa5
commit e8c398e996
4 changed files with 185 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
briar-api/src/org/briarproject/api/feed/Feed.java
View File

@@ -43,6 +43,11 @@ public class Feed {
this(url, blogId, title, description, author, added, 0L, 0L);
}
public Feed(String url, GroupId blogId, long added) {
this(url, blogId, null, null, null, added, 0L, 0L);
}
public String getUrl() {
return url;
}

11
briar-api/src/org/briarproject/api/feed/FeedConstants.java
View File

@@ -1,13 +1,20 @@
package org.briarproject.api.feed;
import java.util.concurrent.TimeUnit;
import static java.util.concurrent.TimeUnit.MINUTES;
public interface FeedConstants {
/* delay after start before fetching feed, in minutes */
/* delay after start before fetching feed */
int FETCH_DELAY_INITIAL = 1;
/* the interval the feed should be fetched, in minutes */
/* the interval the feed should be fetched */
int FETCH_INTERVAL = 30;
/* the unit that applies to the fetch times */
TimeUnit FETCH_UNIT = MINUTES;
// group metadata keys
String KEY_FEEDS = "feeds";
String KEY_FEED_URL = "feedURL";

1
briar-core/src/org/briarproject/blogs/BlogManagerImpl.java
View File

@@ -191,6 +191,7 @@ class BlogManagerImpl extends BdfIncomingMessageHook implements BlogManager,
@Override
public void removeBlog(Blog b) throws DbException {
// TODO if this gets used, check for RSS feeds posting into this blog
Transaction txn = db.startTransaction(false);
try {
for (RemoveBlogHook hook : removeHooks)

232
briar-core/src/org/briarproject/feed/FeedManagerImpl.java
View File

@@ -8,7 +8,10 @@ import com.rometools.rome.io.SyndFeedInput;
import com.rometools.rome.io.XmlReader;
import org.briarproject.api.FormatException;
import org.briarproject.api.blogs.Blog;
import org.briarproject.api.blogs.BlogManager;
import org.briarproject.api.blogs.BlogPost;
import org.briarproject.api.blogs.BlogPostFactory;
import org.briarproject.api.clients.Client;
import org.briarproject.api.clients.ClientHelper;
import org.briarproject.api.clients.PrivateGroupFactory;
@@ -21,18 +24,27 @@ import org.briarproject.api.db.Transaction;
import org.briarproject.api.feed.Feed;
import org.briarproject.api.feed.FeedManager;
import org.briarproject.api.lifecycle.IoExecutor;
import org.briarproject.api.identity.AuthorId;
import org.briarproject.api.identity.IdentityManager;
import org.briarproject.api.identity.LocalAuthor;
import org.briarproject.api.lifecycle.Service;
import org.briarproject.api.lifecycle.ServiceException;
import org.briarproject.api.sync.ClientId;
import org.briarproject.api.sync.Group;
import org.briarproject.api.sync.GroupId;
import org.briarproject.api.system.Clock;
import org.briarproject.util.StringUtils;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledExecutorService;
@@ -44,10 +56,12 @@ import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.logging.Level.INFO;
import static java.util.logging.Level.WARNING;
import static org.briarproject.api.blogs.BlogConstants.MAX_BLOG_POST_BODY_LENGTH;
import static org.briarproject.api.feed.FeedConstants.FETCH_DELAY_INITIAL;
import static org.briarproject.api.feed.FeedConstants.FETCH_INTERVAL;
import static org.briarproject.api.feed.FeedConstants.FETCH_UNIT;
import static org.briarproject.api.feed.FeedConstants.KEY_FEEDS;
class FeedManagerImpl implements FeedManager, Service, Client {
@@ -65,19 +79,28 @@ class FeedManagerImpl implements FeedManager, Service, Client {
private final DatabaseComponent db;
private final PrivateGroupFactory privateGroupFactory;
private final ClientHelper clientHelper;
private IdentityManager identityManager;
private final BlogManager blogManager;
@Inject
@SuppressWarnings("WeakerAccess")
BlogPostFactory blogPostFactory;
@Inject
@SuppressWarnings("WeakerAccess")
Clock clock;
@Inject
FeedManagerImpl(ScheduledExecutorService feedExecutor,
@IoExecutor Executor ioExecutor, DatabaseComponent db,
PrivateGroupFactory privateGroupFactory, ClientHelper clientHelper,
BlogManager blogManager) {
IdentityManager identityManager, BlogManager blogManager) {
this.feedExecutor = feedExecutor;
this.ioExecutor = ioExecutor;
this.db = db;
this.privateGroupFactory = privateGroupFactory;
this.clientHelper = clientHelper;
this.identityManager = identityManager;
this.blogManager = blogManager;
}
@@ -99,7 +122,7 @@ class FeedManagerImpl implements FeedManager, Service, Client {
}
};
feedExecutor.scheduleWithFixedDelay(fetcher, FETCH_DELAY_INITIAL,
FETCH_INTERVAL, MINUTES);
FETCH_INTERVAL, FETCH_UNIT);
}
@Override
@@ -124,16 +147,11 @@ class FeedManagerImpl implements FeedManager, Service, Client {
@Override
public void addFeed(String url, GroupId g) throws DbException, IOException {
LOG.info("Adding new RSS feed...");
Feed feed;
// TODO check for existing feed?
Feed feed = new Feed(url, g, clock.currentTimeMillis());
try {
SyndFeed f = getSyndFeed(getFeedInputStream(url));
String title = StringUtils.isNullOrEmpty(f.getTitle()) ? null :
f.getTitle();
String description = f.getDescription();
String author = f.getAuthor();
long added = System.currentTimeMillis();
feed = new Feed(url, g, title, description, author, added);
feed = fetchFeed(feed);
} catch (FeedException e) {
throw new IOException(e);
}
@@ -142,7 +160,7 @@ class FeedManagerImpl implements FeedManager, Service, Client {
try {
List<Feed> feeds = getFeeds(txn);
feeds.add(feed);
storeFeeds(feeds);
storeFeeds(txn, feeds);
} finally {
db.endTransaction(txn);
}
@@ -150,6 +168,7 @@ class FeedManagerImpl implements FeedManager, Service, Client {
@Override
public void removeFeed(String url) throws DbException {
LOG.info("Removing RSS feed...");
Transaction txn = db.startTransaction(false);
try {
List<Feed> feeds = getFeeds(txn);
@@ -222,6 +241,11 @@ class FeedManagerImpl implements FeedManager, Service, Client {
storeFeeds(null, feeds);
}
/**
* This method is called periodically from a background service.
* It fetches all available feeds and posts new entries to the respective
* blog.
*/
private void fetchFeeds() {
LOG.info("Updating RSS feeds...");
@@ -238,7 +262,15 @@ class FeedManagerImpl implements FeedManager, Service, Client {
// Fetch and update all feeds
List<Feed> newFeeds = new ArrayList<Feed>(feeds.size());
for (Feed feed : feeds) {
newFeeds.add(fetchFeed(feed));
try {
newFeeds.add(fetchFeed(feed));
} catch (FeedException e) {
if (LOG.isLoggable(WARNING))
LOG.log(WARNING, e.toString(), e);
} catch (IOException e) {
if (LOG.isLoggable(WARNING))
LOG.log(WARNING, e.toString(), e);
}
}
// Store updated feeds
@@ -248,61 +280,45 @@ class FeedManagerImpl implements FeedManager, Service, Client {
if (LOG.isLoggable(WARNING))
LOG.log(WARNING, e.toString(), e);
}
LOG.info("Done updating RSS feeds");
}
private Feed fetchFeed(Feed feed) {
private Feed fetchFeed(Feed feed) throws FeedException, IOException {
if (LOG.isLoggable(INFO))
LOG.info("Fetching feed from " + feed.getUrl());
String title, description, author;
long updated = System.currentTimeMillis();
long updated = clock.currentTimeMillis();
long lastEntryTime = feed.getLastEntryTime();
try {
SyndFeed f = getSyndFeed(getFeedInputStream(feed.getUrl()));
title = f.getTitle();
description = f.getDescription();
author = f.getAuthor();
LOG.info("Title: " + f.getTitle());
LOG.info("Description: " + f.getDescription());
LOG.info("Author: " + f.getAuthor());
LOG.info("Number of Entries: " + f.getEntries().size());
LOG.info("------------------------------");
SyndFeed f = getSyndFeed(getFeedInputStream(feed.getUrl()));
title = StringUtils.isNullOrEmpty(f.getTitle()) ? null : f.getTitle();
if (title != null) title = stripHTML(title);
description = StringUtils.isNullOrEmpty(f.getDescription()) ? null :
f.getDescription();
if (description != null) description = stripHTML(description);
author =
StringUtils.isNullOrEmpty(f.getAuthor()) ? null : f.getAuthor();
if (author != null) author = stripHTML(author);
for (SyndEntry entry : f.getEntries()) {
LOG.info("Entry Title: " + entry.getTitle());
LOG.info("Entry Author: " + entry.getAuthor());
LOG.info("Entry Published Date: " + entry.getPublishedDate());
LOG.info("Entry Updated Date: " + entry.getUpdatedDate());
LOG.info("Entry Link: " + entry.getLink());
LOG.info("Entry URI: " + entry.getUri());
//LOG.info("Entry Description: " + entry.getDescription());
long entryTime;
if (entry.getPublishedDate() != null) {
entryTime = entry.getPublishedDate().getTime();
} else if (entry.getUpdatedDate() != null) {
entryTime = entry.getUpdatedDate().getTime();
} else {
// no time information available, ignore this entry
if (LOG.isLoggable(WARNING))
LOG.warning("Entry has no date: " + entry.getTitle());
continue;
}
if (entryTime > feed.getLastEntryTime()) {
LOG.info("Adding new entry...");
// TODO Pass any new entries down the pipeline to be posted (#486)
for (SyndContent content : entry.getContents()) {
LOG.info("Content: " + content.getValue());
}
if (entryTime > lastEntryTime) lastEntryTime = entryTime;
}
LOG.info("------------------------------");
// sort and add new entries
Collections.sort(f.getEntries(), getEntryComparator());
for (SyndEntry entry : f.getEntries()) {
long entryTime;
if (entry.getPublishedDate() != null) {
entryTime = entry.getPublishedDate().getTime();
} else if (entry.getUpdatedDate() != null) {
entryTime = entry.getUpdatedDate().getTime();
} else {
// no time information available, ignore this entry
if (LOG.isLoggable(WARNING))
LOG.warning("Entry has no date: " + entry.getTitle());
continue;
}
if (entryTime > feed.getLastEntryTime()) {
postEntry(feed, entry);
if (entryTime > lastEntryTime) lastEntryTime = entryTime;
}
} catch (FeedException e) {
if (LOG.isLoggable(WARNING))
LOG.log(WARNING, e.toString(), e);
return feed;
} catch (IOException e) {
if (LOG.isLoggable(WARNING))
LOG.log(WARNING, e.toString(), e);
return feed;
}
return new Feed(feed.getUrl(), feed.getBlogId(), title, description,
author, feed.getAdded(), updated, lastEntryTime);
@@ -313,7 +329,7 @@ class FeedManagerImpl implements FeedManager, Service, Client {
// TODO verify and use local Tor proxy address/port
String proxyHost = "localhost";
int proxyPort = 59050;
Proxy proxy = new Proxy(Proxy.Type.HTTP,
Proxy proxy = new Proxy(Proxy.Type.SOCKS,
new InetSocketAddress(proxyHost, proxyPort));
// Build HTTP Client
@@ -338,6 +354,98 @@ class FeedManagerImpl implements FeedManager, Service, Client {
return input.build(new XmlReader(stream));
}
private void postEntry(Feed feed, SyndEntry entry) {
LOG.info("Adding new entry...");
// build post body
StringBuilder b = new StringBuilder();
if (!StringUtils.isNullOrEmpty(entry.getTitle())) {
b.append(stripHTML(entry.getTitle())).append("\n\n");
}
for (SyndContent content : entry.getContents()) {
// extract content and do a very simple HTML tag stripping
if (content.getValue() != null)
b.append(stripHTML(content.getValue()));
}
if (entry.getContents().size() == 0) {
if (entry.getDescription().getValue() != null)
b.append(stripHTML(entry.getDescription().getValue()));
}
if (!StringUtils.isNullOrEmpty(entry.getAuthor())) {
b.append("\n\n-- ").append(stripHTML(entry.getAuthor()));
}
if (entry.getPublishedDate() != null) {
b.append(" (").append(entry.getPublishedDate().toString())
.append(")");
} else if (entry.getUpdatedDate() != null) {
b.append(" (").append(entry.getUpdatedDate().toString())
.append(")");
}
if (!StringUtils.isNullOrEmpty(entry.getLink())) {
b.append("\n\n").append(stripHTML(entry.getLink()));
}
// get other information for post
GroupId groupId = feed.getBlogId();
long time = clock.currentTimeMillis();
byte[] body = getPostBody(b.toString());
try {
// create and store post
Blog blog = blogManager.getBlog(groupId);
AuthorId authorId = blog.getAuthor().getId();
LocalAuthor author = identityManager.getLocalAuthor(authorId);
BlogPost post = blogPostFactory
.createBlogPost(groupId, null, time, null, author,
"text/plain", body);
blogManager.addLocalPost(post);
} catch (DbException e) {
if (LOG.isLoggable(WARNING))
LOG.log(WARNING, e.toString(), e);
} catch (GeneralSecurityException e) {
if (LOG.isLoggable(WARNING))
LOG.log(WARNING, e.toString(), e);
} catch (FormatException e) {
if (LOG.isLoggable(WARNING))
LOG.log(WARNING, e.toString(), e);
} catch (IllegalArgumentException e) {
// yes even catch this, so we at least get a stacktrace
// and the executor doesn't just die a silent death
if (LOG.isLoggable(WARNING))
LOG.log(WARNING, e.toString(), e);
}
}
private String stripHTML(String s) {
return StringUtils.trim(s.replaceAll("<.*?>", ""));
}
private byte[] getPostBody(String text) {
byte[] body = StringUtils.toUtf8(text);
if (body.length <= MAX_BLOG_POST_BODY_LENGTH) return body;
else return Arrays.copyOfRange(body, 0, MAX_BLOG_POST_BODY_LENGTH - 1);
}
/**
* This Comparator assumes that SyndEntry returns a valid Date either for
* getPublishedDate() or getUpdatedDate().
*/
private Comparator<SyndEntry> getEntryComparator() {
return new Comparator<SyndEntry>() {
@Override
public int compare(SyndEntry e1, SyndEntry e2) {
Date d1 =
e1.getPublishedDate() != null ? e1.getPublishedDate() :
e1.getUpdatedDate();
Date d2 =
e2.getPublishedDate() != null ? e2.getPublishedDate() :
e2.getUpdatedDate();
if (d1.after(d2)) return 1;
if (d1.before(d2)) return -1;
return 0;
}
};
}
private Group getLocalGroup() {
return privateGroupFactory.createLocalGroup(getClientId());
}