cominch
13 years ago
parent
a3d5959981
commit
dc468dad01
@ -0,0 +1,116 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>YaCy '#[clientname]#': Content Control</title>
|
||||
#%env/templates/metas.template%#
|
||||
</head>
|
||||
<body id="Settings">
|
||||
#%env/templates/header.template%#
|
||||
#%env/templates/submenuBlacklist.template%#
|
||||
|
||||
<h2>Content Control</h2>
|
||||
|
||||
<form id="contentcontrolsettings" action="ContentControl_p.html" method="post" enctype="multipart/form-data">
|
||||
|
||||
<fieldset><legend id="augmentation">Peer Content Control URL Filter</legend>
|
||||
<p>
|
||||
With this settings you can activate or deactivate content control on this peer.
|
||||
</p>
|
||||
|
||||
|
||||
<dl>
|
||||
|
||||
<dt><label for="content">Use content control filtering:</label></dt>
|
||||
<dd>
|
||||
<input type="checkbox" name="contentcontrolenabled" id="contentcontrolenabled" #(contentcontrolenabled_checked)#:: checked="checked"#(/contentcontrolenabled_checked)# />Enabled<br/>
|
||||
<p class="help">
|
||||
Enables or disables content control.
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">Mandatory default filter list (category):</label></dt>
|
||||
<dd>
|
||||
<input type="text" name="contentcontrolmfl" value="#[contentcontrolmfl]#" size="60" /><br/><br/>
|
||||
<p class="help">
|
||||
Define a category string. If defined, all URLs will be filtered out during crawling and DHT which do not belong to this category.
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">Use this bookmark list:</label></dt>
|
||||
<dd>
|
||||
<input type="text" name="contentcontrolbml" value="#[contentcontrolbml]#" size="60" /><br/><br/>
|
||||
<p class="help">
|
||||
Define a bookmark list. Default: contentcontrol
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
|
||||
</dl>
|
||||
|
||||
|
||||
<input type="submit" name="contentcontrolSettings" value="Submit"/>
|
||||
</fieldset>
|
||||
</form>
|
||||
|
||||
|
||||
|
||||
<form id="contentcontrolExtraSettings" action="ContentControl_p.html" method="post" enctype="multipart/form-data">
|
||||
<fieldset><legend id="urlproxy">Content Control Settings</legend>
|
||||
<p>
|
||||
With this settings you can define the content control settings.
|
||||
</p>
|
||||
|
||||
<dl>
|
||||
|
||||
<dt><label for="content">SMW import to content control list:</label></dt>
|
||||
<dd>
|
||||
<input type="checkbox" name="ccsmwimport" id="ccsmwimport" #(ccsmwimport_checked)#:: checked="checked"#(/ccsmwimport_checked)# />Enabled<br/>
|
||||
<p class="help">
|
||||
Enable or disable constant background synchronisation of content control list from SMW (Semantic Mediawiki). Requires restart!
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">SMW import base URL:</label></dt>
|
||||
<dd>
|
||||
<input type="text" name="ccsmwimporturl" value="#[ccsmwimporturl]#" size="60" /><br/><br/>
|
||||
<p class="help">
|
||||
Define base URL for SMW special page "Ask". Example: http://my.wiki.cc/wiki/Special:Ask
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">SMW import target bookmark list:</label></dt>
|
||||
<dd>
|
||||
<input type="text" name="ccsmwimportlist" value="#[ccsmwimportlist]#" size="60" /><br/><br/>
|
||||
<p class="help">
|
||||
Define import target bookmark list. Default: contentcontrol
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">SMW import default category:</label></dt>
|
||||
<dd>
|
||||
<input type="text" name="ccsmwimportcat" value="#[ccsmwimportcat]#" size="60" /><br/><br/>
|
||||
<p class="help">
|
||||
Define default category which is added to each entry. This category can be defined as mandatory default filter list.
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">Purge content control list on initial sync:</label></dt>
|
||||
<dd>
|
||||
<input type="checkbox" name="ccsmwpurge" id="ccsmwpurge" #(ccsmwpurge_checked)#:: checked="checked"#(/ccsmwpurge_checked)# />Enabled<br/>
|
||||
<p class="help">
|
||||
Purge content control list on initial synchronisation after startup.
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
|
||||
</dl>
|
||||
|
||||
|
||||
<input type="submit" name="contentcontrolExtraSettings" value="Submit"/>
|
||||
</fieldset>
|
||||
</form>
|
||||
|
||||
|
||||
#%env/templates/footer.template%#
|
||||
</body>
|
||||
</html>
|
||||
@ -0,0 +1,77 @@
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import de.anomic.server.serverObjects;
|
||||
import de.anomic.server.serverSwitch;
|
||||
|
||||
public final class ContentControl_p {
|
||||
|
||||
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header,
|
||||
final serverObjects post, final serverSwitch env) {
|
||||
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
if (post != null) {
|
||||
|
||||
if (post.containsKey("contentcontrolExtraSettings")) {
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.baseurl",
|
||||
post.get("ccsmwimporturl"));
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.enabled",
|
||||
"on".equals(post.get("ccsmwimport")) ? true : false);
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.purgelistoninit",
|
||||
"on".equals(post.get("ccsmwpurge")) ? true : false);
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.targetlist",
|
||||
post.get("ccsmwimportlist"));
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.defaultcategory",
|
||||
post.get("ccsmwimportcat"));
|
||||
|
||||
}
|
||||
|
||||
if (post.containsKey("contentcontrolSettings")) {
|
||||
|
||||
env.setConfig("contentcontrol.enabled",
|
||||
"on".equals(post.get("contentcontrolenabled")) ? true : false);
|
||||
|
||||
env.setConfig("contentcontrol.mandatoryfilterlist",
|
||||
post.get("contentcontrolmfl"));
|
||||
|
||||
env.setConfig("contentcontrol.bookmarklist",
|
||||
post.get("contentcontrolbml"));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
prop.putHTML("ccsmwimportcat",
|
||||
env.getConfig("contentcontrol.smwimport.defaultcategory", "yacy"));
|
||||
|
||||
prop.putHTML("ccsmwimportlist",
|
||||
env.getConfig("contentcontrol.smwimport.targetlist", "contentcontrol"));
|
||||
|
||||
prop.put("ccsmwpurge_checked", env.getConfigBool(
|
||||
"contentcontrol.smwimport.purgelistoninit", false) ? "1" : "0");
|
||||
|
||||
prop.putHTML("ccsmwimporturl",
|
||||
env.getConfig("contentcontrol.smwimport.baseurl", ""));
|
||||
|
||||
prop.put("ccsmwimport_checked", env.getConfigBool(
|
||||
"contentcontrol.smwimport.enabled", false) ? "1" : "0");
|
||||
|
||||
|
||||
prop.put("contentcontrolenabled_checked",
|
||||
env.getConfigBool("contentcontrol.enabled", false) ? "1" : "0");
|
||||
|
||||
prop.putHTML("contentcontrolmfl",
|
||||
env.getConfig("contentcontrol.mandatoryfilterlist", "yacy"));
|
||||
|
||||
prop.putHTML("contentcontrolbml",
|
||||
env.getConfig("contentcontrol.bookmarklist", ""));
|
||||
|
||||
// return rewrite properties
|
||||
return prop;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,202 @@
|
||||
package de.anomic.data.ymark;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.search.Switchboard;
|
||||
|
||||
import org.json.simple.parser.ContentHandler;
|
||||
import org.json.simple.parser.JSONParser;
|
||||
import org.json.simple.parser.ParseException;
|
||||
|
||||
public class YMarkSMWJSONImporter implements Runnable, ContentHandler{
|
||||
|
||||
// Importer Variables
|
||||
private final ArrayBlockingQueue<YMarkEntry> bookmarks;
|
||||
private final Reader bmk_file;
|
||||
private final String RootFolder;
|
||||
private final StringBuilder folderstring;
|
||||
private YMarkEntry bmk;
|
||||
private final JSONParser parser;
|
||||
|
||||
private boolean empty = true;
|
||||
private int count = 0;
|
||||
|
||||
// Parser Variables
|
||||
private final StringBuilder value;
|
||||
private final StringBuilder key;
|
||||
private final StringBuilder date;
|
||||
private final HashMap<String,String> obj;
|
||||
|
||||
private Boolean isBookmark;
|
||||
|
||||
public YMarkSMWJSONImporter(final Reader bmk_file, final int queueSize, final String root) {
|
||||
this.bookmarks = new ArrayBlockingQueue<YMarkEntry>(queueSize);
|
||||
this.bmk_file = bmk_file;
|
||||
this.RootFolder = root;
|
||||
this.folderstring = new StringBuilder(YMarkTables.BUFFER_LENGTH);
|
||||
this.folderstring.append(this.RootFolder);
|
||||
this.bmk = new YMarkEntry();
|
||||
|
||||
this.parser = new JSONParser();
|
||||
|
||||
this.value = new StringBuilder(128);
|
||||
this.key = new StringBuilder(16);
|
||||
this.date = new StringBuilder(32);
|
||||
this.obj = new HashMap<String,String>();
|
||||
|
||||
this.isBookmark = false;
|
||||
this.empty = true;
|
||||
this.count = 0;
|
||||
}
|
||||
|
||||
public void startJSON() throws ParseException, IOException {
|
||||
}
|
||||
|
||||
public void endJSON() throws ParseException, IOException {
|
||||
}
|
||||
|
||||
public boolean startArray() throws ParseException, IOException {
|
||||
final String key = this.key.toString();
|
||||
|
||||
if(key.equals("items") ) {
|
||||
|
||||
this.isBookmark = true;
|
||||
this.count = 0;
|
||||
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean endArray() throws ParseException, IOException {
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean startObject() throws ParseException, IOException {
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean endObject() throws ParseException, IOException {
|
||||
|
||||
if(this.isBookmark) {
|
||||
|
||||
if(this.obj.containsKey("category")) {
|
||||
String catstr = obj.get("category");
|
||||
|
||||
HashSet<String> tags = YMarkUtil.keysStringToSet (catstr);
|
||||
|
||||
HashSet<String> categories = YMarkUtil.keysStringToSet("");
|
||||
|
||||
for (String c: tags) {
|
||||
|
||||
c = c.split(":")[1];
|
||||
|
||||
c = c.replace("/", "_");
|
||||
c = c.replace(" ", "_");
|
||||
|
||||
if (!c.equals("") && (!c.equals(" "))) {
|
||||
categories.add ("sc:"+c);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (!Switchboard.getSwitchboard().getConfig("contentcontrol.smwimport.defaultcategory", "").equals("")) {
|
||||
categories.add ("sc:"+Switchboard.getSwitchboard().getConfig("contentcontrol.smwimport.defaultcategory", ""));
|
||||
}
|
||||
|
||||
catstr = YMarkUtil.keySetToString(categories);
|
||||
|
||||
this.bmk.put(YMarkEntry.BOOKMARK.TAGS.key(), catstr);
|
||||
}
|
||||
|
||||
if(this.obj.containsKey("article_has_average_rating")) {
|
||||
this.bmk.put(YMarkEntry.BOOKMARK.STARRATING.key(),obj.get("article_has_average_rating"));
|
||||
}
|
||||
|
||||
this.bmk.put(YMarkEntry.BOOKMARK.TITLE.key(),obj.get("label"));
|
||||
this.bmk.put(YMarkEntry.BOOKMARK.URL.key(),obj.get("url"));
|
||||
if(this.obj.containsKey("filter")) {
|
||||
this.bmk.put(YMarkEntry.BOOKMARK.FILTER.key(),obj.get("filter"));
|
||||
} else {
|
||||
this.bmk.put(YMarkEntry.BOOKMARK.FILTER.key(),"");
|
||||
}
|
||||
try {
|
||||
this.bookmarks.put(this.bmk);
|
||||
this.count++;
|
||||
} catch (InterruptedException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
this.obj.clear();
|
||||
this.bmk = new YMarkEntry();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean startObjectEntry(String key) throws ParseException, IOException {
|
||||
this.key.setLength(0);
|
||||
this.key.append(key);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean primitive(Object value) throws ParseException, IOException {
|
||||
|
||||
this.value.setLength(0);
|
||||
if(value instanceof java.lang.String) {
|
||||
this.value.append((String)value);
|
||||
} else if(value instanceof java.lang.Boolean) {
|
||||
this.value.append((Boolean)value);
|
||||
} else if(value instanceof java.lang.Number) {
|
||||
this.value.append((Number)value);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean endObjectEntry() throws ParseException, IOException {
|
||||
|
||||
final String key = this.key.toString();
|
||||
final String value = this.value.toString();
|
||||
|
||||
this.obj.put(key, value);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public void run() {
|
||||
try {
|
||||
Log.logInfo(YMarkTables.BOOKMARKS_LOG, "SMWJSON Importer run()");
|
||||
this.empty = true;
|
||||
this.parser.parse(this.bmk_file, this, true);
|
||||
|
||||
} catch (IOException e) {
|
||||
Log.logException(e);
|
||||
} catch (ParseException e) {
|
||||
Log.logException(e);
|
||||
} finally {
|
||||
|
||||
try {
|
||||
Log.logInfo(YMarkTables.BOOKMARKS_LOG, "SMWJSON Importer inserted poison pill in queue");
|
||||
this.bookmarks.put(YMarkEntry.POISON);
|
||||
} catch (InterruptedException e) {
|
||||
Log.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public YMarkEntry take() {
|
||||
try {
|
||||
return this.bookmarks.take();
|
||||
} catch (InterruptedException e) {
|
||||
Log.logException(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,114 @@
|
||||
package net.yacy.interaction.contentcontrol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
|
||||
import net.yacy.kelondro.blob.Tables;
|
||||
import net.yacy.kelondro.blob.Tables.Row;
|
||||
import net.yacy.repository.FilterEngine;
|
||||
import net.yacy.search.Switchboard;
|
||||
|
||||
public class ContentControlFilterUpdateThread {
|
||||
|
||||
private Switchboard sb;
|
||||
|
||||
private Boolean locked = false;
|
||||
|
||||
private static FilterEngine networkfilter;
|
||||
|
||||
public ContentControlFilterUpdateThread(final Switchboard sb) {
|
||||
final long time = System.currentTimeMillis();
|
||||
|
||||
this.sb = sb;
|
||||
|
||||
|
||||
if (this.sb.getConfigBool("contentcontrol.smwimport.purgelistoninit",
|
||||
false)) {
|
||||
this.sb.tables.clear(this.sb.getConfig(
|
||||
"contentcontrol.smwimport.targetlist", "contentcontrol"));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
public final void run() {
|
||||
|
||||
if (!locked) {
|
||||
|
||||
locked = true;
|
||||
|
||||
if (this.sb.getConfigBool("contentcontrol.enabled", false) == true) {
|
||||
|
||||
if (!this.sb
|
||||
.getConfig("contentcontrol.mandatoryfilterlist", "")
|
||||
.equals("")) {
|
||||
|
||||
if (sb.tables.bookmarks.dirty) {
|
||||
|
||||
networkfilter = updateFilter();
|
||||
|
||||
sb.tables.bookmarks.dirty = false;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
locked = false;
|
||||
|
||||
}
|
||||
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
private static FilterEngine updateFilter () {
|
||||
|
||||
FilterEngine newfilter = new FilterEngine();
|
||||
|
||||
Switchboard sb = Switchboard.getSwitchboard();
|
||||
|
||||
Iterator<Tables.Row> it;
|
||||
try {
|
||||
it = sb.tables.bookmarks.getBookmarksByTag(
|
||||
sb.getConfig(
|
||||
"contentcontrol.bookmarklist",
|
||||
"contentcontrol"),
|
||||
"^((?!sc:"
|
||||
+ sb
|
||||
.getConfig(
|
||||
"contentcontrol.mandatoryfilterlist",
|
||||
"") + ").*)$");
|
||||
while (it.hasNext()) {
|
||||
Row b = it.next();
|
||||
|
||||
if (!b.get("filter", "").equals("")) {
|
||||
|
||||
newfilter.add(b.get("filter", ""), null);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
return newfilter;
|
||||
}
|
||||
|
||||
|
||||
public static FilterEngine getNetworkFilter() {
|
||||
FilterEngine f = networkfilter;
|
||||
|
||||
if (f != null && f.size() > 0)
|
||||
return f;
|
||||
|
||||
return null;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,253 @@
|
||||
package net.yacy.interaction.contentcontrol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import net.yacy.cora.document.UTF8;
|
||||
import net.yacy.cora.protocol.http.HTTPClient;
|
||||
import net.yacy.cora.util.SpaceExceededException;
|
||||
import net.yacy.kelondro.blob.Tables.Row;
|
||||
import net.yacy.kelondro.data.meta.DigestURI;
|
||||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.search.Switchboard;
|
||||
import de.anomic.data.ymark.YMarkEntry;
|
||||
import de.anomic.data.ymark.YMarkSMWJSONImporter;
|
||||
import de.anomic.data.ymark.YMarkUtil;
|
||||
|
||||
public class ContentControlImportThread {
|
||||
|
||||
private Switchboard sb;
|
||||
|
||||
private Boolean locked = false;
|
||||
|
||||
private String lastsync = "1900-01-01T01:00:00";
|
||||
|
||||
private String currenttimestamp = "1900-01-01T01:00:00";
|
||||
|
||||
private long offset = 0;
|
||||
|
||||
private long limit = 500;
|
||||
|
||||
private long currentmax = 0;
|
||||
|
||||
private boolean runningjob = false;
|
||||
|
||||
public ContentControlImportThread(final Switchboard sb) {
|
||||
final long time = System.currentTimeMillis();
|
||||
|
||||
this.sb = sb;
|
||||
|
||||
|
||||
if (this.sb.getConfigBool("contentcontrol.smwimport.purgelistoninit",
|
||||
false)) {
|
||||
this.sb.tables.clear(this.sb.getConfig(
|
||||
"contentcontrol.smwimport.targetlist", "contentcontrol"));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private final String wikiurlify (String s) {
|
||||
|
||||
String ret = s;
|
||||
|
||||
ret = ret.replace("-", "-2D");
|
||||
ret = ret.replace("+", "-2B");
|
||||
ret = ret.replace(" ", "-20");
|
||||
|
||||
ret = ret.replace("[", "-5B");
|
||||
ret = ret.replace("]", "-5D");
|
||||
|
||||
ret = ret.replace(":", "-3A");
|
||||
ret = ret.replace(">", "-3E");
|
||||
|
||||
ret = ret.replace("?", "-3F");
|
||||
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
public final void run() {
|
||||
|
||||
if (!locked) {
|
||||
|
||||
locked = true;
|
||||
|
||||
if (sb.getConfigBool("contentcontrol.smwimport.enabled", false) == true) {
|
||||
|
||||
if (runningjob) {
|
||||
|
||||
Log.logInfo("CONTENTCONTROL",
|
||||
"CONTENTCONTROL importing max. " + limit
|
||||
+ " elements at " + offset + " of "
|
||||
+ currentmax + ", since "
|
||||
+ currenttimestamp);
|
||||
|
||||
URL bmks_json;
|
||||
|
||||
String currenttimestampurl = wikiurlify (currenttimestamp);
|
||||
|
||||
try {
|
||||
|
||||
if (!sb.getConfig("contentcontrol.smwimport.baseurl",
|
||||
"").equals("")) {
|
||||
|
||||
|
||||
|
||||
bmks_json = new URL(
|
||||
sb.getConfig(
|
||||
"contentcontrol.smwimport.baseurl",
|
||||
"")
|
||||
+ wikiurlify ("/[[Category:Web Page]] [[Modification date::>" +currenttimestamp+ "]]")
|
||||
|
||||
+ wikiurlify ("/?Url/?Filter/?Article has average rating/?Category")
|
||||
+ "/mainlabel%3D"
|
||||
+ "/offset%3D" + offset
|
||||
+ "/limit%3D" + limit
|
||||
+ "/format%3Djson");
|
||||
|
||||
offset += limit;
|
||||
|
||||
if (offset > currentmax) {
|
||||
runningjob = false;
|
||||
}
|
||||
|
||||
InputStreamReader reader = null;
|
||||
try {
|
||||
reader = new InputStreamReader(
|
||||
bmks_json.openStream(), "UTF-8");
|
||||
} catch (Exception e) {
|
||||
|
||||
Log.logException(e);
|
||||
runningjob = false;
|
||||
}
|
||||
|
||||
if (reader != null) {
|
||||
YMarkSMWJSONImporter bookmarkImporter = null;
|
||||
try {
|
||||
bookmarkImporter = new YMarkSMWJSONImporter(
|
||||
reader, 200, "");
|
||||
} catch (final Exception e) {
|
||||
// TODO: display an error message
|
||||
Log.logException(e);
|
||||
runningjob = false;
|
||||
}
|
||||
|
||||
Thread t;
|
||||
YMarkEntry bmk;
|
||||
|
||||
t = new Thread(bookmarkImporter,
|
||||
"YMarks - Network bookmark importer");
|
||||
t.start();
|
||||
|
||||
while ((bmk = bookmarkImporter.take()) != YMarkEntry.POISON) {
|
||||
|
||||
if (bmk == YMarkEntry.EMPTY) {
|
||||
|
||||
runningjob = false;
|
||||
|
||||
} else {
|
||||
|
||||
try {
|
||||
sb.tables.bookmarks.addBookmark(
|
||||
sb.getConfig("contentcontrol.smwimport.targetlist", "contentcontrol"), bmk,
|
||||
true, true);
|
||||
|
||||
} catch (Exception e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
else {
|
||||
|
||||
}
|
||||
|
||||
} catch (MalformedURLException e2) {
|
||||
// TODO Auto-generated catch block
|
||||
e2.printStackTrace();
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
try {
|
||||
|
||||
if (!sb.getConfig("contentcontrol.smwimport.baseurl",
|
||||
"").equals("")) {
|
||||
|
||||
URL bmks_count;
|
||||
|
||||
bmks_count = new URL(
|
||||
sb.getConfig(
|
||||
"contentcontrol.smwimport.baseurl",
|
||||
"")
|
||||
+ wikiurlify ("/[[Category:Web Page]] [[Modification date::>" +lastsync+ "]]")
|
||||
|
||||
|
||||
+ wikiurlify ("/?Url/?Filter/?Article has average rating/?Category")
|
||||
+ "/mainlabel%3D"
|
||||
+ "/format%3Dsupercount");
|
||||
|
||||
String reply = UTF8.String(new HTTPClient()
|
||||
.GETbytes(bmks_count.toString()));
|
||||
|
||||
String overallcount = reply.split(",")[0];
|
||||
|
||||
String lastsyncstring = reply.split(",")[1];
|
||||
|
||||
currentmax = Integer.parseInt(overallcount);
|
||||
|
||||
if (currentmax > 0) {
|
||||
|
||||
Log.logInfo("CONTENTCONTROL",
|
||||
"CONTENTCONTROL import job counts "
|
||||
+ currentmax
|
||||
+ " new elements between "
|
||||
+ lastsync + " and "
|
||||
+ currenttimestamp);
|
||||
|
||||
currenttimestamp = lastsync;
|
||||
|
||||
runningjob = true;
|
||||
lastsync = lastsyncstring;
|
||||
offset = 0;
|
||||
}
|
||||
} else {
|
||||
Log.logWarning("CONTENTCONTROL",
|
||||
"No SMWimport URL defined");
|
||||
}
|
||||
|
||||
} catch (MalformedURLException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
locked = false;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue