This was not used at all (as I know) and was blocking a smooth integration of ivy in the context of an existing JSON parser.pull/533/head
Michael Peter Christen
2 years ago
parent
b54f4ad35f
commit
fc98ca7a9c
@ -1,95 +0,0 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>YaCy '#[clientname]#': Content Control</title>
|
||||
#%env/templates/metas.template%#
|
||||
</head>
|
||||
<body id="Settings">
|
||||
#%env/templates/header.template%#
|
||||
#%env/templates/submenuBlacklist.template%#
|
||||
|
||||
<h2>Content Control</h2>
|
||||
|
||||
<form id="contentcontrolsettings" action="ContentControl_p.html" method="post" enctype="multipart/form-data">
|
||||
|
||||
<fieldset><legend id="augmentation">Peer Content Control URL Filter</legend>
|
||||
<p>
|
||||
With this settings you can activate or deactivate content control on this peer.
|
||||
</p>
|
||||
|
||||
|
||||
<dl>
|
||||
|
||||
<dt><label for="content">Use content control filtering:</label></dt>
|
||||
<dd>
|
||||
<input type="checkbox" name="contentcontrolenabled" id="contentcontrolenabled" #(contentcontrolenabled_checked)#:: checked="checked"#(/contentcontrolenabled_checked)# />Enabled<br/>
|
||||
<p class="help">
|
||||
Enables or disables content control.
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
|
||||
<dt><label for="content">Use this table to create filter:</label></dt>
|
||||
<dd>
|
||||
<input type="text" name="contentcontrolbml" value="#[contentcontrolbml]#" size="60" /><br/><br/>
|
||||
<p class="help">
|
||||
Define a table. Default: contentcontrol
|
||||
</p>
|
||||
</dd>
|
||||
<dt></dt>
|
||||
<dd><input type="submit" name="contentcontrolSettings" value="Submit" class="btn btn-primary"/></dd>
|
||||
</dl>
|
||||
</fieldset>
|
||||
</form>
|
||||
|
||||
|
||||
|
||||
<form id="contentcontrolExtraSettings" action="ContentControl_p.html" method="post" enctype="multipart/form-data">
|
||||
<fieldset><legend id="urlproxy">Content Control SMW Import Settings</legend>
|
||||
<p>
|
||||
With this settings you can define the content control import settings. You can define a <a href="http://wiki.sciety.org/mediawiki/extensions/yacy-smwextension/" target="_blank">Semantic Media Wiki with the appropriate extensions.</a>
|
||||
</p>
|
||||
|
||||
<dl>
|
||||
|
||||
<dt><label for="content">SMW import to content control list:</label></dt>
|
||||
<dd>
|
||||
<input type="checkbox" name="ccsmwimport" id="ccsmwimport" #(ccsmwimport_checked)#:: checked="checked"#(/ccsmwimport_checked)# />Enabled<br/>
|
||||
<p class="help">
|
||||
Enable or disable constant background synchronization of content control list from SMW (Semantic Mediawiki). Requires restart!
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">SMW import base URL:</label></dt>
|
||||
<dd>
|
||||
<input type="text" name="ccsmwimporturl" value="#[ccsmwimporturl]#" size="60" /><br/><br/>
|
||||
<p class="help">
|
||||
Define base URL for SMW special page "Ask". Example: http://my.wiki.cc/wiki/Special:Ask
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">SMW import target table:</label></dt>
|
||||
<dd>
|
||||
<input type="text" name="ccsmwimportlist" value="#[ccsmwimportlist]#" size="60" /><br/><br/>
|
||||
<p class="help">
|
||||
Define import target table. Default: contentcontrol
|
||||
</p>
|
||||
</dd>
|
||||
|
||||
<dt><label for="content">Purge content control list on initial sync:</label></dt>
|
||||
<dd>
|
||||
<input type="checkbox" name="ccsmwpurge" id="ccsmwpurge" #(ccsmwpurge_checked)#:: checked="checked"#(/ccsmwpurge_checked)# />Enabled<br/>
|
||||
<p class="help">
|
||||
Purge content control list on initial synchronisation after startup.
|
||||
</p>
|
||||
</dd>
|
||||
<dt></dt>
|
||||
<dd><input type="submit" name="contentcontrolExtraSettings" value="Submit" class="btn btn-primary"/></dd>
|
||||
</dl>
|
||||
</fieldset>
|
||||
</form>
|
||||
|
||||
|
||||
#%env/templates/footer.template%#
|
||||
</body>
|
||||
</html>
|
||||
@ -1,68 +0,0 @@
|
||||
import net.yacy.cora.protocol.RequestHeader;
|
||||
import net.yacy.server.serverObjects;
|
||||
import net.yacy.server.serverSwitch;
|
||||
|
||||
public final class ContentControl_p {
|
||||
|
||||
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header,
|
||||
final serverObjects post, final serverSwitch env) {
|
||||
|
||||
final serverObjects prop = new serverObjects();
|
||||
|
||||
if (post != null) {
|
||||
|
||||
if (post.containsKey("contentcontrolExtraSettings")) {
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.baseurl",
|
||||
post.get("ccsmwimporturl"));
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.enabled",
|
||||
"on".equals(post.get("ccsmwimport")) ? true : false);
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.purgelistoninit",
|
||||
"on".equals(post.get("ccsmwpurge")) ? true : false);
|
||||
|
||||
env.setConfig("contentcontrol.smwimport.targetlist",
|
||||
post.get("ccsmwimportlist"));
|
||||
|
||||
|
||||
}
|
||||
|
||||
if (post.containsKey("contentcontrolSettings")) {
|
||||
|
||||
env.setConfig("contentcontrol.enabled",
|
||||
"on".equals(post.get("contentcontrolenabled")) ? true : false);
|
||||
|
||||
|
||||
env.setConfig("contentcontrol.bookmarklist",
|
||||
post.get("contentcontrolbml"));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
prop.putHTML("ccsmwimportlist",
|
||||
env.getConfig("contentcontrol.smwimport.targetlist", "contentcontrol"));
|
||||
|
||||
prop.put("ccsmwpurge_checked", env.getConfigBool(
|
||||
"contentcontrol.smwimport.purgelistoninit", false) ? "1" : "0");
|
||||
|
||||
prop.putHTML("ccsmwimporturl",
|
||||
env.getConfig("contentcontrol.smwimport.baseurl", ""));
|
||||
|
||||
prop.put("ccsmwimport_checked", env.getConfigBool(
|
||||
"contentcontrol.smwimport.enabled", false) ? "1" : "0");
|
||||
|
||||
|
||||
prop.put("contentcontrolenabled_checked",
|
||||
env.getConfigBool("contentcontrol.enabled", false) ? "1" : "0");
|
||||
|
||||
prop.putHTML("contentcontrolbml",
|
||||
env.getConfig("contentcontrol.bookmarklist", ""));
|
||||
|
||||
// return rewrite properties
|
||||
return prop;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,90 +0,0 @@
|
||||
package net.yacy.contentcontrol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
|
||||
import net.yacy.kelondro.blob.Tables;
|
||||
import net.yacy.kelondro.blob.Tables.Row;
|
||||
import net.yacy.repository.FilterEngine;
|
||||
import net.yacy.search.Switchboard;
|
||||
|
||||
public class ContentControlFilterUpdateThread implements Runnable {
|
||||
|
||||
private final Switchboard sb;
|
||||
|
||||
private Boolean locked = false;
|
||||
|
||||
private static FilterEngine networkfilter;
|
||||
|
||||
public ContentControlFilterUpdateThread(final Switchboard sb) {
|
||||
|
||||
this.sb = sb;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void run() {
|
||||
|
||||
if (!this.locked) {
|
||||
|
||||
this.locked = true;
|
||||
|
||||
if (this.sb.getConfigBool("contentcontrol.enabled", false) == true) {
|
||||
|
||||
if (SMWListSyncThread.dirty) {
|
||||
|
||||
networkfilter = updateFilter();
|
||||
|
||||
SMWListSyncThread.dirty = false;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
this.locked = false;
|
||||
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
private static FilterEngine updateFilter () {
|
||||
|
||||
FilterEngine newfilter = new FilterEngine();
|
||||
|
||||
Switchboard sb = Switchboard.getSwitchboard();
|
||||
|
||||
Iterator<Tables.Row> it;
|
||||
try {
|
||||
it = sb.tables.iterator(sb.getConfig("contentcontrol.bookmarklist",
|
||||
"contentcontrol"));
|
||||
|
||||
while (it.hasNext()) {
|
||||
Row b = it.next();
|
||||
|
||||
if (!b.get("filter", "").equals("")) {
|
||||
|
||||
newfilter.add(b.get("filter", ""), null);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (final IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
return newfilter;
|
||||
}
|
||||
|
||||
|
||||
public static FilterEngine getNetworkFilter() {
|
||||
FilterEngine f = networkfilter;
|
||||
|
||||
if (f != null && f.size() > 0)
|
||||
return f;
|
||||
|
||||
return null;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,163 +0,0 @@
|
||||
package net.yacy.contentcontrol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
|
||||
import org.json.simple.parser.ContentHandler;
|
||||
import org.json.simple.parser.JSONParser;
|
||||
import org.json.simple.parser.ParseException;
|
||||
|
||||
public class SMWListImporter implements Runnable, ContentHandler{
|
||||
|
||||
// Importer Variables
|
||||
private final ArrayBlockingQueue<SMWListRow> listEntries;
|
||||
private final Reader importFile;
|
||||
|
||||
private SMWListRow row;
|
||||
private final JSONParser parser;
|
||||
|
||||
// Parser Variables
|
||||
private final StringBuilder value;
|
||||
private final StringBuilder key;
|
||||
private final HashMap<String,String> obj;
|
||||
|
||||
private Boolean isElement;
|
||||
|
||||
public SMWListImporter(final Reader importFile, final int queueSize) {
|
||||
this.listEntries = new ArrayBlockingQueue<SMWListRow>(queueSize);
|
||||
this.importFile = importFile;
|
||||
|
||||
this.row = new SMWListRow();
|
||||
|
||||
this.parser = new JSONParser();
|
||||
|
||||
this.value = new StringBuilder(128);
|
||||
this.key = new StringBuilder(16);
|
||||
this.obj = new HashMap<String,String>();
|
||||
|
||||
this.isElement = false;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void startJSON() throws ParseException, IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void endJSON() throws ParseException, IOException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean startArray() throws ParseException, IOException {
|
||||
final String key = this.key.toString();
|
||||
|
||||
if (key.equals("items")) {
|
||||
|
||||
this.isElement = true;
|
||||
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean endArray() throws ParseException, IOException {
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean startObject() throws ParseException, IOException {
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean endObject() throws ParseException, IOException {
|
||||
|
||||
if(this.isElement) {
|
||||
|
||||
for (Entry<String, String> e: this.obj.entrySet()) {
|
||||
this.row.add (e.getKey(), e.getValue());
|
||||
}
|
||||
try {
|
||||
this.listEntries.put(this.row);
|
||||
//this.count++;
|
||||
} catch (final InterruptedException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
this.obj.clear();
|
||||
this.row = new SMWListRow();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean startObjectEntry(String key) throws ParseException, IOException {
|
||||
this.key.setLength(0);
|
||||
this.key.append(key);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean primitive(Object value) throws ParseException, IOException {
|
||||
|
||||
this.value.setLength(0);
|
||||
if(value instanceof java.lang.String) {
|
||||
this.value.append((String)value);
|
||||
} else if(value instanceof java.lang.Boolean) {
|
||||
this.value.append(value);
|
||||
} else if(value instanceof java.lang.Number) {
|
||||
this.value.append(value);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean endObjectEntry() throws ParseException, IOException {
|
||||
|
||||
final String key = this.key.toString();
|
||||
final String value = this.value.toString();
|
||||
|
||||
this.obj.put(key, value);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
ConcurrentLog.info("SMWLISTSYNC", "Importer run()");
|
||||
this.parser.parse(this.importFile, this, true);
|
||||
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
} catch (final ParseException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
} finally {
|
||||
|
||||
try {
|
||||
ConcurrentLog.info("SMWLISTSYNC", "Importer inserted poison pill in queue");
|
||||
this.listEntries.put(SMWListRow.POISON);
|
||||
} catch (final InterruptedException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public SMWListRow take() {
|
||||
try {
|
||||
return this.listEntries.take();
|
||||
} catch (final InterruptedException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,117 +0,0 @@
|
||||
package net.yacy.contentcontrol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.Iterator;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
|
||||
import org.json.simple.JSONArray;
|
||||
import org.json.simple.JSONObject;
|
||||
import org.json.simple.parser.JSONParser;
|
||||
import org.json.simple.parser.ParseException;
|
||||
|
||||
public class SMWListImporterFormatObsolete implements Runnable{
|
||||
|
||||
private final ArrayBlockingQueue<SMWListRow> listEntries;
|
||||
private final Reader importFile;
|
||||
private final JSONParser parser;
|
||||
|
||||
public SMWListImporterFormatObsolete(final Reader importFile, final int queueSize) {
|
||||
this.listEntries = new ArrayBlockingQueue<SMWListRow>(queueSize);
|
||||
this.importFile = importFile;
|
||||
this.parser = new JSONParser();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
ConcurrentLog.info("SMWLISTSYNC", "Importer run()");
|
||||
Object obj = this.parser.parse(this.importFile);
|
||||
|
||||
JSONObject jsonObject = (JSONObject) obj;
|
||||
|
||||
JSONArray items = (JSONArray) jsonObject.get("items");
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<JSONObject> iterator = items.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
this.parseItem (iterator.next());
|
||||
}
|
||||
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
} catch (final ParseException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
} finally {
|
||||
|
||||
try {
|
||||
ConcurrentLog.info("SMWLISTSYNC", "Importer inserted poison pill in queue");
|
||||
this.listEntries.put(SMWListRow.POISON);
|
||||
} catch (final InterruptedException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void parseItem(JSONObject jsonObject) {
|
||||
|
||||
try {
|
||||
SMWListRow row = new SMWListRow();
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<String> iterator = jsonObject.keySet().iterator();
|
||||
|
||||
while (iterator.hasNext()) {
|
||||
String entryKey = iterator.next();
|
||||
|
||||
Object value = jsonObject.get (entryKey);
|
||||
String valueKey = "";
|
||||
|
||||
if (value instanceof java.lang.String) {
|
||||
valueKey = value.toString();
|
||||
} else if (value instanceof JSONArray) {
|
||||
valueKey = jsonListAll ((JSONArray) value);
|
||||
}
|
||||
|
||||
row.add (entryKey, valueKey);
|
||||
}
|
||||
|
||||
this.listEntries.put(row);
|
||||
|
||||
} catch (final Exception e) {
|
||||
ConcurrentLog.info("SMWLISTSYNC", "import of entry failed");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private String jsonListAll(JSONArray value) {
|
||||
String res = "";
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<Object> iterator = value.listIterator();
|
||||
while (iterator.hasNext()) {
|
||||
Object val = iterator.next();
|
||||
res += val.toString()+",";
|
||||
}
|
||||
|
||||
if (res.endsWith (",")) {
|
||||
res = res.substring (0, res.length()-1);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
public SMWListRow take() {
|
||||
try {
|
||||
return this.listEntries.take();
|
||||
} catch (final InterruptedException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
package net.yacy.contentcontrol;
|
||||
|
||||
import net.yacy.kelondro.blob.Tables;
|
||||
|
||||
public class SMWListRow {
|
||||
|
||||
private Tables.Data data;
|
||||
|
||||
public static final SMWListRow POISON = new SMWListRow();
|
||||
public static final SMWListRow EMPTY = new SMWListRow();
|
||||
|
||||
public SMWListRow() {
|
||||
this.data = new Tables.Data();
|
||||
}
|
||||
|
||||
public void add (String key, String value) {
|
||||
this.data.put(key, value);
|
||||
}
|
||||
|
||||
public Tables.Data getData() {
|
||||
return this.data;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,201 +0,0 @@
|
||||
package net.yacy.contentcontrol;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import net.yacy.cora.document.encoding.UTF8;
|
||||
import net.yacy.cora.protocol.ClientIdentification;
|
||||
import net.yacy.cora.protocol.http.HTTPClient;
|
||||
import net.yacy.cora.util.CommonPattern;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.search.Switchboard;
|
||||
|
||||
public class SMWListSyncThread implements Runnable {
|
||||
|
||||
private final Switchboard sb;
|
||||
private Boolean locked = false;
|
||||
private String lastsync = "1900-01-01T01:00:00";
|
||||
private String currenttimestamp = "1900-01-01T01:00:00";
|
||||
private long offset = 0;
|
||||
private final long limit = 500;
|
||||
private long currentmax = 0;
|
||||
private boolean runningjob = false;
|
||||
|
||||
private String targetList;
|
||||
private String parameters;
|
||||
private String query;
|
||||
|
||||
public static Boolean dirty = false;
|
||||
|
||||
public SMWListSyncThread(final Switchboard sb, final String targetList, final String query, final String parameters, final Boolean purgeOnInit) {
|
||||
this.sb = sb;
|
||||
this.targetList = targetList;
|
||||
this.parameters = parameters;
|
||||
this.query = query;
|
||||
if (purgeOnInit) {
|
||||
this.sb.tables.clear(targetList);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private final String wikiurlify (String s) {
|
||||
String ret = s;
|
||||
ret = ret.replace("-", "-2D");
|
||||
ret = ret.replace("+", "-2B");
|
||||
ret = ret.replace(" ", "-20");
|
||||
ret = ret.replace("[", "-5B");
|
||||
ret = ret.replace("]", "-5D");
|
||||
ret = ret.replace(":", "-3A");
|
||||
ret = ret.replace(">", "-3E");
|
||||
ret = ret.replace("?", "-3F");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void run() {
|
||||
|
||||
if (!this.locked) {
|
||||
this.locked = true;
|
||||
if (this.sb.getConfigBool("contentcontrol.smwimport.enabled", false) == true) {
|
||||
|
||||
if (!this.runningjob) {
|
||||
|
||||
// we have to count all new elements first
|
||||
try {
|
||||
if (!this.sb.getConfig("contentcontrol.smwimport.baseurl","").equals("")) {
|
||||
URL urlCount;
|
||||
|
||||
urlCount = new URL(
|
||||
this.sb.getConfig(
|
||||
"contentcontrol.smwimport.baseurl",
|
||||
"")
|
||||
+ wikiurlify ("/[["+this.query+"]] [[Modification date::>" +this.lastsync+ "]]")
|
||||
|
||||
+ wikiurlify (this.parameters)
|
||||
|
||||
+ "/mainlabel%3D"
|
||||
+ "/offset%3D0"
|
||||
+ "/limit%3D200000"
|
||||
+ "/format%3Dystat");
|
||||
|
||||
String reply = UTF8.String(new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent).GETbytes(urlCount.toString(), null, null, false));
|
||||
String overallcount = CommonPattern.COMMA.split(reply)[0];
|
||||
String lastsyncstring = CommonPattern.COMMA.split(reply)[1];
|
||||
this.currentmax = Integer.parseInt(overallcount);
|
||||
|
||||
if (this.currentmax > 0) {
|
||||
ConcurrentLog.info("SMWLISTSYNC",
|
||||
"import job counts "
|
||||
+ this.currentmax
|
||||
+ " new elements between "
|
||||
+ this.lastsync + " and "
|
||||
+ this.currenttimestamp);
|
||||
|
||||
this.currenttimestamp = this.lastsync;
|
||||
|
||||
this.runningjob = true;
|
||||
this.lastsync = lastsyncstring;
|
||||
this.offset = 0;
|
||||
}
|
||||
} else {
|
||||
ConcurrentLog.warn("SMWLISTSYNC",
|
||||
"No SMWimport URL defined");
|
||||
}
|
||||
} catch (final MalformedURLException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (final IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
|
||||
// there are new elements to be imported
|
||||
ConcurrentLog.info("SMWLISTSYNC",
|
||||
"importing max. " + this.limit
|
||||
+ " elements at " + this.offset + " of "
|
||||
+ this.currentmax + ", since "
|
||||
+ this.currenttimestamp);
|
||||
URL urlImport;
|
||||
try {
|
||||
if (!this.sb.getConfig("contentcontrol.smwimport.baseurl","").equals("")) {
|
||||
urlImport = new URL(
|
||||
this.sb.getConfig(
|
||||
"contentcontrol.smwimport.baseurl",
|
||||
"")
|
||||
+ wikiurlify ("/[["+this.query+"]] [[Modification date::>" +this.currenttimestamp+ "]]")
|
||||
|
||||
+ wikiurlify (this.parameters)
|
||||
|
||||
+ "/mainlabel%3D"
|
||||
+ "/syntax%3Dobsolete"
|
||||
+ "/offset%3D" + this.offset
|
||||
+ "/limit%3D" + this.limit
|
||||
+ "/format%3Djson");
|
||||
|
||||
this.offset += this.limit;
|
||||
if (this.offset > this.currentmax) {
|
||||
this.runningjob = false;
|
||||
}
|
||||
|
||||
InputStreamReader reader = null;
|
||||
try {
|
||||
reader = new InputStreamReader(
|
||||
urlImport.openStream(), StandardCharsets.UTF_8);
|
||||
} catch (final Exception e) {
|
||||
ConcurrentLog.logException(e);
|
||||
this.runningjob = false;
|
||||
}
|
||||
|
||||
if (reader != null) {
|
||||
SMWListImporterFormatObsolete smwListImporter = null;
|
||||
try {
|
||||
smwListImporter = new SMWListImporterFormatObsolete(
|
||||
reader, 200);
|
||||
} catch (final Exception e) {
|
||||
// TODO: display an error message
|
||||
ConcurrentLog.logException(e);
|
||||
this.runningjob = false;
|
||||
}
|
||||
Thread t;
|
||||
SMWListRow row;
|
||||
t = new Thread(smwListImporter,"SMW List Importer");
|
||||
t.start();
|
||||
while ((row = smwListImporter.take()) != SMWListRow.POISON) {
|
||||
if (row == SMWListRow.EMPTY) {
|
||||
this.runningjob = false;
|
||||
} else {
|
||||
try {
|
||||
this.sb.tables.insert(targetList, row.getData());
|
||||
|
||||
dirty = true;
|
||||
|
||||
} catch (final Exception e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} catch (final MalformedURLException e2) {
|
||||
// TODO Auto-generated catch block
|
||||
e2.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
this.locked = false;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue