boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

removed many warnings

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent c72c435517
commit a427a68bac
8 changed files with 176 additions and 206 deletions
Show Stats Download Patch File Download Diff File

2
htroot/Table_YMark_p.java
Unescape View File

@ -167,8 +167,6 @@ public class Table_YMark_p {
sb.tables.bookmarks.addBookmark(bmk_user, bmk, false, false);
} catch (final IOException e) {
Log.logException(e);
} catch (final SpaceExceededException e) {
Log.logException(e);
}
}

7
htroot/api/ymarks/add_ymark.java
Unescape View File

@ -1,7 +1,6 @@
import java.io.IOException;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.document.Parser.Failure;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
@ -46,9 +45,6 @@ public class add_ymark {
} catch (final Failure e) {
// TODO Auto-generated catch block
Log.logException(e);
} catch (final SpaceExceededException e) {
// TODO Auto-generated catch block
Log.logException(e);
}
} else if(post.containsKey(YMarkEntry.BOOKMARK.URL.key())) {
@ -77,8 +73,7 @@ public class add_ymark {
sb.tables.bookmarks.addBookmark(bmk_user, bmk, false, false);
} catch (final IOException e) {
Log.logException(e);
} catch (final SpaceExceededException e) {
}
}
prop.put("status", "1");
} else {
prop.put("status", "0");

9
htroot/api/ymarks/import_ymark.java
Unescape View File

@ -11,7 +11,6 @@ import java.util.regex.Pattern;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.services.federated.yacy.CacheStrategy;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.document.Parser.Failure;
import net.yacy.document.content.SurrogateReader;
import net.yacy.kelondro.blob.Tables;
@ -169,9 +168,6 @@ public class import_ymark {
} catch (final IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (final SpaceExceededException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (final Failure e) {
// TODO Auto-generated catch block
e.printStackTrace();
@ -206,9 +202,6 @@ public class import_ymark {
} catch (final IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (final SpaceExceededException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
@ -254,8 +247,6 @@ public class import_ymark {
}
} catch (final IOException e) {
Log.logException(e);
} catch (final SpaceExceededException e) {
Log.logException(e);
} catch (final InterruptedException e) {
Log.logException(e);
}

3
source/de/anomic/data/ymark/YMarkAutoTagger.java
Unescape View File

@ -11,7 +11,6 @@ import java.util.TreeSet;
import java.util.concurrent.ArrayBlockingQueue;
import net.yacy.cora.services.federated.yacy.CacheStrategy;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.document.Condenser;
import net.yacy.document.Document;
import net.yacy.document.LibraryProvider;
@ -267,8 +266,6 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle
Log.logException(e);
} catch (final IOException e) {
Log.logWarning(YMarkTables.BOOKMARKS_LOG.toString(), "autoTagger - IOException for URL: "+url);
} catch (final SpaceExceededException e) {
Log.logException(e);
} finally {
}
}

134
source/de/anomic/data/ymark/YMarkSMWJSONImporter.java
Unescape View File

@ -14,153 +14,162 @@ import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
public class YMarkSMWJSONImporter implements Runnable, ContentHandler{
// Importer Variables
private final ArrayBlockingQueue<YMarkEntry> bookmarks;
private final ArrayBlockingQueue<YMarkEntry> bookmarks;
private final Reader bmk_file;
private final String RootFolder;
private final StringBuilder folderstring;
private YMarkEntry bmk;
private final JSONParser parser;
private boolean empty = true;
private int count = 0;
//private boolean empty = true;
//private int count = 0;
// Parser Variables
private final StringBuilder value;
private final StringBuilder key;
private final StringBuilder date;
//private final StringBuilder date;
private final HashMap<String,String> obj;
private Boolean isBookmark;
private Boolean isBookmark;
public YMarkSMWJSONImporter(final Reader bmk_file, final int queueSize, final String root) {
this.bookmarks = new ArrayBlockingQueue<YMarkEntry>(queueSize);
this.bookmarks = new ArrayBlockingQueue<YMarkEntry>(queueSize);
this.bmk_file = bmk_file;
this.RootFolder = root;
this.folderstring = new StringBuilder(YMarkTables.BUFFER_LENGTH);
this.folderstring.append(this.RootFolder);
this.bmk = new YMarkEntry();
this.parser = new JSONParser();
this.value = new StringBuilder(128);
this.key = new StringBuilder(16);
this.date = new StringBuilder(32);
//this.date = new StringBuilder(32);
this.obj = new HashMap<String,String>();
this.isBookmark = false;
this.empty = true;
this.count = 0;
//this.empty = true;
//this.count = 0;
}
public void startJSON() throws ParseException, IOException {
@Override
public void startJSON() throws ParseException, IOException {
}
public void endJSON() throws ParseException, IOException {
@Override
public void endJSON() throws ParseException, IOException {
}
public boolean startArray() throws ParseException, IOException {
@Override
public boolean startArray() throws ParseException, IOException {
final String key = this.key.toString();
if(key.equals("items") ) {
this.isBookmark = true;
this.count = 0;
//this.count = 0;
}
return true;
}
public boolean endArray() throws ParseException, IOException {
@Override
public boolean endArray() throws ParseException, IOException {
return true;
}
public boolean startObject() throws ParseException, IOException {
@Override
public boolean startObject() throws ParseException, IOException {
return true;
}
public boolean endObject() throws ParseException, IOException {
@Override
public boolean endObject() throws ParseException, IOException {
if(this.isBookmark) {
if(this.obj.containsKey("category")) {
String catstr = obj.get("category");
String catstr = this.obj.get("category");
HashSet<String> tags = YMarkUtil.keysStringToSet (catstr);
HashSet<String> categories = YMarkUtil.keysStringToSet("");
for (String c: tags) {
c = c.split(":")[1];
c = c.replace("/", "_");
c = c.replace(" ", "_");
if (!c.equals("") && (!c.equals(" "))) {
if (!c.equals("") && (!c.equals(" "))) {
categories.add ("sc:"+c);
}
}
if (!Switchboard.getSwitchboard().getConfig("contentcontrol.smwimport.defaultcategory", "").equals("")) {
if (!Switchboard.getSwitchboard().getConfig("contentcontrol.smwimport.defaultcategory", "").equals("")) {
categories.add ("sc:"+Switchboard.getSwitchboard().getConfig("contentcontrol.smwimport.defaultcategory", ""));
}
catstr = YMarkUtil.keySetToString(categories);
this.bmk.put(YMarkEntry.BOOKMARK.TAGS.key(), catstr);
}
if(this.obj.containsKey("article_has_average_rating")) {
this.bmk.put(YMarkEntry.BOOKMARK.STARRATING.key(),obj.get("article_has_average_rating"));
this.bmk.put(YMarkEntry.BOOKMARK.STARRATING.key(),this.obj.get("article_has_average_rating"));
}
this.bmk.put(YMarkEntry.BOOKMARK.TITLE.key(),obj.get("label"));
this.bmk.put(YMarkEntry.BOOKMARK.URL.key(),obj.get("url"));
this.bmk.put(YMarkEntry.BOOKMARK.TITLE.key(),this.obj.get("label"));
this.bmk.put(YMarkEntry.BOOKMARK.URL.key(),this.obj.get("url"));
if(this.obj.containsKey("filter")) {
this.bmk.put(YMarkEntry.BOOKMARK.FILTER.key(),obj.get("filter"));
this.bmk.put(YMarkEntry.BOOKMARK.FILTER.key(),this.obj.get("filter"));
} else {
this.bmk.put(YMarkEntry.BOOKMARK.FILTER.key(),"");
}
try {
this.bookmarks.put(this.bmk);
this.count++;
//this.count++;
} catch (InterruptedException e) {
Log.logException(e);
}
this.obj.clear();
this.bmk = new YMarkEntry();
this.bmk = new YMarkEntry();
}
return true;
}
public boolean startObjectEntry(String key) throws ParseException, IOException {
@Override
public boolean startObjectEntry(String key) throws ParseException, IOException {
this.key.setLength(0);
this.key.append(key);
this.key.append(key);
return true;
}
public boolean primitive(Object value) throws ParseException, IOException {
@Override
public boolean primitive(Object value) throws ParseException, IOException {
this.value.setLength(0);
if(value instanceof java.lang.String) {
this.value.append((String)value);
} else if(value instanceof java.lang.Boolean) {
this.value.append((Boolean)value);
this.value.append(value);
} else if(value instanceof java.lang.Number) {
this.value.append((Number)value);
this.value.append(value);
}
return true;
}
public boolean endObjectEntry() throws ParseException, IOException {
@Override
public boolean endObjectEntry() throws ParseException, IOException {
final String key = this.key.toString();
final String value = this.value.toString();
@ -170,18 +179,19 @@ public class YMarkSMWJSONImporter implements Runnable, ContentHandler{
return true;
}
public void run() {
@Override
public void run() {
try {
Log.logInfo(YMarkTables.BOOKMARKS_LOG, "SMWJSON Importer run()");
this.empty = true;
//this.empty = true;
this.parser.parse(this.bmk_file, this, true);
} catch (IOException e) {
Log.logException(e);
} catch (ParseException e) {
Log.logException(e);
} finally {
} finally {
try {
Log.logInfo(YMarkTables.BOOKMARKS_LOG, "SMWJSON Importer inserted poison pill in queue");
this.bookmarks.put(YMarkEntry.POISON);
@ -190,7 +200,7 @@ public class YMarkSMWJSONImporter implements Runnable, ContentHandler{
}
}
}
public YMarkEntry take() {
try {
return this.bookmarks.take();

32
source/de/anomic/data/ymark/YMarkTables.java
Unescape View File

@ -101,12 +101,12 @@ public class YMarkTables {
public final static int BUFFER_LENGTH = 256;
private final WorkTables worktables;
public boolean dirty = false;
public YMarkTables(final Tables wt) {
this.worktables = (WorkTables)wt;
dirty = true;
this.dirty = true;
}
public void deleteBookmark(final String bmk_user, final byte[] urlHash) throws IOException, SpaceExceededException {
@ -116,7 +116,7 @@ public class YMarkTables {
if(bmk_row != null) {
this.worktables.delete(bmk_table,urlHash);
}
dirty = true;
this.dirty = true;
}
public void deleteBookmark(final String bmk_user, final String url) throws IOException, SpaceExceededException {
@ -221,13 +221,13 @@ public class YMarkTables {
final Pattern p = Pattern.compile(patternBuilder.toString(), Pattern.CASE_INSENSITIVE);
return this.worktables.iterator(bmk_table, YMarkEntry.BOOKMARK.TAGS.key(), p);
}
public Iterator<Tables.Row> getBookmarksByTag(final String bmk_user, String regex) throws IOException {
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final StringBuilder patternBuilder = new StringBuilder(BUFFER_LENGTH);
patternBuilder.setLength(0);
patternBuilder.append(regex);
final Pattern p = Pattern.compile(patternBuilder.toString(), Pattern.CASE_INSENSITIVE);
return this.worktables.iterator(bmk_table, YMarkEntry.BOOKMARK.TAGS.key(), p);
}
@ -244,7 +244,7 @@ public class YMarkTables {
return sortList;
}
public void addTags(final String bmk_user, final String url, final String tagString, final boolean merge) throws IOException, SpaceExceededException {
public void addTags(final String bmk_user, final String url, final String tagString, final boolean merge) throws IOException {
if(!tagString.isEmpty()) {
// do not set defaults as we only want to update tags
final YMarkEntry bmk = new YMarkEntry(false);
@ -252,7 +252,7 @@ public class YMarkTables {
bmk.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkUtil.cleanTagsString(tagString));
addBookmark(bmk_user, bmk, merge, true);
}
dirty = true;
this.dirty = true;
}
public void replaceTags(final Iterator<Row> rowIterator, final String bmk_user, final String tagString, final String replaceString) throws IOException {
@ -272,10 +272,10 @@ public class YMarkTables {
row.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkUtil.cleanTagsString(t.toString()));
this.worktables.update(TABLES.BOOKMARKS.tablename(bmk_user), row);
}
dirty = true;
this.dirty = true;
}
public void addFolder(final String bmk_user, final String url, final String folder) throws IOException, SpaceExceededException {
public void addFolder(final String bmk_user, final String url, final String folder) throws IOException {
if(!folder.isEmpty()) {
// do not set defaults as we only want to add a folder
final YMarkEntry bmk = new YMarkEntry(false);
@ -285,7 +285,7 @@ public class YMarkTables {
}
}
public void visited(final String bmk_user, final String url) throws IOException, SpaceExceededException {
public void visited(final String bmk_user, final String url) throws IOException {
// do not set defaults
final YMarkEntry bmk = new YMarkEntry(false);
bmk.put(YMarkEntry.BOOKMARK.URL.key(), url);
@ -293,11 +293,11 @@ public class YMarkTables {
addBookmark(bmk_user, bmk, true, true);
}
public void createBookmark(final LoaderDispatcher loader, final String url, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure, SpaceExceededException {
public void createBookmark(final LoaderDispatcher loader, final String url, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure {
createBookmark(loader, new DigestURI(url), bmk_user, autotag, tagsString, foldersString);
}
public void createBookmark(final LoaderDispatcher loader, final DigestURI url, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure, SpaceExceededException {
public void createBookmark(final LoaderDispatcher loader, final DigestURI url, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure {
final YMarkEntry bmk_entry = new YMarkEntry(false);
final YMarkMetadata meta = new YMarkMetadata(url);
@ -337,7 +337,7 @@ public class YMarkTables {
}
}
public void addBookmark(final String bmk_user, final YMarkEntry bmk, final boolean mergeTags, final boolean mergeFolders) throws IOException, SpaceExceededException {
public void addBookmark(final String bmk_user, final YMarkEntry bmk, final boolean mergeTags, final boolean mergeFolders) throws IOException {
final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user);
final String date = String.valueOf(System.currentTimeMillis());
byte[] urlHash = null;
@ -352,7 +352,7 @@ public class YMarkTables {
try {
bmk_row = this.worktables.select(bmk_table, urlHash);
} catch (Exception e) {
}
if (bmk_row == null) {
// create and insert new entry
@ -418,8 +418,8 @@ public class YMarkTables {
// update bmk_table
this.worktables.update(bmk_table, bmk_row);
}
dirty = true;
this.dirty = true;
}
}
}

58
source/net/yacy/interaction/contentcontrol/ContentControlFilterUpdateThread.java
Unescape View File

@ -9,69 +9,63 @@ import net.yacy.repository.FilterEngine;
import net.yacy.search.Switchboard;
public class ContentControlFilterUpdateThread {
private Switchboard sb;
private final Switchboard sb;
private Boolean locked = false;
private static FilterEngine networkfilter;
public ContentControlFilterUpdateThread(final Switchboard sb) {
final long time = System.currentTimeMillis();
//final long time = System.currentTimeMillis();
this.sb = sb;
if (this.sb.getConfigBool("contentcontrol.smwimport.purgelistoninit",
false)) {
this.sb.tables.clear(this.sb.getConfig(
"contentcontrol.smwimport.targetlist", "contentcontrol"));
}
}
@SuppressWarnings("deprecation")
public final void run() {
if (!locked) {
if (!this.locked) {
this.locked = true;
locked = true;
if (this.sb.getConfigBool("contentcontrol.enabled", false) == true) {
if (!this.sb
.getConfig("contentcontrol.mandatoryfilterlist", "")
.equals("")) {
if (sb.tables.bookmarks.dirty) {
.equals("")) {
if (this.sb.tables.bookmarks.dirty) {
networkfilter = updateFilter();
sb.tables.bookmarks.dirty = false;
this.sb.tables.bookmarks.dirty = false;
}
}
}
locked = false;
this.locked = false;
}
return;
}
private static FilterEngine updateFilter () {
FilterEngine newfilter = new FilterEngine();
Switchboard sb = Switchboard.getSwitchboard();
Iterator<Tables.Row> it;
try {
it = sb.tables.bookmarks.getBookmarksByTag(
@ -96,11 +90,11 @@ public class ContentControlFilterUpdateThread {
// TODO Auto-generated catch block
e.printStackTrace();
}
return newfilter;
}
public static FilterEngine getNetworkFilter() {
FilterEngine f = networkfilter;

137
source/net/yacy/interaction/contentcontrol/ContentControlImportThread.java
Unescape View File

@ -2,123 +2,108 @@ package net.yacy.interaction.contentcontrol;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.kelondro.blob.Tables.Row;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
import de.anomic.data.ymark.YMarkEntry;
import de.anomic.data.ymark.YMarkSMWJSONImporter;
import de.anomic.data.ymark.YMarkUtil;
public class ContentControlImportThread {
private Switchboard sb;
private final Switchboard sb;
private Boolean locked = false;
private String lastsync = "1900-01-01T01:00:00";
private String currenttimestamp = "1900-01-01T01:00:00";
private long offset = 0;
private long limit = 500;
private final long limit = 500;
private long currentmax = 0;
private boolean runningjob = false;
public ContentControlImportThread(final Switchboard sb) {
final long time = System.currentTimeMillis();
//final long time = System.currentTimeMillis();
this.sb = sb;
if (this.sb.getConfigBool("contentcontrol.smwimport.purgelistoninit",
false)) {
this.sb.tables.clear(this.sb.getConfig(
"contentcontrol.smwimport.targetlist", "contentcontrol"));
}
}
private final String wikiurlify (String s) {
String ret = s;
ret = ret.replace("-", "-2D");
ret = ret.replace("+", "-2B");
ret = ret.replace(" ", "-20");
ret = ret.replace("[", "-5B");
ret = ret.replace("]", "-5D");
ret = ret.replace(":", "-3A");
ret = ret.replace(">", "-3E");
ret = ret.replace("?", "-3F");
return ret;
}
@SuppressWarnings("deprecation")
public final void run() {
if (!locked) {
if (!this.locked) {
locked = true;
this.locked = true;
if (sb.getConfigBool("contentcontrol.smwimport.enabled", false) == true) {
if (this.sb.getConfigBool("contentcontrol.smwimport.enabled", false) == true) {
if (runningjob) {
if (this.runningjob) {
Log.logInfo("CONTENTCONTROL",
"CONTENTCONTROL importing max. " + limit
+ " elements at " + offset + " of "
+ currentmax + ", since "
+ currenttimestamp);
"CONTENTCONTROL importing max. " + this.limit
+ " elements at " + this.offset + " of "
+ this.currentmax + ", since "
+ this.currenttimestamp);
URL bmks_json;
String currenttimestampurl = wikiurlify (currenttimestamp);
//String currenttimestampurl = wikiurlify (this.currenttimestamp);
try {
if (!sb.getConfig("contentcontrol.smwimport.baseurl",
if (!this.sb.getConfig("contentcontrol.smwimport.baseurl",
"").equals("")) {
bmks_json = new URL(
sb.getConfig(
this.sb.getConfig(
"contentcontrol.smwimport.baseurl",
"")
+ wikiurlify ("/[[Category:Web Page]] [[Modification date::>" +currenttimestamp+ "]]")
+ wikiurlify ("/[[Category:Web Page]] [[Modification date::>" +this.currenttimestamp+ "]]")
+ wikiurlify ("/?Url/?Filter/?Article has average rating/?Category")
+ "/mainlabel%3D"
+ "/offset%3D" + offset
+ "/limit%3D" + limit
+ "/mainlabel%3D"
+ "/offset%3D" + this.offset
+ "/limit%3D" + this.limit
+ "/format%3Djson");
offset += limit;
this.offset += this.limit;
if (offset > currentmax) {
runningjob = false;
if (this.offset > this.currentmax) {
this.runningjob = false;
}
InputStreamReader reader = null;
@ -128,7 +113,7 @@ public class ContentControlImportThread {
} catch (Exception e) {
Log.logException(e);
runningjob = false;
this.runningjob = false;
}
if (reader != null) {
@ -139,7 +124,7 @@ public class ContentControlImportThread {
} catch (final Exception e) {
// TODO: display an error message
Log.logException(e);
runningjob = false;
this.runningjob = false;
}
Thread t;
@ -153,13 +138,13 @@ public class ContentControlImportThread {
if (bmk == YMarkEntry.EMPTY) {
runningjob = false;
this.runningjob = false;
} else {
try {
sb.tables.bookmarks.addBookmark(
sb.getConfig("contentcontrol.smwimport.targetlist", "contentcontrol"), bmk,
this.sb.tables.bookmarks.addBookmark(
this.sb.getConfig("contentcontrol.smwimport.targetlist", "contentcontrol"), bmk,
true, true);
} catch (Exception e) {
@ -187,20 +172,20 @@ public class ContentControlImportThread {
try {
if (!sb.getConfig("contentcontrol.smwimport.baseurl",
if (!this.sb.getConfig("contentcontrol.smwimport.baseurl",
"").equals("")) {
URL bmks_count;
bmks_count = new URL(
sb.getConfig(
this.sb.getConfig(
"contentcontrol.smwimport.baseurl",
"")
+ wikiurlify ("/[[Category:Web Page]] [[Modification date::>" +lastsync+ "]]")
+ wikiurlify ("/[[Category:Web Page]] [[Modification date::>" +this.lastsync+ "]]")
+ wikiurlify ("/?Url/?Filter/?Article has average rating/?Category")
+ "/mainlabel%3D"
+ "/mainlabel%3D"
+ "/format%3Dsupercount");
String reply = UTF8.String(new HTTPClient()
@ -210,22 +195,22 @@ public class ContentControlImportThread {
String lastsyncstring = reply.split(",")[1];
currentmax = Integer.parseInt(overallcount);
this.currentmax = Integer.parseInt(overallcount);
if (currentmax > 0) {
if (this.currentmax > 0) {
Log.logInfo("CONTENTCONTROL",
"CONTENTCONTROL import job counts "
+ currentmax
+ this.currentmax
+ " new elements between "
+ lastsync + " and "
+ currenttimestamp);
+ this.lastsync + " and "
+ this.currenttimestamp);
currenttimestamp = lastsync;
this.currenttimestamp = this.lastsync;
runningjob = true;
lastsync = lastsyncstring;
offset = 0;
this.runningjob = true;
this.lastsync = lastsyncstring;
this.offset = 0;
}
} else {
Log.logWarning("CONTENTCONTROL",
@ -242,7 +227,7 @@ public class ContentControlImportThread {
}
locked = false;
this.locked = false;
}
}

Write Preview
Loading…
Cancel
Save