diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java index 0e1718e49..1c5e3667b 100644 --- a/htroot/Blacklist_p.java +++ b/htroot/Blacklist_p.java @@ -43,6 +43,7 @@ import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist; +import de.anomic.data.Tables; import de.anomic.data.listManager; import de.anomic.http.server.RequestHeader; import de.anomic.search.SearchEventCache; @@ -253,7 +254,7 @@ public class Blacklist_p { String blentry = post.get("newEntry", "").trim(); // store this call as api call - listManager.switchboard.recordAPICall(post, "Blacklist_p.html", "blacklist", "add to blacklist: " + blentry); + listManager.switchboard.tables.recordAPICall(post, "Blacklist_p.html", Tables.API_TYPE_CONFIGURATION, "add to blacklist: " + blentry); final String temp = addBlacklistEntry(blacklistToUse, blentry, header, supportedBlacklistTypes); if (temp != null) { diff --git a/htroot/ConfigBasic.java b/htroot/ConfigBasic.java index 054247c0d..cbf0732b3 100644 --- a/htroot/ConfigBasic.java +++ b/htroot/ConfigBasic.java @@ -34,6 +34,7 @@ import java.util.regex.Pattern; import net.yacy.kelondro.util.Domains; import net.yacy.kelondro.workflow.InstantBusyThread; +import de.anomic.data.Tables; import de.anomic.data.translator; import de.anomic.http.server.HTTPDemon; import de.anomic.http.server.HTTPDFileHandler; @@ -72,7 +73,7 @@ public class ConfigBasic { // store this call as api call if (post != null && post.containsKey("set")) { - sb.recordAPICall(post, "ConfigBasic.html", "configuration", "basic settings"); + sb.tables.recordAPICall(post, "ConfigBasic.html", Tables.API_TYPE_CONFIGURATION, "basic settings"); } //boolean doPeerPing = false; diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java index 277897012..17f3f3c11 100644 --- a/htroot/ConfigLanguage_p.java +++ b/htroot/ConfigLanguage_p.java @@ -42,6 +42,7 @@ import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import de.anomic.crawler.retrieval.HTTPLoader; +import de.anomic.data.Tables; import de.anomic.data.translator; import de.anomic.http.client.Client; import de.anomic.http.server.HeaderFramework; @@ -74,7 +75,7 @@ public class ConfigLanguage_p { String selectedLanguage = post.get("language"); // store this call as api call - ((Switchboard) env).recordAPICall(post, "ConfigLanguage.html", "configuration", "language settings: " + selectedLanguage); + ((Switchboard) env).tables.recordAPICall(post, "ConfigLanguage.html", Tables.API_TYPE_CONFIGURATION, "language settings: " + selectedLanguage); //change language if(post.containsKey("use_button") && selectedLanguage != null){ diff --git a/htroot/ConfigNetwork_p.java b/htroot/ConfigNetwork_p.java index ef0d1c2c2..86fbc24ba 100644 --- a/htroot/ConfigNetwork_p.java +++ b/htroot/ConfigNetwork_p.java @@ -32,6 +32,7 @@ import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MapTools; import net.yacy.kelondro.workflow.BusyThread; +import de.anomic.data.Tables; import de.anomic.http.server.HTTPDemon; import de.anomic.http.server.RequestHeader; import de.anomic.search.Switchboard; @@ -55,7 +56,7 @@ public class ConfigNetwork_p { if (post != null) { // store this call as api call - sb.recordAPICall(post, "ConfigNetwork.html", "configuration", "network settings"); + sb.tables.recordAPICall(post, "ConfigNetwork.html", Tables.API_TYPE_CONFIGURATION, "network settings"); if (post.containsKey("changeNetwork")) { final String networkDefinition = post.get("networkDefinition", "defaults/yacy.network.freeworld.unit"); diff --git a/htroot/ConfigPortal.java b/htroot/ConfigPortal.java index 4206419fd..0f517ef8b 100644 --- a/htroot/ConfigPortal.java +++ b/htroot/ConfigPortal.java @@ -25,6 +25,7 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import de.anomic.data.Tables; import de.anomic.http.server.RequestHeader; import de.anomic.search.Switchboard; import de.anomic.search.SwitchboardConstants; @@ -59,7 +60,7 @@ public class ConfigPortal { if (post.containsKey("searchpage_set")) { String newGreeting = post.get(SwitchboardConstants.GREETING, ""); // store this call as api call - sb.recordAPICall(post, "ConfigPortal.html", "appearance", "new portal design. greeting: " + newGreeting); + sb.tables.recordAPICall(post, "ConfigPortal.html", Tables.API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting); sb.setConfig(SwitchboardConstants.GREETING, newGreeting); sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, "")); diff --git a/htroot/Crawler_p.java b/htroot/Crawler_p.java index 4dadbc9c4..409dad7ed 100644 --- a/htroot/Crawler_p.java +++ b/htroot/Crawler_p.java @@ -45,6 +45,7 @@ import net.yacy.kelondro.util.FileUtils; import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.SitemapImporter; import de.anomic.crawler.retrieval.Request; +import de.anomic.data.Tables; import de.anomic.data.bookmarksDB; import de.anomic.data.listManager; import de.anomic.http.server.RequestHeader; @@ -141,7 +142,7 @@ public class Crawler_p { crawlingStart = (crawlingStartURL == null) ? null : crawlingStartURL.toNormalform(true, true); // store this call as api call - sb.recordAPICall(post, "Crawler_p.html", "crawler", "crawl start for " + crawlingStartURL.getHost()); + sb.tables.recordAPICall(post, "Crawler_p.html", Tables.API_TYPE_CRAWLER, "crawl start for " + crawlingStart); // set new properties final boolean fullDomain = post.get("range", "wide").equals("domain"); // special property in simple crawl start diff --git a/htroot/Tables_p.java b/htroot/Tables_p.java index e9d61d804..e6d5998d1 100644 --- a/htroot/Tables_p.java +++ b/htroot/Tables_p.java @@ -17,15 +17,11 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import java.io.IOException; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import net.yacy.kelondro.index.RowSpaceExceededException; -import net.yacy.kelondro.logging.Log; - import de.anomic.http.server.RequestHeader; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; @@ -37,7 +33,7 @@ public class Tables_p { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); String table = (post == null) ? null : post.get("table", null); - if (table != null && !sb.tables.hasHeap(table)) table = null; + if (table != null && !sb.tables.has(table)) table = null; // show table selection int count = 0; @@ -52,30 +48,20 @@ public class Tables_p { prop.put("tables", count); List columns = null; - if (table != null) try { + if (table != null) { columns = sb.tables.columns(table); - } catch (IOException e) { - Log.logException(e); } // apply deletion requests if (post != null && post.get("deletetable", "").length() > 0) { - try { - sb.tables.clear(table); - } catch (IOException e) { - Log.logException(e); - } + sb.tables.clear(table); } if (post != null && post.get("deleterows", "").length() > 0) { - try { - for (Map.Entry entry: post.entrySet()) { - if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) { - sb.tables.delete(table, entry.getKey().substring(5).getBytes()); - } + for (Map.Entry entry: post.entrySet()) { + if (entry.getKey().startsWith("mark_") && entry.getValue().equals("on")) { + sb.tables.delete(table, entry.getKey().substring(5).getBytes()); } - } catch (IOException e) { - Log.logException(e); } } @@ -87,20 +73,14 @@ public class Tables_p { map.put(entry.getKey().substring(4), entry.getValue().getBytes()); } } - try { - sb.tables.insert(table, pk.getBytes(), map); - } catch (IOException e) { - Log.logException(e); - } catch (RowSpaceExceededException e) { - Log.logException(e); - } + sb.tables.insert(table, pk.getBytes(), map); } // generate table prop.put("showtable", 0); prop.put("showedit", 0); - if (table != null && !post.containsKey("editrow") && !post.containsKey("addrow")) try { + if (table != null && !post.containsKey("editrow") && !post.containsKey("addrow")) { prop.put("showtable", 1); prop.put("showtable_table", table); @@ -136,9 +116,9 @@ public class Tables_p { count++; } prop.put("showtable_list", count); - } catch (IOException e) {} + } - if (post != null && table != null && post.containsKey("editrow")) try { + if (post != null && table != null && post.containsKey("editrow")) { // check if we can find a key String pk = null; for (Map.Entry entry: post.entrySet()) { @@ -150,16 +130,12 @@ public class Tables_p { if (pk != null && sb.tables.has(table, pk.getBytes())) { setEdit(sb, prop, table, pk, columns); } - } catch (IOException e) {} + } - if (post != null && table != null && post.containsKey("addrow")) try { + if (post != null && table != null && post.containsKey("addrow")) { // get a new key - String pk = new String(sb.tables.insert(table, new HashMap())); + String pk = sb.tables.createRow(table); setEdit(sb, prop, table, pk, columns); - } catch (IOException e) { - Log.logException(e); - } catch (RowSpaceExceededException e) { - Log.logException(e); } // adding the peer address @@ -169,7 +145,7 @@ public class Tables_p { return prop; } - private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List columns) throws IOException { + private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, List columns) { prop.put("showedit", 1); prop.put("showedit_table", table); prop.put("showedit_pk", pk); diff --git a/source/de/anomic/data/Tables.java b/source/de/anomic/data/Tables.java new file mode 100644 index 000000000..071b5ecbe --- /dev/null +++ b/source/de/anomic/data/Tables.java @@ -0,0 +1,144 @@ +package de.anomic.data; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import net.yacy.kelondro.blob.BEncodedHeapArray; +import net.yacy.kelondro.index.RowSpaceExceededException; +import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.util.DateFormatter; +import de.anomic.server.serverObjects; + +public class Tables { + + public final static String API_TYPE_CONFIGURATION = "configuration"; + public final static String API_TYPE_CRAWLER = "crawler"; + + private BEncodedHeapArray tables; + + public Tables(File workPath) { + this.tables = new BEncodedHeapArray(workPath, 12); + } + + public boolean has(String table) { + return tables.hasHeap(table); + } + + public boolean has(String table, byte[] pk) { + try { + return tables.has(table, pk); + } catch (IOException e) { + Log.logException(e); + return false; + } + } + + public Iterator tables() { + return this.tables.tables(); + } + + public List columns(String table) { + try { + return this.tables.columns(table); + } catch (IOException e) { + Log.logException(e); + return new ArrayList(0); + } + } + + public void clear(String table) { + try { + this.tables.clear(table); + } catch (IOException e) { + Log.logException(e); + } + } + + public void delete(String table, byte[] pk) { + try { + this.tables.delete(table, pk); + } catch (IOException e) { + Log.logException(e); + } + } + + public int size(String table) { + try { + return this.tables.size(table); + } catch (IOException e) { + Log.logException(e); + return 0; + } + } + + public Iterator>> iterator(String table) { + try { + return this.tables.iterator(table); + } catch (IOException e) { + Log.logException(e); + return new TreeMap>().entrySet().iterator(); + } + } + + public void close() { + this.tables.close(); + } + + public Map select(String table, byte[] pk) { + try { + return tables.select(table, pk); + } catch (IOException e) { + Log.logException(e); + return new TreeMap(); + } + } + + public void insert(String table, byte[] pk, Map map) { + try { + this.tables.insert(table, pk, map); + } catch (RowSpaceExceededException e) { + Log.logException(e); + } catch (IOException e) { + Log.logException(e); + } + } + + public String createRow(String table) { + try { + return new String(this.tables.insert(table, new HashMap())); + } catch (RowSpaceExceededException e) { + Log.logException(e); + return null; + } catch (IOException e) { + Log.logException(e); + return null; + } + } + + public void recordAPICall(final serverObjects post, final String servletName, String type, String comment) { + String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString(); + try { + this.tables.insert( + "api", + "type", type.getBytes(), + "comment", comment.getBytes(), + "date", DateFormatter.formatShortMilliSecond(new Date()).getBytes(), + "url", apiurl.getBytes() + ); + } catch (RowSpaceExceededException e2) { + Log.logException(e2); + } catch (IOException e2) { + Log.logException(e2); + } + Log.logInfo("APICALL", apiurl); + } + + +} diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index fdc62d088..bbecde34f 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -70,12 +70,10 @@ import net.yacy.document.content.RSSMessage; import net.yacy.document.content.SurrogateReader; import net.yacy.document.parser.html.ImageEntry; import net.yacy.document.parser.xml.RSSFeed; -import net.yacy.kelondro.blob.BEncodedHeapArray; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.meta.URIMetadataRow.Components; import net.yacy.kelondro.data.word.Word; -import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Digest; @@ -112,6 +110,7 @@ import de.anomic.crawler.retrieval.HTTPLoader; import de.anomic.crawler.retrieval.Request; import de.anomic.crawler.retrieval.Response; import de.anomic.data.LibraryProvider; +import de.anomic.data.Tables; import de.anomic.data.URLLicense; import de.anomic.data.blogBoard; import de.anomic.data.blogBoardComments; @@ -132,7 +131,6 @@ import de.anomic.http.server.ResponseHeader; import de.anomic.http.server.RobotsTxtConfig; import de.anomic.net.UPnP; import de.anomic.search.blockrank.CRDistribution; -import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.serverCore; import de.anomic.tools.crypt; @@ -218,7 +216,7 @@ public final class Switchboard extends serverSwitch { public Dispatcher dhtDispatcher; public List trail; public yacySeedDB peers; - public BEncodedHeapArray tables; + public Tables tables; public WorkflowProcessor indexingDocumentProcessor; public WorkflowProcessor indexingCondensementProcessor; @@ -279,7 +277,7 @@ public final class Switchboard extends serverSwitch { this.log.logConfig("Dictionaries Path:" + this.dictionariesPath.toString()); // init tables - this.tables = new BEncodedHeapArray(this.workPath, 12); + this.tables = new Tables(this.workPath); // init libraries this.log.logConfig("initializing libraries"); @@ -2100,26 +2098,7 @@ public final class Switchboard extends serverSwitch { } yacyCore.log.logInfo("BOOTSTRAP: " + (peers.sizeConnected() - sc) + " new seeds while bootstraping."); } - - - public void recordAPICall(final serverObjects post, final String servletName, String type, String comment) { - String apiurl = /*"http://localhost:" + getConfig("port", "8080") +*/ "/" + servletName + "?" + post.toString(); - try { - sb.tables.insert( - "api", - "type", type.getBytes(), - "comment", comment.getBytes(), - "date", DateFormatter.formatShortMilliSecond(new Date()).getBytes(), - "url", apiurl.getBytes() - ); - } catch (RowSpaceExceededException e2) { - Log.logException(e2); - } catch (IOException e2) { - Log.logException(e2); - } - Log.logInfo("APICALL", apiurl); - } - + public void checkInterruption() throws InterruptedException { final Thread curThread = Thread.currentThread(); if ((curThread instanceof WorkflowThread) && ((WorkflowThread)curThread).shutdownInProgress()) throw new InterruptedException("Shutdown in progress ..."); diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index 099f2761e..a5eb9209f 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -283,7 +283,7 @@ public class serverSwitch { if (f == null) { ret = null; } else { - ret = (f.isAbsolute() ? f : new File(this.rootPath, path)); + ret = (f.isAbsolute() ? new File(f.getAbsolutePath()) : new File(this.rootPath, path)); } return ret; diff --git a/source/net/yacy/kelondro/blob/BEncodedHeapArray.java b/source/net/yacy/kelondro/blob/BEncodedHeapArray.java index fa3c03d3c..eb0d10e09 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeapArray.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeapArray.java @@ -52,6 +52,7 @@ public class BEncodedHeapArray { public BEncodedHeapArray(final File location, final int keymaxlen) { this.location = new File(location.getAbsolutePath()); + if (!this.location.exists()) this.location.mkdirs(); this.keymaxlen = keymaxlen; this.tables = new ConcurrentHashMap(); String[] files = this.location.list(); diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index b19a5276f..0c1733602 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -555,17 +555,22 @@ public class HeapReader { final int keylen1 = this.keylen - 1; while (true) { int len = is.readInt(); - if (len == 0) continue; - b = is.readByte(); // check for empty record + if (len == 0) continue; // rare, but possible: zero length record (takes 4 bytes) + b = is.readByte(); // read a single by te to check for empty record if (b == 0) { + // this is empty // read some more bytes to consume the empty record - is.skip(len - 1); + is.skip(len - 1); // all that is remaining continue; } + // we are now ahead of remaining this.keylen - 1 bytes of the key key = new byte[this.keylen]; - key[0] = b; - if (is.read(key, 1, keylen1) < keylen1) return null; - payload = new byte[len - this.keylen]; + key[0] = b; // the first entry that we know already + if (is.read(key, 1, keylen1) < keylen1) return null; // read remaining key bytes + // so far we have read this.keylen - 1 + 1 = this.keylen bytes. + // there must be a remaining number of len - this.keylen bytes left for the BLOB + if (len < this.keylen) return null; // a strange case that can only happen in case of corrupted data + payload = new byte[len - this.keylen]; // the remaining record entries if (is.read(payload) < payload.length) return null; return new entry(key, payload); } @@ -574,6 +579,22 @@ public class HeapReader { } } + /* the old code: + + private Map.Entry next0() { + try { + while (true) { + int len = is.readInt(); + byte[] key = new byte[this.keylen]; + if (is.read(key) < key.length) return null; + byte[] payload = new byte[len - this.keylen]; + if (is.read(payload) < payload.length) return null; + if (key[0] == 0) continue; // this is an empty gap + return new entry(key, payload); + } + } + */ + public Map.Entry next() { final Map.Entry n = this.nextEntry; this.nextEntry = next0();