From 2a8f70f0ca3f8b2bdcf4de11f6136ec2957b16cf Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 14 May 2010 23:50:07 +0000 Subject: [PATCH] - fix for caching of OSM tiles. if you want that this fix applies to your peer, please delete the crawl profiles - fix for initial generation of crawl profiles (one more reason to remove your crawl profiles) - more String -> byte[] migration - more logging for cache store/hit git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6874 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/CacheResource_p.java | 10 +-- htroot/ViewFile.java | 9 +-- source/de/anomic/crawler/CrawlProfile.java | 21 ++--- .../de/anomic/crawler/CrawlSwitchboard.java | 2 +- source/de/anomic/data/BookmarkDate.java | 6 +- source/de/anomic/data/blogBoard.java | 12 +-- source/de/anomic/data/blogBoardComments.java | 6 +- source/de/anomic/data/bookmarksDB.java | 6 +- source/de/anomic/data/messageBoard.java | 6 +- source/de/anomic/data/userDB.java | 10 +-- source/de/anomic/data/wiki/wikiBoard.java | 6 +- source/de/anomic/http/client/Cache.java | 36 +++++---- source/de/anomic/search/Switchboard.java | 6 +- source/de/anomic/yacy/graphics/OSMTile.java | 7 +- source/de/anomic/yacy/yacySeedDB.java | 40 +++++----- source/net/yacy/kelondro/blob/Heap.java | 14 ++-- .../net/yacy/kelondro/blob/MapDataMining.java | 40 +++------- source/net/yacy/kelondro/blob/MapHeap.java | 79 ++++--------------- .../net/yacy/repository/LoaderDispatcher.java | 8 +- source/net/yacy/yacy.java | 2 +- 20 files changed, 114 insertions(+), 212 deletions(-) diff --git a/htroot/CacheResource_p.java b/htroot/CacheResource_p.java index 45f9e22e4..a975f358f 100644 --- a/htroot/CacheResource_p.java +++ b/htroot/CacheResource_p.java @@ -22,8 +22,6 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -import java.io.IOException; import java.net.MalformedURLException; import net.yacy.kelondro.data.meta.DigestURI; @@ -52,13 +50,7 @@ public class CacheResource_p { } byte[] resource = null; - // trying to load the resource body - try { - resource = Cache.getContent(url); - } catch (IOException e) { - Log.logException(e); - return prop; - } + resource = Cache.getContent(url); if (resource == null) return prop; //ResponseHeader responseHeader = Cache.getResponseHeader(url); //String resMime = responseHeader.mime(); diff --git a/htroot/ViewFile.java b/htroot/ViewFile.java index 285a13dfc..099b28e74 100644 --- a/htroot/ViewFile.java +++ b/htroot/ViewFile.java @@ -167,16 +167,9 @@ public class ViewFile { // loading the resource content as byte array prop.put("error_incache", Cache.has(url) ? 1 : 0); - byte[] resource = null; ResponseHeader responseHeader = null; String resMime = null; - // trying to load the resource body - try { - resource = Cache.getContent(url); - } catch (IOException e) { - Log.logException(e); - resource = null; - } + byte[] resource = Cache.getContent(url); if (resource == null && authorized) { // load resource from net diff --git a/source/de/anomic/crawler/CrawlProfile.java b/source/de/anomic/crawler/CrawlProfile.java index 2a581dddf..6c24a797e 100644 --- a/source/de/anomic/crawler/CrawlProfile.java +++ b/source/de/anomic/crawler/CrawlProfile.java @@ -147,11 +147,11 @@ public class CrawlProfile { public entry newEntry(final Map mem) { final entry ne = new entry(mem); try { - profileTable.put(ne.handle(), ne.map()); + profileTable.put(ne.handle().getBytes(), ne.map()); } catch (final Exception e) { clear(); try { - profileTable.put(ne.handle(), ne.map()); + profileTable.put(ne.handle().getBytes(), ne.map()); } catch (final Exception ee) { Log.logException(e); System.exit(0); @@ -184,11 +184,11 @@ public class CrawlProfile { xsstopw, xdstopw, xpstopw, cacheStrategy); try { - profileTable.put(ne.handle(), ne.map()); + profileTable.put(ne.handle().getBytes(), ne.map()); } catch (final Exception e) { clear(); try { - profileTable.put(ne.handle(), ne.map()); + profileTable.put(ne.handle().getBytes(), ne.map()); } catch (final Exception ee) { Log.logException(e); System.exit(0); @@ -198,19 +198,14 @@ public class CrawlProfile { } public boolean hasEntry(final String handle) { - try { - return profileTable.has(handle); - } catch (final IOException e) { - Log.logException(e); - return false; - } + return profileTable.has(handle.getBytes()); } public entry getEntry(final String handle) { if (profileTable == null) return null; Map m; try { - m = profileTable.get(handle); + m = profileTable.get(handle.getBytes()); } catch (final IOException e) { Log.logException(e); return null; @@ -222,7 +217,7 @@ public class CrawlProfile { public void changeEntry(final entry e, final String propName, final String newValue) throws IOException, RowSpaceExceededException { e.mem.put(propName, newValue); assert e.handle() != null; - profileTable.put(e.handle(), e.mem); + profileTable.put(e.handle().getBytes(), e.mem); } public long getRecrawlDate(final long oldTimeMinutes) { @@ -305,7 +300,7 @@ public class CrawlProfile { final boolean xsstopw, final boolean xdstopw, final boolean xpstopw, final CacheStrategy cacheStrategy) { if (name == null || name.length() == 0) throw new NullPointerException("name must not be null"); - final String handle = (startURL == null) ? Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(0, Word.commonHashLength) : new String(startURL.hash()); + final String handle = (startURL == null) ? Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(name)).substring(0, Word.commonHashLength) : new String(startURL.hash()); mem = new HashMap(40); mem.put(HANDLE, handle); mem.put(NAME, name); diff --git a/source/de/anomic/crawler/CrawlSwitchboard.java b/source/de/anomic/crawler/CrawlSwitchboard.java index 7db84194f..71d6817da 100644 --- a/source/de/anomic/crawler/CrawlSwitchboard.java +++ b/source/de/anomic/crawler/CrawlSwitchboard.java @@ -188,7 +188,7 @@ public final class CrawlSwitchboard { if (this.defaultMediaSnippetLocalProfile == null) { // generate new default entry for snippet fetch and optional crawling defaultMediaSnippetLocalProfile = this.profilesActiveCrawls.newEntry(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_BAD_URL, 0, - this.profilesActiveCrawls.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE), -1, -1, true, false, false, false, false, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST); + this.profilesActiveCrawls.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE), -1, -1, true, false, false, true, false, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST); } if (this.defaultMediaSnippetGlobalProfile == null) { // generate new default entry for snippet fetch and optional crawling diff --git a/source/de/anomic/data/BookmarkDate.java b/source/de/anomic/data/BookmarkDate.java index 34eb5b043..786b04ced 100644 --- a/source/de/anomic/data/BookmarkDate.java +++ b/source/de/anomic/data/BookmarkDate.java @@ -54,7 +54,7 @@ public class BookmarkDate { public Entry getDate(final String date) { Map map; try { - map = datesTable.get(date); + map = datesTable.get(date.getBytes()); } catch (final IOException e) { map = null; } @@ -141,13 +141,13 @@ public class BookmarkDate { public void setDatesTable() { if (this.size() >0) { try { - datesTable.put(getDateString(), mem); + datesTable.put(getDateString().getBytes(), mem); } catch (Exception e) { Log.logException(e); } } else { try { - datesTable.remove(getDateString()); + datesTable.remove(getDateString().getBytes()); } catch (IOException e) { Log.logException(e); } diff --git a/source/de/anomic/data/blogBoard.java b/source/de/anomic/data/blogBoard.java index 6e52f4bfd..a23bf7a05 100644 --- a/source/de/anomic/data/blogBoard.java +++ b/source/de/anomic/data/blogBoard.java @@ -81,11 +81,7 @@ public class blogBoard { * @return true if the database contains the element, else false */ public boolean contains(final String key) { - try { - return database.has(key); - } catch (IOException ex) { - return false; - } + return database.has(key.getBytes()); } public void close() { @@ -126,7 +122,7 @@ public class blogBoard { public String writeBlogEntry(final BlogEntry page) { String ret = null; try { - database.put(page.key, page.record); + database.put(page.key.getBytes(), page.record); ret = page.key; } catch (IOException ex) { Log.logException(ex); @@ -144,7 +140,7 @@ public class blogBoard { final String normalized = normalize(key); Map record; try { - record = base.get(normalized.substring(0, Math.min(normalized.length(), KEY_LENGTH))); + record = base.get(normalized.substring(0, Math.min(normalized.length(), KEY_LENGTH)).getBytes()); } catch (final IOException e) { record = null; } @@ -240,7 +236,7 @@ public class blogBoard { public void deleteBlogEntry(final String key) { try { - database.remove(normalize(key)); + database.remove(normalize(key).getBytes()); } catch (final IOException e) { } } diff --git a/source/de/anomic/data/blogBoardComments.java b/source/de/anomic/data/blogBoardComments.java index a6b88d83c..51d34ef5a 100644 --- a/source/de/anomic/data/blogBoardComments.java +++ b/source/de/anomic/data/blogBoardComments.java @@ -111,7 +111,7 @@ public class blogBoardComments { public String write(final CommentEntry page) { // writes a new page and returns key try { - database.put(page.key, page.record); + database.put(page.key.getBytes(), page.record); return page.key; } catch (final Exception e) { Log.logException(e); @@ -128,7 +128,7 @@ public class blogBoardComments { copyOfKey = copyOfKey.substring(0, Math.min(copyOfKey.length(), KEY_LENGTH)); Map record; try { - record = base.get(copyOfKey); + record = base.get(copyOfKey.getBytes()); } catch (final IOException e) { record = null; } @@ -226,7 +226,7 @@ public class blogBoardComments { public void delete(final String key) { try { - database.remove(normalize(key)); + database.remove(normalize(key).getBytes()); } catch (final IOException e) { } } diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index 94f75cef6..a69be7e9b 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -317,7 +317,7 @@ public class bookmarksDB { // adding a bookmark to the bookmarksDB public void saveBookmark(final Bookmark bookmark){ try { - bookmarks.put(bookmark.getUrlHash(), bookmark.entry); + bookmarks.put(bookmark.getUrlHash().getBytes(), bookmark.entry); } catch (final Exception e) { Log.logException(e); } @@ -330,7 +330,7 @@ public class bookmarksDB { public Bookmark getBookmark(final String urlHash){ try { - final Map map = bookmarks.get(urlHash); + final Map map = bookmarks.get(urlHash.getBytes()); return (map == null) ? null : new Bookmark(map); } catch (final IOException e) { return null; @@ -353,7 +353,7 @@ public class bookmarksDB { Bookmark b; try { b = getBookmark(urlHash); - bookmarks.remove(urlHash); + bookmarks.remove(urlHash.getBytes()); } catch (final IOException e) { b = null; } diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index 3dba25512..5b1115b03 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -182,7 +182,7 @@ public class messageBoard { public String write(final entry message) { // writes a message and returns key try { - database.put(message.key, message.record); + database.put(message.key.getBytes(), message.record); return message.key; } catch (final Exception e) { Log.logException(e); @@ -193,7 +193,7 @@ public class messageBoard { public entry read(final String key) { Map record; try { - record = database.get(key); + record = database.get(key.getBytes()); } catch (final IOException e) { return null; } @@ -202,7 +202,7 @@ public class messageBoard { public void remove(final String key) { try { - database.remove(key); + database.remove(key.getBytes()); } catch (final IOException e) { } } diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index 97490c829..0998b2512 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -85,7 +85,7 @@ public final class userDB { public void removeEntry(final String hostName) { try { - userTable.remove(hostName.toLowerCase()); + userTable.remove(hostName.toLowerCase().getBytes()); } catch (final IOException e) {} } @@ -95,7 +95,7 @@ public final class userDB { } Map record; try { - record = userTable.get(userName); + record = userTable.get(userName.getBytes()); } catch (final IOException e) { return null; } @@ -110,7 +110,7 @@ public final class userDB { public String addEntry(final Entry entry) { try { - userTable.put(entry.userName,entry.mem); + userTable.put(entry.userName.getBytes(), entry.mem); return entry.userName; } catch (final Exception e) { Log.logException(e); @@ -484,7 +484,7 @@ public final class userDB { } try { - userDB.this.userTable.put(getUserName(), this.mem); + userDB.this.userTable.put(getUserName().getBytes(), this.mem); } catch(final Exception e){ Log.logException(e); } @@ -501,7 +501,7 @@ public final class userDB { public void setProperty(final String propName, final String newValue) throws IOException, RowSpaceExceededException { this.mem.put(propName, newValue); - userDB.this.userTable.put(getUserName(), this.mem); + userDB.this.userTable.put(getUserName().getBytes(), this.mem); } public String getProperty(final String propName, final String defaultValue) { diff --git a/source/de/anomic/data/wiki/wikiBoard.java b/source/de/anomic/data/wiki/wikiBoard.java index 7b192bab0..2e5eab96b 100644 --- a/source/de/anomic/data/wiki/wikiBoard.java +++ b/source/de/anomic/data/wiki/wikiBoard.java @@ -265,9 +265,9 @@ public class wikiBoard { //System.out.println("key = " + page.key); //System.out.println("oldDate = " + oldDate); //System.out.println("record = " + oldEntry.record.toString()); - bkpbase.put(page.key + dateString(oldDate), oldEntry.record); + bkpbase.put((page.key + dateString(oldDate)).getBytes(), oldEntry.record); // write the new page - datbase.put(page.key, page.record); + datbase.put(page.key.getBytes(), page.record); return page.key; } catch (final Exception e) { Log.logException(e); @@ -283,7 +283,7 @@ public class wikiBoard { try { key = normalize(key); if (key.length() > keyLength) key = key.substring(0, keyLength); - final Map record = base.get(key); + final Map record = base.get(key.getBytes()); if (record == null) return newEntry(key, "anonymous", "127.0.0.1", "New Page", "".getBytes()); return new entry(key, record); } catch (final IOException e) { diff --git a/source/de/anomic/http/client/Cache.java b/source/de/anomic/http/client/Cache.java index 24019fbd3..5e57b928e 100644 --- a/source/de/anomic/http/client/Cache.java +++ b/source/de/anomic/http/client/Cache.java @@ -59,7 +59,7 @@ public final class Cache { private static Compressor fileDB = null; private static ArrayStack fileDBunbuffered = null; - private static long maxCacheSize = 0l; + private static long maxCacheSize = Long.MAX_VALUE; private static File cachePath = null; private static String prefix; public static final Log log = new Log("HTCACHE"); @@ -111,13 +111,14 @@ public final class Cache { public static void store(final DigestURI url, final ResponseHeader responseHeader, final byte[] file) throws IOException { if (responseHeader == null) throw new IOException("Cache.store of url " + url.toString() + " not possible: responseHeader == null"); if (file == null) throw new IOException("Cache.store of url " + url.toString() + " not possible: file == null"); + log.logInfo("storing content of url " + url.toString() + ", " + file.length + " bytes"); // store the response header into the header database final HashMap hm = new HashMap(); hm.putAll(responseHeader); hm.put("@@URL", url.toNormalform(true, false)); try { - responseHeaderDB.put(new String(url.hash()), hm); + responseHeaderDB.put(url.hash(), hm); } catch (Exception e) { throw new IOException("Cache.store: cannot write to headerDB: " + e.getMessage()); } @@ -140,20 +141,17 @@ public final class Cache { */ public static boolean has(final DigestURI url) { boolean headerExists; - try { - headerExists = responseHeaderDB.has(new String(url.hash())); - } catch (IOException e1) { - try { - fileDB.remove(url.hash()); - } catch (IOException e) {} - return false; - } + headerExists = responseHeaderDB.has(url.hash()); boolean fileExists = fileDB.has(url.hash()); if (headerExists && fileExists) return true; + if (!headerExists && !fileExists) return false; + // if not both is there then we do a clean-up if (headerExists) try { - responseHeaderDB.remove(new String(url.hash())); + log.logWarning("header but not content of url " + url.toString() + " in cache; cleaned up"); + responseHeaderDB.remove(url.hash()); } catch (IOException e) {} if (fileExists) try { + log.logWarning("content but not header of url " + url.toString() + " in cache; cleaned up"); fileDB.remove(url.hash()); } catch (IOException e) {} return false; @@ -173,8 +171,9 @@ public final class Cache { // loading data from database Map hdb; try { - hdb = responseHeaderDB.get(new String(url.hash())); + hdb = responseHeaderDB.get(url.hash()); } catch (final IOException e) { + Log.logException(e); return null; } if (hdb == null) return null; @@ -191,24 +190,29 @@ public final class Cache { * is returned. * @throws IOException */ - public static byte[] getContent(final DigestURI url) throws IOException { + public static byte[] getContent(final DigestURI url) { // load the url as resource from the cache try { - return fileDB.get(url.hash()); + byte[] b = fileDB.get(url.hash()); + if (b == null) return null; + log.logInfo("cache hit for url " + url.toString() + ", " + b.length + " bytes"); + return b; } catch (UnsupportedEncodingException e) { Log.logException(e); return null; + } catch (IOException e) { + Log.logException(e); + return null; } } - /** * removed response header and cached content from the database * @param url * @throws IOException */ public static void delete(final DigestURI url) throws IOException { - responseHeaderDB.remove(new String(url.hash())); + responseHeaderDB.remove(url.hash()); fileDB.remove(url.hash()); } } diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index 3778edbfc..8a44d9184 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -1700,7 +1700,7 @@ public final class Switchboard extends serverSwitch { final long parsingStartTime = System.currentTimeMillis(); // fetch the document from the response byte[] b = response.getContent(); - if (b == null) try { + if (b == null) { // fetch the document from cache b = Cache.getContent(response.url()); if (b == null) { @@ -1708,10 +1708,6 @@ public final class Switchboard extends serverSwitch { addURLtoErrorDB(response.url(), response.referrerHash(), response.initiator(), response.name(), "missing"); return null; } - } catch (IOException e) { - this.log.logWarning("Unable fetch the resource '" + response.url() + "'. from the cache: " + e.getMessage()); - addURLtoErrorDB(response.url(), response.referrerHash(), response.initiator(), response.name(), e.getMessage()); - return null; } try { diff --git a/source/de/anomic/yacy/graphics/OSMTile.java b/source/de/anomic/yacy/graphics/OSMTile.java index b98c11c4e..459e212e3 100644 --- a/source/de/anomic/yacy/graphics/OSMTile.java +++ b/source/de/anomic/yacy/graphics/OSMTile.java @@ -80,12 +80,7 @@ public class OSMTile { return null; } //System.out.println("*** DEBUG: fetching OSM tile: " + tileURL.toNormalform(true, true)); - byte[] tileb = null; - try { - tileb = Cache.getContent(tileURL); - } catch (IOException e1) { - Log.logException(e1); - } + byte[] tileb = Cache.getContent(tileURL); if (tileb == null) { // download resource using the crawler and keep resource in memory if possible Response entry = null; diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index d73eddda1..93b4ecb7d 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -268,9 +268,9 @@ public final class yacySeedDB implements AlternativeDomainNames { if (seedActiveDB.isEmpty() && seedPassiveDB.isEmpty() && seedPotentialDB.isEmpty()) return; // avoid that the own seed is initialized too early if (this.mySeed == null) initMySeed(); try { - seedActiveDB.remove(mySeed.hash); - seedPassiveDB.remove(mySeed.hash); - seedPotentialDB.remove(mySeed.hash); + seedActiveDB.remove(mySeed.hash.getBytes()); + seedPassiveDB.remove(mySeed.hash.getBytes()); + seedPotentialDB.remove(mySeed.hash.getBytes()); } catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } } @@ -292,12 +292,12 @@ public final class yacySeedDB implements AlternativeDomainNames { Log.logWarning("yacySeedDB", "could not create directories for "+ seedDBFile.getParent()); } try { - return new MapDataMining(seedDBFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, sortFields, longaccFields, doubleaccFields, null, this); + return new MapDataMining(seedDBFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, sortFields, longaccFields, doubleaccFields, this); } catch (final Exception e) { // try again FileUtils.deletedelete(seedDBFile); try { - return new MapDataMining(seedDBFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, sortFields, longaccFields, doubleaccFields, null, this); + return new MapDataMining(seedDBFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, sortFields, longaccFields, doubleaccFields, this); } catch (IOException e1) { Log.logException(e1); System.exit(-1); @@ -442,10 +442,10 @@ public final class yacySeedDB implements AlternativeDomainNames { nameLookupCache.put(seed.getName(), seed.hash); final Map seedPropMap = seed.getMap(); synchronized (seedPropMap) { - seedActiveDB.put(seed.hash, seedPropMap); + seedActiveDB.put(seed.hash.getBytes(), seedPropMap); } - seedPassiveDB.remove(seed.hash); - seedPotentialDB.remove(seed.hash); + seedPassiveDB.remove(seed.hash.getBytes()); + seedPotentialDB.remove(seed.hash.getBytes()); } catch (final Exception e) { yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); resetActiveTable(); @@ -456,14 +456,14 @@ public final class yacySeedDB implements AlternativeDomainNames { if (seed.isProper(false) != null) return; try { nameLookupCache.remove(seed.getName()); - seedActiveDB.remove(seed.hash); - seedPotentialDB.remove(seed.hash); + seedActiveDB.remove(seed.hash.getBytes()); + seedPotentialDB.remove(seed.hash.getBytes()); } catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } //seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime()))); try { final Map seedPropMap = seed.getMap(); synchronized (seedPropMap) { - seedPassiveDB.put(seed.hash, seedPropMap); + seedPassiveDB.put(seed.hash.getBytes(), seedPropMap); } } catch (final Exception e) { yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); @@ -475,14 +475,14 @@ public final class yacySeedDB implements AlternativeDomainNames { if (seed.isProper(false) != null) return; try { nameLookupCache.remove(seed.getName()); - seedActiveDB.remove(seed.hash); - seedPassiveDB.remove(seed.hash); + seedActiveDB.remove(seed.hash.getBytes()); + seedPassiveDB.remove(seed.hash.getBytes()); } catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } //seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime()))); try { final Map seedPropMap = seed.getMap(); synchronized (seedPropMap) { - seedPotentialDB.put(seed.hash, seedPropMap); + seedPotentialDB.put(seed.hash.getBytes(), seedPropMap); } } catch (final Exception e) { yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e); @@ -493,14 +493,14 @@ public final class yacySeedDB implements AlternativeDomainNames { public synchronized void removeDisconnected(final String peerHash) { if(peerHash == null) return; try { - seedPassiveDB.remove(peerHash); + seedPassiveDB.remove(peerHash.getBytes()); } catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } } public synchronized void removePotential(final String peerHash) { if(peerHash == null) return; try { - seedPotentialDB.remove(peerHash); + seedPotentialDB.remove(peerHash.getBytes()); } catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); } } @@ -521,7 +521,7 @@ public final class yacySeedDB implements AlternativeDomainNames { if ((this.mySeed != null) && (hash.equals(mySeed.hash))) return mySeed; ConcurrentHashMap entry = new ConcurrentHashMap(); try { - Map map = database.get(hash); + Map map = database.get(hash.getBytes()); if (map == null) return null; entry.putAll(map); } catch (final IOException e) { @@ -558,13 +558,13 @@ public final class yacySeedDB implements AlternativeDomainNames { } yacySeed s = get(hash, seedActiveDB); - if (s != null) try { seedActiveDB.put(hash, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} + if (s != null) try { seedActiveDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} s = get(hash, seedPassiveDB); - if (s != null) try { seedPassiveDB.put(hash, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} + if (s != null) try { seedPassiveDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} s = get(hash, seedPotentialDB); - if (s != null) try { seedPotentialDB.put(hash, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} + if (s != null) try { seedPotentialDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);} } public yacySeed lookupByName(String peerName) { diff --git a/source/net/yacy/kelondro/blob/Heap.java b/source/net/yacy/kelondro/blob/Heap.java index 954b83f7b..e9282a941 100755 --- a/source/net/yacy/kelondro/blob/Heap.java +++ b/source/net/yacy/kelondro/blob/Heap.java @@ -516,13 +516,13 @@ public final class Heap extends HeapModifier implements BLOB { try { //f.delete(); final MapHeap heap = new MapHeap(f, 12, NaturalOrder.naturalOrder, 1024 * 512, 500, '_'); - heap.put("aaaaaaaaaaaa", map("aaaaaaaaaaaa", "eins zwei drei")); - heap.put("aaaaaaaaaaab", map("aaaaaaaaaaab", "vier fuenf sechs")); - heap.put("aaaaaaaaaaac", map("aaaaaaaaaaac", "sieben acht neun")); - heap.put("aaaaaaaaaaad", map("aaaaaaaaaaad", "zehn elf zwoelf")); - heap.remove("aaaaaaaaaaab"); - heap.remove("aaaaaaaaaaac"); - heap.put("aaaaaaaaaaaX", map("aaaaaaaaaaad", "WXYZ")); + heap.put("aaaaaaaaaaaa".getBytes(), map("aaaaaaaaaaaa", "eins zwei drei")); + heap.put("aaaaaaaaaaab".getBytes(), map("aaaaaaaaaaab", "vier fuenf sechs")); + heap.put("aaaaaaaaaaac".getBytes(), map("aaaaaaaaaaac", "sieben acht neun")); + heap.put("aaaaaaaaaaad".getBytes(), map("aaaaaaaaaaad", "zehn elf zwoelf")); + heap.remove("aaaaaaaaaaab".getBytes()); + heap.remove("aaaaaaaaaaac".getBytes()); + heap.put("aaaaaaaaaaaX".getBytes(), map("aaaaaaaaaaad", "WXYZ")); heap.close(); } catch (final IOException e) { Log.logException(e); diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java index bd5d27ba7..861c35a6f 100644 --- a/source/net/yacy/kelondro/blob/MapDataMining.java +++ b/source/net/yacy/kelondro/blob/MapDataMining.java @@ -29,14 +29,11 @@ package net.yacy.kelondro.blob; import java.io.File; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import net.yacy.kelondro.index.RowSpaceExceededException; -import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.util.ScoreCluster; @@ -61,7 +58,6 @@ public class MapDataMining extends MapHeap { final String[] sortfields, final String[] longaccfields, final String[] doubleaccfields, - final Method externalInitializer, final Object externalHandler) throws IOException { super(heapFile, keylength, ordering, buffermax, cachesize, '_'); @@ -103,19 +99,19 @@ public class MapDataMining extends MapHeap { // fill cluster and accumulator with values if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) try { final CloneableIterator it = super.keys(true, false); - String mapname; + byte[] mapnameb; String cell; long valuel; double valued; Map map; while (it.hasNext()) { - mapname = new String(it.next()); - map = super.get(mapname); + mapnameb = it.next(); + map = super.get(mapnameb); if (map == null) break; if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) { cell = map.get(sortfields[i]); - if (cell != null) cluster[i].setScore(mapname, ScoreCluster.object2score(cell)); + if (cell != null) cluster[i].setScore(new String(mapnameb), ScoreCluster.object2score(cell)); } if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) { @@ -135,18 +131,6 @@ public class MapDataMining extends MapHeap { doubleaccumulator[i] = new Double(doubleaccumulator[i].doubleValue() + valued); } catch (final NumberFormatException e) {} } - - if ((externalHandler != null) && (externalInitializer != null)) { - try { - externalInitializer.invoke(externalHandler, new Object[]{mapname, map}); - } catch (final IllegalArgumentException e) { - Log.logException(e); - } catch (final IllegalAccessException e) { - Log.logException(e); - } catch (final InvocationTargetException e) { - Log.logException(e); - } - } } } catch (final IOException e) {} @@ -187,9 +171,9 @@ public class MapDataMining extends MapHeap { } @Override - public synchronized void put(final String key, final Map newMap) throws IOException, RowSpaceExceededException { + public synchronized void put(final byte[] key, final Map newMap) throws IOException, RowSpaceExceededException { assert (key != null); - assert (key.length() > 0); + assert (key.length > 0); assert (newMap != null); // update elementCount @@ -207,7 +191,7 @@ public class MapDataMining extends MapHeap { super.put(key, newMap); // update sortCluster - if (sortClusterMap != null) updateSortCluster(key, newMap); + if (sortClusterMap != null) updateSortCluster(new String(key), newMap); } private void updateAcc(final Map map, final boolean add) { @@ -260,7 +244,7 @@ public class MapDataMining extends MapHeap { } @Override - public synchronized void remove(final String key) throws IOException { + public synchronized void remove(final byte[] key) throws IOException { if (key == null) return; // update elementCount @@ -272,7 +256,7 @@ public class MapDataMining extends MapHeap { if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(map, false); // remove from sortCluster - if (sortfields != null) deleteSortCluster(key); + if (sortfields != null) deleteSortCluster(new String(key)); } } super.remove(key); @@ -390,10 +374,10 @@ public class MapDataMining extends MapHeap { private Map next0() { if (keyIterator == null) return null; - String nextKey; + byte[] nextKey; Map map; while (keyIterator.hasNext()) { - nextKey = new String(keyIterator.next()); + nextKey = keyIterator.next(); try { map = get(nextKey); } catch (final IOException e) { @@ -401,7 +385,7 @@ public class MapDataMining extends MapHeap { } assert map != null; if (map == null) continue; // circumvention of a modified exception - map.put("key", nextKey); + map.put("key", new String(nextKey)); return map; } return null; diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index e85283880..7ee64fe9c 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -128,21 +128,20 @@ public class MapHeap { * @throws IOException * @throws RowSpaceExceededException */ - public void put(String key, final Map newMap) throws IOException, RowSpaceExceededException { + public void put(byte[] key, final Map newMap) throws IOException, RowSpaceExceededException { assert key != null; - assert key.length() > 0; + assert key.length > 0; assert newMap != null; key = normalizeKey(key); - byte[] keyb = key.getBytes(); String s = map2string(newMap, "W" + DateFormatter.formatShortSecond() + " "); assert s != null; byte[] sb = s.getBytes(); synchronized (this) { // write entry - if (blob != null) blob.put(keyb, sb); + if (blob != null) blob.put(key, sb); // write map to cache - if (cache != null) cache.put(key, newMap); + if (cache != null) cache.put(new String(key), newMap); } } @@ -164,26 +163,6 @@ public class MapHeap { blob.remove(key); } } - - /** - * remove a Map - * @param key the primary key - * @throws IOException - */ - public void remove(String key) throws IOException { - // update elementCount - if (key == null) return; - key = normalizeKey(key); - byte[] keyb = key.getBytes(); - - synchronized (this) { - // remove from cache - cache.remove(key); - - // remove from file - blob.remove(keyb); - } - } /** * check if a specific key is in the database @@ -191,17 +170,6 @@ public class MapHeap { * @return * @throws IOException */ - public boolean has(String key) throws IOException { - assert key != null; - if (cache == null) return false; // case may appear during shutdown - key = normalizeKey(key); - byte[] keyb = key.getBytes(); - boolean h; - synchronized (this) { - h = this.cache.containsKey(key) || this.blob.has(keyb); - } - return h; - } public boolean has(byte[] key) { assert key != null; @@ -220,25 +188,10 @@ public class MapHeap { * @return * @throws IOException */ - public Map get(final String key) throws IOException { + public Map get(final byte[] key) throws IOException { if (key == null) return null; return get(key, true); } - - private String normalizeKey(String key) { - if (blob == null || key == null) return key; - if (key.length() > blob.keylength()) { - return key.substring(0, blob.keylength()); - } - if (key.length() < blob.keylength()) { - byte[] k = key.getBytes(); - byte[] b = new byte[blob.keylength()]; - System.arraycopy(k, 0, b, 0, k.length); - for (int i = k.length; i < b.length; i++) b[i] = (byte) fillchar; - return new String(b); - } - return key; - } private byte[] normalizeKey(byte[] key) { if (blob == null || key == null) return key; @@ -256,21 +209,21 @@ public class MapHeap { return key; } - protected Map get(String key, final boolean storeCache) throws IOException { + protected Map get(byte[] key, final boolean storeCache) throws IOException { // load map from cache assert key != null; if (cache == null) return null; // case may appear during shutdown key = normalizeKey(key); - byte[] keyb = key.getBytes(); Map map; if (storeCache) { synchronized (this) { - map = cache.get(key); + String keys = new String(key); + map = cache.get(keys); if (map != null) return map; // read object - final byte[] b = blob.get(keyb); + final byte[] b = blob.get(key); if (b == null) return null; try { map = bytes2map(b); @@ -279,7 +232,7 @@ public class MapHeap { } // write map to cache - cache.put(key, map); + cache.put(keys, map); } // return value @@ -287,9 +240,9 @@ public class MapHeap { } else { byte[] b; synchronized (this) { - map = cache.get(key); + map = cache.get(new String(key)); if (map != null) return map; - b = blob.get(keyb); + b = blob.get(key); } if (b == null) return null; try { @@ -390,7 +343,7 @@ public class MapHeap { return null; } try { - final Map obj = get(new String(nextKey)); + final Map obj = get(nextKey); if (obj == null) throw new kelondroException("no more elements available"); return obj; } catch (final IOException e) { @@ -413,9 +366,9 @@ public class MapHeap { MapHeap map = new MapHeap(f, 12, NaturalOrder.naturalOrder, 1024 * 1024, 1024, '_'); // put some values into the map Map m = new HashMap(); - m.put("k", "000"); map.put("123", m); - m.put("k", "111"); map.put("456", m); - m.put("k", "222"); map.put("789", m); + m.put("k", "000"); map.put("123".getBytes(), m); + m.put("k", "111"); map.put("456".getBytes(), m); + m.put("k", "222"); map.put("789".getBytes(), m); // iterate over keys Iterator i = map.keys(true, false); while (i.hasNext()) { diff --git a/source/net/yacy/repository/LoaderDispatcher.java b/source/net/yacy/repository/LoaderDispatcher.java index 39ac98ec7..a33a6e817 100644 --- a/source/net/yacy/repository/LoaderDispatcher.java +++ b/source/net/yacy/repository/LoaderDispatcher.java @@ -190,13 +190,7 @@ public final class LoaderDispatcher { // now see if there is a cache entry ResponseHeader cachedResponse = (request.url().isLocal()) ? null : Cache.getResponseHeader(request.url()); - byte[] content = null; - try { - content = (cachedResponse == null) ? null : Cache.getContent(request.url()); - } catch (IOException e) { - Log.logException(e); - content = null; - } + byte[] content = (cachedResponse == null) ? null : Cache.getContent(request.url()); if (cachedResponse != null && content != null) { // yes we have the content diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index a85b48c14..15dfe248e 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -917,7 +917,7 @@ public final class yacy { final String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"}; for (int i=0; i < dbFileNames.length; i++) { final File dbFile = new File(yacyDBPath,dbFileNames[i]); - final MapDataMining db = new MapDataMining(dbFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null); + final MapDataMining db = new MapDataMining(dbFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null); MapDataMining.mapIterator it; it = db.maps(true, false);