From ef664386623a097ee50e768e2055e776152be0f1 Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 11 Nov 2008 21:42:12 +0000 Subject: [PATCH] - more space in error db to store larger error messages - added hash to HTCACHE storage files which will make it possible to join separate caches by just copying files git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5329 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- source/de/anomic/crawler/CrawlQueues.java | 4 ++-- source/de/anomic/crawler/ZURL.java | 2 +- source/de/anomic/kelondro/kelondroBLOBArray.java | 9 ++++++--- source/de/anomic/plasma/plasmaHTCache.java | 8 +++++--- source/de/anomic/plasma/plasmaSwitchboard.java | 6 +++--- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java index 0e60cf1d4..36450e566 100644 --- a/source/de/anomic/crawler/CrawlQueues.java +++ b/source/de/anomic/crawler/CrawlQueues.java @@ -78,8 +78,8 @@ public class CrawlQueues { // this is useful because there is currently no re-use of the data in this table. if (errorDBFile.isDirectory()) kelondroFlexWidthArray.delete(plasmaPath, "urlError2.db"); else errorDBFile.delete(); } - errorURL = new ZURL(plasmaPath, "urlError2.db", false); - delegatedURL = new ZURL(plasmaPath, "urlDelegated2.db", true); + errorURL = new ZURL(plasmaPath, "urlError3.db", false); + delegatedURL = new ZURL(plasmaPath, "urlDelegated3.db", true); } public String urlExists(final String hash) { diff --git a/source/de/anomic/crawler/ZURL.java b/source/de/anomic/crawler/ZURL.java index 6209df0f7..789a0598a 100755 --- a/source/de/anomic/crawler/ZURL.java +++ b/source/de/anomic/crawler/ZURL.java @@ -51,7 +51,7 @@ public class ZURL { "String executor-" + yacySeedDB.commonHashLength + ", " + // the crawling executor "Cardinal workdate-8 {b256}, " + // the time when the url was last time tried to load "Cardinal workcount-4 {b256}, " + // number of load retries - "String anycause-80, " + // string describing load failure + "String anycause-132, " + // string describing load failure "byte[] entry-" + CrawlEntry.rowdef.objectsize, // extra space kelondroBase64Order.enhancedCoder, 0); diff --git a/source/de/anomic/kelondro/kelondroBLOBArray.java b/source/de/anomic/kelondro/kelondroBLOBArray.java index c01ef48e4..e4e1d7850 100755 --- a/source/de/anomic/kelondro/kelondroBLOBArray.java +++ b/source/de/anomic/kelondro/kelondroBLOBArray.java @@ -64,12 +64,15 @@ public class kelondroBLOBArray implements kelondroBLOB { private long repositoryAgeMax; private long repositorySizeMax; private List blobs; + private String blobSalt; public kelondroBLOBArray( final File heapLocation, + final String blobSalt, final int keylength, final kelondroByteOrder ordering) throws IOException { this.keylength = keylength; + this.blobSalt = blobSalt; this.ordering = ordering; this.heapLocation = heapLocation; this.fileAgeLimit = oneMonth; @@ -91,7 +94,7 @@ public class kelondroBLOBArray implements kelondroBLOB { kelondroBLOB oneBlob; File f; for (int i = 0; i < files.length; i++) { - if (files[i].length() == 19 && files[i].endsWith("blob")) { + if (files[i].length() >= 19 && files[i].endsWith(".blob")) { try { d = serverDate.parseShortSecond(files[i].substring(0, 14)); } catch (ParseException e) {continue;} @@ -164,7 +167,7 @@ public class kelondroBLOBArray implements kelondroBLOB { public blobItem() throws IOException { // make a new blob file and assign it in this item this.creation = new Date(); - this.location = new File(heapLocation, serverDate.formatShortSecond(creation) + ".blob"); + this.location = new File(heapLocation, serverDate.formatShortSecond(creation) + "." + blobSalt + ".blob"); this.blob = new kelondroBLOBHeap(location, keylength, ordering); } } @@ -316,7 +319,7 @@ public class kelondroBLOBArray implements kelondroBLOB { final File f = new File("/Users/admin/blobarraytest"); try { //f.delete(); - final kelondroBLOBArray heap = new kelondroBLOBArray(f, 12, kelondroNaturalOrder.naturalOrder); + final kelondroBLOBArray heap = new kelondroBLOBArray(f, "test", 12, kelondroNaturalOrder.naturalOrder); heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes()); heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes()); heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes()); diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 49c168aa6..674139ede 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -67,6 +67,7 @@ public final class plasmaHTCache { public static long maxCacheSize = 0l; public static File cachePath = null; + public static String salt; public static final serverLog log = new serverLog("HTCACHE"); @@ -80,11 +81,12 @@ public final class plasmaHTCache { public static final char LT_GLOBAL = 'G'; - public static void init(final File htCachePath, final long CacheSizeMax) { + public static void init(final File htCachePath, String peerSalt, final long CacheSizeMax) { cachePath = htCachePath; maxCacheSize = CacheSizeMax; - + salt = peerSalt; + // reset old HTCache ? String[] list = cachePath.list(); if (list != null) { @@ -134,7 +136,7 @@ public final class plasmaHTCache { } responseHeaderDB = new kelondroMap(blob, 500); try { - fileDBunbuffered = new kelondroBLOBArray(new File(cachePath, FILE_DB_NAME), 12, kelondroBase64Order.enhancedCoder); + fileDBunbuffered = new kelondroBLOBArray(new File(cachePath, FILE_DB_NAME), salt, 12, kelondroBase64Order.enhancedCoder); fileDBunbuffered.setMaxSize(maxCacheSize); fileDB = new kelondroBLOBBuffer(fileDBunbuffered, 2 * 1024 * 1024, true); } catch (IOException e) { diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index c984097aa..5cf3ca03f 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -265,8 +265,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch