- more space in error db to store larger error messages

- added hash to HTCACHE storage files which will make it possible to join separate caches by just copying files

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5329 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/crawler/CrawlQueues.java

@@ -78,8 +78,8 @@ public class CrawlQueues {
             // this is useful because there is currently no re-use of the data in this table.
             if (errorDBFile.isDirectory()) kelondroFlexWidthArray.delete(plasmaPath, "urlError2.db"); else errorDBFile.delete();
         }
-        errorURL = new ZURL(plasmaPath, "urlError2.db", false);
+        errorURL = new ZURL(plasmaPath, "urlError3.db", false);
-        delegatedURL = new ZURL(plasmaPath, "urlDelegated2.db", true);
+        delegatedURL = new ZURL(plasmaPath, "urlDelegated3.db", true);
     }
     
     public String urlExists(final String hash) {

source/de/anomic/crawler/ZURL.java

@@ -51,7 +51,7 @@ public class ZURL {
             "String executor-"  + yacySeedDB.commonHashLength + ", " + // the crawling executor
             "Cardinal workdate-8 {b256}, " +                           // the time when the url was last time tried to load
             "Cardinal workcount-4 {b256}, " +                          // number of load retries
-            "String anycause-80, " +                                   // string describing load failure
+            "String anycause-132, " +                                   // string describing load failure
             "byte[] entry-" + CrawlEntry.rowdef.objectsize,                                          // extra space
             kelondroBase64Order.enhancedCoder,
             0);

source/de/anomic/kelondro/kelondroBLOBArray.java

@@ -64,12 +64,15 @@ public class kelondroBLOBArray implements kelondroBLOB {
     private long repositoryAgeMax;
     private long repositorySizeMax;
     private List<blobItem> blobs;
+    private String blobSalt;
     
     public kelondroBLOBArray(
             final File heapLocation,
+            final String blobSalt,
             final int keylength,
             final kelondroByteOrder ordering) throws IOException {
         this.keylength = keylength;
+        this.blobSalt = blobSalt;
         this.ordering = ordering;
         this.heapLocation = heapLocation;
         this.fileAgeLimit = oneMonth;
@@ -91,7 +94,7 @@ public class kelondroBLOBArray implements kelondroBLOB {
         kelondroBLOB oneBlob;
         File f;
         for (int i = 0; i < files.length; i++) {
-            if (files[i].length() == 19 && files[i].endsWith("blob")) {
+            if (files[i].length() >= 19 && files[i].endsWith(".blob")) {
                try {
                    d = serverDate.parseShortSecond(files[i].substring(0, 14));
                } catch (ParseException e) {continue;}
@@ -164,7 +167,7 @@ public class kelondroBLOBArray implements kelondroBLOB {
         public blobItem() throws IOException {
             // make a new blob file and assign it in this item
             this.creation = new Date();
-            this.location = new File(heapLocation, serverDate.formatShortSecond(creation) + ".blob");
+            this.location = new File(heapLocation, serverDate.formatShortSecond(creation) + "." + blobSalt + ".blob");
             this.blob = new kelondroBLOBHeap(location, keylength, ordering);
         }
     }
@@ -316,7 +319,7 @@ public class kelondroBLOBArray implements kelondroBLOB {
         final File f = new File("/Users/admin/blobarraytest");
         try {
             //f.delete();
-            final kelondroBLOBArray heap = new kelondroBLOBArray(f, 12, kelondroNaturalOrder.naturalOrder);
+            final kelondroBLOBArray heap = new kelondroBLOBArray(f, "test", 12, kelondroNaturalOrder.naturalOrder);
             heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes());
             heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes());
             heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes());

source/de/anomic/plasma/plasmaHTCache.java

@@ -67,6 +67,7 @@ public final class plasmaHTCache {
     
     public static long maxCacheSize = 0l;
     public static File cachePath = null;
+    public static String salt;
     public static final serverLog log = new serverLog("HTCACHE");
     
 
@@ -80,10 +81,11 @@ public final class plasmaHTCache {
     public static final char LT_GLOBAL  = 'G';
 
     
-    public static void init(final File htCachePath, final long CacheSizeMax) {
+    public static void init(final File htCachePath, String peerSalt, final long CacheSizeMax) {
         
         cachePath = htCachePath;
         maxCacheSize = CacheSizeMax;
+        salt = peerSalt;
         
         // reset old HTCache ?
         String[] list = cachePath.list();
@@ -134,7 +136,7 @@ public final class plasmaHTCache {
         }
         responseHeaderDB = new kelondroMap(blob, 500);
         try {
-            fileDBunbuffered = new kelondroBLOBArray(new File(cachePath, FILE_DB_NAME), 12, kelondroBase64Order.enhancedCoder);
+            fileDBunbuffered = new kelondroBLOBArray(new File(cachePath, FILE_DB_NAME), salt, 12, kelondroBase64Order.enhancedCoder);
             fileDBunbuffered.setMaxSize(maxCacheSize);
             fileDB = new kelondroBLOBBuffer(fileDBunbuffered, 2 * 1024 * 1024, true);
         } catch (IOException e) {

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -406,7 +406,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
         htCachePath = getConfigPath(plasmaSwitchboardConstants.HTCACHE_PATH, plasmaSwitchboardConstants.HTCACHE_PATH_DEFAULT);
         this.log.logInfo("HTCACHE Path = " + htCachePath.getAbsolutePath());
         final long maxCacheSize = 1024 * 1024 * Long.parseLong(getConfig(plasmaSwitchboardConstants.PROXY_CACHE_SIZE, "2")); // this is megabyte
-        plasmaHTCache.init(htCachePath, maxCacheSize);
+        plasmaHTCache.init(htCachePath, webIndex.seedDB.mySeed().hash, maxCacheSize);
         
         // create the release download directory
         releasePath = getConfigPath(plasmaSwitchboardConstants.RELEASE_PATH, plasmaSwitchboardConstants.RELEASE_PATH_DEFAULT);