a different naming scheme for BLOBArray files. This may be necessary if blobs are written more often than once in a second.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5771 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/PerformanceGraph.java

@@ -41,7 +41,7 @@ public class PerformanceGraph {
         final int width = post.getInt("width", 660);
         final int height = post.getInt("height", 240);
         
-        return plasmaProfiling.performanceGraph(width, height, sb.webIndex.metadata().size() + " URLS / " + sb.webIndex.index().getBackendSize() + " WORDS IN COLLECTIONS / " + sb.webIndex.index().getBufferSize() + " WORDS IN CACHE");
+        return plasmaProfiling.performanceGraph(width, height, sb.webIndex.metadata().size() + " URLS / " + sb.webIndex.index().getBackendSize() + " WORDS IN INDEX / " + sb.webIndex.index().getBufferSize() + " WORDS IN CACHE");
     }
     
 }

source/de/anomic/kelondro/blob/BLOBArray.java

@@ -74,17 +74,17 @@ public class BLOBArray implements BLOB {
     private long repositoryAgeMax;
     private long repositorySizeMax;
     private List<blobItem> blobs;
-    private String blobSalt;
+    private String prefix;
     private int buffersize;
     
     public BLOBArray(
             final File heapLocation,
-            final String blobSalt,
+            final String prefix,
             final int keylength,
             final ByteOrder ordering,
             final int buffersize) throws IOException {
         this.keylength = keylength;
-        this.blobSalt = blobSalt;
+        this.prefix = prefix;
         this.ordering = ordering;
         this.buffersize = buffersize;
         this.heapLocation = heapLocation;
@@ -120,26 +120,39 @@ public class BLOBArray implements BLOB {
             }
         }
         if (deletions) files = heapLocation.list(); // make a fresh list
+        // migrate old file names
+        Date d;
+        long time;
+        deletions = false;
+        for (int i = 0; i < files.length; i++) {
+            if (files[i].length() >= 19 && files[i].endsWith(".blob")) {
+               try {
+                   d = DateFormatter.parseShortSecond(files[i].substring(0, 14));
+                   new File(heapLocation, files[i]).renameTo(newBLOB(d));
+                   deletions = true;
+               } catch (ParseException e) {continue;}
+            }
+        }
+        if (deletions) files = heapLocation.list(); // make a fresh list
         
         // find maximum time: the file with this time will be given a write buffer
-        Date d;
         TreeMap<Long, blobItem> sortedItems = new TreeMap<Long, blobItem>();
         BLOB oneBlob;
         File f;
-        long time, maxtime = 0;
+        long maxtime = 0;
         for (int i = 0; i < files.length; i++) {
-            if (files[i].length() >= 19 && files[i].endsWith(".blob")) {
+            if (files[i].length() >= 22 && files[i].startsWith(prefix) && files[i].endsWith(".blob")) {
                try {
-                   d = DateFormatter.parseShortSecond(files[i].substring(0, 14));
+                   d = DateFormatter.parseShortMilliSecond(files[i].substring(prefix.length() + 1, prefix.length() + 18));
                    time = d.getTime();
                    if (time > maxtime) maxtime = time;
                } catch (ParseException e) {continue;}
             }
         }
         for (int i = 0; i < files.length; i++) {
-            if (files[i].length() >= 19 && files[i].endsWith(".blob")) {
+            if (files[i].length() >= 22 && files[i].startsWith(prefix) && files[i].endsWith(".blob")) {
-               try {
+                try {
-                   d = DateFormatter.parseShortSecond(files[i].substring(0, 14));
+                   d = DateFormatter.parseShortMilliSecond(files[i].substring(prefix.length() + 1, prefix.length() + 18));
                    f = new File(heapLocation, files[i]);
                    time = d.getTime();
                    oneBlob = (time == maxtime && buffersize > 0) ? new BLOBHeap(f, keylength, ordering, buffersize) : new BLOBHeapModifier(f, keylength, ordering);
@@ -164,7 +177,7 @@ public class BLOBArray implements BLOB {
     public synchronized void mountBLOB(File location) throws IOException {
         Date d;
         try {
-            d = DateFormatter.parseShortSecond(location.getName().substring(0, 14));
+            d = DateFormatter.parseShortMilliSecond(location.getName().substring(prefix.length() + 1, prefix.length() + 18));
         } catch (ParseException e) {
             throw new IOException("date parse problem with file " + location.toString() + ": " + e.getMessage());
         }
@@ -291,7 +304,8 @@ public class BLOBArray implements BLOB {
      * @return
      */
     public synchronized File newBLOB(Date creation) {
-        return new File(heapLocation, DateFormatter.formatShortSecond(creation) + "." + blobSalt + ".blob");
+        //return new File(heapLocation, DateFormatter.formatShortSecond(creation) + "." + blobSalt + ".blob");
+        return new File(heapLocation, prefix + "." + DateFormatter.formatShortMilliSecond(creation) + ".blob");
     }
     
     public String name() {

source/de/anomic/kelondro/index/ObjectIndex.java

@@ -48,8 +48,6 @@ public interface ObjectIndex {
     public void put(Row.Entry row) throws IOException;
     public void put(List<Row.Entry> rows) throws IOException; // for R/W head path optimization
     public void addUnique(Row.Entry row) throws IOException; // no double-check
-    public void addUnique(List<Row.Entry> rows) throws IOException; // no double-check
-    //public long inc(final byte[] key, int col, long add, Row.Entry initrow); // replace a column with a recomputed value
     public ArrayList<RowCollection> removeDoubles() throws IOException; // removes all elements that are double (to be used after all addUnique)
     public Row.Entry remove(byte[] key) throws IOException;
     public Row.Entry removeOne() throws IOException;

source/de/anomic/kelondro/table/FlexTable.java

@@ -223,7 +223,18 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex {
         super.setMultiple(old_rows_ordered);
         
         // write new entries to index
-        addUnique(new_rows_sequential);
+        
+        // add a list of entries in a ordered way.
+        // this should save R/W head positioning time
+        final TreeMap<Integer, byte[]> indexed_result = super.addMultiple(new_rows_sequential);
+        // indexed_result is a Integer/byte[] relation
+        // that is used here to store the index
+        final Iterator<Map.Entry<Integer, byte[]>> j = indexed_result.entrySet().iterator();
+        Map.Entry<Integer, byte[]> entry;
+        while (j.hasNext()) {
+            entry = j.next();
+            index.put(entry.getValue(), entry.getKey().intValue());
+        }
         assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
     }
 
@@ -291,21 +302,6 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex {
 		index.putUnique(row.getColBytes(0), super.add(row));
     }
     
-    public synchronized void addUnique(final List<Row.Entry> rows) throws IOException {
-        // add a list of entries in a ordered way.
-        // this should save R/W head positioning time
-        final TreeMap<Integer, byte[]> indexed_result = super.addMultiple(rows);
-        // indexed_result is a Integer/byte[] relation
-        // that is used here to store the index
-        final Iterator<Map.Entry<Integer, byte[]>> i = indexed_result.entrySet().iterator();
-        Map.Entry<Integer, byte[]> entry;
-        while (i.hasNext()) {
-            entry = i.next();
-            index.put(entry.getValue(), entry.getKey().intValue());
-        }
-        assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size();
-    }
-    
     public synchronized ArrayList<RowCollection> removeDoubles() throws IOException {
         final ArrayList<RowCollection> report = new ArrayList<RowCollection>();
         RowSet rows;

source/de/anomic/kelondro/table/SplitTable.java

@@ -328,12 +328,8 @@ public class SplitTable implements ObjectIndex {
     }
     
     public synchronized void addUnique(final Row.Entry row) throws IOException {
-        addUnique(row, null);
-    }
-    
-    public synchronized void addUnique(final Row.Entry row, Date entryDate) throws IOException {
         assert row.objectsize() <= this.rowdef.objectsize;
-        if ((entryDate == null) || (entryDate.after(new Date()))) entryDate = new Date(); // fix date
+        Date entryDate = new Date();
         final String suffix = dateSuffix(entryDate);
         if (suffix == null) return;
         ObjectIndex table = tables.get(suffix);
@@ -345,16 +341,6 @@ public class SplitTable implements ObjectIndex {
         table.addUnique(row);
     }
     
-    public synchronized void addUnique(final List<Row.Entry> rows) throws IOException {
-        final Iterator<Row.Entry> i = rows.iterator();
-        while (i.hasNext()) addUnique(i.next());
-    }
-    
-    public synchronized void addUniqueMultiple(final List<Row.Entry> rows, final Date entryDate) throws IOException {
-        final Iterator<Row.Entry> i = rows.iterator();
-        while (i.hasNext()) addUnique(i.next(), entryDate);
-    }
-    
     public ArrayList<RowCollection> removeDoubles() throws IOException {
         final Iterator<ObjectIndex> i = tables.values().iterator();
         final ArrayList<RowCollection> report = new ArrayList<RowCollection>();

source/de/anomic/kelondro/text/ReferenceContainerArray.java

@@ -143,6 +143,7 @@ public final class ReferenceContainerArray {
         }
         
         public boolean hasNext() {
+            if (this.iterator == null) return false;
             if (rot) return true;
             return iterator.hasNext();
         }

source/de/anomic/kelondro/util/DateFormatter.java

@@ -349,6 +349,9 @@ public final class DateFormatter {
     public static Date parseShortSecond(final String timeString) throws ParseException {
             return parse(FORMAT_SHORT_SECOND, timeString);
     }
+    public static Date parseShortMilliSecond(final String timeString) throws ParseException {
+        return parse(FORMAT_SHORT_MILSEC, timeString);
+}
 
     /**
      * Like {@link #parseShortSecond(String)} using additional timezone information provided in an

source/de/anomic/plasma/plasmaHTCache.java

@@ -67,7 +67,7 @@ public final class plasmaHTCache {
     
     public static long maxCacheSize = 0l;
     public static File cachePath = null;
-    public static String salt;
+    public static String prefix;
     public static final Log log = new Log("HTCACHE");
     
 
@@ -85,7 +85,7 @@ public final class plasmaHTCache {
         
         cachePath = htCachePath;
         maxCacheSize = CacheSizeMax;
-        salt = peerSalt;
+        prefix = peerSalt;
         
         // reset old HTCache ?
         String[] list = cachePath.list();
@@ -136,7 +136,7 @@ public final class plasmaHTCache {
         }
         responseHeaderDB = new MapView(blob, 500);
         try {
-            fileDBunbuffered = new BLOBArray(new File(cachePath, FILE_DB_NAME), salt, 12, Base64Order.enhancedCoder, 1024 * 1024 * 2);
+            fileDBunbuffered = new BLOBArray(new File(cachePath, FILE_DB_NAME), prefix, 12, Base64Order.enhancedCoder, 1024 * 1024 * 2);
             fileDBunbuffered.setMaxSize(maxCacheSize);
             fileDB = new BLOBCompressor(fileDBunbuffered, 2 * 1024 * 1024);
         } catch (IOException e) {