diff --git a/htroot/PerformanceGraph.java b/htroot/PerformanceGraph.java index 9b72660ff..da0b4b393 100644 --- a/htroot/PerformanceGraph.java +++ b/htroot/PerformanceGraph.java @@ -41,7 +41,7 @@ public class PerformanceGraph { final int width = post.getInt("width", 660); final int height = post.getInt("height", 240); - return plasmaProfiling.performanceGraph(width, height, sb.webIndex.metadata().size() + " URLS / " + sb.webIndex.index().getBackendSize() + " WORDS IN COLLECTIONS / " + sb.webIndex.index().getBufferSize() + " WORDS IN CACHE"); + return plasmaProfiling.performanceGraph(width, height, sb.webIndex.metadata().size() + " URLS / " + sb.webIndex.index().getBackendSize() + " WORDS IN INDEX / " + sb.webIndex.index().getBufferSize() + " WORDS IN CACHE"); } } \ No newline at end of file diff --git a/source/de/anomic/kelondro/blob/BLOBArray.java b/source/de/anomic/kelondro/blob/BLOBArray.java index 104d6f4cc..6a83a0080 100755 --- a/source/de/anomic/kelondro/blob/BLOBArray.java +++ b/source/de/anomic/kelondro/blob/BLOBArray.java @@ -74,17 +74,17 @@ public class BLOBArray implements BLOB { private long repositoryAgeMax; private long repositorySizeMax; private List blobs; - private String blobSalt; + private String prefix; private int buffersize; public BLOBArray( final File heapLocation, - final String blobSalt, + final String prefix, final int keylength, final ByteOrder ordering, final int buffersize) throws IOException { this.keylength = keylength; - this.blobSalt = blobSalt; + this.prefix = prefix; this.ordering = ordering; this.buffersize = buffersize; this.heapLocation = heapLocation; @@ -120,26 +120,39 @@ public class BLOBArray implements BLOB { } } if (deletions) files = heapLocation.list(); // make a fresh list + // migrate old file names + Date d; + long time; + deletions = false; + for (int i = 0; i < files.length; i++) { + if (files[i].length() >= 19 && files[i].endsWith(".blob")) { + try { + d = DateFormatter.parseShortSecond(files[i].substring(0, 14)); + new File(heapLocation, files[i]).renameTo(newBLOB(d)); + deletions = true; + } catch (ParseException e) {continue;} + } + } + if (deletions) files = heapLocation.list(); // make a fresh list // find maximum time: the file with this time will be given a write buffer - Date d; TreeMap sortedItems = new TreeMap(); BLOB oneBlob; File f; - long time, maxtime = 0; + long maxtime = 0; for (int i = 0; i < files.length; i++) { - if (files[i].length() >= 19 && files[i].endsWith(".blob")) { + if (files[i].length() >= 22 && files[i].startsWith(prefix) && files[i].endsWith(".blob")) { try { - d = DateFormatter.parseShortSecond(files[i].substring(0, 14)); + d = DateFormatter.parseShortMilliSecond(files[i].substring(prefix.length() + 1, prefix.length() + 18)); time = d.getTime(); if (time > maxtime) maxtime = time; } catch (ParseException e) {continue;} } } for (int i = 0; i < files.length; i++) { - if (files[i].length() >= 19 && files[i].endsWith(".blob")) { - try { - d = DateFormatter.parseShortSecond(files[i].substring(0, 14)); + if (files[i].length() >= 22 && files[i].startsWith(prefix) && files[i].endsWith(".blob")) { + try { + d = DateFormatter.parseShortMilliSecond(files[i].substring(prefix.length() + 1, prefix.length() + 18)); f = new File(heapLocation, files[i]); time = d.getTime(); oneBlob = (time == maxtime && buffersize > 0) ? new BLOBHeap(f, keylength, ordering, buffersize) : new BLOBHeapModifier(f, keylength, ordering); @@ -164,7 +177,7 @@ public class BLOBArray implements BLOB { public synchronized void mountBLOB(File location) throws IOException { Date d; try { - d = DateFormatter.parseShortSecond(location.getName().substring(0, 14)); + d = DateFormatter.parseShortMilliSecond(location.getName().substring(prefix.length() + 1, prefix.length() + 18)); } catch (ParseException e) { throw new IOException("date parse problem with file " + location.toString() + ": " + e.getMessage()); } @@ -291,7 +304,8 @@ public class BLOBArray implements BLOB { * @return */ public synchronized File newBLOB(Date creation) { - return new File(heapLocation, DateFormatter.formatShortSecond(creation) + "." + blobSalt + ".blob"); + //return new File(heapLocation, DateFormatter.formatShortSecond(creation) + "." + blobSalt + ".blob"); + return new File(heapLocation, prefix + "." + DateFormatter.formatShortMilliSecond(creation) + ".blob"); } public String name() { diff --git a/source/de/anomic/kelondro/index/ObjectIndex.java b/source/de/anomic/kelondro/index/ObjectIndex.java index 1b51d58e8..6646f7218 100644 --- a/source/de/anomic/kelondro/index/ObjectIndex.java +++ b/source/de/anomic/kelondro/index/ObjectIndex.java @@ -48,8 +48,6 @@ public interface ObjectIndex { public void put(Row.Entry row) throws IOException; public void put(List rows) throws IOException; // for R/W head path optimization public void addUnique(Row.Entry row) throws IOException; // no double-check - public void addUnique(List rows) throws IOException; // no double-check - //public long inc(final byte[] key, int col, long add, Row.Entry initrow); // replace a column with a recomputed value public ArrayList removeDoubles() throws IOException; // removes all elements that are double (to be used after all addUnique) public Row.Entry remove(byte[] key) throws IOException; public Row.Entry removeOne() throws IOException; diff --git a/source/de/anomic/kelondro/table/FlexTable.java b/source/de/anomic/kelondro/table/FlexTable.java index 92017f895..cf480b3ad 100644 --- a/source/de/anomic/kelondro/table/FlexTable.java +++ b/source/de/anomic/kelondro/table/FlexTable.java @@ -223,7 +223,18 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex { super.setMultiple(old_rows_ordered); // write new entries to index - addUnique(new_rows_sequential); + + // add a list of entries in a ordered way. + // this should save R/W head positioning time + final TreeMap indexed_result = super.addMultiple(new_rows_sequential); + // indexed_result is a Integer/byte[] relation + // that is used here to store the index + final Iterator> j = indexed_result.entrySet().iterator(); + Map.Entry entry; + while (j.hasNext()) { + entry = j.next(); + index.put(entry.getValue(), entry.getKey().intValue()); + } assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); } @@ -291,21 +302,6 @@ public class FlexTable extends FlexWidthArray implements ObjectIndex { index.putUnique(row.getColBytes(0), super.add(row)); } - public synchronized void addUnique(final List rows) throws IOException { - // add a list of entries in a ordered way. - // this should save R/W head positioning time - final TreeMap indexed_result = super.addMultiple(rows); - // indexed_result is a Integer/byte[] relation - // that is used here to store the index - final Iterator> i = indexed_result.entrySet().iterator(); - Map.Entry entry; - while (i.hasNext()) { - entry = i.next(); - index.put(entry.getValue(), entry.getKey().intValue()); - } - assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size(); - } - public synchronized ArrayList removeDoubles() throws IOException { final ArrayList report = new ArrayList(); RowSet rows; diff --git a/source/de/anomic/kelondro/table/SplitTable.java b/source/de/anomic/kelondro/table/SplitTable.java index 68afa85c9..063896704 100644 --- a/source/de/anomic/kelondro/table/SplitTable.java +++ b/source/de/anomic/kelondro/table/SplitTable.java @@ -328,12 +328,8 @@ public class SplitTable implements ObjectIndex { } public synchronized void addUnique(final Row.Entry row) throws IOException { - addUnique(row, null); - } - - public synchronized void addUnique(final Row.Entry row, Date entryDate) throws IOException { assert row.objectsize() <= this.rowdef.objectsize; - if ((entryDate == null) || (entryDate.after(new Date()))) entryDate = new Date(); // fix date + Date entryDate = new Date(); final String suffix = dateSuffix(entryDate); if (suffix == null) return; ObjectIndex table = tables.get(suffix); @@ -345,16 +341,6 @@ public class SplitTable implements ObjectIndex { table.addUnique(row); } - public synchronized void addUnique(final List rows) throws IOException { - final Iterator i = rows.iterator(); - while (i.hasNext()) addUnique(i.next()); - } - - public synchronized void addUniqueMultiple(final List rows, final Date entryDate) throws IOException { - final Iterator i = rows.iterator(); - while (i.hasNext()) addUnique(i.next(), entryDate); - } - public ArrayList removeDoubles() throws IOException { final Iterator i = tables.values().iterator(); final ArrayList report = new ArrayList(); diff --git a/source/de/anomic/kelondro/text/ReferenceContainerArray.java b/source/de/anomic/kelondro/text/ReferenceContainerArray.java index 61ec6ed32..19e1a6d5f 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainerArray.java +++ b/source/de/anomic/kelondro/text/ReferenceContainerArray.java @@ -143,6 +143,7 @@ public final class ReferenceContainerArray { } public boolean hasNext() { + if (this.iterator == null) return false; if (rot) return true; return iterator.hasNext(); } diff --git a/source/de/anomic/kelondro/util/DateFormatter.java b/source/de/anomic/kelondro/util/DateFormatter.java index 34b6c9edc..dda87a1f3 100644 --- a/source/de/anomic/kelondro/util/DateFormatter.java +++ b/source/de/anomic/kelondro/util/DateFormatter.java @@ -349,6 +349,9 @@ public final class DateFormatter { public static Date parseShortSecond(final String timeString) throws ParseException { return parse(FORMAT_SHORT_SECOND, timeString); } + public static Date parseShortMilliSecond(final String timeString) throws ParseException { + return parse(FORMAT_SHORT_MILSEC, timeString); +} /** * Like {@link #parseShortSecond(String)} using additional timezone information provided in an diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 6bae99bb9..5ddebf492 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -67,7 +67,7 @@ public final class plasmaHTCache { public static long maxCacheSize = 0l; public static File cachePath = null; - public static String salt; + public static String prefix; public static final Log log = new Log("HTCACHE"); @@ -85,7 +85,7 @@ public final class plasmaHTCache { cachePath = htCachePath; maxCacheSize = CacheSizeMax; - salt = peerSalt; + prefix = peerSalt; // reset old HTCache ? String[] list = cachePath.list(); @@ -136,7 +136,7 @@ public final class plasmaHTCache { } responseHeaderDB = new MapView(blob, 500); try { - fileDBunbuffered = new BLOBArray(new File(cachePath, FILE_DB_NAME), salt, 12, Base64Order.enhancedCoder, 1024 * 1024 * 2); + fileDBunbuffered = new BLOBArray(new File(cachePath, FILE_DB_NAME), prefix, 12, Base64Order.enhancedCoder, 1024 * 1024 * 2); fileDBunbuffered.setMaxSize(maxCacheSize); fileDB = new BLOBCompressor(fileDBunbuffered, 2 * 1024 * 1024); } catch (IOException e) { diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 186ece738..36cac1180 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -109,7 +109,7 @@ public final class plasmaWordIndex { final File indexPrimaryRoot, final File indexSecondaryRoot, final int entityCacheMaxSize, - final boolean useCommons, + final boolean useCommons, final int redundancy, final int partitionExponent, final boolean useCell) throws IOException {