- fix for NPE when loading the cytag image

- some hacks for less memory usage:
-- less usage of buffer and cache memory in EcoFS
-- buffer allocation on-demand in BufferedIOChunks
-- removed largest ybr idx

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5595 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/data/URLLicense.java

@@ -77,13 +77,6 @@ public class URLLicense {
     public yacyURL releaseLicense(final String license) {
         yacyURL url = null;
         url = permissions.remove(license);
-        /*
-        if (url == null) {
-            System.out.println("DEBUG-URLLICENSE: no URL license present for code=" + license);
-        } else {
-            System.out.println("DEBUG-URLLICENSE: granted download of " + url.toString());
-        }
-        */
         return url;
     }
     

source/de/anomic/http/HttpClient.java

@@ -75,6 +75,9 @@ public abstract class HttpClient {
      * @param timeout in milliseconds
      * @return
      */
+    public static byte[] wget(final String uri) {
+        return wget(uri, new httpRequestHeader(), 10000, null);
+    }
     public static byte[] wget(final String uri, final httpRequestHeader header, final int timeout) {
         return wget(uri, header, timeout, null);
     }

source/de/anomic/kelondro/index/RowCollection.java

@@ -56,11 +56,11 @@ public class RowCollection implements Iterable<Row.Entry> {
     public static final ExecutorService sortingthreadexecutor = (serverProcessor.useCPU > 1) ? Executors.newCachedThreadPool(new NamePrefixThreadFactory("sorting")) : null;
     public static final ExecutorService partitionthreadexecutor = (serverProcessor.useCPU > 1) ? Executors.newCachedThreadPool(new NamePrefixThreadFactory("partition")) : null;
     
-    protected byte[]        chunkcache;
-    protected int           chunkcount;
-    protected long          lastTimeRead, lastTimeWrote;    
-    public Row   rowdef;
-    protected int           sortBound;
+    protected byte[] chunkcache;
+    protected int    chunkcount;
+    protected long   lastTimeRead, lastTimeWrote;    
+    public Row       rowdef;
+    protected int    sortBound;
 
     private static final int exp_chunkcount  = 0;
     private static final int exp_last_read   = 1;

source/de/anomic/kelondro/io/BufferedIOChunks.java

@@ -33,7 +33,7 @@ public final class BufferedIOChunks extends AbstractIOChunks implements IOChunks
     protected RandomAccessInterface ra;
     private int bufferSize;
     private final long commitTimeout;
-    private final byte[] buffer;
+    private byte[] buffer;
     private long lastCommit = 0;
     
     public BufferedIOChunks(final RandomAccessInterface ra, final String name, final int buffersize, final long commitTimeout) {
@@ -41,7 +41,7 @@ public final class BufferedIOChunks extends AbstractIOChunks implements IOChunks
         this.ra = ra;
         this.bufferSize = 0;
         this.commitTimeout = commitTimeout;
-        this.buffer = new byte[buffersize]; // this is a buffer at the end of the file
+        this.buffer = null; // this is a buffer at the end of the file. It will be initialized if necessary
         this.lastCommit = System.currentTimeMillis();
     }
 
@@ -64,6 +64,7 @@ public final class BufferedIOChunks extends AbstractIOChunks implements IOChunks
         // do the read
         if (pos >= this.ra.length()) {
             // read from the buffer
+            if (this.buffer == null) this.buffer = new byte[this.bufferSize];
             System.arraycopy(this.buffer, (int) (pos - this.ra.length()), b, off, len);
         } else if (pos + len >= this.ra.length()) {
             // the content is partly in the file and partly in the buffer
@@ -84,13 +85,14 @@ public final class BufferedIOChunks extends AbstractIOChunks implements IOChunks
         if (len == 0) return;
         if (pos >= this.ra.length()) {
             // the position is fully outside of the file
-            if (pos - this.ra.length() + len > this.buffer.length) {
+            if (this.buffer != null && pos - this.ra.length() + len > this.buffer.length) {
                 // this does not fit into the buffer
                 commit();
                 this.ra.seek(pos);
                 this.ra.write(b, off, len);
                 return;
             }
+            if (this.buffer == null) this.buffer = new byte[this.bufferSize];
             System.arraycopy(b, off, this.buffer, (int) (pos - this.ra.length()), len);
             this.bufferSize = (int) Math.max(this.bufferSize, pos - this.ra.length() + len);
             return;
@@ -110,7 +112,7 @@ public final class BufferedIOChunks extends AbstractIOChunks implements IOChunks
 
     public synchronized void commit() throws IOException {
         this.lastCommit = System.currentTimeMillis();
-        if (this.bufferSize == 0) return;
+        if (this.buffer == null || this.bufferSize == 0) return;
         this.ra.seek(this.ra.length()); // move to end of file
         this.ra.write(this.buffer, 0, this.bufferSize);
         this.bufferSize = 0;

source/de/anomic/kelondro/io/EcoFS.java

@@ -67,8 +67,8 @@ public class EcoFS {
     /**
      * stay below hard disc cache (is that necessary?)
      */
-    private static final int maxReadCache = 256 * 1024;
-    private static final int maxWriteBuffer = 256 * 1024;
+    private static final int maxReadCache = 16 * 1024;
+    private static final int maxWriteBuffer = 16 * 1024;
     
     
     public EcoFS(final File tablefile, final int recordsize) throws IOException {

source/de/anomic/plasma/plasmaSwitchboard.java

@@ -635,6 +635,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack.
         //plasmaSnippetCache.result scr = snippetCache.retrieve(new URL("http://www.heise.de/kiosk/archiv/ct/2003/4/20"), query, true, 260);
 
         this.dbImportManager = new ImporterManager();
+        this.trail = new ArrayList<String>();
         
         log.logConfig("Finished Switchboard Initialization");
     }