- fixed several bugs in database management functions

- fixed a display bug for the performance graph
- fixed deadlock when initialization of awt happens simultanously
- removed some debugging output

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5245 6c8d7289-2bf4-0310-a012-ef5d649a1542

build.properties

@@ -3,7 +3,7 @@ javacSource=1.5
 javacTarget=1.5
 
 # Release Configuration
-releaseVersion=0.611
+releaseVersion=0.61
 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz

htroot/xml/queues_p.java

@@ -38,6 +38,7 @@ import de.anomic.crawler.CrawlEntry;
 import de.anomic.crawler.IndexingStack;
 import de.anomic.crawler.NoticedURL;
 import de.anomic.http.httpRequestHeader;
+import de.anomic.kelondro.kelondroException;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.plasma.plasmaSwitchboardConstants;
 import de.anomic.server.serverObjects;
@@ -87,7 +88,11 @@ public class queues_p {
             // getting all enqueued entries
             if ((sb.webIndex.queuePreStack.size() > 0)) {
                 final Iterator<IndexingStack.QueueEntry> i1 = sb.webIndex.queuePreStack.entryIterator(false);
-                while (i1.hasNext()) entryList.add(i1.next());
+                while (i1.hasNext()) try {
+                    entryList.add(i1.next());
+                } catch (kelondroException e) {
+                    e.printStackTrace();
+                }
             }
             
             int size = (post == null) ? entryList.size() : post.getInt("num", entryList.size());

source/de/anomic/crawler/Balancer.java

@@ -365,7 +365,7 @@ public class Balancer {
     public synchronized void push(final CrawlEntry entry) throws IOException {
         assert entry != null;
         if (urlFileIndex.has(entry.url().hash().getBytes())) {
-            serverLog.logWarning("BALANCER", "double-check has failed for urlhash " + entry.url().hash()  + " in " + stackname + " - fixed");
+            //serverLog.logWarning("BALANCER", "double-check has failed for urlhash " + entry.url().hash()  + " in " + stackname + " - fixed");
             return;
         }
         

source/de/anomic/kelondro/kelondroCollectionIndex.java

@@ -535,7 +535,12 @@ public class kelondroCollectionIndex {
             final int oldSerialNumber = 0;
 
             // load the old collection and join it
-            collection.addAllUnique(getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false));
+            try {
+                collection.addAllUnique(getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false));
+            } catch (kelondroException e) {
+                // an error like "array does not contain expected row" may appear here. Just go on like if the collection does not exist
+                e.printStackTrace();
+            }
             collection.sort();
             collection.uniq(); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries
             collection.trim(false);
@@ -672,17 +677,27 @@ public class kelondroCollectionIndex {
         int removed = 0;
         assert (removekeys != null);
         // load the old collection and remove keys
-        final kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, serialNumber, false);
+        kelondroRowSet oldcollection = null;
-
+        try {
-        // remove the keys from the set
+            oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, serialNumber, false);
-        final Iterator<String> i = removekeys.iterator();
+        } catch (kelondroException e) {
-        while (i.hasNext()) {
+            // some internal problems occurred. however, go on like there was no old collection
-            if (oldcollection.remove(i.next().getBytes()) != null) removed++;
+            e.printStackTrace();
         }
-        oldcollection.sort();
+        // the oldcollection may be null here!
-        oldcollection.trim(false);
+        
-
+        if (oldcollection != null && oldcollection.size() > 0) {
-        if (oldcollection.size() == 0) {
+            // remove the keys from the set
+            final Iterator<String> i = removekeys.iterator();
+            while (i.hasNext()) {
+                if (oldcollection.remove(i.next().getBytes()) != null) removed++;
+            }
+            oldcollection.sort();
+            oldcollection.trim(false);
+        }
+        
+        // in case of an error or an empty collection, remove the collection:
+        if (oldcollection == null || oldcollection.size() == 0) {
             // delete the index entry and the array
             array_remove(
                     oldPartitionNumber, serialNumber, this.payloadrow.objectsize,
@@ -691,6 +706,7 @@ public class kelondroCollectionIndex {
             return removed;
         }
         
+        // now write to a new partition (which may be the same partition as the old one)
         final int newPartitionNumber = arrayIndex(oldcollection.size());
 
         // see if we need new space or if we can overwrite the old space
@@ -761,7 +777,8 @@ public class kelondroCollectionIndex {
             // the index appears to be corrupted
             this.indexErrors++;
             if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
-            throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row (error #" + indexErrors + ")");
+            serverLog.logWarning("kelondroCollectionIndex", "array " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + " does not contain expected row (error #" + indexErrors + ")");
+            return new kelondroRowSet(this.payloadrow, 0);
         }
 
         // read the row and define a collection
@@ -838,7 +855,7 @@ public class kelondroCollectionIndex {
             if (indexrow == null) return null;
             try {
                 return new Object[]{indexrow.getColBytes(0), getdelete(indexrow, false)};
-            } catch (final IOException e) {
+            } catch (final Exception e) {
                 e.printStackTrace();
                 return null;
             }

source/de/anomic/kelondro/kelondroEcoTable.java

@@ -29,6 +29,7 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.ConcurrentModificationException;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -74,7 +75,7 @@ public class kelondroEcoTable implements kelondroIndex {
     public kelondroEcoTable(final File tablefile, final kelondroRow rowdef, final int useTailCache, final int buffersize, final int initialSpace) {
         this.rowdef = rowdef;
         this.buffersize = buffersize;
-        this.fail = 0;
+        //this.fail = 0;
         assert rowdef.primaryKeyIndex == 0;
         // define the taildef, a row like the rowdef but without the first column
         final kelondroColumn[] cols = new kelondroColumn[rowdef.columns() - 1];
@@ -270,7 +271,10 @@ public class kelondroEcoTable implements kelondroIndex {
         assert ((table == null) || (table.size() == index.size()));
         final int i = (int) file.size();
         final boolean added = index.addi(row.getPrimaryKeyBytes(), i);
-        if (!added) return false;
+        if (!added) {
+            fail++;
+            return false;
+        }
         if (table != null) {
             assert table.size() == i;
             table.addUnique(taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true));
@@ -620,8 +624,7 @@ public class kelondroEcoTable implements kelondroIndex {
                 e.printStackTrace();
                 return null;
             }
-            assert this.c >= 0;
+            if (this.c < 0) throw new ConcurrentModificationException(); // this should only happen if the table was modified during the iteration
-            if (this.c < 0) return null;
             final byte[] b = new byte[rowdef.objectsize];
             if (table == null) {
                 // read from file

source/de/anomic/kelondro/kelondroRow.java

@@ -151,8 +151,10 @@ public final class kelondroRow {
     public final Entry newEntry(final byte[] rowinstance) {
         if (rowinstance == null) return null;
         //assert (rowinstance[0] != 0);
-        assert (this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth)) : "rowinstance[0] = " + new String(rowinstance, 0, row[0].cellwidth) + " / " + serverLog.arrayList(rowinstance, 0, row[0].cellwidth);
+        if (!(this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth))) {
-        if (!(this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth))) return null;
+            serverLog.logWarning("kelondroRow", "row not well-formed: rowinstance[0] = " + new String(rowinstance, 0, row[0].cellwidth) + " / " + serverLog.arrayList(rowinstance, 0, row[0].cellwidth));
+            return null;
+        }
         return new Entry(rowinstance, false);
     }
     

source/de/anomic/kelondro/kelondroStack.java

@@ -113,7 +113,8 @@ public final class kelondroStack extends kelondroFullRecords {
                 nextHandle = new EcoNode(nextHandle).getOHHandle((up) ? right : left);
                 return row().newEntry(new EcoNode(lastHandle).getValueRow());
             } catch (final IOException e) {
-                throw new kelondroException(filename, "IO error at Counter:next()");
+                e.printStackTrace();
+                throw new kelondroException(filename, "IO error at stackIterator.next(): " + e.getMessage());
             }
         }
 

source/de/anomic/plasma/plasmaGrafics.java

@@ -62,9 +62,9 @@ public class plasmaGrafics {
     private static final String COL_WE_LINE        = "FFAAAA";
     private static final String COL_WE_TEXT        = "FFCCCC";
     
-    private static final Color  COL_BORDER      = new Color(  0,   0,   0);
+    private static final String COL_BORDER         = "000000";
-    private static final Color  COL_NORMAL_TEXT = new Color(  0,   0,   0);
+    private static final String COL_NORMAL_TEXT    = "000000";
-    private static final Color  COL_LOAD_BG     = new Color(247, 247, 247);
+    private static final String COL_LOAD_BG        = "F7F7F7";
     
     public static class CircleThreadPiece {
         private final String pieceName;
@@ -294,7 +294,7 @@ public class plasmaGrafics {
     	//prepare image
     	peerloadPicture = new BufferedImage(width,height,BufferedImage.TYPE_INT_RGB);
         final Graphics2D g = peerloadPicture.createGraphics();
-        g.setBackground(COL_LOAD_BG);
+        g.setBackground(Color.decode("0x"+COL_LOAD_BG));
         g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
         g.clearRect(0,0,width,height);
         
@@ -321,7 +321,7 @@ public class plasmaGrafics {
         drawLegendLine(g, 5, height - 5 - 15 * i, fillRest.getPieceName()+" ("+fillRest.getFractionPercent()+" %)", fillRest.getColor());
         
         //draw border around the circle
-        g.setColor(COL_BORDER);
+        g.setColor(Color.decode("0x"+COL_BORDER));
         g.drawArc(circ_x, circ_y, circ_w, circ_w, 0, 360);
         
         peerloadPictureDate = System.currentTimeMillis();
@@ -330,10 +330,10 @@ public class plasmaGrafics {
     private static void drawLegendLine(final Graphics2D g, final int x, final int y, final String caption, final Color item_color) {
     	g.setColor(item_color);
     	g.fillRect(x, y-LEGEND_BOX_SIZE, LEGEND_BOX_SIZE, LEGEND_BOX_SIZE);
-    	g.setColor(COL_BORDER);
+    	g.setColor(Color.decode("0x"+COL_BORDER));
     	g.drawRect(x, y-LEGEND_BOX_SIZE, LEGEND_BOX_SIZE, LEGEND_BOX_SIZE);
     	
-    	g.setColor(COL_NORMAL_TEXT);
+    	g.setColor(Color.decode("0x"+COL_NORMAL_TEXT));
     	g.drawChars(caption.toCharArray(), 0, caption.length(), x+LEGEND_BOX_SIZE+5,y);
     }
 

source/de/anomic/plasma/plasmaProfiling.java

@@ -53,7 +53,7 @@ public class plasmaProfiling {
     public static ymageMatrix performanceGraph(final int width, final int height, final String subline) {        
         // find maximum values for automatic graph dimension adoption
         final int maxppm = (int) maxPayload("ppm", 25);
-        final int maxwords = (int) maxPayload("wordcache", 10000);
+        final int maxwords = (int) maxPayload("wordcache", 12000);
         final long maxbytes = maxPayload("memory", 110 * 1024 * 1024);
         
         // declare graph and set dimensions
@@ -61,7 +61,7 @@ public class plasmaProfiling {
         final int rightborder = 30;
         final int topborder = 20;
         final int bottomborder = 20;
-        final int leftscale = 20000;
+        final int leftscale = 10000;
         final int rightscale = 100;
         final int anotscale = 50;
         final int bottomscale = 60;
@@ -70,7 +70,7 @@ public class plasmaProfiling {
         final int maxtime = 600;
         ymageChart chart = new ymageChart(width, height, "FFFFFF", "000000", "AAAAAA", leftborder, rightborder, topborder, bottomborder, "YACY PEER PERFORMANCE: MAIN MEMORY, WORD CACHE AND PAGES/MINUTE (PPM)", subline);
         chart.declareDimension(ymageChart.DIMENSION_BOTTOM, bottomscale, hspace / (maxtime / bottomscale), -maxtime, "000000", "CCCCCC", "TIME/SECONDS");
-        chart.declareDimension(ymageChart.DIMENSION_LEFT, leftscale, vspace * leftscale / maxwords, 0, "008800", null , "WORDS IN CACHE");
+        chart.declareDimension(ymageChart.DIMENSION_LEFT, leftscale, vspace * leftscale / maxwords, 0, "008800", null , "INDEXING, WORDS IN CACHE");
         chart.declareDimension(ymageChart.DIMENSION_RIGHT, rightscale, vspace * rightscale / (int)(maxbytes / 1024 / 1024), 0, "0000FF", "CCCCCC", "MEMORY/MEGABYTE");
         chart.declareDimension(ymageChart.DIMENSION_ANOT, anotscale, vspace * anotscale / maxppm, 0, "008800", null , "PPM [PAGES/MINUTE]");