diff --git a/build.properties b/build.properties index 37585ee80..0b10f10de 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.5 javacTarget=1.5 # Release Configuration -releaseVersion=0.611 +releaseVersion=0.61 stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz diff --git a/htroot/xml/queues_p.java b/htroot/xml/queues_p.java index 2fb2ab792..ce8ca9103 100644 --- a/htroot/xml/queues_p.java +++ b/htroot/xml/queues_p.java @@ -38,6 +38,7 @@ import de.anomic.crawler.CrawlEntry; import de.anomic.crawler.IndexingStack; import de.anomic.crawler.NoticedURL; import de.anomic.http.httpRequestHeader; +import de.anomic.kelondro.kelondroException; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboardConstants; import de.anomic.server.serverObjects; @@ -87,7 +88,11 @@ public class queues_p { // getting all enqueued entries if ((sb.webIndex.queuePreStack.size() > 0)) { final Iterator i1 = sb.webIndex.queuePreStack.entryIterator(false); - while (i1.hasNext()) entryList.add(i1.next()); + while (i1.hasNext()) try { + entryList.add(i1.next()); + } catch (kelondroException e) { + e.printStackTrace(); + } } int size = (post == null) ? entryList.size() : post.getInt("num", entryList.size()); diff --git a/source/de/anomic/crawler/Balancer.java b/source/de/anomic/crawler/Balancer.java index 5fdf873d3..e0cde994a 100644 --- a/source/de/anomic/crawler/Balancer.java +++ b/source/de/anomic/crawler/Balancer.java @@ -365,7 +365,7 @@ public class Balancer { public synchronized void push(final CrawlEntry entry) throws IOException { assert entry != null; if (urlFileIndex.has(entry.url().hash().getBytes())) { - serverLog.logWarning("BALANCER", "double-check has failed for urlhash " + entry.url().hash() + " in " + stackname + " - fixed"); + //serverLog.logWarning("BALANCER", "double-check has failed for urlhash " + entry.url().hash() + " in " + stackname + " - fixed"); return; } diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index 33834c4ff..e635de3ec 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -535,7 +535,12 @@ public class kelondroCollectionIndex { final int oldSerialNumber = 0; // load the old collection and join it - collection.addAllUnique(getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false)); + try { + collection.addAllUnique(getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false)); + } catch (kelondroException e) { + // an error like "array does not contain expected row" may appear here. Just go on like if the collection does not exist + e.printStackTrace(); + } collection.sort(); collection.uniq(); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries collection.trim(false); @@ -672,17 +677,27 @@ public class kelondroCollectionIndex { int removed = 0; assert (removekeys != null); // load the old collection and remove keys - final kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, serialNumber, false); - - // remove the keys from the set - final Iterator i = removekeys.iterator(); - while (i.hasNext()) { - if (oldcollection.remove(i.next().getBytes()) != null) removed++; + kelondroRowSet oldcollection = null; + try { + oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, serialNumber, false); + } catch (kelondroException e) { + // some internal problems occurred. however, go on like there was no old collection + e.printStackTrace(); } - oldcollection.sort(); - oldcollection.trim(false); - - if (oldcollection.size() == 0) { + // the oldcollection may be null here! + + if (oldcollection != null && oldcollection.size() > 0) { + // remove the keys from the set + final Iterator i = removekeys.iterator(); + while (i.hasNext()) { + if (oldcollection.remove(i.next().getBytes()) != null) removed++; + } + oldcollection.sort(); + oldcollection.trim(false); + } + + // in case of an error or an empty collection, remove the collection: + if (oldcollection == null || oldcollection.size() == 0) { // delete the index entry and the array array_remove( oldPartitionNumber, serialNumber, this.payloadrow.objectsize, @@ -691,6 +706,7 @@ public class kelondroCollectionIndex { return removed; } + // now write to a new partition (which may be the same partition as the old one) final int newPartitionNumber = arrayIndex(oldcollection.size()); // see if we need new space or if we can overwrite the old space @@ -761,7 +777,8 @@ public class kelondroCollectionIndex { // the index appears to be corrupted this.indexErrors++; if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild - throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row (error #" + indexErrors + ")"); + serverLog.logWarning("kelondroCollectionIndex", "array " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + " does not contain expected row (error #" + indexErrors + ")"); + return new kelondroRowSet(this.payloadrow, 0); } // read the row and define a collection @@ -838,7 +855,7 @@ public class kelondroCollectionIndex { if (indexrow == null) return null; try { return new Object[]{indexrow.getColBytes(0), getdelete(indexrow, false)}; - } catch (final IOException e) { + } catch (final Exception e) { e.printStackTrace(); return null; } diff --git a/source/de/anomic/kelondro/kelondroEcoTable.java b/source/de/anomic/kelondro/kelondroEcoTable.java index 6987ab431..e7f84eb29 100644 --- a/source/de/anomic/kelondro/kelondroEcoTable.java +++ b/source/de/anomic/kelondro/kelondroEcoTable.java @@ -29,6 +29,7 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.ConcurrentModificationException; import java.util.Date; import java.util.HashMap; import java.util.Iterator; @@ -74,7 +75,7 @@ public class kelondroEcoTable implements kelondroIndex { public kelondroEcoTable(final File tablefile, final kelondroRow rowdef, final int useTailCache, final int buffersize, final int initialSpace) { this.rowdef = rowdef; this.buffersize = buffersize; - this.fail = 0; + //this.fail = 0; assert rowdef.primaryKeyIndex == 0; // define the taildef, a row like the rowdef but without the first column final kelondroColumn[] cols = new kelondroColumn[rowdef.columns() - 1]; @@ -270,7 +271,10 @@ public class kelondroEcoTable implements kelondroIndex { assert ((table == null) || (table.size() == index.size())); final int i = (int) file.size(); final boolean added = index.addi(row.getPrimaryKeyBytes(), i); - if (!added) return false; + if (!added) { + fail++; + return false; + } if (table != null) { assert table.size() == i; table.addUnique(taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true)); @@ -620,8 +624,7 @@ public class kelondroEcoTable implements kelondroIndex { e.printStackTrace(); return null; } - assert this.c >= 0; - if (this.c < 0) return null; + if (this.c < 0) throw new ConcurrentModificationException(); // this should only happen if the table was modified during the iteration final byte[] b = new byte[rowdef.objectsize]; if (table == null) { // read from file diff --git a/source/de/anomic/kelondro/kelondroRow.java b/source/de/anomic/kelondro/kelondroRow.java index dee5201d2..2b1f370fc 100644 --- a/source/de/anomic/kelondro/kelondroRow.java +++ b/source/de/anomic/kelondro/kelondroRow.java @@ -151,8 +151,10 @@ public final class kelondroRow { public final Entry newEntry(final byte[] rowinstance) { if (rowinstance == null) return null; //assert (rowinstance[0] != 0); - assert (this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth)) : "rowinstance[0] = " + new String(rowinstance, 0, row[0].cellwidth) + " / " + serverLog.arrayList(rowinstance, 0, row[0].cellwidth); - if (!(this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth))) return null; + if (!(this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth))) { + serverLog.logWarning("kelondroRow", "row not well-formed: rowinstance[0] = " + new String(rowinstance, 0, row[0].cellwidth) + " / " + serverLog.arrayList(rowinstance, 0, row[0].cellwidth)); + return null; + } return new Entry(rowinstance, false); } diff --git a/source/de/anomic/kelondro/kelondroStack.java b/source/de/anomic/kelondro/kelondroStack.java index 8dc4f2960..17c576411 100644 --- a/source/de/anomic/kelondro/kelondroStack.java +++ b/source/de/anomic/kelondro/kelondroStack.java @@ -113,7 +113,8 @@ public final class kelondroStack extends kelondroFullRecords { nextHandle = new EcoNode(nextHandle).getOHHandle((up) ? right : left); return row().newEntry(new EcoNode(lastHandle).getValueRow()); } catch (final IOException e) { - throw new kelondroException(filename, "IO error at Counter:next()"); + e.printStackTrace(); + throw new kelondroException(filename, "IO error at stackIterator.next(): " + e.getMessage()); } } diff --git a/source/de/anomic/plasma/plasmaGrafics.java b/source/de/anomic/plasma/plasmaGrafics.java index c57410402..d1bd1c0c6 100644 --- a/source/de/anomic/plasma/plasmaGrafics.java +++ b/source/de/anomic/plasma/plasmaGrafics.java @@ -62,9 +62,9 @@ public class plasmaGrafics { private static final String COL_WE_LINE = "FFAAAA"; private static final String COL_WE_TEXT = "FFCCCC"; - private static final Color COL_BORDER = new Color( 0, 0, 0); - private static final Color COL_NORMAL_TEXT = new Color( 0, 0, 0); - private static final Color COL_LOAD_BG = new Color(247, 247, 247); + private static final String COL_BORDER = "000000"; + private static final String COL_NORMAL_TEXT = "000000"; + private static final String COL_LOAD_BG = "F7F7F7"; public static class CircleThreadPiece { private final String pieceName; @@ -294,7 +294,7 @@ public class plasmaGrafics { //prepare image peerloadPicture = new BufferedImage(width,height,BufferedImage.TYPE_INT_RGB); final Graphics2D g = peerloadPicture.createGraphics(); - g.setBackground(COL_LOAD_BG); + g.setBackground(Color.decode("0x"+COL_LOAD_BG)); g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON); g.clearRect(0,0,width,height); @@ -321,7 +321,7 @@ public class plasmaGrafics { drawLegendLine(g, 5, height - 5 - 15 * i, fillRest.getPieceName()+" ("+fillRest.getFractionPercent()+" %)", fillRest.getColor()); //draw border around the circle - g.setColor(COL_BORDER); + g.setColor(Color.decode("0x"+COL_BORDER)); g.drawArc(circ_x, circ_y, circ_w, circ_w, 0, 360); peerloadPictureDate = System.currentTimeMillis(); @@ -330,10 +330,10 @@ public class plasmaGrafics { private static void drawLegendLine(final Graphics2D g, final int x, final int y, final String caption, final Color item_color) { g.setColor(item_color); g.fillRect(x, y-LEGEND_BOX_SIZE, LEGEND_BOX_SIZE, LEGEND_BOX_SIZE); - g.setColor(COL_BORDER); + g.setColor(Color.decode("0x"+COL_BORDER)); g.drawRect(x, y-LEGEND_BOX_SIZE, LEGEND_BOX_SIZE, LEGEND_BOX_SIZE); - g.setColor(COL_NORMAL_TEXT); + g.setColor(Color.decode("0x"+COL_NORMAL_TEXT)); g.drawChars(caption.toCharArray(), 0, caption.length(), x+LEGEND_BOX_SIZE+5,y); } diff --git a/source/de/anomic/plasma/plasmaProfiling.java b/source/de/anomic/plasma/plasmaProfiling.java index 8d6b2bf77..9ed2f8b9c 100644 --- a/source/de/anomic/plasma/plasmaProfiling.java +++ b/source/de/anomic/plasma/plasmaProfiling.java @@ -53,7 +53,7 @@ public class plasmaProfiling { public static ymageMatrix performanceGraph(final int width, final int height, final String subline) { // find maximum values for automatic graph dimension adoption final int maxppm = (int) maxPayload("ppm", 25); - final int maxwords = (int) maxPayload("wordcache", 10000); + final int maxwords = (int) maxPayload("wordcache", 12000); final long maxbytes = maxPayload("memory", 110 * 1024 * 1024); // declare graph and set dimensions @@ -61,7 +61,7 @@ public class plasmaProfiling { final int rightborder = 30; final int topborder = 20; final int bottomborder = 20; - final int leftscale = 20000; + final int leftscale = 10000; final int rightscale = 100; final int anotscale = 50; final int bottomscale = 60; @@ -70,7 +70,7 @@ public class plasmaProfiling { final int maxtime = 600; ymageChart chart = new ymageChart(width, height, "FFFFFF", "000000", "AAAAAA", leftborder, rightborder, topborder, bottomborder, "YACY PEER PERFORMANCE: MAIN MEMORY, WORD CACHE AND PAGES/MINUTE (PPM)", subline); chart.declareDimension(ymageChart.DIMENSION_BOTTOM, bottomscale, hspace / (maxtime / bottomscale), -maxtime, "000000", "CCCCCC", "TIME/SECONDS"); - chart.declareDimension(ymageChart.DIMENSION_LEFT, leftscale, vspace * leftscale / maxwords, 0, "008800", null , "WORDS IN CACHE"); + chart.declareDimension(ymageChart.DIMENSION_LEFT, leftscale, vspace * leftscale / maxwords, 0, "008800", null , "INDEXING, WORDS IN CACHE"); chart.declareDimension(ymageChart.DIMENSION_RIGHT, rightscale, vspace * rightscale / (int)(maxbytes / 1024 / 1024), 0, "0000FF", "CCCCCC", "MEMORY/MEGABYTE"); chart.declareDimension(ymageChart.DIMENSION_ANOT, anotscale, vspace * anotscale / maxppm, 0, "008800", null , "PPM [PAGES/MINUTE]");