boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- fixed several bugs in database management functions

Browse Source 
- fixed a display bug for the performance graph
- fixed deadlock when initialization of awt happens simultanously
- removed some debugging output

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5245 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 1976b89bef
commit 9ac16f565b
9 changed files with 61 additions and 33 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.5
javacTarget=1.5
# Release Configuration
releaseVersion=0.611
releaseVersion=0.61
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz

7
htroot/xml/queues_p.java
Unescape View File

@ -38,6 +38,7 @@ import de.anomic.crawler.CrawlEntry;
import de.anomic.crawler.IndexingStack;
import de.anomic.crawler.NoticedURL;
import de.anomic.http.httpRequestHeader;
import de.anomic.kelondro.kelondroException;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardConstants;
import de.anomic.server.serverObjects;
@ -87,7 +88,11 @@ public class queues_p {
// getting all enqueued entries
if ((sb.webIndex.queuePreStack.size() > 0)) {
final Iterator<IndexingStack.QueueEntry> i1 = sb.webIndex.queuePreStack.entryIterator(false);
while (i1.hasNext()) entryList.add(i1.next());
while (i1.hasNext()) try {
entryList.add(i1.next());
} catch (kelondroException e) {
e.printStackTrace();
}
}
int size = (post == null) ? entryList.size() : post.getInt("num", entryList.size());

2
source/de/anomic/crawler/Balancer.java
Unescape View File

@ -365,7 +365,7 @@ public class Balancer {
public synchronized void push(final CrawlEntry entry) throws IOException {
assert entry != null;
if (urlFileIndex.has(entry.url().hash().getBytes())) {
serverLog.logWarning("BALANCER", "double-check has failed for urlhash " + entry.url().hash() + " in " + stackname + " - fixed");
//serverLog.logWarning("BALANCER", "double-check has failed for urlhash " + entry.url().hash() + " in " + stackname + " - fixed");
return;
}

25
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -535,7 +535,12 @@ public class kelondroCollectionIndex {
final int oldSerialNumber = 0;
// load the old collection and join it
try {
collection.addAllUnique(getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false));
} catch (kelondroException e) {
// an error like "array does not contain expected row" may appear here. Just go on like if the collection does not exist
e.printStackTrace();
}
collection.sort();
collection.uniq(); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries
collection.trim(false);
@ -672,8 +677,16 @@ public class kelondroCollectionIndex {
int removed = 0;
assert (removekeys != null);
// load the old collection and remove keys
final kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, serialNumber, false);
kelondroRowSet oldcollection = null;
try {
oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, serialNumber, false);
} catch (kelondroException e) {
// some internal problems occurred. however, go on like there was no old collection
e.printStackTrace();
}
// the oldcollection may be null here!
if (oldcollection != null && oldcollection.size() > 0) {
// remove the keys from the set
final Iterator<String> i = removekeys.iterator();
while (i.hasNext()) {
@ -681,8 +694,10 @@ public class kelondroCollectionIndex {
}
oldcollection.sort();
oldcollection.trim(false);
}
if (oldcollection.size() == 0) {
// in case of an error or an empty collection, remove the collection:
if (oldcollection == null || oldcollection.size() == 0) {
// delete the index entry and the array
array_remove(
oldPartitionNumber, serialNumber, this.payloadrow.objectsize,
@ -691,6 +706,7 @@ public class kelondroCollectionIndex {
return removed;
}
// now write to a new partition (which may be the same partition as the old one)
final int newPartitionNumber = arrayIndex(oldcollection.size());
// see if we need new space or if we can overwrite the old space
@ -761,7 +777,8 @@ public class kelondroCollectionIndex {
// the index appears to be corrupted
this.indexErrors++;
if (this.indexErrors == errorLimit) deleteIndexOnExit(); // delete index on exit for rebuild
throw new kelondroException(arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString(), "array does not contain expected row (error #" + indexErrors + ")");
serverLog.logWarning("kelondroCollectionIndex", "array " + arrayFile(this.path, this.filenameStub, this.loadfactor, chunksize, clusteridx, serialnumber).toString() + " does not contain expected row (error #" + indexErrors + ")");
return new kelondroRowSet(this.payloadrow, 0);
}
// read the row and define a collection
@ -838,7 +855,7 @@ public class kelondroCollectionIndex {
if (indexrow == null) return null;
try {
return new Object[]{indexrow.getColBytes(0), getdelete(indexrow, false)};
} catch (final IOException e) {
} catch (final Exception e) {
e.printStackTrace();
return null;
}

11
source/de/anomic/kelondro/kelondroEcoTable.java
Unescape View File

@ -29,6 +29,7 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.ConcurrentModificationException;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
@ -74,7 +75,7 @@ public class kelondroEcoTable implements kelondroIndex {
public kelondroEcoTable(final File tablefile, final kelondroRow rowdef, final int useTailCache, final int buffersize, final int initialSpace) {
this.rowdef = rowdef;
this.buffersize = buffersize;
this.fail = 0;
//this.fail = 0;
assert rowdef.primaryKeyIndex == 0;
// define the taildef, a row like the rowdef but without the first column
final kelondroColumn[] cols = new kelondroColumn[rowdef.columns() - 1];
@ -270,7 +271,10 @@ public class kelondroEcoTable implements kelondroIndex {
assert ((table == null) || (table.size() == index.size()));
final int i = (int) file.size();
final boolean added = index.addi(row.getPrimaryKeyBytes(), i);
if (!added) return false;
if (!added) {
fail++;
return false;
}
if (table != null) {
assert table.size() == i;
table.addUnique(taildef.newEntry(row.bytes(), rowdef.primaryKeyLength, true));
@ -620,8 +624,7 @@ public class kelondroEcoTable implements kelondroIndex {
e.printStackTrace();
return null;
}
assert this.c >= 0;
if (this.c < 0) return null;
if (this.c < 0) throw new ConcurrentModificationException(); // this should only happen if the table was modified during the iteration
final byte[] b = new byte[rowdef.objectsize];
if (table == null) {
// read from file

6
source/de/anomic/kelondro/kelondroRow.java
Unescape View File

@ -151,8 +151,10 @@ public final class kelondroRow {
public final Entry newEntry(final byte[] rowinstance) {
if (rowinstance == null) return null;
//assert (rowinstance[0] != 0);
assert (this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth)) : "rowinstance[0] = " + new String(rowinstance, 0, row[0].cellwidth) + " / " + serverLog.arrayList(rowinstance, 0, row[0].cellwidth);
if (!(this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth))) return null;
if (!(this.objectOrder.wellformed(rowinstance, 0, row[0].cellwidth))) {
serverLog.logWarning("kelondroRow", "row not well-formed: rowinstance[0] = " + new String(rowinstance, 0, row[0].cellwidth) + " / " + serverLog.arrayList(rowinstance, 0, row[0].cellwidth));
return null;
}
return new Entry(rowinstance, false);
}

3
source/de/anomic/kelondro/kelondroStack.java
Unescape View File

@ -113,7 +113,8 @@ public final class kelondroStack extends kelondroFullRecords {
nextHandle = new EcoNode(nextHandle).getOHHandle((up) ? right : left);
return row().newEntry(new EcoNode(lastHandle).getValueRow());
} catch (final IOException e) {
throw new kelondroException(filename, "IO error at Counter:next()");
e.printStackTrace();
throw new kelondroException(filename, "IO error at stackIterator.next(): " + e.getMessage());
}
}

14
source/de/anomic/plasma/plasmaGrafics.java
Unescape View File

@ -62,9 +62,9 @@ public class plasmaGrafics {
private static final String COL_WE_LINE = "FFAAAA";
private static final String COL_WE_TEXT = "FFCCCC";
private static final Color COL_BORDER = new Color( 0, 0, 0);
private static final Color COL_NORMAL_TEXT = new Color( 0, 0, 0);
private static final Color COL_LOAD_BG = new Color(247, 247, 247);
private static final String COL_BORDER = "000000";
private static final String COL_NORMAL_TEXT = "000000";
private static final String COL_LOAD_BG = "F7F7F7";
public static class CircleThreadPiece {
private final String pieceName;
@ -294,7 +294,7 @@ public class plasmaGrafics {
//prepare image
peerloadPicture = new BufferedImage(width,height,BufferedImage.TYPE_INT_RGB);
final Graphics2D g = peerloadPicture.createGraphics();
g.setBackground(COL_LOAD_BG);
g.setBackground(Color.decode("0x"+COL_LOAD_BG));
g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
g.clearRect(0,0,width,height);
@ -321,7 +321,7 @@ public class plasmaGrafics {
drawLegendLine(g, 5, height - 5 - 15 * i, fillRest.getPieceName()+" ("+fillRest.getFractionPercent()+" %)", fillRest.getColor());
//draw border around the circle
g.setColor(COL_BORDER);
g.setColor(Color.decode("0x"+COL_BORDER));
g.drawArc(circ_x, circ_y, circ_w, circ_w, 0, 360);
peerloadPictureDate = System.currentTimeMillis();
@ -330,10 +330,10 @@ public class plasmaGrafics {
private static void drawLegendLine(final Graphics2D g, final int x, final int y, final String caption, final Color item_color) {
g.setColor(item_color);
g.fillRect(x, y-LEGEND_BOX_SIZE, LEGEND_BOX_SIZE, LEGEND_BOX_SIZE);
g.setColor(COL_BORDER);
g.setColor(Color.decode("0x"+COL_BORDER));
g.drawRect(x, y-LEGEND_BOX_SIZE, LEGEND_BOX_SIZE, LEGEND_BOX_SIZE);
g.setColor(COL_NORMAL_TEXT);
g.setColor(Color.decode("0x"+COL_NORMAL_TEXT));
g.drawChars(caption.toCharArray(), 0, caption.length(), x+LEGEND_BOX_SIZE+5,y);
}

6
source/de/anomic/plasma/plasmaProfiling.java
Unescape View File

@ -53,7 +53,7 @@ public class plasmaProfiling {
public static ymageMatrix performanceGraph(final int width, final int height, final String subline) {
// find maximum values for automatic graph dimension adoption
final int maxppm = (int) maxPayload("ppm", 25);
final int maxwords = (int) maxPayload("wordcache", 10000);
final int maxwords = (int) maxPayload("wordcache", 12000);
final long maxbytes = maxPayload("memory", 110 * 1024 * 1024);
// declare graph and set dimensions
@ -61,7 +61,7 @@ public class plasmaProfiling {
final int rightborder = 30;
final int topborder = 20;
final int bottomborder = 20;
final int leftscale = 20000;
final int leftscale = 10000;
final int rightscale = 100;
final int anotscale = 50;
final int bottomscale = 60;
@ -70,7 +70,7 @@ public class plasmaProfiling {
final int maxtime = 600;
ymageChart chart = new ymageChart(width, height, "FFFFFF", "000000", "AAAAAA", leftborder, rightborder, topborder, bottomborder, "YACY PEER PERFORMANCE: MAIN MEMORY, WORD CACHE AND PAGES/MINUTE (PPM)", subline);
chart.declareDimension(ymageChart.DIMENSION_BOTTOM, bottomscale, hspace / (maxtime / bottomscale), -maxtime, "000000", "CCCCCC", "TIME/SECONDS");
chart.declareDimension(ymageChart.DIMENSION_LEFT, leftscale, vspace * leftscale / maxwords, 0, "008800", null , "WORDS IN CACHE");
chart.declareDimension(ymageChart.DIMENSION_LEFT, leftscale, vspace * leftscale / maxwords, 0, "008800", null , "INDEXING, WORDS IN CACHE");
chart.declareDimension(ymageChart.DIMENSION_RIGHT, rightscale, vspace * rightscale / (int)(maxbytes / 1024 / 1024), 0, "0000FF", "CCCCCC", "MEMORY/MEGABYTE");
chart.declareDimension(ymageChart.DIMENSION_ANOT, anotscale, vspace * anotscale / maxppm, 0, "008800", null , "PPM [PAGES/MINUTE]");

Write Preview
Loading…
Cancel
Save