boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

tried to avoid endless loop, see also:

Browse Source 
http://forum.yacy-websuche.de/viewtopic.php?f=6&t=467&hilit=

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4175 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 9d539ec621
commit 9a7b093eed
1 changed files with 18 additions and 28 deletions
Show Stats Download Patch File Download Diff File

46
source/de/anomic/kelondro/kelondroCollectionIndex.java
Unescape View File

@ -585,25 +585,20 @@ public class kelondroCollectionIndex {
int oldSerialNumber = 0; int oldSerialNumber = 0;
// load the old collection and join it // load the old collection and join it
kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false); collection.addAllUnique(getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false));
collection.sort();
// join with new collection collection.uniq(-1); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries
oldcollection.addAllUnique(collection); collection.trim(false);
oldcollection.sort();
oldcollection.uniq(-1); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries
oldcollection.trim(false);
// check for size of collection: // check for size of collection:
// if necessary shrink the collection and dump a part of that collection // if necessary shrink the collection and dump a part of that collection
// to avoid that this grows too big // to avoid that this grows too big
int newPartitionNumber; if (arrayIndex(collection.size()) > maxPartitions) {
while ((newPartitionNumber = arrayIndex(oldcollection.size())) > maxPartitions) { shrinkCollection(key, collection, arrayCapacity(maxPartitions));
shrinkCollection(key, oldcollection, arrayCapacity(maxPartitions));
} }
// work on with oldcollection // determine new partition position
collection = oldcollection; int newPartitionNumber = arrayIndex(collection.size());
newPartitionNumber = arrayIndex(collection.size());
// see if we need new space or if we can overwrite the old space // see if we need new space or if we can overwrite the old space
if (oldPartitionNumber == newPartitionNumber) { if (oldPartitionNumber == newPartitionNumber) {
@ -697,26 +692,20 @@ public class kelondroCollectionIndex {
int oldSerialNumber = 0; int oldSerialNumber = 0;
// load the old collection and join it // load the old collection and join it
kelondroRowSet oldcollection = getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false); collection.addAllUnique(getwithparams(indexrow, oldchunksize, oldchunkcount, oldPartitionNumber, oldrownumber, oldSerialNumber, false));
collection.sort();
// join with new collection collection.uniq(-1); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries
oldcollection.addAllUnique(collection); collection.trim(false);
oldcollection.sort();
oldcollection.uniq(-1); // FIXME: not clear if it would be better to insert the collection with put to avoid double-entries
oldcollection.trim(false);
collection = oldcollection;
// check for size of collection: // check for size of collection:
// if necessary shrink the collection and dump a part of that collection // if necessary shrink the collection and dump a part of that collection
// to avoid that this grows too big // to avoid that this grows too big
int newPartitionNumber; if (arrayIndex(collection.size()) > maxPartitions) {
while ((newPartitionNumber = arrayIndex(oldcollection.size())) > maxPartitions) { shrinkCollection(key, collection, arrayCapacity(maxPartitions));
shrinkCollection(key, oldcollection, arrayCapacity(maxPartitions));
} }
// work on with oldcollection // determine new partition location
collection = oldcollection; int newPartitionNumber = arrayIndex(collection.size());
newPartitionNumber = arrayIndex(collection.size());
// see if we need new space or if we can overwrite the old space // see if we need new space or if we can overwrite the old space
if (oldPartitionNumber == newPartitionNumber) { if (oldPartitionNumber == newPartitionNumber) {
@ -742,7 +731,7 @@ public class kelondroCollectionIndex {
} }
} }
private void shrinkCollection(byte[] key, kelondroRowSet collection, int targetSize) { private void shrinkCollection(byte[] key, kelondroRowCollection collection, int targetSize) {
//TODO Remove timing before release //TODO Remove timing before release
// removes entries from collection // removes entries from collection
// the removed entries are stored in a 'commons' dump file // the removed entries are stored in a 'commons' dump file
@ -795,6 +784,7 @@ public class kelondroCollectionIndex {
} }
} }
tot2 = System.currentTimeMillis() - t1; tot2 = System.currentTimeMillis() - t1;
collection.trim(false);
serverLog.logFine("kelondroCollectionIndex", "tot= "+tot1+'/'+tot2+" # add/rem(1)= "+sadd1+'/'+srem1+" # add/rem(2)= "+sadd2+'/'+srem2); serverLog.logFine("kelondroCollectionIndex", "tot= "+tot1+'/'+tot2+" # add/rem(1)= "+sadd1+'/'+srem1+" # add/rem(2)= "+sadd2+'/'+srem2);
serverLog.logInfo("kelondroCollectionIndex", "shrinked common word " + new String(key) + "; old size = " + oldsize + ", new size = " + collection.size() + ", maximum size = " + targetSize + ", newcommon size = " + newcommon.size() + ", first newcommon = " + firstnewcommon); serverLog.logInfo("kelondroCollectionIndex", "shrinked common word " + new String(key) + "; old size = " + oldsize + ", new size = " + collection.size() + ", maximum size = " + targetSize + ", newcommon size = " + newcommon.size() + ", first newcommon = " + firstnewcommon);

Write Preview
Loading…
Cancel
Save