boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed bugs with open files and caching

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@175 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent f8f8dd05db
commit 21110dcd5e
3 changed files with 25 additions and 15 deletions
Show Stats Download Patch File Download Diff File

7
htroot/IndexControl_p.java
Unescape View File

@ -126,6 +126,7 @@ public class IndexControl_p {
int i = 0;
urlx = new String[index.size()];
while (en.hasMoreElements()) urlx[i++] = ((plasmaWordIndexEntry) en.nextElement()).getUrlHash();
index.close();
} catch (IOException e) {
urlx = new String[0];
}
@ -199,6 +200,7 @@ public class IndexControl_p {
indexes[0] = switchboard.wordIndex.getEntity(keyhash, true);
result = yacyClient.transferIndex(yacyCore.seedDB.getConnected(post.get("hostHash", "")), indexes, switchboard.loadedURL);
prop.put("result", (result == null) ? ("Successfully transferred " + indexes[0].size() + " words in " + ((System.currentTimeMillis() - starttime) / 1000) + " seconds") : result);
try {indexes[0].close();} catch (IOException e) {}
}
if (post.containsKey("keyhashsimilar")) {
@ -206,7 +208,7 @@ public class IndexControl_p {
String result = "Sequential List of Word-Hashes:<br>";
String hash;
int i = 0;
while (hashIt.hasNext()) {
while ((hashIt.hasNext()) && (i < 256)) {
hash = (String) hashIt.next();
result += "<a href=\"/IndexControl_p.html?" +
"keystring=" +
@ -251,7 +253,7 @@ public class IndexControl_p {
String result = "Sequential List of URL-Hashes:<br>";
String hash;
int i = 0;
while (hashIt.hasNext()) {
while ((hashIt.hasNext()) && (i < 256)) {
hash = (String) hashIt.next();
result += "<a href=\"/IndexControl_p.html?" +
"keystring=" +
@ -376,6 +378,7 @@ public class IndexControl_p {
"<span class=\"small\">for every resolveable and deleted URL reference, delete the same reference at every other word where the reference exists (very extensive, but prevents further unresolved references)</span>" +
"</td></tr></table></fieldset></form>";
}
index.close();
return result;
} catch (IOException e) {
return "";

12
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -1422,6 +1422,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
ee.printStackTrace();
return -1;
}
} else {
// simply close the indexEntities
for (int i = 0; i < indexEntities.length; i++) try {
indexEntities[i].close();
} catch (IOException ee) {}
}
return indexCount;
} else {
@ -1514,17 +1519,18 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
indexEntities[i].close();
} else {
// delete complete file
if (!(indexEntities[i].deleteComplete())) {
if (indexEntities[i].deleteComplete()) {
indexEntities[i].close();
} else {
indexEntities[i].close();
// have another try...
if (!(plasmaWordIndexEntity.wordHash2path(plasmaPath, indexEntities[i].wordHash()).delete())) {
success = false;
log.logError("Could not delete whole Index for word " + indexEntities[i].wordHash());
}
} else {
indexEntities[i].close();
}
}
indexEntities[i] = null;
}
return success;
}

19
source/de/anomic/plasma/plasmaWordIndexCache.java
Unescape View File

@ -56,7 +56,8 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
private static final String oldSingletonFileName = "indexSingletons0.db";
private static final String newSingletonFileName = "indexAssortment001.db";
private static final String indexAssortmentClusterPath = "ACLUSTER";
private static final int assortmentLimit = 8;
private static final int assortmentLimit = 20;
private static final int ramcacheLimit = 60;
// class variables
@ -353,15 +354,15 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
}
// print statistics
//for (int k = 0; k < clusterCandidate.length; k++)
// log.logDebug("FLUSH-LIST " + (k + 1) + ": " + clusterCandidate[k].size() + " entries");
for (int k = 0; k < clusterCandidate.length; k++)
log.logDebug("FLUSH-LIST " + (k + 1) + ": " + clusterCandidate[k].size() + " entries");
Map.Entry entry;
int candidateCounter;
count = 0;
// flush high-scores that accumultated too much
for (int cluster = clusterCandidate.length; cluster >= 50; cluster--) {
for (int cluster = clusterCandidate.length; cluster >= ramcacheLimit; cluster--) {
candidateCounter = 0;
i = clusterCandidate[cluster - 1].entrySet().iterator();
while (i.hasNext()) {
@ -384,8 +385,8 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
entry = (Map.Entry) i.next();
key = (String) entry.getValue();
createTime = (Long) entry.getKey();
if ((createTime != null) && ((System.currentTimeMillis() - createTime.longValue()) > (cluster * 30000))) {
//log.logDebug("flushing singleton-key " + key + ", count=" + count + ", cachesize=" + cache.size() + ", singleton-size=" + singletons.size());
if ((createTime != null) && ((System.currentTimeMillis() - createTime.longValue()) > (20000 + (cluster * 5000)))) {
//log.logDebug("flushing key " + key + ", count=" + count + ", cachesize=" + cache.size() + ", singleton-size=" + singletons.size());
count += java.lang.Math.abs(flushFromMem(key, true));
candidateCounter += cluster;
}
@ -395,10 +396,10 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
// stop flushing if cache is shrinked enough
// avoid as possible to flush high-scores
if (cache.size() < this.maxWords) return count;
if (cache.size() < this.maxWords - 100) return count;
// flush high-scores
for (int cluster = java.lang.Math.min(clusterCandidate.length, 50); cluster > assortmentLimit; cluster--) {
for (int cluster = java.lang.Math.min(clusterCandidate.length, ramcacheLimit); cluster > assortmentLimit; cluster--) {
candidateCounter = 0;
i = clusterCandidate[cluster - 1].entrySet().iterator();
while (i.hasNext()) {
@ -410,7 +411,7 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
candidateCounter += cluster;
log.logDebug("flushed high-cluster below limit #" + cluster + ", key=" + key + ", count=" + count + ", cachesize=" + cache.size());
}
if (cache.size() < this.maxWords) return count;
if (cache.size() < this.maxWords - 100) return count;
}
}

Write Preview
Loading…
Cancel
Save