boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more synchronization in plasmaWordIndex

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2385 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 19 years ago
parent f43c90fa98
commit 328f9859a5
1 changed files with 86 additions and 72 deletions
Show Stats Download Patch File Download Diff File

158
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -200,10 +200,10 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
} }
public void flushCacheSome() { public void flushCacheSome() {
int flushCount = ramCache.wSize() / 500; int flushCount = ramCache.wSize() / 350;
if (flushCount > 80) flushCount = 80; if (flushCount > 80) flushCount = 80;
if (flushCount < 10) flushCount = Math.min(10, ramCache.wSize()); if (flushCount < 20) flushCount = Math.min(20, ramCache.wSize());
flushCache(flushCount); flushCache(flushCount);
} }
public void flushCache(int count) { public void flushCache(int count) {
@ -320,40 +320,42 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
public indexContainer getContainer(String wordHash, boolean deleteIfEmpty, long maxTime) { public indexContainer getContainer(String wordHash, boolean deleteIfEmpty, long maxTime) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
synchronized (ramCache) {
// get from cache
indexContainer container = ramCache.getContainer(wordHash, true, -1);
// We must not use the container from cache to store everything we find,
// as that container remains linked to in the cache and might be changed later
// while the returned container is still in use.
// create a clone from the container
if (container != null) container = container.topLevelClone();
// get from cache // get from collection index
indexContainer container = ramCache.getContainer(wordHash, true, -1); if (useCollectionIndex) {
if (container == null) {
// We must not use the container from cache to store everything we find, container = collections.getContainer(wordHash, true, (maxTime < 0) ? -1 : maxTime);
// as that container remains linked to in the cache and might be changed later } else {
// while the returned container is still in use. container.add(collections.getContainer(wordHash, true, (maxTime < 0) ? -1 : maxTime), -1);
// create a clone from the container }
if (container != null) container = container.topLevelClone(); }
// get from collection index // get from assortments
if (useCollectionIndex) {
if (container == null) { if (container == null) {
container = collections.getContainer(wordHash, true, (maxTime < 0) ? -1 : maxTime); container = assortmentCluster.getContainer(wordHash, true, (maxTime < 0) ? -1 : maxTime);
} else { } else {
container.add(collections.getContainer(wordHash, true, (maxTime < 0) ? -1 : maxTime), -1); // add containers from assortment cluster
container.add(assortmentCluster.getContainer(wordHash, true, (maxTime < 0) ? -1 : maxTime), -1);
} }
}
// get from assortments
if (container == null) {
container = assortmentCluster.getContainer(wordHash, true, (maxTime < 0) ? -1 : maxTime);
} else {
// add containers from assortment cluster
container.add(assortmentCluster.getContainer(wordHash, true, (maxTime < 0) ? -1 : maxTime), -1);
}
// get from backend // get from backend
if (maxTime > 0) { if (maxTime > 0) {
maxTime = maxTime - (System.currentTimeMillis() - start); maxTime = maxTime - (System.currentTimeMillis() - start);
if (maxTime < 0) maxTime = 100; if (maxTime < 0) maxTime = 100;
}
container.add(backend.getContainer(wordHash, deleteIfEmpty, (maxTime < 0) ? -1 : maxTime), -1);
return container;
} }
container.add(backend.getContainer(wordHash, deleteIfEmpty, (maxTime < 0) ? -1 : maxTime), -1);
return container;
} }
public Set getContainers(Set wordHashes, boolean deleteIfEmpty, boolean interruptIfEmpty, long maxTime) { public Set getContainers(Set wordHashes, boolean deleteIfEmpty, boolean interruptIfEmpty, long maxTime) {
@ -362,59 +364,67 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
HashSet containers = new HashSet(); HashSet containers = new HashSet();
String singleHash; String singleHash;
indexContainer singleContainer; indexContainer singleContainer;
Iterator i = wordHashes.iterator(); synchronized (ramCache) {
long start = System.currentTimeMillis(); Iterator i = wordHashes.iterator();
long remaining; long start = System.currentTimeMillis();
while (i.hasNext()) { long remaining;
// check time while (i.hasNext()) {
remaining = maxTime - (System.currentTimeMillis() - start); // check time
//if ((maxTime > 0) && (remaining <= 0)) break; remaining = maxTime - (System.currentTimeMillis() - start);
if ((maxTime >= 0) && (remaining <= 0)) remaining = 100; //if ((maxTime > 0) && (remaining <= 0)) break;
if ((maxTime >= 0) && (remaining <= 0)) remaining = 100;
// get next hash: // get next hash:
singleHash = (String) i.next(); singleHash = (String) i.next();
// retrieve index // retrieve index
singleContainer = getContainer(singleHash, deleteIfEmpty, (maxTime < 0) ? -1 : remaining / (wordHashes.size() - containers.size())); singleContainer = getContainer(singleHash, deleteIfEmpty, (maxTime < 0) ? -1 : remaining / (wordHashes.size() - containers.size()));
// check result // check result
if (((singleContainer == null) || (singleContainer.size() == 0)) && (interruptIfEmpty)) return new HashSet(); if (((singleContainer == null) || (singleContainer.size() == 0)) && (interruptIfEmpty)) return new HashSet();
containers.add(singleContainer); containers.add(singleContainer);
}
} }
return containers; return containers;
} }
public int size() { public int size() {
if (useCollectionIndex) synchronized (ramCache) {
return java.lang.Math.max(collections.size(), if (useCollectionIndex)
return java.lang.Math.max(collections.size(),
java.lang.Math.max(assortmentCluster.size(), java.lang.Math.max(assortmentCluster.size(),
java.lang.Math.max(backend.size(), ramCache.size()))); java.lang.Math.max(backend.size(), ramCache.size())));
else else
return java.lang.Math.max(assortmentCluster.size(), return java.lang.Math.max(assortmentCluster.size(),
java.lang.Math.max(backend.size(), ramCache.size())); java.lang.Math.max(backend.size(), ramCache.size()));
}
} }
public int indexSize(String wordHash) { public int indexSize(String wordHash) {
int size = 0; int size = 0;
try { synchronized (ramCache) {
plasmaWordIndexFile entity = backend.getEntity(wordHash, true, -1); try {
if (entity != null) { plasmaWordIndexFile entity = backend.getEntity(wordHash, true, -1);
size += entity.size(); if (entity != null) {
entity.close(); size += entity.size();
} entity.close();
} catch (IOException e) {} }
if (useCollectionIndex) size += collections.indexSize(wordHash); } catch (IOException e) {}
size += assortmentCluster.indexSize(wordHash); if (useCollectionIndex) size += collections.indexSize(wordHash);
size += ramCache.indexSize(wordHash); size += assortmentCluster.indexSize(wordHash);
size += ramCache.indexSize(wordHash);
}
return size; return size;
} }
public synchronized void close(int waitingBoundSeconds) { public synchronized void close(int waitingBoundSeconds) {
ramCache.close(waitingBoundSeconds); synchronized (ramCache) {
if (useCollectionIndex) collections.close(-1); ramCache.close(waitingBoundSeconds);
assortmentCluster.close(-1); if (useCollectionIndex) collections.close(-1);
backend.close(10); assortmentCluster.close(-1);
backend.close(10);
}
} }
public synchronized indexContainer deleteContainer(String wordHash) { public synchronized indexContainer deleteContainer(String wordHash) {
@ -458,7 +468,9 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
// urlHash assigned. This can only work if the entry is really fresh // urlHash assigned. This can only work if the entry is really fresh
// and can be found in the RAM cache // and can be found in the RAM cache
// this returns the number of deletion that had been possible // this returns the number of deletion that had been possible
return ramCache.tryRemoveURLs(urlHash); synchronized (ramCache) {
return ramCache.tryRemoveURLs(urlHash);
}
} }
public static final int RL_RAMCACHE = 0; public static final int RL_RAMCACHE = 0;
@ -472,14 +484,16 @@ public final class plasmaWordIndex extends indexAbstractRI implements indexRI {
kelondroOrder containerOrder = new indexContainerOrder((kelondroOrder) indexOrder.clone()); kelondroOrder containerOrder = new indexContainerOrder((kelondroOrder) indexOrder.clone());
containerOrder.rotate(startHash.getBytes()); containerOrder.rotate(startHash.getBytes());
TreeSet containers = new TreeSet(containerOrder); TreeSet containers = new TreeSet(containerOrder);
Iterator i = wordContainers(startHash, resourceLevel, rot); synchronized (ramCache) {
if (resourceLevel == plasmaWordIndex.RL_RAMCACHE) count = Math.min(ramCache.wSize(), count); Iterator i = wordContainers(startHash, resourceLevel, rot);
indexContainer container; if (resourceLevel == plasmaWordIndex.RL_RAMCACHE) count = Math.min(ramCache.wSize(), count);
while ((count > 0) && (i.hasNext())) { indexContainer container;
container = (indexContainer) i.next(); while ((count > 0) && (i.hasNext())) {
if ((container != null) && (container.size() > 0)) { container = (indexContainer) i.next();
containers.add(container); if ((container != null) && (container.size() > 0)) {
count--; containers.add(container);
count--;
}
} }
} }
return containers; return containers;

Write Preview
Loading…
Cancel
Save