boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fix for NPE during DHT RWI selection

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6527 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 959b38b61b
commit 37245430c3
3 changed files with 43 additions and 27 deletions
Show Stats Download Patch File Download Diff File

8
source/de/anomic/yacy/dht/Dispatcher.java
Unescape View File

@ -210,9 +210,11 @@ public class Dispatcher {
// but to avoid race conditions return the results from the deletes
rc = new ArrayList<ReferenceContainer<WordReference>>(containers.size());
for (ReferenceContainer<WordReference> c: containers) {
container = this.segment.termIndex().delete(c.getTermHash());
if (this.log.isFine()) this.log.logFine("selected " + container.size() + " urls for word '" + c.getTermHashAsString() + "'");
if (!container.isEmpty()) rc.add(container);
container = this.segment.termIndex().delete(c.getTermHash()); // be aware this might be null!
if (container != null && !container.isEmpty()) {
if (this.log.isFine()) this.log.logFine("selected " + container.size() + " urls for word '" + c.getTermHashAsString() + "'");
rc.add(container);
}
}
}

10
source/net/yacy/kelondro/rwi/IndexCell.java
Unescape View File

@ -107,8 +107,7 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
/**
* add entries to the cell: this adds the new entries always to the RAM part, never to BLOBs
* @throws IOException
* @throws RowSpaceExceededException
* @throws RowSpaceExceededException
* @throws RowSpaceExceededException
*/
public void add(ReferenceContainer<ReferenceType> newEntries) throws IOException, RowSpaceExceededException {
try {
@ -186,8 +185,11 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
}
/**
* all containers in the BLOBs and the RAM are merged and returned
* @throws IOException
* all containers in the BLOBs and the RAM are merged and returned.
* Please be aware that the returned values may be top-level cloned ReferenceContainers or direct links to containers
* If the containers are modified after they are returned, they MAY alter the stored index.
* @throws IOException
* @return a container with merged ReferenceContainer from RAM and the file array or null if there is no data to be returned
*/
public ReferenceContainer<ReferenceType> get(byte[] termHash, Set<String> urlselection) throws IOException {
ReferenceContainer<ReferenceType> c0 = this.ram.get(termHash, null);

52
source/net/yacy/kelondro/rwi/ReferenceContainerCache.java
Unescape View File

@ -28,7 +28,8 @@ package net.yacy.kelondro.rwi;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
@ -45,7 +46,15 @@ import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.util.ByteArray;
import net.yacy.kelondro.util.FileUtils;
/**
* A ReferenceContainerCache is the ram cache for word indexes or other entity type indexes
* The <ReferenceType> defines the index reference specification and attributes that can be
* accessed during a search without using the metadata reference that shall be contained within
* the <ReferenceType>. A ReferenceContainerCache has no active backup in a file, it must be flushed to
* a file to save the content of the cache.
*
* @param <ReferenceType>
*/
public final class ReferenceContainerCache<ReferenceType extends Reference> extends AbstractIndex<ReferenceType> implements Index<ReferenceType>, IndexReader<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>> {
private final Row payloadrow;
@ -54,7 +63,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
protected Map<ByteArray, ReferenceContainer<ReferenceType>> cache;
/**
* opens an existing heap file in undefined mode
* open an existing heap file in undefined mode
* after this a initialization should be made to use the heap:
* either a read-only or read/write mode initialization
* @param payloadrow
@ -95,7 +104,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
final long startTime = System.currentTimeMillis();
// sort the map
ReferenceContainer<ReferenceType>[] cachecopy = sortedClone();
ArrayList<ReferenceContainer<ReferenceType>> cachecopy = sortedClone();
// write wCache
long wordcount = 0, urlcount = 0;
@ -132,16 +141,18 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
}
}
@SuppressWarnings("unchecked")
public ReferenceContainer<ReferenceType>[] sortedClone() {
ReferenceContainer<ReferenceType>[] cachecopy = new ReferenceContainer[cache.size()];
/**
* create a clone of the cache content that is sorted using the this.containerOrder
* @return the sorted ReferenceContainer[]
*/
private ArrayList<ReferenceContainer<ReferenceType>> sortedClone() {
ArrayList<ReferenceContainer<ReferenceType>> cachecopy = new ArrayList<ReferenceContainer<ReferenceType>>(cache.size());
synchronized (cache) {
int p = 0;
for (final Map.Entry<ByteArray, ReferenceContainer<ReferenceType>> entry: cache.entrySet()) {
cachecopy[p++] = entry.getValue();
if (entry.getValue() != null && entry.getValue().getTermHash() != null) cachecopy.add(entry.getValue());
}
}
Arrays.sort(cachecopy, this.containerOrder);
Collections.sort(cachecopy, this.containerOrder);
return cachecopy;
}
@ -190,7 +201,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
// plus the mentioned features
private final boolean rot;
private ReferenceContainer<ReferenceType>[] cachecopy;
private ArrayList<ReferenceContainer<ReferenceType>> cachecopy;
private int p;
private byte[] latestTermHash;
@ -199,10 +210,11 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
if (startWordHash != null && startWordHash.length == 0) startWordHash = null;
this.cachecopy = sortedClone();
assert this.cachecopy != null;
assert termOrder != null;
this.p = 0;
if (startWordHash != null) {
while ( (this.p < this.cachecopy.length) &&
(termOrder.compare(this.cachecopy[this.p].getTermHash(), startWordHash) < 0)
while ( this.p < this.cachecopy.size() &&
termOrder.compare(this.cachecopy.get(this.p).getTermHash(), startWordHash) < 0
) this.p++;
}
this.latestTermHash = null;
@ -214,13 +226,13 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
}
public boolean hasNext() {
if (rot) return this.cachecopy.length > 0;
return this.p < this.cachecopy.length;
if (rot) return this.cachecopy.size() > 0;
return this.p < this.cachecopy.size();
}
public ReferenceContainer<ReferenceType> next() {
if (this.p < this.cachecopy.length) {
ReferenceContainer<ReferenceType> c = this.cachecopy[this.p++];
if (this.p < this.cachecopy.size()) {
ReferenceContainer<ReferenceType> c = this.cachecopy.get(this.p++);
this.latestTermHash = c.getTermHash();
try {
return c.topLevelClone();
@ -233,9 +245,9 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
if (!rot) {
return null;
}
if (this.cachecopy.length == 0) return null;
if (this.cachecopy.size() == 0) return null;
p = 0;
ReferenceContainer<ReferenceType> c = this.cachecopy[this.p++];
ReferenceContainer<ReferenceType> c = this.cachecopy.get(this.p++);
this.latestTermHash = c.getTermHash();
try {
return c.topLevelClone();
@ -246,7 +258,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
}
public void remove() {
System.arraycopy(this.cachecopy, this.p, this.cachecopy, this.p - 1, this.cachecopy.length - p);
System.arraycopy(this.cachecopy, this.p, this.cachecopy, this.p - 1, this.cachecopy.size() - p);
cache.remove(new ByteArray(this.latestTermHash));
}

Write Preview
Loading…
Cancel
Save