boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

a variety of possible memory leak fixes

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent d7eb18cdf2
commit 00f2df1120
9 changed files with 58 additions and 39 deletions
Show Stats Download Patch File Download Diff File

1
source/net/yacy/cora/protocol/Domains.java
Unescape View File

@ -453,6 +453,7 @@ public class Domains {
globalHosts = null;
} else try {
globalHosts = new KeyList(globalHostsnameCache);
Log.logInfo("Domains", "loaded globalHosts cache of hostnames, size = " + globalHosts.size());
} catch (final IOException e) {
globalHosts = null;
}

71
source/net/yacy/cora/sorting/WeakPriorityBlockingQueue.java
Unescape View File

@ -12,12 +12,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
@ -41,7 +41,7 @@ import java.util.concurrent.TimeUnit;
*/
public class WeakPriorityBlockingQueue<E> {
private final TreeSet<Element<E>> queue; // object within the stack, ordered using a TreeSet
private final Semaphore enqueued; // semaphore for elements in the stack
private final ArrayList<Element<E>> drained; // objects that had been on the stack but had been removed
@ -70,7 +70,7 @@ public class WeakPriorityBlockingQueue<E> {
this.queue.clear();
this.enqueued.drainPermits();
}
/**
* test if the queue is empty
* @return true if the queue is empty, false if not
@ -78,7 +78,7 @@ public class WeakPriorityBlockingQueue<E> {
public boolean isEmpty() {
return this.queue.isEmpty() & this.drained.isEmpty();
}
/**
* get the number of elements in the queue, waiting to be removed with take() or poll()
* @return
@ -89,7 +89,7 @@ public class WeakPriorityBlockingQueue<E> {
/**
* get the number of elements that had been drained so far and are wainting
* get the number of elements that had been drained so far and are waiting
* in a list to get enumerated with element()
* @return
*/
@ -103,9 +103,9 @@ public class WeakPriorityBlockingQueue<E> {
* @return
*/
public synchronized int sizeAvailable() {
return this.queue.size() + this.drained.size();
return Math.min(this.maxsize, this.queue.size() + this.drained.size());
}
/**
* put a element on the stack using a order of the weight
* elements that had been on the stack cannot be put in again,
@ -126,7 +126,7 @@ public class WeakPriorityBlockingQueue<E> {
}
assert this.queue.size() >= this.enqueued.availablePermits() : "(put) queue.size() = " + this.queue.size() + ", enqueued.availablePermits() = " + this.enqueued.availablePermits();
}
/**
* return the element with the smallest weight and remove it from the stack
* @return null if no element is on the queue or the head of the queue
@ -138,7 +138,7 @@ public class WeakPriorityBlockingQueue<E> {
return takeUnsafe();
}
}
/**
* Retrieves and removes the head of this queue, waiting if necessary
* up to the specified wait time if no elements are present on this queue.
@ -153,7 +153,7 @@ public class WeakPriorityBlockingQueue<E> {
return takeUnsafe();
}
}
/**
* Retrieves and removes the head of this queue, waiting if no elements are present on this queue.
* @return the head element from the queue
@ -165,17 +165,17 @@ public class WeakPriorityBlockingQueue<E> {
return takeUnsafe();
}
}
private Element<E> takeUnsafe() {
final Element<E> element = this.queue.first();
assert element != null;
this.queue.remove(element);
this.drained.add(element);
if (this.drained.size() < this.maxsize) this.drained.add(element);
assert this.queue.size() >= this.enqueued.availablePermits() : "(take) queue.size() = " + this.queue.size() + ", enqueued.availablePermits() = " + this.enqueued.availablePermits();
return element;
}
/**
* return the element with the smallest weight, but do not remove it
* @return null if no element is on the queue or the head of the queue
@ -184,7 +184,7 @@ public class WeakPriorityBlockingQueue<E> {
if (this.queue.isEmpty()) return null;
return this.queue.first();
}
/**
* all objects that have been returned by poll or take are stored in a back-up list
* where they can be retrieved afterward. The elements from that list are stored in
@ -214,7 +214,7 @@ public class WeakPriorityBlockingQueue<E> {
return this.drained.get(position);
}
}
/**
* retrieve an element from the drained queue but wait until a timeout
* until returning null when no element will be available within the time
@ -237,7 +237,7 @@ public class WeakPriorityBlockingQueue<E> {
if (position >= this.drained.size()) return null; // we still don't have that element
return this.drained.get(position);
}
/**
* return the specific amount of entries as they would be retrievable with element()
* if count is < 0 then all elements are taken
@ -249,11 +249,11 @@ public class WeakPriorityBlockingQueue<E> {
if (count < 0) {
return list();
}
if (count > sizeAvailable()) throw new RuntimeException("list(" + count + ") exceeded avaiable number of elements (" + sizeAvailable() + ")");
if (count > sizeAvailable()) throw new RuntimeException("list(" + count + ") exceeded avaiable number of elements (" + sizeAvailable() + ")");
while (count > this.drained.size()) this.poll();
return this.drained;
}
/**
* return all entries as they would be retrievable with element()
* @return a list of all elements in the stack
@ -263,7 +263,7 @@ public class WeakPriorityBlockingQueue<E> {
while (!this.queue.isEmpty()) this.poll();
return this.drained;
}
/**
* iterate over all elements available. All elements that are still in the queue are drained to recorded positions
* @return an iterator over all drained positions.
@ -283,20 +283,23 @@ public class WeakPriorityBlockingQueue<E> {
@Override
public String toString();
}
protected abstract static class AbstractElement<E> implements Element<E> {
public long weight;
public E element;
@Override
public long getWeight() {
return this.weight;
}
@Override
public E getElement() {
return this.element;
}
@Override
public boolean equals(Element<E> o) {
return this.element.equals(o.getElement());
}
@ -305,13 +308,13 @@ public class WeakPriorityBlockingQueue<E> {
public int hashCode() {
return this.element.hashCode();
}
@Override
public String toString() {
return element.toString() + "/" + weight;
return this.element.toString() + "/" + this.weight;
}
}
/**
* natural ordering elements, can be used as container of objects <E> in the priority queue
* the elements with smallest ordering weights are first in the queue when elements are taken
@ -323,10 +326,12 @@ public class WeakPriorityBlockingQueue<E> {
this.weight = weight;
}
@Override
public int compare(NaturalElement<E> o1, NaturalElement<E> o2) {
return o1.compareTo(o2);
}
@Override
public int compareTo(NaturalElement<E> o) {
if (this.element == o.getElement()) return 0;
if (this.element.equals(o.getElement())) return 0;
@ -338,9 +343,9 @@ public class WeakPriorityBlockingQueue<E> {
if (o1h < o2h) return -1;
return 0;
}
}
/**
* reverse ordering elements, can be used as container of objects <E> in the priority queue
* the elements with highest ordering weights are first in the queue when elements are taken
@ -352,10 +357,12 @@ public class WeakPriorityBlockingQueue<E> {
this.weight = weight;
}
@Override
public int compare(ReverseElement<E> o1, ReverseElement<E> o2) {
return o1.compareTo(o2);
}
@Override
public int compareTo(ReverseElement<E> o) {
if (this.element == o.getElement()) return 0;
if (this.element.equals(o.getElement())) return 0;
@ -368,7 +375,7 @@ public class WeakPriorityBlockingQueue<E> {
return 0;
}
}
public static void main(String[] args) {
final WeakPriorityBlockingQueue<String> a = new WeakPriorityBlockingQueue<String>(3);
//final Element<String> REVERSE_POISON = new ReverseElement<String>("", Long.MIN_VALUE);

4
source/net/yacy/cora/storage/HashARC.java
Unescape View File

@ -37,13 +37,13 @@ public final class HashARC<K, V> extends SimpleARC<K, V> implements Map<K, V>, I
public HashARC(final int cacheSize) {
this.cacheSize = cacheSize / 2;
super.levelA = Collections.synchronizedMap(new LinkedHashMap<K, V>(cacheSize, 0.1f, accessOrder) {
super.levelA = Collections.synchronizedMap(new LinkedHashMap<K, V>(1, 0.1f, accessOrder) {
private static final long serialVersionUID = 1L;
@Override protected boolean removeEldestEntry(final Map.Entry<K, V> eldest) {
return size() > HashARC.this.cacheSize;
}
});
this.levelB = Collections.synchronizedMap(new LinkedHashMap<K, V>(cacheSize, 0.1f, accessOrder) {
this.levelB = Collections.synchronizedMap(new LinkedHashMap<K, V>(1, 0.1f, accessOrder) {
private static final long serialVersionUID = 1L;
@Override protected boolean removeEldestEntry(final Map.Entry<K, V> eldest) {
return size() > HashARC.this.cacheSize;

4
source/net/yacy/cora/storage/KeyList.java
Unescape View File

@ -76,6 +76,10 @@ public class KeyList implements Iterable<String> {
}
public int size() {
return this.keys.size();
}
public boolean contains(final String key) {
return this.keys.containsKey(key.trim().toLowerCase());
}

5
source/net/yacy/kelondro/index/HandleMap.java
Unescape View File

@ -98,6 +98,7 @@ public final class HandleMap implements Iterable<Row.Entry> {
is.close();
is = null;
assert this.index.size() == file.length() / (keylength + idxbytes);
trim();
}
public void trim() {
@ -415,6 +416,7 @@ public final class HandleMap implements Iterable<Row.Entry> {
return this.result.get();
}
@Override
public final HandleMap call() throws IOException {
try {
finishloop: while (true) {
@ -439,7 +441,8 @@ public final class HandleMap implements Iterable<Row.Entry> {
}
}
public Iterator<Row.Entry> iterator() {
@Override
public Iterator<Row.Entry> iterator() {
return rows(true, null);
}
}

1
source/net/yacy/kelondro/rwi/ReferenceContainerArray.java
Unescape View File

@ -473,6 +473,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
}
}
}
references.trim();
System.out.println("CELL REFERENCE COLLECTION finished");
return references;
}

3
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -187,6 +187,7 @@ public class Table implements Index, Iterable<Row.Entry> {
}
}
}
this.index.trim();
// open the file
this.file = new BufferedRecords(new Records(tablefile, rowdef.objectsize), this.buffersize);
@ -594,6 +595,7 @@ public class Table implements Index, Iterable<Row.Entry> {
* @throws IOException
* @throws RowSpaceExceededException
*/
@Override
public boolean put(final Entry row) throws IOException, RowSpaceExceededException {
assert row != null;
if (this.file == null || row == null) return true;
@ -702,6 +704,7 @@ public class Table implements Index, Iterable<Row.Entry> {
}
}
@Override
public boolean delete(final byte[] key) throws IOException {
return remove(key) != null;
}

4
source/net/yacy/search/query/SnippetProcess.java
Unescape View File

@ -109,8 +109,8 @@ public class SnippetProcess {
this.urlRetrievalAllTime = 0;
this.snippetComputationAllTime = 0;
this.result = new WeakPriorityBlockingQueue<ResultEntry>(-1); // this is the result, enriched with snippets, ranked and ordered by ranking
this.images = new WeakPriorityBlockingQueue<MediaSnippet>(-1);
this.result = new WeakPriorityBlockingQueue<ResultEntry>(Math.max(1000, 10 * query.itemsPerPage())); // this is the result, enriched with snippets, ranked and ordered by ranking
this.images = new WeakPriorityBlockingQueue<MediaSnippet>(Math.max(1000, 10 * query.itemsPerPage()));
// snippets do not need to match with the complete query hashes,
// only with the query minus the stopwords which had not been used for the search

4
source/net/yacy/search/snippet/TextSnippet.java
Unescape View File

@ -206,10 +206,10 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
if (de.anomic.crawler.Cache.has(url.hash())) {
// get the sentences from the cache
final Request request = loader.request(url, true, reindexing);
final Request request = loader == null ? null : loader.request(url, true, reindexing);
Response response;
try {
response = loader == null ? null : loader.load(request, CacheStrategy.CACHEONLY, true);
response = loader == null || request == null ? null : loader.load(request, CacheStrategy.CACHEONLY, true);
} catch (IOException e1) {
response = null;
}

Write Preview
Loading…
Cancel
Save