some redesign with a possible fix for the ReferenceContainerCache.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6336 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent e5d04f6b91
commit c3a4aee255

@ -77,7 +77,10 @@ public class yacysearch {
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
int display = (post == null) ? 0 : post.getInt("display", 0);
if ((display == 1) && (!authenticated)) display = 0;
if (!authenticated) display = 2;
// display == 0: shop top menu
// display == 1: show top and left menu
// display == 2: do not show any menu
final boolean browserPopUpTrigger = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_TRIGGER, "true").equals("true");
if (browserPopUpTrigger) {
final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");

@ -135,7 +135,9 @@ public class yacysearchtrailer {
// about box
String aboutBody = env.getConfig("about.body", "");
String aboutHeadline = env.getConfig("about.headline", "");
if (aboutBody.length() == 0 && aboutHeadline.length() == 0) {
if ((aboutBody.length() == 0 && aboutHeadline.length() == 0) ||
theSearch.getRankingResult().getLocalResourceSize() +
theSearch.getRankingResult().getRemoteResourceSize() == 0) {
prop.put("nav-about", 0);
} else {
prop.put("nav-about", 1);

@ -44,7 +44,7 @@ public class wikiBoard {
public static final int keyLength = 64;
private static final String dateFormat = "yyyyMMddHHmmss";
private static final SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat);
protected static final SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat);
static {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));

@ -27,7 +27,6 @@
package de.anomic.document.parser;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.HashSet;

@ -81,7 +81,6 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
this.array = new ReferenceContainerArray<ReferenceType>(cellPath, factory, termOrder, payloadrow, merger);
this.ram = new ReferenceContainerCache<ReferenceType>(factory, payloadrow, termOrder);
this.ram.initWriteMode();
this.maxRamEntries = maxRamEntries;
this.merger = merger;
this.lastCleanup = System.currentTimeMillis();
@ -295,18 +294,18 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
}
public int[] sizes() {
int[] as = this.array.sizes();
int[] asr = new int[as.length + 1];
System.arraycopy(as, 0, asr, 0, as.length);
asr[as.length] = this.ram.size();
return asr;
int[] as = this.array.sizes();
int[] asr = new int[as.length + 1];
System.arraycopy(as, 0, asr, 0, as.length);
asr[as.length] = this.ram.size();
return asr;
}
public int sizesMax() {
int m = 0;
int[] s = sizes();
for (int i = 0; i < s.length; i++) if (s[i] > m) m = s[i];
return m;
int m = 0;
int[] s = sizes();
for (int i = 0; i < s.length; i++) if (s[i] > m) m = s[i];
return m;
}
public int minMem() {
@ -323,29 +322,30 @@ public final class IndexCell<ReferenceType extends Reference> extends AbstractBu
*/
private void cleanCache() {
this.countCache.clear();
this.countCache.clear();
// dump the cache if necessary
if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) synchronized (this) {
if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) {
// dump the ram
File dumpFile = this.array.newContainerBLOBFile();
//this.ram.dump(dumpFile, true);
//this.array.mountBLOBContainer(dumpFile);
merger.dump(this.ram, dumpFile, array);
// get a fresh ram cache
this.ram = new ReferenceContainerCache<ReferenceType>(factory, this.array.rowdef(), this.array.ordering());
this.ram.initWriteMode();
}
}
if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) synchronized (this) {
if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) {
// dump the ram
File dumpFile = this.array.newContainerBLOBFile();
// a critical point: when the ram is handed to the dump job,
// dont write into it any more. Use a fresh one instead
ReferenceContainerCache<ReferenceType> ramdump = this.ram;
// get a fresh ram cache
this.ram = new ReferenceContainerCache<ReferenceType>(factory, this.array.rowdef(), this.array.ordering());
// dump the buffer
merger.dump(ramdump, dumpFile, array);
}
}
// clean-up the cache
if (this.array.entries() > 50 || (this.lastCleanup + cleanupCycle < System.currentTimeMillis())) synchronized (this) {
if (this.array.entries() > 50 || (this.lastCleanup + cleanupCycle < System.currentTimeMillis())) {
//System.out.println("----cleanup check");
this.array.shrink(this.targetFileSize, this.maxFileSize);
this.lastCleanup = System.currentTimeMillis();
}
if (this.array.entries() > 50 || (this.lastCleanup + cleanupCycle < System.currentTimeMillis())) synchronized (this) {
if (this.array.entries() > 50 || (this.lastCleanup + cleanupCycle < System.currentTimeMillis())) {
//System.out.println("----cleanup check");
this.array.shrink(this.targetFileSize, this.maxFileSize);
this.lastCleanup = System.currentTimeMillis();
}
}
}

@ -48,18 +48,9 @@ import de.anomic.yacy.logging.Log;
public final class ReferenceContainerCache<ReferenceType extends Reference> extends AbstractIndex<ReferenceType> implements Index<ReferenceType>, IndexReader<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>> {
public class ContainerOrder implements Comparator<ReferenceContainer<ReferenceType>> {
public int compare(ReferenceContainer<ReferenceType> arg0, ReferenceContainer<ReferenceType> arg1) {
if (arg0 == arg1) return 0;
if (arg0 == null) return -1;
if (arg1 == null) return 1;
return termOrder.compare(arg0.getTermHash(), arg1.getTermHash());
}
}
private final Row payloadrow;
protected final ByteOrder termOrder;
private final ContainerOrder containerOrder;
private final ContainerOrder<ReferenceType> containerOrder;
protected Map<ByteArray, ReferenceContainer<ReferenceType>> cache;
/**
@ -73,8 +64,8 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
super(factory);
this.payloadrow = payloadrow;
this.termOrder = termOrder;
this.containerOrder = new ContainerOrder();
this.cache = null;
this.containerOrder = new ContainerOrder<ReferenceType>(this.termOrder);
this.cache = new ConcurrentHashMap<ByteArray, ReferenceContainer<ReferenceType>>();
}
public Row rowdef() {
@ -83,21 +74,12 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
public void clear() {
if (cache != null) cache.clear();
initWriteMode();
}
public void close() {
this.cache = null;
}
/**
* initializes the heap in read/write mode without reading of a dump first
* another dump reading afterwards is not possible
*/
public void initWriteMode() {
this.cache = new ConcurrentHashMap<ByteArray, ReferenceContainer<ReferenceType>>();
}
public void dump(final File heapFile, int writeBuffer) {
assert this.cache != null;
Log.logInfo("indexContainerRAMHeap", "creating rwi heap dump '" + heapFile.getName() + "', " + cache.size() + " rwi's");
@ -160,18 +142,7 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
Arrays.sort(cachecopy, this.containerOrder);
return cachecopy;
}
/*
public SortedMap<byte[], ReferenceContainer<ReferenceType>> sortedClone() {
SortedMap<byte[], ReferenceContainer<ReferenceType>> cachecopy;
synchronized (cache) {
cachecopy = new TreeMap<byte[], ReferenceContainer<ReferenceType>>(this.termOrder);
for (final Map.Entry<ByteArray, ReferenceContainer<ReferenceType>> entry: cache.entrySet()) {
cachecopy.put(entry.getKey().asBytes(), entry.getValue());
}
}
return cachecopy;
}
*/
public int size() {
return (this.cache == null) ? 0 : this.cache.size();
}
@ -345,8 +316,8 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
* @return the indexContainer if one exist, null otherwise
*/
public ReferenceContainer<ReferenceType> get(final byte[] key, Set<String> urlselection) {
if (urlselection == null) return this.cache.get(new ByteArray(key));
ReferenceContainer<ReferenceType> c = this.cache.get(new ByteArray(key));
if (urlselection == null) return c;
if (c == null) return null;
// because this is all in RAM, we must clone the entries (flat)
ReferenceContainer<ReferenceType> c1 = new ReferenceContainer<ReferenceType>(factory, c.getTermHash(), c.row(), c.size());
@ -471,31 +442,6 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
}
}
}
/*
public void add(final byte[] termHash, final ReferenceType newEntry) {
assert this.cache != null;
ByteArray tha = new ByteArray(termHash);
// first access the cache without synchronization
ReferenceContainer<ReferenceType> container = cache.remove(tha);
if (container == null) container = new ReferenceContainer<ReferenceType>(factory, termHash, this.payloadrow, 1);
container.put(newEntry);
// then try to replace the entry that should be empty,
// but it can be possible that another thread has written something in between
ReferenceContainer<ReferenceType> containerNew = cache.put(tha, container);
if (containerNew == null) return;
container = containerNew;
// finally use synchronization: ensure that the entry is written exclusively
synchronized (cache) {
containerNew = cache.get(tha);
if (containerNew != null) container.putAllRecent(containerNew);
cache.put(tha, container);
}
}
*/
public int minMem() {
return 0;
@ -505,4 +451,17 @@ public final class ReferenceContainerCache<ReferenceType extends Reference> exte
return this.termOrder;
}
public static class ContainerOrder<ReferenceType extends Reference> implements Comparator<ReferenceContainer<ReferenceType>> {
private ByteOrder o;
public ContainerOrder(ByteOrder order) {
this.o = order;
}
public int compare(ReferenceContainer<ReferenceType> arg0, ReferenceContainer<ReferenceType> arg1) {
if (arg0 == arg1) return 0;
if (arg0 == null) return -1;
if (arg1 == null) return 1;
return o.compare(arg0.getTermHash(), arg1.getTermHash());
}
}
}

@ -1,150 +0,0 @@
// ReferenceContainerConcurrentCache.java
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 05.04.2009 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2009-05-05 22:08:23 +0200 (Di, 05 Mai 2009) $
// $LastChangedRevision: 5924 $
// $LastChangedBy: orbiter $
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package de.anomic.kelondro.text;
import de.anomic.kelondro.order.ByteOrder;
import de.anomic.kelondro.index.Row;
public final class ReferenceContainerConcurrentCache<ReferenceType extends Reference> /* extends AbstractIndex<ReferenceType> implements Index<ReferenceType>, IndexReader<ReferenceType>, Iterable<ReferenceContainer<ReferenceType>>*/ {
private final Row payloadrow;
private final ByteOrder termOrder;
private ReferenceContainerCache<ReferenceType> caches[];
private final int concurrency;
private final ReferenceFactory<ReferenceType> factory;
public ReferenceContainerConcurrentCache(final ReferenceFactory<ReferenceType> factory, final Row payloadrow, ByteOrder termOrder, int concurrency) {
//super(factory);
this.payloadrow = payloadrow;
this.termOrder = termOrder;
this.concurrency = concurrency;
this.caches = null;
this.factory = factory;
}
public Row rowdef() {
return this.payloadrow;
}
public void clear() {
if (caches != null) {
for (int i = 0; i < caches.length; i++) {
caches[i].clear();
caches[i].initWriteMode();
}
}
}
public void close() {
if (caches != null) {
for (int i = 0; i < caches.length; i++) {
caches[i].close();
}
}
this.caches = null;
}
@SuppressWarnings("unchecked")
public void initWriteMode() {
caches = new ReferenceContainerCache[concurrency];
for (int i = 0; i < caches.length; i++) {
caches[i] = new ReferenceContainerCache<ReferenceType>(factory, payloadrow, termOrder);
}
}
public int size() {
if (caches == null) return 0;
int count = 0;
for (int i = 0; i < caches.length; i++) {
count += caches[i].size();
}
return count;
}
public int maxReferences() {
if (caches == null) return 0;
int max = 0;
for (int i = 0; i < caches.length; i++) {
max = Math.max(max, caches[i].maxReferences());
}
return max;
}
/*
public synchronized CloneableIterator<ReferenceContainer<ReferenceType>> references(final byte[] startWordHash, final boolean rot) {
ArrayList<CloneableIterator<ReferenceContainer<ReferenceType>>> a = new ArrayList<CloneableIterator<ReferenceContainer<ReferenceType>>>(caches.length);
for (int i = 0; i < caches.length; i++) {
a.add(caches[i].references(startWordHash, rot));
}
return MergeIterator.cascade(a, termOrder, MergeIterator.simpleMerge, true);
}
public Iterator<ReferenceContainer<ReferenceType>> iterator() {
return references(null, false);
}
public boolean has(final byte[] key) {
return this.cache.containsKey(new ByteArray(key));
}
public ReferenceContainer<ReferenceType> get(final byte[] key, Set<String> urlselection) {
}
public int count(final byte[] key) {
}
public ReferenceContainer<ReferenceType> delete(final byte[] termHash) {
}
public boolean remove(final byte[] termHash, final String urlHash) {
}
public int remove(final byte[] termHash, final Set<String> urlHashes) {
}
public void add(final ReferenceContainer<ReferenceType> container) {
}
public void add(final byte[] termHash, final ReferenceType newEntry) {
}
public int minMem() {
return 0;
}
public ByteOrder ordering() {
return this.termOrder;
}
*/
}

@ -108,7 +108,6 @@ public class Transmission {
super();
this.primaryTarget = primaryTarget;
this.containers = new ReferenceContainerCache<WordReference>(Segment.wordReferenceFactory, payloadrow, Segment.wordOrder);
this.containers.initWriteMode();
this.references = new HashMap<String, URLMetadataRow>();
this.badReferences = new HashSet<String>();
this.targets = targets;

Loading…
Cancel
Save