From c3a4aee2552759494680176fe3b7ee40acedc82a Mon Sep 17 00:00:00 2001 From: orbiter Date: Tue, 22 Sep 2009 14:33:57 +0000 Subject: [PATCH] some redesign with a possible fix for the ReferenceContainerCache. git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6336 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/yacysearch.java | 5 +- htroot/yacysearchtrailer.java | 4 +- source/de/anomic/data/wiki/wikiBoard.java | 2 +- .../de/anomic/document/parser/docParser.java | 1 - source/de/anomic/kelondro/text/IndexCell.java | 62 ++++---- .../text/ReferenceContainerCache.java | 77 +++------ .../ReferenceContainerConcurrentCache.java | 150 ------------------ source/de/anomic/yacy/dht/Transmission.java | 1 - 8 files changed, 57 insertions(+), 245 deletions(-) delete mode 100644 source/de/anomic/kelondro/text/ReferenceContainerConcurrentCache.java diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index b31b895da..9a42be07a 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -77,7 +77,10 @@ public class yacysearch { final boolean authenticated = sb.adminAuthenticated(header) >= 2; int display = (post == null) ? 0 : post.getInt("display", 0); - if ((display == 1) && (!authenticated)) display = 0; + if (!authenticated) display = 2; + // display == 0: shop top menu + // display == 1: show top and left menu + // display == 2: do not show any menu final boolean browserPopUpTrigger = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_TRIGGER, "true").equals("true"); if (browserPopUpTrigger) { final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html"); diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index 46848e79a..49184cd87 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -135,7 +135,9 @@ public class yacysearchtrailer { // about box String aboutBody = env.getConfig("about.body", ""); String aboutHeadline = env.getConfig("about.headline", ""); - if (aboutBody.length() == 0 && aboutHeadline.length() == 0) { + if ((aboutBody.length() == 0 && aboutHeadline.length() == 0) || + theSearch.getRankingResult().getLocalResourceSize() + + theSearch.getRankingResult().getRemoteResourceSize() == 0) { prop.put("nav-about", 0); } else { prop.put("nav-about", 1); diff --git a/source/de/anomic/data/wiki/wikiBoard.java b/source/de/anomic/data/wiki/wikiBoard.java index ef7485d57..053b6e8e9 100644 --- a/source/de/anomic/data/wiki/wikiBoard.java +++ b/source/de/anomic/data/wiki/wikiBoard.java @@ -44,7 +44,7 @@ public class wikiBoard { public static final int keyLength = 64; private static final String dateFormat = "yyyyMMddHHmmss"; - private static final SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat); + protected static final SimpleDateFormat SimpleFormatter = new SimpleDateFormat(dateFormat); static { SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT")); diff --git a/source/de/anomic/document/parser/docParser.java b/source/de/anomic/document/parser/docParser.java index d50ff7641..6c4a77448 100644 --- a/source/de/anomic/document/parser/docParser.java +++ b/source/de/anomic/document/parser/docParser.java @@ -27,7 +27,6 @@ package de.anomic.document.parser; -import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.util.HashSet; diff --git a/source/de/anomic/kelondro/text/IndexCell.java b/source/de/anomic/kelondro/text/IndexCell.java index 451cb4ae3..f081796d9 100644 --- a/source/de/anomic/kelondro/text/IndexCell.java +++ b/source/de/anomic/kelondro/text/IndexCell.java @@ -81,7 +81,6 @@ public final class IndexCell extends AbstractBu this.array = new ReferenceContainerArray(cellPath, factory, termOrder, payloadrow, merger); this.ram = new ReferenceContainerCache(factory, payloadrow, termOrder); - this.ram.initWriteMode(); this.maxRamEntries = maxRamEntries; this.merger = merger; this.lastCleanup = System.currentTimeMillis(); @@ -295,18 +294,18 @@ public final class IndexCell extends AbstractBu } public int[] sizes() { - int[] as = this.array.sizes(); - int[] asr = new int[as.length + 1]; - System.arraycopy(as, 0, asr, 0, as.length); - asr[as.length] = this.ram.size(); - return asr; + int[] as = this.array.sizes(); + int[] asr = new int[as.length + 1]; + System.arraycopy(as, 0, asr, 0, as.length); + asr[as.length] = this.ram.size(); + return asr; } public int sizesMax() { - int m = 0; - int[] s = sizes(); - for (int i = 0; i < s.length; i++) if (s[i] > m) m = s[i]; - return m; + int m = 0; + int[] s = sizes(); + for (int i = 0; i < s.length; i++) if (s[i] > m) m = s[i]; + return m; } public int minMem() { @@ -323,29 +322,30 @@ public final class IndexCell extends AbstractBu */ private void cleanCache() { - this.countCache.clear(); - + this.countCache.clear(); + // dump the cache if necessary - if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) synchronized (this) { - if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) { - // dump the ram - File dumpFile = this.array.newContainerBLOBFile(); - //this.ram.dump(dumpFile, true); - //this.array.mountBLOBContainer(dumpFile); - merger.dump(this.ram, dumpFile, array); - // get a fresh ram cache - this.ram = new ReferenceContainerCache(factory, this.array.rowdef(), this.array.ordering()); - this.ram.initWriteMode(); - } - } - + if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) synchronized (this) { + if (this.ram.size() >= this.maxRamEntries || (this.ram.size() > 3000 && !MemoryControl.request(80L * 1024L * 1024L, false))) { + // dump the ram + File dumpFile = this.array.newContainerBLOBFile(); + // a critical point: when the ram is handed to the dump job, + // dont write into it any more. Use a fresh one instead + ReferenceContainerCache ramdump = this.ram; + // get a fresh ram cache + this.ram = new ReferenceContainerCache(factory, this.array.rowdef(), this.array.ordering()); + // dump the buffer + merger.dump(ramdump, dumpFile, array); + } + } + // clean-up the cache - if (this.array.entries() > 50 || (this.lastCleanup + cleanupCycle < System.currentTimeMillis())) synchronized (this) { - if (this.array.entries() > 50 || (this.lastCleanup + cleanupCycle < System.currentTimeMillis())) { - //System.out.println("----cleanup check"); - this.array.shrink(this.targetFileSize, this.maxFileSize); - this.lastCleanup = System.currentTimeMillis(); - } + if (this.array.entries() > 50 || (this.lastCleanup + cleanupCycle < System.currentTimeMillis())) synchronized (this) { + if (this.array.entries() > 50 || (this.lastCleanup + cleanupCycle < System.currentTimeMillis())) { + //System.out.println("----cleanup check"); + this.array.shrink(this.targetFileSize, this.maxFileSize); + this.lastCleanup = System.currentTimeMillis(); + } } } diff --git a/source/de/anomic/kelondro/text/ReferenceContainerCache.java b/source/de/anomic/kelondro/text/ReferenceContainerCache.java index cbcb87cee..d17de927a 100644 --- a/source/de/anomic/kelondro/text/ReferenceContainerCache.java +++ b/source/de/anomic/kelondro/text/ReferenceContainerCache.java @@ -48,18 +48,9 @@ import de.anomic.yacy.logging.Log; public final class ReferenceContainerCache extends AbstractIndex implements Index, IndexReader, Iterable> { - public class ContainerOrder implements Comparator> { - public int compare(ReferenceContainer arg0, ReferenceContainer arg1) { - if (arg0 == arg1) return 0; - if (arg0 == null) return -1; - if (arg1 == null) return 1; - return termOrder.compare(arg0.getTermHash(), arg1.getTermHash()); - } - } - private final Row payloadrow; protected final ByteOrder termOrder; - private final ContainerOrder containerOrder; + private final ContainerOrder containerOrder; protected Map> cache; /** @@ -73,8 +64,8 @@ public final class ReferenceContainerCache exte super(factory); this.payloadrow = payloadrow; this.termOrder = termOrder; - this.containerOrder = new ContainerOrder(); - this.cache = null; + this.containerOrder = new ContainerOrder(this.termOrder); + this.cache = new ConcurrentHashMap>(); } public Row rowdef() { @@ -83,21 +74,12 @@ public final class ReferenceContainerCache exte public void clear() { if (cache != null) cache.clear(); - initWriteMode(); } public void close() { this.cache = null; } - /** - * initializes the heap in read/write mode without reading of a dump first - * another dump reading afterwards is not possible - */ - public void initWriteMode() { - this.cache = new ConcurrentHashMap>(); - } - public void dump(final File heapFile, int writeBuffer) { assert this.cache != null; Log.logInfo("indexContainerRAMHeap", "creating rwi heap dump '" + heapFile.getName() + "', " + cache.size() + " rwi's"); @@ -160,18 +142,7 @@ public final class ReferenceContainerCache exte Arrays.sort(cachecopy, this.containerOrder); return cachecopy; } - /* - public SortedMap> sortedClone() { - SortedMap> cachecopy; - synchronized (cache) { - cachecopy = new TreeMap>(this.termOrder); - for (final Map.Entry> entry: cache.entrySet()) { - cachecopy.put(entry.getKey().asBytes(), entry.getValue()); - } - } - return cachecopy; - } - */ + public int size() { return (this.cache == null) ? 0 : this.cache.size(); } @@ -345,8 +316,8 @@ public final class ReferenceContainerCache exte * @return the indexContainer if one exist, null otherwise */ public ReferenceContainer get(final byte[] key, Set urlselection) { - if (urlselection == null) return this.cache.get(new ByteArray(key)); ReferenceContainer c = this.cache.get(new ByteArray(key)); + if (urlselection == null) return c; if (c == null) return null; // because this is all in RAM, we must clone the entries (flat) ReferenceContainer c1 = new ReferenceContainer(factory, c.getTermHash(), c.row(), c.size()); @@ -471,31 +442,6 @@ public final class ReferenceContainerCache exte } } } - - /* - public void add(final byte[] termHash, final ReferenceType newEntry) { - assert this.cache != null; - ByteArray tha = new ByteArray(termHash); - - // first access the cache without synchronization - ReferenceContainer container = cache.remove(tha); - if (container == null) container = new ReferenceContainer(factory, termHash, this.payloadrow, 1); - container.put(newEntry); - - // then try to replace the entry that should be empty, - // but it can be possible that another thread has written something in between - ReferenceContainer containerNew = cache.put(tha, container); - if (containerNew == null) return; - container = containerNew; - - // finally use synchronization: ensure that the entry is written exclusively - synchronized (cache) { - containerNew = cache.get(tha); - if (containerNew != null) container.putAllRecent(containerNew); - cache.put(tha, container); - } - } - */ public int minMem() { return 0; @@ -505,4 +451,17 @@ public final class ReferenceContainerCache exte return this.termOrder; } + public static class ContainerOrder implements Comparator> { + private ByteOrder o; + public ContainerOrder(ByteOrder order) { + this.o = order; + } + public int compare(ReferenceContainer arg0, ReferenceContainer arg1) { + if (arg0 == arg1) return 0; + if (arg0 == null) return -1; + if (arg1 == null) return 1; + return o.compare(arg0.getTermHash(), arg1.getTermHash()); + } + } + } diff --git a/source/de/anomic/kelondro/text/ReferenceContainerConcurrentCache.java b/source/de/anomic/kelondro/text/ReferenceContainerConcurrentCache.java deleted file mode 100644 index 8ab2b4d37..000000000 --- a/source/de/anomic/kelondro/text/ReferenceContainerConcurrentCache.java +++ /dev/null @@ -1,150 +0,0 @@ -// ReferenceContainerConcurrentCache.java -// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany -// first published 05.04.2009 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate: 2009-05-05 22:08:23 +0200 (Di, 05 Mai 2009) $ -// $LastChangedRevision: 5924 $ -// $LastChangedBy: orbiter $ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package de.anomic.kelondro.text; - -import de.anomic.kelondro.order.ByteOrder; -import de.anomic.kelondro.index.Row; - -public final class ReferenceContainerConcurrentCache /* extends AbstractIndex implements Index, IndexReader, Iterable>*/ { - - private final Row payloadrow; - private final ByteOrder termOrder; - private ReferenceContainerCache caches[]; - private final int concurrency; - private final ReferenceFactory factory; - - public ReferenceContainerConcurrentCache(final ReferenceFactory factory, final Row payloadrow, ByteOrder termOrder, int concurrency) { - //super(factory); - this.payloadrow = payloadrow; - this.termOrder = termOrder; - this.concurrency = concurrency; - this.caches = null; - this.factory = factory; - } - - public Row rowdef() { - return this.payloadrow; - } - - public void clear() { - if (caches != null) { - for (int i = 0; i < caches.length; i++) { - caches[i].clear(); - caches[i].initWriteMode(); - } - } - } - - public void close() { - if (caches != null) { - for (int i = 0; i < caches.length; i++) { - caches[i].close(); - } - } - this.caches = null; - } - - @SuppressWarnings("unchecked") - public void initWriteMode() { - caches = new ReferenceContainerCache[concurrency]; - for (int i = 0; i < caches.length; i++) { - caches[i] = new ReferenceContainerCache(factory, payloadrow, termOrder); - } - } - - public int size() { - if (caches == null) return 0; - int count = 0; - for (int i = 0; i < caches.length; i++) { - count += caches[i].size(); - } - return count; - } - - public int maxReferences() { - if (caches == null) return 0; - int max = 0; - for (int i = 0; i < caches.length; i++) { - max = Math.max(max, caches[i].maxReferences()); - } - return max; - } - -/* - public synchronized CloneableIterator> references(final byte[] startWordHash, final boolean rot) { - ArrayList>> a = new ArrayList>>(caches.length); - for (int i = 0; i < caches.length; i++) { - a.add(caches[i].references(startWordHash, rot)); - } - return MergeIterator.cascade(a, termOrder, MergeIterator.simpleMerge, true); - } - - public Iterator> iterator() { - return references(null, false); - } - - public boolean has(final byte[] key) { - return this.cache.containsKey(new ByteArray(key)); - } - - public ReferenceContainer get(final byte[] key, Set urlselection) { - - } - - public int count(final byte[] key) { - - } - - public ReferenceContainer delete(final byte[] termHash) { - - } - - public boolean remove(final byte[] termHash, final String urlHash) { - - } - - public int remove(final byte[] termHash, final Set urlHashes) { - - } - - public void add(final ReferenceContainer container) { - - } - - public void add(final byte[] termHash, final ReferenceType newEntry) { - - } - - public int minMem() { - return 0; - } - - public ByteOrder ordering() { - return this.termOrder; - } - */ -} diff --git a/source/de/anomic/yacy/dht/Transmission.java b/source/de/anomic/yacy/dht/Transmission.java index afe9fa20c..629edd04d 100644 --- a/source/de/anomic/yacy/dht/Transmission.java +++ b/source/de/anomic/yacy/dht/Transmission.java @@ -108,7 +108,6 @@ public class Transmission { super(); this.primaryTarget = primaryTarget; this.containers = new ReferenceContainerCache(Segment.wordReferenceFactory, payloadrow, Segment.wordOrder); - this.containers.initWriteMode(); this.references = new HashMap(); this.badReferences = new HashSet(); this.targets = targets;