refactoring of plasmaWordIndex: less methods in the class, separated the index to CachedIndexCollection
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5710 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
16 years ago
parent
14a1c33823
commit
7f67238f8b
@ -0,0 +1,449 @@
|
||||
// plasmaWordIndex.java
|
||||
// (C) 2005, 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 2005 on http://www.anomic.de
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate: 2009-03-13 11:34:51 +0100 (Fr, 13 Mrz 2009) $
|
||||
// $LastChangedRevision: 5709 $
|
||||
// $LastChangedBy: orbiter $
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package de.anomic.kelondro.text;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import de.anomic.kelondro.index.RowCollection;
|
||||
import de.anomic.kelondro.order.Base64Order;
|
||||
import de.anomic.kelondro.order.ByteOrder;
|
||||
import de.anomic.kelondro.order.CloneableIterator;
|
||||
import de.anomic.kelondro.order.Order;
|
||||
import de.anomic.kelondro.order.RotateIterator;
|
||||
import de.anomic.kelondro.text.Index;
|
||||
import de.anomic.kelondro.text.IndexCache;
|
||||
import de.anomic.kelondro.text.IndexCollection;
|
||||
import de.anomic.kelondro.text.ReferenceContainer;
|
||||
import de.anomic.kelondro.text.ReferenceContainerOrder;
|
||||
import de.anomic.kelondro.text.ReferenceRow;
|
||||
import de.anomic.kelondro.text.Word;
|
||||
import de.anomic.kelondro.util.MemoryControl;
|
||||
import de.anomic.kelondro.util.Log;
|
||||
import de.anomic.server.serverProfiling;
|
||||
|
||||
public final class CachedIndexCollection implements Index {
|
||||
|
||||
// environment constants
|
||||
public static final long wCacheMaxAge = 1000 * 60 * 30; // milliseconds; 30 minutes
|
||||
public static final int wCacheMaxChunk = 800; // maximum number of references for each urlhash
|
||||
public static final int lowcachedivisor = 900;
|
||||
public static final int maxCollectionPartition = 7; // should be 7
|
||||
private static final ByteOrder indexOrder = Base64Order.enhancedCoder;
|
||||
|
||||
|
||||
|
||||
private final IndexCache indexCache;
|
||||
private final IndexCollection collections; // new database structure to replace AssortmentCluster and FileCluster
|
||||
|
||||
public CachedIndexCollection(
|
||||
File indexPrimaryTextLocation,
|
||||
final int entityCacheMaxSize,
|
||||
final boolean useCommons,
|
||||
final int redundancy,
|
||||
Log log) throws IOException {
|
||||
|
||||
final File textindexcache = new File(indexPrimaryTextLocation, "RICACHE");
|
||||
if (!(textindexcache.exists())) textindexcache.mkdirs();
|
||||
if (new File(textindexcache, "index.dhtin.blob").exists()) {
|
||||
// migration of the both caches into one
|
||||
this.indexCache = new IndexCache(textindexcache, ReferenceRow.urlEntryRow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log);
|
||||
IndexCache dhtInCache = new IndexCache(textindexcache, ReferenceRow.urlEntryRow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtin.blob", log);
|
||||
for (ReferenceContainer c: dhtInCache) {
|
||||
this.indexCache.addReferences(c);
|
||||
}
|
||||
new File(textindexcache, "index.dhtin.blob").delete();
|
||||
} else {
|
||||
// read in new BLOB
|
||||
this.indexCache = new IndexCache(textindexcache, ReferenceRow.urlEntryRow, entityCacheMaxSize, wCacheMaxChunk, wCacheMaxAge, "index.dhtout.blob", log);
|
||||
}
|
||||
|
||||
// create collections storage path
|
||||
final File textindexcollections = new File(indexPrimaryTextLocation, "RICOLLECTION");
|
||||
if (!(textindexcollections.exists())) textindexcollections.mkdirs();
|
||||
this.collections = new IndexCollection(
|
||||
textindexcollections,
|
||||
"collection",
|
||||
12,
|
||||
Base64Order.enhancedCoder,
|
||||
maxCollectionPartition,
|
||||
ReferenceRow.urlEntryRow,
|
||||
useCommons);
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
indexCache.clear();
|
||||
try {
|
||||
collections.clear();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public int minMem() {
|
||||
return 1024*1024 /* indexing overhead */ + indexCache.minMem() + collections.minMem();
|
||||
}
|
||||
|
||||
public int maxURLinCache() {
|
||||
return indexCache.maxURLinCache();
|
||||
}
|
||||
|
||||
public long minAgeOfCache() {
|
||||
return indexCache.minAgeOfCache();
|
||||
}
|
||||
|
||||
public long maxAgeOfCache() {
|
||||
return indexCache.maxAgeOfCache();
|
||||
}
|
||||
|
||||
public int indexCacheSize() {
|
||||
return indexCache.size();
|
||||
}
|
||||
|
||||
public long indexCacheSizeBytes() {
|
||||
// calculate the real size in bytes of the index cache
|
||||
long cacheBytes = 0;
|
||||
final long entryBytes = ReferenceRow.urlEntryRow.objectsize;
|
||||
final IndexCache cache = (indexCache);
|
||||
synchronized (cache) {
|
||||
final Iterator<ReferenceContainer> it = cache.referenceIterator(null, false, true);
|
||||
while (it.hasNext()) cacheBytes += it.next().size() * entryBytes;
|
||||
}
|
||||
return cacheBytes;
|
||||
}
|
||||
|
||||
public void setMaxWordCount(final int maxWords) {
|
||||
indexCache.setMaxWordCount(maxWords);
|
||||
}
|
||||
|
||||
public void cacheFlushControl(final IndexCache theCache) {
|
||||
// check for forced flush
|
||||
int cs = cacheSize();
|
||||
if (cs > 0) {
|
||||
// flush elements that are too big. This flushing depends on the fact that the flush rule
|
||||
// selects the biggest elements first for flushing. If it does not for any reason, the following
|
||||
// loop would not terminate.
|
||||
serverProfiling.update("wordcache", Long.valueOf(cs), true);
|
||||
// To ensure termination an additional counter is used
|
||||
int l = 0;
|
||||
while (theCache.size() > 0 && (l++ < 100) && (theCache.maxURLinCache() > wCacheMaxChunk)) {
|
||||
flushCacheOne(theCache);
|
||||
}
|
||||
// next flush more entries if the size exceeds the maximum size of the cache
|
||||
while (theCache.size() > 0 &&
|
||||
((theCache.size() > theCache.getMaxWordCount()) ||
|
||||
(MemoryControl.available() < collections.minMem()))) {
|
||||
flushCacheOne(theCache);
|
||||
}
|
||||
if (cacheSize() != cs) serverProfiling.update("wordcache", Long.valueOf(cacheSize()), true);
|
||||
}
|
||||
}
|
||||
|
||||
public static ReferenceContainer emptyContainer(final String wordHash, final int elementCount) {
|
||||
return new ReferenceContainer(wordHash, ReferenceRow.urlEntryRow, elementCount);
|
||||
}
|
||||
|
||||
public void addEntry(final String wordHash, final ReferenceRow entry, final long updateTime) {
|
||||
// add the entry
|
||||
indexCache.addEntry(wordHash, entry, updateTime, true);
|
||||
cacheFlushControl(this.indexCache);
|
||||
}
|
||||
|
||||
public void addReferences(final ReferenceContainer entries) {
|
||||
assert (entries.row().objectsize == ReferenceRow.urlEntryRow.objectsize);
|
||||
|
||||
// add the entry
|
||||
indexCache.addReferences(entries);
|
||||
cacheFlushControl(this.indexCache);
|
||||
}
|
||||
|
||||
public void flushCacheFor(int time) {
|
||||
flushCacheUntil(System.currentTimeMillis() + time);
|
||||
}
|
||||
|
||||
private synchronized void flushCacheUntil(long timeout) {
|
||||
while (System.currentTimeMillis() < timeout && indexCache.size() > 0) {
|
||||
flushCacheOne(indexCache);
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized void flushCacheOne(final IndexCache ram) {
|
||||
if (ram.size() > 0) collections.addReferences(flushContainer(ram));
|
||||
}
|
||||
|
||||
private ReferenceContainer flushContainer(final IndexCache ram) {
|
||||
String wordHash;
|
||||
ReferenceContainer c;
|
||||
wordHash = ram.maxScoreWordHash();
|
||||
c = ram.getReferences(wordHash, null);
|
||||
if ((c != null) && (c.size() > wCacheMaxChunk)) {
|
||||
return ram.deleteAllReferences(wordHash);
|
||||
} else {
|
||||
return ram.deleteAllReferences(ram.bestFlushWordHash());
|
||||
}
|
||||
}
|
||||
|
||||
public boolean hasReferences(final String wordHash) {
|
||||
if (indexCache.hasReferences(wordHash)) return true;
|
||||
if (collections.hasReferences(wordHash)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
public ReferenceContainer getReferences(final String wordHash, final Set<String> urlselection) {
|
||||
if (wordHash == null) {
|
||||
// wrong input
|
||||
return null;
|
||||
}
|
||||
|
||||
// get from cache
|
||||
ReferenceContainer container;
|
||||
container = indexCache.getReferences(wordHash, urlselection);
|
||||
|
||||
// get from collection index
|
||||
if (container == null) {
|
||||
container = collections.getReferences(wordHash, urlselection);
|
||||
} else {
|
||||
container.addAllUnique(collections.getReferences(wordHash, urlselection));
|
||||
}
|
||||
|
||||
if (container == null) return null;
|
||||
|
||||
// check doubles
|
||||
final int beforeDouble = container.size();
|
||||
container.sort();
|
||||
final ArrayList<RowCollection> d = container.removeDoubles();
|
||||
RowCollection set;
|
||||
for (int i = 0; i < d.size(); i++) {
|
||||
// for each element in the double-set, take that one that is the most recent one
|
||||
set = d.get(i);
|
||||
ReferenceRow e, elm = null;
|
||||
long lm = 0;
|
||||
for (int j = 0; j < set.size(); j++) {
|
||||
e = new ReferenceRow(set.get(j, true));
|
||||
if ((elm == null) || (e.lastModified() > lm)) {
|
||||
elm = e;
|
||||
lm = e.lastModified();
|
||||
}
|
||||
}
|
||||
if(elm != null) {
|
||||
container.addUnique(elm.toKelondroEntry());
|
||||
}
|
||||
}
|
||||
if (container.size() < beforeDouble) System.out.println("*** DEBUG DOUBLECHECK - removed " + (beforeDouble - container.size()) + " index entries from word container " + container.getWordHash());
|
||||
|
||||
return container;
|
||||
}
|
||||
|
||||
/**
|
||||
* return map of wordhash:indexContainer
|
||||
*
|
||||
* @param wordHashes
|
||||
* @param urlselection
|
||||
* @param deleteIfEmpty
|
||||
* @param interruptIfEmpty
|
||||
* @return
|
||||
*/
|
||||
public HashMap<String, ReferenceContainer> getContainers(final Set<String> wordHashes, final Set<String> urlselection, final boolean interruptIfEmpty) {
|
||||
// retrieve entities that belong to the hashes
|
||||
final HashMap<String, ReferenceContainer> containers = new HashMap<String, ReferenceContainer>(wordHashes.size());
|
||||
String singleHash;
|
||||
ReferenceContainer singleContainer;
|
||||
final Iterator<String> i = wordHashes.iterator();
|
||||
while (i.hasNext()) {
|
||||
|
||||
// get next word hash:
|
||||
singleHash = i.next();
|
||||
|
||||
// retrieve index
|
||||
singleContainer = getReferences(singleHash, urlselection);
|
||||
|
||||
// check result
|
||||
if (((singleContainer == null) || (singleContainer.size() == 0)) && (interruptIfEmpty)) return new HashMap<String, ReferenceContainer>(0);
|
||||
|
||||
containers.put(singleHash, singleContainer);
|
||||
}
|
||||
return containers;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public HashMap<String, ReferenceContainer>[] localSearchContainers(
|
||||
final TreeSet<String> queryHashes,
|
||||
final TreeSet<String> excludeHashes,
|
||||
final Set<String> urlselection) {
|
||||
// search for the set of hashes and return a map of of wordhash:indexContainer containing the seach result
|
||||
|
||||
// retrieve entities that belong to the hashes
|
||||
HashMap<String, ReferenceContainer> inclusionContainers = (queryHashes.size() == 0) ? new HashMap<String, ReferenceContainer>(0) : getContainers(
|
||||
queryHashes,
|
||||
urlselection,
|
||||
true);
|
||||
if ((inclusionContainers.size() != 0) && (inclusionContainers.size() < queryHashes.size())) inclusionContainers = new HashMap<String, ReferenceContainer>(0); // prevent that only a subset is returned
|
||||
final HashMap<String, ReferenceContainer> exclusionContainers = (inclusionContainers.size() == 0) ? new HashMap<String, ReferenceContainer>(0) : getContainers(
|
||||
excludeHashes,
|
||||
urlselection,
|
||||
true);
|
||||
return new HashMap[]{inclusionContainers, exclusionContainers};
|
||||
}
|
||||
|
||||
public int size() {
|
||||
return java.lang.Math.max(collections.size(), indexCache.size());
|
||||
}
|
||||
|
||||
public int collectionsSize() {
|
||||
return collections.size();
|
||||
}
|
||||
|
||||
public int cacheSize() {
|
||||
return indexCache.size();
|
||||
}
|
||||
|
||||
public void close() {
|
||||
indexCache.close();
|
||||
collections.close();
|
||||
}
|
||||
|
||||
public ReferenceContainer deleteAllReferences(final String wordHash) {
|
||||
final ReferenceContainer c = new ReferenceContainer(
|
||||
wordHash,
|
||||
ReferenceRow.urlEntryRow,
|
||||
indexCache.countReferences(wordHash));
|
||||
c.addAllUnique(indexCache.deleteAllReferences(wordHash));
|
||||
c.addAllUnique(collections.deleteAllReferences(wordHash));
|
||||
return c;
|
||||
}
|
||||
|
||||
public boolean removeReference(final String wordHash, final String urlHash) {
|
||||
boolean removed = false;
|
||||
removed = removed | (indexCache.removeReference(wordHash, urlHash));
|
||||
removed = removed | (collections.removeReference(wordHash, urlHash));
|
||||
return removed;
|
||||
}
|
||||
|
||||
public int removeEntryMultiple(final Set<String> wordHashes, final String urlHash) {
|
||||
// remove the same url hashes for multiple words
|
||||
// this is mainly used when correcting a index after a search
|
||||
final Iterator<String> i = wordHashes.iterator();
|
||||
int count = 0;
|
||||
while (i.hasNext()) {
|
||||
if (removeReference(i.next(), urlHash)) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
public int removeReferences(final String wordHash, final Set<String> urlHashes) {
|
||||
int removed = 0;
|
||||
removed += indexCache.removeReferences(wordHash, urlHashes);
|
||||
removed += collections.removeReferences(wordHash, urlHashes);
|
||||
return removed;
|
||||
}
|
||||
|
||||
public String removeEntriesExpl(final String wordHash, final Set<String> urlHashes) {
|
||||
String removed = "";
|
||||
removed += indexCache.removeReferences(wordHash, urlHashes) + ", ";
|
||||
removed += collections.removeReferences(wordHash, urlHashes);
|
||||
return removed;
|
||||
}
|
||||
|
||||
public void removeEntriesMultiple(final Set<String> wordHashes, final Set<String> urlHashes) {
|
||||
// remove the same url hashes for multiple words
|
||||
// this is mainly used when correcting a index after a search
|
||||
final Iterator<String> i = wordHashes.iterator();
|
||||
while (i.hasNext()) {
|
||||
removeReferences(i.next(), urlHashes);
|
||||
}
|
||||
}
|
||||
|
||||
public int removeWordReferences(final Set<String> words, final String urlhash) {
|
||||
// sequentially delete all word references
|
||||
// returns number of deletions
|
||||
final Iterator<String> iter = words.iterator();
|
||||
int count = 0;
|
||||
while (iter.hasNext()) {
|
||||
// delete the URL reference in this word index
|
||||
if (removeReference(Word.word2hash(iter.next()), urlhash)) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
public synchronized TreeSet<ReferenceContainer> indexContainerSet(final String startHash, final boolean ram, final boolean rot, int count) {
|
||||
// creates a set of indexContainers
|
||||
// this does not use the cache
|
||||
final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(indexOrder.clone());
|
||||
containerOrder.rotate(emptyContainer(startHash, 0));
|
||||
final TreeSet<ReferenceContainer> containers = new TreeSet<ReferenceContainer>(containerOrder);
|
||||
final Iterator<ReferenceContainer> i = referenceIterator(startHash, rot, ram);
|
||||
if (ram) count = Math.min(indexCache.size(), count);
|
||||
ReferenceContainer container;
|
||||
// this loop does not terminate using the i.hasNex() predicate when rot == true
|
||||
// because then the underlying iterator is a rotating iterator without termination
|
||||
// in this case a termination must be ensured with a counter
|
||||
// It must also be ensured that the counter is in/decreased every loop
|
||||
while ((count > 0) && (i.hasNext())) {
|
||||
container = i.next();
|
||||
if ((container != null) && (container.size() > 0)) {
|
||||
containers.add(container);
|
||||
}
|
||||
count--; // decrease counter even if the container was null or empty to ensure termination
|
||||
}
|
||||
return containers; // this may return less containers as demanded
|
||||
}
|
||||
|
||||
public synchronized CloneableIterator<ReferenceContainer> referenceIterator(final String startHash, final boolean rot, final boolean ram) {
|
||||
final CloneableIterator<ReferenceContainer> i = wordContainers(startHash, ram);
|
||||
if (rot) {
|
||||
return new RotateIterator<ReferenceContainer>(i, new String(Base64Order.zero(startHash.length())), indexCache.size() + ((ram) ? 0 : collections.size()));
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
private synchronized CloneableIterator<ReferenceContainer> wordContainers(final String startWordHash, final boolean ram) {
|
||||
final Order<ReferenceContainer> containerOrder = new ReferenceContainerOrder(indexOrder.clone());
|
||||
containerOrder.rotate(emptyContainer(startWordHash, 0));
|
||||
if (ram) {
|
||||
return indexCache.referenceIterator(startWordHash, false, true);
|
||||
}
|
||||
return collections.referenceIterator(startWordHash, false, false);
|
||||
/*
|
||||
return new MergeIterator<ReferenceContainer>(
|
||||
indexCache.referenceIterator(startWordHash, false, true),
|
||||
collections.referenceIterator(startWordHash, false, false),
|
||||
containerOrder,
|
||||
ReferenceContainer.containerMergeMethod,
|
||||
true);
|
||||
*/
|
||||
}
|
||||
|
||||
public int countReferences(String key) {
|
||||
return indexCache.countReferences(key) + collections.countReferences(key);
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in new issue