boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more refactoring

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5719 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 16 years ago
parent 209f25f5f5
commit 474aac65af
11 changed files with 63 additions and 30 deletions
Show Stats Download Patch File Download Diff File

7
htroot/yacy/transferRWI.java
Unescape View File

@ -27,6 +27,7 @@
// javac -classpath .:../classes transferRWI.java
import java.io.IOException;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@ -157,7 +158,11 @@ public final class transferRWI {
}
// learn entry
sb.webIndex.index().addEntry(wordHash, iEntry, System.currentTimeMillis());
try {
sb.webIndex.index().addReference(wordHash, iEntry);
} catch (IOException e) {
e.printStackTrace();
}
serverCore.checkInterruption();
// check if we need to ask for the corresponding URL

16
source/de/anomic/kelondro/text/CachedIndexCollection.java
Unescape View File

@ -109,6 +109,12 @@ public final class CachedIndexCollection extends AbstractIndex implements Index,
cacheFlushControl();
}
public void addReference(final String wordHash, final ReferenceRow entry) throws IOException {
// add the entry
indexCache.addReference(wordHash, entry);
cacheFlushControl();
}
public boolean hasReferences(final String wordHash) {
if (indexCache.hasReferences(wordHash)) return true;
if (collections.hasReferences(wordHash)) return true;
@ -330,17 +336,7 @@ public final class CachedIndexCollection extends AbstractIndex implements Index,
}
/*
* methods to update the index
*/
public void addEntry(final String wordHash, final ReferenceRow entry, final long updateTime) {
// add the entry
indexCache.addEntry(wordHash, entry, updateTime, true);
cacheFlushControl();
}
/*
* methods to search the index
*/

17
source/de/anomic/kelondro/text/Index.java
Unescape View File

@ -37,13 +37,24 @@ public interface Index {
/**
* add references to the reverse index
* when no references are stored, the new Entries are simply added,
* if there are already references to the word that is denoted
* with the reference stored, then merge the old and the new reference
* if no references to the word are stored, the new Entries are added,
* if there are already references to the word that is denoted with the
* reference to be stored, then the old and the new references are merged
* @param newEntries the References to be merged with existing references
* @throws IOException
*/
public void addReferences(ReferenceContainer newEntries) throws IOException;
/**
* add a single reference to the reverse index
* if no references to the word are stored, the a new entry is added,
* if there are already references to the word hash stored,
* then the old and the new references are merged
* @param wordHash
* @param entry
* @throws IOException
*/
public void addReference(final String wordHash, final ReferenceRow entry) throws IOException;
/**
* check if there are references stored to the given word hash

13
source/de/anomic/kelondro/text/IndexCache.java
Unescape View File

@ -299,8 +299,7 @@ public final class IndexCache extends AbstractIndex implements Index, IndexReade
}
public synchronized void addReferences(final ReferenceContainer container) {
// this puts the entries into the cache, not into the assortment directly
if ((container == null) || (container.size() == 0) || heap == null) return;
if (container == null || container.size() == 0 || heap == null) return;
// put new words into cache
heap.addReferences(container);
@ -308,11 +307,13 @@ public final class IndexCache extends AbstractIndex implements Index, IndexReade
hashDate.setScore(container.getWordHash(), intTime(System.currentTimeMillis()));
}
public synchronized void addEntry(final String wordHash, final ReferenceRow newEntry, final long updateTime, final boolean dhtCase) {
if (heap == null) return; // there was already a shutdown
heap.addEntry(wordHash, newEntry);
public void addReference(final String wordHash, final ReferenceRow entry) throws IOException {
if (entry == null || heap == null) return;
// put new words into cache
heap.addReference(wordHash, entry);
hashScore.incScore(wordHash);
hashDate.setScore(wordHash, intTime(updateTime));
hashDate.setScore(wordHash, intTime(System.currentTimeMillis()));
}
public synchronized void close() {

5
source/de/anomic/kelondro/text/IndexCell.java
Unescape View File

@ -84,6 +84,11 @@ public final class IndexCell extends AbstractIndex implements Index {
if (this.ram.size() > this.maxRamEntries) cacheDump();
}
public synchronized void addReference(String hash, ReferenceRow entry) throws IOException {
this.ram.addReference(hash, entry);
if (this.ram.size() > this.maxRamEntries) cacheDump();
}
/**
* clear the RAM and BLOB part, deletes everything in the cell
* @throws IOException

11
source/de/anomic/kelondro/text/IndexCollection.java
Unescape View File

@ -246,6 +246,17 @@ public class IndexCollection extends AbstractIndex implements Index {
}
}
public void addReference(String wordhash, ReferenceRow entry) {
if (entry == null) return;
try {
this.merge(new ReferenceContainer(wordhash, entry));
} catch (final kelondroOutOfLimitsException e) {
e.printStackTrace();
} catch (final IOException e) {
e.printStackTrace();
}
}
public int countReferences(String key) {
try {
final RowSet collection = this.get(key.getBytes());

5
source/de/anomic/kelondro/text/IndexPackage.java
Unescape View File

@ -70,11 +70,6 @@ public interface IndexPackage extends Index {
public int cacheSize();
/*
* methods to update the index
*/
public void addEntry(final String wordHash, final ReferenceRow entry, final long updateTime);
/*
* methods to search the index

6
source/de/anomic/kelondro/text/ReferenceContainer.java
Unescape View File

@ -53,6 +53,12 @@ public class ReferenceContainer extends RowSet {
this.wordHash = wordHash;
}
public ReferenceContainer(String wordHash, ReferenceRow entry) {
super(ReferenceRow.urlEntryRow, 1);
this.add(entry);
this.wordHash = wordHash;
}
public ReferenceContainer(final String wordHash, final Row rowdef, final int objectCount) {
super(rowdef, objectCount);
this.wordHash = wordHash;

2
source/de/anomic/kelondro/text/ReferenceContainerCache.java
Unescape View File

@ -499,7 +499,7 @@ public final class ReferenceContainerCache extends AbstractIndex implements Inde
return;
}
public synchronized void addEntry(final String wordHash, final ReferenceRow newEntry) {
public synchronized void addReference(final String wordHash, final ReferenceRow newEntry) {
assert this.cache != null;
ReferenceContainer container = cache.get(wordHash);
if (container == null) container = new ReferenceContainer(wordHash, this.payloadrow, 1);

5
source/de/anomic/kelondro/text/ReferenceRow.java
Unescape View File

@ -102,7 +102,7 @@ public final class ReferenceRow implements Reference, Cloneable {
final int posofphrase, // number of the phrase where word appears
final long lastmodified, // last-modified time of the document where word appears
final long updatetime, // update time; this is needed to compute a TTL for the word, so it can be removed easily if the TTL is short
String language, // (guessed) language of document
final String language, // (guessed) language of document
final char doctype, // type of document
final int outlinksSame, // outlinks to same domain
final int outlinksOther, // outlinks to other domain
@ -110,7 +110,6 @@ public final class ReferenceRow implements Reference, Cloneable {
) {
assert (urlHash.length() == 12) : "urlhash = " + urlHash;
if ((language == null) || (language.length() != urlEntryRow.width(col_language))) language = "uk";
this.entry = urlEntryRow.newEntry();
final int mddlm = MicroDate.microDateDays(lastmodified);
final int mddct = MicroDate.microDateDays(updatetime);
@ -121,7 +120,7 @@ public final class ReferenceRow implements Reference, Cloneable {
this.entry.setCol(col_wordsInText, wordcount);
this.entry.setCol(col_phrasesInText, phrasecount);
this.entry.setCol(col_doctype, new byte[]{(byte) doctype});
this.entry.setCol(col_language, language, null);
this.entry.setCol(col_language, ((language == null) || (language.length() != urlEntryRow.width(col_language))) ? "uk" : language, null);
this.entry.setCol(col_llocal, outlinksSame);
this.entry.setCol(col_lother, outlinksOther);
this.entry.setCol(col_urlLength, urlLength);

6
source/de/anomic/plasma/plasmaWordIndex.java
Unescape View File

@ -384,7 +384,11 @@ public final class plasmaWordIndex {
doctype,
outlinksSame, outlinksOther,
wprop.flags);
this.index.addEntry(Word.word2hash(word), ientry, System.currentTimeMillis());
try {
this.index.addReference(Word.word2hash(word), ientry);
} catch (IOException e) {
e.printStackTrace();
}
wordCount++;
}

Write Preview
Loading…
Cancel
Save