boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

performance hack: use a hash cache for all hashes that are computed by a

Browse Source 
byte array. If this hash is used in a HashMap (which is very often the
case) then this hack eliminates a lot of re-computations of the same
hash.
pull/1/head
Michael Peter Christen 13 years ago
parent f8a0cf6d7c
commit 10da7335ea
10 changed files with 135 additions and 63 deletions
Show Stats Download Patch File Download Diff File

7
source/net/yacy/kelondro/data/citation/CitationReference.java
Unescape View File

@ -133,9 +133,14 @@ public class CitationReference implements Reference, Serializable {
return false;
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlhash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlhash());
}
return this.hashCache;
}
@Override

18
source/net/yacy/kelondro/data/image/ImageReferenceRow.java
Unescape View File

@ -200,18 +200,21 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
@Override
public ImageReferenceRow clone() {
final byte[] b = new byte[urlEntryRow.objectsize];
System.arraycopy(entry.bytes(), 0, b, 0, urlEntryRow.objectsize);
System.arraycopy(this.entry.bytes(), 0, b, 0, urlEntryRow.objectsize);
return new ImageReferenceRow(b);
}
@Override
public String toPropertyForm() {
return entry.toPropertyForm('=', true, true, false, false);
return this.entry.toPropertyForm('=', true, true, false, false);
}
@Override
public Entry toKelondroEntry() {
return this.entry;
}
@Override
public byte[] urlhash() {
return this.entry.getColBytes(col_urlhash, true);
}
@ -220,6 +223,7 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
return (int) this.entry.getColLong(col_lastModified); // this is the time in MicoDateDays format
}
@Override
public long lastModified() {
return MicroDate.reverseMicroDateDays((int) this.entry.getColLong(col_lastModified));
}
@ -232,6 +236,7 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
return (int) this.entry.getColLong(col_hitcount);
}
@Override
public Collection<Integer> positions() {
return new ArrayList<Integer>(0);
}
@ -271,17 +276,24 @@ public final class ImageReferenceRow extends AbstractReference implements /*Imag
return toPropertyForm();
}
@Override
public boolean isOlder(final Reference other) {
if (other == null) return false;
if (this.lastModified() < other.lastModified()) return true;
return false;
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlhash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlhash());
}
return this.hashCache;
}
@Override
public void join(Reference oe) {
throw new UnsupportedOperationException("");

7
source/net/yacy/kelondro/data/image/ImageReferenceVars.java
Unescape View File

@ -381,9 +381,14 @@ public class ImageReferenceVars extends AbstractReference implements ImageRefere
this.wordsintext = this.wordsintext + oe.wordsintext();
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlHash);
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlHash);
}
return this.hashCache;
}
public void addPosition(int position) {

10
source/net/yacy/kelondro/data/meta/DigestURI.java
Unescape View File

@ -160,12 +160,14 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
this.hash = null;
}
/* (non-Javadoc)
* @see java.lang.Object#hashCode()
*/
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(hash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(hash());
}
return this.hashCache;
}
public static final int flagTypeID(final String hash) {

29
source/net/yacy/kelondro/data/word/WordReferenceRow.java
Unescape View File

@ -215,6 +215,7 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
this.out = new LinkedBlockingQueue<WordReferenceRow>();
for (int i = 0; i < concurrency; i++) {
this.worker[i] = new Thread() {
@Override
public void run() {
String s;
try {
@ -290,22 +291,27 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return new WordReferenceRow(b);
}
@Override
public String toPropertyForm() {
return this.entry.toPropertyForm('=', true, true, false, false);
}
@Override
public Entry toKelondroEntry() {
return this.entry;
}
@Override
public byte[] urlhash() {
return this.entry.getColBytes(col_urlhash, true);
}
@Override
public int virtualAge() {
return (int) this.entry.getColLong(col_lastModified); // this is the time in MicoDateDays format
}
@Override
public long lastModified() {
return MicroDate.reverseMicroDateDays((int) this.entry.getColLong(col_lastModified));
}
@ -314,10 +320,12 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return MicroDate.reverseMicroDateDays((int) this.entry.getColLong(col_freshUntil));
}
@Override
public int hitcount() {
return (0xff & this.entry.getColByte(col_hitcount));
}
@Override
public Collection<Integer> positions() {
return new ArrayList<Integer>(0);
}
@ -327,54 +335,67 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return (int) this.entry.getColLong(col_posintext);
}
@Override
public int posinphrase() {
return (0xff & this.entry.getColByte(col_posinphrase));
}
@Override
public int posofphrase() {
return (0xff & this.entry.getColByte(col_posofphrase));
}
@Override
public int wordsintext() {
return (int) this.entry.getColLong(col_wordsInText);
}
@Override
public int phrasesintext() {
return (int) this.entry.getColLong(col_phrasesInText);
}
@Override
public byte[] getLanguage() {
return this.entry.getColBytes(col_language, true);
}
@Override
public char getType() {
return (char) this.entry.getColByte(col_doctype);
}
@Override
public int wordsintitle() {
return (0xff & this.entry.getColByte(col_wordsInTitle));
}
@Override
public int llocal() {
return (0xff & this.entry.getColByte(col_llocal));
}
@Override
public int lother() {
return (0xff & this.entry.getColByte(col_lother));
}
@Override
public int urllength() {
return (0xff & this.entry.getColByte(col_urlLength));
}
@Override
public int urlcomps() {
return (0xff & this.entry.getColByte(col_urlComps));
}
@Override
public Bitfield flags() {
return new Bitfield(this.entry.getColBytes(col_flags, false));
}
@Override
public double termFrequency() {
return (((double) hitcount()) / ((double) (wordsintext() + wordsintitle() + 1)));
}
@ -393,11 +414,17 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
return Base64Order.enhancedCoder.equal(urlhash(), other.urlhash());
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(urlhash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(urlhash());
}
return this.hashCache;
}
@Override
public void join(final Reference oe) {
throw new UnsupportedOperationException("");

8
source/net/yacy/kelondro/data/word/WordReferenceVars.java
Unescape View File

@ -31,7 +31,6 @@ import java.util.Comparator;
import java.util.Queue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.UTF8;
@ -394,9 +393,14 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
return Base64Order.enhancedCoder.equal(this.urlHash, other.urlHash);
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlHash);
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlHash);
}
return this.hashCache;
}
@Override

19
source/net/yacy/kelondro/util/ByteArray.java
Unescape View File

@ -44,7 +44,7 @@ import net.yacy.cora.order.ByteOrder;
public class ByteArray {
private final byte[] buffer;
private int hash;
private final int hash;
public ByteArray(final byte[] bb) {
@ -53,7 +53,7 @@ public class ByteArray {
}
public int length() {
return buffer.length;
return this.buffer.length;
}
public byte[] asBytes() {
@ -61,7 +61,7 @@ public class ByteArray {
}
public byte readByte(final int pos) {
return buffer[pos];
return this.buffer[pos];
}
public static boolean startsWith(final byte[] buffer, final byte[] pattern) {
@ -83,11 +83,14 @@ public class ByteArray {
return order.compare(this.buffer, aoffset, b.buffer, boffset, blength);
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
if (this.hash != 0) return this.hash;
this.hash = hashCode(this.buffer);
return this.hash;
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.buffer);
}
return this.hashCache;
}
/**
@ -104,8 +107,8 @@ public class ByteArray {
@Override
public boolean equals(Object other) {
ByteArray b = (ByteArray) other;
if (buffer == null && b == null) return true;
if (buffer == null || b == null) return false;
if (this.buffer == null && b == null) return true;
if (this.buffer == null || b == null) return false;
if (this.buffer.length != b.buffer.length) return false;
int l = this.buffer.length;
while (--l >= 0) if (this.buffer[l] != b.buffer[l]) return false;

7
source/net/yacy/search/snippet/MediaSnippet.java
Unescape View File

@ -100,9 +100,14 @@ public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaS
if ((this.attr == null) || (this.attr.length() == 0)) this.attr = "_";
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.href.hash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.href.hash());
}
return this.hashCache;
}
@Override

6
source/net/yacy/search/snippet/ResultEntry.java
Unescape View File

@ -103,9 +103,13 @@ public class ResultEntry implements Comparable<ResultEntry>, Comparator<ResultEn
if ((p = this.alternative_urlname.indexOf('?')) > 0) this.alternative_urlname = this.alternative_urlname.substring(0, p);
}
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlentry.hash());
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlentry.hash());
}
return this.hashCache;
}
@Override
public boolean equals(final Object obj) {

7
source/net/yacy/search/snippet/TextSnippet.java
Unescape View File

@ -400,9 +400,14 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
return o1.compareTo(o2);
}
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
@Override
public int hashCode() {
return ByteArray.hashCode(this.urlhash);
if (this.hashCache == Integer.MIN_VALUE) {
this.hashCache = ByteArray.hashCode(this.urlhash);
}
return this.hashCache;
}
@Override

Write Preview
Loading…
Cancel
Save