boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

performance hack

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 41536eb4a2
commit ef78f22ee1
5 changed files with 44 additions and 37 deletions
Show Stats Download Patch File Download Diff File

42
source/net/yacy/document/SnippetExtractor.java
Unescape View File

@ -7,12 +7,12 @@
* modify it under the terms of the GNU Lesser General Public * modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either * License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version. * version 2.1 of the License, or (at your option) any later version.
* *
* This library is distributed in the hope that it will be useful, * This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of * but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details. * Lesser General Public License for more details.
* *
* You should have received a copy of the GNU Lesser General Public License * You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt * along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>. * If not, see <http://www.gnu.org/licenses/>.
@ -32,10 +32,10 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
public class SnippetExtractor { public class SnippetExtractor {
String snippetString; String snippetString;
HandleSet remainingHashes; HandleSet remainingHashes;
public SnippetExtractor(final Collection<StringBuilder> sentences, final HandleSet queryhashes, int maxLength) throws UnsupportedOperationException { public SnippetExtractor(final Collection<StringBuilder> sentences, final HandleSet queryhashes, int maxLength) throws UnsupportedOperationException {
if (sentences == null) throw new UnsupportedOperationException("sentence == null"); if (sentences == null) throw new UnsupportedOperationException("sentence == null");
if (queryhashes == null || queryhashes.isEmpty()) throw new UnsupportedOperationException("queryhashes == null"); if (queryhashes == null || queryhashes.isEmpty()) throw new UnsupportedOperationException("queryhashes == null");
@ -47,7 +47,7 @@ public class SnippetExtractor {
int linenumber = 0; int linenumber = 0;
int fullmatchcounter = 0; int fullmatchcounter = 0;
lookup: for (final StringBuilder sentence: sentences) { lookup: for (final StringBuilder sentence: sentences) {
hs = WordTokenizer.hashSentence(sentence.toString(), null); hs = WordTokenizer.hashSentence(sentence.toString(), null, 100);
positions = new TreeSet<Integer>(); positions = new TreeSet<Integer>();
for (final byte[] word: queryhashes) { for (final byte[] word: queryhashes) {
pos = hs.get(word); pos = hs.get(word);
@ -69,7 +69,7 @@ public class SnippetExtractor {
} }
linenumber++; linenumber++;
} }
StringBuilder sentence; StringBuilder sentence;
SnippetExtractor tsr; SnippetExtractor tsr;
while (!order.isEmpty()) { while (!order.isEmpty()) {
@ -79,27 +79,27 @@ public class SnippetExtractor {
} catch (UnsupportedOperationException e) { } catch (UnsupportedOperationException e) {
continue; continue;
} }
snippetString = tsr.snippetString; this.snippetString = tsr.snippetString;
if (snippetString != null && snippetString.length() > 0) { if (this.snippetString != null && this.snippetString.length() > 0) {
remainingHashes = tsr.remainingHashes; this.remainingHashes = tsr.remainingHashes;
if (remainingHashes.isEmpty()) { if (this.remainingHashes.isEmpty()) {
// we have found the snippet // we have found the snippet
return; // finished! return; // finished!
} else if (remainingHashes.size() < queryhashes.size()) { } else if (this.remainingHashes.size() < queryhashes.size()) {
// the result has not all words in it. // the result has not all words in it.
// find another sentence that represents the missing other words // find another sentence that represents the missing other words
// and find recursively more sentences // and find recursively more sentences
maxLength = maxLength - snippetString.length(); maxLength = maxLength - this.snippetString.length();
if (maxLength < 20) maxLength = 20; if (maxLength < 20) maxLength = 20;
try { try {
tsr = new SnippetExtractor(order.values(), remainingHashes, maxLength); tsr = new SnippetExtractor(order.values(), this.remainingHashes, maxLength);
} catch (UnsupportedOperationException e) { } catch (UnsupportedOperationException e) {
throw e; throw e;
} }
final String nextSnippet = tsr.snippetString; final String nextSnippet = tsr.snippetString;
if (nextSnippet == null) return; if (nextSnippet == null) return;
snippetString = snippetString + (" / " + nextSnippet); this.snippetString = this.snippetString + (" / " + nextSnippet);
remainingHashes = tsr.remainingHashes; this.remainingHashes = tsr.remainingHashes;
return; return;
} else { } else {
// error // error
@ -110,7 +110,7 @@ public class SnippetExtractor {
} }
throw new UnsupportedOperationException("no snippet computed"); throw new UnsupportedOperationException("no snippet computed");
} }
private static int linelengthKey(int givenlength, int maxlength) { private static int linelengthKey(int givenlength, int maxlength) {
if (givenlength > maxlength) return 1; if (givenlength > maxlength) return 1;
if (givenlength >= maxlength / 2 && givenlength < maxlength) return 7; if (givenlength >= maxlength / 2 && givenlength < maxlength) return 7;
@ -118,15 +118,15 @@ public class SnippetExtractor {
if (givenlength >= maxlength / 8 && givenlength < maxlength / 4) return 3; if (givenlength >= maxlength / 8 && givenlength < maxlength / 4) return 3;
return 0; return 0;
} }
private SnippetExtractor(String sentence, final HandleSet queryhashes, final int maxLength) throws UnsupportedOperationException { private SnippetExtractor(String sentence, final HandleSet queryhashes, final int maxLength) throws UnsupportedOperationException {
try { try {
if (sentence == null) throw new UnsupportedOperationException("no sentence given"); if (sentence == null) throw new UnsupportedOperationException("no sentence given");
if (queryhashes == null || queryhashes.isEmpty()) throw new UnsupportedOperationException("queryhashes == null"); if (queryhashes == null || queryhashes.isEmpty()) throw new UnsupportedOperationException("queryhashes == null");
byte[] hash; byte[] hash;
// find all hashes that appear in the sentence // find all hashes that appear in the sentence
final Map<byte[], Integer> hs = WordTokenizer.hashSentence(sentence, null); final Map<byte[], Integer> hs = WordTokenizer.hashSentence(sentence, null, 100);
final Iterator<byte[]> j = queryhashes.iterator(); final Iterator<byte[]> j = queryhashes.iterator();
Integer pos; Integer pos;
int p, minpos = sentence.length(), maxpos = -1; int p, minpos = sentence.length(), maxpos = -1;
@ -189,11 +189,11 @@ public class SnippetExtractor {
throw new UnsupportedOperationException(e.getMessage()); throw new UnsupportedOperationException(e.getMessage());
} }
} }
public String getSnippet() { public String getSnippet() {
return this.snippetString; return this.snippetString;
} }
public HandleSet getRemainingWords() { public HandleSet getRemainingWords() {
return this.remainingHashes; return this.remainingHashes;
} }

18
source/net/yacy/document/WordTokenizer.java
Unescape View File

@ -68,10 +68,12 @@ public class WordTokenizer implements Enumeration<StringBuilder> {
return null; return null;
} }
@Override
public boolean hasMoreElements() { public boolean hasMoreElements() {
return this.buffer != null; return this.buffer != null;
} }
@Override
public StringBuilder nextElement() { public StringBuilder nextElement() {
final StringBuilder r = (this.buffer == null) ? null : this.buffer; final StringBuilder r = (this.buffer == null) ? null : this.buffer;
this.buffer = nextElement0(); this.buffer = nextElement0();
@ -79,9 +81,9 @@ public class WordTokenizer implements Enumeration<StringBuilder> {
if (this.meaningLib != null) WordCache.learn(r); if (this.meaningLib != null) WordCache.learn(r);
return r; return r;
} }
public void close() { public void close() {
e.close(); this.e.close();
} }
private static class unsievedWordsEnum implements Enumeration<StringBuilder> { private static class unsievedWordsEnum implements Enumeration<StringBuilder> {
@ -139,10 +141,12 @@ public class WordTokenizer implements Enumeration<StringBuilder> {
return r; return r;
} }
@Override
public boolean hasMoreElements() { public boolean hasMoreElements() {
return this.buffer != null; return this.buffer != null;
} }
@Override
public StringBuilder nextElement() { public StringBuilder nextElement() {
final StringBuilder r = this.buffer; final StringBuilder r = this.buffer;
this.buffer = nextElement0(); this.buffer = nextElement0();
@ -150,7 +154,7 @@ public class WordTokenizer implements Enumeration<StringBuilder> {
} }
public void close() { public void close() {
e.close(); this.e.close();
} }
} }
@ -177,7 +181,7 @@ public class WordTokenizer implements Enumeration<StringBuilder> {
* @param sentence the sentence to be tokenized * @param sentence the sentence to be tokenized
* @return a ordered map containing word hashes as key and positions as value. The map is orderd by the hash ordering * @return a ordered map containing word hashes as key and positions as value. The map is orderd by the hash ordering
*/ */
public static SortedMap<byte[], Integer> hashSentence(final String sentence, final WordCache meaningLib) { public static SortedMap<byte[], Integer> hashSentence(final String sentence, final WordCache meaningLib, int maxlength) {
final SortedMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Base64Order.enhancedCoder); final SortedMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Base64Order.enhancedCoder);
final WordTokenizer words = new WordTokenizer(new ByteArrayInputStream(UTF8.getBytes(sentence)), meaningLib); final WordTokenizer words = new WordTokenizer(new ByteArrayInputStream(UTF8.getBytes(sentence)), meaningLib);
try { try {
@ -185,16 +189,16 @@ public class WordTokenizer implements Enumeration<StringBuilder> {
StringBuilder word; StringBuilder word;
byte[] hash; byte[] hash;
Integer oldpos; Integer oldpos;
while (words.hasMoreElements()) { while (words.hasMoreElements() && maxlength-- > 0) {
word = words.nextElement(); word = words.nextElement();
hash = Word.word2hash(word); hash = Word.word2hash(word);
// don't overwrite old values, that leads to too far word distances // don't overwrite old values, that leads to too far word distances
oldpos = map.put(hash, LargeNumberCache.valueOf(pos)); oldpos = map.put(hash, LargeNumberCache.valueOf(pos));
if (oldpos != null) { if (oldpos != null) {
map.put(hash, oldpos); map.put(hash, oldpos);
} }
pos += word.length() + 1; pos += word.length() + 1;
} }
return map; return map;

10
source/net/yacy/kelondro/order/Digest.java
Unescape View File

@ -65,7 +65,7 @@ public class Digest {
md5Cache = new ConcurrentARC<String, byte[]>(1000, Math.max(8, 2 * Runtime.getRuntime().availableProcessors())); md5Cache = new ConcurrentARC<String, byte[]>(1000, Math.max(8, 2 * Runtime.getRuntime().availableProcessors()));
} }
} }
public static String encodeHex(final long in, final int length) { public static String encodeHex(final long in, final int length) {
String s = Long.toHexString(in); String s = Long.toHexString(in);
while (s.length() < length) s = "0" + s; while (s.length() < length) s = "0" + s;
@ -119,7 +119,7 @@ public class Digest {
byte[] h = md5Cache.get(key); byte[] h = md5Cache.get(key);
if (h != null) return h; if (h != null) return h;
MessageDigest digest = digestPool.poll(); MessageDigest digest = digestPool.poll();
if (digest == null) { if (digest == null) {
// if there are no digest objects left, create some on the fly // if there are no digest objects left, create some on the fly
@ -129,12 +129,14 @@ public class Digest {
digest.reset(); digest.reset();
} catch (final NoSuchAlgorithmException e) { } catch (final NoSuchAlgorithmException e) {
} }
} else {
digest.reset(); // they should all be reseted but anyway; this is safe
} }
byte[] keyBytes; byte[] keyBytes;
keyBytes = UTF8.getBytes(key); keyBytes = UTF8.getBytes(key);
digest.update(keyBytes); digest.update(keyBytes);
final byte[] result = digest.digest(); final byte[] result = digest.digest();
digest.reset(); digest.reset(); // to be prepared for next
try { try {
digestPool.put(digest); digestPool.put(digest);
//System.out.println("Digest Pool size = " + digestPool.size()); //System.out.println("Digest Pool size = " + digestPool.size());
@ -390,7 +392,7 @@ public class Digest {
} }
System.out.println("time: " + (System.currentTimeMillis() - start) + " ms"); System.out.println("time: " + (System.currentTimeMillis() - start) + " ms");
// without this this method would never end // without this this method would never end
Log.shutdown(); Log.shutdown();
} }

8
source/net/yacy/search/snippet/MediaSnippet.java
Unescape View File

@ -27,7 +27,7 @@ package net.yacy.search.snippet;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -51,8 +51,8 @@ import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.util.ByteArray; import net.yacy.kelondro.util.ByteArray;
import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import de.anomic.crawler.retrieval.Request;
import de.anomic.crawler.ZURL.FailCategory; import de.anomic.crawler.ZURL.FailCategory;
import de.anomic.crawler.retrieval.Request;
public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaSnippet> { public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaSnippet> {
@ -117,10 +117,12 @@ public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaS
return Base64Order.enhancedCoder.equal(this.href.hash(), other.href.hash()); return Base64Order.enhancedCoder.equal(this.href.hash(), other.href.hash());
} }
@Override
public int compareTo(final MediaSnippet o) { public int compareTo(final MediaSnippet o) {
return Base64Order.enhancedCoder.compare(this.href.hash(), o.href.hash()); return Base64Order.enhancedCoder.compare(this.href.hash(), o.href.hash());
} }
@Override
public int compare(final MediaSnippet o1, final MediaSnippet o2) { public int compare(final MediaSnippet o1, final MediaSnippet o2) {
return o1.compareTo(o2); return o1.compareTo(o2);
} }
@ -217,7 +219,7 @@ public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaS
private static HandleSet removeAppearanceHashes(final String sentence, final HandleSet queryhashes) { private static HandleSet removeAppearanceHashes(final String sentence, final HandleSet queryhashes) {
// remove all hashes that appear in the sentence // remove all hashes that appear in the sentence
if (sentence == null) return queryhashes; if (sentence == null) return queryhashes;
final SortedMap<byte[], Integer> hs = WordTokenizer.hashSentence(sentence, null); final SortedMap<byte[], Integer> hs = WordTokenizer.hashSentence(sentence, null, 100);
final Iterator<byte[]> j = queryhashes.iterator(); final Iterator<byte[]> j = queryhashes.iterator();
byte[] hash; byte[] hash;
Integer pos; Integer pos;

3
source/net/yacy/search/snippet/TextSnippet.java
Unescape View File

@ -497,8 +497,7 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
private static boolean containsAllHashes( private static boolean containsAllHashes(
final String sentence, final HandleSet queryhashes) { final String sentence, final HandleSet queryhashes) {
final SortedMap<byte[], Integer> m = final SortedMap<byte[], Integer> m = WordTokenizer.hashSentence(sentence, null, 100);
WordTokenizer.hashSentence(sentence, null);
for (final byte[] b : queryhashes) { for (final byte[] b : queryhashes) {
if (!(m.containsKey(b))) { if (!(m.containsKey(b))) {
return false; return false;

Write Preview
Loading…
Cancel
Save