by instatiation from URIMetadataNode, by eleminating differentiation of ResultEntry/URIMetadataNode. - moved remaining ResultEntry functionallity to URIMetadataNode - for 1:1 functionallity added a function makeResultEntry() - removed ResultEntry - refactored related code Main difference is after makeResultEntry the text_t content is removed and alternative title/url strings for display are calculated. Main difference left is, thatpull/8/head
reger
10 years ago
parent
29c4aa3991
commit
000dde9511
@ -1,117 +0,0 @@
|
||||
// BinSearch.java
|
||||
// -----------------------
|
||||
// part of The Kelondro Database
|
||||
// (C) by Michael Peter Christen; mc@yacy.net
|
||||
// first published on http://www.anomic.de
|
||||
// Frankfurt, Germany, 2005
|
||||
// created 22.11.2005
|
||||
//
|
||||
// $LastChangedDate$
|
||||
// $LastChangedRevision$
|
||||
// $LastChangedBy$
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.kelondro.index;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import net.yacy.cora.order.ByteOrder;
|
||||
import net.yacy.cora.order.NaturalOrder;
|
||||
|
||||
|
||||
public final class BinSearch {
|
||||
|
||||
private final byte[] chunks;
|
||||
private final int chunksize;
|
||||
private final int count;
|
||||
private static final ByteOrder objectOrder = new NaturalOrder(true); // the natural order is much faster than the b64Order
|
||||
|
||||
public BinSearch(final byte[] chunks, final int chunksize) {
|
||||
this.chunks = chunks;
|
||||
this.chunksize = chunksize;
|
||||
this.count = chunks.length / chunksize;
|
||||
}
|
||||
|
||||
public BinSearch(final List<byte[]> chunkList, final int chunksize) {
|
||||
byte[][] chunksa = new byte[chunkList.size()][];
|
||||
chunksa = chunkList.toArray(chunksa);
|
||||
Arrays.sort(chunksa, objectOrder);
|
||||
this.chunks = new byte[chunkList.size() * chunksize];
|
||||
for (int i = 0; i < chunksa.length; i++) System.arraycopy(chunksa[i], 0, this.chunks, i * chunksize, chunksize);
|
||||
this.chunksize = chunksize;
|
||||
this.count = chunks.length / chunksize;
|
||||
assert this.count == chunkList.size();
|
||||
}
|
||||
|
||||
public final boolean contains(final byte[] t) {
|
||||
return contains(t, 0, this.count);
|
||||
}
|
||||
|
||||
private final boolean contains(final byte[] t, int beginPos, int endPos) {
|
||||
// the endPos is exclusive, beginPos is inclusive
|
||||
// this method is synchronized to make the use of the buffer possible
|
||||
assert t.length == this.chunksize;
|
||||
while (true) {
|
||||
if (beginPos >= endPos) return false;
|
||||
final int pivot = (beginPos + endPos) / 2;
|
||||
if ((pivot < 0) || (pivot >= this.count)) return false;
|
||||
assert this.chunksize == t.length;
|
||||
final int c = objectOrder.compare(this.chunks, pivot * this.chunksize, t, 0, this.chunksize);
|
||||
if (c == 0) return true;
|
||||
if (c < 0) /* buffer < t */ {beginPos = pivot + 1; continue;}
|
||||
if (c > 0) /* buffer > t */ {endPos = pivot; continue;}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public final int size() {
|
||||
return count;
|
||||
}
|
||||
|
||||
public final byte[] get(final int element) {
|
||||
final byte[] a = new byte[chunksize];
|
||||
System.arraycopy(this.chunks, element * this.chunksize, a, 0, chunksize);
|
||||
return a;
|
||||
}
|
||||
|
||||
public final byte[] get(final int element, byte[] a) {
|
||||
assert a.length == chunksize;
|
||||
System.arraycopy(this.chunks, element * this.chunksize, a, 0, chunksize);
|
||||
return a;
|
||||
}
|
||||
|
||||
public final void write(File f) throws IOException {
|
||||
FileOutputStream os = new FileOutputStream(f);
|
||||
os.write(this.chunks);
|
||||
os.flush();
|
||||
os.close();
|
||||
}
|
||||
|
||||
public static void main(final String[] args) {
|
||||
final String s = "4CEvsI8FRczRBo_ApRCkwfEbFLn1pIFXg39QGMgj5RHM6HpIMJq67QX3M5iQYr_LyI_5aGDaa_bYbRgJ9XnQjpmq6QkOoGWAoEaihRqhV3kItLFHjRtqauUR";
|
||||
final BinSearch bs = new BinSearch(s.getBytes(), 6);
|
||||
for (int i = 0; i + 6 <= s.length(); i = i + 6) {
|
||||
System.out.println(s.substring(i, i + 6) + ":" + ((bs.contains(s.substring(i, i + 6).getBytes())) ? "drin" : "draussen"));
|
||||
}
|
||||
for (int i = 0; i + 7 <= s.length(); i = i + 6) {
|
||||
System.out.println(s.substring(i + 1, i + 7) + ":" + ((bs.contains(s.substring(i + 1, i + 7).getBytes())) ? "drin" : "draussen"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,166 +0,0 @@
|
||||
// ResultEntry.java
|
||||
// (C) 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
||||
// first published 10.10.2005 on http://yacy.net
|
||||
//
|
||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||
//
|
||||
// $LastChangedDate$
|
||||
// $LastChangedRevision$
|
||||
// $LastChangedBy$
|
||||
//
|
||||
// LICENSE
|
||||
//
|
||||
// This program is free software; you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation; either version 2 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program; if not, write to the Free Software
|
||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
package net.yacy.search.snippet;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
|
||||
import net.yacy.cora.document.id.MultiProtocolURL;
|
||||
import net.yacy.cora.order.Base64Order;
|
||||
import net.yacy.cora.util.ByteArray;
|
||||
import net.yacy.cora.util.ConcurrentLog;
|
||||
import net.yacy.document.Condenser;
|
||||
import net.yacy.document.parser.pdfParser;
|
||||
import net.yacy.kelondro.data.meta.URIMetadataNode;
|
||||
import net.yacy.kelondro.data.word.Word;
|
||||
import net.yacy.peers.Seed;
|
||||
import net.yacy.peers.SeedDB;
|
||||
import net.yacy.search.index.Segment;
|
||||
import net.yacy.search.schema.CollectionSchema;
|
||||
|
||||
|
||||
public class ResultEntry extends URIMetadataNode implements Comparable<ResultEntry>, Comparator<ResultEntry> {
|
||||
|
||||
private static final long serialVersionUID = -256046934741561978L;
|
||||
// payload objects
|
||||
private String alternative_urlstring;
|
||||
private String alternative_urlname;
|
||||
private final TextSnippet textSnippet;
|
||||
private final Segment indexSegment;
|
||||
|
||||
public ResultEntry(final URIMetadataNode urlentry,
|
||||
final Segment indexSegment,
|
||||
SeedDB peers,
|
||||
final TextSnippet textSnippet) {
|
||||
super(urlentry);
|
||||
this.removeFields(CollectionSchema.text_t.getSolrFieldName()); // clear the text field which eats up most of the space; it was used for snippet computation which is in a separate field here
|
||||
this.indexSegment = indexSegment;
|
||||
this.alternative_urlstring = null;
|
||||
this.alternative_urlname = null;
|
||||
this.textSnippet = textSnippet;
|
||||
final String host = urlentry.url().getHost();
|
||||
if (host != null && host.endsWith(".yacyh")) {
|
||||
// translate host into current IP
|
||||
int p = host.indexOf('.');
|
||||
final String hash = Seed.hexHash2b64Hash(host.substring(p + 1, host.length() - 6));
|
||||
final Seed seed = peers.getConnected(hash);
|
||||
final String path = urlentry.url().getFile();
|
||||
String address = null;
|
||||
if ((seed == null) || ((address = seed.getPublicAddress(seed.getIP())) == null)) {
|
||||
// seed is not known from here
|
||||
try {
|
||||
if (indexSegment.termIndex() != null) indexSegment.termIndex().remove(
|
||||
Word.words2hashesHandles(Condenser.getWords(
|
||||
("yacyshare " +
|
||||
path.replace('?', ' ') +
|
||||
" " +
|
||||
urlentry.dc_title()), null).keySet()),
|
||||
urlentry.hash());
|
||||
} catch (final IOException e) {
|
||||
ConcurrentLog.logException(e);
|
||||
}
|
||||
indexSegment.fulltext().remove(urlentry.hash()); // clean up
|
||||
throw new RuntimeException("index void");
|
||||
}
|
||||
this.alternative_urlstring = "http://" + address + "/" + host.substring(0, p) + path;
|
||||
this.alternative_urlname = "http://share." + seed.getName() + ".yacy" + path;
|
||||
if ((p = this.alternative_urlname.indexOf('?')) > 0) this.alternative_urlname = this.alternative_urlname.substring(0, p);
|
||||
}
|
||||
}
|
||||
private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful
|
||||
@Override
|
||||
public int hashCode() {
|
||||
if (this.hashCache == Integer.MIN_VALUE) {
|
||||
this.hashCache = ByteArray.hashCode(this.hash());
|
||||
}
|
||||
return this.hashCache;
|
||||
}
|
||||
@Override
|
||||
public boolean equals(final Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (obj == null) return false;
|
||||
if (!(obj instanceof ResultEntry)) return false;
|
||||
ResultEntry other = (ResultEntry) obj;
|
||||
return Base64Order.enhancedCoder.equal(this.hash(), other.hash());
|
||||
}
|
||||
|
||||
public String urlstring() {
|
||||
if (this.alternative_urlstring != null) return this.alternative_urlstring;
|
||||
|
||||
if (!pdfParser.individualPages) return this.url().toNormalform(true);
|
||||
if (!"pdf".equals(MultiProtocolURL.getFileExtension(this.url().getFileName()).toLowerCase())) return this.url().toNormalform(true);
|
||||
// for pdf links we rewrite the url
|
||||
// this is a special treatment of pdf files which can be splitted into subpages
|
||||
String pageprop = pdfParser.individualPagePropertyname;
|
||||
String resultUrlstring = this.url().toNormalform(true);
|
||||
int p = resultUrlstring.lastIndexOf(pageprop + "=");
|
||||
if (p > 0) {
|
||||
return resultUrlstring.substring(0, p - 1) + "#page=" + resultUrlstring.substring(p + pageprop.length() + 1);
|
||||
}
|
||||
return resultUrlstring;
|
||||
}
|
||||
public String urlname() {
|
||||
return (this.alternative_urlname == null) ? MultiProtocolURL.unescape(urlstring()) : this.alternative_urlname;
|
||||
}
|
||||
public String title() {
|
||||
String titlestr = this.dc_title();
|
||||
// if title is empty use filename as title
|
||||
if (titlestr.isEmpty()) { // if url has no filename, title is still empty (e.g. "www.host.com/" )
|
||||
titlestr = this.url() != null ? this.url().getFileName() : "";
|
||||
}
|
||||
return titlestr;
|
||||
}
|
||||
public TextSnippet textSnippet() {
|
||||
return this.textSnippet;
|
||||
}
|
||||
public Date[] events() {
|
||||
return this.datesInContent();
|
||||
}
|
||||
public int referencesCount() {
|
||||
// urlCitationIndex index might be null (= configuration option)
|
||||
return this.indexSegment.connectedCitation() ? this.indexSegment.urlCitation().count(this.hash()) : 0;
|
||||
}
|
||||
public boolean hasTextSnippet() {
|
||||
return (this.textSnippet != null) && (!this.textSnippet.getErrorCode().fail());
|
||||
}
|
||||
public String resource() {
|
||||
// generate transport resource
|
||||
if ((this.textSnippet == null) || (!this.textSnippet.exists())) {
|
||||
return this.toString();
|
||||
}
|
||||
return this.toString(this.textSnippet.getLineRaw());
|
||||
}
|
||||
@Override
|
||||
public int compareTo(ResultEntry o) {
|
||||
return Base64Order.enhancedCoder.compare(this.hash(), o.hash());
|
||||
}
|
||||
@Override
|
||||
public int compare(ResultEntry o1, ResultEntry o2) {
|
||||
return Base64Order.enhancedCoder.compare(o1.hash(), o2.hash());
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue