- moved all url and index(RWI) entries to index package - better naming to distinguish RWI entries and URL entries git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@2937 6c8d7289-2bf4-0310-a012-ef5d649a1542pull/1/head
orbiter
19 years ago
parent
6412c926bc
commit
bb7d4b5d5e
@ -0,0 +1,323 @@
|
|||||||
|
// indexURLEntryNew.java
|
||||||
|
// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
|
||||||
|
// first published 21.07.2006 on http://www.anomic.de
|
||||||
|
//
|
||||||
|
// This is a part of YaCy, a peer-to-peer based web search engine
|
||||||
|
//
|
||||||
|
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
|
||||||
|
// $LastChangedRevision: 1986 $
|
||||||
|
// $LastChangedBy: orbiter $
|
||||||
|
//
|
||||||
|
// LICENSE
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program; if not, write to the Free Software
|
||||||
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
|
||||||
|
package de.anomic.index;
|
||||||
|
|
||||||
|
import de.anomic.kelondro.kelondroColumn;
|
||||||
|
import de.anomic.kelondro.kelondroRow;
|
||||||
|
import de.anomic.kelondro.kelondroRow.Entry;
|
||||||
|
import de.anomic.plasma.plasmaWordIndex;
|
||||||
|
import de.anomic.yacy.yacySeedDB;
|
||||||
|
|
||||||
|
public class indexRWIEntryOld implements Cloneable, indexRWIEntry {
|
||||||
|
|
||||||
|
// this object stores attributes to URL references inside RWI collections
|
||||||
|
|
||||||
|
// statics for value lengths
|
||||||
|
public static final int urlStringLength = 256;// not too short for links without parameters
|
||||||
|
public static final int urlDescrLength = 80; // The headline of a web page (meta-tag or <h1>)
|
||||||
|
public static final int urlNameLength = 40; // the tag content between <a> and </a>
|
||||||
|
public static final int urldescrtagsLength = 320;// the url, the description and tags in one string
|
||||||
|
public static final int urlErrorLength = 80; // a reason description for unavailable urls
|
||||||
|
public static final int urlDateLength = 4; // any date, shortened
|
||||||
|
public static final int urlCopyCountLength = 2; // counter for numbers of copies of this index
|
||||||
|
public static final int urlFlagLength = 2; // any stuff
|
||||||
|
public static final int urlLanguageLength = 2; // taken from TLD suffix as quick-hack
|
||||||
|
public static final int urlDoctypeLength = 1; // taken from extension
|
||||||
|
public static final int urlSizeLength = 6; // the source size, from cache
|
||||||
|
public static final int urlWordCountLength = 3; // the number of words, from condenser
|
||||||
|
public static final int urlCrawlProfileHandleLength = 4; // name of the prefetch profile
|
||||||
|
public static final int urlCrawlDepthLength = 2; // prefetch depth, first is '0'
|
||||||
|
public static final int urlParentBranchesLength = 3; // number of anchors of the parent
|
||||||
|
public static final int urlForkFactorLength = 4; // sum of anchors of all ancestors
|
||||||
|
public static final int urlRetryLength = 2; // number of load retries
|
||||||
|
public static final int urlHostLength = 8; // the host as struncated name
|
||||||
|
public static final int urlHandleLength = 4; // a handle
|
||||||
|
public static final int urlQualityLength = 3; // taken from heuristic
|
||||||
|
|
||||||
|
public static kelondroRow urlEntryRow = new kelondroRow(new kelondroColumn[]{
|
||||||
|
new kelondroColumn("h", kelondroColumn.celltype_string, kelondroColumn.encoder_bytes, yacySeedDB.commonHashLength, "urlhash"),
|
||||||
|
new kelondroColumn("q", kelondroColumn.celltype_cardinal, kelondroColumn.encoder_b64e, urlQualityLength, "quality"),
|
||||||
|
new kelondroColumn("a", kelondroColumn.celltype_cardinal, kelondroColumn.encoder_b64e, 3, "lastModified"),
|
||||||
|
new kelondroColumn("c", kelondroColumn.celltype_cardinal, kelondroColumn.encoder_b64e, 2, "hitcount"),
|
||||||
|
new kelondroColumn("l", kelondroColumn.celltype_string, kelondroColumn.encoder_bytes, urlLanguageLength, "language"),
|
||||||
|
new kelondroColumn("d", kelondroColumn.celltype_binary, kelondroColumn.encoder_bytes, 1, "doctype"),
|
||||||
|
new kelondroColumn("f", kelondroColumn.celltype_binary, kelondroColumn.encoder_bytes, 1, "localflag"),
|
||||||
|
new kelondroColumn("t", kelondroColumn.celltype_cardinal, kelondroColumn.encoder_b64e, 2, "posintext"),
|
||||||
|
new kelondroColumn("r", kelondroColumn.celltype_cardinal, kelondroColumn.encoder_b64e, 2, "posinphrase"),
|
||||||
|
new kelondroColumn("o", kelondroColumn.celltype_cardinal, kelondroColumn.encoder_b64e, 2, "posofphrase"),
|
||||||
|
new kelondroColumn("i", kelondroColumn.celltype_cardinal, kelondroColumn.encoder_b64e, 2, "worddistance"),
|
||||||
|
new kelondroColumn("w", kelondroColumn.celltype_cardinal, kelondroColumn.encoder_b64e, 2, "wordcount"),
|
||||||
|
new kelondroColumn("p", kelondroColumn.celltype_cardinal, kelondroColumn.encoder_b64e, 2, "phrasecount")
|
||||||
|
});
|
||||||
|
|
||||||
|
private static final int col_urlhash = 0;
|
||||||
|
private static final int col_quality = 1;
|
||||||
|
private static final int col_lastModified = 2;
|
||||||
|
private static final int col_hitcount = 3;
|
||||||
|
private static final int col_language = 4;
|
||||||
|
private static final int col_doctype = 5;
|
||||||
|
private static final int col_localflag = 6;
|
||||||
|
private static final int col_posintext = 7;
|
||||||
|
private static final int col_posinphrase = 8;
|
||||||
|
private static final int col_posofphrase = 9;
|
||||||
|
private static final int col_worddistance = 10;
|
||||||
|
private static final int col_wordcount = 11;
|
||||||
|
private static final int col_phrasecount = 12;
|
||||||
|
|
||||||
|
|
||||||
|
private kelondroRow.Entry entry;
|
||||||
|
|
||||||
|
public indexRWIEntryOld(String urlHash,
|
||||||
|
int urlLength, // byte-length of complete URL
|
||||||
|
int urlComps, // number of path components
|
||||||
|
int titleLength, // length of description/length (longer are better?)
|
||||||
|
int hitcount, //*how often appears this word in the text
|
||||||
|
int wordcount, //*total number of words
|
||||||
|
int phrasecount, //*total number of phrases
|
||||||
|
int posintext, //*position of word in all words
|
||||||
|
int posinphrase, //*position of word in its phrase
|
||||||
|
int posofphrase, //*number of the phrase where word appears
|
||||||
|
int worddistance, //*word distance; this is 0 by default, and set to the difference of posintext from two indexes if these are combined (simultanous search). If stored, this shows that the result was obtained by remote search
|
||||||
|
int sizeOfPage, // # of bytes of the page
|
||||||
|
long lastmodified, //*last-modified time of the document where word appears
|
||||||
|
long updatetime, // update time; this is needed to compute a TTL for the word, so it can be removed easily if the TTL is short
|
||||||
|
int quality, //*the entropy value
|
||||||
|
String language, //*(guessed) language of document
|
||||||
|
char doctype, //*type of document
|
||||||
|
int outlinksSame, // outlinks to same domain
|
||||||
|
int outlinksOther,// outlinks to other domain
|
||||||
|
boolean local //*flag shows that this index was generated locally; othervise its from a remote peer
|
||||||
|
) {
|
||||||
|
|
||||||
|
// more needed attributes:
|
||||||
|
// - boolean: appearance attributes: title, appears in header, anchor-descr, image-tag, hervorhebungen, meta-tags, word in link etc
|
||||||
|
// - boolean: URL attributes
|
||||||
|
assert (urlHash.length() == 12) : "urlhash = " + urlHash;
|
||||||
|
if ((language == null) || (language.length() != urlLanguageLength)) language = "uk";
|
||||||
|
this.entry = urlEntryRow.newEntry();
|
||||||
|
this.entry.setCol(col_urlhash, urlHash, null);
|
||||||
|
this.entry.setCol(col_quality, quality);
|
||||||
|
this.entry.setCol(col_lastModified, lastmodified);
|
||||||
|
this.entry.setCol(col_hitcount, hitcount);
|
||||||
|
this.entry.setCol(col_language, language, null);
|
||||||
|
this.entry.setCol(col_doctype, (byte) doctype);
|
||||||
|
this.entry.setCol(col_localflag, (byte) ((local) ? indexEntryAttribute.LT_LOCAL : indexEntryAttribute.LT_GLOBAL));
|
||||||
|
this.entry.setCol(col_posintext, posintext);
|
||||||
|
this.entry.setCol(col_posinphrase, posinphrase);
|
||||||
|
this.entry.setCol(col_posofphrase, posofphrase);
|
||||||
|
this.entry.setCol(col_worddistance, worddistance);
|
||||||
|
this.entry.setCol(col_wordcount, wordcount);
|
||||||
|
this.entry.setCol(col_phrasecount, phrasecount);
|
||||||
|
//System.out.println("DEBUG-NEWENTRY " + toPropertyForm());
|
||||||
|
}
|
||||||
|
|
||||||
|
public indexRWIEntryOld(String urlHash, String code) {
|
||||||
|
// the code is the external form of the row minus the leading urlHash entry
|
||||||
|
this.entry = urlEntryRow.newEntry((urlHash + code).getBytes());
|
||||||
|
}
|
||||||
|
|
||||||
|
public indexRWIEntryOld(String external) {
|
||||||
|
this.entry = urlEntryRow.newEntry(external);
|
||||||
|
}
|
||||||
|
|
||||||
|
public indexRWIEntryOld(byte[] row) {
|
||||||
|
this.entry = urlEntryRow.newEntry(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
public indexRWIEntryOld(kelondroRow.Entry rentry) {
|
||||||
|
// FIXME: see if cloning is necessary
|
||||||
|
this.entry = rentry;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Object clone() {
|
||||||
|
byte[] b = new byte[urlEntryRow.objectsize()];
|
||||||
|
System.arraycopy(entry.bytes(), 0, b, 0, urlEntryRow.objectsize());
|
||||||
|
return new indexRWIEntryOld(b);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String toPropertyForm(boolean displayFormat) {
|
||||||
|
return entry.toPropertyForm(true, displayFormat, displayFormat);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Entry toKelondroEntry() {
|
||||||
|
return this.entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String urlHash() {
|
||||||
|
return this.entry.getColString(col_urlhash, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int quality() {
|
||||||
|
return (int) this.entry.getColLong(col_quality);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int virtualAge() {
|
||||||
|
return plasmaWordIndex.microDateDays(lastModified());
|
||||||
|
}
|
||||||
|
|
||||||
|
public long lastModified() {
|
||||||
|
return (int) this.entry.getColLong(col_lastModified);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int hitcount() {
|
||||||
|
return (int) this.entry.getColLong(col_hitcount);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int posintext() {
|
||||||
|
return (int) this.entry.getColLong(col_posintext);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int posinphrase() {
|
||||||
|
return (int) this.entry.getColLong(col_posinphrase);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int posofphrase() {
|
||||||
|
return (int) this.entry.getColLong(col_posofphrase);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int wordcount() {
|
||||||
|
return (int) this.entry.getColLong(col_wordcount);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int phrasecount() {
|
||||||
|
return (int) this.entry.getColLong(col_phrasecount);
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getLanguage() {
|
||||||
|
return this.entry.getColString(col_language, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public char getType() {
|
||||||
|
return (char) this.entry.getColByte(col_doctype);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isLocal() {
|
||||||
|
return this.entry.getColByte(col_localflag) == indexEntryAttribute.LT_LOCAL;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static indexRWIEntryOld combineDistance(indexRWIEntryOld ie1, indexRWIEntry ie2) {
|
||||||
|
// returns a modified entry of the first argument
|
||||||
|
ie1.entry.setCol(col_worddistance, ie1.worddistance() + ie2.worddistance() + Math.abs(ie1.posintext() - ie2.posintext()));
|
||||||
|
ie1.entry.setCol(col_posintext, Math.min(ie1.posintext(), ie2.posintext()));
|
||||||
|
ie1.entry.setCol(col_posinphrase, (ie1.posofphrase() == ie2.posofphrase()) ? ie1.posofphrase() : 0 /*unknown*/);
|
||||||
|
ie1.entry.setCol(col_posofphrase, Math.min(ie1.posofphrase(), ie2.posofphrase()));
|
||||||
|
ie1.entry.setCol(col_wordcount, (ie1.wordcount() + ie2.wordcount()) / 2);
|
||||||
|
return ie1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void combineDistance(indexRWIEntry oe) {
|
||||||
|
combineDistance(this, oe);
|
||||||
|
}
|
||||||
|
|
||||||
|
public int worddistance() {
|
||||||
|
return (int) this.entry.getColLong(col_worddistance);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static final void min(indexRWIEntryOld t, indexRWIEntry other) {
|
||||||
|
if (t.hitcount() > other.hitcount()) t.entry.setCol(col_hitcount, other.hitcount());
|
||||||
|
if (t.wordcount() > other.wordcount()) t.entry.setCol(col_wordcount, other.wordcount());
|
||||||
|
if (t.phrasecount() > other.phrasecount()) t.entry.setCol(col_phrasecount, other.phrasecount());
|
||||||
|
if (t.posintext() > other.posintext()) t.entry.setCol(col_posintext, other.posintext());
|
||||||
|
if (t.posinphrase() > other.posinphrase()) t.entry.setCol(col_posinphrase, other.posinphrase());
|
||||||
|
if (t.posofphrase() > other.posofphrase()) t.entry.setCol(col_posofphrase, other.posofphrase());
|
||||||
|
if (t.worddistance() > other.worddistance()) t.entry.setCol(col_worddistance, other.worddistance());
|
||||||
|
if (t.lastModified() > other.lastModified()) t.entry.setCol(col_lastModified, other.lastModified());
|
||||||
|
if (t.quality() > other.quality()) t.entry.setCol(col_quality, other.quality());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static final void max(indexRWIEntryOld t, indexRWIEntry other) {
|
||||||
|
if (t.hitcount() < other.hitcount()) t.entry.setCol(col_hitcount, other.hitcount());
|
||||||
|
if (t.wordcount() < other.wordcount()) t.entry.setCol(col_wordcount, other.wordcount());
|
||||||
|
if (t.phrasecount() < other.phrasecount()) t.entry.setCol(col_phrasecount, other.phrasecount());
|
||||||
|
if (t.posintext() < other.posintext()) t.entry.setCol(col_posintext, other.posintext());
|
||||||
|
if (t.posinphrase() < other.posinphrase()) t.entry.setCol(col_posinphrase, other.posinphrase());
|
||||||
|
if (t.posofphrase() < other.posofphrase()) t.entry.setCol(col_posofphrase, other.posofphrase());
|
||||||
|
if (t.worddistance() < other.worddistance()) t.entry.setCol(col_worddistance, other.worddistance());
|
||||||
|
if (t.lastModified() < other.lastModified()) t.entry.setCol(col_lastModified, other.lastModified());
|
||||||
|
if (t.quality() < other.quality()) t.entry.setCol(col_quality, other.quality());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public void min(indexRWIEntry other) {
|
||||||
|
min(this, other);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void max(indexRWIEntry other) {
|
||||||
|
max(this, other);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void normalize(indexRWIEntryOld t, indexRWIEntry min, indexRWIEntry max) {
|
||||||
|
assert (t.urlHash().length() == 12) : "turlhash = " + t.urlHash();
|
||||||
|
assert (min.urlHash().length() == 12) : "minurlhash = " + min.urlHash();
|
||||||
|
assert (max.urlHash().length() == 12) : "maxurlhash = " + max.urlHash();
|
||||||
|
if (1 + max.worddistance() - min.worddistance() == 0) System.out.println("min = " + min.toPropertyForm(true) + "\nmax=" + max.toPropertyForm(true));
|
||||||
|
//System.out.println("Normalize:\nentry = " + t.toPropertyForm(true));
|
||||||
|
//System.out.println("min = " + min.toPropertyForm(true));
|
||||||
|
//System.out.println("max = " + max.toPropertyForm(true));
|
||||||
|
t.entry.setCol(col_hitcount , (t.hitcount() == 0) ? 0 : 1 + 255 * (t.hitcount() - min.hitcount() ) / (1 + max.hitcount() - min.hitcount()));
|
||||||
|
t.entry.setCol(col_wordcount , (t.wordcount() == 0) ? 0 : 1 + 255 * (t.wordcount() - min.wordcount() ) / (1 + max.wordcount() - min.wordcount()));
|
||||||
|
t.entry.setCol(col_phrasecount , (t.phrasecount() == 0) ? 0 : 1 + 255 * (t.phrasecount() - min.phrasecount() ) / (1 + max.phrasecount() - min.phrasecount()));
|
||||||
|
t.entry.setCol(col_posintext , (t.posintext() == 0) ? 0 : 1 + 255 * (t.posintext() - min.posintext() ) / (1 + max.posintext() - min.posintext()));
|
||||||
|
t.entry.setCol(col_posinphrase , (t.posinphrase() == 0) ? 0 : 1 + 255 * (t.posinphrase() - min.posinphrase() ) / (1 + max.posinphrase() - min.posinphrase()));
|
||||||
|
t.entry.setCol(col_posofphrase , (t.posofphrase() == 0) ? 0 : 1 + 255 * (t.posofphrase() - min.posofphrase() ) / (1 + max.posofphrase() - min.posofphrase()));
|
||||||
|
t.entry.setCol(col_worddistance , (t.worddistance() == 0) ? 0 : 1 + 255 * (t.worddistance() - min.worddistance()) / (1 + max.worddistance() - min.worddistance())); // FIXME: hier gibts ein division by zero, was nur sein kann wenn die Normalisierung nicht geklappt hat.
|
||||||
|
t.entry.setCol(col_lastModified , (t.lastModified() == 0) ? 0 : 1 + 255 * (t.lastModified() - min.lastModified()) / (1 + max.lastModified() - min.lastModified()));
|
||||||
|
t.entry.setCol(col_quality , (t.quality() == 0) ? 0 : 1 + 255 * (t.quality() - min.quality() ) / (1 + max.quality() - min.quality()));
|
||||||
|
//System.out.println("out = " + t.toPropertyForm(true));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void normalize(indexRWIEntry min, indexRWIEntry max) {
|
||||||
|
normalize(this, min, max);
|
||||||
|
}
|
||||||
|
|
||||||
|
public indexRWIEntry generateNormalized(indexRWIEntry min, indexRWIEntry max) {
|
||||||
|
assert (this.urlHash().length() == 12) : "this.urlhash = " + this.urlHash();
|
||||||
|
indexRWIEntryOld e = (indexRWIEntryOld) this.clone();
|
||||||
|
e.normalize(min, max);
|
||||||
|
return e;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isNewer(indexRWIEntry other) {
|
||||||
|
if (other == null) return true;
|
||||||
|
if (this.lastModified() > other.lastModified()) return true;
|
||||||
|
if (this.lastModified() == other.lastModified()) {
|
||||||
|
if (this.quality() > other.quality()) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isOlder(indexRWIEntry other) {
|
||||||
|
if (other == null) return false;
|
||||||
|
if (this.lastModified() < other.lastModified()) return true;
|
||||||
|
if (this.lastModified() == other.lastModified()) {
|
||||||
|
if (this.quality() < other.quality()) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in new issue