code cleanup: removed unised methods and made more methods and objects

private
pull/1/head
Michael Peter Christen 13 years ago
parent 584663ae8c
commit 2f536cb54d

@ -33,37 +33,7 @@ import net.yacy.visualization.RasterPlotter;
public class imagetest {
public static RasterPlotter respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
/*
BufferedImage bi = new BufferedImage(640, 400, BufferedImage.TYPE_INT_RGB);
Graphics2D g = bi.createGraphics();
g.setBackground(Color.white);
g.clearRect(0, 0, 640, 400);
g.setColor(new Color(200, 200, 0));
g.drawRect(100, 50, 40, 30);
g.setColor(new Color(0, 0, 200));
try {
Class[] pType = {Integer.TYPE, Integer.TYPE, Integer.TYPE, Integer.TYPE};
Object[] pParam = new Integer[]{new Integer(66), new Integer(55), new Integer(80), new Integer(80)};
String com = "drawRect";
Method m = g.getClass().getMethod(com, pType);
Object result = m.invoke(g, pParam);
} catch (NoSuchMethodException e) {
Log.logException(e);
} catch (IllegalAccessException e) {
Log.logException(e);
} catch (InvocationTargetException e) {
Log.logException(e);
}
WritableRaster r = bi.getRaster();
for (int i = 20; i < 100; i++) r.setPixel(i, 30, new int[]{255, 0, 0});
for (int i = 20; i < 100; i++) r.setPixel(i, 32, new int[]{0, 255, 0});
for (int i = 20; i < 100; i++) r.setPixel(i, 34, new int[]{0, 0, 255});
return bi;
*/
final RasterPlotter img = new RasterPlotter(800, 600, RasterPlotter.DrawMode.MODE_SUB, "FFFFFF");
img.setColor(RasterPlotter.GREY);
for (int y = 0; y < 600; y = y + 50) PrintTool.print(img, 0, 6 + y, 0, Integer.toString(y), -1);

@ -200,7 +200,7 @@ public final class transferRWI {
serverCore.checkInterruption();
// check if we need to ask for the corresponding URL
if (!(knownURL.has(urlHash) || unknownURL.has(urlHash))) try {
if (!knownURL.has(urlHash) && !unknownURL.has(urlHash)) try {
if (sb.index.fulltext().exists(urlHash)) {
knownURL.put(urlHash);
} else {
@ -208,9 +208,7 @@ public final class transferRWI {
}
receivedURL++;
} catch (final Exception ex) {
sb.getLog().logWarning(
"transferRWI: DB-Error while trying to determine if URL with hash '" +
ASCII.String(urlHash) + "' is known.", ex);
sb.getLog().logWarning("transferRWI: DB-Error while trying to determine if URL with hash '" + ASCII.String(urlHash) + "' is known.", ex);
}
received++;
}

@ -178,10 +178,10 @@ public class YMarkCrawlStart extends HashMap<String,String>{
urlMustNotMatch,
CrawlProfile.MATCH_ALL_STRING,
CrawlProfile.MATCH_NEVER_STRING,
"",
CrawlProfile.MATCH_NEVER_STRING,
CrawlProfile.MATCH_ALL_STRING,
CrawlProfile.MATCH_NEVER_STRING,
"",
depth,
medialink,
CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_PROXY_RECRAWL_CYCLE),

@ -83,17 +83,17 @@ public class ArrayStack implements BLOB {
* time-out. Deletions are not made automatically, they must be triggered using this method.
*/
public static final long maxFileSize = Integer.MAX_VALUE;
public static final long oneMonth = 1000L * 60L * 60L * 24L * 365L / 12L;
private static final long maxFileSize = Integer.MAX_VALUE;
public static final long oneMonth = 1000L * 60L * 60L * 24L * 365L / 12L;
protected int keylength;
protected ByteOrder ordering;
private int keylength;
private ByteOrder ordering;
private final File heapLocation;
private long fileAgeLimit;
private long fileSizeLimit;
private long repositoryAgeMax;
private long repositorySizeMax;
protected List<blobItem> blobs;
private List<blobItem> blobs;
private final String prefix;
private final int buffersize;
private final boolean trimall;
@ -256,7 +256,7 @@ public class ArrayStack implements BLOB {
this.blobs.add(new blobItem(d, location, oneBlob));
}
public synchronized void unmountBLOB(final File location, final boolean writeIDX) {
private synchronized void unmountBLOB(final File location, final boolean writeIDX) {
blobItem b;
for (int i = 0; i < this.blobs.size(); i++) {
b = this.blobs.get(i);
@ -334,11 +334,7 @@ public class ArrayStack implements BLOB {
return new File[]{f0, f1};
}
public synchronized File unmountSmallestBLOB(final long maxResultSize) {
return smallestBLOB(null, maxResultSize);
}
public synchronized File smallestBLOB(final File excluding, final long maxsize) {
private synchronized File smallestBLOB(final File excluding, final long maxsize) {
if (this.blobs.isEmpty()) return null;
File bestFile = null;
long smallest = Long.MAX_VALUE;
@ -675,7 +671,7 @@ public class ArrayStack implements BLOB {
return new BlobValues(key);
}
public class BlobValues extends LookAheadIterator<byte[]> {
private class BlobValues extends LookAheadIterator<byte[]> {
private final Iterator<blobItem> bii;
private final byte[] key;
@ -732,7 +728,7 @@ public class ArrayStack implements BLOB {
return new BlobLengths(key);
}
public class BlobLengths extends LookAheadIterator<Long> {
private class BlobLengths extends LookAheadIterator<Long> {
private final Iterator<blobItem> bii;
private final byte[] key;

@ -160,12 +160,12 @@ public class BEncodedHeap implements MapStore {
}
public static class b2mEntry implements Map.Entry<byte[], Map<String, byte[]>>
private static class b2mEntry implements Map.Entry<byte[], Map<String, byte[]>>
{
private final byte[] s;
private Map<String, byte[]> b;
public b2mEntry(final byte[] s, final Map<String, byte[]> b) {
private b2mEntry(final byte[] s, final Map<String, byte[]> b) {
this.s = s;
this.b = b;
}
@ -242,7 +242,7 @@ public class BEncodedHeap implements MapStore {
* @param name
* @return true if the row exists
*/
public boolean containsKey(final byte[] pk) {
private boolean containsKey(final byte[] pk) {
return this.table.containsKey(pk);
}

@ -48,8 +48,8 @@ import net.yacy.kelondro.util.MemoryControl;
public class Compressor implements BLOB, Iterable<byte[]> {
static byte[] gzipMagic = {(byte) 'z', (byte) '|'}; // magic for gzip-encoded content
static byte[] plainMagic = {(byte) 'p', (byte) '|'}; // magic for plain content (no encoding)
private static byte[] gzipMagic = {(byte) 'z', (byte) '|'}; // magic for gzip-encoded content
private static byte[] plainMagic = {(byte) 'p', (byte) '|'}; // magic for plain content (no encoding)
private final BLOB backend;
private TreeMap<byte[], byte[]> buffer; // entries which are not yet compressed, format is RAW (without magic)

@ -729,7 +729,7 @@ public class HeapReader {
Iterator<Map.Entry<byte[], byte[]>>,
Iterable<Map.Entry<byte[], byte[]>> {
DataInputStream is;
private DataInputStream is;
int keylen;
private final File blobFile;

@ -41,7 +41,7 @@ import net.yacy.kelondro.util.FileUtils;
public final class HeapWriter {
public final static Log log = new Log("HeapWriter");
private final static Log log = new Log("HeapWriter");
public final static byte[] ZERO = new byte[]{0};
private final int keylength; // the length of the primary key

@ -381,9 +381,9 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
public class KeyIterator implements CloneableIterator<byte[]>, Iterator<byte[]> {
final boolean up, rotating;
final byte[] firstKey, secondKey;
Iterator<byte[]> iterator;
private final boolean up, rotating;
private final byte[] firstKey, secondKey;
private Iterator<byte[]> iterator;
final private CloneableIterator<byte[]> blobkeys;
public KeyIterator(final boolean up, final boolean rotating, final byte[] firstKey, final byte[] secondKey) throws IOException {

@ -1,181 +0,0 @@
// ObjectBuffer.java
// ------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2006
//
// This is a part of the kelondro database, which is a part of YaCy
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
//
//
// A NOTE FROM THE AUTHOR TO THE USERS:
//
// Using this software in any meaning (reading, learning, copying, compiling,
// running) means that you agree that the Author(s) is (are) not responsible
// for cost, loss of data or any harm that may be caused directly or indirectly
// by usage of this softare or this documentation. The usage of this software
// is on your own risk. The installation and usage (starting/running) of this
// software may allow other people or application to access your computer and
// any attached devices and is highly dependent on the configuration of the
// software which must be done by the user of the software; the author(s) is
// (are) also not responsible for proper configuration and usage of the
// software, even if provoked by documentation provided together with
// the software.
//
//
// A NOTE FROM THE AUTHOR TO DEVELOPERS:
//
// Contributions and changes to the program code should be marked as such:
// Please enter your own (C) notice below; they must be compatible with the GPL.
// Please mark also all changes in the code; if you don't mark them then they
// can't be identified; thus all unmarked code belong to the copyright holder
// as mentioned above. A good documentation of code authorities will also help
// to maintain the code and the project.
// A re-distribution must contain the intact and unchanged copyright statement.
package net.yacy.kelondro.blob;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.order.NaturalOrder;
public class ObjectBuffer {
// this is a buffer for a single (only one) key/value object
// without an index-backend
private int readHit, readMiss, writeUnique, writeDouble;
private final String name;
private byte[] key;
private Object value;
public ObjectBuffer(final String name) {
this.name = name;
this.readHit = 0;
this.readMiss = 0;
this.writeUnique = 0;
this.writeDouble = 0;
this.key = null;
this.value = null;
}
public String getName() {
return this.name;
}
public String[] status() {
return new String[]{
Integer.toString(this.readHit),
Integer.toString(this.readMiss),
Integer.toString(this.writeUnique),
Integer.toString(this.writeDouble)
};
}
private static String[] combinedStatus(final String[] a, final String[] b) {
return new String[]{
Integer.toString(Integer.parseInt(a[0]) + Integer.parseInt(b[0])),
Integer.toString(Integer.parseInt(a[1]) + Integer.parseInt(b[1])),
Integer.toString(Integer.parseInt(a[2]) + Integer.parseInt(b[2])),
Integer.toString(Integer.parseInt(a[3]) + Integer.parseInt(b[3]))
};
}
public static String[] combinedStatus(final String[][] a, final int l) {
if ((a == null) || (a.length == 0) || (l == 0)) return null;
if ((a.length >= 1) && (l == 1)) return a[0];
if ((a.length >= 2) && (l == 2)) return combinedStatus(a[0], a[1]);
return combinedStatus(combinedStatus(a, l - 1), a[l - 1]);
}
public void put(final byte[] key, final Object value) {
if ((key == null) || (value == null)) return;
synchronized(this) {
if (NaturalOrder.naturalOrder.equal(this.key, key)){
this.writeDouble++;
} else {
this.writeUnique++;
}
this.key = key;
this.value = value;
}
}
public void put(final String key, final Object value) {
if ((key == null) || (value == null)) return;
synchronized(this) {
if (NaturalOrder.naturalOrder.equal(this.key, UTF8.getBytes(key))){
this.writeDouble++;
} else {
this.writeUnique++;
}
this.key = UTF8.getBytes(key);
this.value = value;
}
}
public Object get(final byte[] key) {
if (key == null) return null;
synchronized(this) {
if (NaturalOrder.naturalOrder.equal(this.key, key)){
this.readHit++;
return this.value;
}
this.readMiss++;
return null;
}
}
public Object get(final String key) {
if (key == null) return null;
synchronized(this) {
if (NaturalOrder.naturalOrder.equal(this.key, UTF8.getBytes(key))){
this.readHit++;
return this.value;
}
this.readMiss++;
return null;
}
}
public void remove(final byte[] key) {
if (key == null) return;
synchronized(this) {
if (NaturalOrder.naturalOrder.equal(this.key, key)){
this.key = null;
this.value = null;
}
}
}
public void remove(final String key) {
if (key == null) return;
synchronized(this) {
if (NaturalOrder.naturalOrder.equal(this.key, UTF8.getBytes(key))){
this.key = null;
this.value = null;
}
}
}
}

@ -207,7 +207,7 @@ public class Stack {
public class Entry {
long h;
private long h;
byte[] b;
/**

@ -57,12 +57,12 @@ import net.yacy.kelondro.util.FileUtils;
public class Tables implements Iterable<String> {
public final static String p1 = "(?:^|.*,)";
public final static String p2 = "((?:";
public final static String p3 = ")(?:,.*|$)){";
public final static String CIDX = "_cidx";
public final static int NOINDEX = 50000;
public final static int RAMINDEX = 100000;
private final static String p1 = "(?:^|.*,)";
private final static String p2 = "((?:";
private final static String p3 = ")(?:,.*|$)){";
private final static String CIDX = "_cidx";
private final static int NOINDEX = 50000;
private final static int RAMINDEX = 100000;
private static final String suffix = ".bheap";
private static final String system_table_pkcounter = "pkcounter";
@ -71,7 +71,7 @@ public class Tables implements Iterable<String> {
private final File location;
private final ConcurrentHashMap<String, BEncodedHeap> tables;
private final ConcurrentHashMap<String, TablesColumnIndex> cidx;
int keymaxlen;
private int keymaxlen;
// use our own formatter to prevent concurrency locks with other processes
private final static GenericFormatter my_SHORT_MILSEC_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_MILSEC, 1);

@ -72,7 +72,7 @@ public abstract class TablesColumnIndex {
* @param separator - a string value used to split column values into an array
* @param table - an iterator over table rows which should be added to the index
*/
public synchronized void buildIndex(final String columnName, final String separator, final Iterator<Tables.Row> table) {
private synchronized void buildIndex(final String columnName, final String separator, final Iterator<Tables.Row> table) {
this.deleteIndex(columnName);
// loop through all rows of the table
while (table.hasNext()) {
@ -80,15 +80,6 @@ public abstract class TablesColumnIndex {
}
}
/**
* create an index for a given table and column
* @param columnName - name of the column you want to build an index for
* @param table - an iterator over table rows which should be added to the index
*/
public synchronized void buildIndex(final String columnName, final Iterator<Tables.Row> table) {
this.buildIndex(columnName, "", table);
}
/**
* create an index for a given table and given columns
* @param columns - a map of column names and booleans for 'valueIsArray' you want to build an index for
@ -113,10 +104,6 @@ public abstract class TablesColumnIndex {
this.removePK(pk);
}
public void delete(final Tables.Row row) {
this.removePK(row.getPK());
}
public void update(final String columnName, final String separator, final Tables.Row row) {
this.removePK(row.getPK());
this.add(columnName, separator, row);

@ -67,23 +67,10 @@ public class CitationReference implements Reference, Serializable {
this.entry.setCol(col_reserve, 0);
}
public CitationReference(final String urlHash, final String code) {
// the code is the external form of the row minus the leading urlHash entry
this.entry = citationRow.newEntry(UTF8.getBytes((urlHash + code)));
}
public CitationReference(final String external) {
this.entry = citationRow.newEntry(external, true);
}
public CitationReference(final byte[] row) {
private CitationReference(final byte[] row) {
this.entry = citationRow.newEntry(row);
}
public CitationReference(final byte[] row, final int offset, final boolean clone) {
this.entry = citationRow.newEntry(row, offset, clone);
}
public CitationReference(final Row.Entry rentry) {
// FIXME: see if cloning is necessary
this.entry = rentry;
@ -172,10 +159,6 @@ public class CitationReference implements Reference, Serializable {
throw new UnsupportedOperationException();
}
public int position(@SuppressWarnings("unused") int p) {
throw new UnsupportedOperationException();
}
@Override
public Collection<Integer> positions() {
throw new UnsupportedOperationException();

@ -282,11 +282,6 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
private static final char rootURLFlag0 = subdomPortPath("", 80, "");
private static final char rootURLFlag1 = subdomPortPath("www", 80, "");
public static final boolean probablyRootURL(final String urlHash) {
final char c = urlHash.charAt(5);
return c == rootURLFlag0 || c == rootURLFlag1;
}
public static final boolean probablyRootURL(final byte[] urlHash) {
final char c = (char) urlHash[5];
return c == rootURLFlag0 || c == rootURLFlag1;
@ -308,7 +303,7 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
* @param port
* @return 6 bytes base64 encoded String representing the domain of the url
*/
public static final String hosthash6(final String protocol, final String host, final int port) {
private static final String hosthash6(final String protocol, final String host, final int port) {
final StringBuilder hash = new StringBuilder(12);
final int id = Domains.getDomainID(host, null); // id=7: tld is local
int p = host.lastIndexOf('.');
@ -355,18 +350,13 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
return domLengthEstimation(urlHashBytes) << 8 / 20;
}
public static final int domDomain(final byte[] urlHash) {
private static final int domDomain(final byte[] urlHash) {
// returns the ID of the domain of the domain
assert (urlHash != null);
assert (urlHash.length == 12 || urlHash.length == 6) : "urlhash = " + ASCII.String(urlHash);
return (Base64Order.enhancedCoder.decodeByte(urlHash[(urlHash.length == 12) ? 11 : 5]) & 28) >> 2;
}
public static boolean isDomDomain(final byte[] urlHash, final int id) {
return domDomain(urlHash) == id;
}
/**
* checks for local/global IP range and local IP
*/

@ -55,7 +55,7 @@ public class URIMetadataNode implements URIMetadata {
private final byte[] hash;
private final String urlRaw, keywords;
private DigestURI url;
Bitfield flags;
private Bitfield flags;
private final int imagec, audioc, videoc, appc;
private double lat, lon;
private long ranking; // during generation of a search result this value is set

@ -188,7 +188,7 @@ public class URIMetadataRow implements URIMetadata {
*/
}
public static byte[] encodeComp(
private static byte[] encodeComp(
final DigestURI url,
final String dc_title,
final String dc_creator,
@ -220,7 +220,7 @@ public class URIMetadataRow implements URIMetadata {
this.collections = new String[0];
}
public URIMetadataRow(final Properties prop) throws kelondroException {
private URIMetadataRow(final Properties prop) throws kelondroException {
// generates an plasmaLURLEntry using the properties from the argument
// the property names must correspond to the one from toString
//System.out.println("DEBUG-ENTRY: prop=" + prop.toString());

@ -1,55 +0,0 @@
// NavigationReferenceFactory.java
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 19.05.2009 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.yacy.kelondro.data.navigation;
import java.io.Serializable;
import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.Row.Entry;
import net.yacy.kelondro.rwi.ReferenceFactory;
public class NavigationReferenceFactory implements ReferenceFactory<NavigationReference>, Serializable {
private static final long serialVersionUID=-3440570952034279619L;
@Override
public NavigationReference produceSlow(final Entry e) {
return new NavigationReferenceRow(e);
}
@Override
public NavigationReference produceFast(final NavigationReference r) {
if (r instanceof NavigationReferenceVars) return r;
return new NavigationReferenceVars(r);
}
@Override
public Row getRow() {
return NavigationReferenceRow.navEntryRow;
}
}

@ -68,15 +68,6 @@ public final class NavigationReferenceRow extends AbstractReference implements N
private static final int col_pos = 2; // p the position of the first occurence
private static final int col_flags = 3; // f reserve, may be used for flags
// appearance flags, used in RWI entry
// some names are derived from the Dublin Core Metadata tag set
public static final int flag_app_dc_description= 1; // word appears in anchor description text (the reference to an url), or any alternative text field of a link
public static final int flag_app_dc_title = 2; // word appears in title or headline or any description part
public static final int flag_app_dc_creator = 3; // word appears in author
public static final int flag_app_dc_subject = 4; // word appears in header tags or other descriptive part
public static final int flag_app_dc_identifier = 5; // word appears in url or document identifier
public static final int flag_app_emphasized = 6; // word is emphasized in text (i.e. bold, italics, special size)
private final Row.Entry entry;
public NavigationReferenceRow(
@ -95,7 +86,7 @@ public final class NavigationReferenceRow extends AbstractReference implements N
this.entry.setCol(col_flags, flags);
}
public NavigationReferenceRow(final byte[] row) {
private NavigationReferenceRow(final byte[] row) {
this.entry = navEntryRow.newEntry(row);
}
@ -165,8 +156,7 @@ public final class NavigationReferenceRow extends AbstractReference implements N
public boolean isOlder(final Reference other) {
return false;
}
// unsupported operations:
public void join(final Reference oe) {

@ -38,11 +38,11 @@ public class NavigationReferenceVars extends AbstractReference implements Navig
private static final long serialVersionUID=2873544331190937594L;
public byte[] termhash, refhash;
public int hitcount, position;
byte flags;
private byte[] termhash, refhash;
private int hitcount, position;
private byte flags;
public NavigationReferenceVars(
private NavigationReferenceVars(
final byte[] termhash,
final byte[] refhash,
final int count,
@ -76,7 +76,7 @@ public class NavigationReferenceVars extends AbstractReference implements Navig
return c;
}
public NavigationReferenceRow toRowEntry() {
private NavigationReferenceRow toRowEntry() {
return new NavigationReferenceRow(
this.termhash,
this.refhash,

@ -80,7 +80,7 @@ public class Word {
public int posInText; // unique handle, is initialized with word position (excluding double occurring words)
public int posInPhrase; // position of word in phrase
public int numOfPhrase; // number of phrase. 'normal' phrases begin with number 100
Set<Integer> phrases; // a set of handles to all phrases where this word appears
private Set<Integer> phrases; // a set of handles to all phrases where this word appears
public Bitfield flags; // the flag bits for each word
public Word(final int handle, final int pip, final int nop) {
@ -100,15 +100,6 @@ public class Word {
return this.count;
}
public void check(final int i) {
this.phrases.add(LargeNumberCache.valueOf(i));
}
public Iterator<Integer> phrases() {
// returns an iterator to handles of all phrases where the word appears
return this.phrases.iterator();
}
@Override
public String toString() {
// this is here for debugging

@ -79,7 +79,7 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
* object for termination of concurrent blocking queue processing
*/
public static final Row.Entry poisonRowEntry = urlEntryRow.newEntry();
public static final WordReferenceRow poison = new WordReferenceRow(poisonRowEntry);
private static final WordReferenceRow poison = new WordReferenceRow(poisonRowEntry);
// static properties
private static final int col_urlhash = 0; // h 12 the url hash b64-encoded
@ -209,7 +209,7 @@ public final class WordReferenceRow extends AbstractReference implements WordRef
private static final String PIN = "_";
private final BlockingQueue<String> in;
private final BlockingQueue<WordReferenceRow> out;
Thread[] worker;
private Thread[] worker;
public ExternalParser(final int concurrency) {
this.in = new LinkedBlockingQueue<String>();
this.out = new LinkedBlockingQueue<WordReferenceRow>();

@ -57,19 +57,19 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
private static int cores = Runtime.getRuntime().availableProcessors();
public static final byte[] default_language = UTF8.getBytes("uk");
public final Bitfield flags;
public long lastModified;
public final byte[] language;
private final Bitfield flags;
private long lastModified;
private final byte[] language;
public final byte[] urlHash;
private String hostHash = null;
public final char type;
public int hitcount, llocal, lother, phrasesintext,
private final char type;
private int hitcount, llocal, lother, phrasesintext,
posinphrase, posofphrase,
urlcomps, urllength,
wordsintext, wordsintitle;
private int virtualAge;
private final Queue<Integer> positions;
public double termFrequency;
private double termFrequency;
public WordReferenceVars(final URIMetadata md) {
this.language = md.language();
@ -472,9 +472,9 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
public static class TransformDistributor extends Thread {
ReferenceContainer<WordReference> container;
BlockingQueue<WordReferenceVars> out;
long maxtime;
private ReferenceContainer<WordReference> container;
private BlockingQueue<WordReferenceVars> out;
private long maxtime;
public TransformDistributor(final ReferenceContainer<WordReference> container, final BlockingQueue<WordReferenceVars> out, final long maxtime) {
this.container = container;
@ -523,9 +523,9 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
public static class TransformWorker extends Thread {
BlockingQueue<Row.Entry> in;
BlockingQueue<WordReferenceVars> out;
long maxtime;
private BlockingQueue<Row.Entry> in;
private BlockingQueue<WordReferenceVars> out;
private long maxtime;
public TransformWorker(final BlockingQueue<WordReferenceVars> out, final long maxtime) {
this.in = new LinkedBlockingQueue<Row.Entry>();

@ -43,7 +43,7 @@ public class GuiHandler extends Handler {
private final static int DEFAULT_SIZE = 400;
private int size = DEFAULT_SIZE;
private LogRecord buffer[];
int start, count;
private int start, count;
public GuiHandler() {

@ -288,11 +288,11 @@ public final class Log {
}
protected final static class logEntry {
public final Level level;
public final String message;
public Logger logger;
public String loggername;
public Throwable thrown;
private final Level level;
private final String message;
private Logger logger;
private String loggername;
private Throwable thrown;
private logEntry(final Level level, final String message) {
this.level = level;
this.message = message == null || message.length() <= 512 ? message : message.substring(0, 512);
@ -330,8 +330,8 @@ public final class Log {
}
}
protected final static logEntry poison = new logEntry();
protected final static BlockingQueue<logEntry> logQueue = new ArrayBlockingQueue<logEntry>(300);
private final static logEntry poison = new logEntry();
private final static BlockingQueue<logEntry> logQueue = new ArrayBlockingQueue<logEntry>(300);
private final static logRunner logRunnerThread = new logRunner();
static {

@ -38,109 +38,109 @@ import java.util.regex.Pattern;
public final class LogParser {
/** the version of the LogParser - <strong>Double</strong>*/
public static final String PARSER_VERSION = "version";
private static final String PARSER_VERSION = "version";
/** the amount of URLs received during DHT - <strong>Integer</strong> */
public static final String URLS_RECEIVED = "urlSum";
private static final String URLS_RECEIVED = "urlSum";
/** the amount of URLs requested during DHT - <strong>Integer</strong> */
public static final String URLS_REQUESTED = "urlReqSum";
private static final String URLS_REQUESTED = "urlReqSum";
/** the amount of URLs blocked during DHT because they match the peer's blacklist - <strong>Integer</strong> */
public static final String URLS_BLOCKED = "blockedURLSum";
private static final String URLS_BLOCKED = "blockedURLSum";
/** the amount of words received during DHT - <strong>Integer</strong> */
public static final String WORDS_RECEIVED = "wordsSum";
private static final String WORDS_RECEIVED = "wordsSum";
/** the amount of RWIs received during DHT - <strong>Integer</strong> */
public static final String RWIS_RECEIVED = "rwiSum";
private static final String RWIS_RECEIVED = "rwiSum";
/** the amount of RWIs blocked during DHT because their entries match the peer's blacklist - <strong>Integer</strong> */
public static final String RWIS_BLOCKED = "blockedRWISum";
private static final String RWIS_BLOCKED = "blockedRWISum";
/** total time receiving RWIs during DHT in milli seconds - <strong>Long</strong> */
public static final String RWIS_RECEIVED_TIME = "rwiTimeSum";
private static final String RWIS_RECEIVED_TIME = "rwiTimeSum";
/** total time receiving URLs during DHT in milli seconds - <strong>Long</strong> */
public static final String URLS_RECEIVED_TIME = "urlTimeSum";
private static final String URLS_RECEIVED_TIME = "urlTimeSum";
/** the traffic sent during DHT in bytes - <strong>Long</strong> */
public static final String DHT_TRAFFIC_SENT = "DHTSendTraffic";
private static final String DHT_TRAFFIC_SENT = "DHTSendTraffic";
/** the amount of URLs requested by other peers and sent by this one - <strong>Integer</strong> */
public static final String DHT_URLS_SENT = "DHTSendURLs";
private static final String DHT_URLS_SENT = "DHTSendURLs";
/** the amount of rejected DHT transfers from other peers (i.e. because this peer was busy) - <strong>Integer</strong> */
public static final String DHT_REJECTED = "RWIRejectCount";
private static final String DHT_REJECTED = "RWIRejectCount";
/** the peer-names from whose DHT transfers were rejected - <strong>HasSet</strong> */
public static final String DHT_REJECTED_PEERS_NAME = "DHTRejectPeerNames";
private static final String DHT_REJECTED_PEERS_NAME = "DHTRejectPeerNames";
/** the peer-hashes from whose DHT transfers were rejected - <strong>HasSet</strong> */
public static final String DHT_REJECTED_PEERS_HASH = "DHTRejectPeerHashs";
private static final String DHT_REJECTED_PEERS_HASH = "DHTRejectPeerHashs";
/** the peer-names this peer sent DHT chunks to - <strong>HasSet</strong> */
public static final String DHT_SENT_PEERS_NAME = "DHTPeerNames";
private static final String DHT_SENT_PEERS_NAME = "DHTPeerNames";
/** the peer-hashes this peer sent DHT chunks to - <strong>HasSet</strong> */
public static final String DHT_SENT_PEERS_HASH = "DHTPeerHashs";
private static final String DHT_SENT_PEERS_HASH = "DHTPeerHashs";
/** total amount of selected peers for index distribution - <strong>Integer</strong> */
public static final String DHT_SELECTED = "DHTSelectionTargetCount";
private static final String DHT_SELECTED = "DHTSelectionTargetCount";
/** total amount of words selected for index distribution - <strong>Integer</strong> */
public static final String DHT_WORDS_SELECTED = "DHTSelectionWordsCount";
private static final String DHT_WORDS_SELECTED = "DHTSelectionWordsCount";
/** total time selecting words for index distribution - <strong>Integer</strong> */
public static final String DHT_WORDS_SELECTED_TIME = "DHTSelectionWordsTimeCount";
private static final String DHT_WORDS_SELECTED_TIME = "DHTSelectionWordsTimeCount";
/** the minimal DHT distance during peer-selection for index distribution - <strong>Long</strong> */
public static final String DHT_DISTANCE_MIN = "minDHTDist";
private static final String DHT_DISTANCE_MIN = "minDHTDist";
/** the maximal DHT distance during peer-selection for index distribution - <strong>Long</strong> */
public static final String DHT_DISTANCE_MAX = "maxDHTDist";
private static final String DHT_DISTANCE_MAX = "maxDHTDist";
/** the average DHT distance during peer-selection for index distribution - <strong>Long</strong> */
public static final String DHT_DISTANCE_AVERAGE = "avgDHTDist";
private static final String DHT_DISTANCE_AVERAGE = "avgDHTDist";
/** how many times remote peers were too busy to accept the index transfer - <strong>Integer</strong> */
public static final String PEERS_BUSY = "busyPeerCount";
private static final String PEERS_BUSY = "busyPeerCount";
/** how many times not enough peers for index distribution were found - <strong>Integer</strong> */
public static final String PEERS_TOO_LESS = "notEnoughDHTPeers";
private static final String PEERS_TOO_LESS = "notEnoughDHTPeers";
/** how many times the index distribution failed (i.e. due to time-out or other reasons) - <strong>Integer</strong> */
public static final String DHT_SENT_FAILED = "failedIndexDistributionCount";
private static final String DHT_SENT_FAILED = "failedIndexDistributionCount";
/** how many times the error "<code>tried to create left child-node twice</code>" occured - <strong>Integer</strong> */
public static final String ERROR_CHILD_TWICE_LEFT = "leftChildTwiceCount";
private static final String ERROR_CHILD_TWICE_LEFT = "leftChildTwiceCount";
/** how many times the error "<code>tried to create right child-node twice</code>" occured - <strong>Integer</strong> */
public static final String ERROR_CHILD_TWICE_RIGHT = "rightChildTwiceCount";
private static final String ERROR_CHILD_TWICE_RIGHT = "rightChildTwiceCount";
/** how many ranking distributions were executed successfully - <strong>Integer</strong> */
public static final String RANKING_DIST = "rankingDistributionCount";
private static final String RANKING_DIST = "rankingDistributionCount";
/** total time the ranking distributions took - <strong>Integer</strong> */
public static final String RANKING_DIST_TIME = "rankingDistributionTime";
private static final String RANKING_DIST_TIME = "rankingDistributionTime";
/** how many ranking distributions failed - <strong>Integer</strong> */
public static final String RANKING_DIST_FAILED = "rankingDistributionFailCount";
private static final String RANKING_DIST_FAILED = "rankingDistributionFailCount";
/** how many times the error "<code>Malformed URL</code>" occured - <strong>Integer</strong> */
public static final String ERROR_MALFORMED_URL = "malformedURLCount";
private static final String ERROR_MALFORMED_URL = "malformedURLCount";
/** the amount of indexed sites - <strong>Integer</strong> */
public static final String INDEXED_SITES = "indexedSites";
private static final String INDEXED_SITES = "indexedSites";
/** total amount of indexed words - <strong>Integer</strong> */
public static final String INDEXED_WORDS = "indexedWords";
private static final String INDEXED_WORDS = "indexedWords";
/** total size of all indexed sites - <strong>Integer</strong> */
public static final String INDEXED_SITES_SIZE = "indexedSiteSizeSum";
private static final String INDEXED_SITES_SIZE = "indexedSiteSizeSum";
/** total amount of indexed anchors - <strong>Integer</strong> */
public static final String INDEXED_ANCHORS = "indexedAnchors";
private static final String INDEXED_ANCHORS = "indexedAnchors";
// /** total time needed for stacking the site of an indexing - <strong>Integer</strong> */
// public static final String INDEXED_STACK_TIME = "indexedStackingTime";
@ -155,16 +155,16 @@ public final class LogParser {
// public static final String INDEXED_STORE_TIME = "indexedStorageTime";
/** total time needed for storing the results of a link indexing - <strong>Integer</strong> */
public static final String INDEXED_LINKSTORE_TIME = "indexedLinkStorageTime";
private static final String INDEXED_LINKSTORE_TIME = "indexedLinkStorageTime";
/** total time needed for storing the results of a word indexing - <strong>Integer</strong> */
public static final String INDEXED_INDEXSTORE_TIME = "indexedIndexStorageTime";
private static final String INDEXED_INDEXSTORE_TIME = "indexedIndexStorageTime";
/** accumulated time needed to parse the log entries up to now (in ms)*/
public static final String TOTAL_PARSER_TIME = "totalParserTime";
private static final String TOTAL_PARSER_TIME = "totalParserTime";
/** times the parser was called, respectively amount of independent log-lines */
public static final String TOTAL_PARSER_RUNS = "totalParserRuns";
private static final String TOTAL_PARSER_RUNS = "totalParserRuns";
private static final float parserVersion = 0.1f;

@ -35,8 +35,8 @@ import java.util.logging.LogRecord;
public final class LogalizerHandler extends Handler {
public static boolean enabled;
public static boolean debug;
private static boolean enabled;
private static boolean debug;
public LogalizerHandler() {
super();

@ -51,8 +51,8 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
private static final Pattern multiDumpFilterPattern = Pattern.compile(multiDumpFilter);
public static class StackTrace {
public String text;
public Thread.State state;
private String text;
private Thread.State state;
public StackTrace(final String text, final Thread.State state) {
this.state = state;
this.text = text;
@ -75,7 +75,7 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
}
public static class Lock {
public String id;
private String id;
public Lock(final String name) {
this.id = name;
}

@ -213,7 +213,7 @@ public final class ReferenceContainerArray<ReferenceType extends Reference> {
public class ReferenceCountIterator implements CloneableIterator<Rating<byte[]>>, Iterable<Rating<byte[]>> {
private final boolean rot, excludePrivate;
protected CloneableIterator<byte[]> iterator;
private CloneableIterator<byte[]> iterator;
public ReferenceCountIterator(final byte[] startWordHash, final boolean rot, final boolean excludePrivate) throws IOException {
this.rot = rot;

Loading…
Cancel
Save