boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

made class methods static where possible

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 5bd3c90907
commit b0c408788b
48 changed files with 831 additions and 779 deletions
Show Stats Download Patch File Download Diff File

4
.settings/org.eclipse.jdt.core.prefs
Unescape View File

@ -33,10 +33,10 @@ org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning
org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled
org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning
org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning
org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore
org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=warning
org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore
org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning
org.eclipse.jdt.core.compiler.problem.missingDefaultCase=warning
org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore
org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore
org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled
org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=warning

4
htroot/BlacklistCleaner_p.java
Unescape View File

@ -109,7 +109,7 @@ public class BlacklistCleaner_p {
final Map<String, BlacklistError> illegalEntries = getIllegalEntries(blacklistToUse, Switchboard.urlBlacklist, allowRegex);
prop.put(RESULTS + "blList", blacklistToUse);
prop.put(RESULTS + "entries", illegalEntries.size());
prop.putHTML(RESULTS + "blEngine", Switchboard.urlBlacklist.getEngineInfo());
prop.putHTML(RESULTS + "blEngine", Blacklist.getEngineInfo());
prop.put(RESULTS + "disabled", (illegalEntries.isEmpty()) ? "1" : "0");
if (!illegalEntries.isEmpty()) {
prop.put(RESULTS + DISABLED + "entries", illegalEntries.size());
@ -255,7 +255,7 @@ public class BlacklistCleaner_p {
}
legalEntries.add(element);
err = blEngine.checkError(element, properties);
err = Blacklist.checkError(element, properties);
if (err.getInt() > 0) {
illegalEntries.put(element, err);

3
htroot/BlacklistImpExp_p.java
Unescape View File

@ -37,6 +37,7 @@ import java.util.TreeMap;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.Seed;
import net.yacy.repository.Blacklist;
import net.yacy.search.Switchboard;
import de.anomic.data.ListManager;
import de.anomic.server.serverObjects;
@ -57,7 +58,7 @@ public class BlacklistImpExp_p {
String blacklistToUse = null;
final serverObjects prop = new serverObjects();
prop.putHTML("blacklistEngine", Switchboard.urlBlacklist.getEngineInfo());
prop.putHTML("blacklistEngine", Blacklist.getEngineInfo());
// if we have not chosen a blacklist until yet we use the first file
if (blacklistToUse == null && dirlist != null && !dirlist.isEmpty()) {

3
htroot/BlacklistTest_p.java
Unescape View File

@ -34,6 +34,7 @@ import java.net.MalformedURLException;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.repository.Blacklist;
import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard;
import de.anomic.data.ListManager;
@ -49,7 +50,7 @@ public class BlacklistTest_p {
ListManager.listsPath = new File(ListManager.switchboard.getDataPath(),ListManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
final serverObjects prop = new serverObjects();
prop.putHTML("blacklistEngine", Switchboard.urlBlacklist.getEngineInfo());
prop.putHTML("blacklistEngine", Blacklist.getEngineInfo());
// do all post operations
if(post != null && post.containsKey("testList")) {

2
htroot/Blacklist_p.java
Unescape View File

@ -70,7 +70,7 @@ public class Blacklist_p {
String blacklistToUse = null;
final serverObjects prop = new serverObjects();
prop.putHTML("blacklistEngine", Switchboard.urlBlacklist.getEngineInfo());
prop.putHTML("blacklistEngine", Blacklist.getEngineInfo());
// do all post operations
if (post != null) {

2
htroot/Connections_p.java
Unescape View File

@ -75,7 +75,7 @@ public final class Connections_p {
// waiting for all threads to finish
int idx = 0, numActiveRunning = 0, numActivePending = 0;
boolean dark = true;
for (final Session s: ((serverCore) httpd).getJobList()) {
for (final Session s: serverCore.getJobList()) {
if (!s.isAlive()) continue;
// get the session runtime

2
htroot/Network.java
Unescape View File

@ -122,7 +122,7 @@ public class Network {
prop.put("table_my-dhtreceive", seed.getFlagAcceptRemoteIndex() ? 1 : 0);
prop.put("table_my-nodestate", seed.getFlagRootNode() ? 1 : 0);
myppm = sb.currentPPM();
myppm = Switchboard.currentPPM();
myqph = 60d * sb.averageQPM();
prop.put("table_my-version", seed.get(Seed.VERSION, "-"));
prop.put("table_my-utc", seed.get(Seed.UTC, "-"));

2
htroot/api/status_p.java
Unescape View File

@ -51,7 +51,7 @@ public class status_p {
prop.put("rejected", "0");
sb.updateMySeed();
final int cacheMaxSize = (int) sb.getConfigLong(SwitchboardConstants.WORDCACHE_MAX_COUNT, 10000);
prop.putNum("ppm", sb.currentPPM());
prop.putNum("ppm", Switchboard.currentPPM());
prop.putNum("qpm", sb.peers.mySeed().getQPM());
prop.putNum("wordCacheSize", segment.termIndex().getBufferSize());
prop.putNum("wordCacheMaxSize", cacheMaxSize);

2
htroot/yacysearch.java
Unescape View File

@ -104,7 +104,7 @@ public class yacysearch {
final UserDB.Entry user = sb.userDB.getUser(header);
authenticated = (user != null && user.hasRight(UserDB.AccessRight.EXTENDED_SEARCH_RIGHT));
}
final boolean localhostAccess = sb.accessFromLocalhost(header);
final boolean localhostAccess = Switchboard.accessFromLocalhost(header);
final String promoteSearchPageGreeting =
(env.getConfigBool(SwitchboardConstants.GREETING_NETWORK_NAME, false)) ? env.getConfig(
"network.unit.description",

2
source/de/anomic/crawler/CrawlQueues.java
Unescape View File

@ -541,7 +541,7 @@ public class CrawlQueues {
* @param url
* @return
*/
private String urlToString(final DigestURI url) {
private static String urlToString(final DigestURI url) {
return (url == null ? "null" : url.toNormalform(true, false));
}

2
source/de/anomic/crawler/CrawlStacker.java
Unescape View File

@ -130,7 +130,7 @@ public final class CrawlStacker {
clear();
}
private boolean prefetchHost(final String host) {
private static boolean prefetchHost(final String host) {
// returns true when the host was known in the dns cache.
// If not, the host is stacked on the fetch stack and false is returned
try {

4
source/de/anomic/crawler/CrawlSwitchboard.java
Unescape View File

@ -549,11 +549,11 @@ public final class CrawlSwitchboard
/**
* Loads crawl profiles from a DB file.
*
*
* @param file DB file
* @return crawl profile data
*/
private MapHeap loadFromDB(final File file) {
private static MapHeap loadFromDB(final File file) {
MapHeap ret;
try {
ret = new MapHeap(file, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');

2
source/de/anomic/crawler/NoticedURL.java
Unescape View File

@ -308,7 +308,7 @@ public class NoticedURL {
}
}
private Request pop(final Balancer balancer, final boolean delay, final CrawlSwitchboard cs, final RobotsTxt robots) throws IOException {
private static Request pop(final Balancer balancer, final boolean delay, final CrawlSwitchboard cs, final RobotsTxt robots) throws IOException {
// this is a filo - pop
int s;
Request entry;

9
source/de/anomic/data/wiki/WikiCode.java
Unescape View File

@ -211,6 +211,7 @@ public class WikiCode extends AbstractWikiParser implements WikiParser {
* @return HTML fragment.
* @throws IOException in case input from reader can not be read.
*/
@Override
protected String transform(final String hostport, final BufferedReader reader, final int length)
throws IOException {
final StringBuilder out = new StringBuilder(length);
@ -345,7 +346,7 @@ public class WikiCode extends AbstractWikiParser implements WikiParser {
* @param properties String which may contain several table properties and/or junk.
* @return String containing only table properties.
*/
private StringBuilder filterTableProperties(final String properties) {
private static StringBuilder filterTableProperties(final String properties) {
final String[] values = properties.replaceAll("&quot;", EMPTY).split("[= ]"); //splitting the string at = and blanks
final StringBuilder stringBuilder = new StringBuilder(properties.length());
String key, value;
@ -376,7 +377,7 @@ public class WikiCode extends AbstractWikiParser implements WikiParser {
* @param stringBuilder this is what key/value are appended to.
* @return
*/
private StringBuilder appendKeyValuePair(final String key, final String value, final StringBuilder stringBuilder) {
private static StringBuilder appendKeyValuePair(final String key, final String value, final StringBuilder stringBuilder) {
return stringBuilder.append(" ").append(key).append("=\"").append(value).append("\"");
}
@ -577,7 +578,7 @@ public class WikiCode extends AbstractWikiParser implements WikiParser {
* @param line line of text to be transformed from wiki code to HTML
* @return HTML fragment
*/
private String processLinksAndImages(final String hostport, String line) {
private static String processLinksAndImages(final String hostport, String line) {
// create links
String kl, kv, alt, align;
@ -1009,7 +1010,7 @@ public class WikiCode extends AbstractWikiParser implements WikiParser {
}
private String processMetadata(String line) {
private static String processMetadata(String line) {
int p, q, s = 0;
while ((p = line.indexOf(WIKI_OPEN_METADATA, s)) >= 0 && (q = line.indexOf(WIKI_CLOSE_METADATA, p + 1)) >= 0) {
s = q; // continue with next position

6
source/de/anomic/http/server/HTTPDemon.java
Unescape View File

@ -222,7 +222,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
* @param prop
* @return <code>true</code> if a persistent connection was requested or <code>false</code> otherwise
*/
private boolean handlePersistentConnection(final RequestHeader header, final HashMap<String, Object> prop) {
private static boolean handlePersistentConnection(final RequestHeader header, final HashMap<String, Object> prop) {
if (!keepAliveSupport) {
prop.put(HeaderFramework.CONNECTION_PROP_PERSISTENT,"close");
@ -256,7 +256,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
return persistent;
}
private boolean handleYaCyHopAuthentication(final RequestHeader header, final HashMap<String, Object> prop) {
private static boolean handleYaCyHopAuthentication(final RequestHeader header, final HashMap<String, Object> prop) {
// check if the user has allowed that his/her peer is used for hops
if (!allowYaCyHop()) return false;
@ -425,7 +425,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
}
}
private void logUnexpectedError(final Exception e, final String address) {
private static void logUnexpectedError(final Exception e, final String address) {
if (e instanceof InterruptedException) {
log.logInfo("Interruption detected");
} else {

7
source/de/anomic/server/serverCore.java
Unescape View File

@ -445,7 +445,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
this.log.logConfig("* terminated");
}
public List<Session> getJobList() {
public static List<Session> getJobList() {
final Thread[] threadList = new Thread[sessionThreadGroup.activeCount()];
serverCore.sessionThreadGroup.enumerate(threadList, false);
final ArrayList<Session> l = new ArrayList<Session>();
@ -477,7 +477,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
}
// idle sensor: the thread is idle if there are no sessions running
public boolean idle() {
public static boolean idle() {
// idleThreadCheck();
final Thread[] threadList = new Thread[sessionThreadGroup.activeCount()];
serverCore.sessionThreadGroup.enumerate(threadList, false);
@ -1086,7 +1086,8 @@ public final class serverCore extends AbstractBusyThread implements BusyThread {
sslsock.addHandshakeCompletedListener(
new HandshakeCompletedListener() {
public void handshakeCompleted(
@Override
public void handshakeCompleted(
final HandshakeCompletedEvent event) {
System.out.println("Handshake finished!");
System.out.println(

88
source/de/anomic/server/serverCoreSocket.java
Unescape View File

@ -1,4 +1,4 @@
//serverCoreSocket.java
//serverCoreSocket.java
//-------------------------------------
//part of YACY
//
@ -34,19 +34,19 @@ import java.net.SocketException;
import java.nio.channels.SocketChannel;
public class serverCoreSocket extends Socket {
private PushbackInputStream input = null;
private Socket sock = null;
private boolean isSSL = false;
private String sslType = null;
public serverCoreSocket(final Socket sock) throws IOException {
this.sock = sock;
// determine the socket type
detectSSL();
}
public boolean isSSL() {
return this.isSSL;
}
@ -54,10 +54,10 @@ public class serverCoreSocket extends Socket {
public String getProtocol() {
return this.sslType;
}
private void detectSSL() throws IOException {
final InputStream in = getInputStream();
final InputStream in = getInputStream();
// read the first 5 bytes to determine the protocol type
final byte[] preRead = new byte[5];
int read, count = 0;
@ -69,7 +69,7 @@ public class serverCoreSocket extends Socket {
((PushbackInputStream) in).unread(preRead,0,count);
return;
}
int idx = 0;
if ((preRead[0] & 0xFF) == 22) {
// we have detected the ContentType field.
@ -96,6 +96,9 @@ public class serverCoreSocket extends Socket {
case 2:
this.sslType = "TLS_1_1";
break;
default:
this.sslType = "SSL_3";
break;
}
this.isSSL = true;
//} else {
@ -103,88 +106,107 @@ public class serverCoreSocket extends Socket {
}
// unread pre read bytes
((PushbackInputStream) in).unread(preRead);
((PushbackInputStream) in).unread(preRead);
}
@Override
public InetAddress getInetAddress() {
return this.sock.getInetAddress();
}
@Override
public InetAddress getLocalAddress() {
return this.sock.getLocalAddress();
}
@Override
public int getPort() {
return this.sock.getPort();
}
@Override
public int getLocalPort() {
return this.sock.getLocalPort();
}
@Override
public SocketAddress getRemoteSocketAddress() {
return this.sock.getRemoteSocketAddress();
}
@Override
public SocketAddress getLocalSocketAddress() {
return this.sock.getLocalSocketAddress();
}
@Override
public SocketChannel getChannel() {
return this.sock.getChannel();
}
@Override
public InputStream getInputStream() throws IOException {
if (this.input == null) {
this.input = new PushbackInputStream(this.sock.getInputStream(),100);
}
return this.input;
}
@Override
public OutputStream getOutputStream() throws IOException {
return this.sock.getOutputStream();
}
@Override
public synchronized void close() throws IOException {
this.sock.close();
}
@Override
public void shutdownInput() throws IOException {
this.sock.shutdownInput();
}
@Override
public void shutdownOutput() throws IOException {
this.sock.shutdownOutput();
}
@Override
public String toString() {
return this.sock.toString();
}
@Override
public boolean isConnected() {
return this.sock.isConnected();
}
@Override
public boolean isBound() {
return this.sock.isBound();
}
@Override
public boolean isClosed() {
return this.sock.isClosed();
}
@Override
public boolean isInputShutdown() {
return this.sock.isInputShutdown();
}
@Override
public boolean isOutputShutdown() {
return this.sock.isOutputShutdown();
}
}
@Override
public synchronized void setSoTimeout(final int timeout) throws SocketException {
this.sock.setSoTimeout(timeout);
}
}

4
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -466,7 +466,7 @@ public class serverSwitch
final List<String> list = new ArrayList<String>();
final WorkflowThread st = getThread(threadName);
for ( final Session s : ((serverCore) st).getJobList() ) {
for ( final Session s : serverCore.getJobList() ) {
if ( !s.isAlive() ) {
continue;
}
@ -483,7 +483,7 @@ public class serverSwitch
}
final WorkflowThread st = getThread(threadName);
for ( final Session s : ((serverCore) st).getJobList() ) {
for ( final Session s : serverCore.getJobList() ) {
if ( (s.isAlive()) && (s.getName().equals(sessionName)) ) {
// try to stop session
s.setStopped(true);

41
source/net/yacy/YaCySearchClient.java
Unescape View File

@ -12,12 +12,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
@ -50,34 +50,35 @@ public class YaCySearchClient {
* YaCy Search Results are produced in Opensearch format which is basically RSS.
* The YaCy Search Result API Client is therefore implemented as a simple RSS reader.
*/
private String host, query;
private int port, offset;
private final String host, query;
private final int port;
private int offset;
public YaCySearchClient(final String host, final int port, final String query) {
this.host = host;
this.port = port;
this.offset = -10;
this.query = query;
}
public SearchResult next() throws IOException {
this.offset += 10; // you may call this again and get the next results
return new SearchResult();
}
public class SearchResult extends ArrayList<RSSEntry> {
private static final long serialVersionUID = 1337L;
public SearchResult() throws IOException {
URL url;
Document doc;
String u = new StringBuilder(120).append("http://")
.append(host)
.append(YaCySearchClient.this.host)
.append(":")
.append(port)
.append(YaCySearchClient.this.port)
.append("/yacysearch.rss?verify=false&startRecord=")
.append(offset)
.append(YaCySearchClient.this.offset)
.append("&maximumRecords=10&resource=local&query=")
.append(query.replaceAll(" ", "+")).toString();
.append(YaCySearchClient.this.query.replaceAll(" ", "+")).toString();
try { url = new URL(u); } catch (MalformedURLException e) { throw new IOException (e); }
try { doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(url.openStream()); }
catch (ParserConfigurationException e) { throw new IOException (e); }
@ -86,7 +87,7 @@ public class YaCySearchClient {
for (int i = 0; i < nodes.getLength(); i++)
this.add(new RSSEntry((Element) nodes.item(i)));
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder(this.size() * 80 + 1);
@ -98,11 +99,11 @@ public class YaCySearchClient {
public static class RSSEntry {
String title, link, snippet;
public RSSEntry(Element element) {
title = val(element, "title", "");
link = val(element, "link", "");
snippet = val(element, "description", "");
this.title = val(element, "title", "");
this.link = val(element, "link", "");
this.snippet = val(element, "description", "");
}
private String val(Element parent, String label, String dflt) {
private static String val(Element parent, String label, String dflt) {
Element e = (Element) parent.getElementsByTagName(label).item(0);
Node child = e.getFirstChild();
return (child instanceof CharacterData) ?
@ -112,15 +113,15 @@ public class YaCySearchClient {
@Override
public String toString() {
return new StringBuilder(80).append("Title : ")
.append(title)
.append(this.title)
.append("\nLink : ")
.append(link)
.append(this.link)
.append("\nDescription: ")
.append(snippet)
.append(this.snippet)
.append("\n").toString();
}
}
/**
* Call the main method with one argument, the query string
* search results are then simply printed out.

28
source/net/yacy/cora/document/JSONTokener.java
Unescape View File

@ -45,7 +45,7 @@ public class JSONTokener {
private int index;
private int line;
private char previous;
private Reader reader;
private final Reader reader;
private boolean usePrevious;
@ -55,7 +55,7 @@ public class JSONTokener {
* @param reader A reader.
*/
public JSONTokener(Reader reader) {
this.reader = reader.markSupported() ?
this.reader = reader.markSupported() ?
reader : new BufferedReader(reader);
this.eof = false;
this.usePrevious = false;
@ -82,7 +82,7 @@ public class JSONTokener {
* the next number or identifier.
*/
public void back() throws JSONException {
if (usePrevious || index <= 0) {
if (this.usePrevious || this.index <= 0) {
throw new JSONException("Stepping back two steps is not supported");
}
this.index -= 1;
@ -110,9 +110,9 @@ public class JSONTokener {
}
return -1;
}
public boolean end() {
return eof && !usePrevious;
return this.eof && !this.usePrevious;
}
@ -125,7 +125,7 @@ public class JSONTokener {
next();
if (end()) {
return false;
}
}
back();
return true;
}
@ -147,11 +147,11 @@ public class JSONTokener {
} catch (IOException exception) {
throw new JSONException(exception);
}
if (c <= 0) { // End of stream
this.eof = true;
c = 0;
}
}
}
this.index += 1;
if (this.previous == '\r') {
@ -205,7 +205,7 @@ public class JSONTokener {
while (pos < n) {
buffer[pos] = next();
if (end()) {
throw syntaxError("Substring bounds error");
throw syntaxError("Substring bounds error");
}
pos += 1;
}
@ -356,6 +356,8 @@ public class JSONTokener {
case '(':
back();
return new JSONArray(this);
default:
break;
}
/*
@ -395,11 +397,11 @@ public class JSONTokener {
int startIndex = this.index;
int startCharacter = this.character;
int startLine = this.line;
reader.mark(Integer.MAX_VALUE);
this.reader.mark(Integer.MAX_VALUE);
do {
c = next();
if (c == 0) {
reader.reset();
this.reader.reset();
this.index = startIndex;
this.character = startCharacter;
this.line = startLine;
@ -413,7 +415,7 @@ public class JSONTokener {
back();
return c;
}
/**
* Make a JSONException to signal a syntax error.
@ -433,6 +435,6 @@ public class JSONTokener {
*/
@Override
public String toString() {
return " at " + index + " [character " + this.character + " line " + this.line + "]";
return " at " + this.index + " [character " + this.character + " line " + this.line + "]";
}
}

4
source/net/yacy/cora/protocol/http/HTTPClient.java
Unescape View File

@ -609,7 +609,7 @@ public class HTTPClient {
}
}
private byte[] getByteArray(final HttpEntity entity, final int maxBytes) throws IOException {
private static byte[] getByteArray(final HttpEntity entity, final int maxBytes) throws IOException {
final InputStream instream = entity.getContent();
if (instream == null) {
return null;
@ -657,7 +657,7 @@ public class HTTPClient {
httpParams.setParameter(HTTP.TARGET_HOST, this.host);
}
private void setProxy(final HttpParams httpParams) {
private static void setProxy(final HttpParams httpParams) {
if (ProxySettings.use)
ConnRouteParams.setDefaultProxy(httpParams, ProxySettings.getProxyHost());
// TODO find a better way for this

4
source/net/yacy/document/content/DCEntry.java
Unescape View File

@ -145,7 +145,7 @@ public class DCEntry extends TreeMap<String, String> {
}
}
private String bestU(String[] urls) {
private static String bestU(String[] urls) {
for (String uu: urls) {
if (uu.startsWith("http://") && (uu.endsWith(".html") || uu.endsWith(".htm") || uu.endsWith(".pdf") || uu.endsWith(".doc") || uu.endsWith(".rss") || uu.endsWith(".xml"))) return uu;
}
@ -250,7 +250,7 @@ public class DCEntry extends TreeMap<String, String> {
return Double.parseDouble(t);
}
private String stripCDATA(String s) {
private static String stripCDATA(String s) {
if (s == null) return null;
s = s.trim();
if (s.startsWith("<![CDATA[")) s = s.substring(9);

2
source/net/yacy/document/geolocation/GeoLocation.java
Unescape View File

@ -82,7 +82,7 @@ public class GeoLocation extends IntegerGeoPoint implements Comparable<GeoLocati
return 0;
}
private long ph(int population) {
private static long ph(int population) {
if (population > 10000) population -= 10000;
return population;
}

2
source/net/yacy/document/parser/augment/AugmentParser.java
Unescape View File

@ -57,7 +57,7 @@ public class AugmentParser extends AbstractParser implements Parser {
return retDocs;
}
private Document analyze (Document alreadyParsedDocument, DigestURI url,
private static Document analyze (Document alreadyParsedDocument, DigestURI url,
String mimeType, String charset) {
Document newDoc = new Document(url, mimeType, charset, null, null, null, "", "",

4
source/net/yacy/document/parser/csvParser.java
Unescape View File

@ -80,7 +80,7 @@ public class csvParser extends AbstractParser implements Parser {
false)};
}
private String concatRow(String[] columns) {
private static String concatRow(String[] columns) {
final StringBuilder sb = new StringBuilder(80);
for (final String column : columns) {
if (sb.length() > 0) sb.append(' ');
@ -90,7 +90,7 @@ public class csvParser extends AbstractParser implements Parser {
return sb.toString();
}
private List<String[]> getTable(String charset, InputStream source) {
private static List<String[]> getTable(String charset, InputStream source) {
final List<String[]> rows = new ArrayList<String[]>();
BufferedReader reader;
try {

122
source/net/yacy/document/parser/images/bmpParser.java
Unescape View File

@ -7,7 +7,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -37,7 +37,7 @@ import net.yacy.kelondro.logging.Log;
public class bmpParser {
// this is a implementation of http://de.wikipedia.org/wiki/Windows_Bitmap
// file offsets
private static int FILEHEADER_offset = 0;
private static int INFOHEADER_offset = 14;
@ -48,27 +48,27 @@ public class bmpParser {
//private static int BI_RLE8 = 1;
//private static int BI_RLE4 = 2;
//private static int BI_BITFIELDS = 3;
public static final boolean isBMP(final byte[] source) {
// check the file magic
return (source != null) && (source.length >= 2) && (source[0] == 'B') && (source[1] == 'M');
}
public static IMAGEMAP parse(final byte[] source) {
// read info-header
final int bfOffBits = DWORD(source, FILEHEADER_offset + 10);
final INFOHEADER infoheader = new INFOHEADER(source, INFOHEADER_offset);
final COLORTABLE colortable = new COLORTABLE(source, INFOHEADER_offset + INFOHEADER_size, infoheader);
// check consistency with bfOffBits
assert bfOffBits == INFOHEADER_offset + 40 + colortable.colorbytes : "bfOffBits = " + bfOffBits + ", colorbytes = " + colortable.colorbytes;
assert infoheader.biSizeImage <= source.length - bfOffBits : "bfOffBits = " + bfOffBits + ", biSizeImage = " + infoheader.biSizeImage + ", source.length = " + source.length;
return new IMAGEMAP(source, bfOffBits, infoheader.biWidth, infoheader.biHeight, infoheader.biCompression, infoheader.biBitCount, colortable);
}
public static final int DWORD(final byte[] b, final int offset) {
if (offset + 3 >= b.length) return 0;
int ret = (b[offset + 3] & 0xff);
@ -82,72 +82,72 @@ public class bmpParser {
final int ret = ((b[offset + 1] & 0xff) << 8) | (b[offset] & 0xff);
return ret;
}
public static final int BYTE(final byte[] b, final int offset) {
final int ret = (b[offset] & 0xff);
return ret;
}
public static class INFOHEADER {
public int biWidth, biHeight, biBitCount, biCompression, biSizeImage, biClrUsed;
public INFOHEADER(final byte[] s, final int offset) {
// read info-header
biWidth = DWORD(s, offset + 4);
biHeight = DWORD(s, offset + 8);
biBitCount = WORD(s, offset + 14);
biCompression = WORD(s, offset + 16);
biSizeImage = DWORD(s, offset + 20);
biClrUsed = DWORD(s, offset + 32);
this.biWidth = DWORD(s, offset + 4);
this.biHeight = DWORD(s, offset + 8);
this.biBitCount = WORD(s, offset + 14);
this.biCompression = WORD(s, offset + 16);
this.biSizeImage = DWORD(s, offset + 20);
this.biClrUsed = DWORD(s, offset + 32);
}
}
public static class COLORTABLE {
public int colorbytes;
public int[] colorindex;
public COLORTABLE(final byte[] s, final int offset, final INFOHEADER infoheader) {
// read colortable
colorbytes = 0; // for consistency check
this.colorbytes = 0; // for consistency check
if (infoheader.biClrUsed == 0 || infoheader.biClrUsed > 1024) {
if ((infoheader.biBitCount == 1) || (infoheader.biBitCount == 4) || (infoheader.biBitCount == 8)) {
colorindex = new int[1 << infoheader.biBitCount];
colorbytes = 4 * colorindex.length;
this.colorindex = new int[1 << infoheader.biBitCount];
this.colorbytes = 4 * this.colorindex.length;
int color;
for (int i = 0; i < colorindex.length; i++) {
for (int i = 0; i < this.colorindex.length; i++) {
// translate BGR into RGB color Scheme
color = 0xffffff & DWORD(s, offset + 4 * i);
colorindex[i] = color;
this.colorindex[i] = color;
}
} else {
colorindex = null;
this.colorindex = null;
}
} else {
colorindex = new int[infoheader.biClrUsed]; // OOM DANGER!
colorbytes = 4 * colorindex.length;
this.colorindex = new int[infoheader.biClrUsed]; // OOM DANGER!
this.colorbytes = 4 * this.colorindex.length;
int color;
for (int i = 0; i < colorindex.length; i++) {
for (int i = 0; i < this.colorindex.length; i++) {
// translate BGR into RGB color Scheme
color = 0xffffff & DWORD(s, offset + 4 * i);
colorindex[i] = color;
this.colorindex[i] = color;
//if (debugmode) System.out.println("Color " + i + " = " + Integer.toHexString(colorindex[i]));
}
}
}
}
public static class IMAGEMAP {
private BufferedImage image;
public IMAGEMAP(final byte[] s, final int offset, final int width, final int height, final int compression, final int bitcount, final COLORTABLE colortable) {
// parse picture content
if ((width != 0) && (height != 0)) {
image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
this.image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
if (compression == BI_RGB) {
if (bitcount == 1) parseBMP1(s, offset, width, height, colortable);
else if (bitcount == 4) parseBMP4(s, offset, width, height, colortable);
@ -160,7 +160,7 @@ public class bmpParser {
}
}
}
private void parseBMP1(final byte[] s, final int offset, final int width, final int height, final COLORTABLE colortable) {
int n = 0;
int b;
@ -169,19 +169,19 @@ public class bmpParser {
if (offset + n >= s.length) return; // emergency break
b = (s[offset + n] & 0xff);
n++;
image.setRGB(columns, (height - rows - 1), colortable.colorindex[(b & 0x80) >> 7]);
image.setRGB(columns + 1, (height - rows - 1), colortable.colorindex[(b & 0x40) >> 6]);
image.setRGB(columns + 2, (height - rows - 1), colortable.colorindex[(b & 0x20) >> 5]);
image.setRGB(columns + 3, (height - rows - 1), colortable.colorindex[(b & 0x10) >> 4]);
image.setRGB(columns + 4, (height - rows - 1), colortable.colorindex[(b & 0x08) >> 3]);
image.setRGB(columns + 5, (height - rows - 1), colortable.colorindex[(b & 0x04) >> 2]);
image.setRGB(columns + 6, (height - rows - 1), colortable.colorindex[(b & 0x02) >> 1]);
image.setRGB(columns + 7, (height - rows - 1), colortable.colorindex[ b & 0x01]);
this.image.setRGB(columns, (height - rows - 1), colortable.colorindex[(b & 0x80) >> 7]);
this.image.setRGB(columns + 1, (height - rows - 1), colortable.colorindex[(b & 0x40) >> 6]);
this.image.setRGB(columns + 2, (height - rows - 1), colortable.colorindex[(b & 0x20) >> 5]);
this.image.setRGB(columns + 3, (height - rows - 1), colortable.colorindex[(b & 0x10) >> 4]);
this.image.setRGB(columns + 4, (height - rows - 1), colortable.colorindex[(b & 0x08) >> 3]);
this.image.setRGB(columns + 5, (height - rows - 1), colortable.colorindex[(b & 0x04) >> 2]);
this.image.setRGB(columns + 6, (height - rows - 1), colortable.colorindex[(b & 0x02) >> 1]);
this.image.setRGB(columns + 7, (height - rows - 1), colortable.colorindex[ b & 0x01]);
}
n += fill4(n);
}
}
private void parseBMP4(final byte[] s, final int offset, final int width, final int height, final COLORTABLE colortable) {
int n = 0;
int b;
@ -190,8 +190,8 @@ public class bmpParser {
if (offset + n >= s.length) return; // emergency break
b = (s[offset + n] & 0xff);
n++;
image.setRGB(columns, (height - rows - 1), colortable.colorindex[(b & 0xf0) >> 4]);
image.setRGB(columns + 1, (height - rows - 1), colortable.colorindex[b & 0xf]);
this.image.setRGB(columns, (height - rows - 1), colortable.colorindex[(b & 0xf0) >> 4]);
this.image.setRGB(columns + 1, (height - rows - 1), colortable.colorindex[b & 0xf]);
}
n += fill4(n);
}
@ -202,54 +202,54 @@ public class bmpParser {
for (int rows = 0; rows < height; rows++) {
for (int columns = 0; columns < width; columns++) {
if (offset + n >= s.length) return; // emergency break
image.setRGB(columns, (height - rows - 1), colortable.colorindex[(s[offset + n] & 0xff)]);
this.image.setRGB(columns, (height - rows - 1), colortable.colorindex[(s[offset + n] & 0xff)]);
n++;
}
n += fill4(n);
}
}
private void parseBMP24(final byte[] s, final int offset, final int width, final int height) {
int n = 0;
for (int rows = 0; rows < height; rows++) {
for (int columns = 0; columns < width; columns++) {
if (offset + n + 3 >= s.length) return; // emergency break
image.setRGB(columns, (height - rows - 1), 0xffffff & DWORD(s, offset + n));
this.image.setRGB(columns, (height - rows - 1), 0xffffff & DWORD(s, offset + n));
n += 3;
}
n += fill4(n);
}
}
private void parseBMP32(final byte[] s, final int offset, final int width, final int height) {
int n = 0;
for (int rows = 0; rows < height; rows++) {
for (int columns = 0; columns < width; columns++) {
if (offset + n + 3 >= s.length) return; // emergency break
image.setRGB(columns, (height - rows - 1), 0xffffff & DWORD(s, offset + n));
this.image.setRGB(columns, (height - rows - 1), 0xffffff & DWORD(s, offset + n));
n += 4;
}
}
}
private final int fill4(final int x) {
private final static int fill4(final int x) {
final int r = x % 4;
if (r == 0) return 0;
return 4 - r;
}
public BufferedImage getImage() {
return this.image;
}
}
public static void main(final String[] args) {
// read a bmp and write it as png
System.setProperty("java.awt.headless", "true");
final File in = new File(args[0]);
final File out = new File(args[1]);
final byte[] file = new byte[(int) in.length()];
FileInputStream fis = null;
try {
@ -260,7 +260,7 @@ public class bmpParser {
} catch (final IOException e) {
Log.logException(e);
}
try {
ImageIO.write(parse(file).getImage(), "PNG", out);
} catch (final IOException e) {

2
source/net/yacy/document/parser/psParser.java
Unescape View File

@ -130,7 +130,7 @@ public class psParser extends AbstractParser implements Parser {
}
}
private void parseUsingJava(final File inputFile, final File outputFile) throws Exception {
private static void parseUsingJava(final File inputFile, final File outputFile) throws Exception {
BufferedReader reader = null;
BufferedWriter writer = null;

6
source/net/yacy/document/parser/rdfa/impl/RDFaParser.java
Unescape View File

@ -65,7 +65,7 @@ public class RDFaParser extends AbstractParser implements Parser {
return htmlDocs;
}
private Document parseRDFa(DigestURI url, String mimeType,
private static Document parseRDFa(DigestURI url, String mimeType,
String charset, InputStream source) {
RDFaTripleImpl triple;
IRDFaTriple[] allTriples = null;
@ -108,7 +108,7 @@ public class RDFaParser extends AbstractParser implements Parser {
return htmlDocs;
}
private Document convertAllTriplesToDocument(DigestURI url,
private static Document convertAllTriplesToDocument(DigestURI url,
String mimeType, String charset, IRDFaTriple[] allTriples) {
//Set<String> languages = new HashSet<String>(2);
@ -142,7 +142,7 @@ public class RDFaParser extends AbstractParser implements Parser {
return doc;
}
private void addNotEmptyValuesToSet(Set<String> set, String value) {
private static void addNotEmptyValuesToSet(Set<String> set, String value) {
if (value != null) {
set.add(value);
}

4
source/net/yacy/kelondro/blob/MapColumnIndex.java
Unescape View File

@ -93,7 +93,7 @@ public class MapColumnIndex implements Serializable {
}
}
private void indexupdate(final byte[] primarykey, final Map<String, Collection<byte[]>> valueIdxMap, final String value) {
private static void indexupdate(final byte[] primarykey, final Map<String, Collection<byte[]>> valueIdxMap, final String value) {
Collection<byte[]> indexes = valueIdxMap.get(value);
if (indexes == null) {
// create a new index entry
@ -120,7 +120,7 @@ public class MapColumnIndex implements Serializable {
}
}
private void indexdelete(final byte[] index, final Map<String, Collection<byte[]>> valueIdxMap) {
private static void indexdelete(final byte[] index, final Map<String, Collection<byte[]>> valueIdxMap) {
Iterator<Map.Entry<String, Collection<byte[]>>> i = valueIdxMap.entrySet().iterator();
Map.Entry<String, Collection<byte[]>> ref;
while (i.hasNext()) {

3
source/net/yacy/kelondro/data/meta/DigestURI.java
Unescape View File

@ -346,8 +346,9 @@ public class DigestURI extends MultiProtocolURI implements Serializable {
return 14;
case 3:
return 20;
default:
return 20;
}
return 20;
}
public static int domLengthNormalized(final byte[] urlHashBytes) {

2
source/net/yacy/kelondro/index/HandleMap.java
Unescape View File

@ -137,7 +137,7 @@ public final class HandleMap implements Iterable<Row.Entry> {
return new int[]{keym, this.rowdef.width(1) - valm};
}
private final int eq(final byte[] a, final byte[] b) {
private final static int eq(final byte[] a, final byte[] b) {
for (int i = 0; i < a.length; i++) {
if (a[i] != b[i]) return i;
}

8
source/net/yacy/kelondro/index/Row.java
Unescape View File

@ -456,6 +456,8 @@ public final class Row implements Serializable {
break;
case Column.encoder_bytes:
throw new kelondroException("ROW", "setColLong of celltype bytes not applicable");
default:
throw new kelondroException("ROW", "setColLong has celltype none, no encoder given");
}
}
@ -473,8 +475,9 @@ public final class Row implements Serializable {
l = c + NaturalOrder.decodeLong(this.rowinstance, this.offset + colstrt, cellwidth);
NaturalOrder.encodeLong(l, this.rowinstance, this.offset + colstrt, cellwidth);
return l;
default:
throw new kelondroException("ROW", "addCol did not find appropriate encoding");
}
throw new kelondroException("ROW", "addCol did not find appropriate encoding");
}
public final byte[] getPrimaryKeyBytes() {
@ -560,8 +563,9 @@ public final class Row implements Serializable {
return NaturalOrder.decodeLong(this.rowinstance, this.offset + clstrt, length);
case Column.encoder_bytes:
throw new kelondroException("ROW", "getColLong of celltype bytes not applicable");
default:
throw new kelondroException("ROW", "getColLong did not find appropriate encoding");
}
throw new kelondroException("ROW", "getColLong did not find appropriate encoding");
}
public final byte getColByte(final int column) {

138
source/net/yacy/kelondro/io/CachedRecords.java
Unescape View File

@ -7,7 +7,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -54,7 +54,7 @@ import net.yacy.kelondro.util.MemoryControl;
* This class only references byte[] that are handed over to methods of this class.
*/
public final class CachedRecords {
private RandomAccessFile raf;
private final File tablefile;
/**
@ -70,14 +70,14 @@ public final class CachedRecords {
private byte[] cache;
private byte[] buffer;
private final byte[] zero;
/**
* stay below hard disc cache (is that necessary?)
*/
private static final int maxReadCache = 16 * 1024;
private static final int maxWriteBuffer = 16 * 1024;
public CachedRecords(final File tablefile, final int recordsize) throws IOException {
this.tablefile = tablefile;
this.recordsize = recordsize;
@ -85,7 +85,7 @@ public final class CachedRecords {
// initialize zero buffer
this.zero = new byte[recordsize];
for (int i = 0; i < recordsize; i++) this.zero[i] = 0;
// initialize table file
if (!tablefile.exists()) {
// make new file
@ -98,15 +98,15 @@ public final class CachedRecords {
}
try { if (fos != null) fos.close(); } catch (final IOException e) {}
}
// open an existing table file
try {
raf = new RandomAccessFile(tablefile,"rw");
this.raf = new RandomAccessFile(tablefile,"rw");
} catch (final FileNotFoundException e) {
// should never happen
Log.logException(e);
}
// initialize cache and buffer
int cachesize = Math.max(1, (maxReadCache / recordsize)) * recordsize;
int buffersize = Math.max(1, (maxWriteBuffer / recordsize)) * recordsize;
@ -118,15 +118,15 @@ public final class CachedRecords {
buffersize = Math.max(1, (int) (lessmem / recordsize)) * recordsize;
//System.out.println("newmem nachher: cachesize = " + cachesize + ", buffersize = " + buffersize);
}
this.cache = new byte[cachesize];
this.buffer = new byte[buffersize];
this.buffercount = 0;
// first-time read of cache
fillCache(0);
}
/**
* @param tablefile
* @param recordsize
@ -138,7 +138,7 @@ public final class CachedRecords {
if (size % recordsize != 0) throw new IOException("wrong file size: file = " + tablefile + ", size = " + size + ", recordsize = " + recordsize);
return size / recordsize;
}
public final static void fixTableSize(final File tablefile, final long recordsize) throws IOException {
if (!tablefile.exists()) return;
final long size = tablefile.length();
@ -149,7 +149,7 @@ public final class CachedRecords {
raf.close();
}
}
/**
* @return the number of records in file plus number of records in buffer
* @throws IOException
@ -157,24 +157,24 @@ public final class CachedRecords {
public final synchronized long size() throws IOException {
return filesize() + this.buffercount;
}
public final File filename() {
return this.tablefile;
}
/**
* @return records in file
* @throws IOException
*/
private final long filesize() throws IOException {
return raf.length() / recordsize;
return this.raf.length() / this.recordsize;
}
/**
* checks if the index is inside the cache
*
*
* @param index
* @return the index offset inside the cache or -1 if the index is not in the cache
* @return the index offset inside the cache or -1 if the index is not in the cache
*/
private final int inCache(final long index) {
if (index >= this.cacheindex && index < this.cacheindex + this.cachecount) {
@ -182,12 +182,12 @@ public final class CachedRecords {
}
return -1;
}
/**
* checks if the index is inside the buffer
*
*
* @param index
* @return the index offset inside the buffer or -1 if the index is not in the buffer
* @return the index offset inside the buffer or -1 if the index is not in the buffer
* @throws IOException
*/
private final int inBuffer(final long index) throws IOException {
@ -197,13 +197,13 @@ public final class CachedRecords {
}
return -1;
}
/**
* load cache with copy of disc content; start with record at index
*
*
* if the record would overlap with the write buffer,
* its start is shifted forward until it fits
*
*
* @param index
* @throws IOException
*/
@ -212,46 +212,46 @@ public final class CachedRecords {
// first check if the index is inside the current cache
assert inCache(index) < 0;
if (inCache(index) >= 0) return;
// calculate new start position
final long fs = this.filesize();
if (index + this.cache.length / this.recordsize > fs) {
index = fs - this.cache.length / this.recordsize;
}
if (index < 0) index = 0;
// calculate number of records that shall be stored in the cache
this.cachecount = (int) Math.min(this.cache.length / this.recordsize, this.filesize() - index);
assert this.cachecount >= 0;
// check if we need to read 0 bytes from the file
this.cacheindex = index;
if (this.cachecount == 0) return;
// copy records from file to cache
raf.seek(this.recordsize * index);
raf.readFully(this.cache, 0, this.recordsize * this.cachecount);
this.raf.seek(this.recordsize * index);
this.raf.readFully(this.cache, 0, this.recordsize * this.cachecount);
}
/**
* write buffer to end of file
* write buffer to end of file
*/
public final void flushBuffer() {
try {
raf.seek(raf.length());
raf.write(this.buffer, 0, this.recordsize * this.buffercount);
this.raf.seek(this.raf.length());
this.raf.write(this.buffer, 0, this.recordsize * this.buffercount);
} catch (final IOException e) {
Log.logException(e);
}
this.buffercount = 0;
}
public final synchronized void close() {
flushBuffer();
// then close the file
if (raf != null) try {
raf.close();
if (this.raf != null) try {
this.raf.close();
} catch (final IOException e) {
Log.logException(e);
}
@ -280,12 +280,12 @@ public final class CachedRecords {
}
if (p >= 0) {
// read entry from the cache
System.arraycopy(this.cache, p * this.recordsize, b, start, this.recordsize);
System.arraycopy(this.cache, p * this.recordsize, b, start, this.recordsize);
return;
}
if (q >= 0) {
// read entry from the buffer
System.arraycopy(this.buffer, q * this.recordsize, b, start, this.recordsize);
System.arraycopy(this.buffer, q * this.recordsize, b, start, this.recordsize);
return;
}
assert false;
@ -295,21 +295,21 @@ public final class CachedRecords {
assert b.length - start >= this.recordsize;
final long s = filesize() + this.buffercount;
if (index > s) throw new IndexOutOfBoundsException("kelondroEcoFS.put(" + index + ") outside bounds (" + this.size() + ")");
// check if this is an empty entry
if (isClean(b , start, this.recordsize)) {
clean(index);
return;
}
// check if index is inside of cache
final int p = inCache(index);
final int q = (p >= 0) ? -1 : inBuffer(index);
if (p >= 0) {
// write entry to the cache and to the file
System.arraycopy(b, start, this.cache, p * this.recordsize, this.recordsize);
raf.seek(index * this.recordsize);
raf.write(b, start, this.recordsize);
this.raf.seek(index * this.recordsize);
this.raf.write(b, start, this.recordsize);
return;
}
if (q >= 0) {
@ -319,7 +319,7 @@ public final class CachedRecords {
}
if (index == s) {
// append the record to the end of the file;
// look if there is space in the buffer
if (this.buffercount >= this.buffer.length / this.recordsize) {
assert this.buffercount == this.buffer.length / this.recordsize;
@ -337,23 +337,23 @@ public final class CachedRecords {
} else {
// write the record directly to the file,
// do not care about the cache; this case was checked before
raf.seek(index * this.recordsize);
raf.write(b, start, this.recordsize);
this.raf.seek(index * this.recordsize);
this.raf.write(b, start, this.recordsize);
}
}
public final synchronized void add(final byte[] b, final int start) throws IOException {
// index == size() == filesize() + (long) this.buffercount
assert b.length - start >= this.recordsize;
// check if this is an empty entry
if (isClean(b , start, this.recordsize)) {
// it is not possible to add a clean record at the end of a EcoFS, because
// such records should cause the record to shrink
throw new IOException("add: record at end is clean");
}
// append the record to the end of the file;
// look if there is space in the buffer
if (this.buffercount >= this.buffer.length / this.recordsize) {
@ -370,14 +370,14 @@ public final class CachedRecords {
}
assert this.buffercount <= this.buffer.length / this.recordsize;
}
private final boolean isClean(final byte[] b, final int offset, final int length) {
private final static boolean isClean(final byte[] b, final int offset, final int length) {
for (int i = 0; i < length; i++) {
if (b[i + offset] != 0) return false;
}
return true;
}
private final boolean isClean(final long index) throws IOException {
assert index < size();
// check if index is inside of cache
@ -413,27 +413,27 @@ public final class CachedRecords {
cleanLast();
return;
}
// check if index is inside of cache
final int p = inCache(index);
final int q = (p >= 0) ? -1 : inBuffer(index);
if (p >= 0) {
// write zero bytes to the cache and to the file
System.arraycopy(zero, 0, this.cache, p * this.recordsize, this.recordsize);
raf.seek(index * this.recordsize);
raf.write(zero, 0, this.recordsize);
System.arraycopy(this.zero, 0, this.cache, p * this.recordsize, this.recordsize);
this.raf.seek(index * this.recordsize);
this.raf.write(this.zero, 0, this.recordsize);
return;
}
if (q >= 0) {
// write zero to the buffer
System.arraycopy(zero, 0, this.buffer, q * this.recordsize, this.recordsize);
System.arraycopy(this.zero, 0, this.buffer, q * this.recordsize, this.recordsize);
return;
}
raf.seek(index * this.recordsize);
raf.write(zero, 0, this.recordsize);
this.raf.seek(index * this.recordsize);
this.raf.write(this.zero, 0, this.recordsize);
}
/**
* @see clean(long, byte[], int)
* @param b
@ -449,11 +449,11 @@ public final class CachedRecords {
//System.out.println(" after size = " + size());
}
}
/**
* this is like
* <code>clean(this.size() - 1, b, start);</code>
*
*
* @see clean(long, byte[], int)
* @param b
* @param start
@ -473,7 +473,7 @@ public final class CachedRecords {
}
if (p >= 0) {
// read entry from the cache
System.arraycopy(this.cache, p * this.recordsize, b, start, this.recordsize);
System.arraycopy(this.cache, p * this.recordsize, b, start, this.recordsize);
// shrink cache and file
assert this.buffercount == 0;
this.raf.setLength((s - 1) * this.recordsize);
@ -490,7 +490,7 @@ public final class CachedRecords {
}
assert false;
}
/**
* @see clean(long, byte[], int)
* @throws IOException
@ -504,7 +504,7 @@ public final class CachedRecords {
//System.out.println(" after size = " + size());
}
}
private final synchronized void cleanLast0() throws IOException {
// check if index is inside of cache
@ -528,7 +528,7 @@ public final class CachedRecords {
assert this.buffercount == 0;
this.raf.setLength((s - 1) * this.recordsize);
}
public final void deleteOnExit() {
this.tablefile.deleteOnExit();
}
@ -577,7 +577,7 @@ public final class CachedRecords {
}
System.out.println("size() needs " + ((System.currentTimeMillis() - start) / 100) + " nanoseconds");
System.out.println("size = " + c);
t.close();
} catch (final IOException e) {
Log.logException(e);

21
source/net/yacy/kelondro/io/RandomAccessIO.java
Unescape View File

@ -1,4 +1,4 @@
// RandomAccessIO.java
// RandomAccessIO.java
// -----------------------
// part of The Kelondro Database
// (C) by Michael Peter Christen; mc@yacy.net
@ -30,18 +30,18 @@ public final class RandomAccessIO {
protected final Writer ra;
protected final String name;
public RandomAccessIO(final Writer ra, final String name) {
this.name = name;
this.ra = ra;
}
public final Writer getRA() {
return this.ra;
}
public final synchronized long length() throws IOException {
return ra.length();
return this.ra.length();
}
public final synchronized void readFully(long pos, final byte[] b, int off, int len) throws IOException {
@ -57,9 +57,9 @@ public final class RandomAccessIO {
}
public final String name() {
return name;
return this.name;
}
public final synchronized byte readByte(final long pos) throws IOException {
final byte[] b = new byte[1];
this.readFully(pos, b, 0, 1);
@ -117,7 +117,7 @@ public final class RandomAccessIO {
public final synchronized void write(final long pos, final byte[] b) throws IOException {
this.write(pos, b, 0, b.length);
}
public final synchronized void writeSpace(long pos, int spaceCount) throws IOException {
if (spaceCount < 512) {
write(pos, space(spaceCount));
@ -133,17 +133,18 @@ public final class RandomAccessIO {
write(pos, space(spaceCount));
}
}
private final byte[] space(int count) {
private final static byte[] space(int count) {
byte[] s = new byte[count];
while (count-- > 0) s[count] = 0;
return s;
}
public final synchronized void close() throws IOException {
if (this.ra != null) this.ra.close();
}
@Override
protected final void finalize() throws Throwable {
if (this.ra != null) this.close();
super.finalize();

118
source/net/yacy/kelondro/io/Records.java
Unescape View File

@ -7,7 +7,7 @@
// $LastChangedBy$
//
// LICENSE
//
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@ -54,7 +54,7 @@ import net.yacy.kelondro.util.MemoryControl;
* This class only references byte[] that are handed over to methods of this class.
*/
public final class Records {
private RandomAccessFile raf;
private final File tablefile;
/**
@ -67,13 +67,13 @@ public final class Records {
private int buffercount;
private byte[] buffer;
private final byte[] zero;
/**
* stay below hard disc cache (is that necessary?)
*/
private static final int maxWriteBuffer = 16 * 1024;
public Records(final File tablefile, final int recordsize) {
this.tablefile = tablefile;
this.recordsize = recordsize;
@ -81,7 +81,7 @@ public final class Records {
// initialize zero buffer
this.zero = new byte[recordsize];
for (int i = 0; i < recordsize; i++) this.zero[i] = 0;
// initialize table file
if (!tablefile.exists()) {
// make new file
@ -94,15 +94,15 @@ public final class Records {
}
try { if (fos != null) fos.close(); } catch (final IOException e) {}
}
// open an existing table file
try {
raf = new RandomAccessFile(tablefile,"rw");
this.raf = new RandomAccessFile(tablefile,"rw");
} catch (final FileNotFoundException e) {
// should never happen
Log.logException(e);
}
// initialize write buffer
int buffersize = Math.max(1, (maxWriteBuffer / recordsize)) * recordsize;
if (!MemoryControl.request(buffersize + 1024 * 1024 * 20, true)) {
@ -112,11 +112,11 @@ public final class Records {
buffersize = Math.max(1, (int) (lessmem / recordsize)) * recordsize;
//System.out.println("newmem nachher: cachesize = " + cachesize + ", buffersize = " + buffersize);
}
this.buffer = new byte[buffersize];
this.buffercount = 0;
}
/**
* @param tablefile
* @param recordsize
@ -128,7 +128,7 @@ public final class Records {
if (size % recordsize != 0) throw new IOException("wrong file size: file = " + tablefile + ", size = " + size + ", recordsize = " + recordsize);
return size / recordsize;
}
public final static void fixTableSize(final File tablefile, final long recordsize) throws IOException {
if (!tablefile.exists()) return;
final long size = tablefile.length();
@ -139,7 +139,7 @@ public final class Records {
raf.close();
}
}
/**
* @return the number of records in file plus number of records in buffer
* @throws IOException
@ -147,11 +147,11 @@ public final class Records {
public final synchronized long size() throws IOException {
return filesize() + this.buffercount;
}
public final File filename() {
return this.tablefile;
}
/**
* @return records in file
* @throws IOException
@ -160,7 +160,7 @@ public final class Records {
long records = 0;
try {
records = raf.length() / recordsize;
records = this.raf.length() / this.recordsize;
} catch (NullPointerException e) {
// This may happen on shutdown while still something is moving on
Log.logException(e);
@ -168,12 +168,12 @@ public final class Records {
return records;
}
/**
* checks if the index is inside the buffer
*
*
* @param index
* @return the index offset inside the buffer or -1 if the index is not in the buffer
* @return the index offset inside the buffer or -1 if the index is not in the buffer
* @throws IOException
*/
private final int inBuffer(final long index) throws IOException {
@ -185,24 +185,24 @@ public final class Records {
}
/**
* write buffer to end of file
* write buffer to end of file
*/
protected final synchronized void flushBuffer() {
if (raf == null) return;
if (this.raf == null) return;
try {
raf.seek(raf.length());
raf.write(this.buffer, 0, this.recordsize * this.buffercount);
this.raf.seek(this.raf.length());
this.raf.write(this.buffer, 0, this.recordsize * this.buffercount);
} catch (final IOException e) {
Log.logException(e);
}
this.buffercount = 0;
}
public final synchronized void close() {
// close the file
if (raf != null) try {
if (this.raf != null) try {
flushBuffer();
raf.close();
this.raf.close();
} catch (final IOException e) {
Log.logException(e);
}
@ -223,25 +223,25 @@ public final class Records {
final int q = inBuffer(index);
if (q < 0) {
// copy records from file to given buffer
raf.seek(this.recordsize * index);
raf.readFully(b, start, this.recordsize);
this.raf.seek(this.recordsize * index);
this.raf.readFully(b, start, this.recordsize);
return;
}
// read entry from the buffer
System.arraycopy(this.buffer, q * this.recordsize, b, start, this.recordsize);
System.arraycopy(this.buffer, q * this.recordsize, b, start, this.recordsize);
}
public final synchronized void put(final long index, final byte[] b, final int start) throws IOException {
assert b.length - start >= this.recordsize;
final long s = filesize() + this.buffercount;
if (index > s) throw new IndexOutOfBoundsException("kelondroEcoFS.put(" + index + ") outside bounds (" + this.size() + ")");
// check if this is an empty entry
if (isClean(b , start, this.recordsize)) {
clean(index);
return;
}
// check if index is inside of cache
final int q = inBuffer(index);
if (q >= 0) {
@ -251,7 +251,7 @@ public final class Records {
}
if (index == s) {
// append the record to the end of the file;
// look if there is space in the buffer
if (this.buffercount >= this.buffer.length / this.recordsize) {
assert this.buffercount == this.buffer.length / this.recordsize;
@ -269,23 +269,23 @@ public final class Records {
} else {
// write the record directly to the file,
// do not care about the cache; this case was checked before
raf.seek(index * this.recordsize);
raf.write(b, start, this.recordsize);
this.raf.seek(index * this.recordsize);
this.raf.write(b, start, this.recordsize);
}
}
public final synchronized void add(final byte[] b, final int start) throws IOException {
// index == size() == filesize() + (long) this.buffercount
assert b.length - start >= this.recordsize;
// check if this is an empty entry
if (isClean(b , start, this.recordsize)) {
// it is not possible to add a clean record at the end of a EcoFS, because
// such records should cause the record to shrink
throw new IOException("add: record at end is clean");
}
// append the record to the end of the file;
// look if there is space in the buffer
if (this.buffercount >= this.buffer.length / this.recordsize) {
@ -302,14 +302,14 @@ public final class Records {
}
assert this.buffercount <= this.buffer.length / this.recordsize;
}
private final boolean isClean(final byte[] b, final int offset, final int length) {
private final static boolean isClean(final byte[] b, final int offset, final int length) {
for (int i = 0; i < length; i++) {
if (b[i + offset] != 0) return false;
}
return true;
}
private final boolean isClean(final long index) throws IOException {
assert index < size();
// check if index is inside of buffer
@ -319,8 +319,8 @@ public final class Records {
return isClean(this.buffer, q * this.recordsize, this.recordsize);
}
byte[] b = new byte[this.recordsize];
raf.seek(index * this.recordsize);
raf.readFully(b, 0, this.recordsize);
this.raf.seek(index * this.recordsize);
this.raf.readFully(b, 0, this.recordsize);
return isClean(b, 0, this.recordsize);
}
@ -336,19 +336,19 @@ public final class Records {
cleanLast();
return;
}
// check if index is inside of cache
final int q = inBuffer(index);
if (q >= 0) {
// write zero to the buffer
System.arraycopy(zero, 0, this.buffer, q * this.recordsize, this.recordsize);
System.arraycopy(this.zero, 0, this.buffer, q * this.recordsize, this.recordsize);
return;
}
raf.seek(index * this.recordsize);
raf.write(zero, 0, this.recordsize);
this.raf.seek(index * this.recordsize);
this.raf.write(this.zero, 0, this.recordsize);
}
/**
* @see clean(long, byte[], int)
* @param b
@ -364,11 +364,11 @@ public final class Records {
//System.out.println(" after size = " + size());
}
}
/**
* this is like
* <code>clean(this.size() - 1, b, start);</code>
*
*
* @see clean(long, byte[], int)
* @param b
* @param start
@ -386,17 +386,17 @@ public final class Records {
}
// read entry from the file
final long endpos = this.raf.length() - this.recordsize;
raf.seek(endpos);
raf.readFully(b, start, this.recordsize);
this.raf.seek(endpos);
this.raf.readFully(b, start, this.recordsize);
// write zero bytes to the cache and to the file
this.raf.seek(endpos);
this.raf.write(zero, 0, this.recordsize);
this.raf.write(this.zero, 0, this.recordsize);
// shrink file
this.raf.setLength(endpos);
}
/**
* @see clean(long, byte[], int)
* @throws IOException
@ -410,7 +410,7 @@ public final class Records {
//System.out.println(" after size = " + size());
}
}
private final void cleanLast0() throws IOException {
// check if index is inside of cache
@ -420,9 +420,9 @@ public final class Records {
return;
}
// shrink file
this.raf.setLength(raf.length() - this.recordsize);
this.raf.setLength(this.raf.length() - this.recordsize);
}
public final void deleteOnExit() {
this.tablefile.deleteOnExit();
}
@ -471,7 +471,7 @@ public final class Records {
}
System.out.println("size() needs " + ((System.currentTimeMillis() - start) / 100) + " nanoseconds");
System.out.println("size = " + c);
t.close();
} catch (final IOException e) {
Log.logException(e);

85
source/net/yacy/kelondro/logging/GuiHandler.java
Unescape View File

@ -1,4 +1,4 @@
//GuiHandler.java
//GuiHandler.java
//-------------------------------------
//part of YACY
//(C) by Michael Peter Christen; mc@yacy.net
@ -41,38 +41,38 @@ import java.util.logging.SimpleFormatter;
public class GuiHandler extends Handler {
private final static int DEFAULT_SIZE = 400;
private int size = DEFAULT_SIZE;
private int size = DEFAULT_SIZE;
private LogRecord buffer[];
int start, count;
public GuiHandler() {
super();
configure();
init();
}
}
/**
* Get any configuration properties set
*/
private void configure() {
final LogManager manager = LogManager.getLogManager();
final String className = getClass().getName();
final String level = manager.getProperty(className + ".level");
setLevel((level == null) ? Level.INFO : Level.parse(level));
final String filter = manager.getProperty(className + ".filter");
setFilter(makeFilter(filter));
final String formatter = manager.getProperty(className + ".formatter");
setFormatter(makeFormatter(formatter));
final String sizeString = manager.getProperty(className + ".size");
this.size = parseSize(sizeString);
}
private int parseSize(final String sizeString) {
}
private static int parseSize(final String sizeString) {
int newSize = DEFAULT_SIZE;
try {
newSize = Integer.parseInt(sizeString);
@ -81,10 +81,10 @@ public class GuiHandler extends Handler {
}
return newSize;
}
private Filter makeFilter(final String name) {
private static Filter makeFilter(final String name) {
if (name == null) return null;
Filter f = null;
try {
final Class<?> c = Class.forName(name);
@ -93,11 +93,11 @@ public class GuiHandler extends Handler {
System.err.println("Unable to load filter: " + name);
}
return f;
}
private Formatter makeFormatter(final String name) {
}
private static Formatter makeFormatter(final String name) {
if (name == null) return null;
Formatter f = null;
try {
final Class<?> c = Class.forName(name);
@ -107,21 +107,22 @@ public class GuiHandler extends Handler {
}
return f;
}
// Initialize. Size is a count of LogRecords.
private void init() {
this.buffer = new LogRecord[this.size];
this.start = 0;
this.count = 0;
}
public final int getSize() {
return this.size;
}
@Override
public final void publish(final LogRecord record) {
if (!isLoggable(record)) return;
// write it to the buffer
final int ix = (this.start+this.count)%this.buffer.length;
this.buffer[ix] = record;
@ -132,7 +133,7 @@ public class GuiHandler extends Handler {
}
flush();
}
public final synchronized LogRecord[] getLogArray() {
return this.getLogArray(null);
}
@ -140,7 +141,7 @@ public class GuiHandler extends Handler {
public final synchronized LogRecord[] getLogArray(final Long sequenceNumberStart) {
final List<LogRecord> tempBuffer = new ArrayList<LogRecord>(this.count);
for (int i = 0; i < this.count; i++) {
final int ix = (this.start+i)%this.buffer.length;
final LogRecord record = this.buffer[ix];
@ -148,17 +149,17 @@ public class GuiHandler extends Handler {
tempBuffer.add(record);
}
}
return tempBuffer.toArray(new LogRecord[tempBuffer.size()]);
}
public final synchronized String getLog(final boolean reversed, int lineCount) {
}
public final synchronized String getLog(final boolean reversed, int lineCount) {
if ((lineCount > this.count)||(lineCount < 0)) lineCount = this.count;
final StringBuilder logMessages = new StringBuilder(this.count*40);
final Formatter logFormatter = getFormatter();
try {
final int theStart = (reversed)?this.start+this.count-1:this.start;
LogRecord record=null;
@ -177,14 +178,14 @@ public class GuiHandler extends Handler {
return "Error while formatting the logging message";
}
}
public final synchronized String[] getLogLines(final boolean reversed, int lineCount) {
public final synchronized String[] getLogLines(final boolean reversed, int lineCount) {
if ((lineCount > this.count)||(lineCount < 0)) lineCount = this.count;
final List<String> logMessages = new ArrayList<String>(this.count);
final Formatter logFormatter = getFormatter();
try {
final int theStart = (reversed) ? this.start+this.count-1 : this.start+this.count-lineCount;
LogRecord record=null;
@ -202,12 +203,14 @@ public class GuiHandler extends Handler {
reportError(null, ex, ErrorManager.FORMAT_FAILURE);
return new String[]{"Error while formatting the logging message"};
}
}
}
@Override
public void flush() {
}
@Override
public synchronized void close() throws SecurityException {
// Nothing implement here
}

520
source/net/yacy/kelondro/logging/LogParser.java
Unescape View File

@ -1,4 +1,4 @@
//LogParser.java
//LogParser.java
//-------------------------------------
//part of YACY
//(C) by Michael Peter Christen; mc@yacy.net
@ -36,137 +36,137 @@ import java.util.regex.Pattern;
public final class LogParser {
/** the version of the LogParser - <strong>Double</strong>*/
public static final String PARSER_VERSION = "version";
/** the amount of URLs received during DHT - <strong>Integer</strong> */
public static final String URLS_RECEIVED = "urlSum";
/** the amount of URLs requested during DHT - <strong>Integer</strong> */
public static final String URLS_REQUESTED = "urlReqSum";
/** the amount of URLs blocked during DHT because they match the peer's blacklist - <strong>Integer</strong> */
public static final String URLS_BLOCKED = "blockedURLSum";
/** the amount of words received during DHT - <strong>Integer</strong> */
public static final String WORDS_RECEIVED = "wordsSum";
/** the amount of RWIs received during DHT - <strong>Integer</strong> */
public static final String RWIS_RECEIVED = "rwiSum";
/** the amount of RWIs blocked during DHT because their entries match the peer's blacklist - <strong>Integer</strong> */
public static final String RWIS_BLOCKED = "blockedRWISum";
/** total time receiving RWIs during DHT in milli seconds - <strong>Long</strong> */
public static final String RWIS_RECEIVED_TIME = "rwiTimeSum";
/** total time receiving URLs during DHT in milli seconds - <strong>Long</strong> */
public static final String URLS_RECEIVED_TIME = "urlTimeSum";
/** the traffic sent during DHT in bytes - <strong>Long</strong> */
public static final String DHT_TRAFFIC_SENT = "DHTSendTraffic";
/** the amount of URLs requested by other peers and sent by this one - <strong>Integer</strong> */
public static final String DHT_URLS_SENT = "DHTSendURLs";
/** the amount of rejected DHT transfers from other peers (i.e. because this peer was busy) - <strong>Integer</strong> */
public static final String DHT_REJECTED = "RWIRejectCount";
/** the peer-names from whose DHT transfers were rejected - <strong>HasSet</strong> */
public static final String DHT_REJECTED_PEERS_NAME = "DHTRejectPeerNames";
/** the peer-hashes from whose DHT transfers were rejected - <strong>HasSet</strong> */
public static final String DHT_REJECTED_PEERS_HASH = "DHTRejectPeerHashs";
/** the peer-names this peer sent DHT chunks to - <strong>HasSet</strong> */
public static final String DHT_SENT_PEERS_NAME = "DHTPeerNames";
/** the peer-hashes this peer sent DHT chunks to - <strong>HasSet</strong> */
public static final String DHT_SENT_PEERS_HASH = "DHTPeerHashs";
/** total amount of selected peers for index distribution - <strong>Integer</strong> */
public static final String DHT_SELECTED = "DHTSelectionTargetCount";
/** total amount of words selected for index distribution - <strong>Integer</strong> */
public static final String DHT_WORDS_SELECTED = "DHTSelectionWordsCount";
/** total time selecting words for index distribution - <strong>Integer</strong> */
public static final String DHT_WORDS_SELECTED_TIME = "DHTSelectionWordsTimeCount";
/** the minimal DHT distance during peer-selection for index distribution - <strong>Long</strong> */
public static final String DHT_DISTANCE_MIN = "minDHTDist";
/** the maximal DHT distance during peer-selection for index distribution - <strong>Long</strong> */
public static final String DHT_DISTANCE_MAX = "maxDHTDist";
/** the average DHT distance during peer-selection for index distribution - <strong>Long</strong> */
public static final String DHT_DISTANCE_AVERAGE = "avgDHTDist";
/** how many times remote peers were too busy to accept the index transfer - <strong>Integer</strong> */
public static final String PEERS_BUSY = "busyPeerCount";
/** how many times not enough peers for index distribution were found - <strong>Integer</strong> */
public static final String PEERS_TOO_LESS = "notEnoughDHTPeers";
/** how many times the index distribution failed (i.e. due to time-out or other reasons) - <strong>Integer</strong> */
public static final String DHT_SENT_FAILED = "failedIndexDistributionCount";
/** how many times the error "<code>tried to create left child-node twice</code>" occured - <strong>Integer</strong> */
public static final String ERROR_CHILD_TWICE_LEFT = "leftChildTwiceCount";
/** how many times the error "<code>tried to create right child-node twice</code>" occured - <strong>Integer</strong> */
public static final String ERROR_CHILD_TWICE_RIGHT = "rightChildTwiceCount";
/** how many ranking distributions were executed successfully - <strong>Integer</strong> */
public static final String RANKING_DIST = "rankingDistributionCount";
/** total time the ranking distributions took - <strong>Integer</strong> */
public static final String RANKING_DIST_TIME = "rankingDistributionTime";
/** how many ranking distributions failed - <strong>Integer</strong> */
public static final String RANKING_DIST_FAILED = "rankingDistributionFailCount";
/** how many times the error "<code>Malformed URL</code>" occured - <strong>Integer</strong> */
public static final String ERROR_MALFORMED_URL = "malformedURLCount";
/** the amount of indexed sites - <strong>Integer</strong> */
public static final String INDEXED_SITES = "indexedSites";
/** total amount of indexed words - <strong>Integer</strong> */
public static final String INDEXED_WORDS = "indexedWords";
/** total size of all indexed sites - <strong>Integer</strong> */
public static final String INDEXED_SITES_SIZE = "indexedSiteSizeSum";
/** total amount of indexed anchors - <strong>Integer</strong> */
public static final String INDEXED_ANCHORS = "indexedAnchors";
// /** total time needed for stacking the site of an indexing - <strong>Integer</strong> */
// public static final String INDEXED_STACK_TIME = "indexedStackingTime";
//
//
// /** total time needed for parsing during indexing - <strong>Integer</strong> */
// public static final String INDEXED_PARSE_TIME = "indexedParsingTime";
//
//
// /** total time needed for the actual indexing during indexing - <strong>Integer</strong> */
// public static final String INDEXED_INDEX_TIME = "indexedIndexingTime";
//
//
// /** total time needed for storing the results of an indexing - <strong>Integer</strong> */
// public static final String INDEXED_STORE_TIME = "indexedStorageTime";
/** total time needed for storing the results of a link indexing - <strong>Integer</strong> */
public static final String INDEXED_LINKSTORE_TIME = "indexedLinkStorageTime";
/** total time needed for storing the results of a word indexing - <strong>Integer</strong> */
public static final String INDEXED_INDEXSTORE_TIME = "indexedIndexStorageTime";
/** accumulated time needed to parse the log entries up to now (in ms)*/
public static final String TOTAL_PARSER_TIME = "totalParserTime";
/** times the parser was called, respectively amount of independent log-lines */
public static final String TOTAL_PARSER_RUNS = "totalParserRuns";
private static final float parserVersion = 0.1f;
private static final String parserType = "PLASMA";
@ -196,7 +196,7 @@ public final class LogParser {
//RegExp for advancedParser
//private Pattern adv1 = Pattern.compile("\\*Indexed (\\d*) words in URL [\\w:.&?/%-=]* \\[[\\w-_]{12}\\]\\n\\tDescription: ([\\w- ]*)\\n\\tMimeType: ([\\w-_/]*) \\| Size: (\\d*) bytes \\| Anchors: (\\d*)\\n\\tStackingTime: (\\d*) ms \\| ParsingTime: (\\d*) ms \\| IndexingTime: (\\d*) ms \\| StorageTime: (\\d*) ms");
private final static Pattern adv1 = Pattern.compile(
"\\*Indexed (\\d+) words in URL [\\w:.&/%-~;$\u00A7@=]* \\[[\\w_-]{12}\\]\\r?\\n?" +
"\\*Indexed (\\d+) words in URL [\\w:.&/%-~;$\u00A7@=]* \\[[\\w_-]{12}\\]\\r?\\n?" +
"\\tDescription: +([\\w-\\.,:!='\"|/+@\\(\\) \\t]*)\\r?\\n?" +
"\\tMimeType: ([\\w_~/-]*) \\| Charset: ([\\w-]*) \\| Size: (\\d+) bytes \\| Anchors: (\\d+)\\r?\\n?" +
"\\tLinkStorageTime: (\\d+) ms \\| indexStorageTime: (\\d+) ms");
@ -244,292 +244,292 @@ public final class LogParser {
// private int indexedStorageTime = 0;
private long totalParserTime = 0;
private int totalParserRuns = 0;
public final int parse(final String logLevel, final String logLine) {
final long start = System.currentTimeMillis();
if ("INFO".equals(logLevel)){
m = i1.matcher (logLine);
if (m.find () && m.groupCount() >= 3) {
urlSum += Integer.parseInt(m.group(1));
urlTimeSum += Integer.parseInt(m.group(2));
blockedURLSum += Integer.parseInt(m.group(3));
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i1.matcher (logLine);
if (this.m.find () && this.m.groupCount() >= 3) {
this.urlSum += Integer.parseInt(this.m.group(1));
this.urlTimeSum += Integer.parseInt(this.m.group(2));
this.blockedURLSum += Integer.parseInt(this.m.group(3));
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = i2.matcher (logLine);
if (m.find () && m.groupCount() >= 6) {
rwiSum += Integer.parseInt(m.group(1));
wordsSum += Integer.parseInt(m.group(2));
rwiTimeSum += Integer.parseInt(m.group(3));
urlReqSum += Integer.parseInt(m.group(4));
blockedRWISum += Integer.parseInt(m.group(6));
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i2.matcher (logLine);
if (this.m.find () && this.m.groupCount() >= 6) {
this.rwiSum += Integer.parseInt(this.m.group(1));
this.wordsSum += Integer.parseInt(this.m.group(2));
this.rwiTimeSum += Integer.parseInt(this.m.group(3));
this.urlReqSum += Integer.parseInt(this.m.group(4));
this.blockedRWISum += Integer.parseInt(this.m.group(6));
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = i2_2.matcher (logLine);
if (m.find () && m.groupCount() >= 6) {
rwiSum += Integer.parseInt(m.group(1));
wordsSum += Integer.parseInt(m.group(2));
rwiTimeSum += Integer.parseInt(m.group(3));
urlReqSum += Integer.parseInt(m.group(4));
blockedRWISum += Integer.parseInt(m.group(6));
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i2_2.matcher (logLine);
if (this.m.find () && this.m.groupCount() >= 6) {
this.rwiSum += Integer.parseInt(this.m.group(1));
this.wordsSum += Integer.parseInt(this.m.group(2));
this.rwiTimeSum += Integer.parseInt(this.m.group(3));
this.urlReqSum += Integer.parseInt(this.m.group(4));
this.blockedRWISum += Integer.parseInt(this.m.group(6));
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = i3.matcher (logLine);
if (m.find () && m.groupCount() >= 6) {
DHTSendTraffic += Integer.parseInt(m.group(6));
DHTPeerNames.add(m.group(2));
DHTPeerHashs.add(m.group(3));
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i3.matcher (logLine);
if (this.m.find () && this.m.groupCount() >= 6) {
this.DHTSendTraffic += Integer.parseInt(this.m.group(6));
this.DHTPeerNames.add(this.m.group(2));
this.DHTPeerHashs.add(this.m.group(3));
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = i4.matcher (logLine);
if (m.find () && m.groupCount() >= 8) {
DHTSendTraffic += Integer.parseInt(m.group(8));
DHTSendURLs += Integer.parseInt(m.group(3));
DHTPeerNames.add(m.group(4));
DHTPeerHashs.add(m.group(5));
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i4.matcher (logLine);
if (this.m.find () && this.m.groupCount() >= 8) {
this.DHTSendTraffic += Integer.parseInt(this.m.group(8));
this.DHTSendURLs += Integer.parseInt(this.m.group(3));
this.DHTPeerNames.add(this.m.group(4));
this.DHTPeerHashs.add(this.m.group(5));
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = i5.matcher (logLine);
if (m.find () && m.groupCount() >= 4) {
minDHTDist = Math.min(minDHTDist, Math.min(Long.parseLong(m.group(3)), Long.parseLong(m.group(4))));
maxDHTDist = Math.max(maxDHTDist, Math.max(Long.parseLong(m.group(3)), Long.parseLong(m.group(4))));
avgDHTDist += Long.parseLong(m.group(3));
DHTSelectionTargetCount++;
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i5.matcher (logLine);
if (this.m.find () && this.m.groupCount() >= 4) {
this.minDHTDist = Math.min(this.minDHTDist, Math.min(Long.parseLong(this.m.group(3)), Long.parseLong(this.m.group(4))));
this.maxDHTDist = Math.max(this.maxDHTDist, Math.max(Long.parseLong(this.m.group(3)), Long.parseLong(this.m.group(4))));
this.avgDHTDist += Long.parseLong(this.m.group(3));
this.DHTSelectionTargetCount++;
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = i6.matcher (logLine);
if (m.find () && m.groupCount() >= 2) {
RWIRejectPeerNames.add(m.group(2));
RWIRejectPeerHashs.add(m.group(1));
RWIRejectCount++;
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i6.matcher (logLine);
if (this.m.find () && this.m.groupCount() >= 2) {
this.RWIRejectPeerNames.add(this.m.group(2));
this.RWIRejectPeerHashs.add(this.m.group(1));
this.RWIRejectCount++;
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = i7.matcher (logLine);
if (m.find ()) {
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i7.matcher (logLine);
if (this.m.find ()) {
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = i8.matcher (logLine);
if (m.find () && m.groupCount() >= 2) {
DHTSelectionWordsCount += Float.parseFloat(m.group(1));
DHTSelectionWordsTimeCount += Float.parseFloat(m.group(2));
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i8.matcher (logLine);
if (this.m.find () && this.m.groupCount() >= 2) {
this.DHTSelectionWordsCount += Float.parseFloat(this.m.group(1));
this.DHTSelectionWordsTimeCount += Float.parseFloat(this.m.group(2));
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = i9.matcher (logLine);
if (m.find () && m.groupCount() >= 1) {
rankingDistributionCount++;
rankingDistributionTime += Integer.parseInt(m.group(1));
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i9.matcher (logLine);
if (this.m.find () && this.m.groupCount() >= 1) {
this.rankingDistributionCount++;
this.rankingDistributionTime += Integer.parseInt(this.m.group(1));
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = i10.matcher (logLine);
if (m.find ()) {
rankingDistributionFailCount++;
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i10.matcher (logLine);
if (this.m.find ()) {
this.rankingDistributionFailCount++;
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = i11.matcher (logLine);
if (m.find ()) {
busyPeerCount++;
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i11.matcher (logLine);
if (this.m.find ()) {
this.busyPeerCount++;
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
// m = i12.matcher (logLine);
//
//
// if (m.find ()) {
// return 3;
// }
m = i13.matcher (logLine);
if (m.find ()) {
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = i13.matcher (logLine);
if (this.m.find ()) {
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = adv1.matcher (logLine);
if (m.find() && m.groupCount() >= 8) {
indexedSites++;
indexedWordSum += Integer.parseInt(m.group(1));
indexedSiteSizeSum += Integer.parseInt(m.group(5));
indexedAnchorsCount += Integer.parseInt(m.group(6));
indexedLinkStorageTime += Integer.parseInt(m.group(7));
indexedIndexStorageTime += Integer.parseInt(m.group(8));
this.m = adv1.matcher (logLine);
if (this.m.find() && this.m.groupCount() >= 8) {
this.indexedSites++;
this.indexedWordSum += Integer.parseInt(this.m.group(1));
this.indexedSiteSizeSum += Integer.parseInt(this.m.group(5));
this.indexedAnchorsCount += Integer.parseInt(this.m.group(6));
this.indexedLinkStorageTime += Integer.parseInt(this.m.group(7));
this.indexedIndexStorageTime += Integer.parseInt(this.m.group(8));
// indexedStackingTime += Integer.parseInt(m.group(7));
// indexedParsingTime += Integer.parseInt(m.group(8));
// indexedIndexingTime += Integer.parseInt(m.group(9));
// indexedStorageTime += Integer.parseInt(m.group(10));
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
} else if ("WARNING".equals(logLevel)){
m = w1.matcher (logLine);
if (m.find ()) {
notEnoughDHTPeers++;
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = w1.matcher (logLine);
if (this.m.find ()) {
this.notEnoughDHTPeers++;
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = w2.matcher (logLine);
if (m.find ()) {
failedIndexDistributionCount++;
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = w2.matcher (logLine);
if (this.m.find ()) {
this.failedIndexDistributionCount++;
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
} else if ("SEVERE".equals(logLevel)){
m = e1.matcher (logLine);
if (m.find () && m.groupCount() >= 1) {
if ("leftchild".equals(m.group(1))) leftChildTwiceCount++;
else if ("rightchild".equals(m.group(1))) rightChildTwiceCount++;
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = e1.matcher (logLine);
if (this.m.find () && this.m.groupCount() >= 1) {
if ("leftchild".equals(this.m.group(1))) this.leftChildTwiceCount++;
else if ("rightchild".equals(this.m.group(1))) this.rightChildTwiceCount++;
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
m = e2.matcher (logLine);
if (m.find ()) {
malformedURLCount++;
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.m = e2.matcher (logLine);
if (this.m.find ()) {
this.malformedURLCount++;
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return 0;
}
}
totalParserTime += (System.currentTimeMillis() - start);
totalParserRuns++;
this.totalParserTime += (System.currentTimeMillis() - start);
this.totalParserRuns++;
return -1;
}
public final Map<String, Object> getResults() {
final Map<String, Object> results = new HashMap<String, Object>();
results.put(PARSER_VERSION , Float.valueOf(parserVersion));
results.put(URLS_RECEIVED , Integer.valueOf(urlSum));
results.put(URLS_REQUESTED , Integer.valueOf(urlReqSum));
results.put(URLS_BLOCKED , Integer.valueOf(blockedURLSum));
results.put(WORDS_RECEIVED , Integer.valueOf(wordsSum));
results.put(RWIS_RECEIVED , Integer.valueOf(rwiSum));
results.put(RWIS_BLOCKED , Integer.valueOf(blockedRWISum));
results.put(URLS_RECEIVED_TIME , Long.valueOf(urlTimeSum));
results.put(RWIS_RECEIVED_TIME , Long.valueOf(rwiTimeSum));
results.put(DHT_TRAFFIC_SENT , Long.valueOf(DHTSendTraffic));
results.put(DHT_URLS_SENT , Integer.valueOf(DHTSendURLs));
results.put(DHT_REJECTED , Integer.valueOf(RWIRejectCount));
results.put(DHT_REJECTED_PEERS_NAME , RWIRejectPeerNames);
results.put(DHT_REJECTED_PEERS_HASH , RWIRejectPeerHashs);
results.put(DHT_SENT_PEERS_NAME , DHTPeerNames);
results.put(DHT_SENT_PEERS_HASH , DHTPeerHashs);
results.put(DHT_SELECTED , Integer.valueOf(DHTSelectionTargetCount));
results.put(DHT_WORDS_SELECTED , Integer.valueOf(DHTSelectionWordsCount));
results.put(DHT_WORDS_SELECTED_TIME , Integer.valueOf(DHTSelectionWordsTimeCount));
results.put(DHT_DISTANCE_MIN , Long.valueOf(minDHTDist));
results.put(DHT_DISTANCE_MAX , Long.valueOf(maxDHTDist));
results.put(DHT_DISTANCE_AVERAGE , Long.valueOf(avgDHTDist / DHTSelectionTargetCount / Long.MAX_VALUE)); //FIXME: broken avg
results.put(PEERS_BUSY , Integer.valueOf(busyPeerCount));
results.put(PEERS_TOO_LESS , Integer.valueOf(notEnoughDHTPeers));
results.put(DHT_SENT_FAILED , Integer.valueOf(failedIndexDistributionCount));
results.put(ERROR_CHILD_TWICE_LEFT , Integer.valueOf(leftChildTwiceCount));
results.put(ERROR_CHILD_TWICE_RIGHT , Integer.valueOf(rightChildTwiceCount));
results.put(RANKING_DIST , Integer.valueOf(rankingDistributionCount));
results.put(RANKING_DIST_TIME , Integer.valueOf(rankingDistributionTime));
results.put(RANKING_DIST_FAILED , Integer.valueOf(rankingDistributionFailCount));
results.put(ERROR_MALFORMED_URL , Integer.valueOf(malformedURLCount));
results.put(INDEXED_SITES , Integer.valueOf(indexedSites));
results.put(INDEXED_WORDS , Integer.valueOf(indexedWordSum));
results.put(INDEXED_SITES_SIZE , Integer.valueOf(indexedSiteSizeSum));
results.put(INDEXED_ANCHORS , Integer.valueOf(indexedAnchorsCount));
results.put(URLS_RECEIVED , Integer.valueOf(this.urlSum));
results.put(URLS_REQUESTED , Integer.valueOf(this.urlReqSum));
results.put(URLS_BLOCKED , Integer.valueOf(this.blockedURLSum));
results.put(WORDS_RECEIVED , Integer.valueOf(this.wordsSum));
results.put(RWIS_RECEIVED , Integer.valueOf(this.rwiSum));
results.put(RWIS_BLOCKED , Integer.valueOf(this.blockedRWISum));
results.put(URLS_RECEIVED_TIME , Long.valueOf(this.urlTimeSum));
results.put(RWIS_RECEIVED_TIME , Long.valueOf(this.rwiTimeSum));
results.put(DHT_TRAFFIC_SENT , Long.valueOf(this.DHTSendTraffic));
results.put(DHT_URLS_SENT , Integer.valueOf(this.DHTSendURLs));
results.put(DHT_REJECTED , Integer.valueOf(this.RWIRejectCount));
results.put(DHT_REJECTED_PEERS_NAME , this.RWIRejectPeerNames);
results.put(DHT_REJECTED_PEERS_HASH , this.RWIRejectPeerHashs);
results.put(DHT_SENT_PEERS_NAME , this.DHTPeerNames);
results.put(DHT_SENT_PEERS_HASH , this.DHTPeerHashs);
results.put(DHT_SELECTED , Integer.valueOf(this.DHTSelectionTargetCount));
results.put(DHT_WORDS_SELECTED , Integer.valueOf(this.DHTSelectionWordsCount));
results.put(DHT_WORDS_SELECTED_TIME , Integer.valueOf(this.DHTSelectionWordsTimeCount));
results.put(DHT_DISTANCE_MIN , Long.valueOf(this.minDHTDist));
results.put(DHT_DISTANCE_MAX , Long.valueOf(this.maxDHTDist));
results.put(DHT_DISTANCE_AVERAGE , Long.valueOf(this.avgDHTDist / this.DHTSelectionTargetCount / Long.MAX_VALUE)); //FIXME: broken avg
results.put(PEERS_BUSY , Integer.valueOf(this.busyPeerCount));
results.put(PEERS_TOO_LESS , Integer.valueOf(this.notEnoughDHTPeers));
results.put(DHT_SENT_FAILED , Integer.valueOf(this.failedIndexDistributionCount));
results.put(ERROR_CHILD_TWICE_LEFT , Integer.valueOf(this.leftChildTwiceCount));
results.put(ERROR_CHILD_TWICE_RIGHT , Integer.valueOf(this.rightChildTwiceCount));
results.put(RANKING_DIST , Integer.valueOf(this.rankingDistributionCount));
results.put(RANKING_DIST_TIME , Integer.valueOf(this.rankingDistributionTime));
results.put(RANKING_DIST_FAILED , Integer.valueOf(this.rankingDistributionFailCount));
results.put(ERROR_MALFORMED_URL , Integer.valueOf(this.malformedURLCount));
results.put(INDEXED_SITES , Integer.valueOf(this.indexedSites));
results.put(INDEXED_WORDS , Integer.valueOf(this.indexedWordSum));
results.put(INDEXED_SITES_SIZE , Integer.valueOf(this.indexedSiteSizeSum));
results.put(INDEXED_ANCHORS , Integer.valueOf(this.indexedAnchorsCount));
// results.put(INDEXED_STACK_TIME , new Integer(indexedStackingTime));
// results.put(INDEXED_PARSE_TIME , new Integer(indexedParsingTime));
// results.put(INDEXED_INDEX_TIME , new Integer(indexedIndexingTime));
// results.put(INDEXED_STORE_TIME , new Integer(indexedStorageTime));
results.put(INDEXED_LINKSTORE_TIME , Integer.valueOf(indexedLinkStorageTime));
results.put(INDEXED_INDEXSTORE_TIME, Integer.valueOf(indexedIndexStorageTime));
results.put(TOTAL_PARSER_TIME , Long.valueOf(totalParserTime));
results.put(TOTAL_PARSER_RUNS , Integer.valueOf(totalParserRuns));
results.put(INDEXED_LINKSTORE_TIME , Integer.valueOf(this.indexedLinkStorageTime));
results.put(INDEXED_INDEXSTORE_TIME, Integer.valueOf(this.indexedIndexStorageTime));
results.put(TOTAL_PARSER_TIME , Long.valueOf(this.totalParserTime));
results.put(TOTAL_PARSER_RUNS , Integer.valueOf(this.totalParserRuns));
return results;
}
public final String getParserType() {
public final static String getParserType() {
return parserType;
}
public final double getParserVersion() {
public final static double getParserVersion() {
return parserVersion;
}
public final void printResults() {
if(rankingDistributionCount == 0) rankingDistributionCount = 1;
if(DHTSelectionWordsTimeCount == 0) DHTSelectionWordsTimeCount = 1;
if(indexedSites != 0) indexedSites++;
System.out.println("INDEXER: Indexed " + indexedSites + " sites in " + (indexedLinkStorageTime + indexedIndexStorageTime) + " milliseconds.");
System.out.println("INDEXER: Indexed " + indexedWordSum + " words on " + indexedSites + " sites. (avg. words per site: " + (indexedWordSum / indexedSites) + ").");
System.out.println("INDEXER: Total Size of indexed sites: " + indexedSiteSizeSum + " bytes (avg. size per site: " + (indexedSiteSizeSum / indexedSites) + " bytes).");
System.out.println("INDEXER: Total Number of Anchors found: " + indexedAnchorsCount + "(avg. Anchors per site: " + (indexedAnchorsCount / indexedSites) + ").");
System.out.println("INDEXER: Total LinkStorageTime: " + indexedLinkStorageTime + " milliseconds (avg. StorageTime: " + (indexedLinkStorageTime / indexedSites) + " milliseconds).");
System.out.println("INDEXER: Total indexStorageTime: " + indexedIndexStorageTime + " milliseconds (avg. StorageTime: " + (indexedIndexStorageTime / indexedSites) + " milliseconds).");
if(this.rankingDistributionCount == 0) this.rankingDistributionCount = 1;
if(this.DHTSelectionWordsTimeCount == 0) this.DHTSelectionWordsTimeCount = 1;
if(this.indexedSites != 0) this.indexedSites++;
System.out.println("INDEXER: Indexed " + this.indexedSites + " sites in " + (this.indexedLinkStorageTime + this.indexedIndexStorageTime) + " milliseconds.");
System.out.println("INDEXER: Indexed " + this.indexedWordSum + " words on " + this.indexedSites + " sites. (avg. words per site: " + (this.indexedWordSum / this.indexedSites) + ").");
System.out.println("INDEXER: Total Size of indexed sites: " + this.indexedSiteSizeSum + " bytes (avg. size per site: " + (this.indexedSiteSizeSum / this.indexedSites) + " bytes).");
System.out.println("INDEXER: Total Number of Anchors found: " + this.indexedAnchorsCount + "(avg. Anchors per site: " + (this.indexedAnchorsCount / this.indexedSites) + ").");
System.out.println("INDEXER: Total LinkStorageTime: " + this.indexedLinkStorageTime + " milliseconds (avg. StorageTime: " + (this.indexedLinkStorageTime / this.indexedSites) + " milliseconds).");
System.out.println("INDEXER: Total indexStorageTime: " + this.indexedIndexStorageTime + " milliseconds (avg. StorageTime: " + (this.indexedIndexStorageTime / this.indexedSites) + " milliseconds).");
// System.out.println("INDEXER: Total StackingTime: " + indexedStackingTime + " milliseconds (avg. StackingTime: " + (indexedStackingTime / indexedSites) + " milliseconds).");
// System.out.println("INDEXER: Total ParsingTime: " + indexedParsingTime + " milliseconds (avg. ParsingTime: " + (indexedParsingTime / indexedSites) + " milliseconds).");
// System.out.println("INDEXER: Total IndexingTime: " + indexedIndexingTime + " milliseconds (avg. IndexingTime: " + (indexedIndexingTime / indexedSites) + " milliseconds).");
// System.out.println("INDEXER: Total StorageTime: " + indexedStorageTime + " milliseconds (avg. StorageTime: " + (indexedStorageTime / indexedSites) + " milliseconds).");
if(urlSum != 0) urlSum++;
System.out.println("DHT: Recieved " + urlSum + " Urls in " + urlTimeSum + " ms. Blocked " + blockedURLSum + " URLs.");
System.out.println("DHT: " + urlTimeSum / urlSum + " milliseconds per URL.");
if(rwiSum != 0) rwiSum++;
System.out.println("DHT: Recieved " + rwiSum + " RWIs from " + wordsSum + " Words in " + rwiTimeSum + " ms. " + urlReqSum + " requested URLs.");
System.out.println("DHT: Blocked " + blockedRWISum + " RWIs before requesting URLs, because URL-Hash was blacklisted.");
System.out.println("DHT: " + rwiTimeSum / rwiSum + " milliseconds per RWI.");
System.out.println("DHT: Rejected " + RWIRejectCount + " Indextransfers from " + RWIRejectPeerNames.size() + " PeerNames with " + RWIRejectPeerHashs.size() + " PeerHashs.");
System.out.println("DHT: " + DHTSendTraffic/(1024*1024l) + " MegaBytes (" + DHTSendTraffic + " Bytes) of DHT-Transfertraffic.");
System.out.println("DHT: Sended " + DHTSendURLs + " URLs via DHT.");
System.out.println("DHT: DHT Transfers send to " + DHTPeerNames.size() + " Peernames with " + DHTPeerHashs.size() + " Peerhashs.");
System.out.println("DHT: Totally selected " + DHTSelectionWordsCount + " words in " + DHTSelectionWordsTimeCount + " seconds (" + (float)DHTSelectionWordsCount/DHTSelectionWordsTimeCount + " words/s)");
System.out.println("DHT: Selected " + DHTSelectionTargetCount + " possible DHT Targets (min. Distance: " + minDHTDist + " max. Distance: " + maxDHTDist + " avg. Distance: " + (avgDHTDist/DHTSelectionTargetCount));
System.out.println("DHT: " + busyPeerCount + " times a targetpeer was too busy to accept a transfer.");
System.out.println("DHT: " + notEnoughDHTPeers + " times there were not enought targetpeers for the selected DHTChunk");
System.out.println("DHT: IndexDistribution failed " + failedIndexDistributionCount + " times.");
System.out.println("RANKING: Transmitted " + rankingDistributionCount + " Rankingfiles in " + rankingDistributionTime + " seconds (" + rankingDistributionTime/rankingDistributionCount + " seconds/file)");
System.out.println("RANKING: RankingDistribution failed " + rankingDistributionFailCount + " times.");
if (leftChildTwiceCount != 0)
System.out.println("ERRORS: tried " + leftChildTwiceCount + " times to create leftchild node twice in db");
if (rightChildTwiceCount != 0)
System.out.println("ERRORS: tried " + rightChildTwiceCount + " times to create rightchild node twice in db");
if (malformedURLCount != 0)
System.out.println("ERRORS: " + malformedURLCount + " MalformedURLExceptions accord.");
// System.out.println("INDEXER: Total StorageTime: " + indexedStorageTime + " milliseconds (avg. StorageTime: " + (indexedStorageTime / indexedSites) + " milliseconds).");
if(this.urlSum != 0) this.urlSum++;
System.out.println("DHT: Recieved " + this.urlSum + " Urls in " + this.urlTimeSum + " ms. Blocked " + this.blockedURLSum + " URLs.");
System.out.println("DHT: " + this.urlTimeSum / this.urlSum + " milliseconds per URL.");
if(this.rwiSum != 0) this.rwiSum++;
System.out.println("DHT: Recieved " + this.rwiSum + " RWIs from " + this.wordsSum + " Words in " + this.rwiTimeSum + " ms. " + this.urlReqSum + " requested URLs.");
System.out.println("DHT: Blocked " + this.blockedRWISum + " RWIs before requesting URLs, because URL-Hash was blacklisted.");
System.out.println("DHT: " + this.rwiTimeSum / this.rwiSum + " milliseconds per RWI.");
System.out.println("DHT: Rejected " + this.RWIRejectCount + " Indextransfers from " + this.RWIRejectPeerNames.size() + " PeerNames with " + this.RWIRejectPeerHashs.size() + " PeerHashs.");
System.out.println("DHT: " + this.DHTSendTraffic/(1024*1024l) + " MegaBytes (" + this.DHTSendTraffic + " Bytes) of DHT-Transfertraffic.");
System.out.println("DHT: Sended " + this.DHTSendURLs + " URLs via DHT.");
System.out.println("DHT: DHT Transfers send to " + this.DHTPeerNames.size() + " Peernames with " + this.DHTPeerHashs.size() + " Peerhashs.");
System.out.println("DHT: Totally selected " + this.DHTSelectionWordsCount + " words in " + this.DHTSelectionWordsTimeCount + " seconds (" + (float)this.DHTSelectionWordsCount/this.DHTSelectionWordsTimeCount + " words/s)");
System.out.println("DHT: Selected " + this.DHTSelectionTargetCount + " possible DHT Targets (min. Distance: " + this.minDHTDist + " max. Distance: " + this.maxDHTDist + " avg. Distance: " + (this.avgDHTDist/this.DHTSelectionTargetCount));
System.out.println("DHT: " + this.busyPeerCount + " times a targetpeer was too busy to accept a transfer.");
System.out.println("DHT: " + this.notEnoughDHTPeers + " times there were not enought targetpeers for the selected DHTChunk");
System.out.println("DHT: IndexDistribution failed " + this.failedIndexDistributionCount + " times.");
System.out.println("RANKING: Transmitted " + this.rankingDistributionCount + " Rankingfiles in " + this.rankingDistributionTime + " seconds (" + this.rankingDistributionTime/this.rankingDistributionCount + " seconds/file)");
System.out.println("RANKING: RankingDistribution failed " + this.rankingDistributionFailCount + " times.");
if (this.leftChildTwiceCount != 0)
System.out.println("ERRORS: tried " + this.leftChildTwiceCount + " times to create leftchild node twice in db");
if (this.rightChildTwiceCount != 0)
System.out.println("ERRORS: tried " + this.rightChildTwiceCount + " times to create rightchild node twice in db");
if (this.malformedURLCount != 0)
System.out.println("ERRORS: " + this.malformedURLCount + " MalformedURLExceptions accord.");
}
}

17
source/net/yacy/kelondro/logging/LogalizerHandler.java
Unescape View File

@ -1,4 +1,4 @@
//LogalizerHandler.java
//LogalizerHandler.java
//-------------------------------------
//part of YACY
//(C) by Michael Peter Christen; mc@yacy.net
@ -37,17 +37,18 @@ public final class LogalizerHandler extends Handler {
public static boolean enabled;
public static boolean debug;
public LogalizerHandler() {
super();
final LogManager manager = LogManager.getLogManager();
final String className = getClass().getName();
enabled = "true".equalsIgnoreCase(manager.getProperty(className + ".enabled"));
debug = "true".equalsIgnoreCase(manager.getProperty(className + ".debug"));
}
@Override
public final void publish(final LogRecord record) {
if (enabled) {
final LogParser temp = new LogParser();
@ -58,14 +59,16 @@ public final class LogalizerHandler extends Handler {
}
flush();
}
public final Map<String, Object> getParserResults(final LogParser parsername) {
public final static Map<String, Object> getParserResults(final LogParser parsername) {
return (parsername == null) ? null : parsername.getResults();
}
@Override
public final void close() throws SecurityException {
}
@Override
public final void flush() {
}
}

7
source/net/yacy/kelondro/util/MemoryControl.java
Unescape View File

@ -34,14 +34,14 @@ public class MemoryControl {
private static MemoryStrategy strategy;
private static MemoryStrategy getStrategy() {
if (strategy == null || strategy.hasError()) {
if (strategy == null || MemoryStrategy.hasError()) {
if (!usingStandardStrategy) {
strategy = new GenerationMemoryStrategy();
// if (strategy.hasError()) { // perhaps we do have a G1
// strategy = new G1MemoryStrategy();
// }
// fall back if error detected
if (strategy.hasError()) {
if (MemoryStrategy.hasError()) {
usingStandardStrategy = true;
strategy = new StandardMemoryStrategy();
}
@ -63,7 +63,8 @@ public class MemoryControl {
* @return the name of the used strategy
*/
public final static String getStrategyName() {
return getStrategy().getName();
getStrategy();
return MemoryStrategy.getName();
}
/**

28
source/net/yacy/kelondro/util/MemoryStrategy.java
Unescape View File

@ -28,23 +28,23 @@ package net.yacy.kelondro.util;
import net.yacy.kelondro.logging.Log;
public abstract class MemoryStrategy {
protected final static Log log = new Log("MEMORY");
protected static long lastGC = 0l;
protected static boolean error = true;
protected static boolean error = true;
protected static String name;
/**
* @return if an error has been detected
*/
protected final boolean hasError() {
protected final static boolean hasError() {
return error;
}
/**
* @return an identifying name
*/
protected final String getName() {
protected final static String getName() {
return name;
}
@ -61,17 +61,17 @@ public abstract class MemoryStrategy {
lastGC = System.currentTimeMillis();
return true;
}
if (log.isFinest()) log.logFinest("[gc] no execute, last run: " + (elapsed / 1000) + " seconds ago, call: " + info);
return false;
}
/**
* memory that is free without increasing of total memory taken from os
* @return bytes
*/
protected abstract long free();
/**
* memory that is available including increasing total memory up to maximum
* @return bytes
@ -83,19 +83,19 @@ public abstract class MemoryStrategy {
* @return used bytes
*/
protected abstract long used();
/**
* currently allocated memory in the Java virtual machine; may vary over time
* @return bytes
*/
protected abstract long total();
/**
* maximum memory the Java virtual will allocate machine; may vary over time in some cases
* @return bytes
*/
protected abstract long maxMemory();
/**
* <p>Tries to free a specified amount of bytes.</p>
* <p>
@ -116,12 +116,12 @@ public abstract class MemoryStrategy {
* @return whether enough memory could be freed (or is free) or not
*/
protected abstract boolean request(final long size, final boolean force, boolean shortStatus);
/**
* @return if Memory seams to be in a proper state
*/
protected abstract boolean properState();
/**
* forced enable properState - StandardMemoryStrategy only
*/

1
source/net/yacy/kelondro/util/StandardMemoryStrategy.java
Unescape View File

@ -49,7 +49,6 @@ public class StandardMemoryStrategy extends MemoryStrategy {
* @param last time which must be passed since lased gc
* @param info additional info for log
*/
@Override
protected final synchronized boolean gc(final int last, final String info) { // thq
assert last >= 10000; // too many forced GCs will cause bad execution performance
final long elapsed = System.currentTimeMillis() - lastGC;

17
source/net/yacy/peers/NewsPool.java
Unescape View File

@ -398,7 +398,7 @@ public class NewsPool {
return pc;
}
private boolean automaticProcessP(final SeedDB seedDB, final NewsDB.Record record) {
private static boolean automaticProcessP(final SeedDB seedDB, final NewsDB.Record record) {
if (record == null) return false;
if (record.category() == null) return true;
final long created = record.created().getTime();
@ -467,8 +467,9 @@ public class NewsPool {
case PROCESSED_DB: return this.processedNews.get(id);
case OUTGOING_DB: return this.outgoingNews.get(id);
case PUBLISHED_DB: return this.publishedNews.get(id);
default:
return null;
}
return null;
}
private NewsQueue switchQueue(final int dbKey) {
@ -477,8 +478,8 @@ public class NewsPool {
case PROCESSED_DB: return this.processedNews;
case OUTGOING_DB: return this.outgoingNews;
case PUBLISHED_DB: return this.publishedNews;
}
return null;
default: return null;
}
}
public void clear(final int dbKey) {
@ -488,7 +489,8 @@ public class NewsPool {
case PROCESSED_DB: this.processedNews.clear(); break;
case OUTGOING_DB: this.outgoingNews.clear(); break;
case PUBLISHED_DB: this.publishedNews.clear(); break;
}
default: return;
}
}
public void moveOff(final int dbKey, final String id) throws IOException, RowSpaceExceededException {
@ -499,7 +501,8 @@ public class NewsPool {
case PROCESSED_DB: moveOff(this.processedNews, null,id); break;
case OUTGOING_DB: moveOff(this.outgoingNews, this.publishedNews, id); break;
case PUBLISHED_DB: moveOff(this.publishedNews, null, id); break;
}
default: return;
}
}
private boolean moveOff(final NewsQueue fromqueue, final NewsQueue toqueue, final String id) throws IOException, RowSpaceExceededException {
@ -527,7 +530,7 @@ public class NewsPool {
}
}
private int moveOffAll(final NewsQueue fromqueue, final NewsQueue toqueue) throws IOException, RowSpaceExceededException {
private static int moveOffAll(final NewsQueue fromqueue, final NewsQueue toqueue) throws IOException, RowSpaceExceededException {
// move off all news from a specific queue to another queue
final Iterator<NewsDB.Record> i = fromqueue.iterator();
NewsDB.Record record;

6
source/net/yacy/peers/SeedDB.java
Unescape View File

@ -306,7 +306,7 @@ public final class SeedDB implements AlternativeDomainNames {
return sizeConnected() <= dhtActivityMagic;
}
private synchronized MapDataMining openSeedTable(final File seedDBFile) {
private synchronized static MapDataMining openSeedTable(final File seedDBFile) {
final File parentDir = new File(seedDBFile.getParent());
if (!parentDir.exists()) {
if(!parentDir.mkdirs())
@ -817,7 +817,7 @@ public final class SeedDB implements AlternativeDomainNames {
return log;
}
private Iterator<String> downloadSeedFile(final DigestURI seedURL) throws IOException {
private static Iterator<String> downloadSeedFile(final DigestURI seedURL) throws IOException {
// Configure http headers
final RequestHeader reqHeader = new RequestHeader();
reqHeader.put(HeaderFramework.PRAGMA, "no-cache");
@ -850,7 +850,7 @@ public final class SeedDB implements AlternativeDomainNames {
}
}
private String checkCache(final ArrayList<String> uv, final Iterator<String> check) {
private static String checkCache(final ArrayList<String> uv, final Iterator<String> check) {
if ((check == null) || (uv == null)) {
if (Log.isFine("YACY")) Log.logFine("YACY", "SaveSeedList: Local and uploades seed-list are different");
return "Entry count is different: uv.size() = " + ((uv == null) ? "null" : Integer.toString(uv.size()));

141
source/net/yacy/peers/operation/yacySeedUploadScp.java
Unescape View File

@ -1,4 +1,4 @@
//yacySeedUploadScp.java
//yacySeedUploadScp.java
//-------------------------------------
//part of YACY
//(C) by Michael Peter Christen; mc@yacy.net
@ -44,30 +44,31 @@ import com.jcraft.jsch.UserInfo;
import de.anomic.server.serverSwitch;
public class yacySeedUploadScp implements yacySeedUploader {
public static final String CONFIG_SCP_SERVER = "seedScpServer";
public static final String CONFIG_SCP_SERVER_PORT = "seedScpServerPort";
public static final String CONFIG_SCP_ACCOUNT = "seedScpAccount";
public static final String CONFIG_SCP_PASSWORD = "seedScpPassword";
public static final String CONFIG_SCP_PATH = "seedScpPath";
@Override
public String uploadSeedFile(final serverSwitch sb, final File seedFile) throws Exception {
try {
try {
if (sb == null) throw new NullPointerException("Reference to serverSwitch nut not be null.");
if ((seedFile == null)||(!seedFile.exists())) throw new Exception("Seed file does not exist.");
final String seedScpServer = sb.getConfig(CONFIG_SCP_SERVER,null);
final String seedScpServerPort = sb.getConfig(CONFIG_SCP_SERVER_PORT,"22");
final String seedScpAccount = sb.getConfig(CONFIG_SCP_ACCOUNT,null);
final String seedScpPassword = sb.getConfig(CONFIG_SCP_PASSWORD,null);
final String seedScpPath = sb.getConfig(CONFIG_SCP_PATH,null);
final String seedScpPath = sb.getConfig(CONFIG_SCP_PATH,null);
if (seedScpServer == null || seedScpServer.length() == 0)
throw new Exception("Seed SCP upload settings not configured properly. Servername must not be null or empty.");
else if (seedScpAccount == null || seedScpAccount.length() == 0)
throw new Exception("Seed SCP upload settings not configured properly. Username must not be null or empty.");
throw new Exception("Seed SCP upload settings not configured properly. Username must not be null or empty.");
else if (seedScpPassword == null || seedScpPassword.length() == 0)
throw new Exception("Seed SCP upload settings not configured properly. Password must not be null or empty.");
throw new Exception("Seed SCP upload settings not configured properly. Password must not be null or empty.");
else if (seedScpPath == null || seedScpPath.length() == 0)
throw new Exception("Seed SCP upload settings not configured properly. File path must not be null or empty.");
else if (seedScpServerPort == null || seedScpServerPort.length() == 0)
@ -78,38 +79,39 @@ public class yacySeedUploadScp implements yacySeedUploader {
} catch (final NumberFormatException ex) {
throw new Exception("Seed SCP upload settings not configured properly. Server port is not a vaild integer.");
}
return sshc.put(seedScpServer, port, seedFile, seedScpPath, seedScpAccount, seedScpPassword);
} catch (final Exception e) {
throw e;
}
}
@Override
public String[] getConfigurationOptions() {
return new String[] {CONFIG_SCP_SERVER,CONFIG_SCP_SERVER_PORT,CONFIG_SCP_ACCOUNT,CONFIG_SCP_PASSWORD,CONFIG_SCP_PATH};
}
}
class sshc {
public static String put(
final String host,
final int port,
final File localFile,
final File localFile,
final String remoteName,
final String account,
final String account,
final String password
) throws Exception {
Session session = null;
try {
try {
// Creating a new secure channel object
final JSch jsch=new JSch();
// setting hostname, username, userpassword
session = jsch.getSession(account, host, port);
session.setPassword(password);
session.setPassword(password);
/*
* Setting the StrictHostKeyChecking to ignore unknown
* hosts because of a missing known_hosts file ...
@ -117,29 +119,29 @@ class sshc {
final java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking","no");
session.setConfig(config);
/*
/*
* we need this user interaction interface to support
* the interactive-keyboard mode
*/
*/
final UserInfo ui=new SchUserInfo(password);
session.setUserInfo(ui);
session.setUserInfo(ui);
// trying to connect ...
session.connect();
session.connect();
String command="scp -p -t " + remoteName;
final Channel channel = session.openChannel("exec");
((ChannelExec)channel).setCommand(command);
((ChannelExec)channel).setCommand(command);
// get I/O streams for remote scp
final OutputStream out=channel.getOutputStream();
final InputStream in=channel.getInputStream();
channel.connect();
checkAck(in);
// send "C0644 filesize filename", where filename should not include '/'
final int filesize=(int)(localFile).length();
command="C0644 "+filesize+" ";
@ -151,14 +153,14 @@ class sshc {
}
command+="\n";
out.write(UTF8.getBytes(command)); out.flush();
checkAck(in);
// send a content of lfile
final byte[] buf=new byte[1024];
BufferedInputStream bufferedIn = null;
try {
bufferedIn=new BufferedInputStream(new FileInputStream(localFile));
bufferedIn=new BufferedInputStream(new FileInputStream(localFile));
while(true){
final int len=bufferedIn.read(buf, 0, buf.length);
if(len<=0) break;
@ -167,12 +169,12 @@ class sshc {
} finally {
if (bufferedIn != null) try{bufferedIn.close();}catch(final Exception e){}
}
// send '\0'
buf[0]=0; out.write(buf, 0, 1); out.flush();
checkAck(in);
checkAck(in);
return "SCP: File uploaded successfully.";
} catch (final Exception e) {
throw new Exception("SCP: File uploading failed: " + e.getMessage());
@ -180,7 +182,7 @@ class sshc {
if ((session != null) && (session.isConnected())) session.disconnect();
}
}
static int checkAck(final InputStream in) throws IOException{
final int b=in.read();
// b may be 0 for success,
@ -189,7 +191,7 @@ class sshc {
// -1
if(b==0) return b;
if(b==-1) return b;
if(b==1 || b==2){
final StringBuilder sb=new StringBuilder();
int c;
@ -198,7 +200,7 @@ class sshc {
sb.append((char)c);
}
while(c!='\n');
if(b==1){ // error
throw new IOException(sb.toString());
}
@ -211,44 +213,51 @@ class sshc {
}
class SchUserInfo
class SchUserInfo
implements UserInfo, UIKeyboardInteractive {
String passwd;
public SchUserInfo(final String password) {
this.passwd = password;
}
public String getPassword() {
return this.passwd;
@Override
public String getPassword() {
return this.passwd;
}
public boolean promptYesNo(final String str){
@Override
public boolean promptYesNo(final String str){
System.err.println("User was prompted from: " + str);
return true;
}
public String getPassphrase() {
return null;
@Override
public String getPassphrase() {
return null;
}
@Override
public boolean promptPassphrase(final String message) {
System.out.println("promptPassphrase : " + message);
System.out.println("promptPassphrase : " + message);
return false;
}
@Override
public boolean promptPassword(final String message) {
System.out.println("promptPassword : " + message);
System.out.println("promptPassword : " + message);
return true;
}
/**
* @see com.jcraft.jsch.UserInfo#showMessage(java.lang.String)
*/
@Override
public void showMessage(final String message) {
System.out.println("Sch has tried to show the following message to the user: " + message);
}
@Override
public String[] promptKeyboardInteractive(final String destination,
final String name,
final String instruction,
@ -258,15 +267,15 @@ implements UserInfo, UIKeyboardInteractive {
"\n\tDestination: " + destination +
"\n\tName: " + name +
"\n\tInstruction: " + instruction +
"\n\tPrompt: " + arrayToString2(prompt,"|") +
"\n\techo: " + arrayToString2(echo,"|"));
"\n\tPrompt: " + arrayToString2(prompt,"|") +
"\n\techo: " + arrayToString2(echo,"|"));
if ((prompt.length >= 1) && (prompt[0].startsWith("Password")))
return new String[]{this.passwd};
return new String[]{};
}
String arrayToString2(final String[] a, final String separator) {
static String arrayToString2(final String[] a, final String separator) {
final StringBuilder result = new StringBuilder();// start with first element
if (a.length > 0) {
result.append(a[0]);
@ -275,10 +284,10 @@ implements UserInfo, UIKeyboardInteractive {
result.append(a[i]);
}
}
return result.toString();
return result.toString();
}
String arrayToString2(final boolean[] a, final String separator) {
static String arrayToString2(final boolean[] a, final String separator) {
final StringBuilder result = new StringBuilder();// start with first element
if (a.length > 0) {
result.append(a[0]);
@ -287,7 +296,7 @@ implements UserInfo, UIKeyboardInteractive {
result.append(a[i]);
}
}
return result.toString();
return result.toString();
}
}

4
source/net/yacy/repository/Blacklist.java
Unescape View File

@ -372,7 +372,7 @@ public class Blacklist {
return (m1.matcher(host).matches() || m2.matcher(host).matches() || m3.matcher(host).matches());
}
public String getEngineInfo() {
public static String getEngineInfo() {
return "Default YaCy Blacklist Engine";
}
@ -457,7 +457,7 @@ public class Blacklist {
return matched;
}
public BlacklistError checkError(final String element, final Map<String, String> properties) {
public static BlacklistError checkError(final String element, final Map<String, String> properties) {
final boolean allowRegex = (properties != null) && properties.get("allowRegex").equalsIgnoreCase("true");
int slashPos;

2
source/net/yacy/repository/LoaderDispatcher.java
Unescape View File

@ -394,7 +394,7 @@ public final class LoaderDispatcher {
return Document.getHyperlinks(documents);
}
public synchronized void cleanupAccessTimeTable(final long timeout) {
public synchronized static void cleanupAccessTimeTable(final long timeout) {
final Iterator<Map.Entry<String, Long>> i = accessTime.entrySet().iterator();
Map.Entry<String, Long> e;
while (i.hasNext()) {

4
source/net/yacy/search/Switchboard.java
Unescape View File

@ -2868,7 +2868,7 @@ public final class Switchboard extends serverSwitch
}
}
public boolean accessFromLocalhost(final RequestHeader requestHeader) {
public static boolean accessFromLocalhost(final RequestHeader requestHeader) {
// authorization for localhost, only if flag is set to grant localhost access as admin
final String clientIP = requestHeader.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "");
@ -3299,7 +3299,7 @@ public final class Switchboard extends serverSwitch
}.start();
}
public int currentPPM() {
public static int currentPPM() {
return EventTracker.countEvents(EventTracker.EClass.INDEX, 20000) * 3;
}

121
source/net/yacy/upnp/impls/InternetGatewayDevice.java
Unescape View File

@ -26,8 +26,8 @@
* prior written permission. For written permission, please contact
* info@sbbi.net.
*
* 5. Products derived from this software may not be called
* "SuperBonBon Industries", nor may "SBBI" appear in their name,
* 5. Products derived from this software may not be called
* "SuperBonBon Industries", nor may "SBBI" appear in their name,
* without prior written permission of SuperBonBon Industries.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
@ -42,7 +42,7 @@
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* This software consists of voluntary contributions made by many individuals
* on behalf of SuperBonBon Industries. For more information on
* on behalf of SuperBonBon Industries. For more information on
* SuperBonBon Industries, please see <http://www.sbbi.net/>.
*/
package net.yacy.upnp.impls;
@ -52,12 +52,8 @@ import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.UnknownHostException;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import net.yacy.upnp.Discovery;
import net.yacy.upnp.devices.UPNPDevice;
import net.yacy.upnp.devices.UPNPRootDevice;
@ -69,8 +65,11 @@ import net.yacy.upnp.messages.UPNPMessageFactory;
import net.yacy.upnp.messages.UPNPResponseException;
import net.yacy.upnp.services.UPNPService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* This class can be used to access some funtionalities on the
* This class can be used to access some funtionalities on the
* InternetGatewayDevice on your network without having to know
* anything about the required input/output parameters.
* All device functions are not provided.
@ -78,16 +77,16 @@ import net.yacy.upnp.services.UPNPService;
* @version 1.0
*/
public class InternetGatewayDevice {
private final static Log log = LogFactory.getLog( InternetGatewayDevice.class );
private UPNPRootDevice igd;
private final UPNPRootDevice igd;
private UPNPMessageFactory msgFactory;
public InternetGatewayDevice( UPNPRootDevice igd ) throws UnsupportedOperationException {
this( igd, true, true );
}
private InternetGatewayDevice( UPNPRootDevice igd, boolean WANIPConnection, boolean WANPPPConnection ) throws UnsupportedOperationException {
this.igd = igd;
UPNPDevice myIGDWANConnDevice = igd.getChildDevice( "urn:schemas-upnp-org:device:WANConnectionDevice:1" );
@ -105,11 +104,11 @@ public class InternetGatewayDevice {
} else if ( ( !WANIPConnection && WANPPPConnection ) && wanPPPSrv == null ) {
throw new UnsupportedOperationException( "Unable to find any urn:schemas-upnp-org:service:WANPPPConnection:1 service" );
}
if ( wanIPSrv != null && wanPPPSrv == null ) {
msgFactory = UPNPMessageFactory.getNewInstance( wanIPSrv );
this.msgFactory = UPNPMessageFactory.getNewInstance( wanIPSrv );
} else if ( wanPPPSrv != null && wanIPSrv == null ) {
msgFactory = UPNPMessageFactory.getNewInstance( wanPPPSrv );
this.msgFactory = UPNPMessageFactory.getNewInstance( wanPPPSrv );
} else {
// Unable to test the following code since no router implementing both IP and PPP connection on hands..
/*// discover the active WAN interface using the WANCommonInterfaceConfig specs
@ -136,34 +135,34 @@ public class InternetGatewayDevice {
serviceID != null && serviceID.trim().length() > 0 ) {
for ( Iterator i = igd.getChildDevices().iterator(); i.hasNext(); ) {
UPNPDevice dv = (UPNPDevice)i.next();
if ( deviceContainer.startsWith( dv.getUDN() ) &&
if ( deviceContainer.startsWith( dv.getUDN() ) &&
dv.getDeviceType().indexOf( ":WANConnectionDevice:" ) != -1 ) {
myIGDWANConnDevice = dv;
break;
}
}
msgFactory = UPNPMessageFactory.getNewInstance( myIGDWANConnDevice.getServiceByID( serviceID ) );
msgFactory = UPNPMessageFactory.getNewInstance( myIGDWANConnDevice.getServiceByID( serviceID ) );
}
}
}*/
// Doing a tricky test with external IP address, the unactive interface should return a null value or none
if ( testWANInterface( wanIPSrv ) ) {
msgFactory = UPNPMessageFactory.getNewInstance( wanIPSrv );
this.msgFactory = UPNPMessageFactory.getNewInstance( wanIPSrv );
} else if( testWANInterface( wanPPPSrv ) ) {
msgFactory = UPNPMessageFactory.getNewInstance( wanPPPSrv );
this.msgFactory = UPNPMessageFactory.getNewInstance( wanPPPSrv );
}
if ( msgFactory == null ) {
if ( this.msgFactory == null ) {
// Nothing found using WANCommonInterfaceConfig! IP by default
log.warn( "Unable to detect active WANIPConnection, dfaulting to urn:schemas-upnp-org:service:WANIPConnection:1" );
msgFactory = UPNPMessageFactory.getNewInstance( wanIPSrv );
this.msgFactory = UPNPMessageFactory.getNewInstance( wanIPSrv );
}
}
}
private boolean testWANInterface( UPNPService srv ) {
private static boolean testWANInterface( UPNPService srv ) {
UPNPMessageFactory tmp = UPNPMessageFactory.getNewInstance( srv );
ActionMessage msg = tmp.getMessage( "GetExternalIPAddress" );
String ipToParse = null;
try {
@ -183,17 +182,17 @@ public class InternetGatewayDevice {
}
return false;
}
/**
* Retreives the IDG UNPNRootDevice object
* @return the UNPNRootDevie object bound to this object
*/
public UPNPRootDevice getIGDRootDevice() {
return igd;
return this.igd;
}
/**
* Lookup all the IGD (IP or PPP) devices on the network. If a device implements both
* Lookup all the IGD (IP or PPP) devices on the network. If a device implements both
* IP and PPP, the active service will be used for nat mappings.
* @param timeout the timeout in ms to listen for devices response, -1 for default value
* @return an array of devices to play with or null if nothing found.
@ -202,10 +201,10 @@ public class InternetGatewayDevice {
public static InternetGatewayDevice[] getDevices( int timeout ) throws IOException {
return lookupDeviceDevices( timeout, Discovery.DEFAULT_TTL, Discovery.DEFAULT_MX, true, true, null );
}
/**
* Lookup all the IGD (IP urn:schemas-upnp-org:service:WANIPConnection:1, or PPP urn:schemas-upnp-org:service:WANPPPConnection:1)
* devices for a given network interface. If a device implements both
* devices for a given network interface. If a device implements both
* IP and PPP, the active service will be used for nat mappings.
* @param timeout the timeout in ms to listen for devices response, -1 for default value
* @param ttl the discovery ttl such as {@link net.yacy.upnp.Discovery#DEFAULT_TTL}
@ -217,7 +216,7 @@ public class InternetGatewayDevice {
public static InternetGatewayDevice[] getDevices( int timeout, int ttl, int mx, NetworkInterface ni ) throws IOException {
return lookupDeviceDevices( timeout, ttl, mx, true, true, ni );
}
/**
* Lookup all the IGD IP devices on the network (urn:schemas-upnp-org:service:WANIPConnection:1 service)
* @param timeout the timeout in ms to listen for devices response, -1 for default value
@ -230,7 +229,7 @@ public class InternetGatewayDevice {
public static InternetGatewayDevice[] getIPDevices( int timeout ) throws IOException {
return lookupDeviceDevices( timeout, Discovery.DEFAULT_TTL, Discovery.DEFAULT_MX, true, false, null );
}
/**
* Lookup all the IGD PPP devices on the network (urn:schemas-upnp-org:service:WANPPPConnection:1 service)
* @param timeout the timeout in ms to listen for devices response, -1 for default value
@ -243,7 +242,7 @@ public class InternetGatewayDevice {
public static InternetGatewayDevice[] getPPPDevices( int timeout ) throws IOException {
return lookupDeviceDevices( timeout, Discovery.DEFAULT_TTL, Discovery.DEFAULT_MX, false, true, null );
}
private static InternetGatewayDevice[] lookupDeviceDevices( int timeout, int ttl, int mx, boolean WANIPConnection, boolean WANPPPConnection, NetworkInterface ni ) throws IOException {
UPNPRootDevice[] devices = null;
InternetGatewayDevice[] rtrVal = null;
@ -255,9 +254,9 @@ public class InternetGatewayDevice {
if ( devices != null ) {
Set <InternetGatewayDevice> valid = new HashSet<InternetGatewayDevice>();
for ( int i = 0; i < devices.length; i++ ) {
for (UPNPRootDevice device : devices) {
try {
valid.add( new InternetGatewayDevice( devices[i], WANIPConnection, WANPPPConnection ) );
valid.add( new InternetGatewayDevice( device, WANIPConnection, WANPPPConnection ) );
} catch ( UnsupportedOperationException ex ) {
// the device is either not IP or PPP
if ( log.isDebugEnabled() ) log.debug( "UnsupportedOperationException during discovery " + ex.getMessage() );
@ -268,10 +267,10 @@ public class InternetGatewayDevice {
}
rtrVal = new InternetGatewayDevice[valid.size()];
int i = 0;
for ( Iterator<InternetGatewayDevice> itr = valid.iterator(); itr.hasNext(); ) {
rtrVal[i++] = itr.next();
for (InternetGatewayDevice internetGatewayDevice : valid) {
rtrVal[i++] = internetGatewayDevice;
}
}
return rtrVal;
}
@ -283,7 +282,7 @@ public class InternetGatewayDevice {
* @throws IOException if some error occurs during communication with the device
*/
public String getExternalIPAddress() throws UPNPResponseException, IOException {
ActionMessage msg = msgFactory.getMessage( "GetExternalIPAddress" );
ActionMessage msg = this.msgFactory.getMessage( "GetExternalIPAddress" );
return msg.service().getOutActionArgumentValue( "NewExternalIPAddress" );
}
@ -298,7 +297,7 @@ public class InternetGatewayDevice {
*/
public ActionResponse getGenericPortMappingEntry( int newPortMappingIndex ) throws IOException, UPNPResponseException {
ActionMessage msg = msgFactory.getMessage( "GetGenericPortMappingEntry" );
ActionMessage msg = this.msgFactory.getMessage( "GetGenericPortMappingEntry" );
msg.setInputParameter( "NewPortMappingIndex", newPortMappingIndex );
try {
@ -309,7 +308,7 @@ public class InternetGatewayDevice {
}
throw ex;
}
}
/**
@ -327,8 +326,8 @@ public class InternetGatewayDevice {
remoteHost = remoteHost == null ? "" : remoteHost;
checkPortMappingProtocol( protocol );
checkPortRange( externalPort );
ActionMessage msg = msgFactory.getMessage( "GetSpecificPortMappingEntry" );
ActionMessage msg = this.msgFactory.getMessage( "GetSpecificPortMappingEntry" );
msg.setInputParameter( "NewRemoteHost", remoteHost )
.setInputParameter( "NewExternalPort", externalPort )
.setInputParameter( "NewProtocol", protocol );
@ -365,7 +364,7 @@ public class InternetGatewayDevice {
* 727 ExternalPortOnlySupportsWildcard ExternalPort must be a wildcard and cannot be a specific port value
*/
public boolean addPortMapping( String description, String remoteHost,
int internalPort, int externalPort,
int internalPort, int externalPort,
String internalClient, int leaseDuration,
String protocol ) throws IOException, UPNPResponseException {
remoteHost = remoteHost == null ? "" : remoteHost;
@ -377,7 +376,7 @@ public class InternetGatewayDevice {
description = description == null ? "" : description;
if ( leaseDuration < 0 ) throw new IllegalArgumentException( "Invalid leaseDuration (" + leaseDuration + ") value" );
ActionMessage msg = msgFactory.getMessage( "AddPortMapping" );
ActionMessage msg = this.msgFactory.getMessage( "AddPortMapping" );
msg.setInputParameter( "NewRemoteHost", remoteHost )
.setInputParameter( "NewExternalPort", externalPort )
.setInputParameter( "NewProtocol", protocol )
@ -396,7 +395,7 @@ public class InternetGatewayDevice {
throw ex;
}
}
/**
* Deletes a port mapping on the IDG device
* @param remoteHost the host ip for which the mapping was done, null value for a wildcard value
@ -407,11 +406,11 @@ public class InternetGatewayDevice {
* @throws UPNPResponseException if the devices returns an error message
*/
public boolean deletePortMapping( String remoteHost, int externalPort, String protocol ) throws IOException, UPNPResponseException {
remoteHost = remoteHost == null ? "" : remoteHost;
checkPortMappingProtocol( protocol );
checkPortRange( externalPort );
ActionMessage msg = msgFactory.getMessage( "DeletePortMapping" );
ActionMessage msg = this.msgFactory.getMessage( "DeletePortMapping" );
msg.setInputParameter( "NewRemoteHost", remoteHost )
.setInputParameter( "NewExternalPort", externalPort )
.setInputParameter( "NewProtocol", protocol );
@ -425,7 +424,7 @@ public class InternetGatewayDevice {
throw ex;
}
}
/**
* Retreives the current number of mapping in the NAT table
* @return the nat table current number of mappings or null if the device does not allow to query state variables
@ -433,9 +432,9 @@ public class InternetGatewayDevice {
* @throws UPNPResponseException if the devices returns an error message with error code other than 404
*/
public Integer getNatMappingsCount() throws IOException, UPNPResponseException {
Integer rtrval = null;
StateVariableMessage natTableSize = msgFactory.getStateVariableMessage( "PortMappingNumberOfEntries" );
StateVariableMessage natTableSize = this.msgFactory.getStateVariableMessage( "PortMappingNumberOfEntries" );
try {
StateVariableResponse resp = natTableSize.service();
rtrval = new Integer( resp.getStateVariableValue() );
@ -447,7 +446,7 @@ public class InternetGatewayDevice {
}
return rtrval;
}
/**
* Computes the total entries in avaliable in the nat table size, not that this method is not guaranteed to work
* with all upnp devices since it is not an generic IGD command
@ -456,7 +455,7 @@ public class InternetGatewayDevice {
* @throws UPNPResponseException if the devices returns an error message with error code other than 713 or 402
*/
public Integer getNatTableSize() throws IOException, UPNPResponseException {
// first let's look at the first index.. some crappy devices do not start with index 0
// we stop at index 50
int startIndex = -1;
@ -493,16 +492,16 @@ public class InternetGatewayDevice {
}
return new Integer( size );
}
private void checkPortMappingProtocol( String prot ) throws IllegalArgumentException {
private static void checkPortMappingProtocol( String prot ) throws IllegalArgumentException {
if ( prot == null || ( !prot.equals( "TCP" ) && !prot.equals( "UDP" ) ) )
throw new IllegalArgumentException( "PortMappingProtocol must be either TCP or UDP" );
}
private void checkPortRange( int port ) throws IllegalArgumentException {
private static void checkPortRange( int port ) throws IllegalArgumentException {
if ( port < 1 || port > 65535 )
throw new IllegalArgumentException( "Port range must be between 1 and 65535" );
}
}

Write Preview
Loading…
Cancel
Save