boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

code cleanup

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1166 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 20 years ago
parent ae6a4650bc
commit 3d8a5ae652
49 changed files with 158 additions and 156 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/http/httpc.java
Unescape View File

@ -535,7 +535,7 @@ public final class httpc {
//serverLog.logDebug("HTTPC", handle + " initialized");
this.remoteProxyUse = false;
//this.timeout = timeout;
if(yacyDebugMode){ this.timeout=60000; }
//if(yacyDebugMode){ this.timeout=60000; }
this.savedRemoteHost = server;
try {

2
source/de/anomic/kelondro/kelondroTree.java
Unescape View File

@ -170,7 +170,6 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
// be returned, together with the information wether the new key shall be left or right child.
//
private byte[] key;
private Node thenode, parentnode;
private boolean found; // property if node was found
private byte child; // -1: left child; 0: root node; 1: right child
@ -1115,7 +1114,6 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
Handle handle;
Node node;
int linelength, width = (1 << (height - 1)) * (columnSize(0) + 1);
Handle[] childs;
String key;
for (int h = 1; h < height; h++) {
linelength = width / (thisline.size() * 2);

2
source/de/anomic/net/natLib.java
Unescape View File

@ -115,7 +115,7 @@ public class natLib {
private static boolean isIP(String ip) {
if (ip == null) return false;
try {
InetAddress dummy = InetAddress.getByName(ip);
/*InetAddress dummy =*/ InetAddress.getByName(ip);
return true;
} catch (Exception e) {
return false;

4
source/de/anomic/net/whois.java
Unescape View File

@ -48,7 +48,7 @@ import java.util.Properties;
public class whois {
public static Properties whois(String dom) {
public static Properties Whois(String dom) {
try {
Process p = Runtime.getRuntime().exec("whois " + dom);
BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()));
@ -84,7 +84,7 @@ public class whois {
}
public static void main(String[] args) {
Properties p = whois(args[0]);
Properties p = Whois(args[0]);
if (p != null) {
System.out.println(p);
System.out.println("---" + evaluateWhois(p));

4
source/de/anomic/plasma/parser/ParserException.java
Unescape View File

@ -47,7 +47,9 @@ package de.anomic.plasma.parser;
public class ParserException extends Exception
{
public ParserException() {
private static final long serialVersionUID = 1L;
public ParserException() {
super();
}

4
source/de/anomic/plasma/parser/pdf/pdfParser.java
Unescape View File

@ -98,7 +98,7 @@ public class pdfParser extends AbstractParser implements Parser {
// Logger theLogger = Logger.getLogger("org.pdfbox");
// theLogger.setLevel(Level.INFO);
String docTitle = null, docSubject = null, docAuthor = null, docKeyWords = null;
String docTitle = null, docSubject = null, /*docAuthor = null,*/ docKeyWords = null;
PDFParser parser = new PDFParser(source);
parser.parse();
@ -111,7 +111,7 @@ public class pdfParser extends AbstractParser implements Parser {
if (theDocInfo != null) {
docTitle = theDocInfo.getTitle();
docSubject = theDocInfo.getSubject();
docAuthor = theDocInfo.getAuthor();
//docAuthor = theDocInfo.getAuthor();
docKeyWords = theDocInfo.getKeywords();
}

2
source/de/anomic/plasma/parser/rss/rssParser.java
Unescape View File

@ -121,7 +121,7 @@ public class rssParser extends AbstractParser implements Parser {
String feedDescription = channel.getDescription();
// getting the channel site url
URL channelSiteURL = channel.getSite();
//URL channelSiteURL = channel.getSite();
ImageIF channelImage = channel.getImage();
if (channelImage != null) {

2
source/de/anomic/plasma/parser/tar/tarParser.java
Unescape View File

@ -142,7 +142,7 @@ public class tarParser extends AbstractParser implements Parser {
byte[] buf = new byte[(int) entry.getSize()];
int bytesRead = tin.read(buf);
/*int bytesRead =*/ tin.read(buf);
tempFile = File.createTempFile("tarParser_" + ((idx>-1)?entryName.substring(0,idx):entryName), (entryExt.length()>0)?"."+entryExt:entryExt);
serverFileUtils.write(buf, tempFile);

2
source/de/anomic/plasma/parser/zip/zipParser.java
Unescape View File

@ -124,7 +124,7 @@ public class zipParser extends AbstractParser implements Parser {
// getting the entry content
ByteArrayOutputStream bos = new ByteArrayOutputStream();
byte[] buf = new byte[(int) entry.getSize()];
int bytesRead = zippedContent.read(buf);
/*int bytesRead =*/ zippedContent.read(buf);
bos.write(buf);
byte[] ut = bos.toByteArray();

6
source/de/anomic/plasma/plasmaCondenser.java
Unescape View File

@ -57,7 +57,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
@ -153,7 +152,6 @@ public final class plasmaCondenser {
statProp sp, sp1;
int wordHandle;
int wordHandleCount = 0;
int sentenceHandle;
int sentenceHandleCount = 0;
int allwordcounter = 0;
int allsentencecounter = 0;
@ -601,6 +599,7 @@ public final class plasmaCondenser {
}
}
/*
private static void addLineSearchProp(Properties prop, String s, String[] searchwords, HashSet foundsearch) {
// we store lines containing a key in search vector
int p;
@ -622,7 +621,8 @@ public final class plasmaCondenser {
}
}
}
*/
public static Set getWords(byte[] text) {
if (text == null) return null;
ByteArrayInputStream buffer = new ByteArrayInputStream(text);

6
source/de/anomic/plasma/plasmaCrawlLURL.java
Unescape View File

@ -390,7 +390,7 @@ public final class plasmaCrawlLURL extends plasmaURL {
private String descr;
private Date moddate;
private Date loaddate;
String urlHash;
private String urlHash;
private String referrerHash;
private int copyCount;
private String flags;
@ -399,7 +399,7 @@ public final class plasmaCrawlLURL extends plasmaURL {
private char doctype;
private long size;
private int wordCount;
private String snippet;
private String snippet;
public Entry(
URL url,
@ -489,7 +489,7 @@ public final class plasmaCrawlLURL extends plasmaURL {
//System.out.println("DEBUG-ENTRY: prop=" + prop.toString());
this.urlHash = prop.getProperty("hash", dummyHash);
try {
byte[][] entry = urlHashCache.get(urlHash.getBytes());
//byte[][] entry = urlHashCache.get(urlHash.getBytes());
//if (entry == null) {
this.referrerHash = prop.getProperty("referrer", dummyHash);
this.moddate = shortDayFormatter.parse(prop.getProperty("mod", "20000101"));

5
source/de/anomic/plasma/plasmaCrawlLoader.java
Unescape View File

@ -427,11 +427,14 @@ final class CrawlerFactory implements org.apache.commons.pool.PoolableObjectFact
* @param obj
*
*/
public void passivateObject(Object obj) {
//log.debug(" passivateObject..." + obj);
/*
if (obj instanceof plasmaCrawlWorker) {
plasmaCrawlWorker theWorker = (plasmaCrawlWorker) obj;
}
*/
}
}

4
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -179,13 +179,15 @@ public class plasmaCrawlNURL extends plasmaURL {
}
}
/*
private static String normalizeHost(String host) {
if (host.length() > urlHostLength) host = host.substring(0, urlHostLength);
host = host.toLowerCase();
while (host.length() < urlHostLength) host = host + " ";
return host;
}
*/
private static String normalizeHandle(int h) {
String d = Integer.toHexString(h);
while (d.length() < urlHandleLength) d = "0" + d;

11
source/de/anomic/plasma/plasmaCrawlRobotsTxt.java
Unescape View File

@ -116,8 +116,12 @@ public class plasmaCrawlRobotsTxt {
public void removeEntry(String hostName) {
try {
robotsTable.remove(hostName.toLowerCase());
} catch (IOException e) {}
}
} catch (IOException e) {
} catch (kelondroException e) {
resetDatabase();
}
}
public Entry getEntry(String hostName) {
try {
@ -126,6 +130,9 @@ public class plasmaCrawlRobotsTxt {
return new Entry(hostName, record);
} catch (IOException e) {
return null;
} catch (kelondroException e) {
resetDatabase();
return null;
}
}

4
source/de/anomic/plasma/plasmaCrawlStacker.java
Unescape View File

@ -75,7 +75,7 @@ public final class plasmaCrawlStacker {
final ThreadGroup theWorkerThreadGroup = new ThreadGroup("stackCrawlThreadGroup");
final serverLog log = new serverLog("STACKCRAWL");
final plasmaSwitchboard sb;
private boolean stopped = false;
//private boolean stopped = false;
private stackCrawlQueue queue;
public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, int dbCacheSize) throws IOException {
@ -152,7 +152,7 @@ public final class plasmaCrawlStacker {
worker.execute(theMsg);
} catch (InterruptedException e) {
Thread.interrupted();
this.stopped = true;
//this.stopped = true;
}
catch (Exception e) {
this.log.logSevere("plasmaStackCrawlThread.run/loop", e);

7
source/de/anomic/plasma/plasmaCrawlWorker.java
Unescape View File

@ -87,7 +87,8 @@ public final class plasmaCrawlWorker extends Thread {
private boolean stopped = false;
private boolean done = false;
private static boolean doCrawlerLogging = false;
//private static boolean doCrawlerLogging = false;
/**
* Do logging configuration for special proxy access log file
*/
@ -324,7 +325,7 @@ public final class plasmaCrawlWorker extends Thread {
}
// TODO: resolve yacy and yacyh domains
String yAddress = yacyCore.seedDB.resolveYacyAddress(host);
//String yAddress = yacyCore.seedDB.resolveYacyAddress(host);
// set referrer; in some case advertise a little bit:
referer = (referer == null) ? "" : referer.trim();
@ -358,7 +359,7 @@ public final class plasmaCrawlWorker extends Thread {
// the transfer is ok
// TODO: aborting download if content is to long ...
long contentLength = res.responseHeader.contentLength();
//long contentLength = res.responseHeader.contentLength();
// reserve cache entry
plasmaHTCache.Entry htCache = cacheManager.newEntry(requestDate, depth, url, name, requestHeader, res.status, res.responseHeader, initiator, profile);

6
source/de/anomic/plasma/plasmaDbImporter.java
Unescape View File

@ -20,13 +20,13 @@ public class plasmaDbImporter extends Thread {
private final plasmaCrawlLURL importUrlDB;
private final plasmaWordIndex importWordIndex;
private final String importPath;
//private final String importPath;
private final File importRoot;
private final int importStartSize;
private final serverLog log;
private boolean stopped = false;
private boolean paused = false;
//private boolean paused = false;
private String wordHash = "------------";
long wordChunkStart = System.currentTimeMillis(), wordChunkEnd = wordChunkStart;
@ -85,7 +85,7 @@ public class plasmaDbImporter extends Thread {
}
if (theImportPath == null) throw new NullPointerException();
this.importPath = theImportPath;
//this.importPath = theImportPath;
this.importRoot = new File(theImportPath);
if (theHomeIndexDB == null) throw new NullPointerException();

4
source/de/anomic/plasma/plasmaParser.java
Unescape View File

@ -865,9 +865,11 @@ final class plasmaParserFactory implements KeyedPoolableObjectFactory {
* @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object)
*/
public void destroyObject(Object key, Object obj) {
/*
if (obj instanceof Parser) {
Parser theParser = (Parser) obj;
}
*/
}
/**
@ -875,7 +877,7 @@ final class plasmaParserFactory implements KeyedPoolableObjectFactory {
*/
public boolean validateObject(Object key, Object obj) {
if (obj instanceof Parser) {
Parser theParser = (Parser) obj;
//Parser theParser = (Parser) obj;
return true;
}
return true;

2
source/de/anomic/plasma/plasmaParserDocument.java
Unescape View File

@ -91,6 +91,7 @@ public class plasmaParserDocument {
this.condenser = null;
}
/*
private String absolutePath(String relativePath) {
try {
return htmlFilterContentScraper.urlNormalform(location, relativePath);
@ -98,6 +99,7 @@ public class plasmaParserDocument {
return "";
}
}
*/
public String getMimeType() {
return this.mimeType;

1
source/de/anomic/plasma/plasmaRankingCRProcess.java
Unescape View File

@ -169,7 +169,6 @@ public class plasmaRankingCRProcess {
acc = new kelondroAttrSeq(to_file, false);
// collect source files
kelondroAttrSeq source_cr = null;
File source_file = null;
String[] files = from_dir.list();
if (files.length < max_files) max_files = files.length;

1
source/de/anomic/plasma/plasmaRankingRCIEvaluation.java
Unescape View File

@ -183,7 +183,6 @@ public class plasmaRankingRCIEvaluation {
}
public static void storeRankingTable(TreeSet[] ranking, File tablePath) throws IOException {
String hash;
String filename;
if (!(tablePath.exists())) tablePath.mkdirs();
for (int i = 0; i < ranking.length - 1; i++) {

6
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -133,7 +133,7 @@ public final class plasmaSearchEvent {
}
} else {
// do a local search
long start = System.currentTimeMillis();
//long start = System.currentTimeMillis();
try {
localSearch();
plasmaSearchResult result = order();
@ -193,7 +193,7 @@ public final class plasmaSearchEvent {
if (rcGlobal.size() >= profileGlobal.getTargetCount(plasmaSearchProfile.PROCESS_POSTSORT) * 3) break; // we have enough
if (yacySearch.remainingWaiting(searchThreads) == 0) break; // we cannot expect more
// wait a little time ..
try {Thread.currentThread().sleep(100);} catch (InterruptedException e) {}
try {Thread.sleep(100);} catch (InterruptedException e) {}
}
return rcGlobal.size();
@ -286,7 +286,7 @@ public final class plasmaSearchEvent {
rcGlobal.deleteComplete();
}
// wait a little bit before trying again
try {Thread.currentThread().sleep(3000);} catch (InterruptedException e) {}
try {Thread.sleep(3000);} catch (InterruptedException e) {}
if (System.currentTimeMillis() - starttime > 90000) {
yacySearch.interruptAlive(searchThreads);
serverLog.logFine("PLASMA", "SEARCH FLUSH: " + remaining + " PEERS STILL BUSY; ABANDONED; SEARCH WAS " + query.queryWords);

2
source/de/anomic/plasma/plasmaSearchResult.java
Unescape View File

@ -277,12 +277,14 @@ public final class plasmaSearchResult {
}
}
/*
private void printSplitLog(String x, String[] y) {
String s = "";
for (int i = 0; i < y.length; i++) s = s + ", " + y[i];
if (s.length() > 0) s = s.substring(2);
System.out.println("Split '" + x + "' = {" + s + "}");
}
*/
public static void main(String[] args) {
URL[] urls = new URL[10];

1
source/de/anomic/plasma/plasmaSnippetCache.java
Unescape View File

@ -390,7 +390,6 @@ public class plasmaSnippetCache {
int i = 0;
plasmaCrawlLURL.Entry urlentry;
String urlstring;
plasmaSnippetCache.result snippet;
long limitTime = (maxTime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxTime;
while ((acc.hasMoreElements()) && (i < fetchcount) && (System.currentTimeMillis() < limitTime)) {
urlentry = acc.nextElement();

42
source/de/anomic/plasma/plasmaStore.java
Unescape View File

@ -38,7 +38,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
//import java.io.RandomAccessFile;
public class plasmaStore {
@ -104,26 +104,28 @@ public class plasmaStore {
}
*/
/*
private static long[] appendFileToStack(File fragment, File dest) throws IOException {
// returns a long[2] with
// long[0] = startOfFileFragemt in dest
// long[1] = lengthOfFileFragment in dest
long l = fragment.length();
long p = dest.length();
RandomAccessFile fo = new RandomAccessFile(dest, "rw");
FileInputStream fi = new FileInputStream(fragment);
byte[] buffer = new byte[1024];
int c;
fo.seek(p);
while ((c = fi.read(buffer)) >= 0) fo.write(buffer, 0, c);
fi.close();
fo.close();
long[] r = new long[2];
r[0] = p;
r[1] = l;
return r;
}
// returns a long[2] with
// long[0] = startOfFileFragemt in dest
// long[1] = lengthOfFileFragment in dest
long l = fragment.length();
long p = dest.length();
RandomAccessFile fo = new RandomAccessFile(dest, "rw");
FileInputStream fi = new FileInputStream(fragment);
byte[] buffer = new byte[1024];
int c;
fo.seek(p);
while ((c = fi.read(buffer)) >= 0)
fo.write(buffer, 0, c);
fi.close();
fo.close();
long[] r = new long[2];
r[0] = p;
r[1] = l;
return r;
}
*/
/*
public static void main(String[] args) {

13
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -391,16 +391,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
// start a loader
log.logConfig("Starting Crawl Loader");
int remoteport;
try { remoteport = Integer.parseInt(getConfig("remoteProxyPort","3128")); }
catch (NumberFormatException e) { remoteport = 3128; }
crawlSlots = Integer.parseInt(getConfig("crawler.MaxActiveThreads", "10"));
this.crawlingIsPaused = Boolean.valueOf(getConfig("crawler.isPaused", "false")).booleanValue();
plasmaCrawlLoader.switchboard = this;
this.cacheLoader = new plasmaCrawlLoader(
this.cacheManager,
this.log);
this.cacheLoader = new plasmaCrawlLoader(this.cacheManager, this.log);
// starting message board
this.log.logConfig("Starting Message Board");
@ -506,9 +501,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
new serverInstantThread(this, "cleanupJob", "cleanupJobSize"), 10000); // all 5 Minutes
deployThread("82_crawlstack", "Crawl URL Stacker", "process that checks url for double-occurrences and for allowance/disallowance by robots.txt", null,
new serverInstantThread(sbStackCrawlThread, "job", "size"), 8000);
serverInstantThread indexingThread = null;
deployThread("80_indexing", "Parsing/Indexing", "thread that performes document parsing and indexing", "/IndexCreateIndexingQueue_p.html",
indexingThread = new serverInstantThread(this, "deQueue", "queueSize"), 10000);
new serverInstantThread(this, "deQueue", "queueSize"), 10000);
for (int i = 1; i < indexing_cluster; i++) {
setConfig((i + 80) + "_indexing_idlesleep", getConfig("80_indexing_idlesleep", ""));
setConfig((i + 80) + "_indexing_busysleep", getConfig("80_indexing_busysleep", ""));
@ -1179,8 +1173,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
Map hl = document.getHyperlinks();
Iterator i = hl.entrySet().iterator();
String nexturlstring;
String rejectReason;
int c = 0;
//String rejectReason;
Map.Entry e;
while (i.hasNext()) {
e = (Map.Entry) i.next();

4
source/de/anomic/plasma/plasmaWordConnotation.java
Unescape View File

@ -47,7 +47,6 @@ package de.anomic.plasma;
import java.io.File;
import java.io.IOException;
import java.util.TreeMap;
import de.anomic.kelondro.kelondroDynTree;
import de.anomic.server.serverCodings;
@ -91,9 +90,10 @@ public class plasmaWordConnotation {
addSentence(sentence.split(" "));
}
/*
public String[] getConnotation(String word, int count) {
TreeMap map = new TreeMap();
return null;
}
*/
}

50
source/de/anomic/plasma/plasmaWordIndexAssortment.java
Unescape View File

@ -158,29 +158,35 @@ public final class plasmaWordIndexAssortment {
}
public plasmaWordIndexEntryContainer remove(String wordHash) {
// deletes a word index from assortment database
// and returns the content record
byte[][] row = null;
try {
row = assortments.remove(wordHash.getBytes());
} catch (IOException e) {
log.logSevere("removeAssortment/IO-error: " + e.getMessage() + " - reset assortment-DB " + assortments.file(), e);
resetDatabase();
return null;
} catch (kelondroException e) {
log.logSevere("removeAssortment/kelondro-error: " + e.getMessage() + " - reset assortment-DB " + assortments.file(), e);
resetDatabase();
return null;
}
if (row == null) return null;
long updateTime = kelondroRecords.bytes2long(row[2]);
plasmaWordIndexEntry[] wordEntries = new plasmaWordIndexEntry[this.bufferStructureLength];
plasmaWordIndexEntryContainer container = new plasmaWordIndexEntryContainer(wordHash);
for (int i = 0; i < assortmentLength; i++) {
container.add(new plasmaWordIndexEntry[]{new plasmaWordIndexEntry(new String(row[3 + 2 * i]), new String(row[4 + 2 * i]))}, updateTime);
// deletes a word index from assortment database
// and returns the content record
byte[][] row = null;
try {
row = assortments.remove(wordHash.getBytes());
} catch (IOException e) {
log.logSevere("removeAssortment/IO-error: " + e.getMessage()
+ " - reset assortment-DB " + assortments.file(), e);
resetDatabase();
return null;
} catch (kelondroException e) {
log.logSevere("removeAssortment/kelondro-error: " + e.getMessage()
+ " - reset assortment-DB " + assortments.file(), e);
resetDatabase();
return null;
}
if (row == null)
return null;
long updateTime = kelondroRecords.bytes2long(row[2]);
// plasmaWordIndexEntry[] wordEntries = new plasmaWordIndexEntry[this.bufferStructureLength];
plasmaWordIndexEntryContainer container = new plasmaWordIndexEntryContainer(wordHash);
for (int i = 0; i < assortmentLength; i++) {
container.add(
new plasmaWordIndexEntry[] { new plasmaWordIndexEntry(
new String(row[3 + 2 * i]), new String(
row[4 + 2 * i])) }, updateTime);
}
return container;
}
return container;
}
private void resetDatabase() {
// deletes the assortment database and creates a new one

5
source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java
Unescape View File

@ -57,11 +57,10 @@ import de.anomic.server.logging.serverLog;
public final class plasmaWordIndexAssortmentCluster {
// class variables
private File assortmentsPath;
private int clusterCount;
public int clusterCapacity;
private serverLog log;
//private serverLog log;
private plasmaWordIndexAssortment[] assortments;
private long completeBufferKB;
@ -71,7 +70,7 @@ public final class plasmaWordIndexAssortmentCluster {
this.clusterCount = clusterCount;
this.clusterCapacity = clusterCount * (clusterCount + 1) / 2;
this.completeBufferKB = bufferkb;
this.log = log;
//this.log = log;
this.assortments = new plasmaWordIndexAssortment[clusterCount];
// open cluster and close it directly again to detect the element sizes

17
source/de/anomic/plasma/plasmaWordIndexCache.java
Unescape View File

@ -81,12 +81,10 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
private final flush flushThread;
// calculated constants
private static String minKey, maxKey;
private static String maxKey;
static {
maxKey = "";
for (int i = 0; i < yacySeedDB.commonHashLength; i++) maxKey += 'z';
minKey = "";
for (int i = 0; i < yacySeedDB.commonHashLength; i++) maxKey += '-';
maxKey = ""; for (int i = 0; i < yacySeedDB.commonHashLength; i++) maxKey += 'z';
//minKey = ""; for (int i = 0; i < yacySeedDB.commonHashLength; i++) maxKey += '-';
}
public plasmaWordIndexCache(File databaseRoot, plasmaWordIndexInterface backend, int assortmentbufferkb, serverLog log) {
@ -201,7 +199,6 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
synchronized (cache) {
int i = dumpArray.size();
String wordHash;
plasmaWordIndexEntryContainer container;
long creationTime;
plasmaWordIndexEntry wordEntry;
byte[][] row;
@ -304,24 +301,22 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface {
}
public void run() {
String nextHash;
Runtime rt = Runtime.getRuntime();
long pausetime;
while (!terminate) {
if (intermission > 0) {
if (this.intermission > System.currentTimeMillis()) {
try {this.sleep(this.intermission - System.currentTimeMillis());} catch (InterruptedException e) {}
try {sleep(this.intermission - System.currentTimeMillis());} catch (InterruptedException e) {}
}
this.intermission = 0;
}
if (pause) {
try {this.sleep(300);} catch (InterruptedException e) {}
try {sleep(300);} catch (InterruptedException e) {}
} else {
flushFromMem();
try {
pausetime = 1 + java.lang.Math.min(1000, 5 * maxWordsHigh/(cache.size() + 1));
if (cache.size() == 0) pausetime = 2000;
this.sleep(pausetime);
sleep(pausetime);
} catch (InterruptedException e) {}
}
}

8
source/de/anomic/plasma/plasmaWordIndexDistribution.java
Unescape View File

@ -202,7 +202,7 @@ public final class plasmaWordIndexDistribution {
log.logFine("Selected hash " + startPointHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, startPointHash));
Object[] selectResult = selectTransferIndexes(startPointHash, indexCount, this.maxOpenFiles);
plasmaWordIndexEntity[] indexEntities = (plasmaWordIndexEntity[]) selectResult[0];
Integer openedFiles = (Integer) selectResult[2];
//Integer openedFiles = (Integer) selectResult[2];
HashMap urlCache = (HashMap) selectResult[1]; // String (url-hash) / plasmaCrawlLURL.Entry
if ((indexEntities == null) || (indexEntities.length == 0)) {
log.logFine("No index available for index transfer, hash start-point " + startPointHash);
@ -415,7 +415,6 @@ public final class plasmaWordIndexDistribution {
}
boolean deleteTransferIndexes(plasmaWordIndexEntity[] indexEntities) throws IOException {
String wordhash;
Iterator urlIter;
plasmaWordIndexEntry indexEntry;
plasmaWordIndexEntity indexEntity;
@ -672,8 +671,7 @@ public final class plasmaWordIndexDistribution {
private int chunkSize = 500;
private final long startingTime = System.currentTimeMillis();
private final plasmaSwitchboard sb;
private final Runtime rt = Runtime.getRuntime();
transferIndexWorkerThread worker = null;
private transferIndexWorkerThread worker = null;
public transferIndexThread(yacySeed seed, boolean delete) {
super(new ThreadGroup("TransferIndexThreadGroup"),"TransferIndex_" + seed.getName());
@ -911,6 +909,7 @@ public final class plasmaWordIndexDistribution {
} catch (IOException ee) {}
}
/*
private boolean isAborted() {
if (finished || Thread.currentThread().isInterrupted()) {
this.status = "aborted";
@ -918,6 +917,7 @@ public final class plasmaWordIndexDistribution {
}
return false;
}
*/
}
}

8
source/de/anomic/server/logging/serverLog.java
Unescape View File

@ -110,20 +110,12 @@ public final class serverLog {
public void logFinest(String message, Throwable thrown) {this.theLogger.log(Level.FINEST,message,thrown);}
public boolean isFinest() { return this.theLogger.isLoggable(Level.FINEST); }
private void log(Level level, String msg, Throwable thrown) {
this.theLogger.log(level, msg, thrown);
}
public boolean isLoggable(Level level) {
return this.theLogger.isLoggable(level);
}
// static log messages: log everything
private static void log(String appName, int messageLevel, String message) {
Logger.getLogger(appName).log(Level.parse(Integer.toString(messageLevel)),message);
}
public static void logSevere(String appName, String message) {
Logger.getLogger(appName).severe(message);
}

6
source/de/anomic/server/serverAbstractThread.java
Unescape View File

@ -194,10 +194,12 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
// If we reach this point, the process is closed
}
/*
private final void logError(String text) {
if (log == null) serverLog.logSevere("THREAD-CONTROL", text);
else log.logSevere(text);
}
*/
private final void logError(String text,Throwable thrown) {
if (log == null) serverLog.logSevere("THREAD-CONTROL", text, thrown);
@ -209,10 +211,12 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
else log.logConfig(text);
}
/*
private void logSystem(String text, Throwable thrown) {
if (log == null) serverLog.logConfig("THREAD-CONTROL", text, thrown);
else log.logConfig(text,thrown);
}
*/
public void jobExceptionHandler(Exception e) {
if (!(e instanceof ClosedByInterruptException)) {
@ -235,8 +239,6 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
else
logSystem("thread '" + this.getName() + "' deployed, " + ((this.busyPause < 0) ? "starting job." : "starting loop."));
}
int outerloop;
long innerpause;
long timestamp;
long memstamp0, memstamp1;
boolean isBusy;

7
source/de/anomic/server/serverByteBuffer.java
Unescape View File

@ -114,11 +114,10 @@ public final class serverByteBuffer extends OutputStream {
FileInputStream fis = new FileInputStream(f);
// byte[] buf = new byte[512];
// int p = 0;
int l;
// while ((l = fis.read(buf)) > 0) {
// System.arraycopy(buf, 0, buffer, p, l);
// p += l;
l = fis.read(buffer);
/*int l =*/ fis.read(buffer);
// }
fis.close();
} catch (FileNotFoundException e) {
@ -230,13 +229,15 @@ public final class serverByteBuffer extends OutputStream {
return tmp;
}
/*
private serverByteBuffer trim(int start) {
if (start > length) throw new IndexOutOfBoundsException("trim: start > length");
offset = offset + start;
length = length - start;
return this;
}
*/
private serverByteBuffer trim(int start, int end) {
// the end value is outside (+1) of the wanted target array
if (start > length) throw new IndexOutOfBoundsException("trim: start > length");

1
source/de/anomic/server/serverCodings.java
Unescape View File

@ -193,7 +193,6 @@ public final class serverCodings {
if (rfc1113compliant) while (in.charAt(in.length() - 1) == '=') in = in.substring(0, in.length() - 1);
byte[] out = new byte[in.length() / 4 * 3 + (((in.length() % 4) == 0) ? 0 : in.length() % 4 - 1)];
long l;
char c1, c2, c3;
while (posIn + 3 < in.length()) {
l = decodeBase64Long(in.substring(posIn, posIn + 4));
out[posOut+2] = (byte) (l % 256); l = l / 256;

16
source/de/anomic/server/serverCore.java
Unescape View File

@ -46,8 +46,6 @@
package de.anomic.server;
// standard server
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -64,14 +62,18 @@ import java.net.SocketException;
import java.net.URL;
import java.net.UnknownHostException;
import java.nio.channels.ClosedByInterruptException;
import java.security.KeyStore;
import java.util.Enumeration;
import java.util.Hashtable;
/*
import java.io.File;
import java.io.FileInputStream;
import java.security.KeyStore;
import javax.net.ServerSocketFactory;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLServerSocketFactory;
*/
import org.apache.commons.pool.impl.GenericObjectPool;
import org.apache.commons.pool.impl.GenericObjectPool.Config;
@ -126,6 +128,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
return this.theSessionThreadGroup;
}
/*
private static ServerSocketFactory getServerSocketFactory(boolean dflt, File keyfile, String passphrase) {
// see doc's at
// http://java.sun.com/developer/technicalArticles/Security/secureinternet/
@ -163,6 +166,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
}
}
}
*/
public static String clientAddress(Socket s) {
InetAddress uAddr = s.getInetAddress();
@ -1001,7 +1005,7 @@ public final class serverCore extends serverAbstractThread implements serverThre
try {
// set up some reflection
Class[] stringType = {"".getClass()};
Class[] exceptionType = {Class.forName("java.lang.Throwable")};
//Class[] exceptionType = {Class.forName("java.lang.Throwable")};
Object result;
// // send greeting
@ -1150,10 +1154,10 @@ public final class serverCore extends serverAbstractThread implements serverThre
shortReset();
} // end of while
} catch (java.lang.ClassNotFoundException e) {
} /* catch (java.lang.ClassNotFoundException e) {
System.out.println("Internal error: Wrapper class not found: " + e.getMessage());
System.exit(0);
} catch (java.io.IOException e) {
} */ catch (java.io.IOException e) {
// connection interruption: more or less normal
}
//announceMoreExecTime(System.currentTimeMillis() - this.start);

8
source/de/anomic/server/serverDate.java
Unescape View File

@ -66,7 +66,7 @@ public final class serverDate {
private final static int[] dimnormal = {january, normalfebruary, march, april, may, june, july, august, september, october, november, december};
private final static int[] dimleap = {january, leapfebruary, march, april, may, june, july, august, september, october, november, december};
private final static String[] wkday = {"Mon","Tue","Wed","Thu","Fri","Sat","Sun"};
private final static String[] month = {"Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"};
//private final static String[] month = {"Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"};
// find out time zone and DST offset
private static Calendar thisCalendar = GregorianCalendar.getInstance();
@ -317,16 +317,16 @@ public final class serverDate {
} catch (java.text.ParseException e) {
System.out.println("Parse Exception: " + e.getMessage() + ", pos " + e.getErrorOffset());
}
String testresult;
//String testresult;
int cycles = 10000;
long start;
start = System.currentTimeMillis();
for (int i = 0; i < cycles; i++) testresult = new serverDate().toShortString(false);
for (int i = 0; i < cycles; i++) /*testresult =*/ new serverDate().toShortString(false);
System.out.println("time for " + cycles + " calls to serverDate:" + (System.currentTimeMillis() - start) + " milliseconds");
start = System.currentTimeMillis();
for (int i = 0; i < cycles; i++) testresult = testSDateShortString();
for (int i = 0; i < cycles; i++) /*testresult =*/ testSDateShortString();
System.out.println("time for " + cycles + " calls to javaDate:" + (System.currentTimeMillis() - start) + " milliseconds");
}
}

4
source/de/anomic/server/serverObjects.java
Unescape View File

@ -68,7 +68,9 @@ import java.util.Map;
public final class serverObjects extends Hashtable implements Cloneable {
public serverObjects() {
private static final long serialVersionUID = 1L;
public serverObjects() {
super();
}

2
source/de/anomic/server/serverSwitchAbstractAction.java
Unescape View File

@ -44,7 +44,7 @@ import de.anomic.server.logging.serverLog;
public abstract class serverSwitchAbstractAction {
private serverLog log = null;
protected serverLog log = null;
private String shortDescr = "", longDescr = "";
public void setDescription(String shortText, String longText) {

2
source/de/anomic/tools/loaderThreads.java
Unescape View File

@ -132,7 +132,6 @@ public class loaderThreads {
protected class loaderThread extends Thread {
private URL url;
private Exception error;
private long starttime;
private loaderProcess process;
private ArrayList page;
private boolean loaded;
@ -141,7 +140,6 @@ public class loaderThreads {
this.url = url;
this.process = process;
this.error = null;
this.starttime = System.currentTimeMillis();
this.page = null;
this.loaded = false;
}

2
source/de/anomic/yacy/seedUpload/yacySeedUploadFile.java
Unescape View File

@ -58,7 +58,7 @@ public class yacySeedUploadFile implements yacySeedUploader {
public String uploadSeedFile(serverSwitch sb, yacySeedDB seedDB, File seedFile) throws Exception {
String logt, seedFilePath = "";
String seedFilePath = "";
try {
seedFilePath = sb.getConfig(CONFIG_FILE_PATH,"");
if (seedFilePath.length() == 0) throw new Exception("Path to seed file is not configured properly");

1
source/de/anomic/yacy/seedUpload/yacySeedUploadScp.java
Unescape View File

@ -144,7 +144,6 @@ class sshc {
channel.connect();
byte[] tmp=new byte[1];
checkAck(in);
// send "C0644 filesize filename", where filename should not include '/'

6
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -672,8 +672,8 @@ public final class yacyClient {
String access = (String) phase1.get("access");
String nextaddress = (String) phase1.get("address");
String protocol = (String) phase1.get("protocol");
String path = (String) phase1.get("path");
String maxsize = (String) phase1.get("maxsize");
//String path = (String) phase1.get("path");
//String maxsize = (String) phase1.get("maxsize");
String response = (String) phase1.get("response");
if ((response == null) || (protocol == null) || (access == null)) return "wrong return values from other peer; phase 1";
if (!(response.equals("ok"))) return "remote peer rejected transfer: " + response;
@ -1011,7 +1011,7 @@ public final class yacyClient {
System.out.println("yacyClient Test");
try {
final plasmaSwitchboard sb = new plasmaSwitchboard(args[0], "httpProxy.init", "DATA/SETTINGS/httpProxy.conf");
final yacyCore core = new yacyCore(sb);
/*final yacyCore core =*/ new yacyCore(sb);
yacyCore.peerActions.loadSeedLists();
final yacySeed target = yacyCore.seedDB.getConnected(args[1]);
final String wordhashe = plasmaWordIndexEntry.word2hash("test");

6
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -102,7 +102,7 @@ public class yacyCore {
// class variables
private int lastSeedUpload_seedDBSize = 0;
public long lastSeedUpload_timeStamp = System.currentTimeMillis();
private String lastSeedUpload_myPeerType = "";
//private String lastSeedUpload_myPeerType = "";
private String lastSeedUpload_myIP = "";
private static int onlineMode = 1;
@ -310,7 +310,7 @@ public class yacyCore {
}
log.logInfo("re-connect own seed");
final String oldAddress = seedDB.mySeed.getAddress();
final int newSeeds = publishMySeed(true);
/*final int newSeeds =*/ publishMySeed(true);
return (oldAddress != null && oldAddress.equals(seedDB.mySeed.getAddress()));
}
@ -731,7 +731,7 @@ public class yacyCore {
this.lastSeedUpload_timeStamp = System.currentTimeMillis();
this.lastSeedUpload_myIP = seedDB.mySeed.get(yacySeed.IP, "127.0.0.1");
this.lastSeedUpload_myPeerType = seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
//this.lastSeedUpload_myPeerType = seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
}
}

1
source/de/anomic/yacy/yacyNewsPool.java
Unescape View File

@ -222,7 +222,6 @@ public class yacyNewsPool {
public synchronized yacyNewsRecord getByOriginator(int dbKey, String category, String originatorHash) throws IOException {
yacyNewsQueue queue = switchQueue(dbKey);
yacyNewsRecord record;
String s;
for (int i = queue.size() - 1; i >= 0; i--) {
record = queue.top(i);
if ((record != null) &&

2
source/de/anomic/yacy/yacySearch.java
Unescape View File

@ -188,7 +188,7 @@ public class yacySearch extends Thread {
searchThreads[i]= new yacySearch(wordhashes, true, targetPeers[i],
urlManager, entityCache, blacklist, snippetCache, profile);
searchThreads[i].start();
try {Thread.currentThread().sleep(20);} catch (InterruptedException e) {}
try {Thread.sleep(20);} catch (InterruptedException e) {}
}
return searchThreads;

5
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -68,7 +68,6 @@ import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.disorderHeap;
import de.anomic.yacy.yacySeed;
public final class yacySeedDB {
@ -87,7 +86,6 @@ public final class yacySeedDB {
// class objects
private File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
private disorderHeap seedQueue;
private kelondroMap seedActiveDB, seedPassiveDB, seedPotentialDB;
private int seedDBBufferKB;
@ -146,9 +144,6 @@ public final class yacySeedDB {
// check if we are in the seedCaches: this can happen if someone else published our seed
removeMySeed();
// set up seed queue (for probing candidates)
seedQueue = null;
}
public synchronized void removeMySeed() {

2
source/de/anomic/ymage/ymageMatrix.java
Unescape View File

@ -114,11 +114,13 @@ public class ymageMatrix implements Cloneable {
//return Integer.parseInt(col.substring(0,2), 16) << 16 | Integer.parseInt(col.substring(2,4), 16) << 8 | Integer.parseInt(col.substring(4,6), 16);
}
/*
private static String colStr(long c) {
String s = Long.toHexString(c);
while (s.length() < 6) s = "0" + s;
return s;
}
*/
public Object clone() {
return new ymageMatrix(this);

6
source/de/anomic/ymage/ymagePNGEncoderJDE.java
Unescape View File

@ -96,8 +96,7 @@ public class ymagePNGEncoderJDE extends Object
public byte[] pngEncode( boolean encodeAlpha )
{
byte[] pngIdBytes = { -119, 80, 78, 71, 13, 10, 26, 10 };
int i;
if (matrix == null)
{
return null;
@ -375,7 +374,6 @@ public class ymagePNGEncoderJDE extends Object
int nBytes = width * bytesPerPixel;
int leftInsert = offset;
int leftExtract = 0;
byte current_byte;
for (i=actualStart; i < startPos + nBytes; i++)
{
@ -433,7 +431,7 @@ public class ymagePNGEncoderJDE extends Object
byte[] compressedLines; // the resultant compressed lines
int nCompressed; // how big is the compressed area?
int depth; // color depth ( handle only 8 or 32 )
//int depth; // color depth ( handle only 8 or 32 )
bytesPerPixel = (encodeAlpha) ? 4 : 3;

Write Preview
Loading…
Cancel
Save