boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- fixed bug with too much RAM in crawler queue

Browse Source 
- fixed dir bug
- better calculation of TF for join
- better waiting-on-result logic

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4424 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent a8a5df4a51
commit acf771d5e1
17 changed files with 93 additions and 71 deletions
Show Stats Download Patch File Download Diff File

2
build.properties
Unescape View File

@ -3,7 +3,7 @@ javacSource=1.5
javacTarget=1.5
# Release Configuration
releaseVersion=0.564
releaseVersion=0.565
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz

3
source/de/anomic/http/httpdSwitchboard.java
Unescape View File

@ -40,6 +40,7 @@
package de.anomic.http;
import java.io.File;
import java.util.LinkedList;
import de.anomic.server.serverAbstractSwitch;
@ -50,7 +51,7 @@ public final class httpdSwitchboard extends serverAbstractSwitch implements serv
private final LinkedList<Object> cacheStack;
public httpdSwitchboard(String rootPath, String initPath, String configPath, boolean applyPro) {
public httpdSwitchboard(File rootPath, String initPath, String configPath, boolean applyPro) {
super(rootPath, initPath, configPath, applyPro);
cacheStack = new LinkedList<Object>();
}

6
source/de/anomic/index/indexContainer.java
Unescape View File

@ -325,7 +325,7 @@ public class indexContainer extends kelondroRowSet {
assert (ie0.urlHash().length() == keylength) : "ie0.urlHash() = " + ie0.urlHash();
assert (ie1.urlHash().length() == keylength) : "ie1.urlHash() = " + ie1.urlHash();
// this is a hit. Calculate word distance:
ie0.combineDistance(ie1);
ie0.join(ie1);
if (ie0.worddistance() <= maxDistance) conj.add(ie0);
}
}
@ -360,7 +360,7 @@ public class indexContainer extends kelondroRowSet {
if (e2.hasNext()) ie2 = (indexRWIEntry) e2.next(); else break;
} else {
// we have found the same urls in different searches!
ie1.combineDistance(ie2);
ie1.join(ie2);
if (ie1.worddistance() <= maxDistance) conj.add(ie1);
if (e1.hasNext()) ie1 = (indexRWIEntry) e1.next(); else break;
if (e2.hasNext()) ie2 = (indexRWIEntry) e2.next(); else break;
@ -435,7 +435,7 @@ public class indexContainer extends kelondroRowSet {
if (e2.hasNext()) ie2 = (indexRWIEntry) e2.next(); else break;
} else {
// we have found the same urls in different searches!
ie1.combineDistance(ie2);
ie1.join(ie2);
e1.remove();
if (e1.hasNext()) ie1 = (indexRWIEntry) e1.next(); else break;
if (e2.hasNext()) ie2 = (indexRWIEntry) e2.next(); else break;

2
source/de/anomic/index/indexRWIEntry.java
Unescape View File

@ -87,7 +87,7 @@ public interface indexRWIEntry {
public String toString();
public void combineDistance(indexRWIEntry oe);
public void join(indexRWIEntry oe);
public int worddistance();

14
source/de/anomic/index/indexRWIRowEntry.java
Unescape View File

@ -263,18 +263,22 @@ public final class indexRWIRowEntry implements indexRWIEntry {
return toPropertyForm();
}
public static indexRWIEntry combineDistance(indexRWIRowEntry ie1, indexRWIEntry ie2) {
public static indexRWIEntry join(indexRWIRowEntry ie1, indexRWIEntry ie2) {
// returns a modified entry of the first argument
// combine the distance
ie1.entry.setCol(col_worddistance, ie1.worddistance() + ie2.worddistance() + Math.abs(ie1.posintext() - ie2.posintext()));
ie1.entry.setCol(col_posintext, Math.min(ie1.posintext(), ie2.posintext()));
ie1.entry.setCol(col_posinphrase, (ie1.posofphrase() == ie2.posofphrase()) ? ie1.posofphrase() : 0 /*unknown*/);
ie1.entry.setCol(col_posinphrase, (ie1.posofphrase() == ie2.posofphrase()) ? Math.min(ie1.posinphrase(), ie2.posinphrase()) : 0 /*unknown*/);
ie1.entry.setCol(col_posofphrase, Math.min(ie1.posofphrase(), ie2.posofphrase()));
ie1.entry.setCol(col_wordsInText, (ie1.wordsintext() + ie2.wordsintext()) / 2);
// combine term frequency
ie1.entry.setCol(col_wordsInText, ie1.wordsintext() + ie2.wordsintext());
return ie1;
}
public void combineDistance(indexRWIEntry oe) {
combineDistance(this, oe);
public void join(indexRWIEntry oe) {
join(this, oe);
}
public int worddistance() {

12
source/de/anomic/index/indexRWIVarEntry.java
Unescape View File

@ -54,7 +54,7 @@ public class indexRWIVarEntry implements indexRWIEntry {
this.phrasesintext = e.phrasesintext();
this.posintext = e.posintext();
this.posinphrase = e.posinphrase();
this.posofphrase= e.posofphrase();
this.posofphrase = e.posofphrase();
this.quality = e.quality();
this.urlcomps = e.urlcomps();
this.urllength = e.urllength();
@ -65,7 +65,15 @@ public class indexRWIVarEntry implements indexRWIEntry {
this.termFrequency = e.termFrequency();
}
public void combineDistance(indexRWIEntry oe) {
public void join(indexRWIEntry oe) {
// combine the distance
this.worddistance = this.worddistance() + oe.worddistance() + Math.abs(this.posintext() - oe.posintext());
this.posintext = Math.min(this.posintext(), oe.posintext());
this.posinphrase = (this.posofphrase() == oe.posofphrase()) ? Math.min(this.posinphrase(), oe.posinphrase()) : 0;
this.posofphrase = Math.min(this.posofphrase(), oe.posofphrase());
// combine term frequency
this.wordsintext = this.wordsintext() + oe.wordsintext();
}
public kelondroBitfield flags() {

4
source/de/anomic/plasma/plasmaCrawlNURL.java
Unescape View File

@ -74,10 +74,10 @@ public class plasmaCrawlNURL {
public plasmaCrawlNURL(File cachePath) {
super();
coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack", true);
coreStack = new plasmaCrawlBalancer(cachePath, "urlNoticeCoreStack", false);
limitStack = new plasmaCrawlBalancer(cachePath, "urlNoticeLimitStack", false);
//overhangStack = new plasmaCrawlBalancer(overhangStackFile);
remoteStack = new plasmaCrawlBalancer(cachePath, "urlNoticeRemoteStack", true);
remoteStack = new plasmaCrawlBalancer(cachePath, "urlNoticeRemoteStack", false);
}
public void close() {

31
source/de/anomic/plasma/plasmaSearchEvent.java
Unescape View File

@ -103,6 +103,7 @@ public final class plasmaSearchEvent {
this.urlRetrievalAllTime = 0;
this.snippetComputationAllTime = 0;
this.workerThreads = null;
this.localSearchThread = null;
this.resultList = new ArrayList<ResultEntry>(10); // this is the result set which is filled up with search results, enriched with snippets
//this.resultListLock = 0; // no locked elements until now
this.failedURLs = new HashMap<String, String>(); // a map of urls to reason strings where a worker thread tried to work on, but failed.
@ -407,6 +408,17 @@ public final class plasmaSearchEvent {
return false;
}
private int countFinishedRemoteSearch() {
int count = 0;
// check only primary search threads
if ((this.primarySearchThreads != null) && (this.primarySearchThreads.length != 0)) {
for (int i = 0; i < this.primarySearchThreads.length; i++) {
if ((this.primarySearchThreads[i] == null) || (!(this.primarySearchThreads[i].isAlive()))) count++;
}
}
return count;
}
public plasmaSearchQuery getQuery() {
return query;
}
@ -563,19 +575,12 @@ public final class plasmaSearchEvent {
public ResultEntry oneResult(int item) {
// first sleep a while to give accumulation threads a chance to work
if (anyWorkerAlive()) {
long sleeptime = Math.min(600, this.eventTime + (this.query.maximumTime / this.query.displayResults() * ((item % this.query.displayResults()) + 1)) - System.currentTimeMillis());
if (this.resultList.size() <= item + 10) sleeptime = Math.min(sleeptime + 300, 600);
if (sleeptime > 0) try {Thread.sleep(sleeptime);} catch (InterruptedException e) {}
//System.out.println("+++DEBUG-oneResult+++ (1) sleeping " + sleeptime);
// then sleep until any result is available (that should not happen)
while ((this.resultList.size() <= item) && (anyWorkerAlive())) {
try {Thread.sleep(100);} catch (InterruptedException e) {}
//System.out.println("+++DEBUG-oneResult+++ (2) sleeping " + 100);
}
while (((localSearchThread != null) && (localSearchThread.isAlive())) ||
((this.primarySearchThreads != null) && (this.primarySearchThreads.length > item) && (anyWorkerAlive()) &&
((this.resultList.size() <= item) || (countFinishedRemoteSearch() <= item)))) {
try {Thread.sleep(100);} catch (InterruptedException e) {}
}
// finally, if there is something, return the result
synchronized (this.resultList) {
// check if we have enough entries
@ -610,7 +615,7 @@ public final class plasmaSearchEvent {
boolean secondarySearchStartet = false;
private void prepareSecondarySearch() {
if (secondarySearchStartet) return; // dont do this twice
if (secondarySearchStartet) return; // don't do this twice
if ((rcAbstracts == null) || (rcAbstracts.size() != query.queryHashes.size())) return; // secondary search not possible (yet)
this.secondarySearchStartet = true;

2
source/de/anomic/plasma/plasmaSwitchboard.java
Unescape View File

@ -877,7 +877,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
private static plasmaSwitchboard sb;
public plasmaSwitchboard(String rootPath, String initPath, String configPath, boolean applyPro) {
public plasmaSwitchboard(File rootPath, String initPath, String configPath, boolean applyPro) {
super(rootPath, initPath, configPath, applyPro);
serverProfiling.startSystemProfiling();
sb=this;

2
source/de/anomic/server/logging/serverLog.java
Unescape View File

@ -186,7 +186,7 @@ public final class serverLog {
Logger.getLogger(appName).isLoggable(Level.FINEST);
}
public static final void configureLogging(String homePath, File loggingConfigFile) throws SecurityException, FileNotFoundException, IOException {
public static final void configureLogging(File homePath, File loggingConfigFile) throws SecurityException, FileNotFoundException, IOException {
FileInputStream fileIn = null;
try {
System.out.println("STARTUP: Trying to load logging configuration from file " + loggingConfigFile.toString());

6
source/de/anomic/server/serverAbstractSwitch.java
Unescape View File

@ -58,7 +58,7 @@ public abstract class serverAbstractSwitch implements serverSwitch {
// configuration management
private File configFile;
private String configComment;
private String rootPath;
private File rootPath;
protected serverLog log;
protected int serverJobs;
protected long maxTrackingTime;
@ -69,7 +69,7 @@ public abstract class serverAbstractSwitch implements serverSwitch {
private TreeMap<String, serverSwitchAction> switchActions;
protected HashMap<String, TreeMap<Long, String>> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries
public serverAbstractSwitch(String rootPath, String initPath, String configPath, boolean applyPro) {
public serverAbstractSwitch(File rootPath, String initPath, String configPath, boolean applyPro) {
// we initialize the switchboard with a property file,
// but maintain these properties then later in a new 'config' file
// to reset all changed configs, the config file must
@ -526,7 +526,7 @@ public abstract class serverAbstractSwitch implements serverSwitch {
public abstract serverObjects action(String actionName, serverObjects actionInput);
public String getRootPath() {
public File getRootPath() {
return rootPath;
}

4
source/de/anomic/server/serverPlainSwitch.java
Unescape View File

@ -26,9 +26,11 @@
package de.anomic.server;
import java.io.File;
public class serverPlainSwitch extends serverAbstractSwitch implements serverSwitch {
public serverPlainSwitch(String rootPath, String initPath, String configPath, boolean applyPro) {
public serverPlainSwitch(File rootPath, String initPath, String configPath, boolean applyPro) {
super(rootPath, initPath, configPath, applyPro);
}

2
source/de/anomic/server/serverSwitch.java
Unescape View File

@ -60,7 +60,7 @@ import de.anomic.server.logging.serverLog;
public interface serverSwitch {
// the root path for the application
public String getRootPath();
public File getRootPath();
// a logger for this switchboard
public void setLog(serverLog log);

3
source/de/anomic/yacy/yacyClient.java
Unescape View File

@ -43,6 +43,7 @@
package de.anomic.yacy;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
@ -1092,7 +1093,7 @@ public final class yacyClient {
public static void main(String[] args) {
System.out.println("yacyClient Test");
try {
final plasmaSwitchboard sb = new plasmaSwitchboard(args[0], "httpProxy.init", "DATA/SETTINGS/httpProxy.conf", false);
final plasmaSwitchboard sb = new plasmaSwitchboard(new File(args[0]), "httpProxy.init", "DATA/SETTINGS/httpProxy.conf", false);
/*final yacyCore core =*/ new yacyCore(sb);
yacyCore.peerActions.loadSeedLists();
final yacySeed target = yacyCore.seedDB.getConnected(args[1]);

5
source/de/anomic/yacy/yacyVersion.java
Unescape View File

@ -384,6 +384,7 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
//byte[] script = ("cd " + plasmaSwitchboard.getSwitchboard().getRootPath() + ";while [ -e ../yacy.running ]; do sleep 1;done;tar xfz " + release + ";cp -Rf yacy/* ../../;rm -Rf yacy;cd ../../;startYACY.sh").getBytes();
try {
plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard();
String apphome = sb.getRootPath().toString();
serverLog.logInfo("UPDATE", "INITIATED");
String script =
"#!/bin/sh" + serverCore.LF_STRING +
@ -394,14 +395,14 @@ public final class yacyVersion implements Comparator<yacyVersion>, Comparable<ya
"while [ -f ../yacy.running ]; do" + serverCore.LF_STRING +
"sleep 1" + serverCore.LF_STRING +
"done" + serverCore.LF_STRING +
"cp -Rf yacy/* ../../" + serverCore.LF_STRING +
"cp -Rf yacy/* " + apphome + serverCore.LF_STRING +
"rm -Rf yacy" + serverCore.LF_STRING +
"else" + serverCore.LF_STRING +
"while [ -f ../yacy.running ]; do" + serverCore.LF_STRING +
"sleep 1" + serverCore.LF_STRING +
"done" + serverCore.LF_STRING +
"fi" + serverCore.LF_STRING +
"cd ../../" + serverCore.LF_STRING +
"cd " + apphome + serverCore.LF_STRING +
"nohup ./startYACY.sh > /dev/null" + serverCore.LF_STRING;
File scriptFile = new File(sb.getRootPath(), "DATA/RELEASE/update.sh");
serverSystem.deployScript(scriptFile, script);

2
source/migration.java
Unescape View File

@ -51,7 +51,7 @@ public class migration {
//SVN constants
public static final int USE_WORK_DIR=1389; //wiki & messages in DATA/WORK
public static final int TAGDB_WITH_TAGHASH=1635; //tagDB keys are tagHashes instead of plain tagname.
public static final int NEW_OVERLAYS=3675;
public static final int NEW_OVERLAYS=4422;
public static void main(String[] args) {
}

64
source/yacy.java
Unescape View File

@ -164,7 +164,7 @@ public final class yacy {
* @param homePath Root-path where all information is to be found.
* @param startupFree free memory at startup time, to be used later for statistics
*/
private static void startup(String homePath, long startupMemFree, long startupMemTotal) {
private static void startup(File homePath, long startupMemFree, long startupMemTotal) {
int oldRev=0;
int newRev=0;
@ -183,7 +183,7 @@ public final class yacy {
}
// ensure that there is a DATA directory, if not, create one and if that fails warn and die
File f = new File(homePath); if (!(f.exists())) f.mkdirs();
File f = homePath; if (!(f.exists())) f.mkdirs();
f = new File(homePath, "DATA/"); if (!(f.exists())) f.mkdirs();
if (!(f.exists())) {
System.err.println("Error creating DATA-directory in " + homePath.toString() + " . Please check your write-permission for this folder. YaCy will now terminate.");
@ -198,7 +198,7 @@ public final class yacy {
System.out.println("could not copy yacy.logging");
}
try{
serverLog.configureLogging(homePath,new File(homePath, "DATA/LOG/yacy.logging"));
serverLog.configureLogging(homePath, new File(homePath, "DATA/LOG/yacy.logging"));
} catch (IOException e) {
System.out.println("could not find logging properties in homePath=" + homePath);
e.printStackTrace();
@ -263,7 +263,7 @@ public final class yacy {
sb.setConfig("version", Double.toString(version));
sb.setConfig("vString", yacyVersion.combined2prettyVersion(Double.toString(version)));
sb.setConfig("vdate", (vDATE.startsWith("@")) ? serverDate.formatShortDay() : vDATE);
sb.setConfig("applicationRoot", homePath);
sb.setConfig("applicationRoot", homePath.toString());
serverLog.logConfig("STARTUP", "YACY Version: " + version + ", Built " + sb.getConfig("vdate", "00000000"));
yacyVersion.latestRelease = version;
@ -451,7 +451,7 @@ public final class yacy {
* @param homePath Root-path where all the information is to be found.
* @return Properties read from the configurationfile.
*/
private static Properties configuration(String mes, String homePath) {
private static Properties configuration(String mes, File homePath) {
serverLog.logConfig(mes, "Application Root Path: " + homePath.toString());
// read data folder
@ -480,7 +480,7 @@ public final class yacy {
// YaCy is running in the same runtime. we can shutdown via interrupt
sb.terminate();
} else {
String applicationRoot = System.getProperty("user.dir").replace('\\', '/');
File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/'));
shutdown(applicationRoot);
}
}
@ -491,7 +491,7 @@ public final class yacy {
*
* @param homePath Root-path where all the information is to be found.
*/
static void shutdown(String homePath) {
static void shutdown(File homePath) {
// start up
System.out.println(copyright);
System.out.println(hline);
@ -544,7 +544,7 @@ public final class yacy {
*
* @param homePath Root-Path where all the information is to be found.
*/
private static void genWordstat(String homePath) {
private static void genWordstat(File homePath) {
// start up
System.out.println(copyright);
System.out.println(hline);
@ -587,12 +587,12 @@ public final class yacy {
* @param homePath path to the YaCy directory
* @param dbcache cache size in MB
*/
public static void minimizeUrlDB(String homePath) {
public static void minimizeUrlDB(File homePath) {
// run with "java -classpath classes yacy -minimizeUrlDB"
try {serverLog.configureLogging(homePath,new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {}
File indexPrimaryRoot = new File(new File(homePath), "DATA/INDEX");
File indexSecondaryRoot = new File(new File(homePath), "DATA/INDEX");
File indexRoot2 = new File(new File(homePath), "DATA/INDEX2");
try {serverLog.configureLogging(homePath, new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {}
File indexPrimaryRoot = new File(homePath, "DATA/INDEX");
File indexSecondaryRoot = new File(homePath, "DATA/INDEX");
File indexRoot2 = new File(homePath, "DATA/INDEX2");
serverLog log = new serverLog("URL-CLEANUP");
try {
log.logInfo("STARTING URL CLEANUP");
@ -777,24 +777,24 @@ public final class yacy {
*
* @param homePath Root-Path where all information is to be found.
*/
private static void urldbcleanup(String homePath) {
File root = new File(homePath);
private static void urldbcleanup(File homePath) {
File root = homePath;
File indexroot = new File(root, "DATA/INDEX");
try {serverLog.configureLogging(homePath,new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {}
try {serverLog.configureLogging(homePath, new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {}
plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexroot, 10000);
currentUrlDB.urldbcleanup();
currentUrlDB.close();
}
private static void RWIHashList(String homePath, String targetName, String resource, String format) {
private static void RWIHashList(File homePath, String targetName, String resource, String format) {
plasmaWordIndex WordIndex = null;
serverLog log = new serverLog("HASHLIST");
File indexPrimaryRoot = new File(new File(homePath), "DATA/INDEX");
File indexSecondaryRoot = new File(new File(homePath), "DATA/INDEX");
File indexPrimaryRoot = new File(homePath, "DATA/INDEX");
File indexSecondaryRoot = new File(homePath, "DATA/INDEX");
String wordChunkStartHash = "AAAAAAAAAAAA";
try {serverLog.configureLogging(homePath,new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {}
try {serverLog.configureLogging(homePath, new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {}
log.logInfo("STARTING CREATION OF RWI-HASHLIST");
File root = new File(homePath);
File root = homePath;
try {
Iterator<indexContainer> indexContainerIterator = null;
if (resource.equals("all")) {
@ -850,10 +850,10 @@ public final class yacy {
* Searching for peers affected by Bug
* @param homePath
*/
public static void testPeerDB(String homePath) {
public static void testPeerDB(File homePath) {
try {
File yacyDBPath = new File(new File(homePath), "DATA/YACYDB");
File yacyDBPath = new File(homePath, "DATA/YACYDB");
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
@ -905,16 +905,16 @@ public final class yacy {
// go into headless awt mode
System.setProperty("java.awt.headless", "true");
String applicationRoot = System.getProperty("user.dir").replace('\\', '/');
File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/'));
//System.out.println("args.length=" + args.length);
//System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]");
if ((args.length >= 1) && ((args[0].toLowerCase().equals("-startup")) || (args[0].equals("-start")))) {
// normal start-up of yacy
if (args.length == 2) applicationRoot= args[1];
if (args.length == 2) applicationRoot= new File(args[1]);
startup(applicationRoot, startupMemFree, startupMemTotal);
} else if ((args.length >= 1) && ((args[0].toLowerCase().equals("-shutdown")) || (args[0].equals("-stop")))) {
// normal shutdown of yacy
if (args.length == 2) applicationRoot= args[1];
if (args.length == 2) applicationRoot= new File(args[1]);
shutdown(applicationRoot);
} else if ((args.length >= 1) && (args[0].toLowerCase().equals("-minimizeurldb"))) {
// migrate words from DATA/PLASMADB/WORDS path to assortment cache, if possible
@ -922,11 +922,11 @@ public final class yacy {
if (args.length >= 3 && args[1].toLowerCase().equals("-cache")) {
args = shift(args, 1, 2);
}
if (args.length == 2) applicationRoot= args[1];
if (args.length == 2) applicationRoot= new File(args[1]);
minimizeUrlDB(applicationRoot);
} else if ((args.length >= 1) && (args[0].toLowerCase().equals("-testpeerdb"))) {
if (args.length == 2) {
applicationRoot= args[1];
applicationRoot = new File(args[1]);
} else if (args.length > 2) {
System.err.println("Usage: -testPeerDB [homeDbRoot]");
}
@ -935,7 +935,7 @@ public final class yacy {
// this can help to create a stop-word list
// to use this, you need a 'yacy.words' file in the root path
// start this with "java -classpath classes yacy -genwordstat [<rootdir>]"
if (args.length == 2) applicationRoot= args[1];
if (args.length == 2) applicationRoot= new File(args[1]);
genWordstat(applicationRoot);
} else if ((args.length == 4) && (args[0].toLowerCase().equals("-cleanwordlist"))) {
// this can be used to organize and clean a word-list
@ -950,7 +950,7 @@ public final class yacy {
transferCR(targetaddress, crfile);
} else if ((args.length >= 1) && (args[0].toLowerCase().equals("-urldbcleanup"))) {
// generate a url list and save it in a file
if (args.length == 2) applicationRoot= args[1];
if (args.length == 2) applicationRoot= new File(args[1]);
urldbcleanup(applicationRoot);
} else if ((args.length >= 1) && (args[0].toLowerCase().equals("-rwihashlist"))) {
// generate a url list and save it in a file
@ -958,11 +958,11 @@ public final class yacy {
String format = "txt";
if (args.length >= 2) domain= args[1];
if (args.length >= 3) format= args[2];
if (args.length == 4) applicationRoot= args[3];
if (args.length == 4) applicationRoot= new File(args[3]);
String outfile = "rwihashlist_" + System.currentTimeMillis();
RWIHashList(applicationRoot, outfile, domain, format);
} else {
if (args.length == 1) applicationRoot= args[0];
if (args.length == 1) applicationRoot= new File(args[0]);
startup(applicationRoot, startupMemFree, startupMemTotal);
}
}

Write Preview
Loading…
Cancel
Save