|
|
|
@ -1,67 +1,67 @@
|
|
|
|
|
// yacy.java
|
|
|
|
|
// -----------------------
|
|
|
|
|
// (C) by Michael Peter Christen; mc@anomic.de
|
|
|
|
|
// first published on http://www.yacy.net
|
|
|
|
|
// Frankfurt, Germany, 2004, 2005
|
|
|
|
|
// last major change: 24.03.2005
|
|
|
|
|
//yacy.java
|
|
|
|
|
//-----------------------
|
|
|
|
|
//(C) by Michael Peter Christen; mc@anomic.de
|
|
|
|
|
//first published on http://www.yacy.net
|
|
|
|
|
//Frankfurt, Germany, 2004, 2005
|
|
|
|
|
//last major change: 24.03.2005
|
|
|
|
|
//
|
|
|
|
|
// This program is free software; you can redistribute it and/or modify
|
|
|
|
|
// it under the terms of the GNU General Public License as published by
|
|
|
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
|
|
|
// (at your option) any later version.
|
|
|
|
|
//This program is free software; you can redistribute it and/or modify
|
|
|
|
|
//it under the terms of the GNU General Public License as published by
|
|
|
|
|
//the Free Software Foundation; either version 2 of the License, or
|
|
|
|
|
//(at your option) any later version.
|
|
|
|
|
//
|
|
|
|
|
// This program is distributed in the hope that it will be useful,
|
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
// GNU General Public License for more details.
|
|
|
|
|
//This program is distributed in the hope that it will be useful,
|
|
|
|
|
//but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
//GNU General Public License for more details.
|
|
|
|
|
//
|
|
|
|
|
// You should have received a copy of the GNU General Public License
|
|
|
|
|
// along with this program; if not, write to the Free Software
|
|
|
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
|
//You should have received a copy of the GNU General Public License
|
|
|
|
|
//along with this program; if not, write to the Free Software
|
|
|
|
|
//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
|
//
|
|
|
|
|
// Using this software in any meaning (reading, learning, copying, compiling,
|
|
|
|
|
// running) means that you agree that the Author(s) is (are) not responsible
|
|
|
|
|
// for cost, loss of data or any harm that may be caused directly or indirectly
|
|
|
|
|
// by usage of this softare or this documentation. The usage of this software
|
|
|
|
|
// is on your own risk. The installation and usage (starting/running) of this
|
|
|
|
|
// software may allow other people or application to access your computer and
|
|
|
|
|
// any attached devices and is highly dependent on the configuration of the
|
|
|
|
|
// software which must be done by the user of the software; the author(s) is
|
|
|
|
|
// (are) also not responsible for proper configuration and usage of the
|
|
|
|
|
// software, even if provoked by documentation provided together with
|
|
|
|
|
// the software.
|
|
|
|
|
//Using this software in any meaning (reading, learning, copying, compiling,
|
|
|
|
|
//running) means that you agree that the Author(s) is (are) not responsible
|
|
|
|
|
//for cost, loss of data or any harm that may be caused directly or indirectly
|
|
|
|
|
//by usage of this softare or this documentation. The usage of this software
|
|
|
|
|
//is on your own risk. The installation and usage (starting/running) of this
|
|
|
|
|
//software may allow other people or application to access your computer and
|
|
|
|
|
//any attached devices and is highly dependent on the configuration of the
|
|
|
|
|
//software which must be done by the user of the software; the author(s) is
|
|
|
|
|
//(are) also not responsible for proper configuration and usage of the
|
|
|
|
|
//software, even if provoked by documentation provided together with
|
|
|
|
|
//the software.
|
|
|
|
|
//
|
|
|
|
|
// Any changes to this file according to the GPL as documented in the file
|
|
|
|
|
// gpl.txt aside this file in the shipment you received can be done to the
|
|
|
|
|
// lines that follows this copyright notice here, but changes must not be
|
|
|
|
|
// done inside the copyright notive above. A re-distribution must contain
|
|
|
|
|
// the intact and unchanged copyright notice.
|
|
|
|
|
// Contributions and changes to the program code must be marked as such.
|
|
|
|
|
//Any changes to this file according to the GPL as documented in the file
|
|
|
|
|
//gpl.txt aside this file in the shipment you received can be done to the
|
|
|
|
|
//lines that follows this copyright notice here, but changes must not be
|
|
|
|
|
//done inside the copyright notive above. A re-distribution must contain
|
|
|
|
|
//the intact and unchanged copyright notice.
|
|
|
|
|
//Contributions and changes to the program code must be marked as such.
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
This is the main class of the proxy.
|
|
|
|
|
From here, several threads are started:
|
|
|
|
|
|
|
|
|
|
- one single instance of the plasmaSwitchboard is generated,
|
|
|
|
|
which itself starts a thread with a plasmaHTMLCache object. This object simply counts
|
|
|
|
|
files sizes in the cache and terminates then.
|
|
|
|
|
It also generates a plasmaCrawlerLoader object, which may itself start
|
|
|
|
|
some more httpc-calling threads to load web pages. They terminate automatically when a page has loaded
|
|
|
|
|
- one serverCore - thread is started, which implements a multi-threaded server.
|
|
|
|
|
The process may start itself many more processes that handle connections.
|
|
|
|
|
- finally, all idle-dependent processes are written in a queue in plasmaSwitchboard
|
|
|
|
|
which are worked off inside an idle-sensitive loop of the main process. (here)
|
|
|
|
|
|
|
|
|
|
On termination, the following must be done:
|
|
|
|
|
- stop feeding of the crawling process because it othervise fills the indexing queue.
|
|
|
|
|
- say goodbye to connected peers and disable new connections. Don't wait for success.
|
|
|
|
|
- first terminate the serverCore thread. This prevents that new cache objects are queued
|
|
|
|
|
- wait that the plasmaHTMLCache terminates (it should be normal that this process already has terminated)
|
|
|
|
|
- then wait for termination of all loader process of the plasmaCrawlerLoader
|
|
|
|
|
- work off the indexing and cache storage queue. These values are inside a RAM cache and would be lost othervise
|
|
|
|
|
- write all settings
|
|
|
|
|
- terminate
|
|
|
|
|
*/
|
|
|
|
|
This is the main class of the proxy.
|
|
|
|
|
From here, several threads are started:
|
|
|
|
|
|
|
|
|
|
- one single instance of the plasmaSwitchboard is generated,
|
|
|
|
|
which itself starts a thread with a plasmaHTMLCache object. This object simply counts
|
|
|
|
|
files sizes in the cache and terminates then.
|
|
|
|
|
It also generates a plasmaCrawlerLoader object, which may itself start
|
|
|
|
|
some more httpc-calling threads to load web pages. They terminate automatically when a page has loaded
|
|
|
|
|
- one serverCore - thread is started, which implements a multi-threaded server.
|
|
|
|
|
The process may start itself many more processes that handle connections.
|
|
|
|
|
- finally, all idle-dependent processes are written in a queue in plasmaSwitchboard
|
|
|
|
|
which are worked off inside an idle-sensitive loop of the main process. (here)
|
|
|
|
|
|
|
|
|
|
On termination, the following must be done:
|
|
|
|
|
- stop feeding of the crawling process because it othervise fills the indexing queue.
|
|
|
|
|
- say goodbye to connected peers and disable new connections. Don't wait for success.
|
|
|
|
|
- first terminate the serverCore thread. This prevents that new cache objects are queued
|
|
|
|
|
- wait that the plasmaHTMLCache terminates (it should be normal that this process already has terminated)
|
|
|
|
|
- then wait for termination of all loader process of the plasmaCrawlerLoader
|
|
|
|
|
- work off the indexing and cache storage queue. These values are inside a RAM cache and would be lost othervise
|
|
|
|
|
- write all settings
|
|
|
|
|
- terminate
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import java.io.BufferedReader;
|
|
|
|
@ -110,10 +110,10 @@ public final class yacy {
|
|
|
|
|
|
|
|
|
|
private static void startup(String homePath) {
|
|
|
|
|
long startup = yacyCore.universalTime();
|
|
|
|
|
try {
|
|
|
|
|
// start up
|
|
|
|
|
System.out.println(copyright);
|
|
|
|
|
System.out.println(hline);
|
|
|
|
|
try {
|
|
|
|
|
// start up
|
|
|
|
|
System.out.println(copyright);
|
|
|
|
|
System.out.println(hline);
|
|
|
|
|
|
|
|
|
|
// check java version
|
|
|
|
|
try {
|
|
|
|
@ -131,20 +131,20 @@ public final class yacy {
|
|
|
|
|
File dataFolder = new File(homePath, "DATA");
|
|
|
|
|
if (!(dataFolder.exists())) dataFolder.mkdir();
|
|
|
|
|
|
|
|
|
|
plasmaSwitchboard sb = new plasmaSwitchboard(homePath, "yacy.init", "DATA/SETTINGS/httpProxy.conf");
|
|
|
|
|
sb.setConfig("version", vString);
|
|
|
|
|
sb.setConfig("vdate", vDATE);
|
|
|
|
|
plasmaSwitchboard sb = new plasmaSwitchboard(homePath, "yacy.init", "DATA/SETTINGS/httpProxy.conf");
|
|
|
|
|
sb.setConfig("version", vString);
|
|
|
|
|
sb.setConfig("vdate", vDATE);
|
|
|
|
|
sb.setConfig("applicationRoot", homePath);
|
|
|
|
|
sb.setConfig("startupTime", "" + startup);
|
|
|
|
|
serverLog.logSystem("STARTUP", "YACY Version: " + vString + ", Built " + vDATE);
|
|
|
|
|
|
|
|
|
|
// read environment
|
|
|
|
|
// read environment
|
|
|
|
|
//new
|
|
|
|
|
int port = Integer.parseInt(sb.getConfig("port", "8080"));
|
|
|
|
|
int httpdLoglevel = Integer.parseInt(sb.getConfig("httpdLoglevel", "2"));
|
|
|
|
|
int timeout = Integer.parseInt(sb.getConfig("httpdTimeout", "60000"));
|
|
|
|
|
if (timeout < 60000) timeout = 60000;
|
|
|
|
|
int maxSessions = Integer.parseInt(sb.getConfig("httpdMaxSessions", "100"));
|
|
|
|
|
int httpdLoglevel = Integer.parseInt(sb.getConfig("httpdLoglevel", "2"));
|
|
|
|
|
int timeout = Integer.parseInt(sb.getConfig("httpdTimeout", "60000"));
|
|
|
|
|
if (timeout < 60000) timeout = 60000;
|
|
|
|
|
int maxSessions = Integer.parseInt(sb.getConfig("httpdMaxSessions", "100"));
|
|
|
|
|
|
|
|
|
|
// hardcoded, forced, temporary value-migration
|
|
|
|
|
sb.setConfig("htTemplatePath", "htroot/env/templates");
|
|
|
|
@ -157,17 +157,17 @@ public final class yacy {
|
|
|
|
|
if (!(htDocsPath.exists())) htDocsPath.mkdir();
|
|
|
|
|
File htdocsDefaultReadme = new File(htDocsPath, "readme.txt");
|
|
|
|
|
if (!(htdocsDefaultReadme.exists())) try {serverFileUtils.write((
|
|
|
|
|
"This is your root directory for individual Web Content\r\n" +
|
|
|
|
|
"\r\n" +
|
|
|
|
|
"Please place your html files into the www subdirectory.\r\n" +
|
|
|
|
|
"The URL of that path is either\r\n" +
|
|
|
|
|
"http://www.<your-peer-name>.yacy or\r\n" +
|
|
|
|
|
"http://<your-ip>:<your-port>/www\r\n" +
|
|
|
|
|
"\r\n" +
|
|
|
|
|
"Other subdirectories may be created; they map to corresponding sub-domains.\r\n" +
|
|
|
|
|
"This directory shares it's content with the applications htroot path, so you\r\n" +
|
|
|
|
|
"may access your yacy search page with\r\n" +
|
|
|
|
|
"http://<your-peer-name>.yacy/\r\n" +
|
|
|
|
|
"This is your root directory for individual Web Content\r\n" +
|
|
|
|
|
"\r\n" +
|
|
|
|
|
"Please place your html files into the www subdirectory.\r\n" +
|
|
|
|
|
"The URL of that path is either\r\n" +
|
|
|
|
|
"http://www.<your-peer-name>.yacy or\r\n" +
|
|
|
|
|
"http://<your-ip>:<your-port>/www\r\n" +
|
|
|
|
|
"\r\n" +
|
|
|
|
|
"Other subdirectories may be created; they map to corresponding sub-domains.\r\n" +
|
|
|
|
|
"This directory shares it's content with the applications htroot path, so you\r\n" +
|
|
|
|
|
"may access your yacy search page with\r\n" +
|
|
|
|
|
"http://<your-peer-name>.yacy/\r\n" +
|
|
|
|
|
"\r\n").getBytes(), htdocsDefaultReadme);} catch (IOException e) {
|
|
|
|
|
System.out.println("Error creating htdocs readme: " + e.getMessage());
|
|
|
|
|
}
|
|
|
|
@ -177,96 +177,101 @@ public final class yacy {
|
|
|
|
|
|
|
|
|
|
File wwwDefaultClass = new File(wwwDefaultPath, "welcome.class");
|
|
|
|
|
//if ((!(wwwDefaultClass.exists())) || (wwwDefaultClass.length() != (new File(htRootPath, "htdocsdefault/welcome.class")).length())) try {
|
|
|
|
|
if((new File(htRootPath, "htdocsdefault/welcome.java")).exists())
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/welcome.java"), new File(wwwDefaultPath, "welcome.java"));
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/welcome.class"), wwwDefaultClass);
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/welcome.html"), new File(wwwDefaultPath, "welcome.html"));
|
|
|
|
|
if((new File(htRootPath, "htdocsdefault/welcome.java")).exists())
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/welcome.java"), new File(wwwDefaultPath, "welcome.java"));
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/welcome.class"), wwwDefaultClass);
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/welcome.html"), new File(wwwDefaultPath, "welcome.html"));
|
|
|
|
|
//} catch (IOException e) {}
|
|
|
|
|
|
|
|
|
|
File shareDefaultPath = new File(htDocsPath, "share");
|
|
|
|
|
if (!(shareDefaultPath.exists())) shareDefaultPath.mkdir();
|
|
|
|
|
|
|
|
|
|
File shareDefaultClass = new File(shareDefaultPath, "dir.class");
|
|
|
|
|
//if ((!(shareDefaultClass.exists())) || (shareDefaultClass.length() != (new File(htRootPath, "htdocsdefault/dir.class")).length())) try {
|
|
|
|
|
if((new File(htRootPath, "htdocsdefault/dir.java")).exists())
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/dir.java"), new File(shareDefaultPath, "dir.java"));
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/dir.class"), shareDefaultClass);
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/dir.html"), new File(shareDefaultPath, "dir.html"));
|
|
|
|
|
//if ((!(shareDefaultClass.exists())) || (shareDefaultClass.length() != (new File(htRootPath, "htdocsdefault/dir.class")).length())) try {
|
|
|
|
|
if((new File(htRootPath, "htdocsdefault/dir.java")).exists())
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/dir.java"), new File(shareDefaultPath, "dir.java"));
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/dir.class"), shareDefaultClass);
|
|
|
|
|
serverFileUtils.copy(new File(htRootPath, "htdocsdefault/dir.html"), new File(shareDefaultPath, "dir.html"));
|
|
|
|
|
//} catch (IOException e) {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// set preset accounts/passwords
|
|
|
|
|
String acc;
|
|
|
|
|
if ((acc = sb.getConfig("proxyAccount", "")).length() > 0) {
|
|
|
|
|
sb.setConfig("proxyAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(serverCodings.standardCoder.encodeBase64String(acc)));
|
|
|
|
|
sb.setConfig("proxyAccount", "");
|
|
|
|
|
}
|
|
|
|
|
// set preset accounts/passwords
|
|
|
|
|
String acc;
|
|
|
|
|
if ((acc = sb.getConfig("proxyAccount", "")).length() > 0) {
|
|
|
|
|
sb.setConfig("proxyAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(serverCodings.standardCoder.encodeBase64String(acc)));
|
|
|
|
|
sb.setConfig("proxyAccount", "");
|
|
|
|
|
}
|
|
|
|
|
if ((acc = sb.getConfig("serverAccount", "")).length() > 0) {
|
|
|
|
|
sb.setConfig("serverAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(serverCodings.standardCoder.encodeBase64String(acc)));
|
|
|
|
|
sb.setConfig("serverAccount", "");
|
|
|
|
|
}
|
|
|
|
|
if ((acc = sb.getConfig("adminAccount", "")).length() > 0) {
|
|
|
|
|
sb.setConfig("adminAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(serverCodings.standardCoder.encodeBase64String(acc)));
|
|
|
|
|
sb.setConfig("adminAccount", "");
|
|
|
|
|
}
|
|
|
|
|
sb.setConfig("serverAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(serverCodings.standardCoder.encodeBase64String(acc)));
|
|
|
|
|
sb.setConfig("serverAccount", "");
|
|
|
|
|
}
|
|
|
|
|
if ((acc = sb.getConfig("adminAccount", "")).length() > 0) {
|
|
|
|
|
sb.setConfig("adminAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(serverCodings.standardCoder.encodeBase64String(acc)));
|
|
|
|
|
sb.setConfig("adminAccount", "");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// fix unsafe old passwords
|
|
|
|
|
if ((acc = sb.getConfig("proxyAccountBase64", "")).length() > 0) {
|
|
|
|
|
sb.setConfig("proxyAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(acc));
|
|
|
|
|
sb.setConfig("proxyAccountBase64", "");
|
|
|
|
|
}
|
|
|
|
|
if ((acc = sb.getConfig("serverAccountBase64", "")).length() > 0) {
|
|
|
|
|
sb.setConfig("serverAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(acc));
|
|
|
|
|
sb.setConfig("serverAccountBase64", "");
|
|
|
|
|
}
|
|
|
|
|
sb.setConfig("proxyAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(acc));
|
|
|
|
|
sb.setConfig("proxyAccountBase64", "");
|
|
|
|
|
}
|
|
|
|
|
if ((acc = sb.getConfig("serverAccountBase64", "")).length() > 0) {
|
|
|
|
|
sb.setConfig("serverAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(acc));
|
|
|
|
|
sb.setConfig("serverAccountBase64", "");
|
|
|
|
|
}
|
|
|
|
|
if ((acc = sb.getConfig("adminAccountBase64", "")).length() > 0) {
|
|
|
|
|
sb.setConfig("adminAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(acc));
|
|
|
|
|
sb.setConfig("adminAccountBase64", "");
|
|
|
|
|
}
|
|
|
|
|
sb.setConfig("adminAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(acc));
|
|
|
|
|
sb.setConfig("adminAccountBase64", "");
|
|
|
|
|
}
|
|
|
|
|
if ((acc = sb.getConfig("uploadAccountBase64", "")).length() > 0) {
|
|
|
|
|
sb.setConfig("uploadAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(acc));
|
|
|
|
|
sb.setConfig("uploadAccountBase64", "");
|
|
|
|
|
}
|
|
|
|
|
sb.setConfig("uploadAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(acc));
|
|
|
|
|
sb.setConfig("uploadAccountBase64", "");
|
|
|
|
|
}
|
|
|
|
|
if ((acc = sb.getConfig("downloadAccountBase64", "")).length() > 0) {
|
|
|
|
|
sb.setConfig("downloadAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(acc));
|
|
|
|
|
sb.setConfig("downloadAccountBase64", "");
|
|
|
|
|
}
|
|
|
|
|
sb.setConfig("downloadAccountBase64MD5", serverCodings.standardCoder.encodeMD5Hex(acc));
|
|
|
|
|
sb.setConfig("downloadAccountBase64", "");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// init parser
|
|
|
|
|
de.anomic.plasma.plasmaParser.initMediaExt(sb.getConfig("mediaExt",""));
|
|
|
|
|
|
|
|
|
|
// start main threads
|
|
|
|
|
try {
|
|
|
|
|
// start main threads
|
|
|
|
|
try {
|
|
|
|
|
httpd protocolHandler = new httpd(sb, new httpdFileHandler(sb), new httpdProxyHandler(sb));
|
|
|
|
|
serverCore server = new serverCore(port,
|
|
|
|
|
maxSessions /*sessions*/,
|
|
|
|
|
timeout /*control socket timeout in milliseconds*/,
|
|
|
|
|
true /* terminate sleeping threads */,
|
|
|
|
|
true /* block attacks (wrong protocol) */,
|
|
|
|
|
protocolHandler /*command class*/,
|
|
|
|
|
sb,
|
|
|
|
|
30000 /*command max length incl. GET args*/,
|
|
|
|
|
httpdLoglevel /*loglevel*/);
|
|
|
|
|
server.setName("httpd:"+port);
|
|
|
|
|
if (server == null) {
|
|
|
|
|
serverLog.logFailure("STARTUP", "Failed to start server. Probably port " + port + " already in use.");
|
|
|
|
|
} else {
|
|
|
|
|
// first start the server
|
|
|
|
|
serverCore server = new serverCore(port,
|
|
|
|
|
maxSessions /*sessions*/,
|
|
|
|
|
timeout /*control socket timeout in milliseconds*/,
|
|
|
|
|
true /* terminate sleeping threads */,
|
|
|
|
|
true /* block attacks (wrong protocol) */,
|
|
|
|
|
protocolHandler /*command class*/,
|
|
|
|
|
sb,
|
|
|
|
|
30000 /*command max length incl. GET args*/,
|
|
|
|
|
httpdLoglevel /*loglevel*/);
|
|
|
|
|
server.setName("httpd:"+port);
|
|
|
|
|
if (server == null) {
|
|
|
|
|
serverLog.logFailure("STARTUP", "Failed to start server. Probably port " + port + " already in use.");
|
|
|
|
|
} else {
|
|
|
|
|
// first start the server
|
|
|
|
|
sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", server, null, 0, 0, 0);
|
|
|
|
|
//server.start();
|
|
|
|
|
|
|
|
|
|
// open the browser window
|
|
|
|
|
boolean browserPopUpTrigger = sb.getConfig("browserPopUpTrigger", "true").equals("true");
|
|
|
|
|
if (browserPopUpTrigger) {
|
|
|
|
|
String browserPopUpPage = sb.getConfig("browserPopUpPage", "Status.html");
|
|
|
|
|
String browserPopUpApplication = sb.getConfig("browserPopUpApplication", "netscape");
|
|
|
|
|
serverSystem.openBrowser("http://localhost:" + port + "/" + browserPopUpPage, browserPopUpApplication);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// wait for server shutdown
|
|
|
|
|
try {
|
|
|
|
|
sb.waitForShutdown();
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
//server.start();
|
|
|
|
|
|
|
|
|
|
// open the browser window
|
|
|
|
|
boolean browserPopUpTrigger = sb.getConfig("browserPopUpTrigger", "true").equals("true");
|
|
|
|
|
if (browserPopUpTrigger) {
|
|
|
|
|
String browserPopUpPage = sb.getConfig("browserPopUpPage", "Status.html");
|
|
|
|
|
String browserPopUpApplication = sb.getConfig("browserPopUpApplication", "netscape");
|
|
|
|
|
serverSystem.openBrowser("http://localhost:" + port + "/" + browserPopUpPage, browserPopUpApplication);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// registering shutdown hook
|
|
|
|
|
serverLog.logSystem("STARTUP", "Registering Shutdown Hook");
|
|
|
|
|
Runtime run = Runtime.getRuntime();
|
|
|
|
|
run.addShutdownHook(new shutdownHookThread(Thread.currentThread(), sb));
|
|
|
|
|
|
|
|
|
|
// wait for server shutdown
|
|
|
|
|
try {
|
|
|
|
|
sb.waitForShutdown();
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
serverLog.logError("MAIN CONTROL LOOP", "PANIK: " + e.getMessage());
|
|
|
|
|
e.printStackTrace();
|
|
|
|
|
}
|
|
|
|
@ -274,8 +279,8 @@ public final class yacy {
|
|
|
|
|
// shut down
|
|
|
|
|
serverLog.logSystem("SHUTDOWN", "catched termination signal");
|
|
|
|
|
server.terminate(false);
|
|
|
|
|
server.interrupt();
|
|
|
|
|
if (server.isAlive()) try {
|
|
|
|
|
server.interrupt();
|
|
|
|
|
if (server.isAlive()) try {
|
|
|
|
|
httpc.wget(new URL("http://localhost:" + port), 1000, null, null, null, 0); // kick server
|
|
|
|
|
serverLog.logSystem("SHUTDOWN", "sent termination signal to server socket");
|
|
|
|
|
} catch (IOException ee) {
|
|
|
|
@ -284,22 +289,24 @@ public final class yacy {
|
|
|
|
|
|
|
|
|
|
// idle until the processes are down
|
|
|
|
|
while (server.isAlive()) {
|
|
|
|
|
Thread.currentThread().sleep(2000); // wait a while
|
|
|
|
|
Thread.currentThread().sleep(2000); // wait a while
|
|
|
|
|
}
|
|
|
|
|
serverLog.logSystem("SHUTDOWN", "server has terminated");
|
|
|
|
|
sb.close();
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
serverLog.logError("STARTUP", "" + e);
|
|
|
|
|
e.printStackTrace();
|
|
|
|
|
//System.exit(1);
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception ee) {
|
|
|
|
|
serverLog.logFailure("STARTUP", "FATAL ERROR: " + ee.getMessage());
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
serverLog.logError("STARTUP", "" + e);
|
|
|
|
|
e.printStackTrace();
|
|
|
|
|
//System.exit(1);
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception ee) {
|
|
|
|
|
serverLog.logFailure("STARTUP", "FATAL ERROR: " + ee.getMessage());
|
|
|
|
|
ee.printStackTrace();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
serverLog.logSystem("SHUTDOWN", "goodbye. (this is the last line)");
|
|
|
|
|
try {System.exit(0);} catch (Exception e) {} // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790)
|
|
|
|
|
try {
|
|
|
|
|
System.exit(0);
|
|
|
|
|
} catch (Exception e) {} // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -328,7 +335,7 @@ public final class yacy {
|
|
|
|
|
return config;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static void shutdown(String homePath) {
|
|
|
|
|
static void shutdown(String homePath) {
|
|
|
|
|
// start up
|
|
|
|
|
System.out.println(copyright);
|
|
|
|
|
System.out.println(hline);
|
|
|
|
@ -343,8 +350,8 @@ public final class yacy {
|
|
|
|
|
if (encodedPassword == null) encodedPassword = ""; // not defined
|
|
|
|
|
|
|
|
|
|
// send 'wget' to web interface
|
|
|
|
|
httpHeader requestHeader = new httpHeader();
|
|
|
|
|
requestHeader.put("Authorization", "realm=" + encodedPassword); // for http-authentify
|
|
|
|
|
httpHeader requestHeader = new httpHeader();
|
|
|
|
|
requestHeader.put("REMOTE-SHUTDOWN", "realm=" + encodedPassword); // for http-authentify
|
|
|
|
|
try {
|
|
|
|
|
httpc con = httpc.getInstance("localhost", port, 10000, false);
|
|
|
|
|
httpc.response res = con.GET("Steering.html?shutdown=", requestHeader);
|
|
|
|
@ -377,7 +384,7 @@ public final class yacy {
|
|
|
|
|
|
|
|
|
|
Properties config = configuration("GEN-WORDSTAT", homePath);
|
|
|
|
|
|
|
|
|
|
// load words
|
|
|
|
|
// load words
|
|
|
|
|
serverLog.logInfo("GEN-WORDSTAT", "loading words...");
|
|
|
|
|
HashMap words = loadWordMap(new File(homePath, "yacy.words"));
|
|
|
|
|
|
|
|
|
@ -411,7 +418,7 @@ public final class yacy {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private static HashMap loadWordMap(File wordlist) {
|
|
|
|
|
// returns a hash-word - Relation
|
|
|
|
|
// returns a hash-word - Relation
|
|
|
|
|
HashMap wordmap = new HashMap();
|
|
|
|
|
try {
|
|
|
|
|
String word;
|
|
|
|
@ -423,7 +430,7 @@ public final class yacy {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static HashSet loadWordSet(File wordlist) {
|
|
|
|
|
// returns a set of words
|
|
|
|
|
// returns a set of words
|
|
|
|
|
HashSet wordset = new HashSet();
|
|
|
|
|
try {
|
|
|
|
|
String word;
|
|
|
|
@ -445,12 +452,12 @@ public final class yacy {
|
|
|
|
|
int count = 0;
|
|
|
|
|
try {
|
|
|
|
|
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(wordlist)));
|
|
|
|
|
String seps = "' .,:/-&";
|
|
|
|
|
String seps = "' .,:/-&";
|
|
|
|
|
while ((word = br.readLine()) != null) {
|
|
|
|
|
word = word.toLowerCase().trim();
|
|
|
|
|
for (int i = 0; i < seps.length(); i++) {
|
|
|
|
|
for (int i = 0; i < seps.length(); i++) {
|
|
|
|
|
if (word.indexOf(seps.charAt(i)) >= 0) word = word.substring(0, word.indexOf(seps.charAt(i)));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if ((word.length() >= minlength) && (word.length() <= maxlength)) wordset.add(word);
|
|
|
|
|
count++;
|
|
|
|
|
}
|
|
|
|
@ -494,23 +501,23 @@ public final class yacy {
|
|
|
|
|
// find all hashes
|
|
|
|
|
File f;
|
|
|
|
|
String w;
|
|
|
|
|
int count = 0;
|
|
|
|
|
long thisamount, totalamount = 0;
|
|
|
|
|
Iterator i = stopwords.iterator();
|
|
|
|
|
int count = 0;
|
|
|
|
|
long thisamount, totalamount = 0;
|
|
|
|
|
Iterator i = stopwords.iterator();
|
|
|
|
|
while (i.hasNext()) {
|
|
|
|
|
w = (String) i.next();
|
|
|
|
|
w = (String) i.next();
|
|
|
|
|
f = plasmaWordIndexEntity.wordHash2path(dbRoot, plasmaWordIndexEntry.word2hash(w));
|
|
|
|
|
if (f.exists()) {
|
|
|
|
|
thisamount = f.length();
|
|
|
|
|
if (f.delete()) {
|
|
|
|
|
count++;
|
|
|
|
|
totalamount += thisamount;
|
|
|
|
|
serverLog.logInfo("DELETE-STOPWORDS", "deleted index for word '" + w + "', " + thisamount + " bytes");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (f.exists()) {
|
|
|
|
|
thisamount = f.length();
|
|
|
|
|
if (f.delete()) {
|
|
|
|
|
count++;
|
|
|
|
|
totalamount += thisamount;
|
|
|
|
|
serverLog.logInfo("DELETE-STOPWORDS", "deleted index for word '" + w + "', " + thisamount + " bytes");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
serverLog.logInfo("DELETE-STOPWORDS", "TOTALS: deleted " + count + " indexes; " + (totalamount / 1024) + " kbytes");
|
|
|
|
|
serverLog.logInfo("DELETE-STOPWORDS", "TOTALS: deleted " + count + " indexes; " + (totalamount / 1024) + " kbytes");
|
|
|
|
|
|
|
|
|
|
// finished
|
|
|
|
|
serverLog.logSystem("DELETE-STOPWORDS", "FINISHED");
|
|
|
|
@ -533,13 +540,13 @@ public final class yacy {
|
|
|
|
|
deleteStopwords(applicationRoot);
|
|
|
|
|
} else if ((args.length >= 1) && (args[0].equals("-genwordstat"))) {
|
|
|
|
|
// this can help to create a stop-word list
|
|
|
|
|
// to use this, you need a 'yacy.words' file in the root path
|
|
|
|
|
// start this with "java -classpath classes yacy -genwordstat [<rootdir>]"
|
|
|
|
|
// to use this, you need a 'yacy.words' file in the root path
|
|
|
|
|
// start this with "java -classpath classes yacy -genwordstat [<rootdir>]"
|
|
|
|
|
if (args.length == 2) applicationRoot= args[1];
|
|
|
|
|
genWordstat(applicationRoot);
|
|
|
|
|
} else if ((args.length == 4) && (args[0].equals("-cleanwordlist"))) {
|
|
|
|
|
// this can be used to organize and clean a word-list
|
|
|
|
|
// start this with "java -classpath classes yacy -cleanwordlist <word-file> <minlength> <maxlength>"
|
|
|
|
|
// start this with "java -classpath classes yacy -cleanwordlist <word-file> <minlength> <maxlength>"
|
|
|
|
|
int minlength = Integer.parseInt(args[2]);
|
|
|
|
|
int maxlength = Integer.parseInt(args[3]);
|
|
|
|
|
cleanwordlist(args[1], minlength, maxlength);
|
|
|
|
@ -550,3 +557,31 @@ public final class yacy {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
class shutdownHookThread extends Thread
|
|
|
|
|
{
|
|
|
|
|
private plasmaSwitchboard sb = null;
|
|
|
|
|
private Thread mainThread = null;
|
|
|
|
|
|
|
|
|
|
public shutdownHookThread(Thread mainThread, plasmaSwitchboard sb) {
|
|
|
|
|
this.sb = sb;
|
|
|
|
|
this.mainThread = mainThread;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void run() {
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
if (!this.sb.isTerminated()) {
|
|
|
|
|
serverLog.logSystem("SHUTDOWN","Shutdown via shutdown hook.");
|
|
|
|
|
|
|
|
|
|
// sending the yacy main thread a shutdown signal
|
|
|
|
|
this.sb.terminate();
|
|
|
|
|
|
|
|
|
|
// waiting for the yacy thread to finish execution
|
|
|
|
|
this.mainThread.join();
|
|
|
|
|
}
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
e.printStackTrace();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|