diff --git a/nbproject/project.xml b/nbproject/project.xml
index a9bb922d4..c9794a254 100644
--- a/nbproject/project.xml
+++ b/nbproject/project.xml
@@ -72,12 +72,13 @@
<ide-action name="test"/>
</context-menu>
</view>
+ <subprojects/>
</general-data>
<java-data xmlns="http://www.netbeans.org/ns/freeform-project-java/3">
<compilation-unit>
<package-root>source</package-root>
<package-root>htroot</package-root>
- <classpath mode="compile">lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/apache-solr-core-4.0.0.jar;lib/apache-solr-solrj-4.0.0.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.1.jar;lib/fontbox-1.7.1.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-r05.jar;lib/htmllexer.jar;lib/htmlparser.jar;lib/httpclient-4.2.3.jar;lib/httpcore-4.2.3.jar;lib/httpmime-4.2.3.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.7.1.jar;lib/jena-2.6.4.jar;lib/jetty-6.1.26-patched-JETTY-1340.jar;lib/jetty-util-6.1.26-patched-JETTY-1340.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.17.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-4.0.0.jar;lib/lucene-core-4.0.0.jar;lib/lucene-highlighter-4.0.0.jar;lib/lucene-phonetic-4.0.0.jar;lib/lucene-spatial-4.0.0.jar;lib/lucene-spellchecker-4.0.0.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.1.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.7.2.jar;lib/slf4j-jdk14-1.7.2.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.3.6.jar</classpath>
+ <classpath mode="compile">lib/activation.jar;lib/apache-mime4j-0.6.jar;lib/arq-2.8.7.jar;lib/bcmail-jdk15-145.jar;lib/bcprov-jdk15-145.jar;lib/commons-codec-1.7.jar;lib/commons-compress-1.4.1.jar;lib/commons-fileupload-1.2.2.jar;lib/commons-httpclient-3.1.jar;lib/commons-io-2.1.jar;lib/commons-jxpath-1.3.jar;lib/commons-lang-2.6.jar;lib/commons-logging-1.1.1.jar;lib/fontbox-1.7.1.jar;lib/geronimo-stax-api_1.0_spec-1.0.1.jar;lib/guava-13.0.1.jar;lib/htmllexer.jar;lib/httpclient-4.2.3.jar;lib/httpcore-4.2.3.jar;lib/httpmime-4.2.3.jar;lib/icu4j-core.jar;lib/iri-0.8.jar;lib/J7Zip-modified.jar;lib/jakarta-oro-2.0.8.jar;lib/jaudiotagger-2.0.4-20111207.115108-15.jar;lib/jcifs-1.3.15.jar;lib/jcl-over-slf4j-1.7.2.jar;lib/jempbox-1.7.1.jar;lib/jena-2.6.4.jar;lib/jsch-0.1.42.jar;lib/json-simple-1.1.jar;lib/jsoup-1.6.3.jar;lib/log4j-1.2.17.jar;lib/log4j-over-slf4j-1.7.2.jar;lib/lucene-analyzers-common-4.1.0.jar;lib/lucene-analyzers-phonetic-4.1.0.jar;lib/lucene-core-4.1.0.jar;lib/lucene-misc-4.1.0.jar;lib/lucene-spatial-4.1.0.jar;lib/metadata-extractor-2.4.0-beta-1.jar;lib/mysql-connector-java-5.1.12-bin.jar;lib/pdfbox-1.7.1.jar;lib/poi-3.6-20091214.jar;lib/poi-scratchpad-3.6-20091214.jar;lib/sax-2.0.1.jar;lib/servlet-api-2.5-20081211.jar;lib/slf4j-api-1.7.2.jar;lib/slf4j-jdk14-1.7.2.jar;lib/solr-core-4.1.0.jar;lib/solr-solrj-4.1.0.jar;lib/spatial4j-0.3.jar;lib/webcat-0.1-swf.jar;lib/wstx-asl-3.2.7.jar;lib/xercesImpl.jar;lib/xml-apis.jar;lib/zookeeper-3.4.5.jar</classpath>
<source-level>1.6</source-level>
</compilation-unit>
</java-data>
diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java
index 29c4a62ae..99cf11916 100644
--- a/source/net/yacy/yacy.java
+++ b/source/net/yacy/yacy.java
@@ -1,692 +1,697 @@
-// yacy.java
-// -----------------------
-// (C) by Michael Peter Christen; mc@yacy.net
-// first published on http://www.yacy.net
-// Frankfurt, Germany, 2004, 2005
-//
-// $LastChangedDate$
-// $LastChangedRevision$
-// $LastChangedBy$
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-package net.yacy;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.RandomAccessFile;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
-import java.util.Properties;
-import java.util.concurrent.Semaphore;
-
-import net.yacy.cora.date.GenericFormatter;
-import net.yacy.cora.lod.JenaTripleStore;
-import net.yacy.cora.protocol.ClientIdentification;
-import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.cora.protocol.TimeoutRequest;
-import net.yacy.cora.protocol.http.HTTPClient;
-import net.yacy.cora.sorting.Array;
-import net.yacy.data.Translator;
-import net.yacy.gui.YaCyApp;
-import net.yacy.gui.framework.Browser;
-import net.yacy.kelondro.logging.Log;
-import net.yacy.kelondro.util.FileUtils;
-import net.yacy.kelondro.util.Formatter;
-import net.yacy.kelondro.util.MemoryControl;
-import net.yacy.kelondro.util.OS;
-import net.yacy.peers.operation.yacyBuildProperties;
-import net.yacy.peers.operation.yacyRelease;
-import net.yacy.peers.operation.yacyVersion;
-import net.yacy.search.Switchboard;
-import net.yacy.search.SwitchboardConstants;
-import net.yacy.server.serverCore;
-import net.yacy.server.http.HTTPDemon;
-
-import com.google.common.io.Files;
-
-
-/**
-* This is the main class of YaCy. Several threads are started from here:
-* <ul>
-* <li>one single instance of the plasmaSwitchboard is generated, which itself
-* starts a thread with a plasmaHTMLCache object. This object simply counts
-* files sizes in the cache and terminates them. It also generates a
-* plasmaCrawlerLoader object, which may itself start some more httpc-calling
-* threads to load web pages. They terminate automatically when a page has
-* loaded.
-* <li>one serverCore - thread is started, which implements a multi-threaded
-* server. The process may start itself many more processes that handle
-* connections.lo
-* <li>finally, all idle-dependent processes are written in a queue in
-* plasmaSwitchboard which are worked off inside an idle-sensitive loop of the
-* main process. (here)
-* </ul>
-*
-* On termination, the following must be done:
-* <ul>
-* <li>stop feeding of the crawling process because it otherwise fills the
-* indexing queue.
-* <li>say goodbye to connected peers and disable new connections. Don't wait for
-* success.
-* <li>first terminate the serverCore thread. This prevents that new cache
-* objects are queued.
-* <li>wait that the plasmaHTMLCache terminates (it should be normal that this
-* process already has terminated).
-* <li>then wait for termination of all loader process of the
-* plasmaCrawlerLoader.
-* <li>work off the indexing and cache storage queue. These values are inside a
-* RAM cache and would be lost otherwise.
-* <li>write all settings.
-* <li>terminate.
-* </ul>
-*/
-
-public final class yacy {
-
- // static objects
- public static final String vString = yacyBuildProperties.getVersion();
- public static float version = 0.1f;
-
- public static final String vDATE = yacyBuildProperties.getBuildDate();
- public static final String copyright = "[ YaCy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
- public static final String hline = "-------------------------------------------------------------------------------";
- public static final Semaphore shutdownSemaphore = new Semaphore(0);
-
- /**
- * a reference to the {@link Switchboard} created by the
- * {@link yacy#startup(String, long, long)} method.
- */
- private static Switchboard sb = null;
- public static String homedir;
- public static File dataHome_g;
-
- /**
- * Starts up the whole application. Sets up all datastructures and starts
- * the main threads.
- *
- * @param homePath Root-path where all information is to be found.
- * @param startupFree free memory at startup time, to be used later for statistics
- */
- private static void startup(final File dataHome, final File appHome, final long startupMemFree, final long startupMemTotal, final boolean gui) {
- try {
- // start up
- System.out.println(copyright);
- System.out.println(hline);
-
- // check java version
- try {
- "a".isEmpty(); // needs at least Java 1.6
- } catch (final NoSuchMethodError e) {
- System.err.println("STARTUP: Java Version too low. You need at least Java 1.6 to run YaCy");
- System.exit(-1);
- }
-
- // ensure that there is a DATA directory, if not, create one and if that fails warn and die
- mkdirsIfNeseccary(dataHome);
- dataHome_g = dataHome;
- mkdirsIfNeseccary(appHome);
- File f = new File(dataHome, "DATA/");
- mkdirsIfNeseccary(f);
- if (!(f.exists())) {
- System.err.println("Error creating DATA-directory in " + dataHome.toString() + " . Please check your write-permission for this folder. YaCy will now terminate.");
- System.exit(-1);
- }
-
- homedir = appHome.toString();
-
- // setting up logging
- f = new File(dataHome, "DATA/LOG/");
- mkdirsIfNeseccary(f);
- f = new File(dataHome, "DATA/LOG/yacy.logging");
- final File f0 = new File(appHome, "defaults/yacy.logging");
- if (!f.exists() || f0.lastModified() > f.lastModified()) try {
- Files.copy(f0, f);
- } catch (final IOException e){
- System.out.println("could not copy yacy.logging");
- }
- try{
- Log.configureLogging(dataHome, appHome, new File(dataHome, "DATA/LOG/yacy.logging"));
- } catch (final IOException e) {
- System.out.println("could not find logging properties in homePath=" + dataHome);
- Log.logException(e);
- }
- Log.logConfig("STARTUP", "YaCy version: " + yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision());
- Log.logConfig("STARTUP", "Java version: " + System.getProperty("java.version", "no-java-version"));
- Log.logConfig("STARTUP", "Operation system: " + System.getProperty("os.name","unknown"));
- Log.logConfig("STARTUP", "Application root-path: " + appHome);
- Log.logConfig("STARTUP", "Data root-path: " + dataHome);
- Log.logConfig("STARTUP", "Time zone: UTC" + GenericFormatter.UTCDiffString() + "; UTC+0000 is " + System.currentTimeMillis());
- Log.logConfig("STARTUP", "Maximum file system path length: " + OS.maxPathLength);
-
- f = new File(dataHome, "DATA/yacy.running");
- final String conf = "DATA/SETTINGS/yacy.conf".replace("/", File.separator);
- if (f.exists()) { // another instance running? VM crash? User will have to care about this
- Log.logSevere("STARTUP", "WARNING: the file " + f + " exists, this usually means that a YaCy instance is still running");
-
- // If YaCy is actually running, then we check if the server port is open.
- // If yes, then we consider that a restart is a user mistake and then we just respond
- // as the user expects and tell the browser to open the start page.
- // That will especially happen if Windows Users double-Click the YaCy Icon on the desktop to simply
- // open the web interface. (They don't think of 'servers' they just want to get to the search page).
- // We need to parse the configuration file for that to get the host port
- File dataFile = new File(dataHome, conf);
- if (dataFile.exists()) {
- Properties p = new Properties();
- p.load(new FileInputStream(dataFile));
- int port = Integer.parseInt(p.getProperty("port", "8090"));
- if (TimeoutRequest.ping("127.0.0.1", port, 1000)) {
- Browser.openBrowser("http://localhost:" + port + "/" + p.getProperty(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html"));
- // Thats it; YaCy was running, the user is happy, we can stop now.
- Log.logSevere("STARTUP", "WARNING: YaCy instance was still running; just opening the browser and exit.");
- System.exit(0);
- }
- }
-
- // YaCy is not running; thus delete the file an go on as nothing was wrong.
- delete(f);
- }
- if (!f.createNewFile()) Log.logSevere("STARTUP", "WARNING: the file " + f + " can not be created!");
- try { new FileOutputStream(f).write(Integer.toString(OS.getPID()).getBytes()); } catch (final Exception e) { } // write PID
- f.deleteOnExit();
- FileChannel channel = null;
- FileLock lock = null;
- try {
- channel = new RandomAccessFile(f,"rw").getChannel();
- lock = channel.tryLock(); // lock yacy.running
- } catch (final Exception e) { }
-
- try {
- sb = new Switchboard(dataHome, appHome, "defaults/yacy.init".replace("/", File.separator), conf);
- } catch (final RuntimeException e) {
- Log.logSevere("STARTUP", "YaCy cannot start: " + e.getMessage(), e);
- System.exit(-1);
- }
- //sbSync.V(); // signal that the sb reference was set
-
- // switch the memory strategy
- MemoryControl.setStandardStrategy(sb.getConfigBool("memory.standardStrategy", true));
-
- // save information about available memory at startup time
- sb.setConfig("memoryFreeAfterStartup", startupMemFree);
- sb.setConfig("memoryTotalAfterStartup", startupMemTotal);
-
- // start gui if wanted
- if (gui) YaCyApp.start("localhost", (int) sb.getConfigLong("port", 8090));
-
- // hardcoded, forced, temporary value-migration
- sb.setConfig("htTemplatePath", "htroot/env/templates");
-
- int oldRev;
- try {
- oldRev = Integer.parseInt(sb.getConfig("svnRevision", "0"));
- } catch (final NumberFormatException e) {
- oldRev = 0;
- }
- final int newRev = Integer.parseInt(yacyBuildProperties.getSVNRevision());
- sb.setConfig("svnRevision", yacyBuildProperties.getSVNRevision());
- sb.setConfig("applicationRoot", appHome.toString());
- sb.setConfig("dataRoot", dataHome.toString());
- yacyVersion.latestRelease = version;
-
- // read environment
- final int timeout = Math.max(5000, Integer.parseInt(sb.getConfig("httpdTimeout", "5000")));
-
- // create some directories
- final File htRootPath = new File(appHome, sb.getConfig("htRootPath", "htroot"));
- final File htDocsPath = sb.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT);
- mkdirIfNeseccary(htDocsPath);
- //final File htTemplatePath = new File(homePath, sb.getConfig("htTemplatePath","htdocs"));
-
- // create default notifier picture
- //TODO: Use templates instead of copying images ...
- if (!((new File(htDocsPath, "notifier.gif")).exists())) try {
- Files.copy(new File(htRootPath, "env/grafics/empty.gif"),
- new File(htDocsPath, "notifier.gif"));
- } catch (final IOException e) {}
-
- final File htdocsReadme = new File(htDocsPath, "readme.txt");
- if (!(htdocsReadme.exists())) try {FileUtils.copy((
- "This is your root directory for individual Web Content\r\n" +
- "\r\n" +
- "Please place your html files into the www subdirectory.\r\n" +
- "The URL of that path is either\r\n" +
- "http://www.<your-peer-name>.yacy or\r\n" +
- "http://<your-ip>:<your-port>/www\r\n" +
- "\r\n" +
- "Other subdirectories may be created; they map to corresponding sub-domains.\r\n" +
- "This directory shares it's content with the applications htroot path, so you\r\n" +
- "may access your yacy search page with\r\n" +
- "http://<your-peer-name>.yacy/\r\n" +
- "\r\n").getBytes(), htdocsReadme);} catch (final IOException e) {
- System.out.println("Error creating htdocs readme: " + e.getMessage());
- }
-
- final File wwwDefaultPath = new File(htDocsPath, "www");
- mkdirIfNeseccary(wwwDefaultPath);
-
-
- final File shareDefaultPath = new File(htDocsPath, "share");
- mkdirIfNeseccary(shareDefaultPath);
-
- migration.migrate(sb, oldRev, newRev);
-
- // delete old release files
- final int deleteOldDownloadsAfterDays = (int) sb.getConfigLong("update.deleteOld", 30);
- yacyRelease.deleteOldDownloads(sb.releasePath, deleteOldDownloadsAfterDays );
-
- // set user-agent
- HTTPClient.setDefaultUserAgent(ClientIdentification.getUserAgent());
-
- // initial fill of the triplestore
- File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath()));
- mkdirIfNeseccary(triplestore);
- for (String s: triplestore.list()) {
- if ((s.endsWith(".rdf") || s.endsWith(".nt")) && !s.equals("local.rdf") && !s.endsWith("_triplestore.rdf") && !s.startsWith("private_store_")) {
- try {
- JenaTripleStore.load(new File(triplestore, s).getAbsolutePath());
- } catch (IOException e) {
- Log.logException(e);
- }
- }
- }
- if (sb.getConfigBool("triplestore.persistent", false)) {
- File local = new File(triplestore, "local.rdf");
- if (local.exists()) {
- try {
- JenaTripleStore.load(local.getAbsolutePath());
- } catch (IOException e) {
- Log.logException(e);
- }
- }
- }
-
- // start main threads
- final String port = sb.getConfig("port", "8090");
- try {
- final HTTPDemon protocolHandler = new HTTPDemon(sb);
- final serverCore server = new serverCore(
- timeout /*control socket timeout in milliseconds*/,
- true /* block attacks (wrong protocol) */,
- protocolHandler /*command class*/,
- sb,
- 30000 /*command max length incl. GET args*/);
- server.setName("httpd:"+port);
- server.setPriority(Thread.MAX_PRIORITY);
- server.setObeyIntermission(false);
-
- // start the server
- sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0);
- //server.start();
-
- // open the browser window
- final boolean browserPopUpTrigger = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_TRIGGER, "true").equals("true");
- if (browserPopUpTrigger) try {
- final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
- //boolean properPW = (sb.getConfig("adminAccount", "").isEmpty()) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0);
- //if (!properPW) browserPopUpPage = "ConfigBasic.html";
- Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage);
- } catch (final Throwable e) {
- // cannot open browser. This may be normal in headless environments
- //Log.logException(e);
- }
-
- // enable browser popup, http server is ready now
- sb.tray.setReady();
-
- //regenerate Locales from Translationlist, if needed
- final File locale_source = sb.getAppPath("locale.source", "locales");
- final String lang = sb.getConfig("locale.language", "");
- if (!lang.equals("") && !lang.equals("default")) { //locale is used
- String currentRev = "";
- try{
- final BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang+"/version" ))));
- currentRev = br.readLine();
- br.close();
- }catch(final IOException e){
- //Error
- }
-
- if (!currentRev.equals(sb.getConfig("svnRevision", ""))) try { //is this another version?!
- final File sourceDir = new File(sb.getConfig("htRootPath", "htroot"));
- final File destDir = new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang);
- if (Translator.translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it
- //write the new Versionnumber
- final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version"))));
- bw.write(sb.getConfig("svnRevision", "Error getting Version"));
- bw.close();
- }
- } catch (final IOException e) {}
- }
- // initialize number formatter with this locale
- Formatter.setLocale(lang);
-
- // registering shutdown hook
- Log.logConfig("STARTUP", "Registering Shutdown Hook");
- final Runtime run = Runtime.getRuntime();
- run.addShutdownHook(new shutdownHookThread(Thread.currentThread(), sb));
-
- // save information about available memory after all initializations
- //try {
- sb.setConfig("memoryFreeAfterInitBGC", MemoryControl.free());
- sb.setConfig("memoryTotalAfterInitBGC", MemoryControl.total());
- System.gc();
- sb.setConfig("memoryFreeAfterInitAGC", MemoryControl.free());
- sb.setConfig("memoryTotalAfterInitAGC", MemoryControl.total());
- //} catch (ConcurrentModificationException e) {}
-
- // wait for server shutdown
- try {
- sb.waitForShutdown();
- } catch (final Exception e) {
- Log.logSevere("MAIN CONTROL LOOP", "PANIC: " + e.getMessage(),e);
- }
- // shut down
- Array.terminate();
- Log.logConfig("SHUTDOWN", "caught termination signal");
- server.terminate(false);
- server.interrupt();
- server.close();
-
- // idle until the processes are down
- if (server.isAlive()) {
- server.interrupt();
- }
- Log.logConfig("SHUTDOWN", "server has terminated");
- sb.close();
- } catch (final Exception e) {
- Log.logSevere("STARTUP", "Unexpected Error: " + e.getClass().getName(),e);
- //System.exit(1);
- }
- if(lock != null) lock.release();
- if(channel != null) channel.close();
- } catch (final Exception ee) {
- Log.logSevere("STARTUP", "FATAL ERROR: " + ee.getMessage(),ee);
- } finally {
- }
-
- // save the triple store
- if (sb.getConfigBool("triplestore.persistent", false)) {
- JenaTripleStore.saveAll();
- }
-
- Log.logConfig("SHUTDOWN", "goodbye. (this is the last line)");
- Log.shutdown();
- shutdownSemaphore.release(1000);
- try {
- System.exit(0);
- } catch (final Exception e) {} // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790)
- }
-
- /**
- * @param f
- */
- private static void delete(final File f) {
- if(!f.delete())
- Log.logSevere("STARTUP", "WARNING: the file " + f + " can not be deleted!");
- }
-
- /**
- * @see File#mkdir()
- * @param path
- */
- private static void mkdirIfNeseccary(final File path) {
- if (!(path.exists()))
- if(!path.mkdir())
- Log.logWarning("STARTUP", "could not create directory "+ path.toString());
- }
-
- /**
- * @see File#mkdirs()
- * @param path
- */
- public static void mkdirsIfNeseccary(final File path) {
- if (!(path.exists()))
- if(!path.mkdirs())
- Log.logWarning("STARTUP", "could not create directories "+ path.toString());
- }
-
- /**
- * Loads the configuration from the data-folder.
- * FIXME: Why is this called over and over again from every method, instead
- * of setting the configurationdata once for this class in main?
- *
- * @param mes Where are we called from, so that the errormessages can be
- * more descriptive.
- * @param homePath Root-path where all the information is to be found.
- * @return Properties read from the configurationfile.
- */
- private static Properties configuration(final String mes, final File homePath) {
- Log.logConfig(mes, "Application Root Path: " + homePath.toString());
-
- // read data folder
- final File dataFolder = new File(homePath, "DATA");
- if (!(dataFolder.exists())) {
- Log.logSevere(mes, "Application was never started or root path wrong.");
- System.exit(-1);
- }
-
- final Properties config = new Properties();
- FileInputStream fis = null;
- try {
- fis = new FileInputStream(new File(homePath, "DATA/SETTINGS/yacy.conf"));
- config.load(fis);
- } catch (final FileNotFoundException e) {
- Log.logSevere(mes, "could not find configuration file.");
- System.exit(-1);
- } catch (final IOException e) {
- Log.logSevere(mes, "could not read configuration file.");
- System.exit(-1);
- } finally {
- if(fis != null) {
- try {
- fis.close();
- } catch (final IOException e) {
- Log.logException(e);
- }
- }
- }
-
- return config;
- }
-
- /**
- * Call the shutdown-page of YaCy to tell it to shut down. This method is
- * called if you start yacy with the argument -shutdown.
- *
- * @param homePath Root-path where all the information is to be found.
- */
- public static void shutdown(final File homePath) {
- // start up
- System.out.println(copyright);
- System.out.println(hline);
- submitURL(homePath, "Steering.html?shutdown=", "Terminate YaCy");
- }
-
- public static void update(final File homePath) {
- // start up
- System.out.println(copyright);
- System.out.println(hline);
- submitURL(homePath, "ConfigUpdate_p.html?autoUpdate=", "Update YaCy to most recent version");
- }
-
- private static void submitURL(final File homePath, final String path, final String processdescription) {
- final Properties config = configuration("COMMAND-STEERING", homePath);
-
- // read port
- final int port = serverCore.getPortNr(config.getProperty("port", "8090"));
-
- // read password
- String encodedPassword = (String) config.get(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5);
- if (encodedPassword == null) encodedPassword = ""; // not defined
-
- // send 'wget' to web interface
- final RequestHeader requestHeader = new RequestHeader();
- requestHeader.put(RequestHeader.AUTHORIZATION, "realm=" + encodedPassword); // for http-authentify
-// final Client con = new Client(10000, requestHeader);
- final HTTPClient con = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT);
- con.setHeader(requestHeader.entrySet());
-// ResponseContainer res = null;
- try {
-// res = con.GET("http://localhost:"+ port +"/" + path);
- con.GETbytes("http://localhost:"+ port +"/" + path);
-
- // read response
-// if (res.getStatusLine().startsWith("2")) {
- if (con.getStatusCode() > 199 && con.getStatusCode() < 300) {
- Log.logConfig("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);
-// final ByteArrayOutputStream bos = new ByteArrayOutputStream(); //This is stream is not used???
-// try {
-// FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(bos));
-// } finally {
-// res.closeStream();
-// }
- } else {
-// Log.logSevere("COMMAND-STEERING", "error response from YACY socket: " + res.getStatusLine());
- Log.logSevere("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine());
- System.exit(-1);
- }
- } catch (final IOException e) {
- Log.logSevere("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage());
- System.exit(-1);
-// } finally {
-// // release connection
-// if(res != null) {
-// res.closeStream();
-// }
- }
-
- try {
- HTTPClient.closeConnectionManager();
- } catch (final InterruptedException e) {
- e.printStackTrace();
- }
-
- // finished
- Log.logConfig("COMMAND-STEERING", "SUCCESSFULLY FINISHED COMMAND: " + processdescription);
- }
-
- /**
- * Main-method which is started by java. Checks for special arguments or
- * starts up the application.
- *
- * @param args
- * Given arguments from the command line.
- */
- public static void main(String args[]) {
-
- try {
-
- // check assertion status
- //ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true);
- boolean assertionenabled = false;
- assert (assertionenabled = true) == true; // compare to true to remove warning: "Possible accidental assignement"
- if (assertionenabled) System.out.println("Asserts are enabled");
-
- // check memory amount
- System.gc();
- final long startupMemFree = MemoryControl.free();
- final long startupMemTotal = MemoryControl.total();
-
- // maybe go into headless awt mode: we have three cases depending on OS and one exception:
- // windows : better do not go into headless mode
- // mac : go into headless mode because an application is shown in gui which may not be wanted
- // linux : go into headless mode because this does not need any head operation
- // exception : if the -gui option is used then do not go into headless mode since that uses a gui
- boolean headless = true;
- if (OS.isWindows) headless = false;
- if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) headless = false;
- System.setProperty("java.awt.headless", headless ? "true" : "false");
- System.setProperty("java.net.preferIPv4Stack", "true");
-
- String s = ""; for (final String a: args) s += a + " ";
- yacyRelease.startParameter = s.trim();
-
- File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/'));
- File dataRoot = applicationRoot;
- //System.out.println("args.length=" + args.length);
- //System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]");
- if ((args.length >= 1) && (args[0].toLowerCase().equals("-startup") || args[0].equals("-start"))) {
- // normal start-up of yacy
- if (args.length > 1) dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
- startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
- } else if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) {
- // start-up of yacy with gui
- if (args.length > 1) dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
- startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, true);
- } else if ((args.length >= 1) && ((args[0].toLowerCase().equals("-shutdown")) || (args[0].equals("-stop")))) {
- // normal shutdown of yacy
- if (args.length == 2) applicationRoot= new File(args[1]);
- shutdown(applicationRoot);
- } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-update"))) {
- // aut-update yacy
- if (args.length == 2) applicationRoot= new File(args[1]);
- update(applicationRoot);
- } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-version"))) {
- // show yacy version
- System.out.println(copyright);
- } else {
- if (args.length == 1) applicationRoot= new File(args[0]);
- startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
- }
- } finally {
- Log.shutdown();
- }
- }
-}
-
-/**
-* This class is a helper class whose instance is started, when the java virtual
-* machine shuts down. Signals the plasmaSwitchboard to shut down.
-*/
-class shutdownHookThread extends Thread {
- private final Switchboard sb;
- private final Thread mainThread;
-
- public shutdownHookThread(final Thread mainThread, final Switchboard sb) {
- super();
- this.sb = sb;
- this.mainThread = mainThread;
- }
-
- @Override
- public void run() {
- try {
- if (!this.sb.isTerminated()) {
- Log.logConfig("SHUTDOWN","Shutdown via shutdown hook.");
-
- // sending the yacy main thread a shutdown signal
- Log.logFine("SHUTDOWN","Signaling shutdown to the switchboard.");
- this.sb.terminate("shutdown hook");
-
- // waiting for the yacy thread to finish execution
- Log.logFine("SHUTDOWN","Waiting for main thread to finish.");
- if (this.mainThread.isAlive() && !this.sb.isTerminated()) {
- this.mainThread.join();
- }
- }
- } catch (final Exception e) {
- Log.logSevere("SHUTDOWN","Unexpected error. " + e.getClass().getName(),e);
- }
- }
-}
+// yacy.java
+// -----------------------
+// (C) by Michael Peter Christen; mc@yacy.net
+// first published on http://www.yacy.net
+// Frankfurt, Germany, 2004, 2005
+//
+// $LastChangedDate$
+// $LastChangedRevision$
+// $LastChangedBy$
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package net.yacy;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.util.Properties;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ExecutionException;
+
+import net.yacy.cora.date.GenericFormatter;
+import net.yacy.cora.lod.JenaTripleStore;
+import net.yacy.cora.protocol.ClientIdentification;
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.protocol.TimeoutRequest;
+import net.yacy.cora.protocol.http.HTTPClient;
+import net.yacy.cora.sorting.Array;
+import net.yacy.data.Translator;
+import net.yacy.gui.YaCyApp;
+import net.yacy.gui.framework.Browser;
+import net.yacy.kelondro.logging.Log;
+import net.yacy.kelondro.util.FileUtils;
+import net.yacy.kelondro.util.Formatter;
+import net.yacy.kelondro.util.MemoryControl;
+import net.yacy.kelondro.util.OS;
+import net.yacy.peers.operation.yacyBuildProperties;
+import net.yacy.peers.operation.yacyRelease;
+import net.yacy.peers.operation.yacyVersion;
+import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
+import net.yacy.server.serverCore;
+import net.yacy.server.http.HTTPDemon;
+
+import com.google.common.io.Files;
+
+
+/**
+* This is the main class of YaCy. Several threads are started from here:
+* <ul>
+* <li>one single instance of the plasmaSwitchboard is generated, which itself
+* starts a thread with a plasmaHTMLCache object. This object simply counts
+* files sizes in the cache and terminates them. It also generates a
+* plasmaCrawlerLoader object, which may itself start some more httpc-calling
+* threads to load web pages. They terminate automatically when a page has
+* loaded.
+* <li>one serverCore - thread is started, which implements a multi-threaded
+* server. The process may start itself many more processes that handle
+* connections.lo
+* <li>finally, all idle-dependent processes are written in a queue in
+* plasmaSwitchboard which are worked off inside an idle-sensitive loop of the
+* main process. (here)
+* </ul>
+*
+* On termination, the following must be done:
+* <ul>
+* <li>stop feeding of the crawling process because it otherwise fills the
+* indexing queue.
+* <li>say goodbye to connected peers and disable new connections. Don't wait for
+* success.
+* <li>first terminate the serverCore thread. This prevents that new cache
+* objects are queued.
+* <li>wait that the plasmaHTMLCache terminates (it should be normal that this
+* process already has terminated).
+* <li>then wait for termination of all loader process of the
+* plasmaCrawlerLoader.
+* <li>work off the indexing and cache storage queue. These values are inside a
+* RAM cache and would be lost otherwise.
+* <li>write all settings.
+* <li>terminate.
+* </ul>
+*/
+
+public final class yacy {
+
+ // static objects
+ public static final String vString = yacyBuildProperties.getVersion();
+ public static float version = 0.1f;
+
+ public static final String vDATE = yacyBuildProperties.getBuildDate();
+ public static final String copyright = "[ YaCy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
+ public static final String hline = "-------------------------------------------------------------------------------";
+ public static final Semaphore shutdownSemaphore = new Semaphore(0);
+
+ /**
+ * a reference to the {@link Switchboard} created by the
+ * {@link yacy#startup(String, long, long)} method.
+ */
+ private static Switchboard sb = null;
+ public static String homedir;
+ public static File dataHome_g;
+
+ /**
+ * Starts up the whole application. Sets up all datastructures and starts
+ * the main threads.
+ *
+ * @param homePath Root-path where all information is to be found.
+ * @param startupFree free memory at startup time, to be used later for statistics
+ */
+ private static void startup(final File dataHome, final File appHome, final long startupMemFree, final long startupMemTotal, final boolean gui) {
+ try {
+ // start up
+ System.out.println(copyright);
+ System.out.println(hline);
+
+ // check java version
+ try {
+ "a".isEmpty(); // needs at least Java 1.6
+ } catch (final NoSuchMethodError e) {
+ System.err.println("STARTUP: Java Version too low. You need at least Java 1.6 to run YaCy");
+ System.exit(-1);
+ }
+
+ // ensure that there is a DATA directory, if not, create one and if that fails warn and die
+ mkdirsIfNeseccary(dataHome);
+ dataHome_g = dataHome;
+ mkdirsIfNeseccary(appHome);
+ File f = new File(dataHome, "DATA/");
+ mkdirsIfNeseccary(f);
+ if (!(f.exists())) {
+ System.err.println("Error creating DATA-directory in " + dataHome.toString() + " . Please check your write-permission for this folder. YaCy will now terminate.");
+ System.exit(-1);
+ }
+
+ homedir = appHome.toString();
+
+ // setting up logging
+ f = new File(dataHome, "DATA/LOG/");
+ mkdirsIfNeseccary(f);
+ f = new File(dataHome, "DATA/LOG/yacy.logging");
+ final File f0 = new File(appHome, "defaults/yacy.logging");
+ if (!f.exists() || f0.lastModified() > f.lastModified()) try {
+ Files.copy(f0, f);
+ } catch (final IOException e){
+ System.out.println("could not copy yacy.logging");
+ }
+ try{
+ Log.configureLogging(dataHome, appHome, new File(dataHome, "DATA/LOG/yacy.logging"));
+ } catch (final IOException e) {
+ System.out.println("could not find logging properties in homePath=" + dataHome);
+ Log.logException(e);
+ }
+ Log.logConfig("STARTUP", "YaCy version: " + yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision());
+ Log.logConfig("STARTUP", "Java version: " + System.getProperty("java.version", "no-java-version"));
+ Log.logConfig("STARTUP", "Operation system: " + System.getProperty("os.name","unknown"));
+ Log.logConfig("STARTUP", "Application root-path: " + appHome);
+ Log.logConfig("STARTUP", "Data root-path: " + dataHome);
+ Log.logConfig("STARTUP", "Time zone: UTC" + GenericFormatter.UTCDiffString() + "; UTC+0000 is " + System.currentTimeMillis());
+ Log.logConfig("STARTUP", "Maximum file system path length: " + OS.maxPathLength);
+
+ f = new File(dataHome, "DATA/yacy.running");
+ final String conf = "DATA/SETTINGS/yacy.conf".replace("/", File.separator);
+ if (f.exists()) { // another instance running? VM crash? User will have to care about this
+ Log.logSevere("STARTUP", "WARNING: the file " + f + " exists, this usually means that a YaCy instance is still running");
+
+ // If YaCy is actually running, then we check if the server port is open.
+ // If yes, then we consider that a restart is a user mistake and then we just respond
+ // as the user expects and tell the browser to open the start page.
+ // That will especially happen if Windows Users double-Click the YaCy Icon on the desktop to simply
+ // open the web interface. (They don't think of 'servers' they just want to get to the search page).
+ // We need to parse the configuration file for that to get the host port
+ File dataFile = new File(dataHome, conf);
+ if (dataFile.exists()) {
+ Properties p = new Properties();
+ p.load(new FileInputStream(dataFile));
+ int port = Integer.parseInt(p.getProperty("port", "8090"));
+ try {
+ if (TimeoutRequest.ping("127.0.0.1", port, 1000)) {
+ Browser.openBrowser("http://localhost:" + port + "/" + p.getProperty(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html"));
+ // Thats it; YaCy was running, the user is happy, we can stop now.
+ Log.logSevere("STARTUP", "WARNING: YaCy instance was still running; just opening the browser and exit.");
+ System.exit(0);
+ }
+ } catch (ExecutionException ex) {
+ Log.logInfo("STARTUP", "INFO: delete old yacy.running file; likely previous YaCy session was not orderly shutdown!");
+ }
+ }
+
+ // YaCy is not running; thus delete the file an go on as nothing was wrong.
+ delete(f);
+ }
+ if (!f.createNewFile()) Log.logSevere("STARTUP", "WARNING: the file " + f + " can not be created!");
+ try { new FileOutputStream(f).write(Integer.toString(OS.getPID()).getBytes()); } catch (final Exception e) { } // write PID
+ f.deleteOnExit();
+ FileChannel channel = null;
+ FileLock lock = null;
+ try {
+ channel = new RandomAccessFile(f,"rw").getChannel();
+ lock = channel.tryLock(); // lock yacy.running
+ } catch (final Exception e) { }
+
+ try {
+ sb = new Switchboard(dataHome, appHome, "defaults/yacy.init".replace("/", File.separator), conf);
+ } catch (final RuntimeException e) {
+ Log.logSevere("STARTUP", "YaCy cannot start: " + e.getMessage(), e);
+ System.exit(-1);
+ }
+ //sbSync.V(); // signal that the sb reference was set
+
+ // switch the memory strategy
+ MemoryControl.setStandardStrategy(sb.getConfigBool("memory.standardStrategy", true));
+
+ // save information about available memory at startup time
+ sb.setConfig("memoryFreeAfterStartup", startupMemFree);
+ sb.setConfig("memoryTotalAfterStartup", startupMemTotal);
+
+ // start gui if wanted
+ if (gui) YaCyApp.start("localhost", (int) sb.getConfigLong("port", 8090));
+
+ // hardcoded, forced, temporary value-migration
+ sb.setConfig("htTemplatePath", "htroot/env/templates");
+
+ int oldRev;
+ try {
+ oldRev = Integer.parseInt(sb.getConfig("svnRevision", "0"));
+ } catch (final NumberFormatException e) {
+ oldRev = 0;
+ }
+ final int newRev = Integer.parseInt(yacyBuildProperties.getSVNRevision());
+ sb.setConfig("svnRevision", yacyBuildProperties.getSVNRevision());
+ sb.setConfig("applicationRoot", appHome.toString());
+ sb.setConfig("dataRoot", dataHome.toString());
+ yacyVersion.latestRelease = version;
+
+ // read environment
+ final int timeout = Math.max(5000, Integer.parseInt(sb.getConfig("httpdTimeout", "5000")));
+
+ // create some directories
+ final File htRootPath = new File(appHome, sb.getConfig("htRootPath", "htroot"));
+ final File htDocsPath = sb.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT);
+ mkdirIfNeseccary(htDocsPath);
+ //final File htTemplatePath = new File(homePath, sb.getConfig("htTemplatePath","htdocs"));
+
+ // create default notifier picture
+ //TODO: Use templates instead of copying images ...
+ if (!((new File(htDocsPath, "notifier.gif")).exists())) try {
+ Files.copy(new File(htRootPath, "env/grafics/empty.gif"),
+ new File(htDocsPath, "notifier.gif"));
+ } catch (final IOException e) {}
+
+ final File htdocsReadme = new File(htDocsPath, "readme.txt");
+ if (!(htdocsReadme.exists())) try {FileUtils.copy((
+ "This is your root directory for individual Web Content\r\n" +
+ "\r\n" +
+ "Please place your html files into the www subdirectory.\r\n" +
+ "The URL of that path is either\r\n" +
+ "http://www.<your-peer-name>.yacy or\r\n" +
+ "http://<your-ip>:<your-port>/www\r\n" +
+ "\r\n" +
+ "Other subdirectories may be created; they map to corresponding sub-domains.\r\n" +
+ "This directory shares it's content with the applications htroot path, so you\r\n" +
+ "may access your yacy search page with\r\n" +
+ "http://<your-peer-name>.yacy/\r\n" +
+ "\r\n").getBytes(), htdocsReadme);} catch (final IOException e) {
+ System.out.println("Error creating htdocs readme: " + e.getMessage());
+ }
+
+ final File wwwDefaultPath = new File(htDocsPath, "www");
+ mkdirIfNeseccary(wwwDefaultPath);
+
+
+ final File shareDefaultPath = new File(htDocsPath, "share");
+ mkdirIfNeseccary(shareDefaultPath);
+
+ migration.migrate(sb, oldRev, newRev);
+
+ // delete old release files
+ final int deleteOldDownloadsAfterDays = (int) sb.getConfigLong("update.deleteOld", 30);
+ yacyRelease.deleteOldDownloads(sb.releasePath, deleteOldDownloadsAfterDays );
+
+ // set user-agent
+ HTTPClient.setDefaultUserAgent(ClientIdentification.getUserAgent());
+
+ // initial fill of the triplestore
+ File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath()));
+ mkdirIfNeseccary(triplestore);
+ for (String s: triplestore.list()) {
+ if ((s.endsWith(".rdf") || s.endsWith(".nt")) && !s.equals("local.rdf") && !s.endsWith("_triplestore.rdf") && !s.startsWith("private_store_")) {
+ try {
+ JenaTripleStore.load(new File(triplestore, s).getAbsolutePath());
+ } catch (IOException e) {
+ Log.logException(e);
+ }
+ }
+ }
+ if (sb.getConfigBool("triplestore.persistent", false)) {
+ File local = new File(triplestore, "local.rdf");
+ if (local.exists()) {
+ try {
+ JenaTripleStore.load(local.getAbsolutePath());
+ } catch (IOException e) {
+ Log.logException(e);
+ }
+ }
+ }
+
+ // start main threads
+ final String port = sb.getConfig("port", "8090");
+ try {
+ final HTTPDemon protocolHandler = new HTTPDemon(sb);
+ final serverCore server = new serverCore(
+ timeout /*control socket timeout in milliseconds*/,
+ true /* block attacks (wrong protocol) */,
+ protocolHandler /*command class*/,
+ sb,
+ 30000 /*command max length incl. GET args*/);
+ server.setName("httpd:"+port);
+ server.setPriority(Thread.MAX_PRIORITY);
+ server.setObeyIntermission(false);
+
+ // start the server
+ sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0);
+ //server.start();
+
+ // open the browser window
+ final boolean browserPopUpTrigger = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_TRIGGER, "true").equals("true");
+ if (browserPopUpTrigger) try {
+ final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
+ //boolean properPW = (sb.getConfig("adminAccount", "").isEmpty()) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0);
+ //if (!properPW) browserPopUpPage = "ConfigBasic.html";
+ Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage);
+ } catch (final Throwable e) {
+ // cannot open browser. This may be normal in headless environments
+ //Log.logException(e);
+ }
+
+ // enable browser popup, http server is ready now
+ sb.tray.setReady();
+
+ //regenerate Locales from Translationlist, if needed
+ final File locale_source = sb.getAppPath("locale.source", "locales");
+ final String lang = sb.getConfig("locale.language", "");
+ if (!lang.equals("") && !lang.equals("default")) { //locale is used
+ String currentRev = "";
+ try{
+ final BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang+"/version" ))));
+ currentRev = br.readLine();
+ br.close();
+ }catch(final IOException e){
+ //Error
+ }
+
+ if (!currentRev.equals(sb.getConfig("svnRevision", ""))) try { //is this another version?!
+ final File sourceDir = new File(sb.getConfig("htRootPath", "htroot"));
+ final File destDir = new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang);
+ if (Translator.translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it
+ //write the new Versionnumber
+ final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version"))));
+ bw.write(sb.getConfig("svnRevision", "Error getting Version"));
+ bw.close();
+ }
+ } catch (final IOException e) {}
+ }
+ // initialize number formatter with this locale
+ Formatter.setLocale(lang);
+
+ // registering shutdown hook
+ Log.logConfig("STARTUP", "Registering Shutdown Hook");
+ final Runtime run = Runtime.getRuntime();
+ run.addShutdownHook(new shutdownHookThread(Thread.currentThread(), sb));
+
+ // save information about available memory after all initializations
+ //try {
+ sb.setConfig("memoryFreeAfterInitBGC", MemoryControl.free());
+ sb.setConfig("memoryTotalAfterInitBGC", MemoryControl.total());
+ System.gc();
+ sb.setConfig("memoryFreeAfterInitAGC", MemoryControl.free());
+ sb.setConfig("memoryTotalAfterInitAGC", MemoryControl.total());
+ //} catch (ConcurrentModificationException e) {}
+
+ // wait for server shutdown
+ try {
+ sb.waitForShutdown();
+ } catch (final Exception e) {
+ Log.logSevere("MAIN CONTROL LOOP", "PANIC: " + e.getMessage(),e);
+ }
+ // shut down
+ Array.terminate();
+ Log.logConfig("SHUTDOWN", "caught termination signal");
+ server.terminate(false);
+ server.interrupt();
+ server.close();
+
+ // idle until the processes are down
+ if (server.isAlive()) {
+ server.interrupt();
+ }
+ Log.logConfig("SHUTDOWN", "server has terminated");
+ sb.close();
+ } catch (final Exception e) {
+ Log.logSevere("STARTUP", "Unexpected Error: " + e.getClass().getName(),e);
+ //System.exit(1);
+ }
+ if(lock != null) lock.release();
+ if(channel != null) channel.close();
+ } catch (final Exception ee) {
+ Log.logSevere("STARTUP", "FATAL ERROR: " + ee.getMessage(),ee);
+ } finally {
+ }
+
+ // save the triple store
+ if (sb.getConfigBool("triplestore.persistent", false)) {
+ JenaTripleStore.saveAll();
+ }
+
+ Log.logConfig("SHUTDOWN", "goodbye. (this is the last line)");
+ Log.shutdown();
+ shutdownSemaphore.release(1000);
+ try {
+ System.exit(0);
+ } catch (final Exception e) {} // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790)
+ }
+
+ /**
+ * @param f
+ */
+ private static void delete(final File f) {
+ if(!f.delete())
+ Log.logSevere("STARTUP", "WARNING: the file " + f + " can not be deleted!");
+ }
+
+ /**
+ * @see File#mkdir()
+ * @param path
+ */
+ private static void mkdirIfNeseccary(final File path) {
+ if (!(path.exists()))
+ if(!path.mkdir())
+ Log.logWarning("STARTUP", "could not create directory "+ path.toString());
+ }
+
+ /**
+ * @see File#mkdirs()
+ * @param path
+ */
+ public static void mkdirsIfNeseccary(final File path) {
+ if (!(path.exists()))
+ if(!path.mkdirs())
+ Log.logWarning("STARTUP", "could not create directories "+ path.toString());
+ }
+
+ /**
+ * Loads the configuration from the data-folder.
+ * FIXME: Why is this called over and over again from every method, instead
+ * of setting the configurationdata once for this class in main?
+ *
+ * @param mes Where are we called from, so that the errormessages can be
+ * more descriptive.
+ * @param homePath Root-path where all the information is to be found.
+ * @return Properties read from the configurationfile.
+ */
+ private static Properties configuration(final String mes, final File homePath) {
+ Log.logConfig(mes, "Application Root Path: " + homePath.toString());
+
+ // read data folder
+ final File dataFolder = new File(homePath, "DATA");
+ if (!(dataFolder.exists())) {
+ Log.logSevere(mes, "Application was never started or root path wrong.");
+ System.exit(-1);
+ }
+
+ final Properties config = new Properties();
+ FileInputStream fis = null;
+ try {
+ fis = new FileInputStream(new File(homePath, "DATA/SETTINGS/yacy.conf"));
+ config.load(fis);
+ } catch (final FileNotFoundException e) {
+ Log.logSevere(mes, "could not find configuration file.");
+ System.exit(-1);
+ } catch (final IOException e) {
+ Log.logSevere(mes, "could not read configuration file.");
+ System.exit(-1);
+ } finally {
+ if(fis != null) {
+ try {
+ fis.close();
+ } catch (final IOException e) {
+ Log.logException(e);
+ }
+ }
+ }
+
+ return config;
+ }
+
+ /**
+ * Call the shutdown-page of YaCy to tell it to shut down. This method is
+ * called if you start yacy with the argument -shutdown.
+ *
+ * @param homePath Root-path where all the information is to be found.
+ */
+ public static void shutdown(final File homePath) {
+ // start up
+ System.out.println(copyright);
+ System.out.println(hline);
+ submitURL(homePath, "Steering.html?shutdown=", "Terminate YaCy");
+ }
+
+ public static void update(final File homePath) {
+ // start up
+ System.out.println(copyright);
+ System.out.println(hline);
+ submitURL(homePath, "ConfigUpdate_p.html?autoUpdate=", "Update YaCy to most recent version");
+ }
+
+ private static void submitURL(final File homePath, final String path, final String processdescription) {
+ final Properties config = configuration("COMMAND-STEERING", homePath);
+
+ // read port
+ final int port = serverCore.getPortNr(config.getProperty("port", "8090"));
+
+ // read password
+ String encodedPassword = (String) config.get(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5);
+ if (encodedPassword == null) encodedPassword = ""; // not defined
+
+ // send 'wget' to web interface
+ final RequestHeader requestHeader = new RequestHeader();
+ requestHeader.put(RequestHeader.AUTHORIZATION, "realm=" + encodedPassword); // for http-authentify
+// final Client con = new Client(10000, requestHeader);
+ final HTTPClient con = new HTTPClient(ClientIdentification.getUserAgent(), ClientIdentification.DEFAULT_TIMEOUT);
+ con.setHeader(requestHeader.entrySet());
+// ResponseContainer res = null;
+ try {
+// res = con.GET("http://localhost:"+ port +"/" + path);
+ con.GETbytes("http://localhost:"+ port +"/" + path);
+
+ // read response
+// if (res.getStatusLine().startsWith("2")) {
+ if (con.getStatusCode() > 199 && con.getStatusCode() < 300) {
+ Log.logConfig("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);
+// final ByteArrayOutputStream bos = new ByteArrayOutputStream(); //This is stream is not used???
+// try {
+// FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(bos));
+// } finally {
+// res.closeStream();
+// }
+ } else {
+// Log.logSevere("COMMAND-STEERING", "error response from YACY socket: " + res.getStatusLine());
+ Log.logSevere("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine());
+ System.exit(-1);
+ }
+ } catch (final IOException e) {
+ Log.logSevere("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage());
+ System.exit(-1);
+// } finally {
+// // release connection
+// if(res != null) {
+// res.closeStream();
+// }
+ }
+
+ try {
+ HTTPClient.closeConnectionManager();
+ } catch (final InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ // finished
+ Log.logConfig("COMMAND-STEERING", "SUCCESSFULLY FINISHED COMMAND: " + processdescription);
+ }
+
+ /**
+ * Main-method which is started by java. Checks for special arguments or
+ * starts up the application.
+ *
+ * @param args
+ * Given arguments from the command line.
+ */
+ public static void main(String args[]) {
+
+ try {
+
+ // check assertion status
+ //ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true);
+ boolean assertionenabled = false;
+ assert (assertionenabled = true) == true; // compare to true to remove warning: "Possible accidental assignement"
+ if (assertionenabled) System.out.println("Asserts are enabled");
+
+ // check memory amount
+ System.gc();
+ final long startupMemFree = MemoryControl.free();
+ final long startupMemTotal = MemoryControl.total();
+
+ // maybe go into headless awt mode: we have three cases depending on OS and one exception:
+ // windows : better do not go into headless mode
+ // mac : go into headless mode because an application is shown in gui which may not be wanted
+ // linux : go into headless mode because this does not need any head operation
+ // exception : if the -gui option is used then do not go into headless mode since that uses a gui
+ boolean headless = true;
+ if (OS.isWindows) headless = false;
+ if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) headless = false;
+ System.setProperty("java.awt.headless", headless ? "true" : "false");
+ System.setProperty("java.net.preferIPv4Stack", "true");
+
+ String s = ""; for (final String a: args) s += a + " ";
+ yacyRelease.startParameter = s.trim();
+
+ File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/'));
+ File dataRoot = applicationRoot;
+ //System.out.println("args.length=" + args.length);
+ //System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]");
+ if ((args.length >= 1) && (args[0].toLowerCase().equals("-startup") || args[0].equals("-start"))) {
+ // normal start-up of yacy
+ if (args.length > 1) dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
+ startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
+ } else if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) {
+ // start-up of yacy with gui
+ if (args.length > 1) dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
+ startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, true);
+ } else if ((args.length >= 1) && ((args[0].toLowerCase().equals("-shutdown")) || (args[0].equals("-stop")))) {
+ // normal shutdown of yacy
+ if (args.length == 2) applicationRoot= new File(args[1]);
+ shutdown(applicationRoot);
+ } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-update"))) {
+ // aut-update yacy
+ if (args.length == 2) applicationRoot= new File(args[1]);
+ update(applicationRoot);
+ } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-version"))) {
+ // show yacy version
+ System.out.println(copyright);
+ } else {
+ if (args.length == 1) applicationRoot= new File(args[0]);
+ startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
+ }
+ } finally {
+ Log.shutdown();
+ }
+ }
+}
+
+/**
+* This class is a helper class whose instance is started, when the java virtual
+* machine shuts down. Signals the plasmaSwitchboard to shut down.
+*/
+class shutdownHookThread extends Thread {
+ private final Switchboard sb;
+ private final Thread mainThread;
+
+ public shutdownHookThread(final Thread mainThread, final Switchboard sb) {
+ super();
+ this.sb = sb;
+ this.mainThread = mainThread;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (!this.sb.isTerminated()) {
+ Log.logConfig("SHUTDOWN","Shutdown via shutdown hook.");
+
+ // sending the yacy main thread a shutdown signal
+ Log.logFine("SHUTDOWN","Signaling shutdown to the switchboard.");
+ this.sb.terminate("shutdown hook");
+
+ // waiting for the yacy thread to finish execution
+ Log.logFine("SHUTDOWN","Waiting for main thread to finish.");
+ if (this.mainThread.isAlive() && !this.sb.isTerminated()) {
+ this.mainThread.join();
+ }
+ }
+ } catch (final Exception e) {
+ Log.logSevere("SHUTDOWN","Unexpected error. " + e.getClass().getName(),e);
+ }
+ }
+}