You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
912 lines
46 KiB
912 lines
46 KiB
// yacy.java
|
|
// -----------------------
|
|
// (C) by Michael Peter Christen; mc@yacy.net
|
|
// first published on http://www.yacy.net
|
|
// Frankfurt, Germany, 2004, 2005
|
|
//
|
|
// $LastChangedDate$
|
|
// $LastChangedRevision$
|
|
// $LastChangedBy$
|
|
//
|
|
// This program is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU General Public License as published by
|
|
// the Free Software Foundation; either version 2 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU General Public License
|
|
// along with this program; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
package net.yacy;
|
|
|
|
import java.io.BufferedReader;
|
|
import java.io.BufferedWriter;
|
|
import java.io.File;
|
|
import java.io.FileInputStream;
|
|
import java.io.FileNotFoundException;
|
|
import java.io.FileOutputStream;
|
|
import java.io.FileWriter;
|
|
import java.io.IOException;
|
|
import java.io.InputStreamReader;
|
|
import java.io.PrintWriter;
|
|
import java.io.RandomAccessFile;
|
|
import java.nio.channels.FileChannel;
|
|
import java.nio.channels.FileLock;
|
|
import java.util.ArrayList;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.List;
|
|
import java.util.Locale;
|
|
import java.util.Map;
|
|
import java.util.Properties;
|
|
import java.util.concurrent.Semaphore;
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
import org.apache.http.Header;
|
|
import org.apache.http.HttpStatus;
|
|
import org.apache.http.entity.mime.content.ContentBody;
|
|
|
|
import com.google.common.io.Files;
|
|
|
|
import net.yacy.cora.date.GenericFormatter;
|
|
import net.yacy.cora.document.encoding.UTF8;
|
|
import net.yacy.cora.document.id.DigestURL;
|
|
import net.yacy.cora.document.id.MultiProtocolURL;
|
|
import net.yacy.cora.federate.solr.instance.RemoteInstance;
|
|
import net.yacy.cora.federate.yacy.CacheStrategy;
|
|
import net.yacy.cora.order.Digest;
|
|
import net.yacy.cora.protocol.ClientIdentification;
|
|
import net.yacy.cora.protocol.ConnectionInfo;
|
|
import net.yacy.cora.protocol.HeaderFramework;
|
|
import net.yacy.cora.protocol.TimeoutRequest;
|
|
import net.yacy.cora.protocol.http.HTTPClient;
|
|
import net.yacy.cora.sorting.Array;
|
|
import net.yacy.cora.util.ConcurrentLog;
|
|
import net.yacy.crawler.retrieval.Response;
|
|
import net.yacy.data.TransactionManager;
|
|
import net.yacy.data.Translator;
|
|
import net.yacy.document.Document;
|
|
import net.yacy.gui.YaCyApp;
|
|
import net.yacy.gui.framework.Browser;
|
|
import net.yacy.http.Jetty9HttpServerImpl;
|
|
import net.yacy.http.YaCyHttpServer;
|
|
import net.yacy.kelondro.util.FileUtils;
|
|
import net.yacy.kelondro.util.Formatter;
|
|
import net.yacy.kelondro.util.MemoryControl;
|
|
import net.yacy.kelondro.util.OS;
|
|
import net.yacy.peers.Seed;
|
|
import net.yacy.peers.operation.yacyBuildProperties;
|
|
import net.yacy.peers.operation.yacyRelease;
|
|
import net.yacy.search.Switchboard;
|
|
import net.yacy.search.SwitchboardConstants;
|
|
import net.yacy.server.serverSwitch;
|
|
import net.yacy.utils.translation.TranslatorXliff;
|
|
|
|
|
|
/**
|
|
* This is the main class of YaCy. Several threads are started from here:
|
|
* <ul>
|
|
* <li>one single instance of the plasmaSwitchboard is generated, which itself
|
|
* starts a thread with a plasmaHTMLCache object. This object simply counts
|
|
* files sizes in the cache and terminates them. It also generates a
|
|
* plasmaCrawlerLoader object, which may itself start some more httpc-calling
|
|
* threads to load web pages. They terminate automatically when a page has
|
|
* loaded.
|
|
* <li>one serverCore - thread is started, which implements a multi-threaded
|
|
* server. The process may start itself many more processes that handle
|
|
* connections.lo
|
|
* <li>finally, all idle-dependent processes are written in a queue in
|
|
* plasmaSwitchboard which are worked off inside an idle-sensitive loop of the
|
|
* main process. (here)
|
|
* </ul>
|
|
*
|
|
* On termination, the following must be done:
|
|
* <ul>
|
|
* <li>stop feeding of the crawling process because it otherwise fills the
|
|
* indexing queue.
|
|
* <li>say goodbye to connected peers and disable new connections. Don't wait for
|
|
* success.
|
|
* <li>first terminate the serverCore thread. This prevents that new cache
|
|
* objects are queued.
|
|
* <li>wait that the plasmaHTMLCache terminates (it should be normal that this
|
|
* process already has terminated).
|
|
* <li>then wait for termination of all loader process of the
|
|
* plasmaCrawlerLoader.
|
|
* <li>work off the indexing and cache storage queue. These values are inside a
|
|
* RAM cache and would be lost otherwise.
|
|
* <li>write all settings.
|
|
* <li>terminate.
|
|
* </ul>
|
|
*/
|
|
|
|
public final class yacy {
|
|
|
|
// static objects
|
|
public static final String vString = yacyBuildProperties.getVersion();
|
|
|
|
public static final String vDATE = yacyBuildProperties.getBuildDate();
|
|
public static final String copyright = "[ YaCy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
|
|
public static final String hline = "-------------------------------------------------------------------------------";
|
|
public static final Semaphore shutdownSemaphore = new Semaphore(0);
|
|
|
|
public static File htDocsPath = null;
|
|
public static File shareDefaultPath = null;
|
|
public static File shareDumpDefaultPath = null;
|
|
|
|
/**
|
|
* a reference to the {@link Switchboard} created by the
|
|
* {@link yacy#startup(String, long, long)} method.
|
|
*/
|
|
private static Switchboard sb = null;
|
|
|
|
/**
|
|
* Starts up the whole application. Sets up all datastructures and starts
|
|
* the main threads.
|
|
*
|
|
* @param homePath Root-path where all information is to be found.
|
|
* @param startupFree free memory at startup time, to be used later for statistics
|
|
*/
|
|
private static void startup(final File dataHome, final File appHome, final long startupMemFree, final long startupMemTotal, final boolean gui) {
|
|
String tmpdir=null;
|
|
try {
|
|
// start up
|
|
System.out.println(copyright);
|
|
System.out.println(hline);
|
|
|
|
// ensure that there is a DATA directory, if not, create one and if that fails warn and die
|
|
mkdirsIfNeseccary(dataHome);
|
|
mkdirsIfNeseccary(appHome);
|
|
File f = new File(dataHome, "DATA/");
|
|
mkdirsIfNeseccary(f);
|
|
if (!(f.exists())) {
|
|
System.err.println("Error creating DATA-directory in " + dataHome.toString() + " . Please check your write-permission for this folder. YaCy will now terminate.");
|
|
System.exit(-1);
|
|
}
|
|
|
|
// setting up logging
|
|
f = new File(dataHome, "DATA/LOG/");
|
|
mkdirsIfNeseccary(f);
|
|
f = new File(dataHome, "DATA/LOG/yacy.logging");
|
|
final File f0 = new File(appHome, "defaults/yacy.logging");
|
|
if (!f.exists() || f0.lastModified() > f.lastModified()) try {
|
|
Files.copy(f0, f);
|
|
} catch (final IOException e){
|
|
System.out.println("could not copy yacy.logging");
|
|
}
|
|
try{
|
|
ConcurrentLog.configureLogging(dataHome, appHome, new File(dataHome, "DATA/LOG/yacy.logging"));
|
|
} catch (final IOException e) {
|
|
System.out.println("could not find logging properties in homePath=" + dataHome);
|
|
ConcurrentLog.logException(e);
|
|
}
|
|
ConcurrentLog.config("STARTUP", "YaCy version: " + yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision());
|
|
ConcurrentLog.config("STARTUP", "Java version: " + System.getProperty("java.version", "no-java-version"));
|
|
ConcurrentLog.config("STARTUP", "Operation system: " + System.getProperty("os.name","unknown"));
|
|
ConcurrentLog.config("STARTUP", "Application root-path: " + appHome);
|
|
ConcurrentLog.config("STARTUP", "Data root-path: " + dataHome);
|
|
ConcurrentLog.config("STARTUP", "Time zone: UTC" + GenericFormatter.UTCDiffString() + "; UTC+0000 is " + System.currentTimeMillis());
|
|
ConcurrentLog.config("STARTUP", "Maximum file system path length: " + OS.maxPathLength);
|
|
|
|
f = new File(dataHome, "DATA/yacy.running");
|
|
final String conf = "DATA/SETTINGS/yacy.conf".replace("/", File.separator);
|
|
if (!f.createNewFile()) ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be created!");
|
|
try { new FileOutputStream(f).write(Integer.toString(OS.getPID()).getBytes()); } catch (final Exception e) { } // write PID
|
|
f.deleteOnExit();
|
|
FileChannel channel = null;
|
|
FileLock lock = null;
|
|
try {
|
|
channel = new RandomAccessFile(f,"rw").getChannel();
|
|
lock = channel.tryLock(); // lock yacy.running
|
|
} catch (final Exception e) { }
|
|
|
|
// set jvm tmpdir to a subdir for easy cleanup (as extensive use file.deleteonexit waists memory during long runs, as todelete files names are collected and never cleaned up during runtime)
|
|
try {
|
|
tmpdir = java.nio.file.Files.createTempDirectory("yacy-tmp-").toString(); // creates sub dir in jvm's temp (see System.property "java.io.tempdir")
|
|
System.setProperty("java.io.tmpdir", tmpdir);
|
|
} catch (IOException ex) { }
|
|
|
|
try {
|
|
sb = new Switchboard(dataHome, appHome, "defaults/yacy.init".replace("/", File.separator), conf);
|
|
} catch (final RuntimeException e) {
|
|
ConcurrentLog.severe("STARTUP", "YaCy cannot start: " + e.getMessage(), e);
|
|
System.exit(-1);
|
|
}
|
|
//sbSync.V(); // signal that the sb reference was set
|
|
|
|
// switch the memory strategy
|
|
MemoryControl.setStandardStrategy(sb.getConfigBool("memory.standardStrategy", true));
|
|
|
|
// save information about available memory at startup time
|
|
sb.setConfig("memoryFreeAfterStartup", startupMemFree);
|
|
sb.setConfig("memoryTotalAfterStartup", startupMemTotal);
|
|
|
|
// start gui if wanted
|
|
if (gui) YaCyApp.start("localhost", sb.getLocalPort());
|
|
|
|
// hardcoded, forced, temporary value-migration
|
|
sb.setConfig("htTemplatePath", "htroot/env/templates");
|
|
|
|
double oldVer;
|
|
try {
|
|
String tmpversion = sb.getConfig(Seed.VERSION, "");
|
|
if (tmpversion.isEmpty()) { // before 1.83009737 only the svnRevision nr was in config (like 9737)
|
|
tmpversion = yacyBuildProperties.getVersion();
|
|
int oldRev = Integer.parseInt(sb.getConfig("svnRevision", "0"));
|
|
if (oldRev > 1) {
|
|
oldVer = Double.parseDouble(tmpversion) + oldRev / 100000000.0;
|
|
} else {
|
|
oldVer = Double.parseDouble(yacyBuildProperties.getLongVersion()); // failsafe (assume current version = no migration)
|
|
}
|
|
} else {
|
|
oldVer = Double.parseDouble(tmpversion);
|
|
}
|
|
} catch (final NumberFormatException e) {
|
|
oldVer = 0.0d;
|
|
}
|
|
final double newRev = Double.parseDouble(yacyBuildProperties.getLongVersion());
|
|
sb.setConfig(Seed.VERSION, yacyBuildProperties.getLongVersion());
|
|
sb.setConfig("applicationRoot", appHome.toString());
|
|
sb.setConfig("dataRoot", dataHome.toString());
|
|
|
|
// create some directories
|
|
final File htRootPath = new File(appHome, sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT));
|
|
mkdirIfNeseccary(htRootPath);
|
|
htDocsPath = sb.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT);
|
|
mkdirIfNeseccary(htDocsPath);
|
|
//final File htTemplatePath = new File(homePath, sb.getConfig("htTemplatePath","htdocs"));
|
|
|
|
// copy the donate iframe (better to copy this once here instead of doing this in an actual iframe in the search result)
|
|
importDonationIFrame(sb, htDocsPath);
|
|
|
|
// create default notifier picture
|
|
File notifierFile = new File(htDocsPath, "notifier.gif");
|
|
if (!notifierFile.exists()) try {Files.copy(new File(htRootPath, "env/grafics/empty.gif"), notifierFile);} catch (final IOException e) {}
|
|
|
|
final File htdocsReadme = new File(htDocsPath, "readme.txt");
|
|
if (!(htdocsReadme.exists())) try {FileUtils.copy((
|
|
"This is your root directory for individual Web Content\r\n" +
|
|
"\r\n" +
|
|
"Please place your html files into the www subdirectory.\r\n" +
|
|
"The URL of that path is either\r\n" +
|
|
"http://www.<your-peer-name>.yacy or\r\n" +
|
|
"http://<your-ip>:<your-port>/www\r\n" +
|
|
"\r\n" +
|
|
"Other subdirectories may be created; they map to corresponding sub-domains.\r\n" +
|
|
"This directory shares it's content with the applications htroot path, so you\r\n" +
|
|
"may access your yacy search page with\r\n" +
|
|
"http://<your-peer-name>.yacy/\r\n" +
|
|
"\r\n").getBytes(), htdocsReadme);} catch (final IOException e) {
|
|
System.out.println("Error creating htdocs readme: " + e.getMessage());
|
|
}
|
|
|
|
shareDefaultPath = new File(htDocsPath, "share");
|
|
mkdirIfNeseccary(shareDefaultPath);
|
|
shareDumpDefaultPath = new File(shareDefaultPath, "dump");
|
|
mkdirIfNeseccary(shareDumpDefaultPath);
|
|
|
|
migration.migrate(sb, oldVer, newRev);
|
|
|
|
// delete old release files
|
|
final int deleteOldDownloadsAfterDays = (int) sb.getConfigLong("update.deleteOld", 30);
|
|
yacyRelease.deleteOldDownloads(sb.releasePath, deleteOldDownloadsAfterDays );
|
|
|
|
// set user-agent
|
|
HTTPClient.setDefaultUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent);
|
|
|
|
// start main threads
|
|
final int port = sb.getLocalPort();
|
|
try {
|
|
// start http server
|
|
YaCyHttpServer httpServer;
|
|
httpServer = new Jetty9HttpServerImpl(port);
|
|
httpServer.startupServer();
|
|
sb.setHttpServer(httpServer);
|
|
// TODO: this has no effect on Jetty (but needed to reflect configured value and limit is still used)
|
|
ConnectionInfo.setServerMaxcount(sb.getConfigInt("connectionsMax", ConnectionInfo.getMaxcount()));
|
|
|
|
ConcurrentLog.info("STARTUP",httpServer.getVersion());
|
|
|
|
// open the browser window
|
|
final boolean browserPopUpTrigger = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_TRIGGER, "true").equals("true");
|
|
if (browserPopUpTrigger) try {
|
|
final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
|
|
//boolean properPW = (sb.getConfig(SwitchboardConstants.ADMIN_ACCOUNT, "").isEmpty()) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0);
|
|
//if (!properPW) browserPopUpPage = "ConfigBasic.html";
|
|
/* YaCy main startup process must not hang because browser opening is long or fails.
|
|
* Let's open try opening the browser in a separate thread */
|
|
new Thread("Browser opening") {
|
|
@Override
|
|
public void run() {
|
|
Browser.openBrowser(("http://localhost:"+port) + "/" + browserPopUpPage);
|
|
}
|
|
}.start();
|
|
// Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage);
|
|
} catch (final Throwable e) {
|
|
// cannot open browser. This may be normal in headless environments
|
|
//Log.logException(e);
|
|
}
|
|
|
|
// enable browser popup, http server is ready now
|
|
sb.tray.setReady();
|
|
|
|
//regenerate Locales from Translationlist, if needed
|
|
final File locale_source = sb.getAppPath("locale.source", "locales");
|
|
final String lang = sb.getConfig("locale.language", "");
|
|
// on lang=browser all active translation should be checked (because any could be requested by client)
|
|
List<String> langlist;
|
|
if (lang.endsWith("browser"))
|
|
langlist = Translator.activeTranslations(); // get all translated languages
|
|
else {
|
|
langlist = new ArrayList<String>();
|
|
langlist.add(lang);
|
|
}
|
|
for (String tmplang : langlist) {
|
|
if (!tmplang.equals("") && !tmplang.equals("default") && !tmplang.equals("browser")) { //locale is used
|
|
String currentRev = null;
|
|
BufferedReader br = null;
|
|
try {
|
|
br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), tmplang + "/version"))));
|
|
currentRev = br.readLine(); // may return null
|
|
} catch (final IOException e) {
|
|
//Error
|
|
} finally {
|
|
try {
|
|
br.close();
|
|
} catch(IOException ioe) {
|
|
ConcurrentLog.warn("STARTUP", "Could not close " + tmplang + " version file");
|
|
}
|
|
}
|
|
|
|
if (currentRev == null || !currentRev.equals(sb.getConfig(Seed.VERSION, ""))) {
|
|
try { //is this another version?!
|
|
final File sourceDir = new File(sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT));
|
|
final File destDir = new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), tmplang);
|
|
if (new TranslatorXliff().translateFilesRecursive(sourceDir, destDir, new File(locale_source, tmplang + ".lng"), "html,template,inc", "locale")) { //translate it
|
|
//write the new Versionnumber
|
|
final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version"))));
|
|
bw.write(sb.getConfig(Seed.VERSION, "Error getting Version"));
|
|
bw.close();
|
|
}
|
|
} catch (final IOException e) {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// initialize number formatter with this locale
|
|
if (!lang.equals("browser")) // "default" is handled by .setLocale()
|
|
Formatter.setLocale(lang);
|
|
|
|
// registering shutdown hook
|
|
ConcurrentLog.config("STARTUP", "Registering Shutdown Hook");
|
|
final Runtime run = Runtime.getRuntime();
|
|
run.addShutdownHook(new shutdownHookThread(sb, shutdownSemaphore));
|
|
|
|
// save information about available memory after all initializations
|
|
//try {
|
|
sb.setConfig("memoryFreeAfterInitBGC", MemoryControl.free());
|
|
sb.setConfig("memoryTotalAfterInitBGC", MemoryControl.total());
|
|
System.gc();
|
|
sb.setConfig("memoryFreeAfterInitAGC", MemoryControl.free());
|
|
sb.setConfig("memoryTotalAfterInitAGC", MemoryControl.total());
|
|
//} catch (final ConcurrentModificationException e) {}
|
|
|
|
// wait for server shutdown
|
|
try {
|
|
sb.waitForShutdown();
|
|
} catch (final Exception e) {
|
|
ConcurrentLog.severe("MAIN CONTROL LOOP", "PANIC: " + e.getMessage(),e);
|
|
}
|
|
// shut down
|
|
Array.terminate();
|
|
ConcurrentLog.config("SHUTDOWN", "caught termination signal");
|
|
httpServer.stop();
|
|
|
|
ConcurrentLog.config("SHUTDOWN", "server has terminated");
|
|
sb.close();
|
|
} catch (final Exception e) {
|
|
ConcurrentLog.severe("STARTUP", "Unexpected Error: " + e.getClass().getName(),e);
|
|
//System.exit(1);
|
|
}
|
|
if(lock != null) lock.release();
|
|
if(channel != null) channel.close();
|
|
} catch (final Exception ee) {
|
|
ConcurrentLog.severe("STARTUP", "FATAL ERROR: " + ee.getMessage(),ee);
|
|
} finally {
|
|
}
|
|
|
|
if (tmpdir != null) FileUtils.deletedelete(new File(tmpdir)); // clean up temp dir (set at startup as subdir of system.propery "java.io.tmpdir")
|
|
|
|
ConcurrentLog.config("SHUTDOWN", "goodbye. (this is the last line)");
|
|
ConcurrentLog.shutdown();
|
|
shutdownSemaphore.release(1000);
|
|
try {
|
|
System.exit(0);
|
|
} catch (final Exception e) {} // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790)
|
|
}
|
|
|
|
/**
|
|
* Concurrently import the donation iframe content to serve it directly from this peer.
|
|
* @param switchBoard the SwitchBoard instance. Must not be null.
|
|
* @param htDocsDirectory the custom htdocs directory. Must not be null.
|
|
*/
|
|
private static void importDonationIFrame(final Switchboard switchBoard, final File htDocsDirectory) {
|
|
final File wwwEnvPath = new File(htDocsDirectory, "env");
|
|
mkdirIfNeseccary(wwwEnvPath);
|
|
final String iframesource = switchBoard.getConfig("donation.iframesource", "");
|
|
final String iframetarget = switchBoard.getConfig("donation.iframetarget", "");
|
|
final File iframefile = new File(htDocsDirectory, iframetarget);
|
|
if (!iframefile.exists()) new Thread("yacy.importDonationIFrame") {
|
|
@Override
|
|
public void run() {
|
|
final ClientIdentification.Agent agent = ClientIdentification.getAgent(ClientIdentification.yacyInternetCrawlerAgentName);
|
|
Response documentResponse;
|
|
try {
|
|
/* Load the donation html frame content */
|
|
documentResponse = switchBoard.loader == null ? null : switchBoard.loader.load(switchBoard.loader.request(new DigestURL(iframesource), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent);
|
|
if (documentResponse != null) {
|
|
Document[] documents = documentResponse.parse();
|
|
if(documents != null && documents.length > 0 && documents[0] != null) {
|
|
Document donateDocument = documents[0];
|
|
String donateDocContent = new String(documentResponse.getContent(), donateDocument.getCharset());
|
|
/* Load image resources contained in the page */
|
|
if(donateDocument.getImages() != null) {
|
|
for(DigestURL imgURL : donateDocument.getImages().keySet()) {
|
|
try {
|
|
Response response = switchBoard.loader.load(switchBoard.loader.request(imgURL, false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent);
|
|
if (response != null) {
|
|
String imgFileName = imgURL.getFileName();
|
|
/* Store each image in the same directory as the iframe target file */
|
|
FileUtils.copy(response.getContent(), new File(iframefile.getParentFile(), imgFileName));
|
|
|
|
/* Transform the original image URL to a relative one */
|
|
donateDocContent = donateDocContent.replace(imgURL.getURL().toString(), imgFileName);
|
|
}
|
|
} catch(IOException e) {
|
|
/* Failing to load one image should not stop the whole task */
|
|
ConcurrentLog.warn("STARTUP", "Donation frame retrieval : could not get an image resource.", e);
|
|
}
|
|
}
|
|
}
|
|
FileUtils.copy(donateDocContent.getBytes(donateDocument.getCharset()), iframefile);
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
ConcurrentLog.warn("STARTUP", "Could not retrieve donation frame content.", e);
|
|
}
|
|
}
|
|
}.start();
|
|
}
|
|
|
|
/**
|
|
* @param f
|
|
*/
|
|
private static void delete(final File f) {
|
|
if(!f.delete())
|
|
ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be deleted!");
|
|
}
|
|
|
|
/**
|
|
* @see File#mkdir()
|
|
* @param path
|
|
*/
|
|
private static void mkdirIfNeseccary(final File path) {
|
|
if (!(path.exists()))
|
|
if(!path.mkdir())
|
|
ConcurrentLog.warn("STARTUP", "could not create directory "+ path.toString());
|
|
}
|
|
|
|
/**
|
|
* @see File#mkdirs()
|
|
* @param path
|
|
*/
|
|
public static void mkdirsIfNeseccary(final File path) {
|
|
if (!(path.exists()))
|
|
if(!path.mkdirs())
|
|
ConcurrentLog.warn("STARTUP", "could not create directories "+ path.toString());
|
|
}
|
|
|
|
/**
|
|
* Loads the configuration from the data-folder.
|
|
* FIXME: Why is this called over and over again from every method, instead
|
|
* of setting the configurationdata once for this class in main?
|
|
*
|
|
* @param mes Where are we called from, so that the errormessages can be
|
|
* more descriptive.
|
|
* @param homePath Root-path where all the information is to be found.
|
|
* @return Properties read from the configurationfile.
|
|
*/
|
|
private static Properties configuration(final String mes, final File homePath) {
|
|
ConcurrentLog.config(mes, "Application Root Path: " + homePath.toString());
|
|
|
|
// read data folder
|
|
final File dataFolder = new File(homePath, "DATA");
|
|
if (!(dataFolder.exists())) {
|
|
ConcurrentLog.severe(mes, "Application was never started or root path wrong.");
|
|
System.exit(-1);
|
|
}
|
|
|
|
final Properties config = new Properties();
|
|
FileInputStream fis = null;
|
|
try {
|
|
fis = new FileInputStream(new File(homePath, "DATA/SETTINGS/yacy.conf"));
|
|
config.load(fis);
|
|
} catch (final FileNotFoundException e) {
|
|
ConcurrentLog.severe(mes, "could not find configuration file.");
|
|
System.exit(-1);
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.severe(mes, "could not read configuration file.");
|
|
System.exit(-1);
|
|
} finally {
|
|
if(fis != null) {
|
|
try {
|
|
fis.close();
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.logException(e);
|
|
}
|
|
}
|
|
}
|
|
|
|
return config;
|
|
}
|
|
|
|
/**
|
|
* Call the shutdown-page of YaCy to tell it to shut down. This method is
|
|
* called if you start yacy with the argument -shutdown.
|
|
*
|
|
* @param homePath Root-path where all the information is to be found.
|
|
*/
|
|
public static void shutdown(final File homePath) {
|
|
// start up
|
|
System.out.println(copyright);
|
|
System.out.println(hline);
|
|
|
|
final LinkedHashMap<String,ContentBody> post = new LinkedHashMap<String,ContentBody>();
|
|
post.put("shutdown", UTF8.StringBody(""));
|
|
submitPostURL(homePath, "Steering.html", "Terminate YaCy", post);
|
|
}
|
|
|
|
public static void update(final File homePath) {
|
|
// start up
|
|
System.out.println(copyright);
|
|
System.out.println(hline);
|
|
submitURL(homePath, "ConfigUpdate_p.html?autoUpdate=", "Update YaCy to most recent version");
|
|
}
|
|
|
|
/**
|
|
* Submits post data to the local peer URL, authenticating as administrator
|
|
* @param homePath directory containing YaCy DATA folder
|
|
* @param path url relative path part
|
|
* @param processdescription description of the operation for logging purpose
|
|
* @param post data to post
|
|
*/
|
|
private static void submitPostURL(final File homePath, final String path, final String processdescription, final Map<String, ContentBody> post) {
|
|
final Properties config = configuration("COMMAND-STEERING", homePath);
|
|
|
|
// read port
|
|
final int port = Integer.parseInt(config.getProperty(SwitchboardConstants.SERVER_PORT, "8090"));
|
|
|
|
// read password
|
|
final String encodedPassword = config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "");
|
|
final String adminUser = config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin");
|
|
|
|
// send 'wget' to web interface
|
|
final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
|
|
// con.setHeader(requestHeader.entrySet());
|
|
try {
|
|
/* First get a valid transaction token using HTTP GET */
|
|
con.GETbytes("http://localhost:"+ port +"/" + path, adminUser, encodedPassword, false);
|
|
|
|
if (con.getStatusCode() != HttpStatus.SC_OK) {
|
|
throw new IOException("Error response from YACY socket: " + con.getHttpResponse().getStatusLine());
|
|
}
|
|
|
|
final Header transactionTokenHeader = con.getHttpResponse().getFirstHeader(HeaderFramework.X_YACY_TRANSACTION_TOKEN);
|
|
if(transactionTokenHeader == null) {
|
|
throw new IOException("Could not retrieve a valid transaction token");
|
|
}
|
|
|
|
/* Then POST the request */
|
|
post.put(TransactionManager.TRANSACTION_TOKEN_PARAM, UTF8.StringBody(transactionTokenHeader.getValue()));
|
|
con.POSTbytes(new MultiProtocolURL("http://localhost:"+ port +"/" + path), null, post, adminUser, encodedPassword, false, false);
|
|
if (con.getStatusCode() >= HttpStatus.SC_OK && con.getStatusCode() < HttpStatus.SC_MULTIPLE_CHOICES) {
|
|
ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);
|
|
} else {
|
|
ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine());
|
|
|
|
try {
|
|
HTTPClient.closeConnectionManager();
|
|
} catch (final InterruptedException e1) {
|
|
e1.printStackTrace();
|
|
}
|
|
|
|
RemoteInstance.closeConnectionManager();
|
|
|
|
System.exit(-1);
|
|
}
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.severe("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage());
|
|
|
|
try {
|
|
HTTPClient.closeConnectionManager();
|
|
} catch (final InterruptedException e1) {
|
|
e1.printStackTrace();
|
|
}
|
|
RemoteInstance.closeConnectionManager();
|
|
|
|
System.exit(-1);
|
|
}
|
|
|
|
try {
|
|
HTTPClient.closeConnectionManager();
|
|
} catch (final InterruptedException e) {
|
|
e.printStackTrace();
|
|
}
|
|
RemoteInstance.closeConnectionManager();
|
|
|
|
// finished
|
|
ConcurrentLog.config("COMMAND-STEERING", "SUCCESSFULLY FINISHED COMMAND: " + processdescription);
|
|
}
|
|
|
|
private static void submitURL(final File homePath, final String path, final String processdescription) {
|
|
final Properties config = configuration("COMMAND-STEERING", homePath);
|
|
|
|
// read port
|
|
final int port = Integer.parseInt(config.getProperty(SwitchboardConstants.SERVER_PORT, "8090"));
|
|
|
|
// read password
|
|
String encodedPassword = (String) config.get(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5);
|
|
if (encodedPassword == null) encodedPassword = ""; // not defined
|
|
|
|
// send 'wget' to web interface
|
|
final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
|
|
// con.setHeader(requestHeader.entrySet());
|
|
try {
|
|
con.GETbytes("http://localhost:"+ port +"/" + path, config.getProperty(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME,"admin"), encodedPassword, false);
|
|
if (con.getStatusCode() > 199 && con.getStatusCode() < 300) {
|
|
ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);
|
|
|
|
} else {
|
|
ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine());
|
|
System.exit(-1);
|
|
}
|
|
} catch (final IOException e) {
|
|
ConcurrentLog.severe("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage());
|
|
System.exit(-1);
|
|
}
|
|
|
|
try {
|
|
HTTPClient.closeConnectionManager();
|
|
} catch (final InterruptedException e) {
|
|
e.printStackTrace();
|
|
}
|
|
RemoteInstance.closeConnectionManager();
|
|
|
|
// finished
|
|
ConcurrentLog.config("COMMAND-STEERING", "SUCCESSFULLY FINISHED COMMAND: " + processdescription);
|
|
}
|
|
|
|
/**
|
|
* read saved config file and perform action which need to be done before main task starts
|
|
* - like check on system alrady running etc.
|
|
*
|
|
* @param dataHome data directory
|
|
*/
|
|
static private void preReadSavedConfigandInit(File dataHome) {
|
|
File lockFile = new File(dataHome, "DATA/yacy.running");
|
|
final String conf = "DATA/SETTINGS/yacy.conf";
|
|
|
|
// If YaCy is actually running, then we check if the server port is open.
|
|
// If yes, then we consider that a restart is a user mistake and then we just respond
|
|
// as the user expects and tell the browser to open the start page.
|
|
// That will especially happen if Windows Users double-Click the YaCy Icon on the desktop to simply
|
|
// open the web interface. (They don't think of 'servers' they just want to get to the search page).
|
|
// We need to parse the configuration file for that to get the host port
|
|
File configFile = new File(dataHome, conf);
|
|
|
|
if (configFile.exists()) {
|
|
Properties p = new Properties();
|
|
FileInputStream fis = null;
|
|
try {
|
|
fis = new FileInputStream(configFile);
|
|
p.load(fis);
|
|
|
|
// test for yacy already running
|
|
if (lockFile.exists()) { // another instance running? VM crash? User will have to care about this
|
|
//standard log system not up yet - use simply stdout
|
|
// prevents also creation of a log file while just opening browser
|
|
System.out.println("WARNING: the file " + lockFile + " exists, this usually means that a YaCy instance is still running. If you want to restart YaCy, try first ./stopYACY.sh, then ./startYACY.sh. If ./stopYACY.sh fails, try ./killYACY.sh");
|
|
|
|
int port = Integer.parseInt(p.getProperty(SwitchboardConstants.SERVER_PORT, "8090"));
|
|
if (TimeoutRequest.ping("127.0.0.1", port, 1000)) {
|
|
Browser.openBrowser("http://localhost:" + port + "/" + p.getProperty(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html"));
|
|
// Thats it; YaCy was running, the user is happy, we can stop now.
|
|
System.out.println("WARNING: YaCy instance was still running; just opening the browser and exit.");
|
|
System.exit(0);
|
|
} else {
|
|
// YaCy is not running; thus delete the file an go on as nothing was wrong.
|
|
System.err.println("INFO: delete old yacy.running file; likely previous YaCy session was not orderly shutdown!");
|
|
delete(lockFile);
|
|
}
|
|
}
|
|
} catch (IOException ex) {
|
|
System.err.println("ERROR: config file seems to be corrupt");
|
|
System.err.println("ERROR: if problem persists, delete file");
|
|
System.err.println(configFile.getAbsolutePath());
|
|
ConcurrentLog.logException(ex);
|
|
ConcurrentLog.severe("Startup", "cannot read " + configFile.toString() + ", please delete the corrupted file if problem persits");
|
|
} finally {
|
|
try {
|
|
fis.close();
|
|
} catch (IOException e) {
|
|
ConcurrentLog.warn("Startup", "Could not close file " + configFile);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Main-method which is started by java. Checks for special arguments or
|
|
* starts up the application.
|
|
*
|
|
* @param args
|
|
* Given arguments from the command line.
|
|
*/
|
|
public static void main(String args[]) {
|
|
|
|
try {
|
|
// check assertion status
|
|
//ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true);
|
|
boolean assertionenabled = false;
|
|
assert (assertionenabled = true) == true; // compare to true to remove warning: "Possible accidental assignement"
|
|
if (assertionenabled) System.out.println("Asserts are enabled");
|
|
|
|
// check memory amount
|
|
System.gc();
|
|
final long startupMemFree = MemoryControl.free();
|
|
final long startupMemTotal = MemoryControl.total();
|
|
|
|
// maybe go into headless awt mode: we have three cases depending on OS and one exception:
|
|
// windows : better do not go into headless mode
|
|
// mac : go into headless mode because an application is shown in gui which may not be wanted
|
|
// linux : go into headless mode because this does not need any head operation
|
|
// exception : if the -gui option is used then do not go into headless mode since that uses a gui
|
|
boolean headless = true;
|
|
if (OS.isWindows) headless = false;
|
|
if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) headless = false;
|
|
System.setProperty("java.awt.headless", headless ? "true" : "false");
|
|
|
|
String s = ""; for (final String a: args) s += a + " ";
|
|
yacyRelease.startParameter = s.trim();
|
|
|
|
File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/'));
|
|
File dataRoot = applicationRoot;
|
|
//System.out.println("args.length=" + args.length);
|
|
//System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]");
|
|
if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-startup") || args[0].equals("-start"))) {
|
|
// normal start-up of yacy
|
|
if (args.length > 1) {
|
|
dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
|
|
}
|
|
preReadSavedConfigandInit(dataRoot);
|
|
startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
|
|
} else if (args.length >= 1 && args[0].toLowerCase(Locale.ROOT).equals("-gui")) {
|
|
// start-up of yacy with gui
|
|
if (args.length > 1) {
|
|
dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
|
|
}
|
|
preReadSavedConfigandInit(dataRoot);
|
|
startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, true);
|
|
} else if ((args.length >= 1) && ((args[0].toLowerCase(Locale.ROOT).equals("-shutdown")) || (args[0].equals("-stop")))) {
|
|
// normal shutdown of yacy
|
|
if (args.length == 2) applicationRoot= new File(args[1]);
|
|
shutdown(applicationRoot);
|
|
} else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-update"))) {
|
|
// aut-update yacy
|
|
if (args.length == 2) applicationRoot= new File(args[1]);
|
|
update(applicationRoot);
|
|
} else if ((args.length >= 1) && (args[0].toLowerCase(Locale.ROOT).equals("-version"))) {
|
|
// show yacy version
|
|
System.out.println(copyright);
|
|
} else if ((args.length > 1) && (args[0].toLowerCase(Locale.ROOT).equals("-config"))) {
|
|
// set config parameter. Special handling of adminAccount=user:pwd (generates md5 encoded password)
|
|
// on Windows parameter should be enclosed in doublequotes to accept = sign (e.g. -config "port=8090" "port.ssl=8043")
|
|
File f = new File (dataRoot,"DATA/SETTINGS/");
|
|
if (!f.exists()) {
|
|
mkdirsIfNeseccary(f);
|
|
} else {
|
|
if (new File(dataRoot, "DATA/yacy.running").exists()) {
|
|
System.out.println("please restart YaCy");
|
|
}
|
|
}
|
|
// use serverSwitch to read config properties (including init values from yacy.init
|
|
serverSwitch ss = new serverSwitch(dataRoot,applicationRoot,"defaults/yacy.init","DATA/SETTINGS/yacy.conf");
|
|
|
|
for (int icnt=1; icnt < args.length ; icnt++) {
|
|
String cfg = args[icnt];
|
|
int pos = cfg.indexOf('=');
|
|
if (pos > 0) {
|
|
String cmd = cfg.substring(0, pos);
|
|
String val = cfg.substring(pos + 1);
|
|
|
|
if (!val.isEmpty()) {
|
|
if (cmd.equalsIgnoreCase(SwitchboardConstants.ADMIN_ACCOUNT)) { // special command to set adminusername and md5-pwd
|
|
int cpos = val.indexOf(':'); //format adminAccount=adminname:adminpwd
|
|
if (cpos >= 0) {
|
|
String username = val.substring(0, cpos);
|
|
String pwdtxt = val.substring(cpos + 1);
|
|
if (!username.isEmpty()) {
|
|
ss.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, username);
|
|
System.out.println("Set property " + SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME + " = " + username);
|
|
} else {
|
|
username = ss.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_USER_NAME, "admin");
|
|
}
|
|
ss.setConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, "MD5:" + Digest.encodeMD5Hex(username + ":" + ss.getConfig(SwitchboardConstants.ADMIN_REALM, "YaCy") + ":" + pwdtxt));
|
|
System.out.println("Set property " + SwitchboardConstants.ADMIN_ACCOUNT_B64MD5 + " = " + ss.getConfig(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5, ""));
|
|
}
|
|
} else {
|
|
ss.setConfig(cmd, val);
|
|
System.out.println("Set property " + cmd + " = " + val);
|
|
}
|
|
}
|
|
} else {
|
|
System.out.println("skip parameter " + cfg + " (equal sign missing, put parameter in doublequotes)");
|
|
}
|
|
System.out.println();
|
|
}
|
|
} else {
|
|
if (args.length == 1) {
|
|
applicationRoot= new File(args[0]);
|
|
}
|
|
preReadSavedConfigandInit(dataRoot);
|
|
startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
|
|
}
|
|
} finally {
|
|
ConcurrentLog.shutdown();
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* This class is a helper class whose instance is started, when the java virtual
|
|
* machine shuts down. Signals the plasmaSwitchboard to shut down.
|
|
*/
|
|
class shutdownHookThread extends Thread {
|
|
private final Switchboard sb;
|
|
private final Semaphore shutdownSemaphore;
|
|
|
|
public shutdownHookThread(final Switchboard sb, final Semaphore shutdownSemaphore) {
|
|
super("yacy.shutdownHookThread");
|
|
this.sb = sb;
|
|
this.shutdownSemaphore = shutdownSemaphore;
|
|
}
|
|
|
|
@Override
|
|
public void run() {
|
|
try {
|
|
if (!this.sb.isTerminated()) {
|
|
ConcurrentLog.config("SHUTDOWN","Shutdown via shutdown hook.");
|
|
/* Print also to the standard output, as the LogManager is likely to have
|
|
* been concurrently reset by its own shutdown hook thread */
|
|
System.out.println("SHUTDOWN Starting shutdown via shutdown hook.");
|
|
|
|
// sending the yacy main thread a shutdown signal
|
|
ConcurrentLog.fine("SHUTDOWN","Signaling shutdown to the switchboard.");
|
|
this.sb.terminate("shutdown hook");
|
|
|
|
// waiting for the yacy thread to finish execution
|
|
ConcurrentLog.fine("SHUTDOWN","Waiting for main thread to finish.");
|
|
|
|
/* Main thread will release the shutdownSemaphore once completely terminated.
|
|
* We do not wait indefinitely as the application is supposed here to quickly terminate */
|
|
this.shutdownSemaphore.tryAcquire(30, TimeUnit.SECONDS);
|
|
}
|
|
} catch (final Exception e) {
|
|
ConcurrentLog.severe("SHUTDOWN","Unexpected error. " + e.getClass().getName(),e);
|
|
}
|
|
}
|
|
}
|