diff --git a/addon/yacyInit.sh b/addon/yacyInit.sh index 31b2477f0..2fb33aca5 100755 --- a/addon/yacyInit.sh +++ b/addon/yacyInit.sh @@ -26,16 +26,16 @@ NICE_VAL=0 JAVA_ARGS="-Djava.awt.headless=true" #get javastart args -if [ -f DATA/SETTINGS/httpProxy.conf ] +if [ -f DATA/SETTINGS/yacy.conf ] then # startup memory for i in Xmx Xms; do - j="`grep javastart_$i DATA/SETTINGS/httpProxy.conf | sed 's/^[^=]*=//'`"; + j="`grep javastart_$i DATA/SETTINGS/yacy.conf | sed 's/^[^=]*=//'`"; if [ -n $j ]; then JAVA_ARGS="-$j $JAVA_ARGS"; fi; done # Priority - j="`grep javastart_priority DATA/SETTINGS/httpProxy.conf | sed 's/^[^=]*=//'`"; + j="`grep javastart_priority DATA/SETTINGS/yacy.conf | sed 's/^[^=]*=//'`"; if [ ! -z "$j" ];then if [ -n $j ]; then NICE_VAL=$j; fi; diff --git a/build.xml b/build.xml index c01249837..bf8115f54 100644 --- a/build.xml +++ b/build.xml @@ -390,19 +390,22 @@ - - - - + + + + + + diff --git a/yacy.init b/defaults/yacy.init similarity index 97% rename from yacy.init rename to defaults/yacy.init index 6b6a146aa..659bfcaa4 100644 --- a/yacy.init +++ b/defaults/yacy.init @@ -4,7 +4,7 @@ # These properties will be loaded upon installation. # They are used only once for set-up. # If you make changes to this file and want these to make any effect, -# you must delete the httpProxy.conf file in DATA/SETTINGS +# you must delete the yacy.conf file in DATA/SETTINGS # ---------------------------------------------------------------------------- # the HTTP service configurations @@ -70,8 +70,8 @@ server.maxTrackingTime = 3600000 # The property here can also be a url where the definition can be loaded. # In case of privately managed networks, this configuration must be changed BEFORE it is released # to the members of the separated network peers. -network.unit.definition = yacy.network.unit -network.group.definition = yacy.network.group +network.unit.definition = defaults/yacy.network.freeworld.unit +network.group.definition = defaults/yacy.network.group # Update process properties # The update server location is given in the network.unit.definition, diff --git a/defaults/yacy.network.freeworld.unit b/defaults/yacy.network.freeworld.unit new file mode 100644 index 000000000..0c85027b5 --- /dev/null +++ b/defaults/yacy.network.freeworld.unit @@ -0,0 +1,31 @@ +# Network definition file for the freeworld network +# this is the default standard network for fresh YaCy Peers + +# -----------------------------------------------------------------# +# for an explanation please see the file yacy.network.readme # +# this is a work in progress. disabled properties are not yet used # +# -----------------------------------------------------------------# + +# general network definition +network.unit.name = freeworld +network.unit.description = Public YaCy Community +network.unit.domain = global +network.unit.search.time = 4 +network.unit.dht = true +network.unit.dhtredundancy.junior = 1 +network.unit.dhtredundancy.senior = 3 +network.unit.bootstrap.seedlist0 = http://www.yacy.net/seed.txt +network.unit.bootstrap.seedlist1 = http://home.arcor.de/hermens/yacy/seed.txt +network.unit.bootstrap.seedlist2 = http://low.audioattack.de/yacy/seed.txt +network.unit.bootstrap.seedlist3 = http://www.lulabad.de/seed.txt +network.unit.bootstrap.seedlist4 = http://www.marcelhenseler.de/yacy/seed.txt + +# each network may use different yacy distributions. +# the auto-updater can access network-specific update locations +network.unit.update.location0 = http://yacy.net/index.html +network.unit.update.location1 = http://latest.yacy.de +network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/index.php +network.unit.update.location3 = http://www.yacystats.de/yacybuild/ + +# properties for in-protocol response authentification: +network.unit.protocol.control = uncontrolled \ No newline at end of file diff --git a/yacy.network.group b/defaults/yacy.network.group similarity index 98% rename from yacy.network.group rename to defaults/yacy.network.group index 2b1d57746..f5a6698fe 100644 --- a/yacy.network.group +++ b/defaults/yacy.network.group @@ -1,7 +1,7 @@ # YaCy Network Group Definition # ----------------------------- # This is an addition to the yacy.network.unit configuration file. -# This file is addressed by the network.group.definition property in yacy.init +# This file is addressed by the network.group.definition property in defaults/yacy.init # The purpose of a group within a network is that some parts of a network may be managed independently, # while the content of the network stays private for the whole network, mostly for a special purpose. # This file needs to be configured if someone wants to participate with several peers to the network, diff --git a/defaults/yacy.network.intranet.unit b/defaults/yacy.network.intranet.unit new file mode 100644 index 000000000..5ef5d5220 --- /dev/null +++ b/defaults/yacy.network.intranet.unit @@ -0,0 +1,25 @@ +# Network definition file for the freeworld network +# this is the default standard network for fresh YaCy Peers + +# -----------------------------------------------------------------# +# for an explanation please see the file yacy.network.readme # +# this is a work in progress. disabled properties are not yet used # +# -----------------------------------------------------------------# + +# general network definition +network.unit.name = intranet +network.unit.description = Private Intranet +network.unit.domain = local +network.unit.dht = false +network.unit.dhtredundancy.junior = 1 +network.unit.dhtredundancy.senior = 1 + +# each network may use different yacy distributions. +# the auto-updater can access network-specific update locations +network.unit.update.location0 = http://yacy.net/index.html +network.unit.update.location1 = http://latest.yacy.de +network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/index.php +network.unit.update.location3 = http://www.yacystats.de/yacybuild/ + +# properties for in-protocol response authentification: +network.unit.protocol.control = uncontrolled \ No newline at end of file diff --git a/yacy.network.unit b/defaults/yacy.network.readme similarity index 62% rename from yacy.network.unit rename to defaults/yacy.network.readme index b785355c7..eda000c3d 100644 --- a/yacy.network.unit +++ b/defaults/yacy.network.readme @@ -6,7 +6,7 @@ # To distinguish between index areas where the shared web index is for a intranet or the public internet, # we distinguish local and global networks. Each network type can have different user groups. # Groups can be uncontrolled, moderated or controlled. The whole network may also have a global master. -# This file is adressed by the network.unit.definition property in yacy.init. If necessary this +# This file is adressed by the network.unit.definition property in defaults/yacy.init. If necessary this # property can also address a URL where a network definition file can be hosted, so that a network # can easily be moderated from a single central. In case of the public network of all independent peers # for global indexing, the network definition is attached to the release and defines no network master or @@ -42,42 +42,3 @@ # network.unit.administration.manager = # network.unit.administration.request.authentification.method = 'salted-magic' # network.unit.administration.request.authentification.essentials = - - -# -----------------------------------------------------------------# -# this is a work in progress. disabled properties are not yet used # -# -----------------------------------------------------------------# - -# general network definition -network.unit.name = freeworld -network.unit.description = Public YaCy Community -network.unit.domain = global -network.unit.search.time = 4 -network.unit.dht = true -network.unit.dhtredundancy.junior = 1 -network.unit.dhtredundancy.senior = 3 -network.unit.bootstrap.seedlist0 = http://www.yacy.net/seed.txt -network.unit.bootstrap.seedlist1 = http://home.arcor.de/hermens/yacy/seed.txt -network.unit.bootstrap.seedlist2 = http://low.audioattack.de/yacy/seed.txt -network.unit.bootstrap.seedlist3 = http://www.lulabad.de/seed.txt -network.unit.bootstrap.seedlist4 = http://www.marcelhenseler.de/yacy/seed.txt - -# each network may use different yacy distributions. -# the auto-updater can access network-specific update locations -network.unit.update.location0 = http://yacy.net/index.html -network.unit.update.location1 = http://latest.yacy.de -network.unit.update.location2 = http://www.findenstattsuchen.info/YaCy/latest/index.php -network.unit.update.location3 = http://www.yacystats.de/yacybuild/ - -# properties for in-protocol response authentification: -network.unit.protocol.control = uncontrolled -#network.unit.protocol.request.authentification.method = salted-magic-sim -#network.unit.protocol.request.authentification.essentials = -#network.unit.protocol.response.authentification.method = challenge-pwresponse -#network.unit.protocol.response.authentification.essentials = - -# properties for unit access administrator-authentification: -#network.unit.administration.control = uncontrolled -#network.unit.administration.manager = -#network.unit.administration.request.authentification.method = salted-magic -#network.unit.administration.request.authentification.essentials = diff --git a/htroot/ConfigProperties_p.html b/htroot/ConfigProperties_p.html index 8764eafbe..94cee5c81 100644 --- a/htroot/ConfigProperties_p.html +++ b/htroot/ConfigProperties_p.html @@ -44,7 +44,7 @@ You can change anything, but some options need a restart, and some options can crash YaCy, if wrong values are used.

- For explanation please look into yacy.init + For explanation please look into defaults/yacy.init

#{options}# diff --git a/htroot/IndexImport_p.java b/htroot/IndexImport_p.java index fa4e693ff..490f70156 100644 --- a/htroot/IndexImport_p.java +++ b/htroot/IndexImport_p.java @@ -50,7 +50,6 @@ import java.io.PrintStream; import java.util.Date; -import java.util.HashMap; import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaSwitchboard; @@ -72,19 +71,10 @@ public final class IndexImport_p { if (post != null) { if (post.containsKey("startIndexDbImport")) { try { - // getting the import path - String importPlasmaPath = (String) post.get("importPlasmaPath"); - //String importIndexPrimaryPath = (String) post.get("importIndexPrimaryPath"); - //String importIndexSecondaryPath = (String) post.get("importIndexSecondaryPath"); String importType = (String) post.get("importType"); - String cacheSizeStr = (String) post.get("cacheSize"); + int cacheSize = post.getInt("cacheSize", 0); boolean startImport = true; - HashMap initParams = new HashMap(); - initParams.put("plasmaPath",importPlasmaPath); - initParams.put("cacheSize",cacheSizeStr); - initParams.put("preloadTime","100"); - // // check if there is an already running thread with the same import path // Thread[] importThreads = new Thread[plasmaDbImporter.runningJobs.activeCount()*2]; // activeCount = plasmaDbImporter.runningJobs.enumerate(importThreads); @@ -101,7 +91,7 @@ public final class IndexImport_p { if (startImport) { dbImporter importerThread = switchboard.dbImportManager.getNewImporter(importType); if (importerThread != null) { - importerThread.init(initParams); + importerThread.init(switchboard, cacheSize); importerThread.startIt(); } prop.put("LOCATION",""); diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index 6588b8b64..4e8e83a59 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -70,7 +70,7 @@ public class PerformanceMemory_p { // return variable that accumulates replacements serverObjects prop = new serverObjects(); if (defaultSettings == null) { - defaultSettings = serverFileUtils.loadHashMap(new File(env.getRootPath(), "yacy.init")); + defaultSettings = serverFileUtils.loadHashMap(new File(env.getRootPath(), "defaults/yacy.init")); } prop.put("gc", "0"); if (post != null) { diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index ff843c691..ed3101c11 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -62,7 +62,7 @@ public class PerformanceQueues_p { // return variable that accumulates replacements plasmaSwitchboard switchboard = (plasmaSwitchboard) sb; serverObjects prop = new serverObjects(); - File defaultSettingsFile = new File(switchboard.getRootPath(), "yacy.init"); + File defaultSettingsFile = new File(switchboard.getRootPath(), "defaults/yacy.init"); Map defaultSettings = ((post == null) || (!(post.containsKey("submitdefault")))) ? null : serverFileUtils.loadHashMap(defaultSettingsFile); Iterator threads = switchboard.threadNames(); String threadName; diff --git a/htroot/Settings_p.html b/htroot/Settings_p.html index f06dee48a..447044d6c 100644 --- a/htroot/Settings_p.html +++ b/htroot/Settings_p.html @@ -26,7 +26,7 @@

Advanced Settings

If you want to restore all settings to the default values, but forgot your administration password, you must stop the proxy, - delete the file 'DATA/SETTINGS/httpProxy.conf' in the YaCy application root folder and start YaCy again. + delete the file 'DATA/SETTINGS/yacy.conf' in the YaCy application root folder and start YaCy again.

  • Server Access Settings
    • diff --git a/htroot/Threaddump_p.html b/htroot/Threaddump_p.html index e78f96c9b..4689c70d3 100644 --- a/htroot/Threaddump_p.html +++ b/htroot/Threaddump_p.html @@ -1,7 +1,7 @@ - YaCy '#[clientname]#': Threaddump + YaCy '#[clientname]#': Thread Dump #%env/templates/metas.template%# @@ -15,7 +15,7 @@
    - #(dump)#::#[content]##(/dump)# + #(dump)#::#[content]##(/dump)#
    #%env/templates/footer.template%# diff --git a/htroot/Threaddump_p.java b/htroot/Threaddump_p.java index 597a095e4..38cdb7a45 100644 --- a/htroot/Threaddump_p.java +++ b/htroot/Threaddump_p.java @@ -65,11 +65,9 @@ public class Threaddump_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) { - prop = new serverObjects(); sb = (plasmaSwitchboard) env; - StringBuffer buffer = new StringBuffer(); - + StringBuffer buffer = new StringBuffer(1000); if (post != null && post.containsKey("createThreaddump")) { prop.put("dump", "1"); @@ -81,17 +79,17 @@ public class Threaddump_p { buffer.append("************* Start Thread Dump " + dt + " *******************").append("
    "); buffer.append("
    YaCy Version: " + versionstring + "
    "); buffer.append("Total Memory = " + (Runtime.getRuntime().totalMemory())).append("
    "); - buffer.append("Used Memory = " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())).append("
    "); - buffer.append("Free Memory = " + (Runtime.getRuntime().freeMemory())).append("
    "); - buffer.append(" --- --- --- ---
    "); - for (Thread thread: stackTraces.keySet()) { - StackTraceElement[] stackTraceElement = (StackTraceElement[]) stackTraces.get(thread); - buffer.append("Thread= " + thread.getName() + " " + (thread.isDaemon()?"daemon":"") + " id=" + thread.getId() + " " + thread.getState()).append("
    "); - for(int i = 0; i <= (stackTraceElement.length -1); i++) { - buffer.append(stackTraceElement[i]).append("
    "); - } - buffer.append("
    "); - } + buffer.append("Used  Memory = " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())).append("
    "); + buffer.append("Free  Memory = " + (Runtime.getRuntime().freeMemory())).append("
    "); + buffer.append(" --- --- --- ---

    "); + + appendStackTraces(buffer, stackTraces, Thread.State.BLOCKED); + appendStackTraces(buffer, stackTraces, Thread.State.RUNNABLE); + appendStackTraces(buffer, stackTraces, Thread.State.TIMED_WAITING); + appendStackTraces(buffer, stackTraces, Thread.State.WAITING); + appendStackTraces(buffer, stackTraces, Thread.State.NEW); + appendStackTraces(buffer, stackTraces, Thread.State.TERMINATED); + buffer.append("************* End Thread Dump " + dt + " *******************").append("
    "); prop.put("dump_content", buffer.toString()); @@ -102,5 +100,19 @@ public class Threaddump_p { return prop; // return from serverObjects respond() } - + private static void appendStackTraces(StringBuffer buffer, Map stackTraces, Thread.State stateIn) { + buffer.append("THREADS WITH STATES: " + stateIn.toString()).append("
    ").append("
    "); + + for (Thread thread: stackTraces.keySet()) { + StackTraceElement[] stackTraceElement = stackTraces.get(thread); + if (stateIn.equals(thread.getState())) { + buffer.append("Thread= " + thread.getName() + " " + (thread.isDaemon()?"daemon":"") + " id=" + thread.getId() + " " + thread.getState().toString()).append("
    "); + for (int i = 0; i < stackTraceElement.length; i++) { + buffer.append("at " + stackTraceElement[i]).append("
    "); + } + buffer.append("
    "); + } + } + buffer.append("
    "); + } } diff --git a/htroot/WatchCrawler_p.java b/htroot/WatchCrawler_p.java index 597d91b2c..adbf2fa22 100644 --- a/htroot/WatchCrawler_p.java +++ b/htroot/WatchCrawler_p.java @@ -41,7 +41,7 @@ import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaCrawlProfile; import de.anomic.plasma.plasmaCrawlZURL; import de.anomic.plasma.plasmaSwitchboard; -import de.anomic.plasma.dbImport.dbImporter; +import de.anomic.plasma.dbImport.SitemapImporter; import de.anomic.server.serverFileUtils; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -325,13 +325,10 @@ public class WatchCrawler_p { storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw); // create a new sitemap importer - dbImporter importerThread = switchboard.dbImportManager.getNewImporter("sitemap"); + SitemapImporter importerThread = (SitemapImporter) switchboard.dbImportManager.getNewImporter("sitemap"); if (importerThread != null) { - HashMap initParams = new HashMap(); - initParams.put("sitemapURL", sitemapURLStr); - initParams.put("crawlingProfile", pe.handle()); - - importerThread.init(initParams); + importerThread.init(switchboard, 0); + importerThread.initSitemap(new yacyURL(sitemapURLStr, null), pe); importerThread.startIt(); } } catch (Exception e) { diff --git a/htroot/env/templates/header.template b/htroot/env/templates/header.template index 052cbbd12..0f72ba887 100644 --- a/htroot/env/templates/header.template +++ b/htroot/env/templates/header.template @@ -6,7 +6,8 @@

    Search & Browse

    • (old) Search Page
      • -
    • (new) Search Page
      • +
    • (alternative a) Search Page
      • +
    • (alternative b) Search Page
    • Ranking Config
    • Surftips
    • Publication
      • diff --git a/htroot/env/templates/submenuViewLog.template b/htroot/env/templates/submenuViewLog.template index 073ba0749..c68063d2c 100644 --- a/htroot/env/templates/submenuViewLog.template +++ b/htroot/env/templates/submenuViewLog.template @@ -3,5 +3,6 @@ \ No newline at end of file diff --git a/locales/de.lng b/locales/de.lng index 605ed4320..300ce4c57 100644 --- a/locales/de.lng +++ b/locales/de.lng @@ -229,7 +229,7 @@ The maximum cache size is==Die maximale Größe beträgt Advanced Config==Erweiterte Einstellungen Here are all configuration options from YaCy.==Hier sind alle Konfigurationseinstellungen von YaCy. You can change anything, but some options need a restart, and some options can crash YaCy, if wrong values are used.==Sie können alles ändern, bedenken Sie aber, dass einige Änderungen einen Neustart von YaCy erfordern und andere Änderungen YaCy ganz zum Absturz bringen können. -For explanation please look into yacy.init==Für eine Erklärung sehen Sie bitte in der Datei yacy.init nach +For explanation please look into defaults/yacy.init==Für eine Erklärung sehen Sie bitte in der Datei defaults/yacy.init nach "Save"=="Speichern" #----------------------------- @@ -371,7 +371,7 @@ here.==im Profil benutzen. Advanced Config==Erweiterte Einstellungen Here are all configuration options from YaCy.==Hier sind alle Konfigurationseinstellungen von YaCy. You can change anything, but some options need a restart, and some options can crash YaCy, if wrong values are used.==Sie können alles ändern, bedenken Sie aber, dass einige Änderungen einen Neustart von YaCy erfordern und andere Änderungen YaCy ganz zum Absturz bringen können. -For explanation please look into yacy.init==Eine Erklärung finden Sie in der Datei yacy.init +For explanation please look into defaults/yacy.init==Eine Erklärung finden Sie in der Datei defaults/yacy.init "Save"=="Speichern" #----------------------------- @@ -1599,7 +1599,7 @@ Post-Ranking==Nach-Ranking Advanced Settings==Erweiterte Einstellungen If you want to restore all settings to the default values,==Wenn Sie alle Einstellungen auf die ursprünglichen Werte zurücksetzen wollen, but forgot your administration password, you must stop the proxy,==aber Ihr Administatorpasswort vergessen haben, müssen Sie YaCy stoppen, -delete the file 'DATA/SETTINGS/httpProxy.conf' in the YaCy application root folder and start YaCy again.==die Datei 'DATA/SETTINGS/httpProxy.conf' im YaCy Hauptordner löschen und YaCy neu starten. +delete the file 'DATA/SETTINGS/yacy.conf' in the YaCy application root folder and start YaCy again.==die Datei 'DATA/SETTINGS/yacy.conf' im YaCy Hauptordner löschen und YaCy neu starten. Server Access Settings==Server Zugangs-Einstellungen Proxy Access Settings==Proxy Zugangs-Einstellungen Content Parser Settings==Inhalt Parser Einstellungen diff --git a/locales/fr.lng b/locales/fr.lng index 71184617c..80a167a12 100644 --- a/locales/fr.lng +++ b/locales/fr.lng @@ -132,7 +132,7 @@ The maximum cache size is==La taille maximale du cache est Advanced Config==Configuration avancée Here are all configuration options from YaCy.==toutes les options de configuration de YaCy sont ici. You can change anything, but some options need a restart, and some options can crash YaCy, if wrong values are used.==Vous pouvez tout changer, mais certaines options nécessitent un redémarrage et certaines autres peuvent provoquer l'arrêt de YaCy si elles sont mal choisies. -For explanation please look into yacy.init==Pour des explications, regardez dans le fichier « yacy.init » s'il vous plait. +For explanation please look into defaults/yacy.init==Pour des explications, regardez dans le fichier « defaults/yacy.init » s'il vous plait. "Save"=="Sauver" #----------------------------- @@ -141,7 +141,7 @@ For explanation please look into yacy.init==Pour des explications, regardez dans Advanced Config==Configuration avancée Here are all configuration options from YaCy.==toutes les options de configuration de YaCy sont ici. You can change anything, but some options need a restart, and some options can crash YaCy, if wrong values are used.==Vous pouvez tout changer, mais certaines options nécessitent un redémarrage et certaines autres peuvent provoquer l'arrêt de YaCy si elles sont mal choisies. -For explanation please look into yacy.init==Pour des explications, regardez dans le fichier « yacy.init » s'il vous plaît. +For explanation please look into defaults/yacy.init==Pour des explications, regardez dans le fichier « defaults/yacy.init » s'il vous plaît. "Save"=="Sauver" #----------------------------- @@ -1193,7 +1193,7 @@ YaCy \'\#\[clientname\]\#\': Settings==YaCy '#[clientname]#': Paramètres

      Settings

      ==

      Paramètres

      If you want to restore all settings to the default values,==Si vous souhaitez rétablir les valeurs par défaut des paramètres, but forgot your administration password, you must stop the proxy,==Mais que vous avez oublié votre mot de passe d'administration , vous devez arrêter le proxy, -delete the file 'DATA/SETTINGS/httpProxy.conf' in the YaCy application root folder and start YaCy again.==Supprimer le fichier 'DATA/SETTINGS/httpProxy.conf'dans le dossier racine de yacy et démarrer YaCy à nouveau. +delete the file 'DATA/SETTINGS/yacy.conf' in the YaCy application root folder and start YaCy again.==Supprimer le fichier 'DATA/SETTINGS/yacy.conf'dans le dossier racine de yacy et démarrer YaCy à nouveau. Administration Account Settings==Paramètres du compte administrateur Server Access Settings==Paramètres d'accès au serveur Proxy Access Settings==Paramètres Proxy diff --git a/locales/it.lng b/locales/it.lng index baa1c7fee..bfb46b3e9 100644 --- a/locales/it.lng +++ b/locales/it.lng @@ -95,7 +95,7 @@ The maximum cache size is==La dimensione massima della cache è Advanced Config==Configurazione avanzata Here are all configuration options from YaCy.==Qui ci sono tutte le opzioni di configurazione di YaCy You can change anything, but some options need a restart, and some options can crash YaCy, if wrong values are used.==Puoi cambiare tutto, ma alcune opzioni richiedono il restart, e altre opzioni possono far crashare YaCy, se sono utilizzati valori sbagliati. -For explanation please look into yacy.init==Per spiegazioni per favore guarda in yacy.init +For explanation please look into defaults/yacy.init==Per spiegazioni per favore guarda in defaults/yacy.init "Save"=="Save" #------------------------------------------------------- diff --git a/locales/sk.lng b/locales/sk.lng index d94e174e9..201b96b55 100644 --- a/locales/sk.lng +++ b/locales/sk.lng @@ -156,7 +156,7 @@ Advanced Config==Pokrocile nastavenia Here are all configuration options from YaCy.==Tu sa nachadzaju vsetky konfiguracne nastavenia YaCy. You can change anything, but some options need a restart, and some options can crash YaCy, if wrong values are used.==Vsetky konfiguracne nastavenia mozu byt zmenene, avsak niektore volby vyzaduju restart a niektore mozu sposobit pad YaCy v pripade zadnia nespravnych hodnot. # NOT USED -#For an explanation of the available options please look into yacy.init==Vysvetlenie jednotlivych volieb najdete v subore yacy.init +#For an explanation of the available options please look into defaults/yacy.init==Vysvetlenie jednotlivych volieb najdete v subore defaults/yacy.init "Save"==Uloz @@ -166,7 +166,7 @@ You can change anything, but some options need a restart, and some options can c Advanced Config==Pokrocile nastavenia Here are all configuration options from YaCy.==Tu sa nachadzaju vsetky konfiguracne nastavenia YaCy. You can change anything, but some options need a restart, and some options can crash YaCy, if wrong values are used.==Vsetky konfiguracne nastavenia mozu byt zmenene, avsak niektore volby vyzaduju restart a niektore mozu sposobit pad YaCy v pripade zadnia nespravnych hodnot. -For explanation please look into yacy.init==Vysvetlenie najdete v subore yacy.init +For explanation please look into defaults/yacy.init==Vysvetlenie najdete v subore defaults/yacy.init "Save"==Uloz #----------------------------------------------------------- @@ -1467,7 +1467,7 @@ To restrict the access to this page, please set an administrator account and pas If you want to restore all settings to the default values,==Ak chcete obnovit povodne nastavenia, # NOT USED #but forgot your administration password, you must stop the proxy,==avsak zabudli ste heslo administratora, tak musite YaCy zastavit (prikaz 'stopYaCy.*'), -delete the file 'DATA/SETTINGS/httpProxy.conf' in the YaCy application root folder and start YaCy again.==zmazat subor 'DATA/SETTINGS/httpProxy.conf' v domovskom adresary YaCy a YaCy restartovat (prikaz 'startYaCy.*') +delete the file 'DATA/SETTINGS/yacy.conf' in the YaCy application root folder and start YaCy again.==zmazat subor 'DATA/SETTINGS/yacy.conf' v domovskom adresary YaCy a YaCy restartovat (prikaz 'startYaCy.*') # NOT USED #Administration Account Settings==Nastavenia konta administratora # NOT USED diff --git a/source/de/anomic/plasma/dbImport/AbstractImporter.java b/source/de/anomic/plasma/dbImport/AbstractImporter.java index f4fd13948..f8dd29093 100644 --- a/source/de/anomic/plasma/dbImport/AbstractImporter.java +++ b/source/de/anomic/plasma/dbImport/AbstractImporter.java @@ -14,7 +14,6 @@ public abstract class AbstractImporter extends Thread implements dbImporter{ protected boolean paused = false; protected int cacheSize; - protected long preloadTime; protected long globalStart = System.currentTimeMillis(); protected long globalEnd; @@ -37,7 +36,7 @@ public abstract class AbstractImporter extends Thread implements dbImporter{ /** * @see dbImporter#init(HashMap) */ - public void init(HashMap initparams) throws ImporterException { + public void init() throws ImporterException { // initializing the logger and setting a more verbose thread name this.log = new serverLog("IMPORT_" + this.jobType + "_" + this.jobID); this.setName("IMPORT_" + this.jobType + "_" + this.jobID); diff --git a/source/de/anomic/plasma/dbImport/SitemapImporter.java b/source/de/anomic/plasma/dbImport/SitemapImporter.java index 65eceaeda..1d7d43ade 100644 --- a/source/de/anomic/plasma/dbImport/SitemapImporter.java +++ b/source/de/anomic/plasma/dbImport/SitemapImporter.java @@ -101,26 +101,21 @@ public class SitemapImporter extends AbstractImporter implements dbImporter { * @see dbImporter#init(HashMap) * @see AbstractImporter#init(HashMap) */ - public void init(HashMap initParams) throws ImporterException { - super.init(initParams); - - if (initParams == null || initParams.size() == 0) throw new IllegalArgumentException("Init parameters are missing"); - if (!initParams.containsKey("crawlingProfile")) throw new IllegalArgumentException("Init parameters 'crawlingProfile' is missing"); - if (!initParams.containsKey("sitemapURL")) throw new IllegalArgumentException("Init parameters 'sitemapURL' is missing"); - + public void init(plasmaSwitchboard switchboard, int cacheSize) throws ImporterException { + super.init(); + } + + public void initSitemap(yacyURL sitemapURL, plasmaCrawlProfile.entry profileEntry) throws ImporterException { try { - // getting the sitemap URL - this.sitemapURL = new yacyURL((String)initParams.get("sitemapURL"), null); - - // getting the crawling profile to use - plasmaCrawlProfile.entry profileEntry = this.sb.profilesActiveCrawls.getEntry((String)initParams.get("crawlingProfile")); - - // creating the sitemap parser - this.parser = new SitemapParser(this.sb,this.sitemapURL,profileEntry); + // getting the sitemap URL + this.sitemapURL = sitemapURL; + + // creating the sitemap parser + this.parser = new SitemapParser(this.sb,this.sitemapURL, profileEntry); } catch (Exception e) { - throw new ImporterException("Unable to initialize Importer",e); + throw new ImporterException("Unable to initialize Importer",e); } - } + } public void run() { try { diff --git a/source/de/anomic/plasma/dbImport/dbImporter.java b/source/de/anomic/plasma/dbImport/dbImporter.java index 1b44e0b09..229ab274e 100644 --- a/source/de/anomic/plasma/dbImport/dbImporter.java +++ b/source/de/anomic/plasma/dbImport/dbImporter.java @@ -1,7 +1,6 @@ package de.anomic.plasma.dbImport; -import java.util.HashMap; - +import de.anomic.plasma.plasmaSwitchboard; public interface dbImporter { @@ -23,8 +22,7 @@ public interface dbImporter { public String getJobName(); public String getJobType(); public String getError(); - public String getStatus(); - //public void init(File plasmaPath, File indexPrimaryPath, File indexSecondaryPath, int cacheSize, long preloadTime); - public void init(HashMap initParams) throws ImporterException; + public String getStatus(); + public void init(plasmaSwitchboard switchboard, int cacheSize) throws ImporterException; public void startIt(); } diff --git a/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java b/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java index dc3188ad3..b213fab74 100644 --- a/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java +++ b/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java @@ -46,22 +46,13 @@ public class plasmaCrawlNURLImporter extends AbstractImporter implements dbImpor return theStatus.toString(); } - public void init(HashMap initParams) throws ImporterException { - super.init(initParams); - - if (initParams == null || initParams.size() == 0) throw new IllegalArgumentException("Init parameters are missing"); - if (!initParams.containsKey("plasmaPath")) throw new IllegalArgumentException("Init parameters 'plasmaPath' is missing"); - if (!initParams.containsKey("cacheSize")) throw new IllegalArgumentException("Init parameters 'cacheSize' is missing"); - if (!initParams.containsKey("preloadTime")) throw new IllegalArgumentException("Init parameters 'preloadTime' is missing"); + public void init(plasmaSwitchboard sb, int cacheSize) throws ImporterException { + super.init(); // TODO: we need more errorhandling here - this.plasmaPath = new File((String)initParams.get("plasmaPath")); - - this.cacheSize = Integer.valueOf((String)initParams.get("cacheSize")).intValue(); + this.plasmaPath = sb.plasmaPath; + this.cacheSize = cacheSize; if (this.cacheSize < 2*1024*1024) this.cacheSize = 8*1024*1024; - - this.preloadTime = Long.valueOf((String)initParams.get("preloadTime")).longValue(); - File noticeUrlDbFile = new File(plasmaPath,"urlNotice1.db"); File profileDbFile = new File(plasmaPath, plasmaSwitchboard.DBFILE_ACTIVE_CRAWL_PROFILES); diff --git a/source/de/anomic/plasma/dbImport/plasmaDbImporter.java b/source/de/anomic/plasma/dbImport/plasmaDbImporter.java index 5eeefd70f..613aa4149 100644 --- a/source/de/anomic/plasma/dbImport/plasmaDbImporter.java +++ b/source/de/anomic/plasma/dbImport/plasmaDbImporter.java @@ -69,20 +69,14 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter { * @throws ImporterException * @see dbImporter#init(HashMap) */ - public void init(HashMap initParams) throws ImporterException { - super.init(initParams); - - if (initParams == null || initParams.size() == 0) throw new IllegalArgumentException("Init parameters are missing"); - if (!initParams.containsKey("primaryPath")) throw new IllegalArgumentException("Init parameters 'primaryPath' is missing"); - if (!initParams.containsKey("secondaryPath")) throw new IllegalArgumentException("Init parameters 'secondaryPath' is missing"); - if (!initParams.containsKey("cacheSize")) throw new IllegalArgumentException("Init parameters 'cacheSize' is missing"); - if (!initParams.containsKey("preloadTime")) throw new IllegalArgumentException("Init parameters 'preloadTime' is missing"); + public void init(plasmaSwitchboard db, int cacheSize) throws ImporterException { + super.init(); // TODO: we need more errorhandling here - this.importPrimaryPath = new File((String)initParams.get("primaryPath")); - this.importSecondaryPath = new File((String)initParams.get("secondaryPath")); + this.importPrimaryPath = sb.indexPrimaryPath; + this.importSecondaryPath = sb.indexSecondaryPath; - this.cacheSize = Integer.valueOf((String)initParams.get("cacheSize")).intValue(); + this.cacheSize = cacheSize; if (this.cacheSize < 2*1024*1024) this.cacheSize = 8*1024*1024; // configure import DB @@ -105,7 +99,7 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter { } this.log.logFine("Initializing source word index db."); - this.importWordIndex = new plasmaWordIndex(this.importPrimaryPath, this.importSecondaryPath, this.log); + this.importWordIndex = new plasmaWordIndex(this.importPrimaryPath, this.importSecondaryPath, sb.getConfig("network.unit.name", ""), this.log); this.importStartSize = this.importWordIndex.size(); } diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 6086b9ea2..a3328d101 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -97,10 +97,27 @@ public final class plasmaCrawlLURL { // the class object kelondroIndex urlIndexFile; - public plasmaCrawlLURL(File indexPath) { + public plasmaCrawlLURL(File indexSecondaryRoot, String networkName) { super(); - - urlIndexFile = new kelondroSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", indexURLEntry.rowdef, false); + File indexSecondaryPath = new File(indexSecondaryRoot, networkName); + File indexSecondaryTextLocation = new File(indexSecondaryPath, "TEXT"); + if (!indexSecondaryTextLocation.exists()) { + // patch old index locations; the secondary path is patched in plasmaCrawlLURL + File oldSecondaryPath = new File(new File(indexSecondaryRoot, "PUBLIC"), "TEXT"); + File oldSecondaryTextLocation = new File(new File(indexSecondaryRoot, "PUBLIC"), "TEXT"); + if (oldSecondaryPath.exists() && oldSecondaryTextLocation.exists()) { + // move the text folder from the old location to the new location + assert !indexSecondaryTextLocation.exists(); + indexSecondaryTextLocation.mkdirs(); + if (oldSecondaryTextLocation.renameTo(indexSecondaryTextLocation)) { + if (!oldSecondaryPath.delete()) oldSecondaryPath.deleteOnExit(); + } else { + indexSecondaryTextLocation = oldSecondaryTextLocation; // emergency case: stay with old directory + } + } + } + + urlIndexFile = new kelondroSplitTable(indexSecondaryTextLocation, "urls", indexURLEntry.rowdef, false); // init result stacks externResultStack = new LinkedList(); @@ -676,7 +693,7 @@ public final class plasmaCrawlLURL { } catch (MalformedURLException e) {} if (args[0].equals("-l")) try { // arg 1 is path to URLCache - final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[2])); + final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[2]), "freeworld"); final Iterator enu = urls.entries(true, null); while (enu.hasNext()) { System.out.println(enu.next().toString()); diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index b56afabb7..deffb71c9 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -192,7 +192,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // storage management public File htCachePath; - private File plasmaPath; + public File plasmaPath; public File indexPrimaryPath, indexSecondaryPath; public File listsPath; public File htDocsPath; @@ -710,8 +710,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser *

      Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all * databases containing queues are stored

      */ - public static final String DBPATH = "dbPath"; - public static final String DBPATH_DEFAULT = "DATA/PLASMADB"; + public static final String PLASMA_PATH = "dbPath"; + public static final String PLASMA_PATH_DEFAULT = "DATA/PLASMADB"; /** *

      public static final String HTCACHE_PATH = "proxyCache"

      *

      Name of the setting specifying the folder beginning from the YaCy-installation's top-folder, where all @@ -854,7 +854,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser *

      public static final String DBFILE_CRAWL_PROFILES = "crawlProfiles0.db" *

      Name of the file containing the database holding all recent crawl profiles

      * - * @see plasmaSwitchboard#DBPATH for the folder this file lies in + * @see plasmaSwitchboard#PLASMA_PATH for the folder this file lies in */ public static final String DBFILE_ACTIVE_CRAWL_PROFILES = "crawlProfilesActive1.db"; public static final String DBFILE_PASSIVE_CRAWL_PROFILES = "crawlProfilesPassive1.db"; @@ -862,7 +862,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser *

      public static final String DBFILE_CRAWL_ROBOTS = "crawlRobotsTxt.db"

      *

      Name of the file containing the database holding all robots.txt-entries of the lately crawled domains

      * - * @see plasmaSwitchboard#DBPATH for the folder this file lies in + * @see plasmaSwitchboard#PLASMA_PATH for the folder this file lies in */ public static final String DBFILE_CRAWL_ROBOTS = "crawlRobotsTxt1.db"; /** @@ -891,8 +891,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser this.log.logConfig("Remote proxy configuration:\n" + this.remoteProxyConfig.toString()); // load network configuration into settings - String networkUnitDefinition = getConfig("network.unit.definition", "yacy.network.unit"); + String networkUnitDefinition = getConfig("network.unit.definition", "defaults/yacy.network.freeworld.unit"); String networkGroupDefinition = getConfig("network.group.definition", "yacy.network.group"); + // patch old values + if (networkUnitDefinition.equals("yacy.network.unit")) networkUnitDefinition = "defaults/yacy.network.freeworld.unit"; // include additional network definition properties into our settings // note that these properties cannot be set in the application because they are @@ -947,7 +949,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser this.acceptLocalURLs = "local.any".indexOf(getConfig("network.unit.domain", "global")) >= 0; // load values from configs - this.plasmaPath = getConfigPath(DBPATH, DBPATH_DEFAULT); + this.plasmaPath = getConfigPath(PLASMA_PATH, PLASMA_PATH_DEFAULT); this.log.logConfig("Plasma DB Path: " + this.plasmaPath.toString()); this.indexPrimaryPath = getConfigPath(INDEX_PRIMARY_PATH, INDEX_PATH_DEFAULT); this.log.logConfig("Index Primary Path: " + this.indexPrimaryPath.toString()); @@ -965,7 +967,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // start indexing management log.logConfig("Starting Indexing Management"); - wordIndex = new plasmaWordIndex(indexPrimaryPath, indexSecondaryPath, log); + String networkName = getConfig("network.unit.name", ""); + wordIndex = new plasmaWordIndex(indexPrimaryPath, indexSecondaryPath, networkName, log); // start yacy core log.logConfig("Starting YaCy Protocol Core"); @@ -1211,7 +1214,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser CRDist1Percent = 30 CRDist1Target = kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080 **/ - rankingOn = getConfig(RANKING_DIST_ON, "true").equals("true") && getConfig("network.unit.name", "").equals("freeworld"); + rankingOn = getConfig(RANKING_DIST_ON, "true").equals("true") && networkName.equals("freeworld"); rankingOwnDistribution = new plasmaRankingDistribution(log, new File(rankingPath, getConfig(RANKING_DIST_0_PATH, plasmaRankingDistribution.CR_OWN)), (int) getConfigLong(RANKING_DIST_0_METHOD, plasmaRankingDistribution.METHOD_ANYSENIOR), (int) getConfigLong(RANKING_DIST_0_METHOD, 0), getConfig(RANKING_DIST_0_TARGET, "")); rankingOtherDistribution = new plasmaRankingDistribution(log, new File(rankingPath, getConfig(RANKING_DIST_1_PATH, plasmaRankingDistribution.CR_OTHER)), (int) getConfigLong(RANKING_DIST_1_METHOD, plasmaRankingDistribution.METHOD_MIXEDSENIOR), (int) getConfigLong(RANKING_DIST_1_METHOD, 30), getConfig(RANKING_DIST_1_TARGET, "kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080")); diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 07a340c6d..f0cdf8c7c 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -73,19 +73,37 @@ public final class plasmaWordIndex implements indexRI { private int flushsize; public final plasmaCrawlLURL loadedURL; - public plasmaWordIndex(File indexPrimaryRoot, File indexSecondaryRoot, serverLog log) { - File textindexcache = new File(indexPrimaryRoot, "PUBLIC/TEXT/RICACHE"); + public plasmaWordIndex(File indexPrimaryRoot, File indexSecondaryRoot, String networkName, serverLog log) { + File indexPrimaryPath = new File(indexPrimaryRoot, networkName); + File indexPrimaryTextLocation = new File(indexPrimaryPath, "TEXT"); + if (!indexPrimaryTextLocation.exists()) { + // patch old index locations; the secondary path is patched in plasmaCrawlLURL + File oldPrimaryPath = new File(new File(indexPrimaryRoot, "PUBLIC"), "TEXT"); + File oldPrimaryTextLocation = new File(new File(indexPrimaryRoot, "PUBLIC"), "TEXT"); + if (oldPrimaryPath.exists() && oldPrimaryTextLocation.exists()) { + // move the text folder from the old location to the new location + assert !indexPrimaryTextLocation.exists(); + indexPrimaryTextLocation.mkdirs(); + if (oldPrimaryTextLocation.renameTo(indexPrimaryTextLocation)) { + if (!oldPrimaryPath.delete()) oldPrimaryPath.deleteOnExit(); + } else { + indexPrimaryTextLocation = oldPrimaryTextLocation; // emergency case: stay with old directory + } + } + } + + File textindexcache = new File(indexPrimaryTextLocation, "RICACHE"); if (!(textindexcache.exists())) textindexcache.mkdirs(); this.dhtOutCache = new indexRAMRI(textindexcache, indexRWIRowEntry.urlEntryRow, wCacheMaxChunk, wCacheMaxAge, "dump1.array", log); this.dhtInCache = new indexRAMRI(textindexcache, indexRWIRowEntry.urlEntryRow, wCacheMaxChunk, wCacheMaxAge, "dump2.array", log); // create collections storage path - File textindexcollections = new File(indexPrimaryRoot, "PUBLIC/TEXT/RICOLLECTION"); + File textindexcollections = new File(indexPrimaryTextLocation, "RICOLLECTION"); if (!(textindexcollections.exists())) textindexcollections.mkdirs(); this.collections = new indexCollectionRI(textindexcollections, "collection", maxCollectionPartition, indexRWIRowEntry.urlEntryRow); // create LURL-db - loadedURL = new plasmaCrawlLURL(indexSecondaryRoot); + loadedURL = new plasmaCrawlLURL(indexSecondaryRoot, networkName); // performance settings busyCacheFlush = false; diff --git a/source/de/anomic/xml/opensearchdescriptionReader.java b/source/de/anomic/xml/opensearchdescriptionReader.java new file mode 100644 index 000000000..917269bd4 --- /dev/null +++ b/source/de/anomic/xml/opensearchdescriptionReader.java @@ -0,0 +1,231 @@ +// opensearchdescriptionReader.java +// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 11.03.2008 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.xml; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; + +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; + +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import de.anomic.server.serverByteBuffer; +import de.anomic.server.logging.serverLog; + +public class opensearchdescriptionReader extends DefaultHandler { + + // statics for item generation and automatic categorization + static int guidcount = 0; + private static final String recordTag = "OpenSearchDescription"; + private static final String[] tagsDef = new String[]{ + "ShortName", + "LongName", + "Image", + "Language", + "OutputEncoding", + "InputEncoding", + "AdultContent", + "Description", + "Url", + "Developer", + "Query", + "Tags", + "Contact", + "Attribution", + "SyndicationRight" + }; + /* + + + YaCy/#[clientname]# + YaCy.net - #[SearchPageGreeting]# + http://#[thisaddress]#/env/grafics/yacy.gif + en-us + UTF-8 + UTF-8 + true + YaCy is an open-source GPL-licensed software that can be used for stand-alone search engine installations or as a client for a multi-user P2P-based web indexing cluster. This is the access to peer '#[clientname]#'. + + See http://developer.berlios.de/projects/yacy/ + + YaCy P2P Web Search + See http://#[thisaddress]#/ViewProfile.html?hash=localhash + YaCy Software &copy; 2004-2007 by Michael Christen et al., YaCy.net; Content: ask peer owner + open + + */ + + private static final HashSet tags = new HashSet(); + static { + for (int i = 0; i < tagsDef.length; i++) { + tags.add(tagsDef[i]); + } + } + + // class variables + private Item channel, item; + private StringBuffer buffer; + private boolean parsingChannel; + private String imageURL; + private ArrayList itemsGUID; // a list of GUIDs, so the items can be retrieved by a specific order + private HashMap items; // a guid:Item map + + + public opensearchdescriptionReader() { + itemsGUID = new ArrayList(); + items = new HashMap(); + buffer = new StringBuffer(); + item = null; + channel = null; + parsingChannel = false; + } + + public opensearchdescriptionReader(String path) { + this(); + try { + SAXParserFactory factory = SAXParserFactory.newInstance(); + SAXParser saxParser = factory.newSAXParser(); + saxParser.parse(path, this); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public opensearchdescriptionReader(InputStream stream) { + this(); + try { + SAXParserFactory factory = SAXParserFactory.newInstance(); + SAXParser saxParser = factory.newSAXParser(); + saxParser.parse(stream, this); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static opensearchdescriptionReader parse(byte[] a) { + + // check integrity of array + if ((a == null) || (a.length == 0)) { + serverLog.logWarning("opensearchdescriptionReader", "response=null"); + return null; + } + if (a.length < 100) { + serverLog.logWarning("opensearchdescriptionReader", "response=" + new String(a)); + return null; + } + if (!serverByteBuffer.equals(a, " map; + + public Item() { + this.map = new HashMap(); + this.map.put("guid", Long.toHexString(System.currentTimeMillis()) + ":" + guidcount++); + } + + public void setValue(String name, String value) { + map.put(name, value); + } + } +} \ No newline at end of file diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 1c03e142c..528410d94 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -1094,7 +1094,7 @@ public final class yacyClient { public static void main(String[] args) { System.out.println("yacyClient Test"); try { - final plasmaSwitchboard sb = new plasmaSwitchboard(new File(args[0]), "httpProxy.init", "DATA/SETTINGS/httpProxy.conf", false); + final plasmaSwitchboard sb = new plasmaSwitchboard(new File(args[0]), "httpProxy.init", "DATA/SETTINGS/yacy.conf", false); /*final yacyCore core =*/ new yacyCore(sb); yacyCore.peerActions.loadSeedLists(); final yacySeed target = yacyCore.seedDB.getConnected(args[1]); diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index fbe3f91c4..aad9bbebe 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -204,7 +204,7 @@ public class yacyPeerActions { } catch (IOException e) { // this is when wget fails, commonly because of timeout - yacyCore.log.logWarning("BOOTSTRAP: failed (1) to load seeds from seed-list URL " + seedListFileURL + ": " + e.getMessage(), e); + yacyCore.log.logWarning("BOOTSTRAP: failed (1) to load seeds from seed-list URL " + seedListFileURL + ": " + e.getMessage()); } catch (Exception e) { // this is when wget fails; may be because of missing internet connection yacyCore.log.logSevere("BOOTSTRAP: failed (2) to load seeds from seed-list URL " + seedListFileURL + ": " + e.getMessage(), e); diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java index ee0f53d95..cc6eb05d6 100644 --- a/source/de/anomic/yacy/yacyVersion.java +++ b/source/de/anomic/yacy/yacyVersion.java @@ -60,9 +60,9 @@ public final class yacyVersion implements Comparator, Comparable latestReleases = new HashMap(); - public static ArrayList latestReleaseLocations = new ArrayList(); // will be initialized with value in yacy.network.unit + public static ArrayList latestReleaseLocations = new ArrayList(); // will be initialized with value in defaults/yacy.network.freeworld.unit // private static release info about this release; is generated only once and can be retrieved by thisVersion() private static yacyVersion thisVersion = null; diff --git a/source/yacy.java b/source/yacy.java index 8a3275696..b2a4a5d71 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -220,7 +220,13 @@ public final class yacy { f.deleteOnExit(); pro = new File(homePath, "libx").exists(); - sb = new plasmaSwitchboard(homePath, "yacy.init", "DATA/SETTINGS/httpProxy.conf", pro); + String oldconf = "DATA/SETTINGS/httpProxy.conf"; + String newconf = "DATA/SETTINGS/yacy.conf"; + File oldconffile = new File(homePath, oldconf); + if (oldconffile.exists()) { + oldconffile.renameTo(new File(homePath, newconf)); + } + sb = new plasmaSwitchboard(homePath, "defaults/yacy.init", newconf, pro); sbSync.V(); // signal that the sb reference was set // save information about available memory at startup time @@ -590,7 +596,7 @@ public final class yacy { * @param homePath path to the YaCy directory * @param dbcache cache size in MB */ - public static void minimizeUrlDB(File homePath) { + public static void minimizeUrlDB(File homePath, String networkName) { // run with "java -classpath classes yacy -minimizeUrlDB" try {serverLog.configureLogging(homePath, new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {} File indexPrimaryRoot = new File(homePath, "DATA/INDEX"); @@ -601,15 +607,15 @@ public final class yacy { log.logInfo("STARTING URL CLEANUP"); // db containing all currently loades urls - plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexSecondaryRoot); + plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexSecondaryRoot, networkName); // db used to hold all neede urls - plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(indexRoot2); + plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(indexRoot2, networkName); int cacheMem = (int)(serverMemory.max() - serverMemory.total()); if (cacheMem < 2048000) throw new OutOfMemoryError("Not enough memory available to start clean up."); - plasmaWordIndex wordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, log); + plasmaWordIndex wordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, networkName, log); Iterator indexContainerIterator = wordIndex.wordContainers("AAAAAAAAAAAA", false, false); long urlCounter = 0, wordCounter = 0; @@ -779,11 +785,11 @@ public final class yacy { * * @param homePath Root-Path where all information is to be found. */ - private static void urldbcleanup(File homePath) { + private static void urldbcleanup(File homePath, String networkName) { File root = homePath; File indexroot = new File(root, "DATA/INDEX"); try {serverLog.configureLogging(homePath, new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {} - plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexroot); + plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexroot, networkName); currentUrlDB.urldbcleanup(); currentUrlDB.close(); } @@ -800,7 +806,7 @@ public final class yacy { try { Iterator indexContainerIterator = null; if (resource.equals("all")) { - WordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, log); + WordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, "freeworld", log); indexContainerIterator = WordIndex.wordContainers(wordChunkStartHash, false, false); } int counter = 0; @@ -925,7 +931,7 @@ public final class yacy { args = shift(args, 1, 2); } if (args.length == 2) applicationRoot= new File(args[1]); - minimizeUrlDB(applicationRoot); + minimizeUrlDB(applicationRoot, "freeworld"); } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-testpeerdb"))) { if (args.length == 2) { applicationRoot = new File(args[1]); @@ -953,7 +959,7 @@ public final class yacy { } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-urldbcleanup"))) { // generate a url list and save it in a file if (args.length == 2) applicationRoot= new File(args[1]); - urldbcleanup(applicationRoot); + urldbcleanup(applicationRoot, "freeworld"); } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-rwihashlist"))) { // generate a url list and save it in a file String domain = "all"; diff --git a/startYACY.bat b/startYACY.bat index f18354bed..f1088b732 100644 --- a/startYACY.bat +++ b/startYACY.bat @@ -14,7 +14,7 @@ set jms= set javacmd=-Xmx64m -Xms10m set priolvl=0 set priority=/NORMAL -if exist DATA\SETTINGS\httpProxy.conf GoTo :GETSTARTOPTS +if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS :STARTJAVA Rem Starting YaCy @@ -38,7 +38,7 @@ del DATA\yacy.restart Rem This target is used to read java runtime parameters out of the yacy config file :GETSTARTOPTS -for /F "tokens=1,2 delims==" %%i in (DATA\SETTINGS\httpProxy.conf) do ( +for /F "tokens=1,2 delims==" %%i in (DATA\SETTINGS\yacy.conf) do ( if "%%i"=="javastart_Xmx" set jmx=%%j if "%%i"=="javastart_Xms" set jms=%%j if "%%i"=="port" set port=%%j diff --git a/startYACY.sh b/startYACY.sh index 2c7293b7b..f8b057402 100755 --- a/startYACY.sh +++ b/startYACY.sh @@ -77,24 +77,24 @@ done #get javastart args java_args="" -if [ -f DATA/SETTINGS/httpProxy.conf ] +if [ -f DATA/SETTINGS/yacy.conf ] then # startup memory for i in Xmx Xms; do - j="`grep javastart_$i DATA/SETTINGS/httpProxy.conf | sed 's/^[^=]*=//'`"; + j="`grep javastart_$i DATA/SETTINGS/yacy.conf | sed 's/^[^=]*=//'`"; if [ -n $j ]; then JAVA_ARGS="-$j $JAVA_ARGS"; fi; done # Priority - j="`grep javastart_priority DATA/SETTINGS/httpProxy.conf | sed 's/^[^=]*=//'`"; + j="`grep javastart_priority DATA/SETTINGS/yacy.conf | sed 's/^[^=]*=//'`"; if [ ! -z "$j" ];then if [ -n $j ]; then JAVA="nice -n $j $JAVA"; fi; fi - PORT="`grep ^port= DATA/SETTINGS/httpProxy.conf | sed 's/^[^=]*=//'`"; + PORT="`grep ^port= DATA/SETTINGS/yacy.conf | sed 's/^[^=]*=//'`"; -# for i in `grep javastart DATA/SETTINGS/httpProxy.conf`;do +# for i in `grep javastart DATA/SETTINGS/yacy.conf`;do # i="${i#javastart_*=}"; # JAVA_ARGS="-$i $JAVA_ARGS"; # done @@ -102,6 +102,25 @@ else JAVA_ARGS="-Xmx120m -Xms120m $JAVA_ARGS"; PORT="8080" fi + +if [ -f DATA/SETTINGS/httpProx.conf ] +then + # startup memory + for i in Xmx Xms; do + j="`grep javastart_$i DATA/SETTINGS/httpProxy.conf | sed 's/^[^=]*=//'`"; + if [ -n $j ]; then JAVA_ARGS="-$j $JAVA_ARGS"; fi; + done + + # Priority + j="`grep javastart_priority DATA/SETTINGS/httpProxy.conf | sed 's/^[^=]*=//'`"; + + if [ ! -z "$j" ];then + if [ -n $j ]; then JAVA="nice -n $j $JAVA"; fi; + fi + + PORT="`grep ^port= DATA/SETTINGS/httpProxy.conf | sed 's/^[^=]*=//'`"; +fi + #echo "JAVA_ARGS: $JAVA_ARGS" #echo "JAVA: $JAVA" diff --git a/startYACY_noconsole.bat b/startYACY_noconsole.bat index ea4937adc..d26d111ab 100644 --- a/startYACY_noconsole.bat +++ b/startYACY_noconsole.bat @@ -14,7 +14,7 @@ set jms= set javacmd=-Xmx64m -Xms10m set priolvl=0 set priority=/NORMAL -if exist DATA\SETTINGS\httpProxy.conf GoTo :GETSTARTOPTS +if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS :STARTJAVA Rem Starting YaCy @@ -28,7 +28,7 @@ GoTo :END Rem This target is used to read java runtime parameters out of the yacy config file :GETSTARTOPTS -for /F "tokens=1,2 delims==" %%i in (DATA\SETTINGS\httpProxy.conf) do ( +for /F "tokens=1,2 delims==" %%i in (DATA\SETTINGS\yacy.conf) do ( if "%%i"=="javastart_Xmx" set jmx=%%j if "%%i"=="javastart_Xms" set jms=%%j if "%%i"=="javastart_priority" set priolvl=%%j diff --git a/stopYACY.sh b/stopYACY.sh index 631a8c19d..ea46d1618 100755 --- a/stopYACY.sh +++ b/stopYACY.sh @@ -3,8 +3,8 @@ cd `dirname $0` if [ -x `which wget` ] then - port=`cat DATA/SETTINGS/httpProxy.conf |grep "^port="|sed "s/.*=//"` - pw=`cat DATA/SETTINGS/httpProxy.conf |grep "^adminAccountBase64MD5="|sed "s/.*=//"` + port=`cat DATA/SETTINGS/yacy.conf |grep "^port="|sed "s/.*=//"` + pw=`cat DATA/SETTINGS/yacy.conf |grep "^adminAccountBase64MD5="|sed "s/.*=//"` wget -q -t 1 --timeout=5 --header "Authorization: realm=$pw" http://localhost:$port/Steering.html?shutdown=true -O /dev/null echo "Please wait until the YaCy daemon process terminates" @@ -23,7 +23,7 @@ then echo "You can monitor this with 'tail -f DATA/LOG/yacy00.log' and 'fuser log/yacy00.log'" else - port=`cat DATA/SETTINGS/httpProxy.conf |grep "^port="|sed "s/.*=//"` + port=`cat DATA/SETTINGS/yacy.conf |grep "^port="|sed "s/.*=//"` echo "Neither wget nor java could be found or are not executable." echo "Visit http://localhost:$port/Steering.html?shutdown=true to stop YaCy or (in emergency case) use ./killYACY.sh" fi diff --git a/yacy-svn-4.spec b/yacy-svn-4.spec index 939bdac07..cc3dae88d 100644 --- a/yacy-svn-4.spec +++ b/yacy-svn-4.spec @@ -83,10 +83,9 @@ install -d ${RPM_BUILD_ROOT}{$YACYCDIR,$YACYDDIR,$YACYLDIR/libx,$DATADIR,$LOGDIR # copy all other files cp -r htroot locales ranking skins ${RPM_BUILD_ROOT}$YACYCDIR/ -cp -r classes lib libx ${RPM_BUILD_ROOT}$YACYLDIR/ +cp -r defaults classes lib libx ${RPM_BUILD_ROOT}$YACYLDIR/ cp -r doc ${RPM_BUILD_ROOT}$YACYDDIR/ - -cp *.sh build.properties superseed.txt httpd.mime yacy.badwords.example yacy.init yacy.logging yacy.stopwords* yacy.yellow ${RPM_BUILD_ROOT}$YACYCDIR/ +cp *.sh build.properties superseed.txt httpd.mime yacy.badwords.example yacy.logging yacy.stopwords* yacy.yellow ${RPM_BUILD_ROOT}$YACYCDIR/ cp AUTHORS COPYRIGHT ChangeLog gpl.txt readme.txt ${RPM_BUILD_ROOT}$YACYDDIR/ install -m 744 *.sh ${RPM_BUILD_ROOT}$YACYCDIR/ # start/stop/kill scripts @@ -129,8 +128,8 @@ chmod +x /usr/share/yacy/killYACY.sh #if [ $LNT == de -o $LNT == it -o $LNT == sk ]; then # translation exists # # test whether settings-directory exists # if [ ! -d /var/lib/yacy/SETTINGS ]; then mkdir /var/lib/yacy/SETTINGS; chown yacy:root /var/lib/yacy/SETTINGS; fi -# echo "htLocaleSelection=$LNT.lng" >> /var/lib/yacy/SETTINGS/httpProxy.conf -# chown yacy:root /var/lib/yacy/SETTINGS/httpProxy.conf +# echo "htLocaleSelection=$LNT.lng" >> /var/lib/yacy/SETTINGS/yacy.conf +# chown yacy:root /var/lib/yacy/SETTINGS/yacy.conf #fi @@ -153,7 +152,7 @@ cat > /etc/init.d/yacy <