From 3661cb692cae5c3d9046aa7568a03b66353029c4 Mon Sep 17 00:00:00 2001 From: orbiter Date: Fri, 14 May 2010 09:52:53 +0000 Subject: [PATCH] added dictionary loader servlet that can be used to get the geolocalization file: /DictionaryLoader_p.html Will also be used for more dictionary files in the future git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6872 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/DictionaryLoader_p.html | 60 +++++++++++ htroot/DictionaryLoader_p.java | 107 ++++++++++++++++++++ htroot/env/base.css | 6 ++ htroot/env/templates/submenuConfig.template | 1 + source/de/anomic/data/LibraryProvider.java | 22 ++++ 5 files changed, 196 insertions(+) create mode 100644 htroot/DictionaryLoader_p.html create mode 100644 htroot/DictionaryLoader_p.java diff --git a/htroot/DictionaryLoader_p.html b/htroot/DictionaryLoader_p.html new file mode 100644 index 000000000..8437a5b7c --- /dev/null +++ b/htroot/DictionaryLoader_p.html @@ -0,0 +1,60 @@ + + + + YaCy '#[clientname]#': Dictionary Loader + #%env/templates/metas.template%# + + + #%env/templates/header.template%# + #%env/templates/submenuConfig.template%# +

Dictionary Loader

+ +

+ YaCy can use external libraries to enable or enhance some functions. These libraries are not + included in the main release of YaCy because they would increase the application file too much. + You can download additional files here. +

+ +
+
+ Geolocalization + The geolocalization file will enable YaCy to present locations from OpenStreetMap according to given search words. + With this file it is possible to find locations using the location (city) name, a zip code, a car sign or a telephone pre-dial number. +
+
+
#[geo0URL]#
+
+
#[geo0Storage]#
+
+
#(geo0Status)#
not loaded
::
loaded
::de-activated#(/geo0Status)#
+
+
#(geo0Status)# + :: + + :: + + + #(/geo0Status)#
+ #(geo0ActionLoaded)#:: +
loaded and activated dictionary file
:: +
loading of dictionary file failed: #[error]#
+ #(/geo0ActionLoaded)# + #(geo0ActionRemoved)#:: +
de-activated and removed dictionary file
:: +
cannot remove dictionary file: #[error]#
+ #(/geo0ActionRemoved)# + #(geo0ActionDeactivated)#:: +
de-activated dictionary file
:: +
cannot de-activate dictionary file: #[error]#
+ #(/geo0ActionDeactivated)# + #(geo0ActionActivated)#:: +
activated dictionary file
:: +
cannot activate dictionary file: #[error]#
+ #(/geo0ActionActivated)# +
+
+
+ + #%env/templates/footer.template%# + + diff --git a/htroot/DictionaryLoader_p.java b/htroot/DictionaryLoader_p.java new file mode 100644 index 000000000..2e801e153 --- /dev/null +++ b/htroot/DictionaryLoader_p.java @@ -0,0 +1,107 @@ +// DictionaryLoader_p.java +// ----------------------- +// part of YaCy +// (C) by Michael Peter Christen; mc@yacy.net +// first published 12.05.2010 on http://yacy.net +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import java.io.IOException; +import java.net.MalformedURLException; + +import net.yacy.document.geolocalization.OpenGeoDB; +import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.util.FileUtils; +import de.anomic.crawler.CrawlProfile; +import de.anomic.crawler.retrieval.Response; +import de.anomic.data.LibraryProvider; +import de.anomic.http.server.RequestHeader; +import de.anomic.search.Switchboard; +import de.anomic.server.serverObjects; +import de.anomic.server.serverSwitch; + +public class DictionaryLoader_p { + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + final Switchboard sb = (Switchboard) env; + final serverObjects prop = new serverObjects(); // return variable that accumulates replacements + + /* + * distinguish the following cases: + * - dictionary file was not loaded -> actions: load the file + * - dictionary file is loaded and enabled -> actions: disable or remove the file + * - dictionary file is loaded but disabled -> actions: enable or remove the file + */ + + for (LibraryProvider.Dictionary dictionary: LibraryProvider.Dictionary.values()) { + prop.put(dictionary.nickname + "URL", dictionary.url); + prop.put(dictionary.nickname + "Storage", dictionary.file().toString()); + prop.put(dictionary.nickname + "Status", dictionary.file().exists() ? 1 : dictionary.fileDisabled().exists() ? 2 : 0); + prop.put(dictionary.nickname + "ActionLoaded", 0); + prop.put(dictionary.nickname + "ActionRemoved", 0); + prop.put(dictionary.nickname + "ActionActivated", 0); + prop.put(dictionary.nickname + "ActionDeactivated", 0); + } + + if (post == null) return prop; + + if (post.containsKey("geo0Load")) { + // load from the net + try { + Response response = sb.loader.load(new DigestURI(LibraryProvider.Dictionary.GEO0.url), false, true, CrawlProfile.CACHE_STRATEGY_NOCACHE); + byte[] b = response.getContent(); + FileUtils.copy(b, LibraryProvider.Dictionary.GEO0.file()); + LibraryProvider.geoDB = new OpenGeoDB(LibraryProvider.Dictionary.GEO0.file()); + prop.put("geo0Status", LibraryProvider.Dictionary.GEO0.file().exists() ? 1 : 0); + prop.put("geo0ActionLoaded", 1); + } catch (MalformedURLException e) { + Log.logException(e); + prop.put("geo0ActionLoaded", 2); + prop.put("geo0ActionLoaded_error", e.getMessage()); + } catch (IOException e) { + Log.logException(e); + prop.put("geo0ActionLoaded", 2); + prop.put("geo0ActionLoaded_error", e.getMessage()); + } + } + + if (post.containsKey("geo0Remove")) { + FileUtils.deletedelete(LibraryProvider.Dictionary.GEO0.file()); + FileUtils.deletedelete(LibraryProvider.Dictionary.GEO0.fileDisabled()); + LibraryProvider.geoDB = new OpenGeoDB(null); + prop.put("geo0ActionRemoved", 1); + } + + if (post.containsKey("geo0Deactivate")) { + LibraryProvider.Dictionary.GEO0.file().renameTo(LibraryProvider.Dictionary.GEO0.fileDisabled()); + LibraryProvider.geoDB = new OpenGeoDB(null); + prop.put("geo0ActionDeactivated", 1); + } + + if (post.containsKey("geo0Activate")) { + LibraryProvider.Dictionary.GEO0.fileDisabled().renameTo(LibraryProvider.Dictionary.GEO0.file()); + LibraryProvider.geoDB = new OpenGeoDB(LibraryProvider.Dictionary.GEO0.file()); + prop.put("geo0ActionActivated", 1); + } + + // check status again + for (LibraryProvider.Dictionary dictionary: LibraryProvider.Dictionary.values()) { + prop.put(dictionary.nickname + "Status", dictionary.file().exists() ? 1 : dictionary.fileDisabled().exists() ? 2 : 0); + } + + return prop; // return rewrite values for templates + } +} diff --git a/htroot/env/base.css b/htroot/env/base.css index 118ec413e..2da09b500 100644 --- a/htroot/env/base.css +++ b/htroot/env/base.css @@ -124,6 +124,12 @@ tt, *.tt { .error { font-weight:bold; + color:red; +} + +.info { + font-weight:bold; + color:yellow; } .commit { diff --git a/htroot/env/templates/submenuConfig.template b/htroot/env/templates/submenuConfig.template index 3560972ad..384d13024 100644 --- a/htroot/env/templates/submenuConfig.template +++ b/htroot/env/templates/submenuConfig.template @@ -5,6 +5,7 @@
  • Basic Configuration
  • Accounts
  • Network Configuration
    • +
  • Dictionary Loader
  • System Update
    • diff --git a/source/de/anomic/data/LibraryProvider.java b/source/de/anomic/data/LibraryProvider.java index 3f71d441d..b45a534c3 100644 --- a/source/de/anomic/data/LibraryProvider.java +++ b/source/de/anomic/data/LibraryProvider.java @@ -47,11 +47,33 @@ public class LibraryProvider { private static final String path_to_source_dictionaries = "source"; private static final String path_to_did_you_mean_dictionaries = "didyoumean"; + public static final String disabledExtension = ".disabled"; + public static DidYouMeanLibrary dymLib = new DidYouMeanLibrary(null); public static OpenGeoDB geoDB = new OpenGeoDB(null); private static File dictSource = null; private static File dictRoot = null; + public static enum Dictionary { + GEO0("geo0", + "http://downloads.sourceforge.net/project/opengeodb/Data/0.2.5a/opengeodb-0.2.5a-UTF8-sql.gz", + "opengeodb-0.2.5a-UTF8-sql.gz"); + + public String nickname, url, filename; + private Dictionary(String nickname, String url, String filename) { + this.nickname = nickname; + this.url = url; + this.filename = filename; + } + + public File file() { + return new File(dictSource, filename); + } + public File fileDisabled() { + return new File(dictSource, filename + disabledExtension); + } + } + /** * initialize the LibraryProvider as static class. * This assigns default paths, and initializes the dictionary classes