diff --git a/.classpath b/.classpath index c36811f6e..e3ddd58fa 100644 --- a/.classpath +++ b/.classpath @@ -32,14 +32,14 @@ - + - - + + @@ -64,26 +64,26 @@ - - + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/build.xml b/build.xml index 5195f29eb..07228fa83 100644 --- a/build.xml +++ b/build.xml @@ -183,7 +183,7 @@ - + @@ -222,41 +222,41 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - + - - + + - + diff --git a/defaults/yacy.init b/defaults/yacy.init index 357dd83ca..479626e61 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -1169,7 +1169,7 @@ content.phpbb3.dumpfile = # search engine teaser: an about box in search results # this is only shown, if the about.body is filled about.headline=Please support YaCy! -about.body=
If you run a YaCy server, feel free to replace our donation plea with your own support message, use the Portal Configuration servlet.
+about.body=
If you run a YaCy server, feel free to replace our donation plea with your own support message, use the Portal Configuration servlet.
donation.iframesource=http://yacy.net/include/donate.html donation.iframetarget=env/donate.html diff --git a/htroot/ConfigAppearance_p.html b/htroot/ConfigAppearance_p.html index c79d626c0..65f4e5f88 100644 --- a/htroot/ConfigAppearance_p.html +++ b/htroot/ConfigAppearance_p.html @@ -31,7 +31,7 @@

You can change the appearance of the YaCy interface with skins. The selected skin and language also affects the appearance of the search page. - If you create a search portal with YaCy then you can + If you create a search portal with YaCy then you can change the appearance of the search page here.

diff --git a/htroot/ConfigPortal.java b/htroot/ConfigPortal.java index 9362d20f6..adeb9ab7c 100644 --- a/htroot/ConfigPortal.java +++ b/htroot/ConfigPortal.java @@ -25,210 +25,22 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.Properties; - -import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.data.WorkTables; -import net.yacy.http.servlets.YaCyDefaultServlet; -import net.yacy.search.Switchboard; -import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; -import net.yacy.server.http.HTTPDFileHandler; +/** + * @deprecated use now {@link ConfigPortal_p} + */ +@Deprecated public class ConfigPortal { - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + @SuppressWarnings("unused") + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final serverObjects prop = new serverObjects(); - final Switchboard sb = (Switchboard) env; - - if (post != null) { - // AUTHENTICATE - if (!sb.verifyAuthentication(header)) { - // force log-in - prop.authenticationRequired(); - return prop; - } - - if (post.containsKey("popup")) { - final String popup = post.get("popup", "status"); - if ("front".equals(popup)) { - sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html"); - } else if ("search".equals(popup)) { - sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "yacysearch.html"); - } else if ("interactive".equals(popup)) { - sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "yacyinteractive.html"); - } else { - sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "Status.html"); - } - sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html")); - HTTPDFileHandler.initDefaultPath(); - } - if (post.containsKey("searchpage_set")) { - final String newGreeting = post.get(SwitchboardConstants.GREETING, ""); - // store this call as api call - sb.tables.recordAPICall(post, "ConfigPortal.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting); - - sb.setConfig(SwitchboardConstants.GREETING, newGreeting); - sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, "")); - sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, post.get(SwitchboardConstants.GREETING_LARGE_IMAGE, "")); - sb.setConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, post.get(SwitchboardConstants.GREETING_SMALL_IMAGE, "")); - sb.setConfig(SwitchboardConstants.GREETING_IMAGE_ALT, post.get(SwitchboardConstants.GREETING_IMAGE_ALT, "")); - sb.setConfig(SwitchboardConstants.SEARCH_TARGET_DEFAULT, post.get("target", "_self")); - sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL, post.get("target_special", "_self")); - sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, post.get("target_special_pattern", "_self")); - sb.setConfig(SwitchboardConstants.SEARCH_ITEMS, post.getInt("maximumRecords", 10)); - sb.setConfig(SwitchboardConstants.INDEX_FORWARD, post.get(SwitchboardConstants.INDEX_FORWARD, "")); - HTTPDFileHandler.indexForward = post.get(SwitchboardConstants.INDEX_FORWARD, ""); - sb.setConfig("publicTopmenu", !post.containsKey("publicTopmenu") || post.getBoolean("publicTopmenu")); - sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, !post.containsKey(SwitchboardConstants.PUBLIC_SEARCHPAGE) || post.getBoolean(SwitchboardConstants.PUBLIC_SEARCHPAGE)); - sb.setConfig("search.options", post.getBoolean("search.options")); - - sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, post.getBoolean(SwitchboardConstants.GREEDYLEARNING_ACTIVE)); - - final boolean storeresult = post.getBoolean(SwitchboardConstants.REMOTESEARCH_RESULT_STORE); - sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, storeresult); - sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, post.getLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1)); - - sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, post.get("search.verify", "ifexist")); - sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, post.getBoolean("search.verify.delete")); - - sb.setConfig("about.headline", post.get("about.headline", "")); - sb.setConfig("about.body", post.get("about.body", "")); - - String excludehosts = post.get("search.excludehosts", ""); - sb.setConfig("search.excludehosts", excludehosts); - try { - sb.setConfig("search.excludehosth", DigestURL.hosthashes(excludehosts)); - } catch (MalformedURLException e) { - ConcurrentLog.logException(e); - sb.setConfig("search.excludehosth", ""); - } - } - if (post.containsKey("searchpage_default")) { - // load defaults from defaults/yacy.init file - final Properties config = new Properties(); - final String mes = "ConfigPortal"; - FileInputStream fis = null; - try { - fis = new FileInputStream(new File(sb.appPath, "defaults/yacy.init")); - config.load(fis); - } catch (final FileNotFoundException e) { - ConcurrentLog.severe(mes, "could not find configuration file."); - return prop; - } catch (final IOException e) { - ConcurrentLog.severe(mes, "could not read configuration file."); - return prop; - } finally { - if (fis != null) { - try { - fis.close(); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - } - } - sb.setConfig(SwitchboardConstants.GREETING, config.getProperty(SwitchboardConstants.GREETING,"P2P Web Search")); - sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, config.getProperty(SwitchboardConstants.GREETING_HOMEPAGE,"http://yacy.net")); - sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, config.getProperty(SwitchboardConstants.GREETING_LARGE_IMAGE,"env/grafics/YaCyLogo_120ppi.png")); - sb.setConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, config.getProperty(SwitchboardConstants.GREETING_SMALL_IMAGE,"env/grafics/YaCyLogo_60ppi.png")); - sb.setConfig(SwitchboardConstants.GREETING_IMAGE_ALT, config.getProperty(SwitchboardConstants.GREETING_IMAGE_ALT,"YaCy project web site")); - sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, config.getProperty(SwitchboardConstants.BROWSER_POP_UP_PAGE,"Status.html")); - sb.setConfig(SwitchboardConstants.INDEX_FORWARD, config.getProperty(SwitchboardConstants.INDEX_FORWARD,"")); - HTTPDFileHandler.indexForward = ""; - sb.setConfig(SwitchboardConstants.SEARCH_TARGET_DEFAULT, config.getProperty(SwitchboardConstants.SEARCH_TARGET_DEFAULT,"_self")); - sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL, config.getProperty(SwitchboardConstants.SEARCH_TARGET_SPECIAL,"_self")); - sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, config.getProperty(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN,"")); - sb.setConfig("publicTopmenu", config.getProperty("publicTopmenu","true")); - sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, config.getProperty(SwitchboardConstants.PUBLIC_SEARCHPAGE,"true")); - sb.setConfig("search.navigation", config.getProperty("search.navigation","hosts,authors,namespace,topics")); - sb.setConfig("search.options", config.getProperty("search.options","true")); - sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, config.getProperty(SwitchboardConstants.GREEDYLEARNING_ACTIVE)); - sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE)); - sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE)); - sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, config.getProperty(SwitchboardConstants.SEARCH_VERIFY,"iffresh")); - sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, config.getProperty(SwitchboardConstants.SEARCH_VERIFY_DELETE,"true")); - sb.setConfig("about.headline", config.getProperty("about.headline","")); - sb.setConfig("about.body", config.getProperty("about.body","")); - sb.setConfig("search.excludehosts", config.getProperty("search.excludehosts","")); - sb.setConfig("search.excludehosth", config.getProperty("search.excludehosth","")); - } - } - - prop.putHTML(SwitchboardConstants.GREETING, sb.getConfig(SwitchboardConstants.GREETING, "")); - prop.putHTML(SwitchboardConstants.GREETING_HOMEPAGE, sb.getConfig(SwitchboardConstants.GREETING_HOMEPAGE, "")); - prop.putHTML(SwitchboardConstants.GREETING_LARGE_IMAGE, sb.getConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, "")); - prop.putHTML(SwitchboardConstants.GREETING_SMALL_IMAGE, sb.getConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, "")); - prop.putHTML(SwitchboardConstants.GREETING_IMAGE_ALT, sb.getConfig(SwitchboardConstants.GREETING_IMAGE_ALT, "")); - prop.putHTML(SwitchboardConstants.INDEX_FORWARD, sb.getConfig(SwitchboardConstants.INDEX_FORWARD, "")); - prop.put("publicTopmenu", sb.getConfigBool("publicTopmenu", false) ? 1 : 0); - prop.put(SwitchboardConstants.PUBLIC_SEARCHPAGE, sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, false) ? 1 : 0); - prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0); - - prop.put(SwitchboardConstants.GREEDYLEARNING_ACTIVE, sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ACTIVE, false) ? 1 : 0); - prop.put(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, sb.getConfig(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, "0")); - - prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, sb.getConfigBool(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, true) ? 1 : 0); - long resultStoredMaxSize = sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1); - if(resultStoredMaxSize > 0) { - prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, resultStoredMaxSize); - } else { - prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, ""); - } - - prop.put("search.verify.nocache", sb.getConfig("search.verify", "").equals("nocache") ? 1 : 0); - prop.put("search.verify.iffresh", sb.getConfig("search.verify", "").equals("iffresh") ? 1 : 0); - prop.put("search.verify.ifexist", sb.getConfig("search.verify", "").equals("ifexist") ? 1 : 0); - prop.put("search.verify.cacheonly", sb.getConfig("search.verify", "").equals("cacheonly") ? 1 : 0); - prop.put("search.verify.false", sb.getConfig("search.verify", "").equals("false") ? 1 : 0); - prop.put("search.verify.delete", sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true) ? 1 : 0); - - prop.put("about.headline", sb.getConfig("about.headline", "")); - prop.put("about.body", sb.getConfig("about.body", "")); - - prop.put("search.excludehosts", sb.getConfig("search.excludehosts", "")); - prop.put("search.excludehosth", sb.getConfig("search.excludehosth", "")); - - final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html"); - prop.put("popupFront", 0); - prop.put("popupSearch", 0); - prop.put("popupInteractive", 0); - prop.put("popupStatus", 0); - if (browserPopUpPage.startsWith("index")) { - prop.put("popupFront", 1); - } else if (browserPopUpPage.startsWith("yacysearch")) { - prop.put("popupSearch", 1); - } else if (browserPopUpPage.startsWith("yacyinteractive")) { - prop.put("popupInteractive", 1); - } else { - prop.put("popupStatus", 1); - } - - prop.put("maximumRecords", sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10)); - - final String target = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_DEFAULT, "_self"); - prop.put("target_selected_blank", "_blank".equals(target) ? 1 : 0); - prop.put("target_selected_self", "_self".equals(target) ? 1 : 0); - prop.put("target_selected_parent", "_parent".equals(target) ? 1 : 0); - prop.put("target_selected_top", "_top".equals(target) ? 1 : 0); - prop.put("target_selected_searchresult", "searchresult".equals(target) ? 1 : 0); - - final String target_special = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL, "_self"); - prop.put("target_selected_special_blank", "_blank".equals(target_special) ? 1 : 0); - prop.put("target_selected_special_self", "_self".equals(target_special) ? 1 : 0); - prop.put("target_selected_special_parent", "_parent".equals(target_special) ? 1 : 0); - prop.put("target_selected_special_top", "_top".equals(target_special) ? 1 : 0); - prop.put("target_selected_special_searchresult", "searchresult".equals(target_special) ? 1 : 0); - prop.put("target_special_pattern", sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "")); - prop.put("myContext", YaCyDefaultServlet.getContext(header, sb)); + /* Redirect to ConfigPortal_p */ + prop.put(serverObjects.ACTION_LOCATION, "ConfigPortal_p.html"); return prop; } diff --git a/htroot/ConfigPortal.html b/htroot/ConfigPortal_p.html similarity index 98% rename from htroot/ConfigPortal.html rename to htroot/ConfigPortal_p.html index 0938a4767..d0de8afaf 100644 --- a/htroot/ConfigPortal.html +++ b/htroot/ConfigPortal_p.html @@ -15,7 +15,8 @@ and a link to a home page that is reached when the 'corporate identity'-images are clicked. To change also colours and styles use the Appearance Servlet for different skins and languages.

-
+ +
Greeting Line
diff --git a/htroot/ConfigPortal_p.java b/htroot/ConfigPortal_p.java new file mode 100644 index 000000000..60576bebb --- /dev/null +++ b/htroot/ConfigPortal_p.java @@ -0,0 +1,235 @@ +// ConfigPortal_p.java +// ----------------------- +// part of YaCy +// (C) by Michael Peter Christen; mc@yacy.net +// first published on http://yacy.net +// Frankfurt, Germany, 4.7.2008 +// +//$LastChangedDate$ +//$LastChangedRevision$ +//$LastChangedBy$ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.MalformedURLException; +import java.util.Properties; + +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.data.TransactionManager; +import net.yacy.data.WorkTables; +import net.yacy.http.servlets.YaCyDefaultServlet; +import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; +import net.yacy.server.http.HTTPDFileHandler; + +public class ConfigPortal_p { + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + final serverObjects prop = new serverObjects(); + final Switchboard sb = (Switchboard) env; + + if (post != null) { + /* Check this is a valid transaction */ + TransactionManager.checkPostTransaction(header, post); + + if (post.containsKey("popup")) { + final String popup = post.get("popup", "status"); + if ("front".equals(popup)) { + sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html"); + } else if ("search".equals(popup)) { + sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "yacysearch.html"); + } else if ("interactive".equals(popup)) { + sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "yacyinteractive.html"); + } else { + sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "Status.html"); + } + sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html")); + HTTPDFileHandler.initDefaultPath(); + } + if (post.containsKey("searchpage_set")) { + final String newGreeting = post.get(SwitchboardConstants.GREETING, ""); + // store this call as api call + sb.tables.recordAPICall(post, "ConfigPortal_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting); + + sb.setConfig(SwitchboardConstants.GREETING, newGreeting); + sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, "")); + sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, post.get(SwitchboardConstants.GREETING_LARGE_IMAGE, "")); + sb.setConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, post.get(SwitchboardConstants.GREETING_SMALL_IMAGE, "")); + sb.setConfig(SwitchboardConstants.GREETING_IMAGE_ALT, post.get(SwitchboardConstants.GREETING_IMAGE_ALT, "")); + sb.setConfig(SwitchboardConstants.SEARCH_TARGET_DEFAULT, post.get("target", "_self")); + sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL, post.get("target_special", "_self")); + sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, post.get("target_special_pattern", "_self")); + sb.setConfig(SwitchboardConstants.SEARCH_ITEMS, post.getInt("maximumRecords", 10)); + sb.setConfig(SwitchboardConstants.INDEX_FORWARD, post.get(SwitchboardConstants.INDEX_FORWARD, "")); + HTTPDFileHandler.indexForward = post.get(SwitchboardConstants.INDEX_FORWARD, ""); + sb.setConfig("publicTopmenu", !post.containsKey("publicTopmenu") || post.getBoolean("publicTopmenu")); + sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, !post.containsKey(SwitchboardConstants.PUBLIC_SEARCHPAGE) || post.getBoolean(SwitchboardConstants.PUBLIC_SEARCHPAGE)); + sb.setConfig("search.options", post.getBoolean("search.options")); + + sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, post.getBoolean(SwitchboardConstants.GREEDYLEARNING_ACTIVE)); + + final boolean storeresult = post.getBoolean(SwitchboardConstants.REMOTESEARCH_RESULT_STORE); + sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, storeresult); + sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, post.getLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1)); + + sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, post.get("search.verify", "ifexist")); + sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, post.getBoolean("search.verify.delete")); + + sb.setConfig("about.headline", post.get("about.headline", "")); + sb.setConfig("about.body", post.get("about.body", "")); + + String excludehosts = post.get("search.excludehosts", ""); + sb.setConfig("search.excludehosts", excludehosts); + try { + sb.setConfig("search.excludehosth", DigestURL.hosthashes(excludehosts)); + } catch (MalformedURLException e) { + ConcurrentLog.logException(e); + sb.setConfig("search.excludehosth", ""); + } + } + if (post.containsKey("searchpage_default")) { + // load defaults from defaults/yacy.init file + final Properties config = new Properties(); + final String mes = "ConfigPortal"; + FileInputStream fis = null; + try { + fis = new FileInputStream(new File(sb.appPath, "defaults/yacy.init")); + config.load(fis); + } catch (final FileNotFoundException e) { + ConcurrentLog.severe(mes, "could not find configuration file."); + return prop; + } catch (final IOException e) { + ConcurrentLog.severe(mes, "could not read configuration file."); + return prop; + } finally { + if (fis != null) { + try { + fis.close(); + } catch (final IOException e) { + ConcurrentLog.logException(e); + } + } + } + sb.setConfig(SwitchboardConstants.GREETING, config.getProperty(SwitchboardConstants.GREETING,"P2P Web Search")); + sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, config.getProperty(SwitchboardConstants.GREETING_HOMEPAGE,"http://yacy.net")); + sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, config.getProperty(SwitchboardConstants.GREETING_LARGE_IMAGE,"env/grafics/YaCyLogo_120ppi.png")); + sb.setConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, config.getProperty(SwitchboardConstants.GREETING_SMALL_IMAGE,"env/grafics/YaCyLogo_60ppi.png")); + sb.setConfig(SwitchboardConstants.GREETING_IMAGE_ALT, config.getProperty(SwitchboardConstants.GREETING_IMAGE_ALT,"YaCy project web site")); + sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, config.getProperty(SwitchboardConstants.BROWSER_POP_UP_PAGE,"Status.html")); + sb.setConfig(SwitchboardConstants.INDEX_FORWARD, config.getProperty(SwitchboardConstants.INDEX_FORWARD,"")); + HTTPDFileHandler.indexForward = ""; + sb.setConfig(SwitchboardConstants.SEARCH_TARGET_DEFAULT, config.getProperty(SwitchboardConstants.SEARCH_TARGET_DEFAULT,"_self")); + sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL, config.getProperty(SwitchboardConstants.SEARCH_TARGET_SPECIAL,"_self")); + sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, config.getProperty(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN,"")); + sb.setConfig("publicTopmenu", config.getProperty("publicTopmenu","true")); + sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, config.getProperty(SwitchboardConstants.PUBLIC_SEARCHPAGE,"true")); + sb.setConfig("search.navigation", config.getProperty("search.navigation","hosts,authors,namespace,topics")); + sb.setConfig("search.options", config.getProperty("search.options","true")); + sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, config.getProperty(SwitchboardConstants.GREEDYLEARNING_ACTIVE)); + sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE)); + sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE)); + sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, config.getProperty(SwitchboardConstants.SEARCH_VERIFY,"iffresh")); + sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, config.getProperty(SwitchboardConstants.SEARCH_VERIFY_DELETE,"true")); + sb.setConfig("about.headline", config.getProperty("about.headline","")); + sb.setConfig("about.body", config.getProperty("about.body","")); + sb.setConfig("search.excludehosts", config.getProperty("search.excludehosts","")); + sb.setConfig("search.excludehosth", config.getProperty("search.excludehosth","")); + } + } + + /* Acquire a transaction token for the next POST form submission */ + prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header)); + + prop.putHTML(SwitchboardConstants.GREETING, sb.getConfig(SwitchboardConstants.GREETING, "")); + prop.putHTML(SwitchboardConstants.GREETING_HOMEPAGE, sb.getConfig(SwitchboardConstants.GREETING_HOMEPAGE, "")); + prop.putHTML(SwitchboardConstants.GREETING_LARGE_IMAGE, sb.getConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, "")); + prop.putHTML(SwitchboardConstants.GREETING_SMALL_IMAGE, sb.getConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, "")); + prop.putHTML(SwitchboardConstants.GREETING_IMAGE_ALT, sb.getConfig(SwitchboardConstants.GREETING_IMAGE_ALT, "")); + prop.putHTML(SwitchboardConstants.INDEX_FORWARD, sb.getConfig(SwitchboardConstants.INDEX_FORWARD, "")); + prop.put("publicTopmenu", sb.getConfigBool("publicTopmenu", false) ? 1 : 0); + prop.put(SwitchboardConstants.PUBLIC_SEARCHPAGE, sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, false) ? 1 : 0); + prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0); + + prop.put(SwitchboardConstants.GREEDYLEARNING_ACTIVE, sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ACTIVE, false) ? 1 : 0); + prop.put(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, sb.getConfig(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, "0")); + + prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, sb.getConfigBool(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, true) ? 1 : 0); + long resultStoredMaxSize = sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1); + if(resultStoredMaxSize > 0) { + prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, resultStoredMaxSize); + } else { + prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, ""); + } + + prop.put("search.verify.nocache", sb.getConfig("search.verify", "").equals("nocache") ? 1 : 0); + prop.put("search.verify.iffresh", sb.getConfig("search.verify", "").equals("iffresh") ? 1 : 0); + prop.put("search.verify.ifexist", sb.getConfig("search.verify", "").equals("ifexist") ? 1 : 0); + prop.put("search.verify.cacheonly", sb.getConfig("search.verify", "").equals("cacheonly") ? 1 : 0); + prop.put("search.verify.false", sb.getConfig("search.verify", "").equals("false") ? 1 : 0); + prop.put("search.verify.delete", sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true) ? 1 : 0); + + prop.put("about.headline", sb.getConfig("about.headline", "")); + prop.put("about.body", sb.getConfig("about.body", "")); + + prop.put("search.excludehosts", sb.getConfig("search.excludehosts", "")); + prop.put("search.excludehosth", sb.getConfig("search.excludehosth", "")); + + final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html"); + prop.put("popupFront", 0); + prop.put("popupSearch", 0); + prop.put("popupInteractive", 0); + prop.put("popupStatus", 0); + if (browserPopUpPage.startsWith("index")) { + prop.put("popupFront", 1); + } else if (browserPopUpPage.startsWith("yacysearch")) { + prop.put("popupSearch", 1); + } else if (browserPopUpPage.startsWith("yacyinteractive")) { + prop.put("popupInteractive", 1); + } else { + prop.put("popupStatus", 1); + } + + prop.put("maximumRecords", sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10)); + + final String target = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_DEFAULT, "_self"); + prop.put("target_selected_blank", "_blank".equals(target) ? 1 : 0); + prop.put("target_selected_self", "_self".equals(target) ? 1 : 0); + prop.put("target_selected_parent", "_parent".equals(target) ? 1 : 0); + prop.put("target_selected_top", "_top".equals(target) ? 1 : 0); + prop.put("target_selected_searchresult", "searchresult".equals(target) ? 1 : 0); + + final String target_special = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL, "_self"); + prop.put("target_selected_special_blank", "_blank".equals(target_special) ? 1 : 0); + prop.put("target_selected_special_self", "_self".equals(target_special) ? 1 : 0); + prop.put("target_selected_special_parent", "_parent".equals(target_special) ? 1 : 0); + prop.put("target_selected_special_top", "_top".equals(target_special) ? 1 : 0); + prop.put("target_selected_special_searchresult", "searchresult".equals(target_special) ? 1 : 0); + prop.put("target_special_pattern", sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, "")); + + prop.put("myContext", YaCyDefaultServlet.getContext(header, sb)); + return prop; + } + +} diff --git a/htroot/ConfigSearchPage_p.html b/htroot/ConfigSearchPage_p.html index 9a576f440..fa133af3b 100644 --- a/htroot/ConfigSearchPage_p.html +++ b/htroot/ConfigSearchPage_p.html @@ -17,7 +17,7 @@

Below is a generic template of the search result page. Mark the check boxes for features you would like to be displayed. To change colors and styles use the Appearance menu for different skins. - Other portal settings can be adjusted in Generic Search Portal menu. + Other portal settings can be adjusted in Generic Search Portal menu.

Page Template

diff --git a/htroot/ConfigSearchPage_p.java b/htroot/ConfigSearchPage_p.java index c46ac9d93..54dc08d2a 100644 --- a/htroot/ConfigSearchPage_p.java +++ b/htroot/ConfigSearchPage_p.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.sql.Date; import java.util.Map; import java.util.Properties; -import java.util.TreeMap; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; @@ -62,7 +61,7 @@ public class ConfigSearchPage_p { if (post.containsKey("searchpage_set")) { final String newGreeting = post.get(SwitchboardConstants.GREETING, ""); // store this call as api call - sb.tables.recordAPICall(post, "ConfigPortal.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting); + sb.tables.recordAPICall(post, "ConfigPortal_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting); sb.setConfig("publicTopmenu", post.getBoolean("publicTopmenu")); sb.setConfig("search.options", post.getBoolean("search.options")); diff --git a/htroot/IndexImportOAIPMH_p.html b/htroot/IndexImportOAIPMH_p.html index 671d3acdd..84c9da15f 100644 --- a/htroot/IndexImportOAIPMH_p.html +++ b/htroot/IndexImportOAIPMH_p.html @@ -19,12 +19,12 @@ #(import-one)#:: -

+
Source:
#[source]#
Processed:
#[count]# records
ResumptionToken:
#[rt]#
-

- ::

Import failed: #[error]#

+
+ ::

Import failed: #[error]#

#(/import-one)#
diff --git a/htroot/IndexImportWarc_p.html b/htroot/IndexImportWarc_p.html new file mode 100644 index 000000000..d6003bc9e --- /dev/null +++ b/htroot/IndexImportWarc_p.html @@ -0,0 +1,51 @@ + + + + YaCy '#[clientname]#': Warc Import + #%env/templates/metas.template%# + #(import)#:: + + #(/import)# + + + #%env/templates/header.template%# + #%env/templates/submenuIndexImport.template%# +

Web Archive File Import

+ + #(import)# +

No import thread is running, you can start a new thread here

+
+ +
+ Warc File Selection: select an warc file (which may be gz compressed) +

+ You can download warc archives for example here + Internet Archive. +

+
+ + +
+ +
+
+
+
+ +
+ :: +
Import Process +
+
Thread:
#[thread]#
+
Warc File:
#[warcfile]#
+
Processed:
#[count]# Entries
+
Speed:
#[speed]# pages per second
+
Running Time:
#[runningHours]# hours, #[runningMinutes]# minutes
+
Remaining Time:
#[remainingHours]# hours, #[remainingMinutes]# minutes
+
+
+ #(/import)# + + #%env/templates/footer.template%# + + \ No newline at end of file diff --git a/htroot/IndexImportWarc_p.java b/htroot/IndexImportWarc_p.java new file mode 100644 index 000000000..f503fe98b --- /dev/null +++ b/htroot/IndexImportWarc_p.java @@ -0,0 +1,76 @@ +// IndexImportWarc_p.java +// ------------------------- +// (c) 2017 by reger24; https://github.com/reger24 +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import java.io.File; +import java.io.FileNotFoundException; + +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.document.importer.WarcImporter; +import net.yacy.search.Switchboard; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; + +public class IndexImportWarc_p { + + public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + final serverObjects prop = new serverObjects(); + final Switchboard sb = (Switchboard) env; + + if (WarcImporter.job != null && WarcImporter.job.isAlive()) { + // one import is running, no option to insert anything + prop.put("import", 1); + prop.put("import_thread", "running"); + prop.put("import_warcfile", WarcImporter.job.source()); + prop.put("import_count", WarcImporter.job.count()); + prop.put("import_speed", WarcImporter.job.speed()); + prop.put("import_runningHours", (WarcImporter.job.runningTime() / 60) / 60); + prop.put("import_runningMinutes", (WarcImporter.job.runningTime() / 60) % 60); + prop.put("import_remainingHours", (WarcImporter.job.remainingTime() / 60) / 60); + prop.put("import_remainingMinutes", (WarcImporter.job.remainingTime() / 60) % 60); + } else { + prop.put("import", 0); + if (post != null) { + if (post.containsKey("file")) { + String file = post.get("file"); + final File sourcefile = new File(file); + if (sourcefile.exists()) { + try { + WarcImporter wi = new WarcImporter(sourcefile); + wi.start(); + prop.put("import_thread", "started"); + } catch (FileNotFoundException ex) { + prop.put("import_thread", "Error: file not found [" + file + "]"); + } + prop.put("import_warcfile", file); + } else { + prop.put("import_warcfile", ""); + prop.put("import_thread", "Error: file not found [" + file + "]"); + } + prop.put("import", 1); + prop.put("import_count", 0); + prop.put("import_speed", 0); + prop.put("import_runningHours", 0); + prop.put("import_runningMinutes", 0); + prop.put("import_remainingHours", 0); + prop.put("import_remainingMinutes", 0); + } + } + } + return prop; + } +} diff --git a/htroot/Table_API_p.html b/htroot/Table_API_p.html index 230d5b013..8ad8f4f94 100644 --- a/htroot/Table_API_p.html +++ b/htroot/Table_API_p.html @@ -46,6 +46,7 @@ To see a list of all APIs, please visit the +
Recorded Actions
diff --git a/htroot/Table_API_p.java b/htroot/Table_API_p.java index 43ee5bca3..17a51a124 100644 --- a/htroot/Table_API_p.java +++ b/htroot/Table_API_p.java @@ -35,6 +35,7 @@ import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.data.TransactionManager; import net.yacy.data.WorkTables; import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.blob.Tables.Row; @@ -46,10 +47,10 @@ import net.yacy.server.serverSwitch; public class Table_API_p { - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard) env; final serverObjects prop = new serverObjects(); - + prop.put("showexec", 0); prop.put("showtable", 0); @@ -85,6 +86,10 @@ public class Table_API_p { current_pk = post.get("current_pk", ""); } if (post != null && scheduleeventaction && !current_pk.isEmpty()) { + + /* Check this is a valid transaction */ + TransactionManager.checkPostTransaction(header, post); + try { Tables.Row row = sb.tables.select(WorkTables.TABLE_API_NAME, current_pk.getBytes()); if (row != null) { @@ -150,6 +155,10 @@ public class Table_API_p { } if (post != null && !post.get("deleterows", "").isEmpty()) { + + /* Check this is a valid transaction */ + TransactionManager.checkPostTransaction(header, post); + for (final Map.Entry entry : post.entrySet()) { if (entry.getValue().startsWith("mark_")) { try { @@ -162,6 +171,10 @@ public class Table_API_p { } if (post != null && !post.get("deleteold", "").isEmpty()) { + + /* Check this is a valid transaction */ + TransactionManager.checkPostTransaction(header, post); + int days = post.getInt("deleteoldtime", 365); try { Iterator ri = sb.tables.iterator(WorkTables.TABLE_API_NAME); @@ -199,6 +212,10 @@ public class Table_API_p { } if (post != null && !post.get("execrows", "").isEmpty()) { + + /* Check this is a valid transaction */ + TransactionManager.checkPostTransaction(header, post); + // create a time-ordered list of events to execute final Set pks = new TreeSet(); for (final Map.Entry entry : post.entrySet()) { @@ -234,6 +251,11 @@ public class Table_API_p { // generate table prop.put("showtable", 1); prop.put("showtable_inline", inline ? 1 : 0); + + /* Acquire a transaction token for the next POST form submission */ + final String nextTransactionToken = TransactionManager.getTransactionToken(header); + prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken); + prop.put("showtable_" + TransactionManager.TRANSACTION_TOKEN_PARAM, nextTransactionToken); // insert rows final List table = new ArrayList(maximumRecords); @@ -287,7 +309,7 @@ public class Table_API_p { prop.put("showtable_list_" + count + "_dateLastExec", date_last_exec == null ? "-" : GenericFormatter.FORMAT_SIMPLE.format(date_last_exec)); prop.put("showtable_list_" + count + "_dateNextExec", date_next_exec == null ? "-" : GenericFormatter.FORMAT_SIMPLE.format(date_next_exec)); prop.put("showtable_list_" + count + "_type", row.get(WorkTables.TABLE_API_COL_TYPE)); - prop.put("showtable_list_" + count + "_comment", row.get(WorkTables.TABLE_API_COL_COMMENT)); + prop.putHTML("showtable_list_" + count + "_comment", row.get(WorkTables.TABLE_API_COL_COMMENT)); // check type & action to link crawl start URLs back to CrawlStartExpert.html if (prop.get("showtable_list_" + count + "_type", "").equals(WorkTables.TABLE_API_TYPE_CRAWLER) && prop.get("showtable_list_" + count + "_comment", "").startsWith("crawl start for")) { diff --git a/htroot/Translator_p.html b/htroot/Translator_p.html index 79b0deeca..e4925791e 100644 --- a/htroot/Translator_p.html +++ b/htroot/Translator_p.html @@ -13,6 +13,7 @@

Translate untranslated text of the user interface (current language). The modified translation file is stored in DATA/LOCALE directory.

+
diff --git a/htroot/Translator_p.java b/htroot/Translator_p.java index 31d97760f..ffbd14dc2 100644 --- a/htroot/Translator_p.java +++ b/htroot/Translator_p.java @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.Map; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.data.TransactionManager; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; @@ -33,7 +34,7 @@ import net.yacy.utils.translation.TranslationManager; public class Translator_p { - public static servletProperties respond(@SuppressWarnings("unused") final RequestHeader requestHeader, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { + public static servletProperties respond(final RequestHeader requestHeader, final serverObjects post, final serverSwitch env) { try { final servletProperties prop = new servletProperties(); final Switchboard sb = (Switchboard) env; @@ -112,6 +113,9 @@ public class Translator_p { } // handle (modified) input text if (i == textlistid && post != null) { + /* Check this is a valid transaction */ + TransactionManager.checkPostTransaction(requestHeader, post); + if (editapproved) { // switch already translated in edit mode by copying to local translation // not saved here as not yet modified/approved localTransMgr.addTranslation(localTrans, filename, sourcetext, targettxt); @@ -138,6 +142,9 @@ public class Translator_p { changed = true; } if (changed) { + /* Check this is a valid transaction */ + TransactionManager.checkPostTransaction(requestHeader, post); + localTransMgr.saveAsLngFile(langcfg, locallngfile, localTrans); // adhoc translate this file // 1. get/calc the path @@ -150,6 +157,10 @@ public class Translator_p { localTransMgr.translateFile(sourceFile, destFile, origTextList); // do the translation } } + + /* Acquire a transaction token for the next POST form submission */ + prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(requestHeader)); + prop.put("textlist", i); return prop; } catch (IOException ex) { diff --git a/htroot/env/templates/header.template b/htroot/env/templates/header.template index 9b684a293..32e70cc59 100644 --- a/htroot/env/templates/header.template +++ b/htroot/env/templates/header.template @@ -176,7 +176,7 @@ diff --git a/htroot/env/templates/submenuIndexImport.template b/htroot/env/templates/submenuIndexImport.template index b44d9d95a..d85b313c2 100644 --- a/htroot/env/templates/submenuIndexImport.template +++ b/htroot/env/templates/submenuIndexImport.template @@ -13,6 +13,7 @@ diff --git a/htroot/env/templates/submenuPortalConfiguration.template b/htroot/env/templates/submenuPortalConfiguration.template index 896f9b592..21b177ab7 100644 --- a/htroot/env/templates/submenuPortalConfiguration.template +++ b/htroot/env/templates/submenuPortalConfiguration.template @@ -1,7 +1,7 @@

Portal Configuration

    -
  • Generic Search Portal
    • +
  • Generic Search Portal
  • Search Box Anywhere
  • User Profile
  • Local robots.txt
    • diff --git a/lib/fontbox-2.0.4.License b/lib/fontbox-2.0.5.License similarity index 100% rename from lib/fontbox-2.0.4.License rename to lib/fontbox-2.0.5.License diff --git a/lib/fontbox-2.0.4.jar b/lib/fontbox-2.0.5.jar similarity index 92% rename from lib/fontbox-2.0.4.jar rename to lib/fontbox-2.0.5.jar index 43cf6b1e0..b1d1a4e56 100644 Binary files a/lib/fontbox-2.0.4.jar and b/lib/fontbox-2.0.5.jar differ diff --git a/lib/lucene-analyzers-common-5.5.3.jar b/lib/lucene-analyzers-common-5.5.4.jar similarity index 94% rename from lib/lucene-analyzers-common-5.5.3.jar rename to lib/lucene-analyzers-common-5.5.4.jar index 523032cfe..6c490d421 100644 Binary files a/lib/lucene-analyzers-common-5.5.3.jar and b/lib/lucene-analyzers-common-5.5.4.jar differ diff --git a/lib/lucene-analyzers-phonetic-5.5.3.jar b/lib/lucene-analyzers-phonetic-5.5.4.jar similarity index 89% rename from lib/lucene-analyzers-phonetic-5.5.3.jar rename to lib/lucene-analyzers-phonetic-5.5.4.jar index 1595657c6..9e29e004d 100644 Binary files a/lib/lucene-analyzers-phonetic-5.5.3.jar and b/lib/lucene-analyzers-phonetic-5.5.4.jar differ diff --git a/lib/lucene-backward-codecs-5.5.3.jar b/lib/lucene-backward-codecs-5.5.4.jar similarity index 92% rename from lib/lucene-backward-codecs-5.5.3.jar rename to lib/lucene-backward-codecs-5.5.4.jar index ab482cc08..76e02b103 100644 Binary files a/lib/lucene-backward-codecs-5.5.3.jar and b/lib/lucene-backward-codecs-5.5.4.jar differ diff --git a/lib/lucene-classification-5.5.3.jar b/lib/lucene-classification-5.5.4.jar similarity index 92% rename from lib/lucene-classification-5.5.3.jar rename to lib/lucene-classification-5.5.4.jar index f1db172eb..0a9cfb498 100644 Binary files a/lib/lucene-classification-5.5.3.jar and b/lib/lucene-classification-5.5.4.jar differ diff --git a/lib/lucene-codecs-5.5.3.jar b/lib/lucene-codecs-5.5.4.jar similarity index 93% rename from lib/lucene-codecs-5.5.3.jar rename to lib/lucene-codecs-5.5.4.jar index 476d4d37f..6653b8244 100644 Binary files a/lib/lucene-codecs-5.5.3.jar and b/lib/lucene-codecs-5.5.4.jar differ diff --git a/lib/lucene-core-5.5.3.jar b/lib/lucene-core-5.5.4.jar similarity index 88% rename from lib/lucene-core-5.5.3.jar rename to lib/lucene-core-5.5.4.jar index ffb39a6d0..311992ca0 100644 Binary files a/lib/lucene-core-5.5.3.jar and b/lib/lucene-core-5.5.4.jar differ diff --git a/lib/lucene-facet-5.5.3.jar b/lib/lucene-facet-5.5.4.jar similarity index 91% rename from lib/lucene-facet-5.5.3.jar rename to lib/lucene-facet-5.5.4.jar index 885c92f3f..967b2a609 100644 Binary files a/lib/lucene-facet-5.5.3.jar and b/lib/lucene-facet-5.5.4.jar differ diff --git a/lib/lucene-grouping-5.5.3.jar b/lib/lucene-grouping-5.5.4.jar similarity index 71% rename from lib/lucene-grouping-5.5.3.jar rename to lib/lucene-grouping-5.5.4.jar index 8a64b0f3c..c7749aadf 100644 Binary files a/lib/lucene-grouping-5.5.3.jar and b/lib/lucene-grouping-5.5.4.jar differ diff --git a/lib/lucene-highlighter-5.5.3.jar b/lib/lucene-highlighter-5.5.4.jar similarity index 69% rename from lib/lucene-highlighter-5.5.3.jar rename to lib/lucene-highlighter-5.5.4.jar index 079491575..b31415e2a 100644 Binary files a/lib/lucene-highlighter-5.5.3.jar and b/lib/lucene-highlighter-5.5.4.jar differ diff --git a/lib/lucene-join-5.5.3.jar b/lib/lucene-join-5.5.4.jar similarity index 68% rename from lib/lucene-join-5.5.3.jar rename to lib/lucene-join-5.5.4.jar index a78d60d13..103b682b2 100644 Binary files a/lib/lucene-join-5.5.3.jar and b/lib/lucene-join-5.5.4.jar differ diff --git a/lib/lucene-memory-5.5.3.jar b/lib/lucene-memory-5.5.4.jar similarity index 91% rename from lib/lucene-memory-5.5.3.jar rename to lib/lucene-memory-5.5.4.jar index 5dbae077f..e3ff559b8 100644 Binary files a/lib/lucene-memory-5.5.3.jar and b/lib/lucene-memory-5.5.4.jar differ diff --git a/lib/lucene-misc-5.5.3.jar b/lib/lucene-misc-5.5.4.jar similarity index 91% rename from lib/lucene-misc-5.5.3.jar rename to lib/lucene-misc-5.5.4.jar index 6b9e3cd05..390b80574 100644 Binary files a/lib/lucene-misc-5.5.3.jar and b/lib/lucene-misc-5.5.4.jar differ diff --git a/lib/lucene-queries-5.5.3.jar b/lib/lucene-queries-5.5.4.jar similarity index 89% rename from lib/lucene-queries-5.5.3.jar rename to lib/lucene-queries-5.5.4.jar index a5f7be8be..c8fecce21 100644 Binary files a/lib/lucene-queries-5.5.3.jar and b/lib/lucene-queries-5.5.4.jar differ diff --git a/lib/lucene-queryparser-5.5.3.jar b/lib/lucene-queryparser-5.5.4.jar similarity index 90% rename from lib/lucene-queryparser-5.5.3.jar rename to lib/lucene-queryparser-5.5.4.jar index cbd9b10a8..22e280345 100644 Binary files a/lib/lucene-queryparser-5.5.3.jar and b/lib/lucene-queryparser-5.5.4.jar differ diff --git a/lib/lucene-spatial-5.5.3.jar b/lib/lucene-spatial-5.5.4.jar similarity index 91% rename from lib/lucene-spatial-5.5.3.jar rename to lib/lucene-spatial-5.5.4.jar index 65629d546..e8ea7c930 100644 Binary files a/lib/lucene-spatial-5.5.3.jar and b/lib/lucene-spatial-5.5.4.jar differ diff --git a/lib/lucene-suggest-5.5.3.jar b/lib/lucene-suggest-5.5.4.jar similarity index 90% rename from lib/lucene-suggest-5.5.3.jar rename to lib/lucene-suggest-5.5.4.jar index d947a22aa..1ad6b5a48 100644 Binary files a/lib/lucene-suggest-5.5.3.jar and b/lib/lucene-suggest-5.5.4.jar differ diff --git a/lib/pdfbox-2.0.4.License b/lib/pdfbox-2.0.5.License similarity index 100% rename from lib/pdfbox-2.0.4.License rename to lib/pdfbox-2.0.5.License diff --git a/lib/pdfbox-2.0.4.jar b/lib/pdfbox-2.0.5.jar similarity index 80% rename from lib/pdfbox-2.0.4.jar rename to lib/pdfbox-2.0.5.jar index 4f675dd81..f3776b61c 100644 Binary files a/lib/pdfbox-2.0.4.jar and b/lib/pdfbox-2.0.5.jar differ diff --git a/lib/solr-core-5.5.3.jar b/lib/solr-core-5.5.4.jar similarity index 92% rename from lib/solr-core-5.5.3.jar rename to lib/solr-core-5.5.4.jar index d2e2dafce..4a560a3ae 100644 Binary files a/lib/solr-core-5.5.3.jar and b/lib/solr-core-5.5.4.jar differ diff --git a/lib/solr-solrj-5.5.3.jar b/lib/solr-solrj-5.5.4.jar similarity index 91% rename from lib/solr-solrj-5.5.3.jar rename to lib/solr-solrj-5.5.4.jar index 26512a529..17e495f90 100644 Binary files a/lib/solr-solrj-5.5.3.jar and b/lib/solr-solrj-5.5.4.jar differ diff --git a/lib/xmpcore-5.1.2.jar b/lib/xmpcore-5.1.2.jar deleted file mode 100644 index ecd5db142..000000000 Binary files a/lib/xmpcore-5.1.2.jar and /dev/null differ diff --git a/lib/xmpcore-5.1.3.jar b/lib/xmpcore-5.1.3.jar new file mode 100644 index 000000000..dfd0f37cd Binary files /dev/null and b/lib/xmpcore-5.1.3.jar differ diff --git a/lib/xmpcore-5.1.2.license b/lib/xmpcore-5.1.3.license similarity index 100% rename from lib/xmpcore-5.1.2.license rename to lib/xmpcore-5.1.3.license diff --git a/locales/cn.lng b/locales/cn.lng index 18f58ec16..838303423 100644 --- a/locales/cn.lng +++ b/locales/cn.lng @@ -342,8 +342,8 @@ Appearance and Integration==外观界面 You can change the appearance of the YaCy interface with skins.==您可以在这里修改YaCy的外观界面. #You can change the appearance of YaCy with skins==Sie können hier das Erscheinungsbild von YaCy mit Skins ändern The selected skin and language also affects the appearance of the search page.==选择的皮肤和语言也会影响到搜索页面的外观. -If you create a search portal with YaCy then you can==如果您创建YaCy门户, -change the appearance of the search page here.==那么您能在这里 改变搜索页面的外观. +If you create a search portal with YaCy then you can==如果您创建YaCy门户, +change the appearance of the search page here.==那么您能在这里 改变搜索页面的外观. #and the default icons and links on the search page can be replaced with you own.==und die standard Grafiken und Links auf der Suchseite durch Ihre eigenen ersetzen. Skin Selection==选择皮肤 Select one of the default skins, download new skins, or create your own skin.==选择一个默认皮肤, 下载新皮肤或者创建属于您自己的皮肤. @@ -619,7 +619,7 @@ enable/disable Parser==Parser 开启 / 关闭 "Submit"=="提交" #----------------------------- -#File: ConfigPortal.html +#File: ConfigPortal_p.html #--------------------------- Integration of a Search Portal==搜索门户设置 If you like to integrate YaCy as portal for your web pages, you may want to change icons and messages on the search page.==如果您想将YaCy作为您的网站搜索门户, 您可能需要在这改变搜索页面的图标和信息. diff --git a/locales/de.lng b/locales/de.lng index 98c4477c3..2827f5aca 100644 --- a/locales/de.lng +++ b/locales/de.lng @@ -383,8 +383,8 @@ Save User==Benutzer speichern Appearance and Integration==Erscheinungsbild und Integration You can change the appearance of the YaCy interface with skins.==Sie können hier das Aussehen der YaCy Oberfläche mit Skins verändern, The selected skin and language also affects the appearance of the search page.==das ausgewählte Design und die gewählte Sprache wirken sich auch auf das Erscheinungsbild der Suchseite aus. -If you create a search portal with YaCy then you can==Wenn Sie ein Such-Portal mit Yacy erstellen möchten können Sie -change the appearance of the search page here.==das Erscheinungsbild der Suchseite hier weitgehend ändern und die standard Grafiken und Links auf der Suchseite durch Ihre eigenen ersetzen. +If you create a search portal with YaCy then you can==Wenn Sie ein Such-Portal mit Yacy erstellen möchten können Sie +change the appearance of the search page here.==das Erscheinungsbild der Suchseite hier weitgehend ändern und die standard Grafiken und Links auf der Suchseite durch Ihre eigenen ersetzen. #and the default icons and links on the search page can be replaced with you own.==und die standard Grafiken und Links auf der Suchseite durch Ihre eigenen ersetzen. Skin Selection==Skinauswahl Select one of the default skins, download new skins, or create your own skin.==Wählen Sie einen der mitgelieferten Skins, laden Sie einen neuen herunter oder erstellen Sie selbst einen neuen Skin. @@ -708,7 +708,7 @@ If you want to test a specific parser you can do so using the==Wenn Sie einen be "Submit"=="Speichern" #----------------------------- -#File: ConfigPortal.html +#File: ConfigPortal_p.html #--------------------------- Integration of a Search Portal==Integration eines Suchportals If you like to integrate YaCy as portal for your web pages, you may want to change icons and messages on the search page.==Wenn Sie YaCy als Suchportal für Ihre Webseiten integrieren wollen, können Sie auch die Icons und Nachrichten auf den Suchseiten verändern. diff --git a/locales/fr.lng b/locales/fr.lng index 55a5872b2..2c94223e9 100644 --- a/locales/fr.lng +++ b/locales/fr.lng @@ -334,7 +334,7 @@ Set Access Rules==Définir les règles d'accès Appearance and Integration==Apparence et intégration You can change the appearance of the YaCy interface with skins.==Vous pouvez personnaliser l'interface YaCy avec des thèmes. The selected skin and language also affects the appearance of the search page.==Le thème et la langue sélectionnés affectent aussi l'aspect de la page de recherche. -If you create a search portal with YaCy then you can==Si vous utilisez YaCy pour créer un portail de recherche vous pouvez alors +If you create a search portal with YaCy then you can==Si vous utilisez YaCy pour créer un portail de recherche vous pouvez alors change the appearance of the search page here.==modifier l'aspect de la page de recherche ici. Skin Selection==Sélection du thème Select one of the default skins, download new skins, or create your own skin.==Sélectionnez l'un des thèmes par défaut, téléchargez de nouveaux thèmes ou créez le vôtre. diff --git a/locales/hi.lng b/locales/hi.lng index 5277f8a31..b02f965aa 100644 --- a/locales/hi.lng +++ b/locales/hi.lng @@ -328,7 +328,7 @@ Appearance and Integration==अप्पेअरेञ्चे और इं You can change the appearance of the YaCy interface with skins.==Yacy के इंटरफ़ेस और स्किन्स को आप बदल सकते हैं, #You can change the appearance of YaCy with skins==Yacy के इंटरफ़ेस और स्किन्स को आप बदल सकते हैं, The selected skin and language also affects the appearance of the search page.==चयनित स्किन्स और भाषा भी सर्च पेज की अपीयरेंस को प्रभावित करता है. -If you create a search portal with YaCy then you can==अगर आप YaCy साथ एक सर्च पोर्टल बनाते हैं तो आप +If you create a search portal with YaCy then you can==अगर आप YaCy साथ एक सर्च पोर्टल बनाते हैं तो आप change the appearance of the search page here.==यहाँ सर्च पेज के स्वरूप को बदल सकते हैं. #and the default icons and links on the search page can be replaced with you own.==और सर्च पेज पर डिफ़ॉल्ट चिह्न और लिंक आप के अनुसार बदला जा सकता है Skin Selection==स्किन चयन @@ -613,7 +613,7 @@ BMP Image Parser==बीएमपी छवि पार्सर "Submit"==" सबमिट " #----------------------------- -#File: ConfigPortal.html +#File: ConfigPortal_p.html #--------------------------- Integration of a Search Portal== एक खोज पोर्टल की इंटीग्रेशन If you like to integrate YaCy as portal for your web pages, you may want to change icons and messages on the search page.== यदि आप अपने वेब पृष्ठों के लिए पोर्टल के रूप में याची एकीकृत करना चाहते हैं, तो आप खोज पृष्ठ पर माउस और संदेशों को बदलने के लिए चाह सकते हैं. diff --git a/locales/ja.lng b/locales/ja.lng index 9e3cbd29e..91b1125e5 100644 --- a/locales/ja.lng +++ b/locales/ja.lng @@ -403,7 +403,7 @@ Appearance and Integration==外観と統合 You can change the appearance of the YaCy interface with skins.==あなたはスキンによってYaCyのインターフェースの外観を変更する事ができます. #You can change the appearance of YaCy with skins==あなたはスキンによってYaCyのインターフェースの外観を変更する事ができます The selected skin and language also affects the appearance of the search page.==選択されたスキンと言語は検索ページの外観にも影響を与えます. -If you create a search portal with YaCy then you can==もしあなたがYaCyで検索ポータルを作成するならば +If you create a search portal with YaCy then you can==もしあなたがYaCyで検索ポータルを作成するならば change the appearance of the search page here.==ここで検索ページの外観を変更できます. #and the default icons and links on the search page can be replaced with you own.==そして検索ページの既定のアイコンとリンクを自分のもので置き換える事ができます. Skin Selection==スキンの選択 @@ -554,7 +554,7 @@ Content Parser Settings==コンテントの構文解析器の構成 "Submit"=="確定" #----------------------------- -#File: ConfigPortal.html +#File: ConfigPortal_p.html #--------------------------- Integration of a Search Portal==検索ポータルの統合 #----------------------------- @@ -619,7 +619,7 @@ Search Page<==検索ページ< >Search Result Page Layout Configuration<==>検索結果のページのレイアウトの構成< >Appearance<==>外観< menu for different skins.== 別のスキンの為のメニュー. -Other portal settings can be adjusted in Generic Search Portal menu.==他のポータルの設定は一般的な検索ポータル メニューで調整できます +Other portal settings can be adjusted in Generic Search Portal menu.==他のポータルの設定は一般的な検索ポータル メニューで調整できます >Page Template<==>ページのテンプレート< #>Administration<==>管理< >Web Search<==>ウェブ検索< diff --git a/locales/master.lng.xlf b/locales/master.lng.xlf index f1c6d09cc..6ca2b5827 100644 --- a/locales/master.lng.xlf +++ b/locales/master.lng.xlf @@ -948,7 +948,7 @@ The selected skin and language also affects the appearance of the search page. - If you <a href="ConfigPortal.html">create a search portal with YaCy</a> then you can + If you <a href="ConfigPortal_p.html">create a search portal with YaCy</a> then you can change the appearance of the search page here. @@ -1685,7 +1685,7 @@ - + Integration of a Search Portal @@ -2659,6 +2659,47 @@ + + + + Cookie - Test Page + + + Here is a cookie test page. + + + Just clean it + + + Name: + + + Value: + + + Dear server, set this cookie for me! + + + Cookies at this browser: + + + Cookies coming to server: + + + Cookies server sent: + + + YaCy is a GPL'ed project + + + with the target of implementing a P2P-based global search engine. + + + Architecture (C) by + + + + @@ -4704,6 +4745,9 @@ (Rich and full-text Solr data, one document per line in one large xml file, can be processed with shell tools, can be imported with DATA/SURROGATE/in/) + + (Rich and full-text Elasticsearch data, one document per line in one flat JSON file, can be bulk-imported to elasticsearch with the command "curl -XPOST localhost:9200/collection1/yacy/_bulk --data-binary @yacy_dump_XXX.flatjson") + Full URL List: @@ -5036,6 +5080,65 @@ + + + + Warc Import + + + Web Archive File Import + + + No import thread is running, you can start a new thread here + + + Warc File Selection: select an warc file (which may be gz compressed) + + + You can download warc archives for example here + + + Internet Archive + + + Import Warc File + + + Import Process + + + Thread: + + + Warc File: + + + Processed: + + + Entries + + + Speed: + + + pages per second + + + Running Time: + + + hours, + + + minutes< + + + Remaining Time: + + + + @@ -8941,6 +9044,9 @@ Save translation + + Check for remote translation proposals and/or share your own added translations + diff --git a/locales/ru.lng b/locales/ru.lng index 2cb843f9e..7635bfe7e 100644 --- a/locales/ru.lng +++ b/locales/ru.lng @@ -420,7 +420,7 @@ Appearance and Integration==Внешний вид You can change the appearance of the YaCy interface with skins.==Вы можете изменить внешний вид YaCy используя скины. #You can change the appearance of YaCy with skins==Вы можете изменить внешний вид YaCy со скинами The selected skin and language also affects the appearance of the search page.==Выбранный скин и язык, также влияют на внешний вид страницы поиска. -If you create a search portal with YaCy then you can==Если вы создадите поиск с Yacy, +If you create a search portal with YaCy then you can==Если вы создадите поиск с Yacy, change the appearance of the search page here.==то изменить внешний вид страницы поиска вы можете здесь. #and the default icons and links on the search page can be replaced with you own.==и стандартные иконки и ссылки на странице поиска будут заменены на ваши. Skin Selection==Выбор скина @@ -806,7 +806,7 @@ If you want to test a specific parser you can do so using the==Если вы ж "Submit"=="Установить" #----------------------------- -#File: ConfigPortal.html +#File: ConfigPortal_p.html #--------------------------- Integration of a Search Portal==Интеграция поиска If you like to integrate YaCy as portal for your web pages, you may want to change icons and messages on the search page.==Если вы хотите интегрировать YaCy как поиск для веб-страниц, то вы можете менять иконки и сообщения на странице поиска. diff --git a/locales/uk.lng b/locales/uk.lng index f8ba64bb3..aa2634752 100644 --- a/locales/uk.lng +++ b/locales/uk.lng @@ -341,8 +341,8 @@ Appearance and Integration==Зовнішній вигляд та інтегра You can change the appearance of the YaCy interface with skins.==Ви можете змінити зовнішній вигляд інтерфейсу YaCy з допомогою шкури. #You can change the appearance of YaCy with skins==Ви можете змінити зовнішній вигляд YaCy зі шкурами The selected skin and language also affects the appearance of the search page.==Вибрані дизайн і мова також впливають на зовнішній вигляд сторінки пошуку. -If you create a search portal with YaCy then you can==Якщо ви створюєте пошуковий портал з Yacy, то можете -change the appearance of the search page here.==змінити появу пошукової сторінки тут, а також змінити графіку та стиль посилань пошукової сторінки на власні. +If you create a search portal with YaCy then you can==Якщо ви створюєте пошуковий портал з Yacy, то можете +change the appearance of the search page here.==змінити появу пошукової сторінки тут, а також змінити графіку та стиль посилань пошукової сторінки на власні. #and the default icons and links on the search page can be replaced with you own.==і стандартні іконки та посилання на сторінці пошуку можуть бути замінені на власні. Skin Selection==Вибір шкурки Select one of the default skins, download new skins, or create your own skin.==Виберіть один з існуючих скінів, завантажте новий або створіть для себе новий. @@ -655,7 +655,7 @@ enable/disable==ввімкнути/вимкнути >Extension==>Розширення #----------------------------- -#File: ConfigPortal.html +#File: ConfigPortal_p.html #--------------------------- Integration of a Search Portal==Інтегрування пошукового порталу If you like to integrate YaCy as portal for your web pages, you may want to change icons and messages on the search page.==Якщо ви хочете вбудувати YaCy в якості пошукового порталу у веб-сайт, ви також можете змінити значки та повідомлення на сторінках пошуку. diff --git a/pom.xml b/pom.xml index 19272fea2..70fd8b4a5 100644 --- a/pom.xml +++ b/pom.xml @@ -34,7 +34,7 @@ 1.7 1.7 - 5.5.3 + 5.5.4 9.2.21.v20170120 @@ -526,7 +526,7 @@ org.apache.pdfbox pdfbox - 2.0.4 + 2.0.5 jar diff --git a/source/net/yacy/document/importer/WarcImporter.java b/source/net/yacy/document/importer/WarcImporter.java index e3463a02c..e921765ce 100644 --- a/source/net/yacy/document/importer/WarcImporter.java +++ b/source/net/yacy/document/importer/WarcImporter.java @@ -22,6 +22,9 @@ */ package net.yacy.document.importer; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import net.yacy.cora.document.id.DigestURL; @@ -52,7 +55,29 @@ import org.jwat.warc.WarcRecord; * http://archive-access.sourceforge.net/warc/warc_file_format-0.9.html * http://archive-access.sourceforge.net/warc/ */ -public class WarcImporter { +public class WarcImporter extends Thread implements Importer { + + static public Importer job; // static object to assure only one importer is running (if started from a servlet, this object is used to store the thread) + + private final InputStream source; // current input warc archive + private String name; // file name of input source + + private int recordCnt; // number of responses indexed (for statistic) + private long startTime; // (for statistic) + private final long sourceSize; // length of the input source (for statistic) + private long consumed; // bytes consumed from input source (for statistic) + + public WarcImporter(InputStream f) { + source = f; + recordCnt = 0; + sourceSize = -1; + } + + public WarcImporter(File f) throws FileNotFoundException{ + name = f.getName(); + sourceSize = f.length(); + source = new FileInputStream(f); + } /** * Reads a Warc file and adds all contained responses to the index. @@ -64,7 +89,8 @@ public class WarcImporter { public void indexWarcRecords(InputStream f) throws IOException { byte[] content; - int cnt = 0; + job = this; + startTime = System.currentTimeMillis(); WarcReader localwarcReader = WarcReaderFactory.getReader(f); WarcRecord wrec = localwarcReader.getNextRecord(); @@ -126,13 +152,82 @@ public class WarcImporter { ); Switchboard.getSwitchboard().toIndexer(response); - cnt++; + recordCnt++; } } } + this.consumed = localwarcReader.getConsumed(); wrec = localwarcReader.getNextRecord(); } localwarcReader.close(); - ConcurrentLog.info("WarcImporter", "Indexed " + cnt + " documents"); + ConcurrentLog.info("WarcImporter", "Indexed " + recordCnt + " documents"); + job = null; + } + + @Override + public void run() { + try { + this.indexWarcRecords(this.source); + } catch (IOException ex) { + ConcurrentLog.info("WarcImporter", ex.getMessage()); + } + } + + /** + * Filename of the input source + * @return + */ + @Override + public String source() { + return this.name; + } + + /** + * Number of responses (pages) indexed + * @return + */ + @Override + public int count() { + return this.recordCnt; + } + + /** + * Indexed responses per second + * @return + */ + @Override + public int speed() { + if (this.recordCnt == 0) return 0; + return (int) (this.recordCnt / Math.max(0L, runningTime() )); + } + + /** + * Duration in seconds running, working on the current import source + * @return duration in seconds + */ + @Override + public long runningTime() { + return (System.currentTimeMillis() - this.startTime) / 1000L; } + + /** + * Estimate on time remaining calculated from length of input source and + * processed bytes. + * @return duration in seconds + */ + @Override + public long remainingTime() { + if (this.consumed == 0) { + return 0; + } else { + long speed = this.consumed / runningTime(); + return (this.sourceSize - this.consumed) / speed; + } + } + + @Override + public String status() { + return ""; + } + } diff --git a/source/net/yacy/http/servlets/GSAsearchServlet.java b/source/net/yacy/http/servlets/GSAsearchServlet.java index 4885b9f95..7f7571860 100644 --- a/source/net/yacy/http/servlets/GSAsearchServlet.java +++ b/source/net/yacy/http/servlets/GSAsearchServlet.java @@ -109,7 +109,7 @@ public class GSAsearchServlet extends HttpServlet { sb.peers.peerActions.setUserAgent(clientip, userAgent); // --- handled by Servlet securityHandler - // check if user is allowed to search (can be switched in /ConfigPortal.html) + // check if user is allowed to search (can be switched in /ConfigPortal_p.html) boolean authenticated = header.isUserInRole(UserDB.AccessRight.ADMIN_RIGHT.toString()); //sb.adminAuthenticated(header) >= 2; // final boolean searchAllowed = authenticated || sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, true); // if (!searchAllowed) return null; diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index bcdf98814..5bed37940 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -2010,9 +2010,13 @@ public final class Switchboard extends serverSwitch { return moved; } else if (s.endsWith(".warc") || s.endsWith(".warc.gz")) { try { - InputStream is = new BufferedInputStream(new FileInputStream(infile)); - WarcImporter wri = new WarcImporter(); - wri.indexWarcRecords(is); + WarcImporter wri = new WarcImporter(infile); + wri.start(); + try { + wri.join(); + } catch (InterruptedException ex) { + return moved; + } moved = infile.renameTo(outfile); } catch (IOException ex) { log.warn("IO Error processing warc file " + infile); diff --git a/source/net/yacy/search/navigator/NavigatorPlugins.java b/source/net/yacy/search/navigator/NavigatorPlugins.java index 278ff1941..843ea1f59 100644 --- a/source/net/yacy/search/navigator/NavigatorPlugins.java +++ b/source/net/yacy/search/navigator/NavigatorPlugins.java @@ -48,6 +48,7 @@ public class NavigatorPlugins { defaultnavplugins.put("namespace", "Wiki Name Space"); defaultnavplugins.put("year", "Year"); // defaultnavplugins.put("year:dates_in_content_dts:Event","Event"); + defaultnavplugins.put("keywords", "Keywords"); return defaultnavplugins; } @@ -118,6 +119,10 @@ public class NavigatorPlugins { navigatorPlugins.put("year", new YearNavigator("Year", CollectionSchema.last_modified)); } } + + if (navname.contains("keywords")) { + navigatorPlugins.put("keywords", new TokenizedStringNavigator("Keywords", CollectionSchema.keywords)); + } } return navigatorPlugins; } diff --git a/source/net/yacy/search/navigator/TokenizedStringNavigator.java b/source/net/yacy/search/navigator/TokenizedStringNavigator.java new file mode 100644 index 000000000..2298ea37b --- /dev/null +++ b/source/net/yacy/search/navigator/TokenizedStringNavigator.java @@ -0,0 +1,77 @@ +/** + * TokenizedStringNavigator.java + * (C) 2017 by reger24; https://github.com/reger24 + * + * This is a part of YaCy, a peer-to-peer based web search engine + * + * LICENSE + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. + * If not, see . + */ +package net.yacy.search.navigator; + +import java.util.Collection; +import java.util.StringTokenizer; +import net.yacy.kelondro.data.meta.URIMetadataNode; +import net.yacy.search.Switchboard; +import net.yacy.search.schema.CollectionSchema; + +/** + * Search navigator for string entries based on ScoreMap to count and + * order the result list by counted occurence. The string values are tokenized + * and each word is added (lowercased) to the score map. + */ +public class TokenizedStringNavigator extends StringNavigator implements Navigator { + + public TokenizedStringNavigator(String title, CollectionSchema field) { + super(title, field); + } + + /** + * Increase the score for the key value contained in the defined field in + * the doc. The value string is tokenized using delimiter " ,;" + * @param doc Solrdocument with field for the key content + */ + @Override + public void incDoc(URIMetadataNode doc) { + if (field != null) { + Object val = doc.getFieldValue(field.getSolrFieldName()); + if (val != null) { + if (val instanceof Collection) { + Collection ll = (Collection) val; + for (String s : ll) { + if (!s.isEmpty()) { + StringTokenizer token = new StringTokenizer(s.toLowerCase()," ,;"); // StringTokenizer faster than regex pattern + while (token.hasMoreTokens()) { + String word = token.nextToken(); + if (word.length() > 1 && !Switchboard.stopwords.contains(word)) { + this.inc(word); + } + } + } + } + } else { + StringTokenizer token = new StringTokenizer((String) val, " ,;"); + while (token.hasMoreTokens()) { + String word = token.nextToken().toLowerCase(); + if (word.length() > 1 && !Switchboard.stopwords.contains(word)) { + this.inc(word); + } + } + } + } + } + } +} diff --git a/test/java/net/yacy/kelondro/util/MemoryControlTest.java b/test/java/net/yacy/kelondro/util/MemoryControlTest.java deleted file mode 100644 index 5c8666436..000000000 --- a/test/java/net/yacy/kelondro/util/MemoryControlTest.java +++ /dev/null @@ -1,43 +0,0 @@ - -package net.yacy.kelondro.util; - -import static org.junit.Assert.assertTrue; -import org.junit.Test; - - -public class MemoryControlTest { - - final int onemb = 1024 * 1024; - - /** - * Test of request method, of class MemoryControl. - */ - @Test - public void testRequest_StandardStrategy() { - MemoryControl.setStandardStrategy(true); - MemoryControl.setProperMbyte(24); - - int memblock = onemb * 13; // memsize to allocate - - int iterations = (int) MemoryControl.available() / memblock; - int arraysize = (int) MemoryControl.maxMemory() / memblock + 10; - - byte[][] x = new byte[arraysize][]; - - int i = 0; - - while (i < arraysize && MemoryControl.request(memblock, false)) { - x[i] = new byte[memblock]; - // for realistic test produce some memory avail to GC - if (MemoryControl.request(memblock, false)) { - x[i] = new byte[memblock]; - } - - i++; - } - System.out.println("allocated " + i + " * " + memblock/onemb + " MB = " + i*memblock/onemb + " MB"); - - assertTrue(i >= iterations); - } - -}