Michael Peter Christen
8 years ago
commit
f5ad29edb1
@ -0,0 +1,235 @@
|
|||||||
|
// ConfigPortal_p.java
|
||||||
|
// -----------------------
|
||||||
|
// part of YaCy
|
||||||
|
// (C) by Michael Peter Christen; mc@yacy.net
|
||||||
|
// first published on http://yacy.net
|
||||||
|
// Frankfurt, Germany, 4.7.2008
|
||||||
|
//
|
||||||
|
//$LastChangedDate$
|
||||||
|
//$LastChangedRevision$
|
||||||
|
//$LastChangedBy$
|
||||||
|
//
|
||||||
|
// LICENSE
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program; if not, write to the Free Software
|
||||||
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.util.Properties;
|
||||||
|
|
||||||
|
import net.yacy.cora.document.id.DigestURL;
|
||||||
|
import net.yacy.cora.protocol.RequestHeader;
|
||||||
|
import net.yacy.cora.util.ConcurrentLog;
|
||||||
|
import net.yacy.data.TransactionManager;
|
||||||
|
import net.yacy.data.WorkTables;
|
||||||
|
import net.yacy.http.servlets.YaCyDefaultServlet;
|
||||||
|
import net.yacy.search.Switchboard;
|
||||||
|
import net.yacy.search.SwitchboardConstants;
|
||||||
|
import net.yacy.server.serverObjects;
|
||||||
|
import net.yacy.server.serverSwitch;
|
||||||
|
import net.yacy.server.http.HTTPDFileHandler;
|
||||||
|
|
||||||
|
public class ConfigPortal_p {
|
||||||
|
|
||||||
|
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||||
|
final serverObjects prop = new serverObjects();
|
||||||
|
final Switchboard sb = (Switchboard) env;
|
||||||
|
|
||||||
|
if (post != null) {
|
||||||
|
/* Check this is a valid transaction */
|
||||||
|
TransactionManager.checkPostTransaction(header, post);
|
||||||
|
|
||||||
|
if (post.containsKey("popup")) {
|
||||||
|
final String popup = post.get("popup", "status");
|
||||||
|
if ("front".equals(popup)) {
|
||||||
|
sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html");
|
||||||
|
} else if ("search".equals(popup)) {
|
||||||
|
sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "yacysearch.html");
|
||||||
|
} else if ("interactive".equals(popup)) {
|
||||||
|
sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "yacyinteractive.html");
|
||||||
|
} else {
|
||||||
|
sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "Status.html");
|
||||||
|
}
|
||||||
|
sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html"));
|
||||||
|
HTTPDFileHandler.initDefaultPath();
|
||||||
|
}
|
||||||
|
if (post.containsKey("searchpage_set")) {
|
||||||
|
final String newGreeting = post.get(SwitchboardConstants.GREETING, "");
|
||||||
|
// store this call as api call
|
||||||
|
sb.tables.recordAPICall(post, "ConfigPortal_p.html", WorkTables.TABLE_API_TYPE_CONFIGURATION, "new portal design. greeting: " + newGreeting);
|
||||||
|
|
||||||
|
sb.setConfig(SwitchboardConstants.GREETING, newGreeting);
|
||||||
|
sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, post.get(SwitchboardConstants.GREETING_HOMEPAGE, ""));
|
||||||
|
sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, post.get(SwitchboardConstants.GREETING_LARGE_IMAGE, ""));
|
||||||
|
sb.setConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, post.get(SwitchboardConstants.GREETING_SMALL_IMAGE, ""));
|
||||||
|
sb.setConfig(SwitchboardConstants.GREETING_IMAGE_ALT, post.get(SwitchboardConstants.GREETING_IMAGE_ALT, ""));
|
||||||
|
sb.setConfig(SwitchboardConstants.SEARCH_TARGET_DEFAULT, post.get("target", "_self"));
|
||||||
|
sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL, post.get("target_special", "_self"));
|
||||||
|
sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, post.get("target_special_pattern", "_self"));
|
||||||
|
sb.setConfig(SwitchboardConstants.SEARCH_ITEMS, post.getInt("maximumRecords", 10));
|
||||||
|
sb.setConfig(SwitchboardConstants.INDEX_FORWARD, post.get(SwitchboardConstants.INDEX_FORWARD, ""));
|
||||||
|
HTTPDFileHandler.indexForward = post.get(SwitchboardConstants.INDEX_FORWARD, "");
|
||||||
|
sb.setConfig("publicTopmenu", !post.containsKey("publicTopmenu") || post.getBoolean("publicTopmenu"));
|
||||||
|
sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, !post.containsKey(SwitchboardConstants.PUBLIC_SEARCHPAGE) || post.getBoolean(SwitchboardConstants.PUBLIC_SEARCHPAGE));
|
||||||
|
sb.setConfig("search.options", post.getBoolean("search.options"));
|
||||||
|
|
||||||
|
sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, post.getBoolean(SwitchboardConstants.GREEDYLEARNING_ACTIVE));
|
||||||
|
|
||||||
|
final boolean storeresult = post.getBoolean(SwitchboardConstants.REMOTESEARCH_RESULT_STORE);
|
||||||
|
sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, storeresult);
|
||||||
|
sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, post.getLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1));
|
||||||
|
|
||||||
|
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, post.get("search.verify", "ifexist"));
|
||||||
|
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, post.getBoolean("search.verify.delete"));
|
||||||
|
|
||||||
|
sb.setConfig("about.headline", post.get("about.headline", ""));
|
||||||
|
sb.setConfig("about.body", post.get("about.body", ""));
|
||||||
|
|
||||||
|
String excludehosts = post.get("search.excludehosts", "");
|
||||||
|
sb.setConfig("search.excludehosts", excludehosts);
|
||||||
|
try {
|
||||||
|
sb.setConfig("search.excludehosth", DigestURL.hosthashes(excludehosts));
|
||||||
|
} catch (MalformedURLException e) {
|
||||||
|
ConcurrentLog.logException(e);
|
||||||
|
sb.setConfig("search.excludehosth", "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (post.containsKey("searchpage_default")) {
|
||||||
|
// load defaults from defaults/yacy.init file
|
||||||
|
final Properties config = new Properties();
|
||||||
|
final String mes = "ConfigPortal";
|
||||||
|
FileInputStream fis = null;
|
||||||
|
try {
|
||||||
|
fis = new FileInputStream(new File(sb.appPath, "defaults/yacy.init"));
|
||||||
|
config.load(fis);
|
||||||
|
} catch (final FileNotFoundException e) {
|
||||||
|
ConcurrentLog.severe(mes, "could not find configuration file.");
|
||||||
|
return prop;
|
||||||
|
} catch (final IOException e) {
|
||||||
|
ConcurrentLog.severe(mes, "could not read configuration file.");
|
||||||
|
return prop;
|
||||||
|
} finally {
|
||||||
|
if (fis != null) {
|
||||||
|
try {
|
||||||
|
fis.close();
|
||||||
|
} catch (final IOException e) {
|
||||||
|
ConcurrentLog.logException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sb.setConfig(SwitchboardConstants.GREETING, config.getProperty(SwitchboardConstants.GREETING,"P2P Web Search"));
|
||||||
|
sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, config.getProperty(SwitchboardConstants.GREETING_HOMEPAGE,"http://yacy.net"));
|
||||||
|
sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, config.getProperty(SwitchboardConstants.GREETING_LARGE_IMAGE,"env/grafics/YaCyLogo_120ppi.png"));
|
||||||
|
sb.setConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, config.getProperty(SwitchboardConstants.GREETING_SMALL_IMAGE,"env/grafics/YaCyLogo_60ppi.png"));
|
||||||
|
sb.setConfig(SwitchboardConstants.GREETING_IMAGE_ALT, config.getProperty(SwitchboardConstants.GREETING_IMAGE_ALT,"YaCy project web site"));
|
||||||
|
sb.setConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, config.getProperty(SwitchboardConstants.BROWSER_POP_UP_PAGE,"Status.html"));
|
||||||
|
sb.setConfig(SwitchboardConstants.INDEX_FORWARD, config.getProperty(SwitchboardConstants.INDEX_FORWARD,""));
|
||||||
|
HTTPDFileHandler.indexForward = "";
|
||||||
|
sb.setConfig(SwitchboardConstants.SEARCH_TARGET_DEFAULT, config.getProperty(SwitchboardConstants.SEARCH_TARGET_DEFAULT,"_self"));
|
||||||
|
sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL, config.getProperty(SwitchboardConstants.SEARCH_TARGET_SPECIAL,"_self"));
|
||||||
|
sb.setConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, config.getProperty(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN,""));
|
||||||
|
sb.setConfig("publicTopmenu", config.getProperty("publicTopmenu","true"));
|
||||||
|
sb.setConfig(SwitchboardConstants.PUBLIC_SEARCHPAGE, config.getProperty(SwitchboardConstants.PUBLIC_SEARCHPAGE,"true"));
|
||||||
|
sb.setConfig("search.navigation", config.getProperty("search.navigation","hosts,authors,namespace,topics"));
|
||||||
|
sb.setConfig("search.options", config.getProperty("search.options","true"));
|
||||||
|
sb.setConfig(SwitchboardConstants.GREEDYLEARNING_ACTIVE, config.getProperty(SwitchboardConstants.GREEDYLEARNING_ACTIVE));
|
||||||
|
sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE));
|
||||||
|
sb.setConfig(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, config.getProperty(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE));
|
||||||
|
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY, config.getProperty(SwitchboardConstants.SEARCH_VERIFY,"iffresh"));
|
||||||
|
sb.setConfig(SwitchboardConstants.SEARCH_VERIFY_DELETE, config.getProperty(SwitchboardConstants.SEARCH_VERIFY_DELETE,"true"));
|
||||||
|
sb.setConfig("about.headline", config.getProperty("about.headline",""));
|
||||||
|
sb.setConfig("about.body", config.getProperty("about.body",""));
|
||||||
|
sb.setConfig("search.excludehosts", config.getProperty("search.excludehosts",""));
|
||||||
|
sb.setConfig("search.excludehosth", config.getProperty("search.excludehosth",""));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Acquire a transaction token for the next POST form submission */
|
||||||
|
prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));
|
||||||
|
|
||||||
|
prop.putHTML(SwitchboardConstants.GREETING, sb.getConfig(SwitchboardConstants.GREETING, ""));
|
||||||
|
prop.putHTML(SwitchboardConstants.GREETING_HOMEPAGE, sb.getConfig(SwitchboardConstants.GREETING_HOMEPAGE, ""));
|
||||||
|
prop.putHTML(SwitchboardConstants.GREETING_LARGE_IMAGE, sb.getConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, ""));
|
||||||
|
prop.putHTML(SwitchboardConstants.GREETING_SMALL_IMAGE, sb.getConfig(SwitchboardConstants.GREETING_SMALL_IMAGE, ""));
|
||||||
|
prop.putHTML(SwitchboardConstants.GREETING_IMAGE_ALT, sb.getConfig(SwitchboardConstants.GREETING_IMAGE_ALT, ""));
|
||||||
|
prop.putHTML(SwitchboardConstants.INDEX_FORWARD, sb.getConfig(SwitchboardConstants.INDEX_FORWARD, ""));
|
||||||
|
prop.put("publicTopmenu", sb.getConfigBool("publicTopmenu", false) ? 1 : 0);
|
||||||
|
prop.put(SwitchboardConstants.PUBLIC_SEARCHPAGE, sb.getConfigBool(SwitchboardConstants.PUBLIC_SEARCHPAGE, false) ? 1 : 0);
|
||||||
|
prop.put("search.options", sb.getConfigBool("search.options", false) ? 1 : 0);
|
||||||
|
|
||||||
|
prop.put(SwitchboardConstants.GREEDYLEARNING_ACTIVE, sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ACTIVE, false) ? 1 : 0);
|
||||||
|
prop.put(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, sb.getConfig(SwitchboardConstants.GREEDYLEARNING_LIMIT_DOCCOUNT, "0"));
|
||||||
|
|
||||||
|
prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, sb.getConfigBool(SwitchboardConstants.REMOTESEARCH_RESULT_STORE, true) ? 1 : 0);
|
||||||
|
long resultStoredMaxSize = sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, -1);
|
||||||
|
if(resultStoredMaxSize > 0) {
|
||||||
|
prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, resultStoredMaxSize);
|
||||||
|
} else {
|
||||||
|
prop.put(SwitchboardConstants.REMOTESEARCH_RESULT_STORE_MAXSIZE, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
prop.put("search.verify.nocache", sb.getConfig("search.verify", "").equals("nocache") ? 1 : 0);
|
||||||
|
prop.put("search.verify.iffresh", sb.getConfig("search.verify", "").equals("iffresh") ? 1 : 0);
|
||||||
|
prop.put("search.verify.ifexist", sb.getConfig("search.verify", "").equals("ifexist") ? 1 : 0);
|
||||||
|
prop.put("search.verify.cacheonly", sb.getConfig("search.verify", "").equals("cacheonly") ? 1 : 0);
|
||||||
|
prop.put("search.verify.false", sb.getConfig("search.verify", "").equals("false") ? 1 : 0);
|
||||||
|
prop.put("search.verify.delete", sb.getConfigBool(SwitchboardConstants.SEARCH_VERIFY_DELETE, true) ? 1 : 0);
|
||||||
|
|
||||||
|
prop.put("about.headline", sb.getConfig("about.headline", ""));
|
||||||
|
prop.put("about.body", sb.getConfig("about.body", ""));
|
||||||
|
|
||||||
|
prop.put("search.excludehosts", sb.getConfig("search.excludehosts", ""));
|
||||||
|
prop.put("search.excludehosth", sb.getConfig("search.excludehosth", ""));
|
||||||
|
|
||||||
|
final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
|
||||||
|
prop.put("popupFront", 0);
|
||||||
|
prop.put("popupSearch", 0);
|
||||||
|
prop.put("popupInteractive", 0);
|
||||||
|
prop.put("popupStatus", 0);
|
||||||
|
if (browserPopUpPage.startsWith("index")) {
|
||||||
|
prop.put("popupFront", 1);
|
||||||
|
} else if (browserPopUpPage.startsWith("yacysearch")) {
|
||||||
|
prop.put("popupSearch", 1);
|
||||||
|
} else if (browserPopUpPage.startsWith("yacyinteractive")) {
|
||||||
|
prop.put("popupInteractive", 1);
|
||||||
|
} else {
|
||||||
|
prop.put("popupStatus", 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
prop.put("maximumRecords", sb.getConfigInt(SwitchboardConstants.SEARCH_ITEMS, 10));
|
||||||
|
|
||||||
|
final String target = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_DEFAULT, "_self");
|
||||||
|
prop.put("target_selected_blank", "_blank".equals(target) ? 1 : 0);
|
||||||
|
prop.put("target_selected_self", "_self".equals(target) ? 1 : 0);
|
||||||
|
prop.put("target_selected_parent", "_parent".equals(target) ? 1 : 0);
|
||||||
|
prop.put("target_selected_top", "_top".equals(target) ? 1 : 0);
|
||||||
|
prop.put("target_selected_searchresult", "searchresult".equals(target) ? 1 : 0);
|
||||||
|
|
||||||
|
final String target_special = sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL, "_self");
|
||||||
|
prop.put("target_selected_special_blank", "_blank".equals(target_special) ? 1 : 0);
|
||||||
|
prop.put("target_selected_special_self", "_self".equals(target_special) ? 1 : 0);
|
||||||
|
prop.put("target_selected_special_parent", "_parent".equals(target_special) ? 1 : 0);
|
||||||
|
prop.put("target_selected_special_top", "_top".equals(target_special) ? 1 : 0);
|
||||||
|
prop.put("target_selected_special_searchresult", "searchresult".equals(target_special) ? 1 : 0);
|
||||||
|
prop.put("target_special_pattern", sb.getConfig(SwitchboardConstants.SEARCH_TARGET_SPECIAL_PATTERN, ""));
|
||||||
|
|
||||||
|
prop.put("myContext", YaCyDefaultServlet.getContext(header, sb));
|
||||||
|
return prop;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,51 @@
|
|||||||
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||||
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||||
|
<head>
|
||||||
|
<title>YaCy '#[clientname]#': Warc Import</title>
|
||||||
|
#%env/templates/metas.template%#
|
||||||
|
#(import)#::<meta http-equiv="REFRESH" content="10;url=IndexImportWarc_p.html" />
|
||||||
|
<!-- the url= removes http get parameters on refresh, preventing restart of import -->
|
||||||
|
#(/import)#
|
||||||
|
</head>
|
||||||
|
<body id="IndexImportWarc">
|
||||||
|
#%env/templates/header.template%#
|
||||||
|
#%env/templates/submenuIndexImport.template%#
|
||||||
|
<h2>Web Archive File Import</h2>
|
||||||
|
|
||||||
|
#(import)#
|
||||||
|
<p>No import thread is running, you can start a new thread here</p>
|
||||||
|
<form action="IndexImportWarc_p.html" method="get" accept-charset="UTF-8">
|
||||||
|
<!-- no post method here, we don't want to transmit the whole file, only the path-->
|
||||||
|
<fieldset>
|
||||||
|
<legend>Warc File Selection: select an warc file (which may be gz compressed)</legend>
|
||||||
|
<p>
|
||||||
|
You can download warc archives for example here
|
||||||
|
<a href="https://archive.org/search.php?query=subject%3A%22warcarchives%22&and[]=subject%3A%22warcarchives%22" target="_blank">Internet Archive</a>.
|
||||||
|
</p>
|
||||||
|
<div class="input-group">
|
||||||
|
<span style="display: inline-block">
|
||||||
|
<input name="file" type="file" value="" size="75" /></span>
|
||||||
|
<div class="btn-group">
|
||||||
|
<input name="submit" class="btn btn-primary" type="submit" value="Import Warc File" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</fieldset>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
<br />
|
||||||
|
::
|
||||||
|
<form><fieldset><legend>Import Process</legend>
|
||||||
|
<dl>
|
||||||
|
<dt>Thread:</dt><dd>#[thread]#</dd>
|
||||||
|
<dt>Warc File:</dt><dd>#[warcfile]#</dd>
|
||||||
|
<dt>Processed:</dt><dd>#[count]# Entries</dd>
|
||||||
|
<dt>Speed:</dt><dd>#[speed]# pages per second</dd>
|
||||||
|
<dt>Running Time:</dt><dd>#[runningHours]# hours, #[runningMinutes]# minutes</dd>
|
||||||
|
<dt>Remaining Time:</dt><dd>#[remainingHours]# hours, #[remainingMinutes]# minutes</dd>
|
||||||
|
</dl>
|
||||||
|
</fieldset></form>
|
||||||
|
#(/import)#
|
||||||
|
|
||||||
|
#%env/templates/footer.template%#
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@ -0,0 +1,76 @@
|
|||||||
|
// IndexImportWarc_p.java
|
||||||
|
// -------------------------
|
||||||
|
// (c) 2017 by reger24; https://github.com/reger24
|
||||||
|
//
|
||||||
|
// This program is free software; you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as published by
|
||||||
|
// the Free Software Foundation; either version 2 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program; if not, write to the Free Software
|
||||||
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
|
||||||
|
import net.yacy.cora.protocol.RequestHeader;
|
||||||
|
import net.yacy.document.importer.WarcImporter;
|
||||||
|
import net.yacy.search.Switchboard;
|
||||||
|
import net.yacy.server.serverObjects;
|
||||||
|
import net.yacy.server.serverSwitch;
|
||||||
|
|
||||||
|
public class IndexImportWarc_p {
|
||||||
|
|
||||||
|
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
||||||
|
final serverObjects prop = new serverObjects();
|
||||||
|
final Switchboard sb = (Switchboard) env;
|
||||||
|
|
||||||
|
if (WarcImporter.job != null && WarcImporter.job.isAlive()) {
|
||||||
|
// one import is running, no option to insert anything
|
||||||
|
prop.put("import", 1);
|
||||||
|
prop.put("import_thread", "running");
|
||||||
|
prop.put("import_warcfile", WarcImporter.job.source());
|
||||||
|
prop.put("import_count", WarcImporter.job.count());
|
||||||
|
prop.put("import_speed", WarcImporter.job.speed());
|
||||||
|
prop.put("import_runningHours", (WarcImporter.job.runningTime() / 60) / 60);
|
||||||
|
prop.put("import_runningMinutes", (WarcImporter.job.runningTime() / 60) % 60);
|
||||||
|
prop.put("import_remainingHours", (WarcImporter.job.remainingTime() / 60) / 60);
|
||||||
|
prop.put("import_remainingMinutes", (WarcImporter.job.remainingTime() / 60) % 60);
|
||||||
|
} else {
|
||||||
|
prop.put("import", 0);
|
||||||
|
if (post != null) {
|
||||||
|
if (post.containsKey("file")) {
|
||||||
|
String file = post.get("file");
|
||||||
|
final File sourcefile = new File(file);
|
||||||
|
if (sourcefile.exists()) {
|
||||||
|
try {
|
||||||
|
WarcImporter wi = new WarcImporter(sourcefile);
|
||||||
|
wi.start();
|
||||||
|
prop.put("import_thread", "started");
|
||||||
|
} catch (FileNotFoundException ex) {
|
||||||
|
prop.put("import_thread", "Error: file not found [" + file + "]");
|
||||||
|
}
|
||||||
|
prop.put("import_warcfile", file);
|
||||||
|
} else {
|
||||||
|
prop.put("import_warcfile", "");
|
||||||
|
prop.put("import_thread", "Error: file not found [" + file + "]");
|
||||||
|
}
|
||||||
|
prop.put("import", 1);
|
||||||
|
prop.put("import_count", 0);
|
||||||
|
prop.put("import_speed", 0);
|
||||||
|
prop.put("import_runningHours", 0);
|
||||||
|
prop.put("import_runningMinutes", 0);
|
||||||
|
prop.put("import_remainingHours", 0);
|
||||||
|
prop.put("import_remainingMinutes", 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return prop;
|
||||||
|
}
|
||||||
|
}
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,77 @@
|
|||||||
|
/**
|
||||||
|
* TokenizedStringNavigator.java
|
||||||
|
* (C) 2017 by reger24; https://github.com/reger24
|
||||||
|
*
|
||||||
|
* This is a part of YaCy, a peer-to-peer based web search engine
|
||||||
|
*
|
||||||
|
* LICENSE
|
||||||
|
*
|
||||||
|
* This program is free software; you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU General Public License as published by
|
||||||
|
* the Free Software Foundation; either version 2 of the License, or
|
||||||
|
* (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* along with this program.
|
||||||
|
* If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package net.yacy.search.navigator;
|
||||||
|
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.StringTokenizer;
|
||||||
|
import net.yacy.kelondro.data.meta.URIMetadataNode;
|
||||||
|
import net.yacy.search.Switchboard;
|
||||||
|
import net.yacy.search.schema.CollectionSchema;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Search navigator for string entries based on ScoreMap to count and
|
||||||
|
* order the result list by counted occurence. The string values are tokenized
|
||||||
|
* and each word is added (lowercased) to the score map.
|
||||||
|
*/
|
||||||
|
public class TokenizedStringNavigator extends StringNavigator implements Navigator {
|
||||||
|
|
||||||
|
public TokenizedStringNavigator(String title, CollectionSchema field) {
|
||||||
|
super(title, field);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Increase the score for the key value contained in the defined field in
|
||||||
|
* the doc. The value string is tokenized using delimiter " ,;"
|
||||||
|
* @param doc Solrdocument with field for the key content
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void incDoc(URIMetadataNode doc) {
|
||||||
|
if (field != null) {
|
||||||
|
Object val = doc.getFieldValue(field.getSolrFieldName());
|
||||||
|
if (val != null) {
|
||||||
|
if (val instanceof Collection) {
|
||||||
|
Collection<String> ll = (Collection) val;
|
||||||
|
for (String s : ll) {
|
||||||
|
if (!s.isEmpty()) {
|
||||||
|
StringTokenizer token = new StringTokenizer(s.toLowerCase()," ,;"); // StringTokenizer faster than regex pattern
|
||||||
|
while (token.hasMoreTokens()) {
|
||||||
|
String word = token.nextToken();
|
||||||
|
if (word.length() > 1 && !Switchboard.stopwords.contains(word)) {
|
||||||
|
this.inc(word);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
StringTokenizer token = new StringTokenizer((String) val, " ,;");
|
||||||
|
while (token.hasMoreTokens()) {
|
||||||
|
String word = token.nextToken().toLowerCase();
|
||||||
|
if (word.length() > 1 && !Switchboard.stopwords.contains(word)) {
|
||||||
|
this.inc(word);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,43 +0,0 @@
|
|||||||
|
|
||||||
package net.yacy.kelondro.util;
|
|
||||||
|
|
||||||
import static org.junit.Assert.assertTrue;
|
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
|
|
||||||
public class MemoryControlTest {
|
|
||||||
|
|
||||||
final int onemb = 1024 * 1024;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Test of request method, of class MemoryControl.
|
|
||||||
*/
|
|
||||||
@Test
|
|
||||||
public void testRequest_StandardStrategy() {
|
|
||||||
MemoryControl.setStandardStrategy(true);
|
|
||||||
MemoryControl.setProperMbyte(24);
|
|
||||||
|
|
||||||
int memblock = onemb * 13; // memsize to allocate
|
|
||||||
|
|
||||||
int iterations = (int) MemoryControl.available() / memblock;
|
|
||||||
int arraysize = (int) MemoryControl.maxMemory() / memblock + 10;
|
|
||||||
|
|
||||||
byte[][] x = new byte[arraysize][];
|
|
||||||
|
|
||||||
int i = 0;
|
|
||||||
|
|
||||||
while (i < arraysize && MemoryControl.request(memblock, false)) {
|
|
||||||
x[i] = new byte[memblock];
|
|
||||||
// for realistic test produce some memory avail to GC
|
|
||||||
if (MemoryControl.request(memblock, false)) {
|
|
||||||
x[i] = new byte[memblock];
|
|
||||||
}
|
|
||||||
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
System.out.println("allocated " + i + " * " + memblock/onemb + " MB = " + i*memblock/onemb + " MB");
|
|
||||||
|
|
||||||
assertTrue(i >= iterations);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
Loading…
Reference in new issue