From be4c96f3b10cf5ca7db2ad415538cd2d84716183 Mon Sep 17 00:00:00 2001 From: orbiter Date: Sun, 30 Sep 2012 13:23:06 +0200 Subject: [PATCH] The HostBrowser now offers to index files that are discovered because they are linked in the web interface. --- htroot/HostBrowser.html | 19 ++++++++++++++++--- htroot/HostBrowser.java | 42 +++++++++++++++++++++++++++++++++++++---- htroot/env/base.css | 10 ++++++++++ 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/htroot/HostBrowser.html b/htroot/HostBrowser.html index d53a202b0..e56617a61 100644 --- a/htroot/HostBrowser.html +++ b/htroot/HostBrowser.html @@ -91,7 +91,7 @@ function updatepage(str) { Path - Documents + #(root)# @@ -104,18 +104,31 @@ function updatepage(str) { #(type)# - #[file]#  + #[file]#  + #(stored)# + #(load)#::load & index#(/load)#:: + indexed:: + loading + #(/stored)# :: #[file]# - #[count]# + #[count]# files #(/type)# #{/list}# #(/files)# + + #(admin)#:: +
+
Administration Options + +
+
+ #(/admin)# #%env/templates/footer.template%# diff --git a/htroot/HostBrowser.java b/htroot/HostBrowser.java index 57854b358..5151a9ed3 100644 --- a/htroot/HostBrowser.java +++ b/htroot/HostBrowser.java @@ -19,7 +19,9 @@ */ import java.io.IOException; +import java.net.MalformedURLException; import java.util.Collection; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -31,11 +33,12 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.solr.common.SolrDocument; -import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.federate.solr.YaCySchema; import net.yacy.cora.federate.solr.connector.AbstractSolrConnector; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.ReversibleScoreMap; +import net.yacy.crawler.retrieval.Request; +import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; import net.yacy.search.Switchboard; import net.yacy.search.index.Fulltext; @@ -49,7 +52,9 @@ public class HostBrowser { // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; Fulltext fulltext = sb.index.fulltext(); - final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header); + final boolean admin = sb.verifyAuthentication(header); + final boolean loadRight = admin; // add config later + final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || admin; final serverObjects prop = new serverObjects(); @@ -59,6 +64,7 @@ public class HostBrowser { prop.putNum("ucount", fulltext.size()); prop.put("hosts", 0); prop.put("files", 0); + prop.put("admin", 0); if (!searchAllowed) { prop.put("result", "You are not allowed to use this page. Please ask an administrator for permission."); @@ -80,6 +86,25 @@ public class HostBrowser { !path.startsWith("file://"))) { path = "http://" + path; } prop.putHTML("path", path); + String load = post.get("load", ""); + //if (path.length() != 0 && load.length() == 0 && loadRight && !sb.index.exists(urlhash)) + if (load.length() > 0 && loadRight) { + // stack URL + DigestURI url; + try { + url = new DigestURI(load); + String reasonString = sb.crawlStacker.stackCrawl(new Request( + sb.peers.mySeed().hash.getBytes(), + url, null, load, new Date(), + sb.crawler.defaultRemoteProfile.handle(), + 0, 0, 0, 0 + )); + prop.put("result", reasonString == null ? ("added url to indexer: " + load) : ("not indexed url '" + load + "': " + reasonString)); + } catch (MalformedURLException e) { + prop.put("result", "bad url '" + load + "'"); + } + } + if (post.containsKey("hosts")) { // generate host list try { @@ -112,7 +137,7 @@ public class HostBrowser { } try { // generate file list from path - MultiProtocolURI uri = new MultiProtocolURI(path); + DigestURI uri = new DigestURI(path); String host = uri.getHost(); // get all files for a specific host from the index @@ -169,7 +194,16 @@ public class HostBrowser { // this is a file prop.put("files_list_" + c + "_type", 0); prop.put("files_list_" + c + "_type_file", entry.getKey()); - prop.put("files_list_" + c + "_type_stored", ((Boolean) entry.getValue()).booleanValue() ? 1 : 0); + boolean stored = ((Boolean) entry.getValue()).booleanValue(); + try {uri = new DigestURI(entry.getKey());} catch (MalformedURLException e) {uri = null;} + boolean loading = load.equals(entry.getKey()) || + (uri != null && sb.crawlQueues.urlExists(uri.hash()) != null); + prop.put("files_list_" + c + "_type_stored", stored ? 1 : loading ? 2 : 0); + prop.put("files_list_" + c + "_type_stored_load", loadRight ? 1 : 0); + if (loadRight) { + prop.put("files_list_" + c + "_type_stored_load_file", entry.getKey()); + prop.put("files_list_" + c + "_type_stored_load_path", path); + } } else { // this is a folder prop.put("files_list_" + c + "_type", 1); diff --git a/htroot/env/base.css b/htroot/env/base.css index 3a22fcc7c..862dbf4f2 100644 --- a/htroot/env/base.css +++ b/htroot/env/base.css @@ -160,6 +160,16 @@ tt, *.tt { color:DimGray; } +.listingok { + font-family:Courier,Terminal,sans-serif; + color:green; +} + +.listingnok { + font-family:Courier,Terminal,sans-serif; + color:red; +} + /* .snippetLoaded { font-style:italic; color: #666;