boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

The HostBrowser now offers to index files that are discovered because

Browse Source 
they are linked in the web interface.
pull/1/head
orbiter 13 years ago
parent c4a3d8870f
commit be4c96f3b1
3 changed files with 64 additions and 7 deletions
Show Stats Download Patch File Download Diff File

19
htroot/HostBrowser.html
Unescape View File

@ -91,7 +91,7 @@ function updatepage(str) {
<tr>
<th align="center" width="32"></th>
<th align="left" width="600" class="listing">Path</th>
<th align="right" width="80" class="listing">Documents</th>
<th align="right" width="80" class="listing"></th>
</tr>
#(root)#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
@ -104,18 +104,31 @@ function updatepage(str) {
#(type)#
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td align="center"><a href="/ViewFile.html?url=#[file]#"><img src="/env/grafics/doc.gif"/></a></td>
<td align="left" nowrap class=#(stored)#"listingem"::"listing"#(/stored)# colspan="2">#[file]#&nbsp;<a href="#[file]#"><img src="/env/grafics/link.gif"/></a></td>
<td align="left" nowrap class=#(stored)#"listingem"::"listing"#(/stored)#>#[file]#&nbsp;<a href="#[file]#"><img src="/env/grafics/link.gif"/></a></td>
#(stored)#
#(load)#::<td align="right" nowrap class="listingnok"><a href="/HostBrowser.html?load=#[file]#&path=#[path]#">load &amp; index</a>#(/load)#</td>::
<td align="right" nowrap class="listingok">indexed</td>::
<td align="right" nowrap class="listingnok">loading</td>
#(/stored)#
</tr>::
<tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#">
<td align="center"><img src="/env/grafics/dir.gif"/></td>
<td align="left" nowrap class="listing"><a href="/HostBrowser.html?path=#[file]#" class="listing">#[file]#</a></td>
<td align="right" nowrap>#[count]#</td>
<td align="right" nowrap class="listing">#[count]# files</td>
</tr>
#(/type)#
#{/list}#
</table>
</fieldset>
#(/files)#
#(admin)#::
<form action="HostBrowser.html" method="get">
<fieldset><legend>Administration Options</legend>
</fieldset>
</form>
#(/admin)#
#%env/templates/footer.template%#
</body>

42
htroot/HostBrowser.java
Unescape View File

@ -19,7 +19,9 @@
*/
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@ -31,11 +33,12 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.solr.common.SolrDocument;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.federate.solr.YaCySchema;
import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.sorting.ReversibleScoreMap;
import net.yacy.crawler.retrieval.Request;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
import net.yacy.search.index.Fulltext;
@ -49,7 +52,9 @@ public class HostBrowser {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
Fulltext fulltext = sb.index.fulltext();
final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || sb.verifyAuthentication(header);
final boolean admin = sb.verifyAuthentication(header);
final boolean loadRight = admin; // add config later
final boolean searchAllowed = sb.getConfigBool("publicSearchpage", true) || admin;
final serverObjects prop = new serverObjects();
@ -59,6 +64,7 @@ public class HostBrowser {
prop.putNum("ucount", fulltext.size());
prop.put("hosts", 0);
prop.put("files", 0);
prop.put("admin", 0);
if (!searchAllowed) {
prop.put("result", "You are not allowed to use this page. Please ask an administrator for permission.");
@ -80,6 +86,25 @@ public class HostBrowser {
!path.startsWith("file://"))) { path = "http://" + path; }
prop.putHTML("path", path);
String load = post.get("load", "");
//if (path.length() != 0 && load.length() == 0 && loadRight && !sb.index.exists(urlhash))
if (load.length() > 0 && loadRight) {
// stack URL
DigestURI url;
try {
url = new DigestURI(load);
String reasonString = sb.crawlStacker.stackCrawl(new Request(
sb.peers.mySeed().hash.getBytes(),
url, null, load, new Date(),
sb.crawler.defaultRemoteProfile.handle(),
0, 0, 0, 0
));
prop.put("result", reasonString == null ? ("added url to indexer: " + load) : ("not indexed url '" + load + "': " + reasonString));
} catch (MalformedURLException e) {
prop.put("result", "bad url '" + load + "'");
}
}
if (post.containsKey("hosts")) {
// generate host list
try {
@ -112,7 +137,7 @@ public class HostBrowser {
}
try {
// generate file list from path
MultiProtocolURI uri = new MultiProtocolURI(path);
DigestURI uri = new DigestURI(path);
String host = uri.getHost();
// get all files for a specific host from the index
@ -169,7 +194,16 @@ public class HostBrowser {
// this is a file
prop.put("files_list_" + c + "_type", 0);
prop.put("files_list_" + c + "_type_file", entry.getKey());
prop.put("files_list_" + c + "_type_stored", ((Boolean) entry.getValue()).booleanValue() ? 1 : 0);
boolean stored = ((Boolean) entry.getValue()).booleanValue();
try {uri = new DigestURI(entry.getKey());} catch (MalformedURLException e) {uri = null;}
boolean loading = load.equals(entry.getKey()) ||
(uri != null && sb.crawlQueues.urlExists(uri.hash()) != null);
prop.put("files_list_" + c + "_type_stored", stored ? 1 : loading ? 2 : 0);
prop.put("files_list_" + c + "_type_stored_load", loadRight ? 1 : 0);
if (loadRight) {
prop.put("files_list_" + c + "_type_stored_load_file", entry.getKey());
prop.put("files_list_" + c + "_type_stored_load_path", path);
}
} else {
// this is a folder
prop.put("files_list_" + c + "_type", 1);

10
htroot/env/base.css vendored
Unescape View File

@ -160,6 +160,16 @@ tt, *.tt {
color:DimGray;
}
.listingok {
font-family:Courier,Terminal,sans-serif;
color:green;
}
.listingnok {
font-family:Courier,Terminal,sans-serif;
color:red;
}
/* .snippetLoaded {
font-style:italic;
color: #666;

Write Preview
Loading…
Cancel
Save