boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

Browse Source
pull/1/head
orbiter 13 years ago
parent 6171143b4a e6330f648a
commit fcd5c7eec3
6 changed files with 92 additions and 17 deletions
Show Stats Download Patch File Download Diff File

5
htroot/Crawler_p.java
Unescape View File

@ -96,6 +96,11 @@ public class Crawler_p {
prop.put("info", "0");
if (post != null) {
String c = post.toString();
if (c.length() < 1000) Log.logInfo("Crawl Start", c);
}
if (post != null && post.containsKey("continue")) {
// continue queue
final String queue = post.get("continue", "");

63
htroot/ViewFile.html
Unescape View File

@ -1,6 +1,59 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "DTD/xhtml1-transitional.dtd">
<!-- This page is only XHTML 1.0 Transitional and not Strict because iframes are in use -->
<html xmlns="http://www.w3.org/1999/xhtml">
<script type="text/javascript">
//<![CDATA[
function xmlhttpPost() {
var searchform = document.getElementById('searchform');
search(searchform.url.value);
}
function search(query) {
var xmlHttpReq = false;
var self = this;
if (window.XMLHttpRequest) { // Mozilla/Safari
self.xmlHttpReq = new XMLHttpRequest();
}
else if (window.ActiveXObject) { // IE
self.xmlHttpReq = new ActiveXObject("Microsoft.XMLHTTP");
}
self.xmlHttpReq.open('GET', "/solr/select?q=sku:\"" + query + "\" OR host_s:\"" + query + "\" OR host_dnc_s:\"" + query + "\" OR host_organization_s:\"" + query + "\" OR host_organizationdnc_s:\"" + query + "\" OR host_subdomain_s:\"" + query + "\"&start=0&rows=100&wt=json", true);
self.xmlHttpReq.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
self.xmlHttpReq.onreadystatechange = function() {
if (self.xmlHttpReq.readyState == 4) {
updatepage(self.xmlHttpReq.responseText);
}
}
self.xmlHttpReq.send(null);
}
function updatepage(str) {
var raw = document.getElementById("raw");
if (raw != null) raw.innerHTML = str;
var rsp = eval("("+str+")");
var firstChannel = rsp.channels[0];
var totalResults = firstChannel.totalResults.replace(/[,.]/,"");
var startIndex = firstChannel.startIndex;
var itemsPerPage = firstChannel.itemsPerPage;
var navigation = firstChannel.navigation;
var html = "";
if (totalResults > 0 && firstChannel.items.length > 0) {
var item;
html += "<table class=\"networkTable\" border=\"0\" cellpadding=\"2\" cellspacing=\"1\" width=\"99%\">";
html += "<tr class=\"TableHeader\" valign=\"bottom\">";
html += "<td>URL from index (total results = " + totalResults + ")<\/td>";
for (var i = 0; i < firstChannel.items.length; i++) {
item = firstChannel.items[i];
html += "<tr class=\"TableCellLight\"><td align=\"left\"><a href=\"HostBrowser.html?urlstringsearch=&amp;urlstring=" + item.link + "\">" + item.link + "<\/a><\/td>";
}
html += "<\/table>";
}
document.getElementById("searchresults").innerHTML = html;
}
//]]>
</script>
<head>
<title>YaCy '#[clientname]#': View URL Content</title>
#%env/templates/metas.template%#
@ -22,11 +75,15 @@
<h2>View URL Content</h2>
<form method="get" action="ViewFile.html" accept-charset="ascii">
<form method="get" action="ViewFile.html" id="searchform" accept-charset="ascii" onkeyup="xmlhttpPost(); return false;">
<fieldset><legend>Get URL Viewer</legend>
<dl>
<dt>URL:</dt><dd><input type="text" size="60" name="url" value="#[url]#" />&nbsp;<input type="submit" name="show" value="Show" /></dd>
<dd>&nbsp;</dd>
<dt>URL:</dt>
<dd>
<input type="text" size="60" name="url" value="#[url]#" />
<input type="submit" name="show" value="Show" />
<div id="searchresults"></div>
</dd>
</dl>
</fieldset>
</form>

2
source/de/anomic/http/server/HTTPDFileHandler.java
Unescape View File

@ -336,7 +336,7 @@ public final class HTTPDFileHandler {
if (!accessGranted) {
final boolean accessFromLocalhost = Domains.isLocalhost(clientIP) && (refererHost == null || refererHost.isEmpty() || Domains.isLocalhost(refererHost));
accessGranted = accessFromLocalhost && realmValue != null && realmProp.length() > 6 && (adminAccountBase64MD5.equals(realmValue));
Log.logInfo("HTTPDFileHandler", "access from localhost blocked, clientIP=" + clientIP);
if (!accessGranted) Log.logInfo("HTTPDFileHandler", "access blocked, clientIP=" + clientIP);
}
// -5- a password is configured and access comes with matching http-authentify

7
source/net/yacy/cora/services/federated/solr/EnhancedXMLResponseWriter.java
Unescape View File

@ -83,6 +83,7 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
}
private static void writeProps(final Writer writer, final String name, final NamedList<?> val) throws IOException {
if (val == null) return;
int sz = val.size();
if (sz <= 0) startTagClose(writer, "lst", name); else startTagOpen(writer, "lst", name);
Object v;
@ -153,7 +154,8 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
if (fidx1 + 1 == fidx2) {
if (sf.multiValued()) {
startTagOpen(writer, "arr", fieldName);
writeField(writer, type, null, value.stringValue()); //sf.write(this, null, f1);
String sv = value.stringValue();
writeField(writer, type, null, sv); //sf.write(this, null, f1);
writer.write("</arr>");
} else {
writeField(writer, type, value.name(), value.stringValue()); //sf.write(this, f1.name(), f1);
@ -161,7 +163,8 @@ public class EnhancedXMLResponseWriter implements QueryResponseWriter {
} else {
startTagOpen(writer, "arr", fieldName);
for (int i = fidx1; i < fidx2; i++) {
writeField(writer, type, null, fields.get(i).stringValue()); //sf.write(this, null, (Fieldable)this.tlst.get(i));
String sv = fields.get(i).stringValue();
writeField(writer, type, null, sv); //sf.write(this, null, (Fieldable)this.tlst.get(i));
}
writer.write("</arr>");
writer.write(lb);

21
source/net/yacy/cora/services/federated/solr/JsonResponseWriter.java
Unescape View File

@ -109,17 +109,17 @@ public class JsonResponseWriter implements QueryResponseWriter {
List<Fieldable> fields = doc.getFields();
int fieldc = fields.size();
List<String> texts = new ArrayList<String>();
String description = "";
String description = "", title = "";
StringBuilder path = new StringBuilder(80);
for (int j = 0; j < fieldc; j++) {
Fieldable value = fields.get(j);
String fieldName = value.name();
if (YaCySchema.title.name().equals(fieldName)) {
solitaireTag(writer, "title", value.stringValue());
texts.add(value.stringValue());
title = value.stringValue();
texts.add(title);
continue;
}
if (YaCySchema.sku.name().equals(fieldName)) {
urlhash = value.stringValue();
solitaireTag(writer, "link", value.stringValue());
continue;
}
@ -133,14 +133,20 @@ public class JsonResponseWriter implements QueryResponseWriter {
solitaireTag(writer, "guid", urlhash);
continue;
}
if (YaCySchema.host_s.name().equals(fieldName)) {
solitaireTag(writer, "host", value.stringValue());
continue;
}
if (YaCySchema.url_paths_sxt.name().equals(fieldName)) {
path.append('/').append(value.stringValue());
continue;
}
if (YaCySchema.last_modified.name().equals(fieldName)) {
Date d = new Date(Long.parseLong(value.stringValue()));
solitaireTag(writer, "pubDate", HeaderFramework.formatRFC1123(d));
texts.add(value.stringValue());
continue;
}
if (YaCySchema.size_i.equals(fieldName)) {
urlhash = value.stringValue();
int size = value.stringValue() != null && value.stringValue().length() > 0 ? Integer.parseInt(value.stringValue()) : -1;
int sizekb = size / 1024;
int sizemb = sizekb / 1024;
@ -161,6 +167,9 @@ public class JsonResponseWriter implements QueryResponseWriter {
}
}
// compute snippet from texts
solitaireTag(writer, "path", path.toString());
solitaireTag(writer, "title", title.length() == 0 ? (texts.size() == 0 ? path.toString() : texts.get(0)) : title);
List<String> snippet = urlhash == null ? null : snippets.get(urlhash);
writer.write("\"description\":\""); writer.write(serverObjects.toJSON(snippet == null || snippet.size() == 0 ? description : snippet.get(0))); writer.write("\"\n}\n");
if (i < responseCount - 1) {

9
source/net/yacy/cora/services/federated/solr/OpensearchResponseWriter.java
Unescape View File

@ -151,7 +151,7 @@ public class OpensearchResponseWriter implements QueryResponseWriter {
List<Fieldable> fields = doc.getFields();
int fieldc = fields.size();
List<String> texts = new ArrayList<String>();
String description = "";
String description = "", title = "";
for (int j = 0; j < fieldc; j++) {
Fieldable value = fields.get(j);
String fieldName = value.name();
@ -170,14 +170,13 @@ public class OpensearchResponseWriter implements QueryResponseWriter {
continue;
}
if (YaCySchema.title.name().equals(fieldName)) {
solitaireTag(writer, RSSMessage.Token.title.name(), value.stringValue());
texts.add(value.stringValue());
title = value.stringValue();
texts.add(title);
continue;
}
if (YaCySchema.last_modified.name().equals(fieldName)) {
Date d = new Date(Long.parseLong(value.stringValue()));
solitaireTag(writer, RSSMessage.Token.pubDate.name(), HeaderFramework.formatRFC1123(d));
texts.add(value.stringValue());
continue;
}
if (YaCySchema.description.name().equals(fieldName)) {
@ -199,6 +198,8 @@ public class OpensearchResponseWriter implements QueryResponseWriter {
}
}
// compute snippet from texts
solitaireTag(writer, RSSMessage.Token.title.name(), title.length() == 0 ? (texts.size() == 0 ? "" : texts.get(0)) : title);
List<String> snippet = urlhash == null ? null : snippets.get(urlhash);
String tagname = RSSMessage.Token.description.name();
writer.write("<"); writer.write(tagname); writer.write('>');

Write Preview
Loading…
Cancel
Save