boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) Implementing snippet fetching via ajax

Browse Source 
Snippets that are not available on page load time will be fetched using ajax requests.
   see: http://www.yacy-forum.de/viewtopic.php?p=16479

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1748 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 19 years ago
parent 7e7a72b108
commit dc9174c809
8 changed files with 181 additions and 26 deletions
Show Stats Download Patch File Download Diff File

27
htroot/ViewFile.java
Unescape View File

@ -52,6 +52,7 @@ import java.net.URLEncoder;
import de.anomic.http.httpHeader; import de.anomic.http.httpHeader;
import de.anomic.http.httpc; import de.anomic.http.httpc;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.plasmaParserDocument;
import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaCrawlLURL.Entry; import de.anomic.plasma.plasmaCrawlLURL.Entry;
@ -126,7 +127,11 @@ public class ViewFile {
try { try {
resource = sb.cacheManager.loadResource(url); resource = sb.cacheManager.loadResource(url);
if (resource == null) { if (resource == null) {
sb.snippetCache.loadResourceFromWeb(url, 5000); plasmaHTCache.Entry entry = sb.snippetCache.loadResourceFromWeb(url, 5000);
if (entry != null) {
resHeader = entry.responseHeader;
}
resource = sb.cacheManager.loadResource(url); resource = sb.cacheManager.loadResource(url);
if (resource == null) { if (resource == null) {
@ -135,15 +140,17 @@ public class ViewFile {
return prop; return prop;
} }
} }
resHeader = sb.cacheManager.getCachedResponse(urlEntry.hash());
if (resHeader == null) { if (resHeader == null) {
resHeader = httpc.whead(url,5000,null,null,sb.remoteProxyConfig); resHeader = sb.cacheManager.getCachedResponse(urlEntry.hash());
if (resource == null) { if (resHeader == null) {
prop.put("error",4); resHeader = httpc.whead(url,5000,null,null,sb.remoteProxyConfig);
prop.put("viewMode",VIEW_MODE_NO_TEXT); if (resource == null) {
return prop; prop.put("error",4);
} prop.put("viewMode",VIEW_MODE_NO_TEXT);
resMime = resHeader.mime(); return prop;
}
resMime = resHeader.mime();
}
} }
} catch (IOException e) { } catch (IOException e) {
if (url == null) { if (url == null) {
@ -165,7 +172,7 @@ public class ViewFile {
prop.put("viewMode_plainText",content); prop.put("viewMode_plainText",content);
} else if (viewMode.equals("parsed") || viewMode.equals("sentences") || viewMode.equals("iframe")) { } else if (viewMode.equals("parsed") || viewMode.equals("sentences") || viewMode.equals("iframe")) {
// parsing the resource content // parsing the resource content
plasmaParserDocument document = sb.snippetCache.parseDocument(url, resource); plasmaParserDocument document = sb.snippetCache.parseDocument(url, resource,header);
if (document == null) { if (document == null) {
prop.put("error",5); prop.put("error",5);
prop.put("viewMode",VIEW_MODE_NO_TEXT); prop.put("viewMode",VIEW_MODE_NO_TEXT);

19
htroot/env/style.css vendored
Unescape View File

@ -166,6 +166,9 @@ background-color: #ffbbaa;
.TableCellSummary { .TableCellSummary {
background-color: #ffbbaa; background-color: #ffbbaa;
} }
.TableCellPassive {
background-color: lightgrey;
}
.WikiBackground { .WikiBackground {
background-color: #eeeeee; background-color: #eeeeee;
} }
@ -213,7 +216,21 @@ width: 16px;
} }
.ResultDateYBR { .ResultDateYBR {
color:#007050 color:#007050;
}
.snippetLoading {
color:#999999;
}
.snippetLoaded {
color:black;
font-style:italic;
}
.snippetError {
color:red;
font-style:italic;
} }
.settingsValue { .settingsValue {

22
htroot/index.html
Unescape View File

@ -4,6 +4,10 @@
<title>YaCy '#[clientname]#': Search Page</title> <title>YaCy '#[clientname]#': Search Page</title>
#%env/templates/metas.template%# #%env/templates/metas.template%#
<link rel="alternate" type="application/rss+xml" title="Search for #[former]#" href="index.rss?search=#[former]#"> <link rel="alternate" type="application/rss+xml" title="Search for #[former]#" href="index.rss?search=#[former]#">
<script src="/js/ajax.js"></script>
<script src="/js/Index.js"></script>
</head> </head>
<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0"> <body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0">
#%env/templates/header.template%# #%env/templates/header.template%#
@ -29,8 +33,8 @@ picPlus.src = "/env/grafics/plus.gif";
<td colspan=2> <td colspan=2>
<center> <center>
<input name="search" type="text" size="52" maxlength="80" value="#[former]#"> <input name="search" type="text" size="52" maxlength="80" value="#[former]#">
<input type="submit" name="Enter" value="Search"><p> <input type="submit" name="Enter" value="Search"><p>
<input type="submit" name="former" value="#[former]#">
</center> </center>
</td> </td>
@ -153,12 +157,13 @@ from 'late' peers.
<!-- link begin --> <!-- link begin -->
<p style="border-bottom:1px #000 dashed; padding-bottom: 1em;"> <p style="border-bottom:1px #000 dashed; padding-bottom: 1em;">
<a href="/Bookmarks_p.html?edit=#[urlhash]#" onMouseover="document.plus_#[urlhexhash]#.src=picPlus.src" onMouseout="document.plus_#[urlhexhash]#.src=picEmpty.src"> <a href="/Bookmarks_p.html?edit=#[urlhash]#" onMouseover="document.plus_#[urlhexhash]#.src=picPlus.src" onMouseout="document.plus_#[urlhexhash]#.src=picEmpty.src">
<img src="/env/grafics/empty.gif" title="bookmark" alt="bookmark" name="plus_#[urlhexhash]#" border="0"></a>&nbsp; <img src="/env/grafics/empty.gif" title="bookmark" alt="bookmark" name="plus_#[urlhexhash]#" border="0"></a>
<span class="ResultDescription">#[description]#</span><br> <span class="ResultDescription">#[description]#</span><br>
<a href="#[delete]#" onMouseover="document.minus_#[urlhexhash]#.src=picMinus.src" onMouseout="document.minus_#[urlhexhash]#.src=picEmpty.src"> <a href="#[delete]#" onMouseover="document.minus_#[urlhexhash]#.src=picMinus.src" onMouseout="document.minus_#[urlhexhash]#.src=picEmpty.src">
<img src="/env/grafics/empty.gif" title="delete" alt="delete" name="minus_#[urlhexhash]#" border="0"></a>&nbsp; <img src="/env/grafics/empty.gif" title="delete" alt="delete" name="minus_#[urlhexhash]#" border="0"></a>
#(snippet)#::<i>#[text]#</i><br>#(/snippet)#<a href="#[url]#">#[urlname]#</a><br> <span class="#(snippet)#snippetLoading::snippetLoaded#(/snippet)#" id="#[urlhash]#">#(snippet)#loading snippet ...::#[text]##(/snippet)#</span><br>
<img src="/env/grafics/empty.gif">&nbsp;<span class="ResultDateYBR">&nbsp;#[date]#</span> | <span class="ResultDateYBR">YBR-#[ybr]#</span> | <a href="ViewFile.html?urlHash=#[urlhash]#&words=#[words]#">Info</a><br> <img src="/env/grafics/empty.gif" border="0">&nbsp;<a id="url#[urlhash]#" href="#[url]#">#[urlname]#</a><br>
<img src="/env/grafics/empty.gif"><span class="ResultDateYBR">&nbsp;#[date]#</span> | <span class="ResultDateYBR">YBR-#[ybr]#</span> | <a href="ViewFile.html?urlHash=#[urlhash]#&words=#[words]#">Info</a><br>
</p> </p>
<!-- link end --> <!-- link end -->
#{/results}# #{/results}#
@ -191,5 +196,12 @@ Architecture (C) by Michael Peter Christen,
<img src="/env/grafics/mcemailh.gif"></i></center> <img src="/env/grafics/mcemailh.gif"></i></center>
#%env/templates/footer.template%# #%env/templates/footer.template%#
<script language="JavaScript">
AllSnippets();
</script>
</body> </body>
</html> </html>

46
htroot/js/Index.js
Unescape View File

@ -0,0 +1,46 @@
function AllSnippets() {
var query = document.getElementsByName("former")[0].value;
var span = document.getElementsByTagName("span");
for(var x=0;x<span.length;x++) {
if (span[x].className == 'snippetLoading') {
var url = document.getElementById("url" + span[x].id);
requestSnippet(url,query);
}
}
}
function requestSnippet(url, query){
var req=createRequestObject();
req.open('get', '/xml/snippet.xml?url=' + escape(url) + '&search=' + escape(query),true);
req.onreadystatechange = function () {handleState(req)};
req.send(null);
}
function handleState(req) {
if(req.readyState != 4){
return;
}
var response = req.responseXML;
var snippetText = response.getElementsByTagName("text")[0].firstChild.data;
var urlHash = response.getElementsByTagName("urlHash")[0].firstChild.data;
var status = response.getElementsByTagName("status")[0].firstChild.data;
var span = document.getElementById(urlHash)
span.removeChild(span.firstChild);
if (status < 11) {
span.className = "snippetLoaded";
//span.setAttribute("class", "snippetLoaded");
} else {
span.className = "snippetError";
//span.setAttribute("class", "snippetError");
}
var snippetNode = document.createTextNode(snippetText);
span.appendChild(snippetNode);
}

57
htroot/xml/snippet.java
Unescape View File

@ -0,0 +1,57 @@
package xml;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Set;
import java.util.TreeSet;
import de.anomic.http.httpHeader;
import de.anomic.kelondro.kelondroMSetTools;
import de.anomic.plasma.plasmaSearchQuery;
import de.anomic.plasma.plasmaSnippetCache;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaURL;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class snippet {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws MalformedURLException {
// return variable that accumulates replacements
plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
// getting url
String urlString = post.get("url", "");
URL url = new URL(urlString);
String querystring = post.get("search", "").trim();
if ((querystring.length() > 2) && (querystring.charAt(0) == '"') && (querystring.charAt(querystring.length() - 1) == '"')) {
querystring = querystring.substring(1, querystring.length() - 1).trim();
}
final TreeSet query = plasmaSearchQuery.cleanQuery(querystring);
// filter out stopwords
final TreeSet filtered = kelondroMSetTools.joinConstructive(query, plasmaSwitchboard.stopwords);
if (filtered.size() > 0) {
kelondroMSetTools.excludeDestructive(query, plasmaSwitchboard.stopwords);
}
// do the search
Set queryHashes = plasmaSearchQuery.words2hashes(query);
plasmaSnippetCache.result snippet = switchboard.snippetCache.retrieve(url, queryHashes, true, 260);
prop.put("status",snippet.source);
if (snippet.source < 11) {
prop.put("text", (snippet.line != null)?snippet.line.trim():"unknown");
} else {
prop.put("text", (snippet.error != null)?snippet.error.trim():"unkown");
}
prop.put("urlHash",plasmaURL.urlHash(url));
// return rewrite properties
return prop;
}
}

6
htroot/xml/snippet.xml
Unescape View File

@ -0,0 +1,6 @@
<?xml version="1.0"?>
<snippet>
<text>#[text]#</text>
<status>#[status]#</status>
<urlHash>#[urlHash]#</urlHash>
</snippet>

4
source/de/anomic/plasma/plasmaCrawlWorker.java
Unescape View File

@ -245,7 +245,7 @@ public final class plasmaCrawlWorker extends Thread {
} }
} }
public static void load( public static plasmaHTCache.Entry load(
URL url, URL url,
String name, String name,
String referer, String referer,
@ -257,7 +257,7 @@ public final class plasmaCrawlWorker extends Thread {
plasmaHTCache cacheManager, plasmaHTCache cacheManager,
serverLog log serverLog log
) throws IOException { ) throws IOException {
load(url, return load(url,
name, name,
referer, referer,
initiator, initiator,

26
source/de/anomic/plasma/plasmaSnippetCache.java
Unescape View File

@ -132,10 +132,14 @@ public class plasmaSnippetCache {
// if the snippet is not in the cache, we can try to get it from the htcache // if the snippet is not in the cache, we can try to get it from the htcache
byte[] resource = null; byte[] resource = null;
httpHeader header = null;
try { try {
resource = cacheManager.loadResource(url); resource = cacheManager.loadResource(url);
if ((fetchOnline) && (resource == null)) { if ((fetchOnline) && (resource == null)) {
loadResourceFromWeb(url, 5000); plasmaHTCache.Entry entry = loadResourceFromWeb(url, 5000);
if (entry != null) {
header = entry.responseHeader;
}
resource = cacheManager.loadResource(url); resource = cacheManager.loadResource(url);
source = SOURCE_WEB; source = SOURCE_WEB;
} }
@ -146,7 +150,7 @@ public class plasmaSnippetCache {
//System.out.println("cannot load document for URL " + url); //System.out.println("cannot load document for URL " + url);
return new result(null, ERROR_RESOURCE_LOADING, "error loading resource from web, cacheManager returned NULL"); return new result(null, ERROR_RESOURCE_LOADING, "error loading resource from web, cacheManager returned NULL");
} }
plasmaParserDocument document = parseDocument(url, resource); plasmaParserDocument document = parseDocument(url, resource, header);
if (document == null) return new result(null, ERROR_PARSER_FAILED, "parser error/failed"); // cannot be parsed if (document == null) return new result(null, ERROR_PARSER_FAILED, "parser error/failed"); // cannot be parsed
//System.out.println("loaded document for URL " + url); //System.out.println("loaded document for URL " + url);
@ -318,11 +322,17 @@ public class plasmaSnippetCache {
} }
public plasmaParserDocument parseDocument(URL url, byte[] resource) { public plasmaParserDocument parseDocument(URL url, byte[] resource) {
return parseDocument(url, resource, null);
}
public plasmaParserDocument parseDocument(URL url, byte[] resource, httpHeader header) {
if (resource == null) return null; if (resource == null) return null;
httpHeader header = null;
try { if (header == null) {
header = this.cacheManager.getCachedResponse(plasmaURL.urlHash(url)); try {
} catch (IOException e) {} header = this.cacheManager.getCachedResponse(plasmaURL.urlHash(url));
} catch (IOException e) {}
}
if (header == null) { if (header == null) {
String filename = this.cacheManager.getCachePath(url).getName(); String filename = this.cacheManager.getCachePath(url).getName();
@ -371,8 +381,8 @@ public class plasmaSnippetCache {
} }
} }
public void loadResourceFromWeb(URL url, int socketTimeout) throws IOException { public plasmaHTCache.Entry loadResourceFromWeb(URL url, int socketTimeout) throws IOException {
plasmaCrawlWorker.load( return plasmaCrawlWorker.load(
url, url,
"", "",
null, null,

Write Preview
Loading…
Cancel
Save