boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) favicons that are specified in the document content via html link-tags

Browse Source 
are now detected and displayed on the search page (requested by allo).

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3845 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
theli 18 years ago
parent 854eb1492f
commit 339153d40e
10 changed files with 87 additions and 18 deletions
Show Stats Download Patch File Download Diff File

7
htroot/js/yacysearch.js
Unescape View File

@ -65,6 +65,7 @@ function handleTextState(req) {
var snippetText = response.getElementsByTagName("text")[0].firstChild.data;
var urlHash = response.getElementsByTagName("urlHash")[0].firstChild.data;
var status = response.getElementsByTagName("status")[0].firstChild.data;
var span = document.getElementById("h" + urlHash);
removeAllChildren(span);
@ -77,6 +78,12 @@ function handleTextState(req) {
span.parentNode.parentNode.setAttribute("style", "display: none");
document.getElementById("hidden_results").innerHTML='Some results were hidden, because they do not contain your searchwords anymore, or because they are not accessible. Click here to <a href="javascript:show_hidden_results()">show them</a>';
}
// set URL to favicon (if a link-tag was found in the document)
if (response.getElementsByTagName("favicon")[0].firstChild != null) {
var img = document.getElementById("f" + urlHash);
img.src = response.getElementsByTagName("favicon")[0].firstChild.data;
}
// replace "<b>" text by <strong> node
var pos1=snippetText.indexOf("<b>");

2
htroot/xml/snippet.java
Unescape View File

@ -70,6 +70,7 @@ public class snippet {
}
prop.put("link", 0);
prop.put("links", 0);
prop.putSafeXML("favicon",snippet.getFavicon()==null?"":snippet.getFavicon().toString());
} else {
// attach media information
ArrayList mediaSnippets = switchboard.snippetCache.retrieveMediaSnippets(url, queryHashes, media, true, mediasnippet_timeout);
@ -85,6 +86,7 @@ public class snippet {
prop.put("text", "");
prop.put("link", mediaSnippets.size());
prop.put("links", mediaSnippets.size());
prop.put("favicon","");
}

1
htroot/xml/snippet.xml
Unescape View File

@ -4,6 +4,7 @@
<status>#[status]#</status>
<urlHash>#[urlHash]#</urlHash>
<links>#[links]#</links>
<favicon>#[favicon]#</favicon>
#{link}#
<link>
<type>#[type]#</type>

17
htroot/xml/util/getpageinfo_p.java
Unescape View File

@ -66,6 +66,7 @@ public class getpageinfo_p {
serverObjects prop = new serverObjects();
prop.put("sitemap", "");
prop.put("title", "");
prop.put("favicon","");
prop.put("robots-allowed", 3); //unknown
String actions="title";
if(post!=null && post.containsKey("url")){
@ -90,15 +91,21 @@ public class getpageinfo_p {
serverFileUtils.write(contentString,writer);
writer.close();
// put the document title
prop.put("title", scraper.getTitle());
// put the favicon that belongs to the document
prop.putSafeXML("favicon", (scraper.getFavicon()==null)?"":scraper.getFavicon().toString());
// put keywords
String list[]=scraper.getKeywords();
for(int i=0;i<list.length;i++){
prop.putSafeXML("tags_"+i+"_tag", list[i]);
}
prop.put("tags", list.length);
} catch (MalformedURLException e) {
} catch (IOException e) {
} catch (MalformedURLException e) { /* ignore this */
} catch (IOException e) { /* ignore this */
}
}
if(actions.indexOf("robots")>=0){
@ -106,11 +113,7 @@ public class getpageinfo_p {
URL theURL = new URL(url);
// determine if crawling of the current URL is allowed
if(robotsParser.isDisallowed(theURL)){
prop.put("robots-allowed", 0);
}else{
prop.put("robots-allowed", 1);
}
prop.put("robots-allowed", robotsParser.isDisallowed(theURL) ? 0:1);
// get the sitemap URL of the domain
URL sitemapURL = robotsParser.getSitemapURL(theURL);

1
htroot/xml/util/getpageinfo_p.xml
Unescape View File

@ -3,6 +3,7 @@
<title>#[title]#</title>
<robots>#(robots-allowed)#0::1::#(/robots-allowed)#</robots>
<sitemap>#[sitemap]#</sitemap>
<favicon>#[favicon]#</favicon>
<tags>
#{tags}#
<tag name="#[tag]#" />

2
htroot/yacysearch.html
Unescape View File

@ -150,7 +150,7 @@ document.getElementById("Enter").value = "search again - catch up more links";
#(/recommend)#
</div>
#(/authorized)#
<h4 class="linktitle"><img src="#[favicon]#" class="favicon" width="16" height="16" /><a href="#[url]#" target="_parent">#[description]#</a></h4>
<h4 class="linktitle"><img src="#[favicon]#" id="f#[urlhash]#" class="favicon" width="16" height="16" /><a href="#[url]#" target="_parent">#[description]#</a></h4>
<p class="snippet iconindented"><span class="#(snippet)#snippetLoading::snippetLoaded#(/snippet)#" id="h#[urlhash]#">#(snippet)#loading snippet ...::#[text]##(/snippet)#</span></p>
<p class="url iconindented"><a href="#[url]#" id="url#[urlhash]#" target="_parent">#[urlname]#</a></p>
<p class="urlinfo iconindented">#[date]# | YBR-#[ybr]# | <a href="ViewFile.html?urlHash=#[urlhash]#&amp;words=#[words]#">Info</a> | <a href="yacysearch.html?cat=image&amp;url=#[url]#&amp;search=#[former]#">Pictures</a></p>

18
source/de/anomic/htmlFilter/htmlFilterContentScraper.java
Unescape View File

@ -109,6 +109,14 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
private serverCharBuffer content;
private EventListenerList htmlFilterEventListeners = new EventListenerList();
/**
* {@link URL} to the favicon that belongs to the document
*/
private URL favicon;
/**
* The document root {@link URL}
*/
private URL root;
public htmlFilterContentScraper(URL root) {
@ -207,7 +215,8 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
if (type.equalsIgnoreCase("shortcut icon")) {
htmlFilterImageEntry ie = new htmlFilterImageEntry(newLink, linktitle, -1,-1);
images.add(ie);
images.add(ie);
this.favicon = newLink;
} else if (!type.equalsIgnoreCase("stylesheet") && !type.equalsIgnoreCase("alternate stylesheet")) {
anchors.put(newLink.toString(), linktitle);
}
@ -346,6 +355,13 @@ public class htmlFilterContentScraper extends htmlFilterAbstractScraper implemen
public Map getMetas() {
return metas;
}
/**
* @return the {@link URL} to the favicon that belongs to the document
*/
public URL getFavicon() {
return this.favicon;
}
public String getDescription() {
String s = (String) metas.get("description");

3
source/de/anomic/plasma/plasmaParser.java
Unescape View File

@ -765,7 +765,8 @@ public final class plasmaParser {
scraper.getText(),
scraper.getAnchors(),
scraper.getImages());
//scraper.close();
//scraper.close();
ppd.setFavicon(scraper.getFavicon());
return ppd;
} catch (MalformedURLException e) {
//e.printStackTrace();

15
source/de/anomic/plasma/plasmaParserDocument.java
Unescape View File

@ -83,6 +83,7 @@ public class plasmaParserDocument {
// text in image tags.
private Map hyperlinks, audiolinks, videolinks, applinks;
private Map emaillinks;
private URL favicon;
private boolean resorted;
private InputStream textStream;
@ -395,6 +396,20 @@ public class plasmaParserDocument {
images.addAll(doc.getImages());
}
/**
* @return the {@link URL} to the favicon that belongs to the document
*/
public URL getFavicon() {
return this.favicon;
}
/**
* @param faviconURL the {@link URL} to the favicon that belongs to the document
*/
public void setFavicon(URL faviconURL) {
this.favicon = faviconURL;
}
public void close() {
// try close the output stream
if (this.textStream != null) {

39
source/de/anomic/plasma/plasmaSnippetCache.java
Unescape View File

@ -89,6 +89,15 @@ public class plasmaSnippetCache {
private int snippetsScoreCounter;
private kelondroMScoreCluster snippetsScore;
private HashMap snippetsCache;
/**
* a cache holding URLs to favicons specified by the page content, e.g. by using the html link-tag. e.g.
* <pre>
* &lt;link rel="shortcut icon" type="image/x-icon" href="../src/favicon.ico"&gt;
* </pre>
*/
private HashMap faviconCache;
private plasmaHTCache cacheManager;
private plasmaParser parser;
private serverLog log;
@ -106,7 +115,8 @@ public class plasmaSnippetCache {
this.sb = theSb;
this.snippetsScoreCounter = 0;
this.snippetsScore = new kelondroMScoreCluster();
this.snippetsCache = new HashMap();
this.snippetsCache = new HashMap();
this.faviconCache = new HashMap();
}
public class TextSnippet {
@ -115,12 +125,19 @@ public class plasmaSnippetCache {
private String error;
private int errorCode;
private Set remaingHashes;
private URL favicon;
public TextSnippet(URL url, String line, int errorCode, Set remaingHashes, String errortext) {
this(url,line,errorCode,remaingHashes,errortext,null);
}
public TextSnippet(URL url, String line, int errorCode, Set remaingHashes, String errortext, URL favicon) {
this.url = url;
this.line = line;
this.errorCode = errorCode;
this.error = errortext;
this.remaingHashes = remaingHashes;
this.favicon = favicon;
}
public URL getUrl() {
return this.url;
@ -213,6 +230,10 @@ public class plasmaSnippetCache {
}
return l.toString().trim();
}
public URL getFavicon() {
return this.favicon;
}
}
public class MediaSnippet {
@ -244,9 +265,9 @@ public class plasmaSnippetCache {
int source = SOURCE_CACHE;
String wordhashes = yacySearch.set2string(queryhashes);
String line = retrieveFromCache(wordhashes, urlhash);
if (line != null) {
if (line != null) {
//System.out.println("found snippet for URL " + url + " in cache: " + line);
return new TextSnippet(url, line, source, null, null);
return new TextSnippet(url, line, source, null, null,(URL)this.faviconCache.get(urlhash));
}
/* ===========================================================================
@ -300,7 +321,7 @@ public class plasmaSnippetCache {
* =========================================================================== */
plasmaParserDocument document = null;
try {
document = parseDocument(url, resContentLength, resContent, resInfo);
document = parseDocument(url, resContentLength, resContent, resInfo);
} catch (ParserException e) {
return new TextSnippet(url, null, ERROR_PARSER_FAILED, queryhashes, e.getMessage()); // cannot be parsed
} finally {
@ -311,12 +332,14 @@ public class plasmaSnippetCache {
/* ===========================================================================
* COMPUTE SNIPPET
* =========================================================================== */
* =========================================================================== */
URL resFavicon = document.getFavicon();
if (resFavicon != null) this.faviconCache.put(urlhash,resFavicon);
// we have found a parseable non-empty file: use the lines
// compute snippet from text
final Iterator sentences = document.getSentences(pre);
if (sentences == null) return new TextSnippet(url, null, ERROR_PARSER_NO_LINES, queryhashes, "parser returned no sentences");
if (sentences == null) return new TextSnippet(url, null, ERROR_PARSER_NO_LINES, queryhashes, "parser returned no sentences",resFavicon);
Object[] tsr = computeTextSnippet(sentences, queryhashes, snippetMaxLength);
String textline = (tsr == null) ? null : (String) tsr[0];
Set remainingHashes = (tsr == null) ? queryhashes : (Set) tsr[1];
@ -335,13 +358,13 @@ public class plasmaSnippetCache {
//if (hrefline != null) line += (line.length() == 0) ? hrefline : "<br />" + hrefline;
if (textline != null) line += (line.length() == 0) ? textline : "<br />" + textline;
if ((line == null) || (remainingHashes.size() > 0)) return new TextSnippet(url, null, ERROR_NO_MATCH, remainingHashes, "no matching snippet found");
if ((line == null) || (remainingHashes.size() > 0)) return new TextSnippet(url, null, ERROR_NO_MATCH, remainingHashes, "no matching snippet found",resFavicon);
if (line.length() > snippetMaxLength) line = line.substring(0, snippetMaxLength);
// finally store this snippet in our own cache
storeToCache(wordhashes, urlhash, line);
document.close();
return new TextSnippet(url, line, source, null, null);
return new TextSnippet(url, line, source, null, null, resFavicon);
}
/**

Write Preview
Loading…
Cancel
Save