boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixes for metadata retrieval and presentation

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6855 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 5c5e6accdb
commit 06ff0c5b06
5 changed files with 38 additions and 24 deletions
Show Stats Download Patch File Download Diff File

4
htroot/CrawlResults.java
Unescape View File

@ -79,7 +79,7 @@ public class CrawlResults {
tabletype == EventOrigin.LOCAL_CRAWLING && tabletype == EventOrigin.LOCAL_CRAWLING &&
sb.crawlResults.getStackSize(EventOrigin.LOCAL_CRAWLING) == 0) { sb.crawlResults.getStackSize(EventOrigin.LOCAL_CRAWLING) == 0) {
// the main menu does a request to the local crawler page, but in case this table is empty, the overview page is shown // the main menu does a request to the local crawler page, but in case this table is empty, the overview page is shown
tabletype = EventOrigin.UNKNOWN; tabletype = (sb.crawlResults.getStackSize(EventOrigin.SURROGATES) == 0) ? EventOrigin.UNKNOWN : EventOrigin.SURROGATES;
} }
// check if authorization is needed and/or given // check if authorization is needed and/or given
@ -100,7 +100,7 @@ public class CrawlResults {
} }
} }
if(post != null) { if (post != null) {
// custom number of lines // custom number of lines
if (post.containsKey("count")) { if (post.containsKey("count")) {
lines = Integer.parseInt(post.get("count", "500")); lines = Integer.parseInt(post.get("count", "500"));

26
htroot/ViewFile.java
Unescape View File

@ -90,15 +90,12 @@ public class ViewFile {
// get segment // get segment
Segment indexSegment = null; Segment indexSegment = null;
if (post.containsKey("segment")) { boolean authorized = sb.verifyAuthentication(header, false);
String segmentName = post.get("segment"); if (post != null && post.containsKey("segment") && authorized) {
if (sb.indexSegments.segmentExist(segmentName)) { indexSegment = sb.indexSegments.segment(post.get("segment"));
indexSegment = sb.indexSegments.segment(segmentName);
}
} else { } else {
// take default segment
indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC); indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
} }
prop.put("display", display); prop.put("display", display);
prop.put("error_display", display); prop.put("error_display", display);
@ -180,7 +177,20 @@ public class ViewFile {
Log.logException(e); Log.logException(e);
resource = null; resource = null;
} }
responseHeader = Cache.getResponseHeader(url);
if (resource == null && authorized) {
// load resource from net
Response response = null;
try {
response = sb.loader.load(url, true, false);
} catch (IOException e) {
Log.logException(e);
}
if (response != null) resource = response.getContent();
responseHeader = response.getResponseHeader();
}
if (responseHeader == null) responseHeader = Cache.getResponseHeader(url);
// if the resource body was not cached we try to load it from web // if the resource body was not cached we try to load it from web
if (resource == null) { if (resource == null) {

20
htroot/api/yacydoc.html
Unescape View File

@ -22,15 +22,17 @@ you can validate it with http://validator.w3.org/
<h1 property="dc:Title">#[dc_title]#</h1> <h1 property="dc:Title">#[dc_title]#</h1>
<dl> <dl>
<dt>Author</dt><dd property="dc:Creator">#[dc_creator]#</dd> <dt>Title</dt><dd property="dc:title">#[dc_title]#</dd>
<dt>Description</dt><dd property="dc:Description">#[dc_description]#</dd> <dt>Author</dt><dd property="dc:creator">#[dc_creator]#</dd>
<dt>Subject</dt><dd property="dc:Subject">#[dc_subject]#</dd> <dt>Description</dt><dd property="dc:description">#[dc_description]#</dd>
<dt>Publisher</dt><dd property="dc:Publisher">#[dc_publisher]#</dd> <dt>Subject</dt><dd property="dc:subject">#[dc_subject]#</dd>
<dt>Contributor</dt><dd property="dc:Contributor">#[dc_contributor]#</dd> <dt>Publisher</dt><dd property="dc:publisher">#[dc_publisher]#</dd>
<dt>Date</dt><dd property="dc:Date">#[dc_date]#</dd> <dt>Contributor</dt><dd property="dc:contributor">#[dc_contributor]#</dd>
<dt>Type</dt><dd property="dc:Type">yacy:doctype:#[dc_type]#</dd> <dt>Date</dt><dd property="dc:date">#[dc_date]#</dd>
<dt>Identifier</dt><dd property="dc:Identifier">#[dc_identifier]#</dd> <dt>Type</dt><dd property="dc:type">yacy:doctype:#[dc_type]#</dd>
<dt>Language</dt><dd property="dc:Language">#[dc_language]#</dd> <dt>YaCy Identifier</dt><dd property="dc:identifier">yacy:urlhash:#[yacy_urlhash]#</dd>
<dt>Identifier</dt><dd><a href="#[dc_identifier]# property="dc:identifier">#[dc_identifier]#</a></dd>
<dt>Language</dt><dd property="dc:language">#[dc_language]#</dd>
<dt>Load Date</dt><dd property="yacy:loaddate">#[yacy_loaddate]#</dd> <dt>Load Date</dt><dd property="yacy:loaddate">#[yacy_loaddate]#</dd>
<dt>Referrer Identifier</dt><dd property="yacy:referrer.hash">yacy:urlhash:#[yacy_referrer_hash]#</dd> <dt>Referrer Identifier</dt><dd property="yacy:referrer.hash">yacy:urlhash:#[yacy_referrer_hash]#</dd>

8
htroot/api/yacydoc.java
Unescape View File

@ -49,7 +49,8 @@ public class yacydoc {
final Segment segment; final Segment segment;
boolean html = post != null && post.containsKey("html"); boolean html = post != null && post.containsKey("html");
prop.setLocalized(html); prop.setLocalized(html);
if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) { boolean authorized = sb.verifyAuthentication(header, false);
if (post != null && post.containsKey("segment") && authorized) {
segment = sb.indexSegments.segment(post.get("segment")); segment = sb.indexSegments.segment(post.get("segment"));
} else { } else {
segment = sb.indexSegments.segment(Segments.Process.PUBLIC); segment = sb.indexSegments.segment(Segments.Process.PUBLIC);
@ -68,7 +69,7 @@ public class yacydoc {
if (post == null) return prop; if (post == null) return prop;
String urlstring = post.get("urlstring", "").trim(); String urlstring = post.get("url", "").trim();
String urlhash = post.get("urlhash", "").trim(); String urlhash = post.get("urlhash", "").trim();
if (urlstring.length() == 0 && urlhash.length() == 0) return prop; if (urlstring.length() == 0 && urlhash.length() == 0) return prop;
@ -93,7 +94,7 @@ public class yacydoc {
prop.putXML("dc_title", metadata.dc_title()); prop.putXML("dc_title", metadata.dc_title());
prop.putXML("dc_creator", metadata.dc_creator()); prop.putXML("dc_creator", metadata.dc_creator());
prop.putXML("dc_description", ""); prop.putXML("dc_description", ""); // this is the fulltext part in the surrogate
prop.putXML("dc_subject", metadata.dc_subject()); prop.putXML("dc_subject", metadata.dc_subject());
prop.putXML("dc_publisher", ""); prop.putXML("dc_publisher", "");
prop.putXML("dc_contributor", ""); prop.putXML("dc_contributor", "");
@ -102,6 +103,7 @@ public class yacydoc {
prop.putXML("dc_identifier", metadata.url().toNormalform(false, true)); prop.putXML("dc_identifier", metadata.url().toNormalform(false, true));
prop.putXML("dc_language", entry.language()); prop.putXML("dc_language", entry.language());
prop.put("yacy_urlhash", metadata.url().hash());
prop.putXML("yacy_loaddate", entry.loaddate().toString()); prop.putXML("yacy_loaddate", entry.loaddate().toString());
prop.putXML("yacy_referrer_hash", (le == null) ? "" : new String(le.hash())); prop.putXML("yacy_referrer_hash", (le == null) ? "" : new String(le.hash()));
prop.putXML("yacy_referrer_url", (le == null) ? "" : le.metadata().url().toNormalform(false, true)); prop.putXML("yacy_referrer_url", (le == null) ? "" : le.metadata().url().toNormalform(false, true));

4
htroot/api/yacydoc.xml
Unescape View File

@ -15,8 +15,8 @@ you can validate it with http://www.stg.brown.edu/service/xmlvalid/
<dc:publisher.URL>#[dc_publisher]#</dc:publisher.URL> <dc:publisher.URL>#[dc_publisher]#</dc:publisher.URL>
<dc:date>#[dc_date]#</dc:date> <dc:date>#[dc_date]#</dc:date>
<dc:type>yacy:doctype:#[dc_type]#</dc:type> <dc:type>yacy:doctype:#[dc_type]#</dc:type>
<dc:identifier SCHEME="yacy:urlhash">yacy:urlhash:#[dc_identifier]#</dc:identifier> <dc:identifier SCHEME="yacy:urlhash">yacy:urlhash:#[yacy_urlhash]#</dc:identifier>
<dc:identifier SCHEME="URL">#[dc_publisher]#</dc:identifier> <dc:identifier SCHEME="URL">#[dc_identifier]#</dc:identifier>
<dc:format.extent>#[yacy_size]#</dc:format.extent> <dc:format.extent>#[yacy_size]#</dc:format.extent>
<dc:language SCHEME="ISO639-2">#[dc_language]#</dc:language> <dc:language SCHEME="ISO639-2">#[dc_language]#</dc:language>
<yacy:loaddate>#[yacy_loaddate]#</yacy:loaddate> <yacy:loaddate>#[yacy_loaddate]#</yacy:loaddate>

Write Preview
Loading…
Cancel
Save