fixes for metadata retrieval and presentation

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6855 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/CrawlResults.java

@@ -79,7 +79,7 @@ public class CrawlResults {
             tabletype == EventOrigin.LOCAL_CRAWLING &&
             sb.crawlResults.getStackSize(EventOrigin.LOCAL_CRAWLING) == 0) {
             // the main menu does a request to the local crawler page, but in case this table is empty, the overview page is shown
-            tabletype = EventOrigin.UNKNOWN;
+            tabletype = (sb.crawlResults.getStackSize(EventOrigin.SURROGATES) == 0) ? EventOrigin.UNKNOWN : EventOrigin.SURROGATES;
         }
         
         // check if authorization is needed and/or given
@@ -100,7 +100,7 @@ public class CrawlResults {
             }
         }
 
-        if(post != null) {
+        if (post != null) {
         // custom number of lines
         if (post.containsKey("count")) {
             lines = Integer.parseInt(post.get("count", "500"));

htroot/ViewFile.java

@@ -90,15 +90,12 @@ public class ViewFile {
         
         // get segment
         Segment indexSegment = null;
-        if (post.containsKey("segment")) {
-            String segmentName = post.get("segment");
-            if (sb.indexSegments.segmentExist(segmentName)) {
-                indexSegment = sb.indexSegments.segment(segmentName);
-            }
+        boolean authorized = sb.verifyAuthentication(header, false);
+        if (post != null && post.containsKey("segment") && authorized) {
+            indexSegment = sb.indexSegments.segment(post.get("segment"));
         } else {
-            // take default segment
             indexSegment = sb.indexSegments.segment(Segments.Process.PUBLIC);
-        }
+        }        
         
         prop.put("display", display);
         prop.put("error_display", display);
@@ -180,7 +177,20 @@ public class ViewFile {
             Log.logException(e);
             resource = null;
         }
-        responseHeader = Cache.getResponseHeader(url);
+        
+        if (resource == null && authorized) {
+            // load resource from net
+            Response response = null;
+            try {
+                response = sb.loader.load(url, true, false);
+            } catch (IOException e) {
+                Log.logException(e);
+            }
+            if (response != null) resource = response.getContent();
+            responseHeader = response.getResponseHeader();
+        }
+        
+        if (responseHeader == null) responseHeader = Cache.getResponseHeader(url);
 
         // if the resource body was not cached we try to load it from web
         if (resource == null) {

htroot/api/yacydoc.html

@@ -22,15 +22,17 @@ you can validate it with http://validator.w3.org/
   <h1 property="dc:Title">#[dc_title]#</h1>
 
   <dl>
-  <dt>Author</dt><dd property="dc:Creator">#[dc_creator]#</dd>
-  <dt>Description</dt><dd property="dc:Description">#[dc_description]#</dd>
-  <dt>Subject</dt><dd property="dc:Subject">#[dc_subject]#</dd>
-  <dt>Publisher</dt><dd property="dc:Publisher">#[dc_publisher]#</dd>
-  <dt>Contributor</dt><dd property="dc:Contributor">#[dc_contributor]#</dd>
-  <dt>Date</dt><dd property="dc:Date">#[dc_date]#</dd>
-  <dt>Type</dt><dd property="dc:Type">yacy:doctype:#[dc_type]#</dd>
-  <dt>Identifier</dt><dd property="dc:Identifier">#[dc_identifier]#</dd>
-  <dt>Language</dt><dd property="dc:Language">#[dc_language]#</dd>
+  <dt>Title</dt><dd property="dc:title">#[dc_title]#</dd>
+  <dt>Author</dt><dd property="dc:creator">#[dc_creator]#</dd>
+  <dt>Description</dt><dd property="dc:description">#[dc_description]#</dd>
+  <dt>Subject</dt><dd property="dc:subject">#[dc_subject]#</dd>
+  <dt>Publisher</dt><dd property="dc:publisher">#[dc_publisher]#</dd>
+  <dt>Contributor</dt><dd property="dc:contributor">#[dc_contributor]#</dd>
+  <dt>Date</dt><dd property="dc:date">#[dc_date]#</dd>
+  <dt>Type</dt><dd property="dc:type">yacy:doctype:#[dc_type]#</dd>
+  <dt>YaCy Identifier</dt><dd property="dc:identifier">yacy:urlhash:#[yacy_urlhash]#</dd>
+  <dt>Identifier</dt><dd><a href="#[dc_identifier]# property="dc:identifier">#[dc_identifier]#</a></dd>
+  <dt>Language</dt><dd property="dc:language">#[dc_language]#</dd>
   
   <dt>Load Date</dt><dd property="yacy:loaddate">#[yacy_loaddate]#</dd>
   <dt>Referrer Identifier</dt><dd property="yacy:referrer.hash">yacy:urlhash:#[yacy_referrer_hash]#</dd>

htroot/api/yacydoc.java

@@ -49,7 +49,8 @@ public class yacydoc {
         final Segment segment;
         boolean html = post != null && post.containsKey("html");
         prop.setLocalized(html);
-        if (post != null && post.containsKey("segment") && sb.verifyAuthentication(header, false)) {
+        boolean authorized = sb.verifyAuthentication(header, false);
+        if (post != null && post.containsKey("segment") && authorized) {
             segment = sb.indexSegments.segment(post.get("segment"));
         } else {
             segment = sb.indexSegments.segment(Segments.Process.PUBLIC);
@@ -68,7 +69,7 @@ public class yacydoc {
 
         if (post == null) return prop;
         
-        String urlstring = post.get("urlstring", "").trim();
+        String urlstring = post.get("url", "").trim();
         String urlhash = post.get("urlhash", "").trim();
         if (urlstring.length() == 0 && urlhash.length() == 0) return prop;
 
@@ -93,7 +94,7 @@ public class yacydoc {
         
         prop.putXML("dc_title", metadata.dc_title());
         prop.putXML("dc_creator", metadata.dc_creator());
-        prop.putXML("dc_description", "");
+        prop.putXML("dc_description", ""); // this is the fulltext part in the surrogate
         prop.putXML("dc_subject", metadata.dc_subject());
         prop.putXML("dc_publisher", "");
         prop.putXML("dc_contributor", "");
@@ -102,6 +103,7 @@ public class yacydoc {
         prop.putXML("dc_identifier", metadata.url().toNormalform(false, true));
         prop.putXML("dc_language", entry.language());
 
+        prop.put("yacy_urlhash", metadata.url().hash());
         prop.putXML("yacy_loaddate", entry.loaddate().toString());
         prop.putXML("yacy_referrer_hash", (le == null) ? "" : new String(le.hash()));
         prop.putXML("yacy_referrer_url", (le == null) ? "" : le.metadata().url().toNormalform(false, true));

htroot/api/yacydoc.xml

@@ -15,8 +15,8 @@ you can validate it with http://www.stg.brown.edu/service/xmlvalid/
   <dc:publisher.URL>#[dc_publisher]#</dc:publisher.URL>
   <dc:date>#[dc_date]#</dc:date>
   <dc:type>yacy:doctype:#[dc_type]#</dc:type>
-  <dc:identifier SCHEME="yacy:urlhash">yacy:urlhash:#[dc_identifier]#</dc:identifier>
-  <dc:identifier SCHEME="URL">#[dc_publisher]#</dc:identifier>
+  <dc:identifier SCHEME="yacy:urlhash">yacy:urlhash:#[yacy_urlhash]#</dc:identifier>
+  <dc:identifier SCHEME="URL">#[dc_identifier]#</dc:identifier>
   <dc:format.extent>#[yacy_size]#</dc:format.extent>
   <dc:language SCHEME="ISO639-2">#[dc_language]#</dc:language>
   <yacy:loaddate>#[yacy_loaddate]#</yacy:loaddate>