- fixed some data types in new search stack

- added image domain presentation to image preview
- added new search page to menu
- added automatic re-search when an old search profile is requested and a crawl is ongoing,
  to fetch newly crawled entries

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4501 6c8d7289-2bf4-0310-a012-ef5d649a1542

htroot/env/templates/header.template

@@ -5,7 +5,8 @@
   <li class="menugroup" id="menugroupGlobalIndex">
     <h3>Search&nbsp;&amp;&nbsp;Browse</h3>
     <ul class="menu">
-      <li><a href="/index.html?display=1" accesskey="s" class="MenuItemLink">Search Page</a></li>
+      <li><a href="/index.html?display=1" accesskey="s" class="MenuItemLink">(old) Search Page</a></li>
+      <li><a href="/yacy/user/ysearch.html?display=1" accesskey="s" class="MenuItemLink">(new) Search Page</a></li>
       <li><a href="/Ranking_p.html" class="MenuItemLink lock">Ranking Config</a></li>
       <li><a href="/Surftips.html?display=1" class="MenuItemLink">Surftips</a></li>
       <li><a href="/Wiki.html?display=1" class="MenuItemLink">Publication</a></li>

htroot/yacy/user/ysearch.java

@@ -166,7 +166,7 @@ public class ysearch {
         int contentdomCode = plasmaSearchQuery.contentdomParser(post.get("contentdom", "text"));
         
         // patch until better search profiles are available
-        if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 30)) itemsPerPage = 30;
+        if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;
         
         serverObjects prop = new serverObjects();
         if (post.get("cat", "href").equals("href")) {

htroot/yacy/user/ysearchitem.html

@@ -12,8 +12,8 @@
     <a href="#[href]#" class="thumblink" onclick="return hs.expand(this)">
       <img src="/ViewImage.png?maxwidth=96&maxheight=96&code=#[code]#" alt="#[name]#">
     </a>
-    <div class="highslide-caption"><a href="#[href]#">#[name]#</a></div>
-    <div class="snippet"><a href="#[href]#">#[name]##[attr]#</a></div>
+    <div class="highslide-caption"><a href="#[href]#">#[name]#<br \><a href="#[source]#">#[sourcedom]#</a></a></div>
+    <div class="TableCellDark"><a href="#[href]#">#[name]#</a><br \>#[attr]#</div>
   </div>
   #{/items}#
   ::

htroot/yacy/user/ysearchitem.java

@@ -235,7 +235,9 @@ public class ysearchitem {
                 prop.putHTML("content_items_0_href", ms.href.toNormalform(true, false));
                 prop.put("content_items_0_code", sb.licensedURLs.aquireLicense(ms.href));
                 prop.putHTML("content_items_0_name", shorten(ms.name, namelength));
-                prop.put("content_items_0_attr", (ms.attr.equals("-1 x -1")) ? "" : " (" + ms.attr + ")"); // attributes, here: original size of image
+                prop.put("content_items_0_attr", (ms.attr.equals("-1 x -1")) ? "" : "(" + ms.attr + ")"); // attributes, here: original size of image
+                prop.put("content_items_0_source", ms.source.toNormalform(true, false));
+                prop.put("content_items_0_sourcedom", ms.source.getHost());
                 prop.put("content_items", 1);
             }
             return prop;

htroot/yacysearch.java

@@ -167,7 +167,7 @@ public class yacysearch {
         int contentdomCode = plasmaSearchQuery.contentdomParser(post.get("contentdom", "text"));
         
         // patch until better search profiles are available
-        if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 30)) itemsPerPage = 30;
+        if ((contentdomCode != plasmaSearchQuery.CONTENTDOM_TEXT) && (itemsPerPage <= 32)) itemsPerPage = 32;
         
         serverObjects prop = new serverObjects();
         if (post.get("cat", "href").equals("href")) {

htroot/yacysearchitem.html

@@ -25,8 +25,8 @@
     <a href="#[href]#" class="thumblink" onclick="return hs.expand(this)">
       <img src="/ViewImage.png?maxwidth=96&maxheight=96&code=#[code]#" alt="#[name]#">
     </a>
-    <div class="highslide-caption"><a href="#[href]#">#[name]#</a></div>
-    <div class="TableCellDark"><a href="#[href]#">#[name]##[attr]#</a></div>
+    <div class="highslide-caption"><a href="#[href]#">#[name]#<br \><a href="#[source]#">#[sourcedom]#</a></a></div>
+    <div class="TableCellDark"><a href="#[href]#">#[name]#</a><br \>#[attr]#</div>
   </div>
   #{/items}#
   ::

htroot/yacysearchitem.java

@@ -236,7 +236,9 @@ public class yacysearchitem {
                 prop.putHTML("content_items_0_href", ms.href.toNormalform(true, false));
                 prop.put("content_items_0_code", sb.licensedURLs.aquireLicense(ms.href));
                 prop.putHTML("content_items_0_name", shorten(ms.name, namelength));
-                prop.put("content_items_0_attr", (ms.attr.equals("-1 x -1")) ? "" : " (" + ms.attr + ")"); // attributes, here: original size of image
+                prop.put("content_items_0_attr", (ms.attr.equals("-1 x -1")) ? "" : "(" + ms.attr + ")"); // attributes, here: original size of image
+                prop.put("content_items_0_source", ms.source.toNormalform(true, false));
+                prop.put("content_items_0_sourcedom", ms.source.getHost());
                 prop.put("content_items", 1);
             }
             return prop;

source/de/anomic/kelondro/kelondroSortStack.java

@@ -57,18 +57,17 @@ public class kelondroSortStack<E> {
         push(se.element, se.weight);
     }
     
-    public synchronized void push(E element, long weight) {
+    public synchronized void push(E element, Long weight) {
         if (exists(element)) return;
         
         // manipulate weight in such a way that it has no conflicts
-        Long w = new Long(weight);
-        while (this.onstack.containsKey(w)) w = new Long(w.longValue() + 1);
+        while (this.onstack.containsKey(weight)) weight = new Long(weight.longValue() + 1);
         
         // put the element on the stack
-        this.onstack.put(w, element);
+        this.onstack.put(weight, element);
         
         // register it for double-check
-        this.instack.add(element.hashCode());
+        this.instack.add(new Integer(element.hashCode()));
 
         // check maximum size of the stack an remove elements if the stack gets too large
         if (this.maxsize <= 0) return;
@@ -82,7 +81,7 @@ public class kelondroSortStack<E> {
         if (this.onstack.size() == 0) return null;
         Long w = this.onstack.firstKey();
         E element = this.onstack.get(w);
-        return new stackElement(element, w.longValue());
+        return new stackElement(element, w);
     }
     
     public synchronized stackElement pop() {
@@ -92,18 +91,18 @@ public class kelondroSortStack<E> {
         if (this.onstack.size() == 0) return null;
         Long w = this.onstack.firstKey();
         E element = this.onstack.remove(w);
-        stackElement se = new stackElement(element, w.longValue());
+        stackElement se = new stackElement(element, w);
         return se;
     }
     
     public boolean exists(E element) {
         // uses the hashCode of the element to find out of the element had been on the list or the stack
-        return this.instack.contains(element.hashCode());
+        return this.instack.contains(new Integer(element.hashCode()));
     }
     
     public boolean exists(int hashcode) {
         // uses the hashCode of the element to find out of the element had been on the list or the stack
-        return this.instack.contains(hashcode);
+        return this.instack.contains(new Integer(hashcode));
     }
     
     public stackElement get(int hashcode) {
@@ -111,7 +110,7 @@ public class kelondroSortStack<E> {
         Map.Entry<Long, E> entry;
         while (i.hasNext()) {
             entry = i.next();
-            if (entry.getValue().hashCode() == hashcode) return new stackElement(entry.getValue(), entry.getKey().longValue());
+            if (entry.getValue().hashCode() == hashcode) return new stackElement(entry.getValue(), entry.getKey());
         }
         return null;
     }
@@ -123,7 +122,7 @@ public class kelondroSortStack<E> {
         while (i.hasNext()) {
             entry = i.next();
             if (entry.getValue().hashCode() == hashcode) {
-                se = new stackElement(entry.getValue(), entry.getKey().longValue());
+                se = new stackElement(entry.getValue(), entry.getKey());
                 this.onstack.remove(se.weight);
                 return se;
             }
@@ -137,9 +136,9 @@ public class kelondroSortStack<E> {
     }
     
     public class stackElement {
-        public long weight;
+        public Long weight;
         public E element;
-        public stackElement(E element, long weight) {
+        public stackElement(E element, Long weight) {
             this.element = element;
             this.weight = weight;
         }

source/de/anomic/kelondro/kelondroSortStore.java

@@ -50,7 +50,7 @@ public class kelondroSortStore<E> extends kelondroSortStack<E> {
         return this.offstack.size();
     }
 
-    public synchronized void push(E element, long weight) {
+    public synchronized void push(E element, Long weight) {
         super.push(element, weight);
         if (this.maxsize <= 0) return;
         while ((this.onstack.size() > 0) && (super.onstack.size() + this.offstack.size() > this.maxsize)) {
@@ -79,7 +79,7 @@ public class kelondroSortStore<E> extends kelondroSortStack<E> {
         while (position >= this.offstack.size()) {
             Long w = this.onstack.firstKey();
             E element = this.onstack.remove(w);
-            stackElement se = new stackElement(element, w.longValue());
+            stackElement se = new stackElement(element, w);
             this.offstack.add(se);
         }
         return this.offstack.get(position);
@@ -94,7 +94,7 @@ public class kelondroSortStore<E> extends kelondroSortStack<E> {
             while (this.onstack.size() > 0) {
                 Long w = this.onstack.firstKey();
                 E element = this.onstack.remove(w);
-                stackElement se = new stackElement(element, w.longValue());
+                stackElement se = new stackElement(element, w);
                 this.offstack.add(se);
             }
             return this.offstack;
@@ -103,7 +103,7 @@ public class kelondroSortStore<E> extends kelondroSortStack<E> {
         while (this.onstack.size() < count) {
             Long w = this.onstack.firstKey();
             E element = this.onstack.remove(w);
-            stackElement se = new stackElement(element, w.longValue());
+            stackElement se = new stackElement(element, w);
             this.offstack.add(se);
         }
         return this.offstack;

source/de/anomic/plasma/plasmaSearchEvent.java

@@ -216,7 +216,7 @@ public final class plasmaSearchEvent {
                     snippetComputationAllTime += resultEntry.snippetComputationTime;
                 
                     // place the result to the result vector
-                    result.push(resultEntry, rankedCache.getOrder().cardinal(resultEntry.word()));
+                    result.push(resultEntry, new Long(rankedCache.getOrder().cardinal(resultEntry.word())));
 
                     // add references
                     synchronized (rankedCache) {
@@ -464,6 +464,14 @@ public final class plasmaSearchEvent {
                 event.query = query;
             }
         
+            // if a local crawl is ongoing, do another local search to enrich the current results with more
+            // entries that can possibly come out of the running crawl
+            if (plasmaSwitchboard.getSwitchboard().crawlQueues.noticeURL.size() > 0) {
+                synchronized (event.rankedCache) {
+                    event.rankedCache.execQuery();
+                }
+            }
+            
             // if worker threads had been alive, but did not succeed, start them again to fetch missing links
             if ((query.onlineSnippetFetch) &&
                 (!event.anyWorkerAlive()) &&
@@ -536,7 +544,7 @@ public final class plasmaSearchEvent {
                 
                 // place the result to the result vector
                 if (!result.exists(resultEntry)) {
-                    result.push(resultEntry, rankedCache.getOrder().cardinal(resultEntry.word()));
+                    result.push(resultEntry, new Long(rankedCache.getOrder().cardinal(resultEntry.word())));
                     rankedCache.addReferences(resultEntry);
                 }
                 //System.out.println("DEBUG SNIPPET_LOADING: thread " + id + " got " + resultEntry.url());
@@ -620,7 +628,7 @@ public final class plasmaSearchEvent {
                 if (imagemedia != null) {
                     for (int j = 0; j < imagemedia.size(); j++) {
                         ms = imagemedia.get(j);
-                        images.push(ms, ms.ranking);
+                        images.push(ms, new Long(ms.ranking));
                     }
                 }
             }

source/de/anomic/plasma/plasmaSearchRankingProcess.java

@@ -154,16 +154,16 @@ public final class plasmaSearchRankingProcess {
             
             // load url
             if (sortorder == 0) {
-                this.stack.push(ientry, ientry.urlHash().hashCode());
-                this.urlhashes.put(ientry.urlHash(), ientry.urlHash().hashCode());
+                this.stack.push(ientry, new Long(ientry.urlHash().hashCode()));
+                this.urlhashes.put(ientry.urlHash(), new Integer(ientry.urlHash().hashCode()));
             } else {
                 uentry = wordIndex.loadedURL.load(ientry.urlHash(), ientry, 0);
                 if (uentry == null) {
                     this.misses.add(ientry.urlHash());
                 } else {
                     u = uentry.comp().url().toNormalform(false, true);
-                    this.stack.push(ientry, u.hashCode());
-                    this.urlhashes.put(ientry.urlHash(), u.hashCode());
+                    this.stack.push(ientry, new Long(u.hashCode()));
+                    this.urlhashes.put(ientry.urlHash(), new Integer(u.hashCode()));
                 }
             }
             
@@ -306,7 +306,7 @@ public final class plasmaSearchRankingProcess {
                 continue;
             }
             o = m.top();
-            if (o.weight < bestEntry.weight) {
+            if (o.weight.longValue() < bestEntry.weight.longValue()) {
                 bestEntry = o;
             }
         }
@@ -322,7 +322,7 @@ public final class plasmaSearchRankingProcess {
         // returns from the current RWI list the best URL entry and removed this entry from the list
         while ((stack.size() > 0) || (size() > 0)) {
             kelondroSortStack<indexRWIVarEntry>.stackElement obrwi = bestRWI(skipDoubleDom);
-            indexURLEntry u = wordIndex.loadedURL.load(obrwi.element.urlHash(), obrwi.element, obrwi.weight);
+            indexURLEntry u = wordIndex.loadedURL.load(obrwi.element.urlHash(), obrwi.element, obrwi.weight.longValue());
             if (u != null) {
             	indexURLEntry.Components comp = u.comp();
             	if (comp.url() != null) this.handover.put(u.hash(), comp.url().toNormalform(true, false)); // remember that we handed over this url

source/de/anomic/plasma/plasmaSnippetCache.java

@@ -230,12 +230,13 @@ public class plasmaSnippetCache {
     
     public static class MediaSnippet {
         public int type;
-        public yacyURL href;
+        public yacyURL href, source;
         public String name, attr;
         public int ranking;
-        public MediaSnippet(int type, yacyURL href, String name, String attr, int ranking) {
+        public MediaSnippet(int type, yacyURL href, String name, String attr, int ranking, yacyURL source) {
             this.type = type;
             this.href = href;
+            this.source = source; // the web page where the media resource appeared
             this.name = name;
             this.attr = attr;
             this.ranking = ranking; // the smaller the better! small values should be shown first
@@ -682,12 +683,12 @@ public class plasmaSnippetCache {
             desc = entry.getValue();
             s = removeAppearanceHashes(url.toNormalform(false, false), queryhashes);
             if (s.size() == 0) {
-                result.add(new MediaSnippet(mediatype, url, desc, null, 0));
+                result.add(new MediaSnippet(mediatype, url, desc, null, 0, document.dc_source()));
                 continue;
             }
             s = removeAppearanceHashes(desc, s);
             if (s.size() == 0) {
-                result.add(new MediaSnippet(mediatype, url, desc, null, 0));
+                result.add(new MediaSnippet(mediatype, url, desc, null, 0, document.dc_source()));
                 continue;
             }
         }
@@ -712,15 +713,13 @@ public class plasmaSnippetCache {
             s = removeAppearanceHashes(url.toNormalform(false, false), queryhashes);
             if (s.size() == 0) {
                 int ranking = ientry.hashCode();
-                System.out.println(ranking);
-                result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height(), ranking));
+                result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height(), ranking, document.dc_source()));
                 continue;
             }
             s = removeAppearanceHashes(desc, s);
             if (s.size() == 0) {
                 int ranking = ientry.hashCode();
-                System.out.println(ranking);
-                result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height(), ranking));
+                result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height(), ranking, document.dc_source()));
                 continue;
             }
         }