From 85282b1d980c44af88a231590e4c68d9aa65a103 Mon Sep 17 00:00:00 2001
From: orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542>
Date: Wed, 23 Nov 2005 01:40:02 +0000
Subject: [PATCH] enhanced YBR recognition and search result heuristics

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@1121 6c8d7289-2bf4-0310-a012-ef5d649a1542
---
 build.properties                              |  2 +-
 htroot/index.html                             |  1 +
 htroot/index.java                             |  4 +-
 .../de/anomic/kelondro/kelondroBinSearch.java | 95 +++++++++++++++++++
 .../de/anomic/kelondro/kelondroHashtable.java |  2 +-
 source/de/anomic/kelondro/kelondroTree.java   |  8 +-
 .../anomic/plasma/plasmaSearchPreOrder.java   | 18 ++--
 .../de/anomic/plasma/plasmaSearchResult.java  |  7 ++
 .../de/anomic/plasma/plasmaSwitchboard.java   |  2 +-
 source/de/anomic/server/serverByteBuffer.java |  2 +-
 10 files changed, 123 insertions(+), 18 deletions(-)
 create mode 100644 source/de/anomic/kelondro/kelondroBinSearch.java

diff --git a/build.properties b/build.properties
index 332ec64c0..d123ceb12 100644
--- a/build.properties
+++ b/build.properties
@@ -3,7 +3,7 @@ javacSource=1.4
 javacTarget=1.4
 
 # Release Configuration
-releaseVersion=0.414
+releaseVersion=0.415
 releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 #releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
 releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}
diff --git a/htroot/index.html b/htroot/index.html
index ae173a7e7..d5980f24a 100644
--- a/htroot/index.html
+++ b/htroot/index.html
@@ -43,6 +43,7 @@ Resource:
 <select NAME="time">
 <option #(time-1)#::selected#(/time-1)#>1</option>
 <option #(time-3)#::selected#(/time-3)#>3</option>
+<option #(time-6)#::selected#(/time-6)#>6</option>
 <option #(time-10)#::selected#(/time-10)#>10</option>
 <option #(time-30)#::selected#(/time-30)#>30</option>
 <option #(time-60)#::selected#(/time-60)#>60</option>
diff --git a/htroot/index.java b/htroot/index.java
index 0aabe1025..c0f8f9991 100644
--- a/htroot/index.java
+++ b/htroot/index.java
@@ -114,7 +114,8 @@ public class index {
             prop.put("resource-local", ((global) ? 0 : 1));
             prop.put("time-1", 0);
             prop.put("time-3", 0);
-            prop.put("time-10", 1);
+            prop.put("time-6", 1);
+            prop.put("time-10", 0);
             prop.put("time-30", 0);
             prop.put("time-60", 0);
             prop.put("results", "");
@@ -263,6 +264,7 @@ public class index {
         prop.put("resource-local", ((global) ? 0 : 1));
         prop.put("time-1", ((searchtime == 1000) ? 1 : 0));
         prop.put("time-3", ((searchtime == 3000) ? 1 : 0));
+        prop.put("time-6", ((searchtime == 6000) ? 1 : 0));
         prop.put("time-10", ((searchtime == 10000) ? 1 : 0));
         prop.put("time-30", ((searchtime == 30000) ? 1 : 0));
         prop.put("time-60", ((searchtime == 60000) ? 1 : 0));
diff --git a/source/de/anomic/kelondro/kelondroBinSearch.java b/source/de/anomic/kelondro/kelondroBinSearch.java
new file mode 100644
index 000000000..2e2457f30
--- /dev/null
+++ b/source/de/anomic/kelondro/kelondroBinSearch.java
@@ -0,0 +1,95 @@
+// kelondroBinSearch.java
+// -----------------------
+// part of The Kelondro Database
+// (C) by Michael Peter Christen; mc@anomic.de
+// first published on http://www.anomic.de
+// Frankfurt, Germany, 2005
+// created 22.11.2005
+//
+// $LastChangedDate: 2005-09-22 22:01:26 +0200 (Thu, 22 Sep 2005) $
+// $LastChangedRevision: 774 $
+// $LastChangedBy: orbiter $
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+//
+// Using this software in any meaning (reading, learning, copying, compiling,
+// running) means that you agree that the Author(s) is (are) not responsible
+// for cost, loss of data or any harm that may be caused directly or indirectly
+// by usage of this softare or this documentation. The usage of this software
+// is on your own risk. The installation and usage (starting/running) of this
+// software may allow other people or application to access your computer and
+// any attached devices and is highly dependent on the configuration of the
+// software which must be done by the user of the software; the author(s) is
+// (are) also not responsible for proper configuration and usage of the
+// software, even if provoked by documentation provided together with
+// the software.
+//
+// Any changes to this file according to the GPL as documented in the file
+// gpl.txt aside this file in the shipment you received can be done to the
+// lines that follows this copyright notice here, but changes must not be
+// done inside the copyright notive above. A re-distribution must contain
+// the intact and unchanged copyright notice.
+// Contributions and changes to the program code must be marked as such.
+
+package de.anomic.kelondro;
+
+
+public class kelondroBinSearch {
+    
+    private byte[] chunks;
+    private int    chunksize;
+    private byte[] buffer;
+    private int    count;
+    
+    public kelondroBinSearch(byte[] chunks, int chunksize) {
+        this.chunks = chunks;
+        this.chunksize = chunksize;
+        this.count = chunks.length / chunksize;
+        this.buffer = new byte[chunksize];
+    }
+    
+    public boolean contains(byte[] t) {
+        return contains(t, 0, this.count);
+    }
+
+    private synchronized boolean contains(byte[] t, int beginPos, int endPos) {
+        // the endPos is exclusive, beginPos is inclusive
+        // this method is synchronized to make the use of the buffer possible
+        if (beginPos >= endPos) return false;
+        int pivot = (beginPos + endPos) / 2;
+        if ((pivot < 0) || (pivot >= this.count)) return false;
+        selectBuffer(pivot);
+        int c = kelondroTree.compare(buffer, t);
+        if (c == 0) return true;
+        if (c < 0) /* buffer < t */ return contains(t, pivot + 1, endPos);
+        if (c > 0) /* buffer > t */ return contains(t, beginPos, pivot);
+        return false;
+    }
+
+    private void selectBuffer(int element) {
+        System.arraycopy(this.chunks, element * this.chunksize, this.buffer, 0, chunksize);
+    }
+    
+    public static void main(String[] args) {
+        String s = "4CEvsI8FRczRBo_ApRCkwfEbFLn1pIFXg39QGMgj5RHM6HpIMJq67QX3M5iQYr_LyI_5aGDaa_bYbRgJ9XnQjpmq6QkOoGWAoEaihRqhV3kItLFHjRtqauUR";
+        kelondroBinSearch bs = new kelondroBinSearch(s.getBytes(), 6);
+        for (int i = 0; i + 6 <= s.length(); i = i + 6) {
+            System.out.println(s.substring(i, i + 6) + ":" + ((bs.contains(s.substring(i, i + 6).getBytes())) ? "drin" : "draussen"));
+        }
+        for (int i = 0; i + 7 <= s.length(); i = i + 6) {
+            System.out.println(s.substring(i + 1, i + 7) + ":" + ((bs.contains(s.substring(i + 1, i + 7).getBytes())) ? "drin" : "draussen"));
+        }
+    }
+}
diff --git a/source/de/anomic/kelondro/kelondroHashtable.java b/source/de/anomic/kelondro/kelondroHashtable.java
index 2c85ae01a..12aee822d 100644
--- a/source/de/anomic/kelondro/kelondroHashtable.java
+++ b/source/de/anomic/kelondro/kelondroHashtable.java
@@ -1,4 +1,4 @@
-// kelondroArray.java
+// kelondroHashtable.java
 // ------------------
 // part of the Kelondro Database
 // (C) by Michael Peter Christen; mc@anomic.de
diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java
index c6ae259f3..df98b3199 100644
--- a/source/de/anomic/kelondro/kelondroTree.java
+++ b/source/de/anomic/kelondro/kelondroTree.java
@@ -1299,11 +1299,11 @@ public class kelondroTree extends kelondroRecords implements Comparator, kelondr
     // Returns -1, 0, or 1 as the first argument
     // is less than, equal to, or greater than the second.
     // two arrays are also equal if one array is a subset of the other's array with filled-up char(0)-values
-    public int compare(byte[] a, byte[] b) {
+    public static int compare(byte[] a, byte[] b) {
 	int i = 0;
-	int al = a.length;
-	int bl = b.length;
-	int len = (al > bl) ? bl : al;
+	final int al = a.length;
+	final int bl = b.length;
+	final int len = (al > bl) ? bl : al;
 	while (i < len) {
 	    if (a[i] > b[i]) return  1;
 	    if (a[i] < b[i]) return -1;
diff --git a/source/de/anomic/plasma/plasmaSearchPreOrder.java b/source/de/anomic/plasma/plasmaSearchPreOrder.java
index afb566450..2a8ac3248 100644
--- a/source/de/anomic/plasma/plasmaSearchPreOrder.java
+++ b/source/de/anomic/plasma/plasmaSearchPreOrder.java
@@ -44,16 +44,16 @@ package de.anomic.plasma;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.Set;
 import java.util.TreeMap;
 import java.util.Iterator;
 
 import de.anomic.server.serverCodings;
 import de.anomic.server.serverFileUtils;
+import de.anomic.kelondro.kelondroBinSearch;
 
 public final class plasmaSearchPreOrder {
     
-    private static Set[] ybrTables = null; // block-rank tables
+    private static kelondroBinSearch[] ybrTables = null; // block-rank tables
     private static boolean useYBR = true;
     
     private TreeMap pageAcc; // key = order hash; value = plasmaLURL.entry
@@ -62,12 +62,12 @@ public final class plasmaSearchPreOrder {
     public static void loadYBR(File rankingPath, int count) {
         // load ranking tables
         if (rankingPath.exists()) {
-            ybrTables = new Set[count];
+            ybrTables = new kelondroBinSearch[count];
             String ybrName;
             try {
                 for (int i = 0; i < count; i++) {
                     ybrName = "YBR-4-" + serverCodings.encodeHex(i, 2) + ".idx";
-                    ybrTables[i] = serverFileUtils.loadSet(new File(rankingPath, ybrName), 6, false);
+                    ybrTables[i] = new kelondroBinSearch(serverFileUtils.read(new File(rankingPath, ybrName)), 6);
                 }
             } catch (IOException e) {
                 ybrTables = null;
@@ -123,13 +123,13 @@ public final class plasmaSearchPreOrder {
     
     public void addEntry(plasmaWordIndexEntry indexEntry) {
         long ranking = 0;
-        long factor = 1024 * 1024;
+        long factor = 4096L*4096L;
         
         for (int i = 0; i < 3; i++) {
-            if (query.order[i].equals(plasmaSearchQuery.ORDER_QUALITY))  ranking  = factor * indexEntry.getQuality();
-            else if (query.order[i].equals(plasmaSearchQuery.ORDER_DATE)) ranking  = factor * indexEntry.getVirtualAge();
+            if (query.order[i].equals(plasmaSearchQuery.ORDER_QUALITY))  ranking  = factor * indexEntry.getQuality() / 64L;
+            else if (query.order[i].equals(plasmaSearchQuery.ORDER_DATE)) ranking  = factor * indexEntry.getVirtualAge() / 64L;
             else if (query.order[i].equals(plasmaSearchQuery.ORDER_YBR))  ranking  = factor * ybr_p(indexEntry.getUrlHash());
-            factor = factor / 1024;
+            factor = factor / 4096L;
         }
 
         pageAcc.put(serverCodings.encodeHex(ranking, 16) + indexEntry.getUrlHash(), indexEntry);
@@ -144,7 +144,7 @@ public final class plasmaSearchPreOrder {
         if (!(useYBR)) return 16;
         final String domHash = urlHash.substring(6);
         for (int i = 0; i < ybrTables.length; i++) {
-            if (ybrTables[i].contains(domHash)) {
+            if (ybrTables[i].contains(domHash.getBytes())) {
                 //System.out.println("YBR FOUND: " + urlHash + " (" + i + ")");
                 return i;
             }
diff --git a/source/de/anomic/plasma/plasmaSearchResult.java b/source/de/anomic/plasma/plasmaSearchResult.java
index 24cba50c2..40296a3a3 100644
--- a/source/de/anomic/plasma/plasmaSearchResult.java
+++ b/source/de/anomic/plasma/plasmaSearchResult.java
@@ -172,6 +172,13 @@ public final class plasmaSearchResult {
                 if (descrcomph.contains(queryhash)) ranking += 40L*4096L*4096L / descrcomps.length / query.queryHashes.size();
             }
             
+            // prefer short urls
+            ranking -= 64L * page.url().toString().length();
+            ranking -= 64L * urlcomps.length;
+            
+            // prefer long descriptions
+            ranking += 64L * (40 - Math.abs(40 - Math.min(40, page.descr().length())));
+            ranking += 64L * ( 8 - Math.abs( 8 - Math.min( 8, descrcomps.length)));
             
             // insert value
             //System.out.println("Ranking " + ranking + ", YBR-" + plasmaSearchPreOrder.ybr(indexEntry.getUrlHash()) + " for URL " + page.url());
diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java
index 1fa173723..60ba3b653 100644
--- a/source/de/anomic/plasma/plasmaSwitchboard.java
+++ b/source/de/anomic/plasma/plasmaSwitchboard.java
@@ -284,7 +284,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser
         // load ranking tables
         File rankingPath = new File(rootPath, "ranking/YBR");
         if (rankingPath.exists()) {
-            plasmaSearchPreOrder.loadYBR(rankingPath, 12);
+            plasmaSearchPreOrder.loadYBR(rankingPath, 15);
         }
 
         // read memory amount
diff --git a/source/de/anomic/server/serverByteBuffer.java b/source/de/anomic/server/serverByteBuffer.java
index 5f9941050..8cb24f1b4 100644
--- a/source/de/anomic/server/serverByteBuffer.java
+++ b/source/de/anomic/server/serverByteBuffer.java
@@ -313,7 +313,7 @@ public final class serverByteBuffer extends OutputStream {
         return equals(buffer, 0, pattern);
     }
     
-        public static boolean equals(byte[] buffer, int offset, byte[] pattern) {
+    public static boolean equals(byte[] buffer, int offset, byte[] pattern) {
         // compares two byte arrays: true, if pattern appears completely at offset position
         if (buffer.length < offset + pattern.length) return false;
         for (int i = 0; i < pattern.length; i++) if (buffer[offset + i] != pattern[i]) return false;