fixed url crawl list display

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6908 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/crawler/Balancer.java

@@ -27,6 +27,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
@@ -515,41 +516,46 @@ public class Balancer {
     }
 
     public ArrayList<Request> top(int count) {
-    	count = Math.min(count, top.size());
     	final ArrayList<Request> cel = new ArrayList<Request>();
     	if (count == 0) return cel;
-    	byte[][] ta = new byte[count][];
+    	byte[][] ta = new byte[Math.min(count, top.size())][];
         ta = top.toArray(ta);
-    	synchronized (this) {
-	    	for (byte[] n: ta) {
-	    		try {
-					final Row.Entry rowEntry = urlFileIndex.get(n);
-					if (rowEntry == null) continue;
-					final Request crawlEntry = new Request(rowEntry);
-					cel.add(crawlEntry);
-					count--;
-					if (count <= 0) break;
-				} catch (IOException e) {}
-	    	}
-	    	
-	    	int depth = 0;
-	    	loop: while (count > 0) {
-    	    	// iterate over the domain stacks
-    	        for (LinkedList<byte[]> list: this.domainStacks.values()) {
-    	            if (list.size() <= depth) continue loop;
-    	            byte[] n = list.get(depth);
-                    try {
-                        Row.Entry rowEntry = urlFileIndex.get(n);
-                        if (rowEntry == null) continue;
-                        final Request crawlEntry = new Request(rowEntry);
-                        cel.add(crawlEntry);
-                        count--;
-                        if (count <= 0) break loop;
-                    } catch (IOException e) {}
-    	        }
-	    	}
-	    	
+    	for (byte[] n: ta) {
+    	    if (n == null) break;
+    		try {
+				final Row.Entry rowEntry = urlFileIndex.get(n);
+				if (rowEntry == null) continue;
+				final Request crawlEntry = new Request(rowEntry);
+				cel.add(crawlEntry);
+				count--;
+				if (count <= 0) break;
+			} catch (IOException e) {}
     	}
+    	
+    	int depth = 0;
+    	loop: while (count > 0) {
+	    	// iterate over the domain stacks
+    	    int celsize = cel.size();
+	        ll: for (LinkedList<byte[]> list: this.domainStacks.values()) {
+	            if (list.size() <= depth) continue ll;
+	            byte[] n = list.get(depth);
+                try {
+                    Row.Entry rowEntry = urlFileIndex.get(n);
+                    if (rowEntry == null) continue;
+                    final Request crawlEntry = new Request(rowEntry);
+                    cel.add(crawlEntry);
+                    count--;
+                    if (count <= 0) break loop;
+                } catch (IOException e) {}
+	        }
+    	    if (cel.size() == celsize) break loop;
+	        depth++;
+    	}
+    	
+    	if (cel.size() < count) try {
+            List<Row.Entry> list = urlFileIndex.top(count - cel.size());
+            for (Row.Entry entry: list) cel.add(new Request(entry));
+        } catch (IOException e) { }
     	return cel;
     }
     

source/net/yacy/kelondro/index/BufferedObjectIndex.java

@@ -25,6 +25,7 @@ package net.yacy.kelondro.index;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 
 import net.yacy.kelondro.index.Row.Entry;
 import net.yacy.kelondro.logging.Log;
@@ -172,6 +173,17 @@ public class BufferedObjectIndex implements ObjectIndex, Iterable<Row.Entry> {
         }
     }
 
+    public List<Row.Entry> top(int count) throws IOException {
+        List<Row.Entry> list = new ArrayList<Row.Entry>();
+        synchronized (this.backend) {
+            List<Row.Entry> list0 = buffer.top(count);
+            list.addAll(list0);
+            list0 = backend.top(count - list.size());
+            list.addAll(list0);
+        }
+        return list;
+    }
+
     public Entry removeOne() throws IOException {
         synchronized (this.backend) {
             if (!this.buffer.isEmpty()) {

source/net/yacy/kelondro/index/Cache.java

@@ -534,6 +534,10 @@ public final class Cache implements ObjectIndex, Iterable<Row.Entry> {
         return entry;
     }
 
+    public synchronized List<Row.Entry> top(int count) throws IOException {
+        return this.index.top(count);
+    }
+
     public final synchronized Row row() {
         return index.row();
     }

source/net/yacy/kelondro/index/HandleMap.java

@@ -34,6 +34,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
@@ -250,6 +251,15 @@ public final class HandleMap implements Iterable<Row.Entry> {
         return report;
     }
     
+    public final synchronized ArrayList<byte[]> top(int count) {
+        List<Row.Entry> list0 = index.top(count);
+        ArrayList<byte[]> list = new ArrayList<byte[]>();
+        for (Row.Entry entry: list0) {
+            list.add(entry.getPrimaryKeyBytes());
+        }
+        return list;
+    }
+    
     public final synchronized long remove(final byte[] key) {
         assert (key != null);
         final Row.Entry indexentry = index.remove(key);

source/net/yacy/kelondro/index/ObjectIndex.java

@@ -34,6 +34,7 @@ package net.yacy.kelondro.index;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 
 import net.yacy.kelondro.order.CloneableIterator;
 
@@ -55,6 +56,7 @@ public interface ObjectIndex extends Iterable<Row.Entry> {
     public boolean delete(byte[] key) throws IOException;
     public Row.Entry remove(byte[] key) throws IOException;
     public Row.Entry removeOne() throws IOException;
+    public List<Row.Entry> top(int count) throws IOException;
     public CloneableIterator<byte[]> keys(boolean up, byte[] firstKey) throws IOException; // iterates only the key
     public CloneableIterator<Row.Entry> rows(boolean up, byte[] firstKey) throws IOException; // iterates the whole row using the order of the keys
     public CloneableIterator<Row.Entry> rows() throws IOException; // iterates the whole row without any order

source/net/yacy/kelondro/index/ObjectIndexCache.java

@@ -24,6 +24,7 @@
 
 package net.yacy.kelondro.index;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -237,6 +238,15 @@ public final class ObjectIndexCache implements ObjectIndex, Iterable<Row.Entry>
         return null;
     }
     
+    public synchronized List<Row.Entry> top(int count) throws IOException {
+        List<Row.Entry> list = new ArrayList<Row.Entry>();
+        List<Row.Entry> list0 = index1.top(count);
+        list.addAll(list0);
+        list0 = index0.top(count - list.size());
+        list.addAll(list0);
+        return list;
+    }
+    
     public final synchronized int size() {
         if (index0 != null && index1 == null) {
             return index0.size();

source/net/yacy/kelondro/index/RowCollection.java

@@ -487,6 +487,20 @@ public class RowCollection implements Iterable<Row.Entry>, Cloneable {
         return r;
     }
     
+    public synchronized List<Row.Entry> top(int count) {
+        ArrayList<Row.Entry> list = new ArrayList<Row.Entry>();
+        if (chunkcount == 0) return list;
+        Row.Entry entry;
+        int cursor = chunkcount - 1;
+        while (count > 0 && cursor >= 0) {
+            entry = get(cursor, true);
+            list.add(entry);
+            count--;
+            cursor--;
+        }
+        return list;
+    }
+    
     public synchronized byte[] smallestKey() {
         if (chunkcount == 0) return null;
         this.sort();

source/net/yacy/kelondro/index/RowSetArray.java

@@ -21,6 +21,7 @@
 
 package net.yacy.kelondro.index;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
@@ -205,6 +206,24 @@ public final class RowSetArray implements ObjectIndex, Iterable<Row.Entry>, Clon
         return null;
     }
 
+    public List<Row.Entry> top(int count) {
+        List<Row.Entry> list = new ArrayList<Row.Entry>();
+        synchronized (this.array) {
+            for (int i = 0; i < this.array.length; i++) {
+                if (this.array[i] != null) {
+                    try {
+                        List<Row.Entry> list0 = this.array[i].top(count - list.size());
+                        list.addAll(list0);
+                    } catch (IOException e) {
+                        continue;
+                    }
+                }
+                if (list.size() >= count) return list;
+            }
+        }
+        return list;
+    }
+    
     public final Entry replace(final Entry row) throws RowSpaceExceededException {
         final int i = indexFor(row);
         if (i < 0) return null;

source/net/yacy/kelondro/table/SQLTable.java

@@ -281,6 +281,10 @@ public class SQLTable implements ObjectIndex, Iterable<Row.Entry> {
         return null;
     }
     
+    public List<Row.Entry> top(int count) throws IOException {
+        return null;
+    }
+    
     public CloneableIterator<Row.Entry> rows(final boolean up, final byte[] startKey) throws IOException {
         // Objects are of type kelondroRow.Entry
         return null;

source/net/yacy/kelondro/table/SplitTable.java

@@ -416,6 +416,23 @@ public class SplitTable implements ObjectIndex, Iterable<Row.Entry> {
         return maxtable.removeOne();
     }
     
+    public List<Row.Entry> top(int count) throws IOException {
+        final Iterator<ObjectIndex> i = tables.values().iterator();
+        ObjectIndex table, maxtable = null;
+        int maxcount = -1;
+        while (i.hasNext()) {
+            table = i.next();
+            if (table.size() > maxcount) {
+                maxtable = table;
+                maxcount = table.size();
+            }
+        }
+        if (maxtable == null) {
+            return null;
+        }
+        return maxtable.top(count);
+    }
+    
     public CloneableIterator<byte[]> keys(final boolean up, final byte[] firstKey) throws IOException {
         final List<CloneableIterator<byte[]>> c = new ArrayList<CloneableIterator<byte[]>>(tables.size());
         final Iterator<ObjectIndex> i = tables.values().iterator();

source/net/yacy/kelondro/table/Table.java

@@ -733,6 +733,20 @@ public class Table implements ObjectIndex, Iterable<Row.Entry> {
         assert table == null || table.size() == index.size() : "table.size() = " + table.size() + ", index.size() = " + index.size();
         return lr;
     }
+    
+    public List<Row.Entry> top(int count) throws IOException {
+        ArrayList<Row.Entry> list = new ArrayList<Row.Entry>();
+        if ((file == null) || (index == null)) return list;
+        long i = file.size() - 1;
+        while (count > 0 && i >= 0) {
+            byte[] b = new byte[rowdef.objectsize];
+            file.get(i, b, 0);
+            list.add(rowdef.newEntry(b));
+            i--;
+            count--;
+        }
+        return list;
+    }
 
     public synchronized void clear() throws IOException {
         final File f = file.filename();