- moved httpd.mime to defaults path

- some documentation fixes
- adopted a default setting for the search window: moves css setting to base.css
- some enhancements for the DocumentIndex class

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6410 6c8d7289-2bf4-0310-a012-ef5d649a1542

defaults/yacy.init

@@ -220,7 +220,7 @@ locale.language=default
 fileHost = localpeer
 
 # specify the path to the MIME matching file table
-mimeConfig = httpd.mime
+mimeTable = defaults/httpd.mime
 
 # a path to the file cache, used for the internal proxy and as crawl buffer
 # This will be used if the server is addressed as a proxy

htroot/env/base.css

@@ -834,6 +834,12 @@ div#api {
   z-index: 100;
 }
 
+div#yacylivesearch {
+  float:right;
+  margin-right: 90px;
+  margin-top: -27px;
+}
+
 div#api span {
   display: none;
 }

htroot/env/templates/header.template

@@ -27,7 +27,7 @@
 
 <div class="head">
 	<h1>YaCy - Distributed Search Engine</h1>
-	<div id="yacylivesearch" style="float: right; margin-right: 90px; margin-top: -27px;">
+	<div id="yacylivesearch">
 		<form id="ysearch" method="get" accept-charset="UTF-8" action="yacysearch.html">
 			<input name="search" id="yquery" class="fancy" type="text" size="20" maxlength="80" value=""/>
 			<input type="hidden" name="verify" value="true" />

source/de/anomic/document/Classification.java

@@ -32,6 +32,8 @@ import java.util.HashSet;
 import java.util.Properties;
 import java.util.Set;
 
+import net.yacy.kelondro.logging.Log;
+
 public class Classification {
 
     private static final HashSet<String> mediaExtSet = new HashSet<String>();
@@ -45,10 +47,12 @@ public class Classification {
     static {
     	// load a list of extensions from file
         BufferedInputStream bufferedIn = null;
+        File mimeFile = new File("defaults/httpd.mime");
+        if (!mimeFile.exists()) mimeFile = new File("config/mime.properties");
         try {
-            ext2mime.load(bufferedIn = new BufferedInputStream(new FileInputStream(new File("httpd.mime"))));
+            ext2mime.load(bufferedIn = new BufferedInputStream(new FileInputStream(mimeFile)));
         } catch (final IOException e) {
-            System.err.println("ERROR: httpd.mime not found in settings path");
+            Log.logSevere("Classification", "httpd.mime not found in " + mimeFile.toString(), e);
         } finally {
             if (bufferedIn != null) try {
                 bufferedIn.close();

source/de/anomic/document/parser/html/ContentScraper.java

@@ -376,6 +376,8 @@ public class ContentScraper extends AbstractScraper implements Scraper {
     }
     
     public HashSet<String> getContentLanguages() {
+        // i.e. <meta name="DC.language" content="en" scheme="DCTERMS.RFC3066">
+        // or <meta http-equiv="content-language" content="en">
         String s = metas.get("content-language");
         if (s == null) s = metas.get("dc.language");
         if (s == null) return null;

source/de/anomic/http/server/HTTPDFileHandler.java

@@ -144,7 +144,7 @@ public final class HTTPDFileHandler {
 
             if (mimeTable.size() == 0) {
                 // load the mime table
-                final String mimeTablePath = theSwitchboard.getConfig("mimeConfig","");
+                final String mimeTablePath = theSwitchboard.getConfig("mimeTable","");
                 BufferedInputStream mimeTableInputStream = null;
                 try {
                     Log.logConfig("HTTPDFiles", "Loading mime mapping file " + mimeTablePath);

source/de/anomic/search/DocumentIndex.java

@@ -53,13 +53,15 @@ public class DocumentIndex extends Segment {
     private static final RankingProfile textRankingDefault = new RankingProfile(QueryParams.CONTENTDOM_TEXT);
     //private Bitfield zeroConstraint = new Bitfield(4);
     
-    File poison = new File(".");
-    BlockingQueue<File> queue;
-    Worker[] worker;
+    private final static File poison = new File(".");
+    private BlockingQueue<File> queue;
+    private Worker[] worker;
+    private CallbackListener callback;
     
-    public DocumentIndex(Log log, final File segmentPath, int cachesize) throws IOException {
+    public DocumentIndex(Log log, final File segmentPath, CallbackListener callback, int cachesize) throws IOException {
         super(log, segmentPath, cachesize, targetFileSize * 4 - 1, false, false);
         int cores = Runtime.getRuntime().availableProcessors() + 1;
+        this.callback = callback;
         this.queue = new LinkedBlockingQueue<File>(cores * 300);
         this.worker = new Worker[cores];
         for (int i = 0; i < cores; i++) {
@@ -68,8 +70,8 @@ public class DocumentIndex extends Segment {
         }
     }
     
-    public DocumentIndex(final File segmentPath, int cachesize) throws IOException {
-        this(new Log("DocumentIndex"), segmentPath, cachesize);
+    public DocumentIndex(final File segmentPath, CallbackListener callback, int cachesize) throws IOException {
+        this(new Log("DocumentIndex"), segmentPath, callback, cachesize);
     }
 	
     class Worker extends Thread {
@@ -78,6 +80,7 @@ public class DocumentIndex extends Segment {
             try {
                 while ((f = queue.take()) != poison) try {
                     add(f);
+                    if (callback != null) callback.commitIndex(f);
                 } catch (IOException e) {
                     if (e.getMessage().indexOf("cannot parse") < 0) e.printStackTrace();
                 }
@@ -85,6 +88,17 @@ public class DocumentIndex extends Segment {
         }
     }
     
+    /**
+     * get the number of pending documents in the indexing queue
+     */
+    public int pending() {
+        return this.queue.size();
+    }
+    
+    public void clearQueue() {
+        this.queue.clear();
+    }
+    
     /**
      * put a single file into the index
      * @param file
@@ -214,6 +228,10 @@ public class DocumentIndex extends Segment {
         super.close();
     }
     
+    public interface CallbackListener {
+        public void commitIndex(File f);
+    }
+    
     public static void main(String[] args) {
         // first argument: path to segment
         // second argument: either 'add' or 'search'
@@ -226,17 +244,22 @@ public class DocumentIndex extends Segment {
         if (args.length < 3) return;
         File segmentPath = new File(args[0]);
         System.out.println("using index files at " + segmentPath.getAbsolutePath());
+        CallbackListener callback = new CallbackListener() {
+            public void commitIndex(File f) {
+                System.out.println("indexed: " + f.toString());
+            }
+        };
         try {
             if (args[1].equals("add")) {
                 File f = new File(args[2]);
-                DocumentIndex di = new DocumentIndex(segmentPath, 100000);
+                DocumentIndex di = new DocumentIndex(segmentPath, callback, 100000);
                 di.addConcurrent(f);
                 di.close();
             } else {
                 String query = "";
                 for (int i = 2; i < args.length; i++) query += args[i];
                 query.trim();
-                DocumentIndex di = new DocumentIndex(segmentPath, 100000);
+                DocumentIndex di = new DocumentIndex(segmentPath, callback, 100000);
                 ArrayList<File> results = di.find(query);
                 for (File f: results) {
                     if (f != null) System.out.println(f.toString());

source/de/anomic/search/Segments.java

@@ -41,7 +41,7 @@ import net.yacy.kelondro.rwi.IndexCell;
 import de.anomic.document.Condenser;
 import de.anomic.document.Document;
 
-public final class Segments implements Iterable<Segment> {
+public class Segments implements Iterable<Segment> {
     
     /**
      * process enumeration type

source/net/yacy/kelondro/blob/Heap.java

@@ -248,7 +248,7 @@ public final class Heap extends HeapModifier implements BLOB {
      * close the BLOB table
      */
     public synchronized void close(boolean writeIDX) {
-    	if (file != null) {
+    	if (file != null && buffer != null) {
             try {
                 flushBuffer();
             } catch (IOException e) {

source/net/yacy/kelondro/util/FileUtils.java

@@ -389,7 +389,7 @@ public final class FileUtils {
             final byte[] b = read(f);
             return table(strings(b));
         } catch (final IOException e2) {
-            System.err.println("ERROR: " + f.toString() + " not found in settings path");
+            Log.logSevere("FileUtils", f.toString() + " not found", e2);
             return null;
         }
     }