diff --git a/httpd.mime b/defaults/httpd.mime similarity index 100% rename from httpd.mime rename to defaults/httpd.mime diff --git a/defaults/yacy.init b/defaults/yacy.init index 66dc80463..1e944824d 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -220,7 +220,7 @@ locale.language=default fileHost = localpeer # specify the path to the MIME matching file table -mimeConfig = httpd.mime +mimeTable = defaults/httpd.mime # a path to the file cache, used for the internal proxy and as crawl buffer # This will be used if the server is addressed as a proxy diff --git a/htroot/env/base.css b/htroot/env/base.css index 01b9a741c..30a88a4d6 100644 --- a/htroot/env/base.css +++ b/htroot/env/base.css @@ -834,6 +834,12 @@ div#api { z-index: 100; } +div#yacylivesearch { + float:right; + margin-right: 90px; + margin-top: -27px; +} + div#api span { display: none; } diff --git a/htroot/env/templates/header.template b/htroot/env/templates/header.template index 7bd044c33..791c6b601 100644 --- a/htroot/env/templates/header.template +++ b/htroot/env/templates/header.template @@ -27,7 +27,7 @@

YaCy - Distributed Search Engine

-
+
diff --git a/source/de/anomic/document/Classification.java b/source/de/anomic/document/Classification.java index 3cc589422..5a319e5dd 100644 --- a/source/de/anomic/document/Classification.java +++ b/source/de/anomic/document/Classification.java @@ -32,6 +32,8 @@ import java.util.HashSet; import java.util.Properties; import java.util.Set; +import net.yacy.kelondro.logging.Log; + public class Classification { private static final HashSet mediaExtSet = new HashSet(); @@ -45,10 +47,12 @@ public class Classification { static { // load a list of extensions from file BufferedInputStream bufferedIn = null; + File mimeFile = new File("defaults/httpd.mime"); + if (!mimeFile.exists()) mimeFile = new File("config/mime.properties"); try { - ext2mime.load(bufferedIn = new BufferedInputStream(new FileInputStream(new File("httpd.mime")))); + ext2mime.load(bufferedIn = new BufferedInputStream(new FileInputStream(mimeFile))); } catch (final IOException e) { - System.err.println("ERROR: httpd.mime not found in settings path"); + Log.logSevere("Classification", "httpd.mime not found in " + mimeFile.toString(), e); } finally { if (bufferedIn != null) try { bufferedIn.close(); diff --git a/source/de/anomic/document/parser/html/ContentScraper.java b/source/de/anomic/document/parser/html/ContentScraper.java index 5ea6b5c3a..42c3d2f25 100644 --- a/source/de/anomic/document/parser/html/ContentScraper.java +++ b/source/de/anomic/document/parser/html/ContentScraper.java @@ -376,6 +376,8 @@ public class ContentScraper extends AbstractScraper implements Scraper { } public HashSet getContentLanguages() { + // i.e. + // or String s = metas.get("content-language"); if (s == null) s = metas.get("dc.language"); if (s == null) return null; diff --git a/source/de/anomic/http/server/HTTPDFileHandler.java b/source/de/anomic/http/server/HTTPDFileHandler.java index b1b6a4cac..dc28c0d8d 100644 --- a/source/de/anomic/http/server/HTTPDFileHandler.java +++ b/source/de/anomic/http/server/HTTPDFileHandler.java @@ -144,7 +144,7 @@ public final class HTTPDFileHandler { if (mimeTable.size() == 0) { // load the mime table - final String mimeTablePath = theSwitchboard.getConfig("mimeConfig",""); + final String mimeTablePath = theSwitchboard.getConfig("mimeTable",""); BufferedInputStream mimeTableInputStream = null; try { Log.logConfig("HTTPDFiles", "Loading mime mapping file " + mimeTablePath); diff --git a/source/de/anomic/search/DocumentIndex.java b/source/de/anomic/search/DocumentIndex.java index f8f7ba9b5..c2ceaa52c 100644 --- a/source/de/anomic/search/DocumentIndex.java +++ b/source/de/anomic/search/DocumentIndex.java @@ -53,13 +53,15 @@ public class DocumentIndex extends Segment { private static final RankingProfile textRankingDefault = new RankingProfile(QueryParams.CONTENTDOM_TEXT); //private Bitfield zeroConstraint = new Bitfield(4); - File poison = new File("."); - BlockingQueue queue; - Worker[] worker; + private final static File poison = new File("."); + private BlockingQueue queue; + private Worker[] worker; + private CallbackListener callback; - public DocumentIndex(Log log, final File segmentPath, int cachesize) throws IOException { + public DocumentIndex(Log log, final File segmentPath, CallbackListener callback, int cachesize) throws IOException { super(log, segmentPath, cachesize, targetFileSize * 4 - 1, false, false); int cores = Runtime.getRuntime().availableProcessors() + 1; + this.callback = callback; this.queue = new LinkedBlockingQueue(cores * 300); this.worker = new Worker[cores]; for (int i = 0; i < cores; i++) { @@ -68,8 +70,8 @@ public class DocumentIndex extends Segment { } } - public DocumentIndex(final File segmentPath, int cachesize) throws IOException { - this(new Log("DocumentIndex"), segmentPath, cachesize); + public DocumentIndex(final File segmentPath, CallbackListener callback, int cachesize) throws IOException { + this(new Log("DocumentIndex"), segmentPath, callback, cachesize); } class Worker extends Thread { @@ -78,6 +80,7 @@ public class DocumentIndex extends Segment { try { while ((f = queue.take()) != poison) try { add(f); + if (callback != null) callback.commitIndex(f); } catch (IOException e) { if (e.getMessage().indexOf("cannot parse") < 0) e.printStackTrace(); } @@ -85,6 +88,17 @@ public class DocumentIndex extends Segment { } } + /** + * get the number of pending documents in the indexing queue + */ + public int pending() { + return this.queue.size(); + } + + public void clearQueue() { + this.queue.clear(); + } + /** * put a single file into the index * @param file @@ -214,6 +228,10 @@ public class DocumentIndex extends Segment { super.close(); } + public interface CallbackListener { + public void commitIndex(File f); + } + public static void main(String[] args) { // first argument: path to segment // second argument: either 'add' or 'search' @@ -226,17 +244,22 @@ public class DocumentIndex extends Segment { if (args.length < 3) return; File segmentPath = new File(args[0]); System.out.println("using index files at " + segmentPath.getAbsolutePath()); + CallbackListener callback = new CallbackListener() { + public void commitIndex(File f) { + System.out.println("indexed: " + f.toString()); + } + }; try { if (args[1].equals("add")) { File f = new File(args[2]); - DocumentIndex di = new DocumentIndex(segmentPath, 100000); + DocumentIndex di = new DocumentIndex(segmentPath, callback, 100000); di.addConcurrent(f); di.close(); } else { String query = ""; for (int i = 2; i < args.length; i++) query += args[i]; query.trim(); - DocumentIndex di = new DocumentIndex(segmentPath, 100000); + DocumentIndex di = new DocumentIndex(segmentPath, callback, 100000); ArrayList results = di.find(query); for (File f: results) { if (f != null) System.out.println(f.toString()); diff --git a/source/de/anomic/search/Segments.java b/source/de/anomic/search/Segments.java index 422e9ad1e..4cc84727b 100644 --- a/source/de/anomic/search/Segments.java +++ b/source/de/anomic/search/Segments.java @@ -41,7 +41,7 @@ import net.yacy.kelondro.rwi.IndexCell; import de.anomic.document.Condenser; import de.anomic.document.Document; -public final class Segments implements Iterable { +public class Segments implements Iterable { /** * process enumeration type diff --git a/source/net/yacy/kelondro/blob/Heap.java b/source/net/yacy/kelondro/blob/Heap.java index 1ae4e0532..f3939eeb0 100755 --- a/source/net/yacy/kelondro/blob/Heap.java +++ b/source/net/yacy/kelondro/blob/Heap.java @@ -248,7 +248,7 @@ public final class Heap extends HeapModifier implements BLOB { * close the BLOB table */ public synchronized void close(boolean writeIDX) { - if (file != null) { + if (file != null && buffer != null) { try { flushBuffer(); } catch (IOException e) { diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index 28d57005c..7a22dfac7 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -389,7 +389,7 @@ public final class FileUtils { final byte[] b = read(f); return table(strings(b)); } catch (final IOException e2) { - System.err.println("ERROR: " + f.toString() + " not found in settings path"); + Log.logSevere("FileUtils", f.toString() + " not found", e2); return null; } }