Merge branch 'master' of git@gitorious.org:yacy/rc1.git

defaults/yacy.init

@@ -1056,7 +1056,6 @@ color_searchurlhover = #008000
 # - to check whats in solr after indexing, open http://localhost:8983/solr/admin/
 federated.service.solr.indexing.enabled = false
 federated.service.solr.indexing.url = http://127.0.0.1:8983/solr
-federated.service.solr.indexing.commitWithinMs = -1
 federated.service.solr.indexing.sharding = MODULO_HOST_MD5
 federated.service.solr.indexing.schemefile = solr.keys.default.list
 # the lazy attribute causes that fields containing "" or 0 are not added and not written

htroot/IndexFederated_p.java

@@ -79,7 +79,6 @@ public class IndexFederated_p {
             final boolean previous_core_fulltext = sb.index.fulltext().connectedLocalSolr() && env.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, false);
             env.setConfig(SwitchboardConstants.CORE_SERVICE_FULLTEXT, post_core_fulltext);
 
-            final int commitWithinMs = post.getInt("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1));
             if (previous_core_fulltext && !post_core_fulltext) {
                 // switch off
                 sb.index.fulltext().disconnectLocalSolr();
@@ -88,7 +87,7 @@ public class IndexFederated_p {
             if (!previous_core_fulltext && post_core_fulltext) {
                 // switch on
                 sb.index.connectUrlDb(sb.useTailCache, sb.exceed134217727);
-                try { sb.index.fulltext().connectLocalSolr(commitWithinMs); } catch (IOException e) { Log.logException(e); }
+                try { sb.index.fulltext().connectLocalSolr(); } catch (IOException e) { Log.logException(e); }
             }
 
             // solr
@@ -114,7 +113,6 @@ public class IndexFederated_p {
             }
             solrurls = s.toString().trim();
             env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, solrurls);
-            env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, commitWithinMs);
             env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, lazy);
             env.setConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, post.get("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5")));
             final String schemename = post.get("solr.indexing.schemefile", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE, "solr.keys.default.list"));
@@ -136,7 +134,6 @@ public class IndexFederated_p {
                 try {
                     if (usesolr) {
                         SolrConnector solr = new ShardSolrConnector(solrurls, ShardSelection.Method.MODULO_HOST_MD5, 10000, true);
-                        if (commitWithinMs >= 0) solr.setCommitWithinMs(commitWithinMs);
                         sb.index.fulltext().connectRemoteSolr(solr);
                     } else {
                         sb.index.fulltext().disconnectRemoteSolr();
@@ -225,7 +222,6 @@ public class IndexFederated_p {
         prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
         prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, false) ? 1 : 0);
         prop.put("solr.indexing.url", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_URL, "http://127.0.0.1:8983/solr").replace(",", "\n"));
-        prop.put("solr.indexing.commitWithinMs", env.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, 180000));
         prop.put("solr.indexing.lazy.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_LAZY, true) ? 1 : 0);
         prop.put("solr.indexing.sharding", env.getConfig(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_SHARDING, "modulo-host-md5"));
         prop.put("solr.indexing.schemefile", schemename);

htroot/yacysearch.html

@@ -69,10 +69,9 @@
 <div id="api">
 <script type="text/javascript">
 //<![CDATA[
-document.write("\<a href=\"yacysearch.rss?" + window.location.search.substring(1) + "\"\>")
+document.write("\<a href=\"yacysearch.rss?" + window.location.search.substring(1) + "\"\><img src=\"env/grafics/api.png\" width=\"60\" height=\"40\" alt=\"API\" /></a>")
 //]]>
 </script>
-<img src="env/grafics/api.png" width="60" height="40" alt="API" /></a>
 <span>This search result can also be retrieved as RSS/<a href="http://www.opensearch.org">opensearch</a> output.
 The query format is similar to <a href="http://www.loc.gov/standards/sru/">SRU</a>.
 Click the API icon to see an example call to the search rss API.

source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java

@@ -148,23 +148,6 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
         this.documentCache.clear();
     }
 
-    @Override
-    public int getCommitWithinMs() {
-        if (this.solr0 != null) this.solr0.getCommitWithinMs();
-        if (this.solr1 != null) this.solr1.getCommitWithinMs();
-        return -1;
-    }
-
-    /**
-     * set the solr autocommit delay
-     * @param c the maximum waiting time after a solr command until it is transported to the server
-     */
-    @Override
-    public void setCommitWithinMs(int c) {
-        if (this.solr0 != null) this.solr0.setCommitWithinMs(c);
-        if (this.solr1 != null) this.solr1.setCommitWithinMs(c);
-    }
-
     @Override
     public void commit(boolean softCommit) {
         if (this.solr0 != null) this.solr0.commit(softCommit);

source/net/yacy/cora/federate/solr/connector/MultipleSolrConnector.java

@@ -36,13 +36,11 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
     private final ArrayBlockingQueue<SolrInputDocument> queue;
     private final AddWorker[] worker;
     private final SolrConnector solr;
-    private int commitWithinMs;
 
     public MultipleSolrConnector(final String url, final int connections) throws IOException {
         this.solr = new RemoteSolrConnector(url);
         this.queue = new ArrayBlockingQueue<SolrInputDocument>(1000);
         this.worker = new AddWorker[connections];
-        this.commitWithinMs = -1;
         for (int i = 0; i < connections; i++) {
             this.worker[i] = new AddWorker(url);
             this.worker[i].start();
@@ -53,7 +51,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
         private final SolrConnector solr;
         public AddWorker(final String url) throws IOException {
             this.solr = new RemoteSolrConnector(url);
-            if (MultipleSolrConnector.this.commitWithinMs >= 0 ) this.solr.setCommitWithinMs(MultipleSolrConnector.this.commitWithinMs);
         }
         @Override
         public void run() {
@@ -75,22 +72,6 @@ public class MultipleSolrConnector extends AbstractSolrConnector implements Solr
         }
     }
 
-    @Override
-    public int getCommitWithinMs() {
-        return this.commitWithinMs;
-    }
-
-    /**
-     * set the solr autocommit delay
-     * @param c the maximum waiting time after a solr command until it is transported to the server
-     */
-    @Override
-    public void setCommitWithinMs(final int c) {
-        this.commitWithinMs = c;
-        this.solr.setCommitWithinMs(c);
-        for (AddWorker w: this.worker) w.solr.setCommitWithinMs(c);
-    }
-
     @Override
     public void commit(boolean softCommit) {
         this.solr.commit(softCommit);

source/net/yacy/cora/federate/solr/connector/RetrySolrConnector.java

@@ -38,20 +38,6 @@ public class RetrySolrConnector extends AbstractSolrConnector implements SolrCon
         this.retryMaxTime = retryMaxTime;
     }
 
-    @Override
-    public int getCommitWithinMs() {
-        return this.solrConnector.getCommitWithinMs();
-    }
-
-    /**
-     * set the solr autocommit delay
-     * @param c the maximum waiting time after a solr command until it is transported to the server
-     */
-    @Override
-    public void setCommitWithinMs(int c) {
-        this.solrConnector.setCommitWithinMs(c);
-    }
-
     @Override
     public void commit(boolean softCommit) {
         this.solrConnector.commit(softCommit);

source/net/yacy/cora/federate/solr/connector/ShardSolrConnector.java

@@ -58,20 +58,6 @@ public class ShardSolrConnector extends AbstractSolrConnector implements SolrCon
         this.sharding = new ShardSelection(method, this.urls.length);
     }
 
-    @Override
-    public int getCommitWithinMs() {
-        return this.connectors.get(0).getCommitWithinMs();
-    }
-
-    /**
-     * set the solr autocommit delay
-     * @param c the maximum waiting time after a solr command until it is transported to the server
-     */
-    @Override
-    public void setCommitWithinMs(int c) {
-        for (final SolrConnector connector: this.connectors) connector.setCommitWithinMs(c);
-    }
-
     @Override
     public void commit(boolean softCommit) {
         for (final SolrConnector connector: this.connectors) connector.commit(softCommit);

source/net/yacy/cora/federate/solr/connector/SolrConnector.java

@@ -36,18 +36,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;
 
 public interface SolrConnector extends Iterable<String> /* Iterable of document IDs */ {
 
-    /**
-     * get the solr autocommit delay
-     * @return the maximum waiting time after a solr command until it is transported to the server
-     */
-    public int getCommitWithinMs();
-
-    /**
-     * set the solr autocommit delay
-     * @param c the maximum waiting time after a solr command until it is transported to the server
-     */
-    public void setCommitWithinMs(int c);
-
     /**
      * force a commit
      */

source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java

@@ -56,11 +56,9 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
     protected final static Logger log = Logger.getLogger(SolrServerConnector.class);
     
     protected SolrServer server;
-    protected int commitWithinMs; // max time (in ms) before a commit will happen
 
     protected SolrServerConnector() {
         this.server = null;
-        this.commitWithinMs = 180000;
     }
 
     protected void init(SolrServer server) {
@@ -71,25 +69,6 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
         return this.server;
     }
 
-    /**
-     * get the solr autocommit delay
-     * @return the maximum waiting time after a solr command until it is transported to the server
-     */
-    @Override
-    public int getCommitWithinMs() {
-        return this.commitWithinMs;
-    }
-
-    /**
-     * set the solr autocommit delay
-     * when doing continuous inserts, don't set this value because it would cause continuous commits
-     * @param c the maximum waiting time after a solr command until it is transported to the server
-     */
-    @Override
-    public void setCommitWithinMs(int c) {
-        this.commitWithinMs = c;
-    }
-
     @Override
     public synchronized void commit(final boolean softCommit) {
         //if (this.server instanceof HttpSolrServer) ((HttpSolrServer) this.server).getHttpClient().getConnectionManager().closeExpiredConnections();
@@ -209,7 +188,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
     public void delete(final String id) throws IOException {
         try {
             synchronized (this.server) {
-                this.server.deleteById(id, this.commitWithinMs);
+                this.server.deleteById(id, -1);
             }
         } catch (final Throwable e) {
             throw new IOException(e);
@@ -220,7 +199,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
     public void delete(final List<String> ids) throws IOException {
         try {
             synchronized (this.server) {
-                this.server.deleteById(ids, this.commitWithinMs);
+                this.server.deleteById(ids, -1);
             }
         } catch (final Throwable e) {
             throw new IOException(e);
@@ -236,7 +215,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
     public void deleteByQuery(final String querystring) throws IOException {
         try {
             synchronized (this.server) {
-                this.server.deleteByQuery(querystring, this.commitWithinMs);
+                this.server.deleteByQuery(querystring, -1);
             }
         } catch (final Throwable e) {
             throw new IOException(e);
@@ -249,7 +228,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
         up.setParam("literal.id", solrId);
         up.setParam("uprefix", "attr_");
         up.setParam("fmap.content", "attr_content");
-        up.setCommitWithin(this.commitWithinMs);
+        up.setCommitWithin(-1);
         //up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
         try {
             synchronized (this.server) {
@@ -266,7 +245,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
         try {
             if (solrdoc.containsKey("_version_")) solrdoc.setField("_version_",0L); // prevent Solr "version conflict"
             synchronized (this.server) {
-                this.server.add(solrdoc, this.commitWithinMs);
+                this.server.add(solrdoc, -1);
             }
         } catch (Throwable e) {
             // catches "version conflict for": try this again and delete the document in advance
@@ -275,7 +254,7 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
             } catch (SolrServerException e1) {}
             try {
                 synchronized (this.server) {
-                    this.server.add(solrdoc, this.commitWithinMs);
+                    this.server.add(solrdoc, -1);
                 }
             } catch (Throwable ee) {
                 log.warn(e.getMessage() + " DOC=" + solrdoc.toString());

source/net/yacy/data/URLLicense.java

@@ -42,6 +42,7 @@ public class URLLicense {
     private static final Map<String, String> permissions = Collections.synchronizedMap(new SizeLimitedMap<String, String>(maxQueue));
 
     public static String aquireLicense(final DigestURI url) {
+        if (url == null) return "";
         // generate license key
         String license = ASCII.String(url.hash());
         // store reference to url with license key

source/net/yacy/search/Switchboard.java

@@ -428,12 +428,11 @@ public final class Switchboard extends serverSwitch {
         ReferenceContainer.maxReferences = getConfigInt("index.maxReferences", 0);
         final File segmentsPath = new File(new File(indexPath, networkName), "SEGMENTS");
         this.index = new Segment(this.log, new File(segmentsPath, "default"), solrScheme);
-        final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1);
         if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
         if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
         if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
             this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
-            this.index.fulltext().connectLocalSolr(connectWithinMs);
+            this.index.fulltext().connectLocalSolr();
         }
 
         // set up the solr interface
@@ -446,7 +445,6 @@ public final class Switchboard extends serverSwitch {
                                 solrurls,
                                 ShardSelection.Method.MODULO_HOST_MD5,
                                 10000, true);
-                solr.setCommitWithinMs(connectWithinMs);
                 this.index.fulltext().connectRemoteSolr(solr);
             } catch ( final IOException e ) {
                 Log.logException(e);
@@ -1282,11 +1280,10 @@ public final class Switchboard extends serverSwitch {
                 this.useTailCache,
                 this.exceed134217727);
             this.index = new Segment(this.log, new File(new File(new File(indexPrimaryPath, networkName), "SEGMENTS"), "default"), solrScheme);
-            final int connectWithinMs = this.getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS, -1);
             if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, true)) this.index.connectRWI(wordCacheMaxCount, fileSizeMax);
             if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, true)) this.index.connectCitation(wordCacheMaxCount, fileSizeMax);
             if (this.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, true)) {
-                this.index.fulltext().connectLocalSolr(connectWithinMs);
+                this.index.fulltext().connectLocalSolr();
                 this.index.connectUrlDb(this.useTailCache, this.exceed134217727);
             }
 
@@ -1300,7 +1297,6 @@ public final class Switchboard extends serverSwitch {
                                     solrurls,
                                     ShardSelection.Method.MODULO_HOST_MD5,
                                     10000, true);
-                    if (connectWithinMs >= 0) solr.setCommitWithinMs(connectWithinMs);
                     this.index.fulltext().connectRemoteSolr(solr);
                 } catch ( final IOException e ) {
                     Log.logException(e);

source/net/yacy/search/SwitchboardConstants.java

@@ -294,7 +294,6 @@ public final class SwitchboardConstants {
 
     public static final String FEDERATED_SERVICE_SOLR_INDEXING_ENABLED = "federated.service.solr.indexing.enabled";
     public static final String FEDERATED_SERVICE_SOLR_INDEXING_URL = "federated.service.solr.indexing.url";
-    public static final String FEDERATED_SERVICE_SOLR_INDEXING_COMMITWITHINMS = "federated.service.solr.indexing.commitWithinMs";
     public static final String FEDERATED_SERVICE_SOLR_INDEXING_SHARDING = "federated.service.solr.indexing.sharding";
     public static final String FEDERATED_SERVICE_SOLR_INDEXING_SCHEMEFILE = "federated.service.solr.indexing.schemefile";
     public static final String FEDERATED_SERVICE_SOLR_INDEXING_LAZY = "federated.service.solr.indexing.lazy";

source/net/yacy/search/index/DocumentIndex.java

@@ -70,7 +70,7 @@ public class DocumentIndex extends Segment {
                 false, // useTailCache
                 false  // exceed134217727
                 );
-        super.fulltext().connectLocalSolr(-1);
+        super.fulltext().connectLocalSolr();
         final int cores = Runtime.getRuntime().availableProcessors() + 1;
         this.callback = callback;
         this.queue = new LinkedBlockingQueue<DigestURI>(cores * 300);

source/net/yacy/search/index/Fulltext.java

@@ -142,7 +142,7 @@ public final class Fulltext {
         return this.solr.isConnected0();
     }
 
-    public void connectLocalSolr(final int commitWithin) throws IOException {
+    public void connectLocalSolr() throws IOException {
         File baseLocation = this.location;
         if (baseLocation.getName().equals("default")) baseLocation = baseLocation.getParentFile();
         File solrLocation = new File(baseLocation, SOLR_PATH);
@@ -152,7 +152,6 @@ public final class Fulltext {
             if (oldLocation.exists()) oldLocation.renameTo(solrLocation);
         }
         EmbeddedSolrConnector esc = new EmbeddedSolrConnector(solrLocation, new File(new File(Switchboard.getSwitchboard().appPath, "defaults"), "solr"));
-        if (commitWithin >= 0) esc.setCommitWithinMs(commitWithin);
         Version luceneVersion = esc.getConfig().getLuceneVersion("luceneMatchVersion");
         String lvn = luceneVersion.name();
         Log.logInfo("Fulltext", "using lucene version " + lvn);
@@ -230,10 +229,6 @@ public final class Fulltext {
         }
         this.solr.close();
     }
-
-    public int getCommitWithinMs() {
-        return this.solr.getCommitWithinMs();
-    }
     
     public void commit(boolean softCommit) {
         this.solr.commit(softCommit);
@@ -571,7 +566,6 @@ public final class Fulltext {
      */
     public File dumpSolr() {
         EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0();
-        int commitWithin = esc.getCommitWithinMs();
         File storagePath = esc.getStoragePath();
         File zipOut = new File(storagePath.toString() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip");
         synchronized (this.solr) {
@@ -582,7 +576,7 @@ public final class Fulltext {
                 Log.logException(e);
             } finally {
                 try {
-                    this.connectLocalSolr(commitWithin);
+                    this.connectLocalSolr();
                 } catch (IOException e) {
                     Log.logException(e);
                 }
@@ -597,7 +591,6 @@ public final class Fulltext {
      */
     public void restoreSolr(File solrDumpZipFile) {
         EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0();
-        int commitWithin = esc.getCommitWithinMs();
         File storagePath = esc.getStoragePath();
         synchronized (this.solr) {
             this.disconnectLocalSolr();
@@ -607,7 +600,7 @@ public final class Fulltext {
                 Log.logException(e);
             } finally {
                 try {
-                    this.connectLocalSolr(commitWithin);
+                    this.connectLocalSolr();
                 } catch (IOException e) {
                     Log.logException(e);
                 }