- re-introduced existById in solr connector.

- intruduced raw-queries for the re-introduced byId-Queries (they are
hopefully faster than full edismax queries)
- removed the cached solr connector (testing this) to rely only on the
solr built-in search caches. That should save some RAM (also). We will
see if this is usable.

source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java

@@ -44,7 +44,6 @@ import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.FacetParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
 
 public abstract class AbstractSolrConnector implements SolrConnector {
 
@@ -66,10 +65,6 @@ public abstract class AbstractSolrConnector implements SolrConnector {
         catchSuccessQuery.setStart(0);
     }
     private final static int pagesize = 100;
-
-    public static String idQuery(String id) {
-        return CollectionSchema.id.getSolrFieldName() + ":\"" + id + "\"";
-    }
     
     @Override
     public boolean existsByQuery(final String query) throws IOException {
@@ -201,6 +196,47 @@ public abstract class AbstractSolrConnector implements SolrConnector {
         return docs;
     }
 
+    @Override
+    public boolean existsById(String id) throws IOException {
+        // construct raw query
+        final SolrQuery params = new SolrQuery();
+        //params.setQuery(CollectionSchema.id.getSolrFieldName() + ":\"" + id + "\"");
+        params.setQuery("{!raw f=" + CollectionSchema.id.getSolrFieldName() + "}" + id);
+        //params.set("defType", "raw");
+        params.setRows(0);
+        params.setStart(0);
+        params.setFacet(false);
+        params.setFields(CollectionSchema.id.getSolrFieldName());
+
+        // query the server
+        QueryResponse rsp = getResponseByParams(params);
+        final SolrDocumentList docs = rsp.getResults();
+        boolean exist = docs == null ? false : docs.getNumFound() > 0;
+        return exist;
+    }
+
+    /**
+     * get the number of results for this id.
+     * This should only be called if the actual result is never used, and only the count is interesting
+     * @param id
+     * @return the number of results for this query
+     */
+    public long getCountById(final String id) throws IOException {
+        // construct raw query
+        final SolrQuery params = new SolrQuery();
+        params.setQuery("{!raw f=" + CollectionSchema.id.getSolrFieldName() + "}" + id);
+        params.setRows(0);
+        params.setStart(0);
+        params.setFacet(false);
+        params.setFields(CollectionSchema.id.getSolrFieldName());
+
+        // query the server
+        QueryResponse rsp = getResponseByParams(params);
+        final SolrDocumentList docs = rsp.getResults();
+        long count = docs == null ? 0 : docs.getNumFound();
+        return count;
+    }
+
     /**
      * get the number of results when this query is done.
      * This should only be called if the actual result is never used, and only the count is interesting
@@ -258,21 +294,13 @@ public abstract class AbstractSolrConnector implements SolrConnector {
         }
         return facets;
     }
-
-    @Override
-    abstract public QueryResponse getResponseByParams(ModifiableSolrParams params) throws IOException;
-
-    private final char[] queryIDTemplate = "id:\"            \"".toCharArray();
     
     @Override
     public SolrDocument getDocumentById(final String key, final String ... fields) throws IOException {
         final SolrQuery query = new SolrQuery();
         assert key.length() == 12;
         // construct query
-        char[] q = new char[17];
-        System.arraycopy(this.queryIDTemplate, 0, q, 0, 17);
-        System.arraycopy(key.toCharArray(), 0, q, 4, 12);
-        query.setQuery(new String(q));
+        query.setQuery("{!raw f=" + CollectionSchema.id.getSolrFieldName() + "}" + key);
         query.setRows(1);
         query.setStart(0);
         if (fields.length > 0) query.setFields(fields);

source/net/yacy/cora/federate/solr/connector/CachedSolrConnector.java

@@ -50,6 +50,10 @@ public class CachedSolrConnector extends AbstractSolrConnector implements SolrCo
     public long hitCache_Hit = 0, hitCache_Miss = 0, hitCache_Insert = 0; // for statistics only; do not write
     public long missCache_Hit = 0, missCache_Miss = 0, missCache_Insert = 0; // for statistics only; do not write
 
+    private static final String idQuery(String id) {
+        return CollectionSchema.id.getSolrFieldName() + ":\"" + id + "\"";
+    }
+    
     public CachedSolrConnector(SolrConnector c, int hitCacheMax, int missCacheMax, int docCacheMax) {
         this.solr = c;
         int partitions = Runtime.getRuntime().availableProcessors() * 2;

source/net/yacy/cora/federate/solr/connector/SolrConnector.java

@@ -87,9 +87,16 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
     public void deleteByQuery(final String querystring) throws IOException;
 
     /**
-     * check if a given key exists in solr at the field fieldName
-     * @param fieldName
-     * @param key
+     * check if a given id exists
+     * @param id
+     * @return true if any entry in solr exists
+     * @throws IOException
+     */
+    public boolean existsById(final String id) throws IOException;
+    
+    /**
+     * check if a given document exists in solr
+     * @param solrquery
      * @return true if any entry in solr exists
      * @throws IOException
      */
@@ -147,6 +154,14 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
      */
     public SolrDocumentList getDocumentListByQuery(final String querystring, final int offset, final int count, final String ... fields) throws IOException, SolrException;
 
+    /**
+     * get the number of results for this id.
+     * This should only be called if the actual result is never used, and only the count is interesting
+     * @param key
+     * @return the number of results for this query
+     */
+    public long getCountById(final String key) throws IOException;
+    
     /**
      * get the number of results when this query is done.
      * This should only be called if the actual result is never used, and only the count is interesting

source/net/yacy/cora/federate/solr/instance/InstanceMirror.java

@@ -34,8 +34,8 @@ public class InstanceMirror {
 
     private EmbeddedInstance solr0;
     private ShardInstance solr1;
-    private CachedSolrConnector defaultConnector;
-    private Map<String, CachedSolrConnector> connectorCache;
+    private SolrConnector defaultConnector;
+    private Map<String, SolrConnector> connectorCache;
     private EmbeddedSolrConnector defaultEmbeddedConnector;
     private Map<String, EmbeddedSolrConnector> embeddedCache;
 
@@ -43,7 +43,7 @@ public class InstanceMirror {
         this.solr0 = null;
         this.solr1 = null;
         this.defaultConnector = null;
-        this.connectorCache = new ConcurrentHashMap<String, CachedSolrConnector>();
+        this.connectorCache = new ConcurrentHashMap<String, SolrConnector>();
         this.defaultEmbeddedConnector = null;
         this.embeddedCache = new ConcurrentHashMap<String, EmbeddedSolrConnector>();
     }
@@ -142,23 +142,25 @@ public class InstanceMirror {
         if (defaultCoreName == null) return null;
         EmbeddedSolrConnector esc = this.solr0 == null ? null : new EmbeddedSolrConnector(this.solr0, defaultCoreName);
         RemoteSolrConnector rsc = this.solr1 == null ? null : new RemoteSolrConnector(this.solr1, defaultCoreName);
-        this.defaultConnector = new CachedSolrConnector(new MirrorSolrConnector(esc, rsc), 10000, 1000, 100);
+        this.defaultConnector = /*new CachedSolrConnector(*/new MirrorSolrConnector(esc, rsc)/*, 10000, 1000, 100)*/;
         this.connectorCache.put(defaultCoreName, this.defaultConnector);
         return this.defaultConnector;
     }
 
     public SolrConnector getMirrorConnector(String corename) {
-        CachedSolrConnector msc = this.connectorCache.get(corename);
+        SolrConnector msc = this.connectorCache.get(corename);
         if (msc != null) return msc;
         EmbeddedSolrConnector esc = this.solr0 == null ? null : new EmbeddedSolrConnector(this.solr0, corename);
         RemoteSolrConnector rsc = this.solr1 == null ? null : new RemoteSolrConnector(this.solr1, corename);
-        msc = new CachedSolrConnector(new MirrorSolrConnector(esc, rsc), 10000, 1000, 100);
+        msc = /*new CachedSolrConnector(*/new MirrorSolrConnector(esc, rsc)/*, 10000, 1000, 100)*/;
         this.connectorCache.put(corename, msc);
         return msc;
     }
     
     public void clearCache() {
-        for (CachedSolrConnector csc: this.connectorCache.values()) csc.clearCache();
+        for (SolrConnector csc: this.connectorCache.values()) {
+            if (csc instanceof CachedSolrConnector) ((CachedSolrConnector) csc).clearCache();
+        }
         for (EmbeddedSolrConnector ssc: this.embeddedCache.values()) ssc.commit(true);
     }
     

source/net/yacy/search/index/Fulltext.java

@@ -733,7 +733,7 @@ public final class Fulltext {
             if (urlHash.equals(doc.getFieldValue(CollectionSchema.id.getSolrFieldName()))) return true;
         }
         try {
-            if (this.getDefaultConnector().existsByQuery(AbstractSolrConnector.idQuery(urlHash))) return true;
+            if (this.getDefaultConnector().existsById(urlHash)) return true;
         } catch (final Throwable e) {
             Log.logException(e);
         }