- more tolerance against failure of table opening

- more connections for solrj

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7968 6c8d7289-2bf4-0310-a012-ef5d649a1542

source/de/anomic/search/MetadataRepository.java

@@ -64,10 +64,11 @@ import de.anomic.crawler.CrawlStacker;
 public final class MetadataRepository implements Iterable<byte[]> {
 
     // class objects
-    protected Index               urlIndexFile;
-    private   Export              exportthread; // will have a export thread assigned if exporter is running
-    private final   File                location;
-    private   ArrayList<HostStat> statsDump;
+    protected     Index               urlIndexFile;
+    private       Export              exportthread; // will have a export thread assigned if exporter is running
+    private final File                location;
+    private final String              tablename;
+    private       ArrayList<HostStat> statsDump;
 
     public MetadataRepository(
             final File path,
@@ -75,16 +76,9 @@ public final class MetadataRepository implements Iterable<byte[]> {
             final boolean useTailCache,
             final boolean exceed134217727) {
         this.location = path;
+        this.tablename = tablename;
         Index backupIndex = null;
-        try {
-            backupIndex = new SplitTable(this.location, tablename, URIMetadataRow.rowdef, useTailCache, exceed134217727);
-        } catch (final RowSpaceExceededException e) {
-            try {
-                backupIndex = new SplitTable(this.location, tablename, URIMetadataRow.rowdef, false, exceed134217727);
-            } catch (final RowSpaceExceededException e1) {
-                Log.logException(e1);
-            }
-        }
+        backupIndex = new SplitTable(this.location, tablename, URIMetadataRow.rowdef, useTailCache, exceed134217727);
         this.urlIndexFile = backupIndex; //new Cache(backupIndex, 20000000, 20000000);
         this.exportthread = null; // will have a export thread assigned if exporter is running
         this.statsDump = null;
@@ -97,7 +91,12 @@ public final class MetadataRepository implements Iterable<byte[]> {
 
     public void clear() throws IOException {
         if (this.exportthread != null) this.exportthread.interrupt();
-        this.urlIndexFile.clear();
+        if (this.urlIndexFile == null) {
+            SplitTable.delete(this.location, this.tablename);
+            this.urlIndexFile = new SplitTable(this.location, this.tablename, URIMetadataRow.rowdef, false, false);
+        } else {
+            this.urlIndexFile.clear();
+        }
         this.statsDump = null;
     }
 

source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java

@@ -47,7 +47,6 @@ import org.apache.commons.httpclient.UsernamePasswordCredentials;
 import org.apache.commons.httpclient.auth.AuthPolicy;
 import org.apache.commons.httpclient.auth.AuthScope;
 import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
 import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
@@ -61,7 +60,7 @@ public class SolrSingleConnector implements SolrConnector {
 
     private final String solrurl, host, solrpath, solraccount, solrpw;
     private final int port;
-    private SolrServer server;
+    private CommonsHttpSolrServer server;
     private final SolrScheme scheme;
 
     private final static int transmissionQueueCount = 4; // allow concurrent http sessions to solr
@@ -118,6 +117,8 @@ public class SolrSingleConnector implements SolrConnector {
                 throw new IOException("bad connector url: " + this.solrurl);
             }
         }
+        this.server.setDefaultMaxConnectionsPerHost( 128 );
+        this.server.setMaxTotalConnections( 256 );
 
         // start worker
         this.transmissionWorker = new Worker[transmissionQueueCount];

source/net/yacy/kelondro/table/SplitTable.java

@@ -80,7 +80,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
     private       String current;
     private final long  fileAgeLimit;
     private final long  fileSizeLimit;
-    private boolean useTailCache;
+    private final boolean useTailCache;
     private final boolean exceed134217727;
 
     public SplitTable(
@@ -88,7 +88,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
             final String tablename,
             final Row rowdef,
             final boolean useTailCache,
-            final boolean exceed134217727) throws RowSpaceExceededException {
+            final boolean exceed134217727) {
         this(path, tablename, rowdef, ArrayStack.oneMonth, Integer.MAX_VALUE, useTailCache, exceed134217727);
     }
 
@@ -99,7 +99,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
             final long fileAgeLimit,
             final long fileSizeLimit,
             final boolean useTailCache,
-            final boolean exceed134217727) throws RowSpaceExceededException {
+            final boolean exceed134217727) {
         this.path = path;
         this.prefix = tablename;
         this.rowdef = rowdef;
@@ -141,7 +141,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
         return this.prefix + "." + GenericFormatter.SHORT_MILSEC_FORMATTER.format() + ".table";
     }
 
-    private void init() throws RowSpaceExceededException {
+    private void init() {
         this.current = null;
 
         // initialized tables map
@@ -196,7 +196,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
         String maxf;
         long maxram;
         final List<Thread> warmingUp = new ArrayList<Thread>(); // for concurrent warming up
-        while (!t.isEmpty()) {
+        maxfind: while (!t.isEmpty()) {
             // find maximum table
             maxram = 0;
             maxf = null;
@@ -218,7 +218,12 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
             try {
                 table = new Table(f, this.rowdef, EcoFSBufferSize, 0, this.useTailCache, this.exceed134217727, false);
             } catch (final RowSpaceExceededException e) {
-                table = new Table(f, this.rowdef, 0, 0, false, this.exceed134217727, false);
+                try {
+                    table = new Table(f, this.rowdef, 0, 0, false, this.exceed134217727, false);
+                } catch (final RowSpaceExceededException ee) {
+                    Log.logSevere("SplitTable", "Table " + f.toString() + " cannot be initialized: " + ee.getMessage(), ee);
+                    continue maxfind;
+                }
             }
             final Table a = table;
             final Thread p = new Thread() {
@@ -252,16 +257,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> {
     		    if (f.isDirectory()) delete(this.path, element); else FileUtils.deletedelete(f);
     		}
     	}
-    	try {
-            init();
-        } catch (final RowSpaceExceededException e) {
-            this.useTailCache = false;
-            try {
-                init();
-            } catch (final RowSpaceExceededException e1) {
-                throw new IOException(e1.getMessage());
-            }
-        }
+    	init();
     }
 
     public static void delete(final File path, final String tablename) {