From 0c6d95e57bd4f8b7fd2042fe7297d5bb27565896 Mon Sep 17 00:00:00 2001 From: orbiter Date: Wed, 21 Sep 2011 15:08:05 +0000 Subject: [PATCH] - more tolerance against failure of table opening - more connections for solrj git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7968 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- .../de/anomic/search/MetadataRepository.java | 27 +++++++++--------- .../federated/solr/SolrSingleConnector.java | 5 ++-- .../net/yacy/kelondro/table/SplitTable.java | 28 ++++++++----------- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/source/de/anomic/search/MetadataRepository.java b/source/de/anomic/search/MetadataRepository.java index 184baebed..120118644 100644 --- a/source/de/anomic/search/MetadataRepository.java +++ b/source/de/anomic/search/MetadataRepository.java @@ -64,10 +64,11 @@ import de.anomic.crawler.CrawlStacker; public final class MetadataRepository implements Iterable { // class objects - protected Index urlIndexFile; - private Export exportthread; // will have a export thread assigned if exporter is running - private final File location; - private ArrayList statsDump; + protected Index urlIndexFile; + private Export exportthread; // will have a export thread assigned if exporter is running + private final File location; + private final String tablename; + private ArrayList statsDump; public MetadataRepository( final File path, @@ -75,16 +76,9 @@ public final class MetadataRepository implements Iterable { final boolean useTailCache, final boolean exceed134217727) { this.location = path; + this.tablename = tablename; Index backupIndex = null; - try { - backupIndex = new SplitTable(this.location, tablename, URIMetadataRow.rowdef, useTailCache, exceed134217727); - } catch (final RowSpaceExceededException e) { - try { - backupIndex = new SplitTable(this.location, tablename, URIMetadataRow.rowdef, false, exceed134217727); - } catch (final RowSpaceExceededException e1) { - Log.logException(e1); - } - } + backupIndex = new SplitTable(this.location, tablename, URIMetadataRow.rowdef, useTailCache, exceed134217727); this.urlIndexFile = backupIndex; //new Cache(backupIndex, 20000000, 20000000); this.exportthread = null; // will have a export thread assigned if exporter is running this.statsDump = null; @@ -97,7 +91,12 @@ public final class MetadataRepository implements Iterable { public void clear() throws IOException { if (this.exportthread != null) this.exportthread.interrupt(); - this.urlIndexFile.clear(); + if (this.urlIndexFile == null) { + SplitTable.delete(this.location, this.tablename); + this.urlIndexFile = new SplitTable(this.location, this.tablename, URIMetadataRow.rowdef, false, false); + } else { + this.urlIndexFile.clear(); + } this.statsDump = null; } diff --git a/source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java b/source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java index 798c09322..e9c9dadf5 100644 --- a/source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java +++ b/source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java @@ -47,7 +47,6 @@ import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.auth.AuthPolicy; import org.apache.commons.httpclient.auth.AuthScope; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer; import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; @@ -61,7 +60,7 @@ public class SolrSingleConnector implements SolrConnector { private final String solrurl, host, solrpath, solraccount, solrpw; private final int port; - private SolrServer server; + private CommonsHttpSolrServer server; private final SolrScheme scheme; private final static int transmissionQueueCount = 4; // allow concurrent http sessions to solr @@ -118,6 +117,8 @@ public class SolrSingleConnector implements SolrConnector { throw new IOException("bad connector url: " + this.solrurl); } } + this.server.setDefaultMaxConnectionsPerHost( 128 ); + this.server.setMaxTotalConnections( 256 ); // start worker this.transmissionWorker = new Worker[transmissionQueueCount]; diff --git a/source/net/yacy/kelondro/table/SplitTable.java b/source/net/yacy/kelondro/table/SplitTable.java index 587f3c703..d23b5e220 100644 --- a/source/net/yacy/kelondro/table/SplitTable.java +++ b/source/net/yacy/kelondro/table/SplitTable.java @@ -80,7 +80,7 @@ public class SplitTable implements Index, Iterable { private String current; private final long fileAgeLimit; private final long fileSizeLimit; - private boolean useTailCache; + private final boolean useTailCache; private final boolean exceed134217727; public SplitTable( @@ -88,7 +88,7 @@ public class SplitTable implements Index, Iterable { final String tablename, final Row rowdef, final boolean useTailCache, - final boolean exceed134217727) throws RowSpaceExceededException { + final boolean exceed134217727) { this(path, tablename, rowdef, ArrayStack.oneMonth, Integer.MAX_VALUE, useTailCache, exceed134217727); } @@ -99,7 +99,7 @@ public class SplitTable implements Index, Iterable { final long fileAgeLimit, final long fileSizeLimit, final boolean useTailCache, - final boolean exceed134217727) throws RowSpaceExceededException { + final boolean exceed134217727) { this.path = path; this.prefix = tablename; this.rowdef = rowdef; @@ -141,7 +141,7 @@ public class SplitTable implements Index, Iterable { return this.prefix + "." + GenericFormatter.SHORT_MILSEC_FORMATTER.format() + ".table"; } - private void init() throws RowSpaceExceededException { + private void init() { this.current = null; // initialized tables map @@ -196,7 +196,7 @@ public class SplitTable implements Index, Iterable { String maxf; long maxram; final List warmingUp = new ArrayList(); // for concurrent warming up - while (!t.isEmpty()) { + maxfind: while (!t.isEmpty()) { // find maximum table maxram = 0; maxf = null; @@ -218,7 +218,12 @@ public class SplitTable implements Index, Iterable { try { table = new Table(f, this.rowdef, EcoFSBufferSize, 0, this.useTailCache, this.exceed134217727, false); } catch (final RowSpaceExceededException e) { - table = new Table(f, this.rowdef, 0, 0, false, this.exceed134217727, false); + try { + table = new Table(f, this.rowdef, 0, 0, false, this.exceed134217727, false); + } catch (final RowSpaceExceededException ee) { + Log.logSevere("SplitTable", "Table " + f.toString() + " cannot be initialized: " + ee.getMessage(), ee); + continue maxfind; + } } final Table a = table; final Thread p = new Thread() { @@ -252,16 +257,7 @@ public class SplitTable implements Index, Iterable { if (f.isDirectory()) delete(this.path, element); else FileUtils.deletedelete(f); } } - try { - init(); - } catch (final RowSpaceExceededException e) { - this.useTailCache = false; - try { - init(); - } catch (final RowSpaceExceededException e1) { - throw new IOException(e1.getMessage()); - } - } + init(); } public static void delete(final File path, final String tablename) {