more robust OAI-PMH client (large time-out, three re-tries). OAI-PMH

server appeart to be very slow sometimes

defaults/yacy.init

@@ -691,7 +691,7 @@ crawlPause.localsearch=50
 crawlPause.remotesearch=10
 
 # Some configuration values for the crawler
-crawler.clientTimeout=9000
+crawler.clientTimeout=30000
 
 # http crawler specific settings; size in bytes
 crawler.http.accept=text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8

source/de/anomic/crawler/retrieval/HTTPLoader.java

@@ -67,7 +67,7 @@ public final class HTTPLoader {
         this.log = theLog;
 
         // refreshing timeout value
-        this.socketTimeout = (int) sb.getConfigLong("crawler.clientTimeout", 10000);
+        this.socketTimeout = (int) sb.getConfigLong("crawler.clientTimeout", 30000);
     }
 
     public Response load(final Request entry, final int maxFileSize, final boolean checkBlacklist) throws IOException {

source/net/yacy/document/importer/OAIPMHLoader.java

@@ -27,6 +27,7 @@ import java.io.IOException;
 
 import net.yacy.cora.services.federated.yacy.CacheStrategy;
 import net.yacy.kelondro.data.meta.DigestURI;
+import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.util.FileUtils;
 import net.yacy.repository.LoaderDispatcher;
 import de.anomic.crawler.retrieval.Response;
@@ -47,7 +48,20 @@ public class OAIPMHLoader {
         this.source = source;
 
         // load the file from the net
-        final Response response = loader.load(loader.request(source, false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, true);
+        Response response = null;
+        IOException ee = null;
+        for (int i = 0; i < 3; i++) {
+            // make some retries if first attempt fails
+            try {
+                response = loader.load(loader.request(source, false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, true);
+                break;
+            } catch (IOException e) {
+                Log.logWarning("OAIPMHLoader", "loading failed at attempt " + (i + 1) + ": " + source.toNormalform(true, false));
+                ee = e;
+                continue;
+            }
+        }
+        if (response == null) throw ee;
         final byte[] b = response.getContent();
         this.resumptionToken = new ResumptionToken(source, b);
         //System.out.println("*** ResumptionToken = " + this.resumptionToken.toString());
@@ -57,19 +71,6 @@ public class OAIPMHLoader {
         // transaction-safe writing
         FileUtils.copy(b, f0);
         f0.renameTo(f1);
-
-        /*
-        SurrogateReader sr = new SurrogateReader(new ByteArrayInputStream(b), 100);
-        Thread srt = new Thread(sr);
-        srt.start();
-        DCEntry dce;
-        while ((dce = sr.take()) != DCEntry.poison) {
-            System.out.println(dce.toString());
-        }
-        try {
-            srt.join();
-        } catch (InterruptedException e) {}
-        */
     }
 
     public ResumptionToken getResumptionToken() {