diff --git a/bin/apicall.sh b/bin/apicall.sh index 60d359167..50a3331ca 100755 --- a/bin/apicall.sh +++ b/bin/apicall.sh @@ -4,9 +4,9 @@ port=$(grep ^port= ../DATA/SETTINGS/yacy.conf |cut -d= -f2) pw=$(grep ^adminAccountBase64MD5= ../DATA/SETTINGS/yacy.conf |cut -d= -f2) if which curl &>/dev/null; then - curl -s --header "Authorization: realm=$pw" "http://127.0.0.1:$port/$1" > /dev/null + curl -s --header "Authorization: realm=$pw" "http://127.0.0.1:$port/$1" elif which wget &>/dev/null; then - wget -q -t 1 --timeout=5 --header "Authorization: realm=$pw" "http://127.0.0.1:$port/$1" -O /dev/null + wget -q -t 1 --timeout=120 --header "Authorization: realm=$pw" "http://127.0.0.1:$port/$1" -O - else exit 1 fi diff --git a/bin/indexdump.sh b/bin/indexdump.sh new file mode 100755 index 000000000..d71b4248e --- /dev/null +++ b/bin/indexdump.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cd "`dirname $0`" +./apicall0.sh "/IndexControlURLs_p.xml?indexdump=" | awk '//{ gsub("","" );gsub("<\/dumpfile>","" ); print $0 }' | awk '{print $1}'; \ No newline at end of file diff --git a/htroot/IndexControlURLs_p.xml b/htroot/IndexControlURLs_p.xml new file mode 100644 index 000000000..79ce730bc --- /dev/null +++ b/htroot/IndexControlURLs_p.xml @@ -0,0 +1,25 @@ + + + #(statisticslines)#:: + + #{domains}# + + #[domain]# + #[count]# + + #{/domains}# + + #(/statisticslines)# + #(indexdump)#:: + #[dumpfile]#:: + #(/indexdump)# + #(urlhashsimilar)#:: + + #{rows}# + #{cols}# + #[urlHash]# + #{/cols}# + #{/rows}# + + #(/urlhashsimilar)# + \ No newline at end of file diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index dfc8fea67..d9ac71377 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -293,10 +293,12 @@ public final class Fulltext implements Iterable { if (this.connectedSolr()) { try { if (this.urlIndexFile != null) this.urlIndexFile.remove(idb); - SolrDocument sd = this.solr.get(id); - if (sd == null || this.solrScheme.getDate(sd, YaCySchema.last_modified).before(this.solrScheme.getDate(doc, YaCySchema.last_modified))) { - this.solr.add(doc); - } + synchronized (this.solr) { + SolrDocument sd = this.solr.get(id); + if (sd == null || this.solrScheme.getDate(sd, YaCySchema.last_modified).before(this.solrScheme.getDate(doc, YaCySchema.last_modified))) { + this.solr.add(doc); + } + } } catch (SolrException e) { throw new IOException(e.getMessage(), e); } @@ -334,10 +336,12 @@ public final class Fulltext implements Iterable { if (this.connectedSolr()) { try { if (this.urlIndexFile != null) this.urlIndexFile.remove(idb); - SolrDocument sd = this.solr.get(id); - if (sd == null || (new URIMetadataNode(sd)).isOlder(row)) { - this.solr.add(getSolrScheme().metadata2solr(row)); - } + synchronized (this.solr) { + SolrDocument sd = this.solr.get(id); + if (sd == null || (new URIMetadataNode(sd)).isOlder(row)) { + this.solr.add(getSolrScheme().metadata2solr(row)); + } + } } catch (SolrException e) { throw new IOException(e.getMessage(), e); } @@ -365,7 +369,9 @@ public final class Fulltext implements Iterable { public boolean remove(final byte[] urlHash) { if (urlHash == null) return false; try { - this.solr.delete(ASCII.String(urlHash)); + synchronized (this.solr) { + this.solr.delete(ASCII.String(urlHash)); + } } catch (final Throwable e) { Log.logException(e); } @@ -470,17 +476,19 @@ public final class Fulltext implements Iterable { EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0(); int commitWithin = esc.getCommitWithinMs(); File storagePath = esc.getStoragePath(); - this.disconnectLocalSolr(); File zipOut = new File(storagePath.toString() + "_" + GenericFormatter.SHORT_DAY_FORMATTER.format() + ".zip"); - try { - ZIPWriter.zip(storagePath, zipOut); - } catch (IOException e) { - Log.logException(e); - } finally { + synchronized (this.solr) { + this.disconnectLocalSolr(); try { - this.connectLocalSolr(commitWithin); + ZIPWriter.zip(storagePath, zipOut); } catch (IOException e) { Log.logException(e); + } finally { + try { + this.connectLocalSolr(commitWithin); + } catch (IOException e) { + Log.logException(e); + } } } return zipOut; @@ -494,16 +502,18 @@ public final class Fulltext implements Iterable { EmbeddedSolrConnector esc = (EmbeddedSolrConnector) this.solr.getSolr0(); int commitWithin = esc.getCommitWithinMs(); File storagePath = esc.getStoragePath(); - this.disconnectLocalSolr(); - try { - ZIPReader.unzip(solrDumpZipFile, storagePath); - } catch (IOException e) { - Log.logException(e); - } finally { + synchronized (this.solr) { + this.disconnectLocalSolr(); try { - this.connectLocalSolr(commitWithin); + ZIPReader.unzip(solrDumpZipFile, storagePath); } catch (IOException e) { Log.logException(e); + } finally { + try { + this.connectLocalSolr(commitWithin); + } catch (IOException e) { + Log.logException(e); + } } } } @@ -783,7 +793,9 @@ public final class Fulltext implements Iterable { // first collect all url hashes that belong to the domain assert hosthash.length() == 6; // delete in solr - this.solr.deleteByQuery(YaCySchema.host_id_s.name() + ":\"" + hosthash + "\""); + synchronized (this.solr) { + this.solr.deleteByQuery(YaCySchema.host_id_s.name() + ":\"" + hosthash + "\""); + } // delete in old metadata structure final ArrayList l = new ArrayList();