![\"arrowleft\"](\"env/grafics/navdl.gif\")
");
+ if ( thispage == 0 ) {
+ resnav
+ .append(" ");
} else {
- resnav.append(" ");
+ resnav.append(" ");
}
final int numberofpages = Math.min(10, 1 + ((indexcount - 1) / theQuery.displayResults()));
- for (int i = 0; i < numberofpages; i++) {
- if (i == thispage) {
+ for ( int i = 0; i < numberofpages; i++ ) {
+ if ( i == thispage ) {
resnav.append("![\"arrowright\"](\"env/grafics/navdr.gif\")
");
+ if ( thispage >= numberofpages ) {
+ resnav
+ .append("");
} else {
resnav.append("");
+ resnav.append(QueryParams.navurl(
+ "html",
+ thispage + 1,
+ theQuery,
+ null,
+ originalUrlMask,
+ navigation).toString());
+ resnav
+ .append("\">");
}
final String resnavs = resnav.toString();
prop.put("num-results_resnav", resnavs);
@@ -791,24 +977,29 @@ public class yacysearch {
prop.put("pageNavBottom_resnav", resnavs);
// generate the search result lines; the content will be produced by another servlet
- for (int i = 0; i < theQuery.displayResults(); i++) {
+ for ( int i = 0; i < theQuery.displayResults(); i++ ) {
prop.put("results_" + i + "_item", startRecord + i);
prop.put("results_" + i + "_eventID", theQuery.id(false));
}
prop.put("results", theQuery.displayResults());
- prop.put("resultTable", (contentdom == ContentDomain.APP || contentdom == ContentDomain.AUDIO || contentdom == ContentDomain.VIDEO) ? 1 : 0);
+ prop
+ .put(
+ "resultTable",
+ (contentdom == ContentDomain.APP || contentdom == ContentDomain.AUDIO || contentdom == ContentDomain.VIDEO)
+ ? 1
+ : 0);
prop.put("eventID", theQuery.id(false)); // for bottomline
// process result of search
- if (!filtered.isEmpty()) {
+ if ( !filtered.isEmpty() ) {
prop.put("excluded", "1");
prop.putHTML("excluded_stopwords", filtered.toString());
} else {
prop.put("excluded", "0");
}
- if (prop == null || prop.isEmpty()) {
- if (post.get("query", post.get("search", "")).length() < 2) {
+ if ( prop == null || prop.isEmpty() ) {
+ if ( post.get("query", post.get("search", "")).length() < 2 ) {
prop.put("num-results", "2"); // no results - at least 2 chars
} else {
prop.put("num-results", "1"); // no results
@@ -822,7 +1013,7 @@ public class yacysearch {
// adding some additional properties needed for the rss feed
String hostName = header.get("Host", "localhost");
- if (hostName.indexOf(':',0) == -1) {
+ if ( hostName.indexOf(':', 0) == -1 ) {
hostName += ":" + serverCore.getPortNr(env.getConfig("port", "8090"));
}
prop.put("searchBaseURL", "http://" + hostName + "/yacysearch.html");
@@ -838,10 +1029,20 @@ public class yacysearch {
prop.putHTML("prefermaskfilter", prefermask);
prop.put("indexof", (indexof) ? "on" : "off");
prop.put("constraint", (constraint == null) ? "" : constraint.exportB64());
- prop.put("search.verify", snippetFetchStrategy == null ? sb.getConfig("search.verify", "iffresh") : snippetFetchStrategy.toName());
- prop.put("search.navigation", (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "all"));
+ prop.put("search.verify", snippetFetchStrategy == null
+ ? sb.getConfig("search.verify", "iffresh")
+ : snippetFetchStrategy.toName());
+ prop.put(
+ "search.navigation",
+ (post == null) ? sb.getConfig("search.navigation", "all") : post.get("nav", "all"));
prop.put("contentdom", (post == null ? "text" : post.get("contentdom", "text")));
- prop.put("searchdomswitches", sb.getConfigBool("search.text", true) || sb.getConfigBool("search.audio", true) || sb.getConfigBool("search.video", true) || sb.getConfigBool("search.image", true) || sb.getConfigBool("search.app", true) ? 1 : 0);
+ prop.put(
+ "searchdomswitches",
+ sb.getConfigBool("search.text", true)
+ || sb.getConfigBool("search.audio", true)
+ || sb.getConfigBool("search.video", true)
+ || sb.getConfigBool("search.image", true)
+ || sb.getConfigBool("search.app", true) ? 1 : 0);
prop.put("searchdomswitches_searchtext", sb.getConfigBool("search.text", true) ? 1 : 0);
prop.put("searchdomswitches_searchaudio", sb.getConfigBool("search.audio", true) ? 1 : 0);
prop.put("searchdomswitches_searchvideo", sb.getConfigBool("search.video", true) ? 1 : 0);
diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java
index 935636d4c..5daaddad9 100644
--- a/source/de/anomic/crawler/CrawlQueues.java
+++ b/source/de/anomic/crawler/CrawlQueues.java
@@ -142,11 +142,19 @@ public class CrawlQueues {
* @return if the hash exists, the name of the database is returned, otherwise null is returned
*/
public String urlExists(final byte[] hash) {
- if (this.delegatedURL.exists(hash)) return "delegated";
- if (this.errorURL.exists(hash)) return "errors";
- if (this.noticeURL.existsInStack(hash)) return "crawler";
+ if (this.delegatedURL.exists(hash)) {
+ return "delegated";
+ }
+ if (this.errorURL.exists(hash)) {
+ return "errors";
+ }
+ if (this.noticeURL.existsInStack(hash)) {
+ return "crawler";
+ }
for (final Loader worker: this.workers.values()) {
- if (Base64Order.enhancedCoder.equal(worker.request.url().hash(), hash)) return "worker";
+ if (Base64Order.enhancedCoder.equal(worker.request.url().hash(), hash)) {
+ return "worker";
+ }
}
return null;
}
@@ -159,16 +167,26 @@ public class CrawlQueues {
public DigestURI getURL(final byte[] urlhash) {
assert urlhash != null;
- if (urlhash == null || urlhash.length == 0) return null;
+ if (urlhash == null || urlhash.length == 0) {
+ return null;
+ }
ZURL.Entry ee = this.delegatedURL.get(urlhash);
- if (ee != null) return ee.url();
+ if (ee != null) {
+ return ee.url();
+ }
ee = this.errorURL.get(urlhash);
- if (ee != null) return ee.url();
+ if (ee != null) {
+ return ee.url();
+ }
for (final Loader w: this.workers.values()) {
- if (Base64Order.enhancedCoder.equal(w.request.url().hash(), urlhash)) return w.request.url();
+ if (Base64Order.enhancedCoder.equal(w.request.url().hash(), urlhash)) {
+ return w.request.url();
+ }
}
final Request ne = this.noticeURL.get(urlhash);
- if (ne != null) return ne.url();
+ if (ne != null) {
+ return ne.url();
+ }
return null;
}
@@ -176,7 +194,9 @@ public class CrawlQueues {
// wait for all workers to finish
final int timeout = (int) this.sb.getConfigLong("crawler.clientTimeout", 10000);
for (final Loader w: this.workers.values()) {
- if (w.age() > timeout) w.interrupt();
+ if (w.age() > timeout) {
+ w.interrupt();
+ }
}
}
@@ -185,7 +205,9 @@ public class CrawlQueues {
final Request[] e = new Request[this.workers.size()];
int i = 0;
for (final Loader w: this.workers.values()) {
- if (i >= e.length) break;
+ if (i >= e.length) {
+ break;
+ }
e[i++] = w.request;
}
return e;
@@ -199,8 +221,7 @@ public class CrawlQueues {
public boolean coreCrawlJob() {
final boolean robinsonPrivateCase = (this.sb.isRobinsonMode() &&
- !this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER) &&
- !this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PRIVATE_CLUSTER));
+ !this.sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER));
if ((robinsonPrivateCase || coreCrawlJobSize() <= 20) && limitCrawlJobSize() > 0) {
// move some tasks to the core crawl job so we have something to do
@@ -216,12 +237,16 @@ public class CrawlQueues {
final String queueCheckCore = loadIsPossible(NoticedURL.StackType.CORE);
final String queueCheckNoload = loadIsPossible(NoticedURL.StackType.NOLOAD);
if (queueCheckCore != null && queueCheckNoload != null) {
- if (this.log.isFine()) this.log.logFine("omitting de-queue/local: " + queueCheckCore + ":" + queueCheckNoload);
+ if (this.log.isFine()) {
+ this.log.logFine("omitting de-queue/local: " + queueCheckCore + ":" + queueCheckNoload);
+ }
return false;
}
if (isPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
- if (this.log.isFine()) this.log.logFine("omitting de-queue/local: paused");
+ if (this.log.isFine()) {
+ this.log.logFine("omitting de-queue/local: paused");
+ }
return false;
}
@@ -238,7 +263,9 @@ public class CrawlQueues {
if (this.noticeURL.stackSize(NoticedURL.StackType.NOLOAD) > 0) {
// get one entry that will not be loaded, just indexed
urlEntry = this.noticeURL.pop(NoticedURL.StackType.NOLOAD, true, this.sb.crawler);
- if (urlEntry == null) continue;
+ if (urlEntry == null) {
+ continue;
+ }
final String profileHandle = urlEntry.profileHandle();
if (profileHandle == null) {
this.log.logSevere(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url());
@@ -259,7 +286,9 @@ public class CrawlQueues {
}
urlEntry = this.noticeURL.pop(NoticedURL.StackType.CORE, true, this.sb.crawler);
- if (urlEntry == null) continue;
+ if (urlEntry == null) {
+ continue;
+ }
final String profileHandle = urlEntry.profileHandle();
// System.out.println("DEBUG plasmaSwitchboard.processCrawling:
// profileHandle = " + profileHandle + ", urlEntry.url = " + urlEntry.url());
@@ -271,7 +300,9 @@ public class CrawlQueues {
return true;
} catch (final IOException e) {
this.log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e);
- if (e.getMessage().indexOf("hash is null",0) > 0) this.noticeURL.clear(NoticedURL.StackType.CORE);
+ if (e.getMessage().indexOf("hash is null",0) > 0) {
+ this.noticeURL.clear(NoticedURL.StackType.CORE);
+ }
}
}
return true;
@@ -293,7 +324,7 @@ public class CrawlQueues {
final DigestURI url = urlEntry.url();
final String urlProtocol = url.getProtocol();
if (this.sb.loader.isSupportedProtocol(urlProtocol)) {
- if (this.log.isFine())
+ if (this.log.isFine()) {
this.log.logFine(stats + ": URL=" + urlEntry.url()
+ ", initiator=" + ((urlEntry.initiator() == null) ? "" : ASCII.String(urlEntry.initiator()))
+ ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false")
@@ -302,6 +333,7 @@ public class CrawlQueues {
+ ", must-match=" + profile.urlMustMatchPattern().toString()
+ ", must-not-match=" + profile.urlMustNotMatchPattern().toString()
+ ", permission=" + ((this.sb.peers == null) ? "undefined" : (((this.sb.peers.mySeed().isSenior()) || (this.sb.peers.mySeed().isPrincipal())) ? "true" : "false")));
+ }
// work off one Crawl stack entry
if (urlEntry == null || urlEntry.url() == null) {
@@ -387,23 +419,31 @@ public class CrawlQueues {
}
// check again
if (this.workers.size() >= this.sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 20)) {
- if (this.log.isFine()) this.log.logFine("remoteCrawlLoaderJob: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.workers.size() + "), httpClients = " + ConnectionInfo.getCount());
+ if (this.log.isFine()) {
+ this.log.logFine("remoteCrawlLoaderJob: too many processes in loader queue, dismissed (" + "cacheLoader=" + this.workers.size() + "), httpClients = " + ConnectionInfo.getCount());
+ }
return false;
}
final String cautionCause = this.sb.onlineCaution();
if (cautionCause != null) {
- if (this.log.isFine()) this.log.logFine("remoteCrawlLoaderJob: online caution for " + cautionCause + ", omitting processing");
+ if (this.log.isFine()) {
+ this.log.logFine("remoteCrawlLoaderJob: online caution for " + cautionCause + ", omitting processing");
+ }
return false;
}
if (remoteTriggeredCrawlJobSize() > 200) {
- if (this.log.isFine()) this.log.logFine("remoteCrawlLoaderJob: the remote-triggered crawl job queue is filled, omitting processing");
+ if (this.log.isFine()) {
+ this.log.logFine("remoteCrawlLoaderJob: the remote-triggered crawl job queue is filled, omitting processing");
+ }
return false;
}
if (coreCrawlJobSize() > 0 /*&& sb.indexingStorageProcessor.queueSize() > 0*/) {
- if (this.log.isFine()) this.log.logFine("remoteCrawlLoaderJob: a local crawl is running, omitting processing");
+ if (this.log.isFine()) {
+ this.log.logFine("remoteCrawlLoaderJob: a local crawl is running, omitting processing");
+ }
return false;
}
@@ -414,27 +454,37 @@ public class CrawlQueues {
final Iterator+ * * @return the IP or null */ public final String getIP() { final String ip = get(Seed.IP, "127.0.0.1"); return (ip == null || ip.length() == 0) ? "127.0.0.1" : ip; } + /** * try to get the peertype
+ * * @return the peertype or null */ - public final String getPeerType() { return get(Seed.PEERTYPE, ""); } + public final String getPeerType() { + return get(Seed.PEERTYPE, ""); + } + /** * try to get the peertype
+ * * @return the peertype or "virgin" */ - public final String orVirgin() { return get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN); } + public final String orVirgin() { + return get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN); + } + /** * try to get the peertype
+ * * @return the peertype or "junior" */ - public final String orJunior() { return get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR); } + public final String orJunior() { + return get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR); + } + /** * try to get the peertype
+ * * @return the peertype or "senior" */ - public final String orSenior() { return get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); } + public final String orSenior() { + return get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); + } + /** * try to get the peertype
+ * * @return the peertype or "principal" */ - public final String orPrincipal() { return get(Seed.PEERTYPE, Seed.PEERTYPE_PRINCIPAL); } + public final String orPrincipal() { + return get(Seed.PEERTYPE, Seed.PEERTYPE_PRINCIPAL); + } /** * Get a value from the peer's DNA (its set of peer defining values, e.g. IP, name, version, ...) + * * @param key the key for the value to fetch * @param dflt the default value */ public final String get(final String key, final String dflt) { final Object o = this.dna.get(key); - if (o == null) { return dflt; } + if ( o == null ) { + return dflt; + } return (String) o; } public final float getFloat(final String key, final float dflt) { final Object o = this.dna.get(key); - if (o == null) { return dflt; } - if (o instanceof String) try { - return Float.parseFloat((String) o); - } catch (final NumberFormatException e) { - return dflt; - } else if (o instanceof Float) { + if ( o == null ) { + return dflt; + } + if ( o instanceof String ) { + try { + return Float.parseFloat((String) o); + } catch ( final NumberFormatException e ) { + return dflt; + } + } else if ( o instanceof Float ) { return ((Float) o).floatValue(); - } else return dflt; + } else { + return dflt; + } } public final long getLong(final String key, final long dflt) { final Object o = this.dna.get(key); - if (o == null) { return dflt; } - if (o instanceof String) try { - return Long.parseLong((String) o); - } catch (final NumberFormatException e) { - return dflt; - } else if (o instanceof Long) { + if ( o == null ) { + return dflt; + } + if ( o instanceof String ) { + try { + return Long.parseLong((String) o); + } catch ( final NumberFormatException e ) { + return dflt; + } + } else if ( o instanceof Long ) { return ((Long) o).longValue(); - } else if (o instanceof Integer) { + } else if ( o instanceof Integer ) { return ((Integer) o).intValue(); - } else return dflt; + } else { + return dflt; + } + } + + public final void setIP(final String ip) { + this.dna.put(Seed.IP, ip); + } + + public final void setPort(final String port) { + this.dna.put(Seed.PORT, port); + } + + public final void setType(final String type) { + this.dna.put(Seed.PEERTYPE, type); + } + + public final void setJunior() { + this.dna.put(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR); } - public final void setIP(final String ip) { this.dna.put(Seed.IP, ip); } - public final void setPort(final String port) { this.dna.put(Seed.PORT, port); } - public final void setType(final String type) { this.dna.put(Seed.PEERTYPE, type); } - public final void setJunior() { this.dna.put(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR); } - public final void setSenior() { this.dna.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); } - public final void setPrincipal() { this.dna.put(Seed.PEERTYPE, Seed.PEERTYPE_PRINCIPAL); } + public final void setSenior() { + this.dna.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); + } + + public final void setPrincipal() { + this.dna.put(Seed.PEERTYPE, Seed.PEERTYPE_PRINCIPAL); + } public final void put(final String key, final String value) { - synchronized (this.dna) { + synchronized ( this.dna ) { this.dna.put(key, value); } } @@ -374,7 +441,7 @@ public class Seed implements Cloneable, Comparable
12 * 6 bit = 72 bit = 24 characters octal-hash
- * Octal hashes are used for cache-dumps that are DHT-ready
*- * Cause: the natural order of octal hashes are the same as the b64-order of b64Hashes. - * a hexhash cannot be used in such cases, and b64Hashes are not appropriate for file names + * Octal hashes are used for cache-dumps that are DHT-ready + *
+ *+ * Cause: the natural order of octal hashes are the same as the b64-order of b64Hashes. a hexhash cannot + * be used in such cases, and b64Hashes are not appropriate for file names *
+ * * @param b64Hash a base64 hash * @return the octal representation of the given base64 hash */ @@ -434,11 +512,14 @@ public class Seed implements Cloneable, Comparable12 * 6 bit = 72 bit = 18 characters hex-hash
+ *
* @param b64Hash a base64 hash
* @return the hexadecimal representation of the given base64 hash
*/
public static String b64Hash2hexHash(final String b64Hash) {
- if (b64Hash.length() > 12) return "";
+ if ( b64Hash.length() > 12 ) {
+ return "";
+ }
// the hash string represents 12 * 6 bit = 72 bits. This is too much for a long integer.
return Digest.encodeHex(Base64Order.enhancedCoder.decode(b64Hash));
}
@@ -453,19 +534,21 @@ public class Seed implements Cloneable, ComparableMAJORVERSION . MINORVERSION 0 SVN REVISION
+ *
* @return the YaCy version of this peer as a float or 0 if no valid value could be retrieved
- * from this yacySeed object
+ * from this yacySeed object
*/
public final float getVersion() {
try {
return Float.parseFloat(get(Seed.VERSION, Seed.ZERO));
- } catch (final NumberFormatException e) {
+ } catch ( final NumberFormatException e ) {
return 0;
}
}
/**
* get the SVN version of the peer
+ *
* @return
*/
public final int getRevision() {
@@ -474,14 +557,18 @@ public class Seed implements Cloneable, Comparablenull if no valid values for
- * either the IP or the port could be retrieved from this yacySeed object
+ * either the IP or the port could be retrieved from this yacySeed object
*/
public final String getPublicAddress() {
String ip = getIP();
- if (ip == null || ip.length() < 8 || ip.length() > 60) ip = "127.0.0.1";
+ if ( ip == null || ip.length() < 8 || ip.length() > 60 ) {
+ ip = "127.0.0.1";
+ }
final String port = this.dna.get(Seed.PORT);
- if (port == null || port.length() < 2 || port.length() > 5) return null;
+ if ( port == null || port.length() < 2 || port.length() > 5 ) {
+ return null;
+ }
final StringBuilder sb = new StringBuilder(ip.length() + port.length() + 1);
sb.append(ip);
@@ -492,16 +579,21 @@ public class Seed implements Cloneable, Comparable-1 if not present */
public final int getPort() {
final String port = this.dna.get(Seed.PORT);
- if (port == null) return -1;
+ if ( port == null ) {
+ return -1;
+ }
/*if (port.length() < 2) return -1; It is possible to use port 0-9*/
return Integer.parseInt(port);
}
@@ -526,10 +620,13 @@ public class Seed implements Cloneable, Comparable- * Then it iterates through all existing {@link CrawlProfiles crawl profiles} and removes - * all profiles which are not hard-coded. + * Then it iterates through all existing {@link CrawlProfiles crawl profiles} and removes all profiles + * which are not hard-coded. *
*
- * If this method encounters DB-failures, the profile DB will be reseted and
- * true will be returned
+ * If this method encounters DB-failures, the profile DB will be reseted and true
+ * will be returned
*