throw exception if crawler hostqueue can't create hostpath directory.

In rare cases hostname may not be a valid filesystem directory name,
which can't be created (e.g. containing '*' char). To prevent crawl queue
looping on this invalid entry by throwing a malformedurlexception.
pull/27/head
reger 9 years ago
parent 97cc03ef6a
commit 297fdb60d3

@ -77,7 +77,7 @@ public class HostQueue implements Balancer {
final String hostName, final String hostName,
final int port, final int port,
final boolean onDemand, final boolean onDemand,
final boolean exceed134217727) { final boolean exceed134217727) throws MalformedURLException {
this.onDemand = onDemand; this.onDemand = onDemand;
this.exceed134217727 = exceed134217727; this.exceed134217727 = exceed134217727;
this.hostName = (hostName == null) ? "localhost" : hostName; // might be null (file://) but hostqueue needs a name (for queue file) this.hostName = (hostName == null) ? "localhost" : hostName; // might be null (file://) but hostqueue needs a name (for queue file)
@ -89,7 +89,7 @@ public class HostQueue implements Balancer {
public HostQueue ( public HostQueue (
final File hostPath, final File hostPath,
final boolean onDemand, final boolean onDemand,
final boolean exceed134217727) { final boolean exceed134217727) throws MalformedURLException {
this.onDemand = onDemand; this.onDemand = onDemand;
this.exceed134217727 = exceed134217727; this.exceed134217727 = exceed134217727;
this.hostPath = hostPath; this.hostPath = hostPath;
@ -102,7 +102,7 @@ public class HostQueue implements Balancer {
init(); init();
} }
private final void init() { private final void init() throws MalformedURLException {
try { try {
if (this.hostName == null) if (this.hostName == null)
this.hostHash=""; this.hostHash="";
@ -111,7 +111,12 @@ public class HostQueue implements Balancer {
} catch (MalformedURLException e) { } catch (MalformedURLException e) {
this.hostHash = ""; this.hostHash = "";
} }
if (!(this.hostPath.exists())) this.hostPath.mkdirs(); if (!(this.hostPath.exists())) {
this.hostPath.mkdirs();
if (!this.hostPath.exists()) { // check if directory created (if not, likely a name violation)
throw new MalformedURLException("hostPath could not be created: " + this.hostPath.toString());
}
}
this.depthStacks = new TreeMap<Integer, Index>(); this.depthStacks = new TreeMap<Integer, Index>();
int size = openAllStacks(); int size = openAllStacks();
if (log.isInfo()) log.info("opened HostQueue " + this.hostPath.getAbsolutePath() + " with " + size + " urls."); if (log.isInfo()) log.info("opened HostQueue " + this.hostPath.getAbsolutePath() + " with " + size + " urls.");

Loading…
Cancel
Save