limitation of reference evaluation (protection against crawler pits)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7902 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 13 years ago
parent a5541751a8
commit 3e6767d66c

@ -147,7 +147,8 @@ public class WebStructureGraph {
final HashSet<MultiProtocolURI> globalRefURLs = new HashSet<MultiProtocolURI>();
final String refhost = url.getHost();
MultiProtocolURI u;
while (it.hasNext()) {
int maxref = 1000;
while (it.hasNext() && maxref-- > 0) {
u = it.next();
if (u == null) continue;
if (refhost != null && u.getHost() != null && !u.getHost().equals(refhost)) {

Loading…
Cancel
Save