addon to http://bugs.yacy.net/view.php?id=72

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@8095 6c8d7289-2bf4-0310-a012-ef5d649a1542
13 years ago · da55a359e9
parent c9a0dbd25a
commit da55a359e9
1 changed files with 10 additions and 4 deletions
--- a/htroot/Crawler_p.java
+++ b/htroot/Crawler_p.java
@ -174,7 +174,13 @@ public class Crawler_p {
                    } else if (crawlingStartURL.isFTP()) {
                        newcrawlingMustMatch = "ftp://" + crawlingStartURL.getHost();
                    } else {
+                        final String host = crawlingStartURL.getHost();
+                        if (host.startsWith("www.")) {
                            newcrawlingMustMatch = "https?://" + crawlingStartURL.getHost();
+                        } else {
+                            // if the www is not given we accept that also
+                            newcrawlingMustMatch = "https?://(www.)?" + crawlingStartURL.getHost();
+                        }
                    }
                    if (subPath) newcrawlingMustMatch += crawlingStartURL.getPath();
                    newcrawlingMustMatch += ".*";