From 29fe401f93c58e35ebcc031d4d590e6d0771d6e4 Mon Sep 17 00:00:00 2001 From: orbiter Date: Thu, 16 Sep 2010 23:00:07 +0000 Subject: [PATCH] - some layout and text enhancement for site crawl start - Quix0rs patch from http://forum.yacy-websuche.de/viewtopic.php?p=20839#p20839 (parts) git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7163 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/AccessTracker_p.java | 7 +-- htroot/CrawlStart_p.html | 8 ++-- htroot/env/base.css | 1 + .../env/templates/submenuIndexCreate.template | 2 +- source/de/anomic/search/RankingProcess.java | 4 +- source/de/anomic/search/ResultFetcher.java | 3 +- .../de/anomic/server/serverAccessTracker.java | 46 ++++++++----------- source/de/anomic/server/serverSwitch.java | 4 +- 8 files changed, 37 insertions(+), 38 deletions(-) diff --git a/htroot/AccessTracker_p.java b/htroot/AccessTracker_p.java index f3c7af80b..8db723bd8 100644 --- a/htroot/AccessTracker_p.java +++ b/htroot/AccessTracker_p.java @@ -25,6 +25,7 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import java.util.ArrayList; +import java.util.Collection; import java.util.ConcurrentModificationException; import java.util.Date; import java.util.Iterator; @@ -53,7 +54,7 @@ public class AccessTracker_p { private static SimpleDateFormat SimpleFormatter = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss", Locale.US); - private static final List listclone(final List m) { + private static final Collection listclone(final Collection m) { final List accessClone = new LinkedList(); try { accessClone.addAll(m); @@ -76,7 +77,7 @@ public class AccessTracker_p { if (page == 0) { final Iterator i = sb.accessHosts(); String host; - List access; + Collection access; int entCount = 0; try { while ((entCount < maxCount) && (i.hasNext())) { @@ -106,7 +107,7 @@ public class AccessTracker_p { if (page == 1) { String host = (post == null) ? "" : post.get("host", ""); int entCount = 0; - List access; + Collection access; Track entry; if (host.length() > 0) { access = sb.accessTrack(host); diff --git a/htroot/CrawlStart_p.html b/htroot/CrawlStart_p.html index 099b4317a..a679b4f4d 100644 --- a/htroot/CrawlStart_p.html +++ b/htroot/CrawlStart_p.html @@ -69,10 +69,10 @@ for new documents automatically. -
+
- full domain
- only sub-path of given url + load all files in domain
+ load only files in a sub-path of given url
@@ -85,7 +85,7 @@
-
allow '?' in path +
allow query-strings (urls with a '?' in the path)
diff --git a/htroot/env/base.css b/htroot/env/base.css index f023b0ce8..e7016b5cb 100644 --- a/htroot/env/base.css +++ b/htroot/env/base.css @@ -72,6 +72,7 @@ h3 { h4 { font-size:1.1em; + margin-bottom: 4px; } a:link { diff --git a/htroot/env/templates/submenuIndexCreate.template b/htroot/env/templates/submenuIndexCreate.template index 581f19419..d7b40e3a9 100644 --- a/htroot/env/templates/submenuIndexCreate.template +++ b/htroot/env/templates/submenuIndexCreate.template @@ -5,7 +5,7 @@

Crawler/Spider

    -
  • Site Crawl
     
    • +
  • Full Site Crawl/
    Sitemap Loader
  • Crawl Start
    (Expert)
  • Crawling of
    Media Wikis
  • Crawling of
    phpBB3 Forums
    • diff --git a/source/de/anomic/search/RankingProcess.java b/source/de/anomic/search/RankingProcess.java index 48b76ae3c..0eb9c1e88 100644 --- a/source/de/anomic/search/RankingProcess.java +++ b/source/de/anomic/search/RankingProcess.java @@ -382,7 +382,9 @@ public final class RankingProcess extends Thread { // in case that we do not have e catchall filter for urls // we must also construct the domain navigator here - this.hostNavigator.inc(new String(urlhash, 6, 6), new String(urlhash)); + if (query.sitehash == null) { + this.hostNavigator.inc(new String(urlhash, 6, 6), new String(urlhash)); + } } // check for more errors diff --git a/source/de/anomic/search/ResultFetcher.java b/source/de/anomic/search/ResultFetcher.java index a60085d53..7865249e4 100644 --- a/source/de/anomic/search/ResultFetcher.java +++ b/source/de/anomic/search/ResultFetcher.java @@ -43,6 +43,7 @@ import net.yacy.kelondro.util.EventTracker; import net.yacy.repository.LoaderDispatcher; import de.anomic.crawler.CrawlProfile; +import de.anomic.crawler.CrawlProfile.CacheStrategy; import de.anomic.search.MediaSnippet; import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.graphics.ProfilingGraph; @@ -168,7 +169,7 @@ public class ResultFetcher { if (page == null) break; if (failedURLs.has(page.hash())) continue; - final ResultEntry resultEntry = fetchSnippet(page, cacheStrategy); // does not fetch snippets if snippetMode == 0 + final ResultEntry resultEntry = fetchSnippet(page, query.host == null ? cacheStrategy : CacheStrategy.CACHEONLY); // does not fetch snippets if snippetMode == 0 if (resultEntry == null) continue; // the entry had some problems, cannot be used //if (result.contains(resultEntry)) continue; diff --git a/source/de/anomic/server/serverAccessTracker.java b/source/de/anomic/server/serverAccessTracker.java index ca6e309a7..4a8c81b41 100644 --- a/source/de/anomic/server/serverAccessTracker.java +++ b/source/de/anomic/server/serverAccessTracker.java @@ -20,11 +20,11 @@ package de.anomic.server; +import java.util.Collection; import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import net.yacy.kelondro.logging.Log; @@ -35,7 +35,7 @@ public class serverAccessTracker { private final long maxTrackingTime; private final int maxTrackingCount; private final int maxHostCount; - private final ConcurrentHashMap> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries + private final ConcurrentHashMap> accessTracker; // mappings from requesting host to an ArrayList of serverTrack-entries private long lastCleanup; public static class Track { @@ -57,7 +57,7 @@ public class serverAccessTracker { this.maxTrackingTime = maxTrackingTime; this.maxTrackingCount = maxTrackingCount; this.maxHostCount = maxTrackingHostCount; - this.accessTracker = new ConcurrentHashMap>(); + this.accessTracker = new ConcurrentHashMap>(); } /* @@ -68,8 +68,8 @@ public class serverAccessTracker { if (System.currentTimeMillis() - this.lastCleanup < cleanupCycle) return; // clear entries which had no entry for the maxTrackingTime time - final Iterator>> i = accessTracker.entrySet().iterator(); - List track; + final Iterator>> i = accessTracker.entrySet().iterator(); + Collection track; while (i.hasNext()) { track = i.next().getValue(); if (tailList(track, Long.valueOf(System.currentTimeMillis() - maxTrackingTime)).isEmpty()) { @@ -93,46 +93,40 @@ public class serverAccessTracker { this.lastCleanup = System.currentTimeMillis(); } - public static List tailList(List timeList, long time) { - List t = new LinkedList(); + public static Collection tailList(Collection timeList, long time) { + Collection t = new ConcurrentLinkedQueue(); for (Track l: timeList) if (l.getTime() > time) t.add(l); return t; } - private List clearTooOldAccess(final List access) { + private Collection clearTooOldAccess(final Collection access) { try { return tailList(access, Long.valueOf(System.currentTimeMillis() - maxTrackingTime)); } catch (IllegalArgumentException e) { Log.logException(e); - return new LinkedList(); + return new ConcurrentLinkedQueue(); } } public void track(final String host, String accessPath) { // check storage size - if (System.currentTimeMillis() - this.lastCleanup > cleanupCycle) synchronized (this) { - if (System.currentTimeMillis() - this.lastCleanup > cleanupCycle) { - cleanupAccessTracker(); - this.lastCleanup = System.currentTimeMillis(); - } + if (System.currentTimeMillis() - this.lastCleanup > cleanupCycle) { + cleanupAccessTracker(); } // learn that a specific host has accessed a specific path if (accessPath == null) accessPath="NULL"; - List track = accessTracker.get(host); - if (track == null) track = new LinkedList(); - - synchronized (track) { - track.add(new Track(System.currentTimeMillis(), accessPath)); - // write back to tracker - accessTracker.put(host, clearTooOldAccess(track)); - } + Collection track = accessTracker.get(host); + if (track == null) track = new ConcurrentLinkedQueue(); + track.add(new Track(System.currentTimeMillis(), accessPath)); + // write back to tracker + accessTracker.put(host, clearTooOldAccess(track)); } - public List accessTrack(final String host) { + public Collection accessTrack(final String host) { // returns mapping from Long(accesstime) to path - List access = accessTracker.get(host); + Collection access = accessTracker.get(host); if (access == null) return null; // clear too old entries synchronized (access) { @@ -150,7 +144,7 @@ public class serverAccessTracker { public Iterator accessHosts() { // returns an iterator of hosts in tracker (String) - final Map> accessTrackerClone = new ConcurrentHashMap>(); + final Map> accessTrackerClone = new ConcurrentHashMap>(); accessTrackerClone.putAll(accessTracker); return accessTrackerClone.keySet().iterator(); } diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index b544adcf4..0caddb10c 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -25,8 +25,8 @@ import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; @@ -541,7 +541,7 @@ public class serverSwitch { this.accessTracker.track(host, accessPath); } - public List accessTrack(String host) { + public Collection accessTrack(String host) { return this.accessTracker.accessTrack(host); }