boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

*) some minor changes for better code readability

Browse Source 
*) added more SVN properties

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6787 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
low012 15 years ago
parent 7a3c19846f
commit b97ad0f380
8 changed files with 34 additions and 35 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/crawler/CrawlProfile.java
Unescape View File

@ -85,6 +85,7 @@ public class CrawlProfile {
this.profileTable = null;
}
@Override
public void finalize() {
this.close();
}
@ -319,6 +320,7 @@ public class CrawlProfile {
doms = new ConcurrentHashMap<String, DomProfile>();
}
@Override
public String toString() {
final StringBuilder str = new StringBuilder();

1
source/de/anomic/crawler/CrawlQueues.java
Unescape View File

@ -535,6 +535,7 @@ public class CrawlQueues {
return System.currentTimeMillis() - start;
}
@Override
public void run() {
try {
// checking robots.txt for http(s) resources

6
source/de/anomic/crawler/CrawlSwitchboard.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2009-05-28 01:51:34 +0200 (Do, 28 Mai 2009) $
// $LastChangedRevision: 5988 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

18
source/de/anomic/crawler/ImporterManager.java
Unescape View File

@ -30,7 +30,7 @@ public class ImporterManager {
final Thread[] importThreads = new Thread[this.runningJobs.activeCount()*2];
final int activeCount = this.runningJobs.enumerate(importThreads);
final Importer[] importers = new Importer[activeCount];
for (int i=0; i<activeCount; i++) {
for (int i = 0; i < activeCount; i++) {
importers[i] = (Importer) importThreads[i];
}
return importers;
@ -43,10 +43,9 @@ public class ImporterManager {
public Importer getImporterByID(final int jobID) {
final Thread[] importThreads = new Thread[this.runningJobs.activeCount()*2];
final int activeCount = this.runningJobs.enumerate(importThreads);
for (int i=0; i < activeCount; i++) {
final Importer currThread = (Importer) importThreads[i];
for(final Thread importThread : importThreads) {
final Importer currThread = (Importer) importThread;
if (currThread.getJobID() == jobID) {
return currThread;
}
@ -73,8 +72,7 @@ public class ImporterManager {
try {
// trying to gracefull stop all still running sessions ...
log.logInfo("Signaling shutdown to " + threadCount + " remaining dbImporter threads ...");
for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) {
final Thread currentThread = threadList[currentThreadIdx];
for (final Thread currentThread : threadList) {
if (currentThread.isAlive()) {
((Importer)currentThread).stopIt();
}
@ -89,10 +87,10 @@ public class ImporterManager {
// we need to use a timeout here because of missing interruptable session threads ...
if (log.isFine()) log.logFine("Waiting for " + runningJobs.activeCount() + " remaining dbImporter threads to finish shutdown ...");
for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) {
final Thread currentThread = threadList[currentThreadIdx];
int currentThreadIdx = 0;
for (final Thread currentThread : threadList) {
if (currentThread.isAlive()) {
if (log.isFine()) log.logFine("Waiting for dbImporter thread '" + currentThread.getName() + "' [" + currentThreadIdx + "] to finish shutdown.");
if (log.isFine()) log.logFine("Waiting for dbImporter thread '" + currentThread.getName() + "' [" + currentThreadIdx++ + "] to finish shutdown.");
try { currentThread.join(500); } catch (final InterruptedException ex) {}
}
}

6
source/de/anomic/crawler/ResultImages.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

6
source/de/anomic/crawler/ResultURLs.java
Unescape View File

@ -5,9 +5,9 @@
// first published on http://yacy.net
// Frankfurt, Germany, 2004
//
// $LastChangedDate: 2008-03-16 23:31:54 +0100 (So, 16 Mrz 2008) $
// $LastChangedRevision: 4575 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by

24
source/de/anomic/crawler/RobotsEntry.java
Unescape View File

@ -28,12 +28,11 @@
package de.anomic.crawler;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import net.yacy.kelondro.data.meta.DigestURI;
@ -53,7 +52,7 @@ public class RobotsEntry {
// this is a simple record structure that holds all properties of a single crawl start
private Map<String, byte[]> mem;
private LinkedList<String> allowPathList, denyPathList;
private List<String> allowPathList, denyPathList;
String hostName;
public RobotsEntry(final String hostName, final Map<String, byte[]> mem) {
@ -88,8 +87,8 @@ public class RobotsEntry {
public RobotsEntry(
final DigestURI theURL,
final ArrayList<String> allowPathList,
final ArrayList<String> disallowPathList,
final List<String> allowPathList,
final List<String> disallowPathList,
final Date loadedDate,
final Date modDate,
final String eTag,
@ -114,8 +113,8 @@ public class RobotsEntry {
this.allowPathList.addAll(allowPathList);
final StringBuilder pathListStr = new StringBuilder(allowPathList.size() * 30);
for (int i=0; i<allowPathList.size();i++) {
pathListStr.append(allowPathList.get(i))
for (String element : allowPathList) {
pathListStr.append(element)
.append(ROBOTS_DB_PATH_SEPARATOR);
}
this.mem.put(ALLOW_PATH_LIST, pathListStr.substring(0,pathListStr.length()-1).getBytes());
@ -125,8 +124,8 @@ public class RobotsEntry {
this.denyPathList.addAll(disallowPathList);
final StringBuilder pathListStr = new StringBuilder(disallowPathList.size() * 30);
for (int i=0; i<disallowPathList.size();i++) {
pathListStr.append(disallowPathList.get(i))
for (String element : disallowPathList) {
pathListStr.append(element)
.append(ROBOTS_DB_PATH_SEPARATOR);
}
this.mem.put(DISALLOW_PATH_LIST,pathListStr.substring(0, pathListStr.length()-1).getBytes());
@ -138,6 +137,7 @@ public class RobotsEntry {
return this.mem;
}
@Override
public String toString() {
final StringBuilder str = new StringBuilder(6000);
str.append((this.hostName == null) ? "null" : this.hostName).append(": ");
@ -198,12 +198,10 @@ public class RobotsEntry {
// escaping all occurences of ; because this char is used as special char in the Robots DB
else path = path.replaceAll(ROBOTS_DB_PATH_SEPARATOR,"%3B");
final Iterator<String> pathIter = this.denyPathList.iterator();
while (pathIter.hasNext()) {
final String nextPath = pathIter.next();
for (String element : this.denyPathList) {
// disallow rule
if (path.startsWith(nextPath)) {
if (path.startsWith(element)) {
return true;
}
}

6
source/de/anomic/crawler/ZURL.java
Unescape View File

@ -4,9 +4,9 @@
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
// $LastChangedRevision: 1986 $
// $LastChangedBy: orbiter $
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//

Write Preview
Loading…
Cancel
Save