boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

more generics

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4380 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 4e70dff8cf
commit 9e7cd4fdbb
2 changed files with 34 additions and 32 deletions
Show Stats Download Patch File Download Diff File

35
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -83,7 +83,7 @@ public class yacyCore {
public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup"); public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup");
public static yacySeedDB seedDB = null; public static yacySeedDB seedDB = null;
public static yacyNewsPool newsPool = null; public static yacyNewsPool newsPool = null;
public static final HashMap seedUploadMethods = new HashMap(); public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>();
public static yacyPeerActions peerActions = null; public static yacyPeerActions peerActions = null;
public static yacyDHTAction dhtAgent = null; public static yacyDHTAction dhtAgent = null;
public static serverLog log; public static serverLog log;
@ -291,10 +291,10 @@ public class yacyCore {
private int added; private int added;
private yacySeed seed; private yacySeed seed;
private final serverSemaphore sync; private final serverSemaphore sync;
private final List syncList; private final List<Thread> syncList;
public publishThread(ThreadGroup tg, yacySeed seed, public publishThread(ThreadGroup tg, yacySeed seed,
serverSemaphore sync, List syncList) throws InterruptedException { serverSemaphore sync, List<Thread> syncList) throws InterruptedException {
super(tg, "PublishSeed_" + seed.getName()); super(tg, "PublishSeed_" + seed.getName());
this.sync = sync; this.sync = sync;
@ -369,24 +369,24 @@ public class yacyCore {
// probed until we get a valid response. // probed until we get a valid response.
// init yacyHello-process // init yacyHello-process
Map seeds; // hash/yacySeed relation Map<String, yacySeed> seeds; // hash/yacySeed relation
int attempts = seedDB.sizeConnected(); int attempts = seedDB.sizeConnected();
// getting a list of peers to contact // getting a list of peers to contact
if (seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) { if (seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
if (attempts > PING_INITIAL) { attempts = PING_INITIAL; } if (attempts > PING_INITIAL) { attempts = PING_INITIAL; }
Map ch = plasmaSwitchboard.getSwitchboard().clusterhashes; Map<String, String> ch = plasmaSwitchboard.getSwitchboard().clusterhashes;
seeds = seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection seeds = seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster // add also all peers from cluster if this is a public robinson cluster
if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) { if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) {
Iterator i = ch.entrySet().iterator(); Iterator<Map.Entry<String, String>> i = ch.entrySet().iterator();
String hash; String hash;
Map.Entry entry; Map.Entry<String, String> entry;
yacySeed seed; yacySeed seed;
while (i.hasNext()) { while (i.hasNext()) {
entry = (Map.Entry) i.next(); entry = i.next();
hash = (String) entry.getKey(); hash = entry.getKey();
seed = (yacySeed) seeds.get(hash); seed = (yacySeed) seeds.get(hash);
if (seed == null) { if (seed == null) {
seed = seedDB.get(hash); seed = seedDB.get(hash);
@ -411,7 +411,7 @@ public class yacyCore {
if (seeds.size() < attempts) { attempts = seeds.size(); } if (seeds.size() < attempts) { attempts = seeds.size(); }
// This will try to get Peers that are not currently in amIAccessibleDB // This will try to get Peers that are not currently in amIAccessibleDB
Iterator si = seeds.values().iterator(); Iterator<yacySeed> si = seeds.values().iterator();
yacySeed seed; yacySeed seed;
// include a YaCyNews record to my seed // include a YaCyNews record to my seed
@ -434,7 +434,7 @@ public class yacyCore {
//if (seeds.length > 1) { //if (seeds.length > 1) {
// holding a reference to all started threads // holding a reference to all started threads
int contactedSeedCount = 0; int contactedSeedCount = 0;
final List syncList = Collections.synchronizedList(new LinkedList()); // memory for threads final List<Thread> syncList = Collections.synchronizedList(new LinkedList<Thread>()); // memory for threads
final serverSemaphore sync = new serverSemaphore(attempts); final serverSemaphore sync = new serverSemaphore(attempts);
// going through the peer list and starting a new publisher thread for each peer // going through the peer list and starting a new publisher thread for each peer
@ -493,7 +493,7 @@ public class yacyCore {
final int dbSize; final int dbSize;
synchronized (amIAccessibleDB) { synchronized (amIAccessibleDB) {
dbSize = amIAccessibleDB.size(); dbSize = amIAccessibleDB.size();
Iterator ai = amIAccessibleDB.keySet().iterator(); Iterator<String> ai = amIAccessibleDB.keySet().iterator();
while (ai.hasNext()) { while (ai.hasNext()) {
yacyAccessible ya = (yacyAccessible) amIAccessibleDB.get(ai.next()); yacyAccessible ya = (yacyAccessible) amIAccessibleDB.get(ai.next());
if (ya.lastUpdated < cutofftime) { if (ya.lastUpdated < cutofftime) {
@ -611,9 +611,10 @@ public class yacyCore {
} }
} }
public static HashMap getSeedUploadMethods() { @SuppressWarnings("unchecked")
public static HashMap<String, String> getSeedUploadMethods() {
synchronized (yacyCore.seedUploadMethods) { synchronized (yacyCore.seedUploadMethods) {
return (HashMap) yacyCore.seedUploadMethods.clone(); return (HashMap<String, String>) yacyCore.seedUploadMethods.clone();
} }
} }
@ -627,7 +628,7 @@ public class yacyCore {
if (className == null) { return null; } if (className == null) { return null; }
try { try {
final Class uploaderClass = Class.forName(className); final Class<?> uploaderClass = Class.forName(className);
final Object uploader = uploaderClass.newInstance(); final Object uploader = uploaderClass.newInstance();
return (yacySeedUploader) uploader; return (yacySeedUploader) uploader;
} catch (Exception e) { } catch (Exception e) {
@ -636,7 +637,7 @@ public class yacyCore {
} }
public static void loadSeedUploadMethods() { public static void loadSeedUploadMethods() {
final HashMap availableUploaders = new HashMap(); final HashMap<String, String> availableUploaders = new HashMap<String, String>();
try { try {
final String uploadersPkgName = yacyCore.class.getPackage().getName() + ".seedUpload"; final String uploadersPkgName = yacyCore.class.getPackage().getName() + ".seedUpload";
final String packageURI = yacyCore.class.getResource("/" + uploadersPkgName.replace('.', '/')).toString(); final String packageURI = yacyCore.class.getResource("/" + uploadersPkgName.replace('.', '/')).toString();
@ -661,7 +662,7 @@ public class yacyCore {
final String className = uploaderClasses[uploaderNr].substring(0, uploaderClasses[uploaderNr].indexOf(".class")); final String className = uploaderClasses[uploaderNr].substring(0, uploaderClasses[uploaderNr].indexOf(".class"));
final String fullClassName = uploadersPkgName + "." + className; final String fullClassName = uploadersPkgName + "." + className;
try { try {
final Class uploaderClass = Class.forName(fullClassName); final Class<?> uploaderClass = Class.forName(fullClassName);
final Object theUploader = uploaderClass.newInstance(); final Object theUploader = uploaderClass.newInstance();
if (!(theUploader instanceof yacySeedUploader)) { continue; } if (!(theUploader instanceof yacySeedUploader)) { continue; }
final String[] neededLibx = ((yacySeedUploader)theUploader).getLibxDependencies(); final String[] neededLibx = ((yacySeedUploader)theUploader).getLibxDependencies();

31
source/yacy.java
Unescape View File

@ -69,6 +69,7 @@ import de.anomic.http.httpc;
import de.anomic.http.httpd; import de.anomic.http.httpd;
import de.anomic.index.indexContainer; import de.anomic.index.indexContainer;
import de.anomic.index.indexRWIEntry; import de.anomic.index.indexRWIEntry;
import de.anomic.index.indexRWIRowEntry;
import de.anomic.index.indexURLEntry; import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroDyn;
@ -552,7 +553,7 @@ public final class yacy {
// load words // load words
serverLog.logInfo("GEN-WORDSTAT", "loading words..."); serverLog.logInfo("GEN-WORDSTAT", "loading words...");
HashMap words = loadWordMap(new File(homePath, "yacy.words")); HashMap<String, String> words = loadWordMap(new File(homePath, "yacy.words"));
// find all hashes // find all hashes
serverLog.logInfo("GEN-WORDSTAT", "searching all word-hash databases..."); serverLog.logInfo("GEN-WORDSTAT", "searching all word-hash databases...");
@ -570,10 +571,10 @@ public final class yacy {
// list the hashes in reverse order // list the hashes in reverse order
serverLog.logInfo("GEN-WORDSTAT", "listing words in reverse size order..."); serverLog.logInfo("GEN-WORDSTAT", "listing words in reverse size order...");
String w; String w;
Iterator i = hs.scores(false); Iterator<String> i = hs.scores(false);
while (i.hasNext()) { while (i.hasNext()) {
h = (String) i.next(); h = i.next();
w = (String) words.get(h); w = words.get(h);
if (w == null) System.out.print("# " + h); else System.out.print(w); if (w == null) System.out.print("# " + h); else System.out.print(w);
System.out.println(" - " + hs.getScore(h)); System.out.println(" - " + hs.getScore(h));
} }
@ -607,7 +608,7 @@ public final class yacy {
if (cacheMem < 2048000) throw new OutOfMemoryError("Not enough memory available to start clean up."); if (cacheMem < 2048000) throw new OutOfMemoryError("Not enough memory available to start clean up.");
plasmaWordIndex wordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, 10000, log); plasmaWordIndex wordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, 10000, log);
Iterator indexContainerIterator = wordIndex.wordContainers("AAAAAAAAAAAA", false, false); Iterator<indexContainer> indexContainerIterator = wordIndex.wordContainers("AAAAAAAAAAAA", false, false);
long urlCounter = 0, wordCounter = 0; long urlCounter = 0, wordCounter = 0;
long wordChunkStart = System.currentTimeMillis(), wordChunkEnd = 0; long wordChunkStart = System.currentTimeMillis(), wordChunkEnd = 0;
@ -617,10 +618,10 @@ public final class yacy {
indexContainer wordIdxContainer = null; indexContainer wordIdxContainer = null;
try { try {
wordCounter++; wordCounter++;
wordIdxContainer = (indexContainer) indexContainerIterator.next(); wordIdxContainer = indexContainerIterator.next();
// the combined container will fit, read the container // the combined container will fit, read the container
Iterator wordIdxEntries = wordIdxContainer.entries(); Iterator<indexRWIRowEntry> wordIdxEntries = wordIdxContainer.entries();
indexRWIEntry iEntry; indexRWIEntry iEntry;
while (wordIdxEntries.hasNext()) { while (wordIdxEntries.hasNext()) {
iEntry = (indexRWIEntry) wordIdxEntries.next(); iEntry = (indexRWIEntry) wordIdxEntries.next();
@ -684,13 +685,13 @@ public final class yacy {
* @param wordlist File where the words are stored. * @param wordlist File where the words are stored.
* @return HashMap with the hash-word - relation. * @return HashMap with the hash-word - relation.
*/ */
private static HashMap loadWordMap(File wordlist) { private static HashMap<String, String> loadWordMap(File wordlist) {
// returns a hash-word - Relation // returns a hash-word - Relation
HashMap wordmap = new HashMap(); HashMap<String, String> wordmap = new HashMap<String, String>();
try { try {
String word; String word;
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(wordlist))); BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(wordlist)));
while ((word = br.readLine()) != null) wordmap.put(plasmaCondenser.word2hash(word),word); while ((word = br.readLine()) != null) wordmap.put(plasmaCondenser.word2hash(word), word);
br.close(); br.close();
} catch (IOException e) {} } catch (IOException e) {}
return wordmap; return wordmap;
@ -712,7 +713,7 @@ public final class yacy {
serverLog.logConfig("CLEAN-WORDLIST", "START"); serverLog.logConfig("CLEAN-WORDLIST", "START");
String word; String word;
TreeSet wordset = new TreeSet(); TreeSet<String> wordset = new TreeSet<String>();
int count = 0; int count = 0;
try { try {
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(wordlist))); BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(wordlist)));
@ -795,7 +796,7 @@ public final class yacy {
log.logInfo("STARTING CREATION OF RWI-HASHLIST"); log.logInfo("STARTING CREATION OF RWI-HASHLIST");
File root = new File(homePath); File root = new File(homePath);
try { try {
Iterator indexContainerIterator = null; Iterator<indexContainer> indexContainerIterator = null;
if (resource.equals("all")) { if (resource.equals("all")) {
WordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, 3000, log); WordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, 3000, log);
indexContainerIterator = WordIndex.wordContainers(wordChunkStartHash, false, false); indexContainerIterator = WordIndex.wordContainers(wordChunkStartHash, false, false);
@ -810,7 +811,7 @@ public final class yacy {
bos.putNextEntry(zipEntry); bos.putNextEntry(zipEntry);
while (indexContainerIterator.hasNext()) { while (indexContainerIterator.hasNext()) {
counter++; counter++;
container = (indexContainer) indexContainerIterator.next(); container = indexContainerIterator.next();
bos.write((container.getWordHash()).getBytes()); bos.write((container.getWordHash()).getBytes());
bos.write(serverCore.CRLF); bos.write(serverCore.CRLF);
if (counter % 500 == 0) { if (counter % 500 == 0) {
@ -825,7 +826,7 @@ public final class yacy {
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file)); BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file));
while (indexContainerIterator.hasNext()) { while (indexContainerIterator.hasNext()) {
counter++; counter++;
container = (indexContainer) indexContainerIterator.next(); container = indexContainerIterator.next();
bos.write((container.getWordHash()).getBytes()); bos.write((container.getWordHash()).getBytes());
bos.write(serverCore.CRLF); bos.write(serverCore.CRLF);
if (counter % 500 == 0) { if (counter % 500 == 0) {
@ -862,7 +863,7 @@ public final class yacy {
kelondroMapObjects.mapIterator it; kelondroMapObjects.mapIterator it;
it = db.maps(true, false); it = db.maps(true, false);
while (it.hasNext()) { while (it.hasNext()) {
Map dna = (Map) it.next(); Map<String, String> dna = it.next();
String peerHash = (String) dna.get("key"); String peerHash = (String) dna.get("key");
if (peerHash.length() < yacySeedDB.commonHashLength) { if (peerHash.length() < yacySeedDB.commonHashLength) {
String peerName = (String) dna.get("Name"); String peerName = (String) dna.get("Name");

Write Preview
Loading…
Cancel
Save