boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 6f8a2fef1f 3e1bc9477f
commit fa735f4f04
14 changed files with 117 additions and 114 deletions
Show Stats Download Patch File Download Diff File

4
htroot/Banner.java
Unescape View File

@ -110,12 +110,12 @@ public class Banner {
}
}
final BannerData data =
final BannerData data =
new BannerData(
width, height, bgcolor, textcolor, bordercolor, name, links,
words, type, myppm, network, peers, nlinks, nwords,
nqph, nppm);
if (!net.yacy.peers.graphics.Banner.logoIsLoaded()) {
// do not write a cache to disc; keep in RAM
ImageIO.setUseCache(false);

2
htroot/IndexFederated_p.html
Unescape View File

@ -65,7 +65,7 @@
</dl>
</div>
<div>
<h3>Index Scheme</h3><p>If you use a custom Solr schema you may enter a different field name in the column 'cutom Solr Field Name' of the YaCy default attribute name</p>
<h3>Index Scheme</h3><p>If you use a custom Solr schema you may enter a different field name in the column 'Custom Solr Field Name' of the YaCy default attribute name</p>
<table class="sortable" border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
<td>Active</td>

2
htroot/Table_API_p.java
Unescape View File

@ -36,7 +36,6 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.search.query.QueryParams;
import de.anomic.data.WorkTables;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -73,7 +72,6 @@ public class Table_API_p {
typefilter = Pattern.compile(post.get("filter", ".*"));
}
String pk;
boolean scheduleevent = false; // flag if schedule info of row changes
String current_schedule_pk = ""; // pk of changed schedule data row
if (post != null && post.containsKey("scheduleevent")) {

2
htroot/yacysearchitem.java
Unescape View File

@ -61,7 +61,7 @@ public class yacysearchitem {
private static final int MAX_NAME_LENGTH = 60;
private static final int MAX_URL_LENGTH = 120;
private static boolean col = true;
//private static boolean col = true;
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;

66
source/de/anomic/data/ymark/YMarkAutoTagger.java
Unescape View File

@ -4,13 +4,13 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.ArrayBlockingQueue;
import net.yacy.cora.document.UTF8;
import net.yacy.cora.services.federated.yacy.CacheStrategy;
import net.yacy.document.Condenser;
@ -28,11 +28,11 @@ import de.anomic.crawler.retrieval.Response;
public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandler {
private static final String EMPTY_STRING = new String();
public final static String SPACE = " ";
public final static String POISON = "";
public final static HashSet<String> stopwords = new HashSet<String>(Arrays.asList(".", "!", "?", "nbsp", "uuml", "ouml", "auml", "amp", "quot", "laquo", "raquo",
public final static HashSet<String> stopwords = new HashSet<String>(Arrays.asList(".", "!", "?", "nbsp", "uuml", "ouml", "auml", "amp", "quot", "laquo", "raquo",
"and", "with", "the", "gt", "lt"));
@ -81,10 +81,10 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle
return null;
}
}
public static String autoTag(final Document document, final int max, final TreeMap<String, YMarkTag> tags) {
final TreeSet<YMarkTag> topwords = new TreeSet<YMarkTag>();
StringBuilder token;
StringBuilder token;
if(document == null) {
return EMPTY_STRING;
@ -92,7 +92,7 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle
//get words from document
final Map<String, Word> words = new Condenser(document, true, true, LibraryProvider.dymLib).words();
// generate potential tags from document title, description and subject
final int bufferSize = document.dc_title().length() + document.dc_description().length() + document.dc_subject(' ').length() + 32;
final StringBuilder buffer = new StringBuilder(bufferSize);
@ -103,21 +103,21 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle
final WordTokenizer tokens = new WordTokenizer(new ByteArrayInputStream(UTF8.getBytes(buffer.toString())), LibraryProvider.dymLib);
try {
int score = 0;
// get phrases
final TreeMap<String, YMarkTag> phrases = getPhrases(document, 2);
phrases.putAll(getPhrases(document, 3));
final Iterator<String> iter = phrases.keySet().iterator();
while(iter.hasNext()) {
score = 10;
final String phrase = iter.next();
final String phrase = iter.next();
if(phrases.get(phrase).size() > 3 && phrases.get(phrase).size() < 10) {
score = phrases.get(phrase).size() * phrase.split(" ").length * 20;
}
if(isDigitSpace(phrase)) {
score = 10;
}
if(phrases.get(phrase).size() > 2 && buffer.indexOf(phrase) > 1) {
if(phrases.get(phrase).size() > 2 && buffer.indexOf(phrase) > 1) {
score = score * 10;
}
if (tags.containsKey(phrase)) {
@ -127,14 +127,14 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle
pwords.append(phrase);
pwords.append(' ');
}
// loop through potential tag and rank them
while(tokens.hasMoreElements()) {
while(tokens.hasMoreElements()) {
score = 0;
token = tokens.nextElement();
// check if the token appears in the text
if (words.containsKey(token.toString())) {
if (words.containsKey(token.toString())) {
final Word word = words.get(token.toString());
// token appears in text and matches an existing bookmark tag
if (tags.containsKey(token.toString())) {
@ -172,8 +172,8 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle
} finally {
tokens.close();
}
}
}
private static TreeMap<String, YMarkTag> getPhrases(final Document document, final int size) {
final TreeMap<String, YMarkTag> phrases = new TreeMap<String, YMarkTag>();
final StringBuilder phrase = new StringBuilder(128);
@ -181,33 +181,33 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle
try {
StringBuilder token;
int count = 0;
// loop through text
while(tokens.hasMoreElements()) {
token = tokens.nextElement();
while(tokens.hasMoreElements()) {
token = tokens.nextElement();
if(stopwords.contains(token.toString()) || isDigitSpace(token.toString()))
continue;
continue;
// if we have a full phrase, delete the first token
count++;
if(count > size)
phrase.delete(0, phrase.indexOf(SPACE)+1);
// append new token
if(phrase.length() > 1)
phrase.append(SPACE);
phrase.append(SPACE);
phrase.append(token);
if(count >= size) { // make sure we really have a phrase
if(phrases.containsKey(phrase.toString())) {
phrases.get(phrase.toString()).inc();
} else {
phrases.put(phrase.toString(), new YMarkTag(phrase.toString()));
}
}
}
}
return phrases;
} finally {
tokens.close();
@ -221,7 +221,7 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle
else
return "/IOExceptions";
}
public static boolean isDigitSpace(String str) {
if (str == null) {
return false;
@ -235,7 +235,8 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle
return true;
}
public void run() {
@Override
public void run() {
Log.logInfo(YMarkTables.BOOKMARKS_LOG, "autoTagger run()");
Thread.currentThread().setUncaughtExceptionHandler(this);
String url = null;
@ -247,9 +248,9 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle
while((url = this.bmkQueue.take()) != POISON) {
tagString = autoTag(url, this.loader, 5, tags);
if (tagString.equals("/IOExceptions")) {
this.ymarks.addFolder(bmk_user, url, tagString);
this.ymarks.addFolder(this.bmk_user, url, tagString);
tagString = "";
}
}
// update tags
this.ymarks.addTags(this.bmk_user, url, tagString, this.merge);
@ -275,7 +276,8 @@ public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandle
}
}
public void uncaughtException(final Thread t, final Throwable e) {
@Override
public void uncaughtException(final Thread t, final Throwable e) {
Log.logWarning(YMarkTables.BOOKMARKS_LOG, "I caught an uncaughtException in thread "+t.getName());
Log.logException(e);
}

4
source/de/anomic/http/server/TemplateEngine.java
Unescape View File

@ -129,7 +129,7 @@ public final class TemplateEngine {
private final static byte lbr = (byte)'[';
private final static byte rbr = (byte)']';
private final static byte[] pOpen = {hashChar, lbr};
//private final static byte[] pOpen = {hashChar, lbr};
private final static byte[] pClose = {rbr, hashChar};
private final static byte lcbr = (byte)'{';
@ -142,7 +142,7 @@ public final class TemplateEngine {
private final static byte[] aOpen = {hashChar, lrbr};
private final static byte[] aClose = {rrbr, hashChar};
private final static byte[] iOpen = {hashChar, pcChar};
//private final static byte[] iOpen = {hashChar, pcChar};
private final static byte[] iClose = {pcChar, hashChar};
private final static byte[] ul = "_".getBytes();

96
source/de/anomic/tools/loaderThreads.java
Unescape View File

@ -1,4 +1,4 @@
// loaderThreads.java
// loaderThreads.java
// ---------------------------
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
@ -24,7 +24,6 @@
package de.anomic.tools;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@ -33,7 +32,7 @@ import net.yacy.cora.protocol.http.ProxySettings;
import net.yacy.kelondro.data.meta.DigestURI;
public class loaderThreads {
// global values for loader threads
protected int timeout;
protected String user;
@ -41,16 +40,16 @@ public class loaderThreads {
protected ProxySettings remoteProxyConfig;
// management objects for collection of threads
private Map<String, Thread> threads;
private final Map<String, Thread> threads;
private int completed, failed;
public loaderThreads() {
this(10000, null, null);
}
public loaderThreads(
final int timeout,
final String user,
final int timeout,
final String user,
final String password
) {
this.timeout = timeout;
@ -60,45 +59,45 @@ public class loaderThreads {
this.completed = 0;
this.failed = 0;
}
public void newThread(final String name, final DigestURI url, final loaderProcess process) {
final Thread t = new loaderThread(url, process);
threads.put(name, t);
this.threads.put(name, t);
t.start();
}
public void terminateThread(final String name) {
final loaderThread t = (loaderThread) threads.get(name);
final loaderThread t = (loaderThread) this.threads.get(name);
if (t == null) throw new RuntimeException("no such thread: " + name);
t.terminate();
}
public int threadCompleted(final String name) {
final loaderThread t = (loaderThread) threads.get(name);
final loaderThread t = (loaderThread) this.threads.get(name);
if (t == null) throw new RuntimeException("no such thread: " + name);
return t.completed();
}
public int threadStatus(final String name) {
final loaderThread t = (loaderThread) threads.get(name);
final loaderThread t = (loaderThread) this.threads.get(name);
if (t == null) throw new RuntimeException("no such thread: " + name);
return t.status();
}
public int completed() {
return completed;
return this.completed;
}
public int failed() {
return failed;
return this.failed;
}
public int count() {
return threads.size();
return this.threads.size();
}
public Exception threadError(final String name) {
final loaderThread t = (loaderThread) threads.get(name);
final loaderThread t = (loaderThread) this.threads.get(name);
if (t == null) throw new RuntimeException("no such thread: " + name);
return t.error();
}
@ -109,7 +108,7 @@ public class loaderThreads {
private final loaderProcess process;
private byte[] page;
private boolean loaded;
public loaderThread(final DigestURI url, final loaderProcess process) {
this.url = url;
this.process = process;
@ -118,47 +117,48 @@ public class loaderThreads {
this.loaded = false;
}
@Override
public void run() {
try {
page = url.get(ClientIdentification.getUserAgent(), timeout);
loaded = true;
process.feed(page);
if (process.status() == loaderCore.STATUS_FAILED) {
error = process.error();
this.page = this.url.get(ClientIdentification.getUserAgent(), loaderThreads.this.timeout);
this.loaded = true;
this.process.feed(this.page);
if (this.process.status() == loaderCore.STATUS_FAILED) {
this.error = this.process.error();
}
if ((process.status() == loaderCore.STATUS_COMPLETED) ||
(process.status() == loaderCore.STATUS_FINALIZED)) completed++;
if ((process.status() == loaderCore.STATUS_ABORTED) ||
(process.status() == loaderCore.STATUS_FAILED)) failed++;
if ((this.process.status() == loaderCore.STATUS_COMPLETED) ||
(this.process.status() == loaderCore.STATUS_FINALIZED)) loaderThreads.this.completed++;
if ((this.process.status() == loaderCore.STATUS_ABORTED) ||
(this.process.status() == loaderCore.STATUS_FAILED)) loaderThreads.this.failed++;
} catch (final Exception e) {
error = e;
failed++;
this.error = e;
loaderThreads.this.failed++;
}
}
public void terminate() {
process.terminate();
this.process.terminate();
}
public boolean loaded() {
return loaded;
return this.loaded;
}
public int completed() {
if (process.status() == loaderCore.STATUS_READY) return 1;
if (process.status() == loaderCore.STATUS_RUNNING) return 9 + ((process.completed() * 9) / 10);
if (process.status() == loaderCore.STATUS_COMPLETED) return 100;
if (this.process.status() == loaderCore.STATUS_READY) return 1;
if (this.process.status() == loaderCore.STATUS_RUNNING) return 9 + ((this.process.completed() * 9) / 10);
if (this.process.status() == loaderCore.STATUS_COMPLETED) return 100;
return 0;
}
public int status() {
return process.status(); // see constants in loaderCore
return this.process.status(); // see constants in loaderCore
}
public Exception error() {
return error;
return this.error;
}
}
}

6
source/net/yacy/cora/storage/ConfigurationSet.java
Unescape View File

@ -32,6 +32,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.Serializable;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
@ -200,7 +201,8 @@ public class ConfigurationSet extends TreeMap<String,Entry> implements Serializa
File bakfile = new File (this.file.getAbsolutePath() + ".bak");
FileUtils.copy (this.file, bakfile);
TreeMap tclone = (TreeMap) this.clone(); // clone to write appended entries
@SuppressWarnings("unchecked")
TreeMap<String,Entry> tclone = (TreeMap<String,Entry>) this.clone(); // clone to write appended entries
final BufferedWriter writer = new BufferedWriter(new FileWriter(this.file));
try {
@ -248,7 +250,7 @@ public class ConfigurationSet extends TreeMap<String,Entry> implements Serializa
} catch (final IOException e) {}
// write remainig entries (not already written)
Iterator ie = tclone.entrySet().iterator();
Iterator<Map.Entry<String,Entry>> ie = tclone.entrySet().iterator();
while (ie.hasNext()) {
Object e = ie.next();
writer.write (e.toString() + "\n");

2
source/net/yacy/document/parser/dwgParser.java
Unescape View File

@ -50,7 +50,7 @@ public class dwgParser extends AbstractParser implements Parser {
* How far to skip after the last standard property, before
* we find any custom properties that might be there.
*/
private static final int CUSTOM_PROPERTIES_SKIP = 20;
//private static final int CUSTOM_PROPERTIES_SKIP = 20;
public dwgParser() {
super("DWG (CAD Drawing) parser (very basic)");

15
source/net/yacy/kelondro/logging/ThreadDump.java
Unescape View File

@ -36,7 +36,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import net.yacy.document.parser.html.CharacterCoding;
@ -210,7 +209,7 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
Thread thread;
// collect single dumps
for (final Entry<Thread, StackTraceElement[]> entry: stackTraces.entrySet()) {
for (final Map.Entry<Thread, StackTraceElement[]> entry: stackTraces.entrySet()) {
thread = entry.getKey();
final StackTraceElement[] stackTraceElements = entry.getValue();
StackTraceElement ste;
@ -274,7 +273,7 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
bufferappend(buffer, plain, "");
// write dumps
for (final Entry<StackTrace, List<String>> entry: entrySet()) {
for (final Map.Entry<StackTrace, List<String>> entry: entrySet()) {
final List<String> threads = entry.getValue();
for (final String t: threads) bufferappend(buffer, plain, t);
bufferappend(buffer, plain, entry.getKey().text);
@ -317,7 +316,7 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
ThreadDump x;
for (final Map<Thread, StackTraceElement[]> trace: stackTraces) {
x = new ThreadDump(rootPath, trace, plain, Thread.State.RUNNABLE);
for (final Entry<StackTrace, List<String>> e: x.entrySet()) {
for (final Map.Entry<StackTrace, List<String>> e: x.entrySet()) {
if (multiDumpFilterPattern.matcher(e.getKey().text).matches()) continue;
Integer c = dumps.get(e.getKey().text);
if (c == null) dumps.put(e.getKey().text, Integer.valueOf(1));
@ -330,16 +329,16 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp
// write dumps
while (!dumps.isEmpty()) {
final Entry<String, Integer> e = removeMax(dumps);
final Map.Entry<String, Integer> e = removeMax(dumps);
bufferappend(buffer, plain, "Occurrences: " + e.getValue());
bufferappend(buffer, plain, e.getKey());
}
bufferappend(buffer, plain, "");
}
private static Entry<String, Integer> removeMax(final Map<String, Integer> result) {
Entry<String, Integer> max = null;
for (final Entry<String, Integer> e: result.entrySet()) {
private static Map.Entry<String, Integer> removeMax(final Map<String, Integer> result) {
Map.Entry<String, Integer> max = null;
for (final Map.Entry<String, Integer> e: result.entrySet()) {
if (max == null || e.getValue().intValue() > max.getValue().intValue()) {
max = e;
}

12
source/net/yacy/peers/graphics/NetworkGraph.java
Unescape View File

@ -324,12 +324,12 @@ public class NetworkGraph {
private static class drawNetworkPicturePeerJob {
private RasterPlotter img;
private int centerX, centerY, innerradius, outerradius, coronaangle;
private Seed seed;
private String colorDot, colorLine, colorText;
private double cyc;
public drawNetworkPicturePeerJob() {} // used to produce a poison pill
private final RasterPlotter img;
private final int centerX, centerY, innerradius, outerradius, coronaangle;
private final Seed seed;
private final String colorDot, colorLine, colorText;
private final double cyc;
//public drawNetworkPicturePeerJob() {} // used to produce a poison pill
public drawNetworkPicturePeerJob(
final RasterPlotter img, final int centerX, final int centerY,
final int innerradius, final int outerradius,

2
source/net/yacy/search/query/RWIProcess.java
Unescape View File

@ -95,7 +95,6 @@ public final class RWIProcess extends Thread
private final ScoreMap<String> ref; // reference score computation for the commonSense heuristic
private final Map<String, byte[]> hostResolver; // a mapping from a host hash (6 bytes) to the full url hash of one of these urls that have the host hash
private final ReferenceOrder order;
private final long startTime;
private boolean addRunning;
private final boolean remote;
@ -141,7 +140,6 @@ public final class RWIProcess extends Thread
this.ref = new ConcurrentScoreMap<String>();
this.feedersAlive = new AtomicInteger(0);
this.feedersTerminated = new AtomicInteger(0);
this.startTime = System.currentTimeMillis();
this.maxExpectedRemoteReferences = new AtomicInteger(0);
this.expectedRemoteReferences = new AtomicInteger(0);
this.receivedRemoteReferences = new AtomicInteger(0);

17
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -505,16 +505,21 @@ public final class SearchEvent
this.urlhash = urlhash;
this.heuristicName = heuristicName;
this.redundant = redundant;
}/*
public int compareTo(HeuristicResult o) {
}
public int compareTo(HeuristicResult o) {
return Base64Order.enhancedCoder.compare(this.urlhash, o.urlhash);
}
public int hashCode() {
@Override
public int hashCode() {
return (int) Base64Order.enhancedCoder.cardinal(this.urlhash);
}
public boolean equals(Object o) {
}
@Override
public boolean equals(Object o) {
return Base64Order.enhancedCoder.equal(this.urlhash, ((HeuristicResult) o).urlhash);
}*/
}
}
public class SecondarySearchSuperviser extends Thread

1
source/net/yacy/yacy.java
Unescape View File

@ -57,7 +57,6 @@ import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.sorting.Array;
import net.yacy.cora.sorting.OrderedScoreMap;
import net.yacy.cora.sorting.ScoreMap;
import net.yacy.gui.Tray;
import net.yacy.gui.YaCyApp;
import net.yacy.gui.framework.Browser;
import net.yacy.kelondro.blob.MapDataMining;

Write Preview
Loading…
Cancel
Save