boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- removed some warinings

Browse Source 
- removed a dead update location

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7970 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent fabda9ad31
commit 6b22865dbc
7 changed files with 129 additions and 132 deletions
Show Stats Download Patch File Download Diff File

1
defaults/yacy.network.freeworld.unit
Unescape View File

@ -82,7 +82,6 @@ network.unit.update.location0 = http://yacy.net/en/index.html
network.unit.update.location1 = http://latest.yacy.de network.unit.update.location1 = http://latest.yacy.de
network.unit.update.location2 = http://www.yacystats.de/yacybuild/ network.unit.update.location2 = http://www.yacystats.de/yacybuild/
network.unit.update.location2.key = MIIBtTCCASwGByqGSM44BAEwggEfAoGBAP1/U4EddRIpUt9KnC7s5Of2EbdSPO9EAMMeP4C2USZpRV1AIlH7WT2NWPq/xfW6MPbLm1Vs14E7gB00b/JmYLdrmVClpJ+f6AR7ECLCT7up1/63xhv4O1fnxqimFQ8E+4P208UewwI1VBNaFpEy9nXzrith1yrv8iIDGZ3RSAHHAhUAl2BQjxUjC8yykrmCouuEC/BYHPUCgYEA9+GghdabPd7LvKtcNrhXuXmUr7v6OuqC+VdMCz0HgmdRWVeOutRZT+ZxBxCBgLRJFnEj6EwoFhO3zwkyjMim4TwWeotUfI0o4KOuHiuzpnWRbqN/C/ohNWLx+2J6ASQ7zKTxvqhRkImog9/hWuWfBpKLZl6Ae1UlZAFMO/7PSSoDgYIAAn8bzmhk8EWOj9h+7zng1o3OlgI+LsG7lI7kVsDxyzMB/WYTpO+NKWoibFjQDFN92TKBQVAA8DQciHfolqcFfVAot9/3ipamXVGz29OAxz8i0Wty6KI6w50YrL2xAkWjx7jSBghJKlnKx3V0PaDCWqz37ogQvuxLKBFORyAjbv3O network.unit.update.location2.key = MIIBtTCCASwGByqGSM44BAEwggEfAoGBAP1/U4EddRIpUt9KnC7s5Of2EbdSPO9EAMMeP4C2USZpRV1AIlH7WT2NWPq/xfW6MPbLm1Vs14E7gB00b/JmYLdrmVClpJ+f6AR7ECLCT7up1/63xhv4O1fnxqimFQ8E+4P208UewwI1VBNaFpEy9nXzrith1yrv8iIDGZ3RSAHHAhUAl2BQjxUjC8yykrmCouuEC/BYHPUCgYEA9+GghdabPd7LvKtcNrhXuXmUr7v6OuqC+VdMCz0HgmdRWVeOutRZT+ZxBxCBgLRJFnEj6EwoFhO3zwkyjMim4TwWeotUfI0o4KOuHiuzpnWRbqN/C/ohNWLx+2J6ASQ7zKTxvqhRkImog9/hWuWfBpKLZl6Ae1UlZAFMO/7PSSoDgYIAAn8bzmhk8EWOj9h+7zng1o3OlgI+LsG7lI7kVsDxyzMB/WYTpO+NKWoibFjQDFN92TKBQVAA8DQciHfolqcFfVAot9/3ipamXVGz29OAxz8i0Wty6KI6w50YrL2xAkWjx7jSBghJKlnKx3V0PaDCWqz37ogQvuxLKBFORyAjbv3O
network.unit.update.location3 = https://latestyacy.f1ori.de/
# properties for in-protocol response authentication: # properties for in-protocol response authentication:
network.unit.protocol.control = uncontrolled network.unit.protocol.control = uncontrolled

2
source/de/anomic/http/server/ServerSideIncludes.java
Unescape View File

@ -94,7 +94,7 @@ public class ServerSideIncludes {
header.put(RequestHeader.AUTHORIZATION, authorization); header.put(RequestHeader.AUTHORIZATION, authorization);
if (requestHeader.containsKey(RequestHeader.COOKIE)) if (requestHeader.containsKey(RequestHeader.COOKIE))
header.put(RequestHeader.COOKIE, requestHeader.get(RequestHeader.COOKIE)); header.put(RequestHeader.COOKIE, requestHeader.get(RequestHeader.COOKIE));
header.put(RequestHeader.REFERER, requestHeader.get(RequestHeader.CONNECTION_PROP_PATH)); header.put(RequestHeader.REFERER, requestHeader.get(HeaderFramework.CONNECTION_PROP_PATH));
HTTPDFileHandler.doGet(conProp, header, out); HTTPDFileHandler.doGet(conProp, header, out);
} }
} }

30
source/de/anomic/search/AccessTracker.java
Unescape View File

@ -36,7 +36,7 @@ import java.util.LinkedList;
import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.UTF8; import net.yacy.cora.document.UTF8;
import net.yacy.document.LibraryProvider; import net.yacy.document.WordCache;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
@ -52,14 +52,14 @@ public class AccessTracker {
private static LinkedList<QueryParams> remoteSearches = new LinkedList<QueryParams>(); private static LinkedList<QueryParams> remoteSearches = new LinkedList<QueryParams>();
private static ArrayList<String> log = new ArrayList<String>(); private static ArrayList<String> log = new ArrayList<String>();
public static void add(Location location, QueryParams query) { public static void add(final Location location, final QueryParams query) {
if (location == Location.local) synchronized (localSearches) {add(localSearches, query);} if (location == Location.local) synchronized (localSearches) {add(localSearches, query);}
if (location == Location.remote) synchronized (remoteSearches) {add(remoteSearches, query);} if (location == Location.remote) synchronized (remoteSearches) {add(remoteSearches, query);}
} }
private static void add(LinkedList<QueryParams> list, QueryParams query) { private static void add(final LinkedList<QueryParams> list, final QueryParams query) {
// learn that this word can be a word completion for the DidYouMeanLibrary // learn that this word can be a word completion for the DidYouMeanLibrary
if (query.resultcount > 10 && query.queryString != null && query.queryString.length() > 0) LibraryProvider.dymLib.learn(query.queryString); if (query.resultcount > 10 && query.queryString != null && query.queryString.length() > 0) WordCache.learn(query.queryString);
// add query to statistics list // add query to statistics list
list.add(query); list.add(query);
@ -75,30 +75,30 @@ public class AccessTracker {
if (list.size() <= minSize) return; if (list.size() <= minSize) return;
// if the list is large we look for too old entries // if the list is large we look for too old entries
long timeout = System.currentTimeMillis() - maxAge; final long timeout = System.currentTimeMillis() - maxAge;
while (list.size() > 0) { while (list.size() > 0) {
QueryParams q = list.getFirst(); final QueryParams q = list.getFirst();
if (q.time.longValue() > timeout) break; if (q.time.longValue() > timeout) break;
addToDump(list.removeFirst()); addToDump(list.removeFirst());
} }
} }
public static Iterator<QueryParams> get(Location location) { public static Iterator<QueryParams> get(final Location location) {
if (location == Location.local) return localSearches.descendingIterator(); if (location == Location.local) return localSearches.descendingIterator();
if (location == Location.remote) return remoteSearches.descendingIterator(); if (location == Location.remote) return remoteSearches.descendingIterator();
return null; return null;
} }
public static int size(Location location) { public static int size(final Location location) {
if (location == Location.local) synchronized (localSearches) {return localSearches.size();} if (location == Location.local) synchronized (localSearches) {return localSearches.size();}
if (location == Location.remote) synchronized (remoteSearches) {return remoteSearches.size();} if (location == Location.remote) synchronized (remoteSearches) {return remoteSearches.size();}
return 0; return 0;
} }
private static void addToDump(QueryParams query) { private static void addToDump(final QueryParams query) {
//if (query.resultcount == 0) return; //if (query.resultcount == 0) return;
if (query.queryString == null || query.queryString.length() == 0) return; if (query.queryString == null || query.queryString.length() == 0) return;
StringBuilder sb = new StringBuilder(40); final StringBuilder sb = new StringBuilder(40);
sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(query.time))); sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(query.time)));
sb.append(' '); sb.append(' ');
sb.append(Integer.toString(query.resultcount)); sb.append(Integer.toString(query.resultcount));
@ -107,21 +107,21 @@ public class AccessTracker {
log.add(sb.toString()); log.add(sb.toString());
} }
public static void dumpLog(File file) { public static void dumpLog(final File file) {
while (localSearches.size() > 0) { while (localSearches.size() > 0) {
addToDump(localSearches.removeFirst()); addToDump(localSearches.removeFirst());
} }
try { try {
RandomAccessFile raf = new RandomAccessFile(file, "rw"); final RandomAccessFile raf = new RandomAccessFile(file, "rw");
raf.seek(raf.length()); raf.seek(raf.length());
for (String s: log) { for (final String s: log) {
raf.write(UTF8.getBytes(s)); raf.write(UTF8.getBytes(s));
raf.writeByte(10); raf.writeByte(10);
} }
log.clear(); log.clear();
} catch (FileNotFoundException e) { } catch (final FileNotFoundException e) {
Log.logException(e); Log.logException(e);
} catch (IOException e) { } catch (final IOException e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }

2
source/de/anomic/search/DocumentIndex.java
Unescape View File

@ -225,7 +225,7 @@ public class DocumentIndex extends Segment {
@Override @Override
public void close() { public void close() {
// send termination signal to worker threads // send termination signal to worker threads
for (final Worker element : this.worker) { for (@SuppressWarnings("unused") final Worker element : this.worker) {
try { try {
this.queue.put(poison); this.queue.put(poison);
} catch (final InterruptedException e) {} } catch (final InterruptedException e) {}

5
source/de/anomic/search/ResultFetcher.java
Unescape View File

@ -358,7 +358,6 @@ public class ResultFetcher {
// start fetching urls and snippets // start fetching urls and snippets
URIMetadataRow page; URIMetadataRow page;
ResultEntry resultEntry; ResultEntry resultEntry;
String rawLine;
//final int fetchAhead = snippetMode == 0 ? 0 : 10; //final int fetchAhead = snippetMode == 0 ? 0 : 10;
final boolean nav_topics = ResultFetcher.this.query.navigators.equals("all") || ResultFetcher.this.query.navigators.indexOf("topics") >= 0; final boolean nav_topics = ResultFetcher.this.query.navigators.equals("all") || ResultFetcher.this.query.navigators.indexOf("topics") >= 0;
try { try {
@ -405,9 +404,9 @@ public class ResultFetcher {
loops++; loops++;
resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0 resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0
if (resultEntry == null) continue; // the entry had some problems, cannot be used if (resultEntry == null) continue; // the entry had some problems, cannot be used
rawLine = resultEntry.textSnippet() == null ? null : resultEntry.textSnippet().getLineRaw(); //final String rawLine = resultEntry.textSnippet() == null ? null : resultEntry.textSnippet().getLineRaw();
//System.out.println("***SNIPPET*** raw='" + rawLine + "', pattern='" + this.snippetPattern.toString() + "'"); //System.out.println("***SNIPPET*** raw='" + rawLine + "', pattern='" + this.snippetPattern.toString() + "'");
if (rawLine != null && !this.snippetPattern.matcher(rawLine).matches()) continue; //if (rawLine != null && !this.snippetPattern.matcher(rawLine).matches()) continue;
//if (result.contains(resultEntry)) continue; //if (result.contains(resultEntry)) continue;
ResultFetcher.this.urlRetrievalAllTime += resultEntry.dbRetrievalTime; ResultFetcher.this.urlRetrievalAllTime += resultEntry.dbRetrievalTime;

93
source/net/yacy/cora/document/RSSReader.java
Unescape View File

@ -33,7 +33,6 @@ import org.xml.sax.Attributes;
import org.xml.sax.EntityResolver; import org.xml.sax.EntityResolver;
import org.xml.sax.InputSource; import org.xml.sax.InputSource;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler; import org.xml.sax.helpers.DefaultHandler;
@ -48,17 +47,17 @@ public class RSSReader extends DefaultHandler {
public enum Type { rss, atom, rdf, none }; public enum Type { rss, atom, rdf, none };
private RSSReader(int maxsize) { private RSSReader(final int maxsize) {
theChannel = new RSSFeed(maxsize); this.theChannel = new RSSFeed(maxsize);
buffer = new StringBuilder(300); this.buffer = new StringBuilder(300);
item = null; this.item = null;
parsingChannel = false; this.parsingChannel = false;
parsingImage = false; this.parsingImage = false;
parsingItem = false; this.parsingItem = false;
type = Type.none; this.type = Type.none;
} }
public RSSReader(int maxsize, final InputStream stream, Type type) throws IOException { public RSSReader(final int maxsize, final InputStream stream, final Type type) throws IOException {
this(maxsize); this(maxsize);
this.type = type; this.type = type;
final SAXParserFactory factory = SAXParserFactory.newInstance(); final SAXParserFactory factory = SAXParserFactory.newInstance();
@ -73,9 +72,9 @@ public class RSSReader extends DefaultHandler {
} }
}); });
saxParser.parse(stream, this); saxParser.parse(stream, this);
} catch (SAXException e) { } catch (final SAXException e) {
throw new IOException (e.getMessage()); throw new IOException (e.getMessage());
} catch (ParserConfigurationException e) { } catch (final ParserConfigurationException e) {
throw new IOException (e.getMessage()); throw new IOException (e.getMessage());
} }
} }
@ -84,7 +83,7 @@ public class RSSReader extends DefaultHandler {
return this.type; return this.type;
} }
public static RSSReader parse(int maxsize, final byte[] a) throws IOException { public static RSSReader parse(final int maxsize, final byte[] a) throws IOException {
// check integrity of array // check integrity of array
if ((a == null) || (a.length == 0)) { if ((a == null) || (a.length == 0)) {
@ -156,25 +155,25 @@ public class RSSReader extends DefaultHandler {
public void startElement(final String uri, final String name, final String tag, final Attributes atts) throws SAXException { public void startElement(final String uri, final String name, final String tag, final Attributes atts) throws SAXException {
if ("channel".equals(tag)) { if ("channel".equals(tag)) {
this.type = Type.rss; this.type = Type.rss;
item = new RSSMessage(); this.item = new RSSMessage();
parsingChannel = true; this.parsingChannel = true;
} else if ("feed".equals(tag)) { } else if ("feed".equals(tag)) {
this.type = Type.atom; this.type = Type.atom;
item = new RSSMessage(); this.item = new RSSMessage();
parsingChannel = true; this.parsingChannel = true;
} else if ("item".equals(tag) || "entry".equals(tag)) { } else if ("item".equals(tag) || "entry".equals(tag)) {
if (parsingChannel) { if (this.parsingChannel) {
// the channel ends with the first item not with the channel close tag // the channel ends with the first item not with the channel close tag
theChannel.setChannel(item); this.theChannel.setChannel(this.item);
parsingChannel = false; this.parsingChannel = false;
} }
item = new RSSMessage(); this.item = new RSSMessage();
parsingItem = true; this.parsingItem = true;
} else if (parsingItem && this.type == Type.atom && "link".equals(tag)) { } else if (this.parsingItem && this.type == Type.atom && "link".equals(tag)) {
String url = atts.getValue("href"); final String url = atts.getValue("href");
if (url != null && url.length() > 0) item.setValue("link", url); if (url != null && url.length() > 0) this.item.setValue("link", url);
} else if ("image".equals(tag)) { } else if ("image".equals(tag)) {
parsingImage = true; this.parsingImage = true;
} }
} }
@ -182,37 +181,37 @@ public class RSSReader extends DefaultHandler {
public void endElement(final String uri, final String name, final String tag) { public void endElement(final String uri, final String name, final String tag) {
if (tag == null) return; if (tag == null) return;
if ("channel".equals(tag) || "feed".equals(tag)) { if ("channel".equals(tag) || "feed".equals(tag)) {
if (parsingChannel) theChannel.setChannel(item); if (this.parsingChannel) this.theChannel.setChannel(this.item);
parsingChannel = false; this.parsingChannel = false;
} else if ("item".equals(tag) || "entry".equals(tag)) { } else if ("item".equals(tag) || "entry".equals(tag)) {
theChannel.addMessage(item); this.theChannel.addMessage(this.item);
parsingItem = false; this.parsingItem = false;
} else if ("image".equals(tag)) { } else if ("image".equals(tag)) {
parsingImage = false; this.parsingImage = false;
} else if ((parsingImage) && (parsingChannel)) { } else if ((this.parsingImage) && (this.parsingChannel)) {
final String value = buffer.toString().trim(); final String value = this.buffer.toString().trim();
buffer.setLength(0); this.buffer.setLength(0);
if ("url".equals(tag)) theChannel.setImage(value); if ("url".equals(tag)) this.theChannel.setImage(value);
} else if (parsingItem) { } else if (this.parsingItem) {
final String value = buffer.toString().trim(); final String value = this.buffer.toString().trim();
buffer.setLength(0); this.buffer.setLength(0);
if (RSSMessage.tags.contains(tag) && value.length() > 0) item.setValue(tag, value); if (RSSMessage.tags.contains(tag) && value.length() > 0) this.item.setValue(tag, value);
} else if (parsingChannel) { } else if (this.parsingChannel) {
final String value = buffer.toString().trim(); final String value = this.buffer.toString().trim();
buffer.setLength(0); this.buffer.setLength(0);
if (RSSMessage.tags.contains(tag)) item.setValue(tag, value); if (RSSMessage.tags.contains(tag)) this.item.setValue(tag, value);
} }
} }
@Override @Override
public void characters(final char ch[], final int start, final int length) { public void characters(final char ch[], final int start, final int length) {
if (parsingItem || parsingChannel) { if (this.parsingItem || this.parsingChannel) {
buffer.append(ch, start, length); this.buffer.append(ch, start, length);
} }
} }
public RSSFeed getFeed() { public RSSFeed getFeed() {
return theChannel; return this.theChannel;
} }
} }

62
source/net/yacy/document/WordTokenizer.java
Unescape View File

@ -41,8 +41,8 @@ public class WordTokenizer implements Enumeration<String> {
// this enumeration removes all words that contain either wrong characters or are too short // this enumeration removes all words that contain either wrong characters or are too short
private StringBuilder buffer = null; private StringBuilder buffer = null;
private unsievedWordsEnum e; private final unsievedWordsEnum e;
private WordCache meaningLib; private final WordCache meaningLib;
public WordTokenizer(final InputStream is, final WordCache meaningLib) { public WordTokenizer(final InputStream is, final WordCache meaningLib) {
assert is != null; assert is != null;
@ -52,13 +52,13 @@ public class WordTokenizer implements Enumeration<String> {
} }
public void pre(final boolean x) { public void pre(final boolean x) {
e.pre(x); this.e.pre(x);
} }
private StringBuilder nextElement0() { private StringBuilder nextElement0() {
StringBuilder s; StringBuilder s;
loop: while (e.hasMoreElements()) { loop: while (this.e.hasMoreElements()) {
s = e.nextElement(); s = this.e.nextElement();
if ((s.length() == 1) && (SentenceReader.punctuation(s.charAt(0)))) return s; if ((s.length() == 1) && (SentenceReader.punctuation(s.charAt(0)))) return s;
for (int i = 0; i < s.length(); i++) { for (int i = 0; i < s.length(); i++) {
if (SentenceReader.invisible(s.charAt(i))) continue loop; if (SentenceReader.invisible(s.charAt(i))) continue loop;
@ -69,85 +69,85 @@ public class WordTokenizer implements Enumeration<String> {
} }
public boolean hasMoreElements() { public boolean hasMoreElements() {
return buffer != null; return this.buffer != null;
} }
public String nextElement() { public String nextElement() {
final String r = (buffer == null) ? null : buffer.toString(); final String r = (this.buffer == null) ? null : this.buffer.toString();
buffer = nextElement0(); this.buffer = nextElement0();
// put word to words statistics cache // put word to words statistics cache
if (meaningLib != null) meaningLib.learn(r); if (this.meaningLib != null) WordCache.learn(r);
return r; return r;
} }
private static class unsievedWordsEnum implements Enumeration<StringBuilder> { private static class unsievedWordsEnum implements Enumeration<StringBuilder> {
// returns an enumeration of StringBuilder Objects // returns an enumeration of StringBuilder Objects
private StringBuilder buffer = null; private StringBuilder buffer = null;
private SentenceReader e; private final SentenceReader e;
private List<StringBuilder> s; private final List<StringBuilder> s;
private int sIndex; private int sIndex;
public unsievedWordsEnum(final InputStream is) { public unsievedWordsEnum(final InputStream is) {
assert is != null; assert is != null;
e = new SentenceReader(is); this.e = new SentenceReader(is);
s = new ArrayList<StringBuilder>(); this.s = new ArrayList<StringBuilder>();
sIndex = 0; this.sIndex = 0;
buffer = nextElement0(); this.buffer = nextElement0();
} }
public void pre(final boolean x) { public void pre(final boolean x) {
e.pre(x); this.e.pre(x);
} }
private StringBuilder nextElement0() { private StringBuilder nextElement0() {
StringBuilder r; StringBuilder r;
StringBuilder sb; StringBuilder sb;
char c; char c;
if (sIndex >= s.size()) { if (this.sIndex >= this.s.size()) {
sIndex = 0; this.sIndex = 0;
s.clear(); this.s.clear();
} }
while (s.isEmpty()) { while (this.s.isEmpty()) {
if (!e.hasNext()) return null; if (!this.e.hasNext()) return null;
r = e.next(); r = this.e.next();
if (r == null) return null; if (r == null) return null;
r = trim(r); r = trim(r);
sb = new StringBuilder(20); sb = new StringBuilder(20);
for (int i = 0; i < r.length(); i++) { for (int i = 0; i < r.length(); i++) {
c = r.charAt(i); c = r.charAt(i);
if (SentenceReader.invisible(c)) { if (SentenceReader.invisible(c)) {
if (sb.length() > 0) {s.add(sb); sb = new StringBuilder(20);} if (sb.length() > 0) {this.s.add(sb); sb = new StringBuilder(20);}
} else if (SentenceReader.punctuation(c)) { } else if (SentenceReader.punctuation(c)) {
if (sb.length() > 0) {s.add(sb); sb = new StringBuilder(1);} if (sb.length() > 0) {this.s.add(sb); sb = new StringBuilder(1);}
sb.append(c); sb.append(c);
s.add(sb); this.s.add(sb);
sb = new StringBuilder(20); sb = new StringBuilder(20);
} else { } else {
sb = sb.append(c); sb = sb.append(c);
} }
} }
if (sb.length() > 0) { if (sb.length() > 0) {
s.add(sb); this.s.add(sb);
sb = null; sb = null;
} }
} }
r = s.get(sIndex++); r = this.s.get(this.sIndex++);
return r; return r;
} }
public boolean hasMoreElements() { public boolean hasMoreElements() {
return buffer != null; return this.buffer != null;
} }
public StringBuilder nextElement() { public StringBuilder nextElement() {
final StringBuilder r = buffer; final StringBuilder r = this.buffer;
buffer = nextElement0(); this.buffer = nextElement0();
return r; return r;
} }
} }
public static StringBuilder trim(StringBuilder sb) { public static StringBuilder trim(final StringBuilder sb) {
int i = 0; int i = 0;
while (i < sb.length() && sb.charAt(i) <= ' ') { while (i < sb.length() && sb.charAt(i) <= ' ') {
i++; i++;

Write Preview
Loading…
Cancel
Save