- removed some warinings

- removed a dead update location

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7970 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent fabda9ad31
commit 6b22865dbc

@ -82,7 +82,6 @@ network.unit.update.location0 = http://yacy.net/en/index.html
network.unit.update.location1 = http://latest.yacy.de
network.unit.update.location2 = http://www.yacystats.de/yacybuild/
network.unit.update.location2.key = MIIBtTCCASwGByqGSM44BAEwggEfAoGBAP1/U4EddRIpUt9KnC7s5Of2EbdSPO9EAMMeP4C2USZpRV1AIlH7WT2NWPq/xfW6MPbLm1Vs14E7gB00b/JmYLdrmVClpJ+f6AR7ECLCT7up1/63xhv4O1fnxqimFQ8E+4P208UewwI1VBNaFpEy9nXzrith1yrv8iIDGZ3RSAHHAhUAl2BQjxUjC8yykrmCouuEC/BYHPUCgYEA9+GghdabPd7LvKtcNrhXuXmUr7v6OuqC+VdMCz0HgmdRWVeOutRZT+ZxBxCBgLRJFnEj6EwoFhO3zwkyjMim4TwWeotUfI0o4KOuHiuzpnWRbqN/C/ohNWLx+2J6ASQ7zKTxvqhRkImog9/hWuWfBpKLZl6Ae1UlZAFMO/7PSSoDgYIAAn8bzmhk8EWOj9h+7zng1o3OlgI+LsG7lI7kVsDxyzMB/WYTpO+NKWoibFjQDFN92TKBQVAA8DQciHfolqcFfVAot9/3ipamXVGz29OAxz8i0Wty6KI6w50YrL2xAkWjx7jSBghJKlnKx3V0PaDCWqz37ogQvuxLKBFORyAjbv3O
network.unit.update.location3 = https://latestyacy.f1ori.de/
# properties for in-protocol response authentication:
network.unit.protocol.control = uncontrolled

@ -94,7 +94,7 @@ public class ServerSideIncludes {
header.put(RequestHeader.AUTHORIZATION, authorization);
if (requestHeader.containsKey(RequestHeader.COOKIE))
header.put(RequestHeader.COOKIE, requestHeader.get(RequestHeader.COOKIE));
header.put(RequestHeader.REFERER, requestHeader.get(RequestHeader.CONNECTION_PROP_PATH));
header.put(RequestHeader.REFERER, requestHeader.get(HeaderFramework.CONNECTION_PROP_PATH));
HTTPDFileHandler.doGet(conProp, header, out);
}
}

@ -12,12 +12,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
@ -36,7 +36,7 @@ import java.util.LinkedList;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.UTF8;
import net.yacy.document.LibraryProvider;
import net.yacy.document.WordCache;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.MemoryControl;
@ -45,22 +45,22 @@ public class AccessTracker {
public static final int minSize = 100;
public static final int maxSize = 1000;
public static final int maxAge = 10 * 60 * 1000;
public enum Location {local, remote}
private static LinkedList<QueryParams> localSearches = new LinkedList<QueryParams>();
private static LinkedList<QueryParams> remoteSearches = new LinkedList<QueryParams>();
private static ArrayList<String> log = new ArrayList<String>();
public static void add(Location location, QueryParams query) {
public static void add(final Location location, final QueryParams query) {
if (location == Location.local) synchronized (localSearches) {add(localSearches, query);}
if (location == Location.remote) synchronized (remoteSearches) {add(remoteSearches, query);}
}
private static void add(LinkedList<QueryParams> list, QueryParams query) {
private static void add(final LinkedList<QueryParams> list, final QueryParams query) {
// learn that this word can be a word completion for the DidYouMeanLibrary
if (query.resultcount > 10 && query.queryString != null && query.queryString.length() > 0) LibraryProvider.dymLib.learn(query.queryString);
if (query.resultcount > 10 && query.queryString != null && query.queryString.length() > 0) WordCache.learn(query.queryString);
// add query to statistics list
list.add(query);
@ -70,35 +70,35 @@ public class AccessTracker {
if (list.size() > 0) addToDump(list.removeFirst()); else break;
}
}
// if the list is small we can terminate
if (list.size() <= minSize) return;
// if the list is large we look for too old entries
long timeout = System.currentTimeMillis() - maxAge;
final long timeout = System.currentTimeMillis() - maxAge;
while (list.size() > 0) {
QueryParams q = list.getFirst();
final QueryParams q = list.getFirst();
if (q.time.longValue() > timeout) break;
addToDump(list.removeFirst());
}
}
public static Iterator<QueryParams> get(Location location) {
public static Iterator<QueryParams> get(final Location location) {
if (location == Location.local) return localSearches.descendingIterator();
if (location == Location.remote) return remoteSearches.descendingIterator();
return null;
}
public static int size(Location location) {
public static int size(final Location location) {
if (location == Location.local) synchronized (localSearches) {return localSearches.size();}
if (location == Location.remote) synchronized (remoteSearches) {return remoteSearches.size();}
return 0;
}
private static void addToDump(QueryParams query) {
private static void addToDump(final QueryParams query) {
//if (query.resultcount == 0) return;
if (query.queryString == null || query.queryString.length() == 0) return;
StringBuilder sb = new StringBuilder(40);
final StringBuilder sb = new StringBuilder(40);
sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(query.time)));
sb.append(' ');
sb.append(Integer.toString(query.resultcount));
@ -106,22 +106,22 @@ public class AccessTracker {
sb.append(query.queryString);
log.add(sb.toString());
}
public static void dumpLog(File file) {
public static void dumpLog(final File file) {
while (localSearches.size() > 0) {
addToDump(localSearches.removeFirst());
}
try {
RandomAccessFile raf = new RandomAccessFile(file, "rw");
final RandomAccessFile raf = new RandomAccessFile(file, "rw");
raf.seek(raf.length());
for (String s: log) {
for (final String s: log) {
raf.write(UTF8.getBytes(s));
raf.writeByte(10);
}
log.clear();
} catch (FileNotFoundException e) {
} catch (final FileNotFoundException e) {
Log.logException(e);
} catch (IOException e) {
} catch (final IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

@ -225,7 +225,7 @@ public class DocumentIndex extends Segment {
@Override
public void close() {
// send termination signal to worker threads
for (final Worker element : this.worker) {
for (@SuppressWarnings("unused") final Worker element : this.worker) {
try {
this.queue.put(poison);
} catch (final InterruptedException e) {}

@ -358,7 +358,6 @@ public class ResultFetcher {
// start fetching urls and snippets
URIMetadataRow page;
ResultEntry resultEntry;
String rawLine;
//final int fetchAhead = snippetMode == 0 ? 0 : 10;
final boolean nav_topics = ResultFetcher.this.query.navigators.equals("all") || ResultFetcher.this.query.navigators.indexOf("topics") >= 0;
try {
@ -405,9 +404,9 @@ public class ResultFetcher {
loops++;
resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0
if (resultEntry == null) continue; // the entry had some problems, cannot be used
rawLine = resultEntry.textSnippet() == null ? null : resultEntry.textSnippet().getLineRaw();
//final String rawLine = resultEntry.textSnippet() == null ? null : resultEntry.textSnippet().getLineRaw();
//System.out.println("***SNIPPET*** raw='" + rawLine + "', pattern='" + this.snippetPattern.toString() + "'");
if (rawLine != null && !this.snippetPattern.matcher(rawLine).matches()) continue;
//if (rawLine != null && !this.snippetPattern.matcher(rawLine).matches()) continue;
//if (result.contains(resultEntry)) continue;
ResultFetcher.this.urlRetrievalAllTime += resultEntry.dbRetrievalTime;

@ -7,12 +7,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
@ -33,32 +33,31 @@ import org.xml.sax.Attributes;
import org.xml.sax.EntityResolver;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.XMLReader;
import org.xml.sax.helpers.DefaultHandler;
public class RSSReader extends DefaultHandler {
// class variables
private RSSMessage item;
private final StringBuilder buffer;
private boolean parsingChannel, parsingImage, parsingItem;
private final RSSFeed theChannel;
private Type type;
public enum Type { rss, atom, rdf, none };
private RSSReader(int maxsize) {
theChannel = new RSSFeed(maxsize);
buffer = new StringBuilder(300);
item = null;
parsingChannel = false;
parsingImage = false;
parsingItem = false;
type = Type.none;
private RSSReader(final int maxsize) {
this.theChannel = new RSSFeed(maxsize);
this.buffer = new StringBuilder(300);
this.item = null;
this.parsingChannel = false;
this.parsingImage = false;
this.parsingItem = false;
this.type = Type.none;
}
public RSSReader(int maxsize, final InputStream stream, Type type) throws IOException {
public RSSReader(final int maxsize, final InputStream stream, final Type type) throws IOException {
this(maxsize);
this.type = type;
final SAXParserFactory factory = SAXParserFactory.newInstance();
@ -73,18 +72,18 @@ public class RSSReader extends DefaultHandler {
}
});
saxParser.parse(stream, this);
} catch (SAXException e) {
} catch (final SAXException e) {
throw new IOException (e.getMessage());
} catch (ParserConfigurationException e) {
} catch (final ParserConfigurationException e) {
throw new IOException (e.getMessage());
}
}
public Type getType() {
return this.type;
}
public static RSSReader parse(int maxsize, final byte[] a) throws IOException {
public static RSSReader parse(final int maxsize, final byte[] a) throws IOException {
// check integrity of array
if ((a == null) || (a.length == 0)) {
@ -96,15 +95,15 @@ public class RSSReader extends DefaultHandler {
if (!equals(a, UTF8.getBytes("<?xml")) && !equals(a, UTF8.getBytes("<rss"))) {
throw new IOException("response does not contain valid xml");
}
final Type type = findOutType(a);
if (type == Type.none) {
throw new IOException("response incomplete");
}
// make input stream
final ByteArrayInputStream bais = new ByteArrayInputStream(a);
// parse stream
RSSReader reader = null;
try {
@ -156,25 +155,25 @@ public class RSSReader extends DefaultHandler {
public void startElement(final String uri, final String name, final String tag, final Attributes atts) throws SAXException {
if ("channel".equals(tag)) {
this.type = Type.rss;
item = new RSSMessage();
parsingChannel = true;
this.item = new RSSMessage();
this.parsingChannel = true;
} else if ("feed".equals(tag)) {
this.type = Type.atom;
item = new RSSMessage();
parsingChannel = true;
this.item = new RSSMessage();
this.parsingChannel = true;
} else if ("item".equals(tag) || "entry".equals(tag)) {
if (parsingChannel) {
if (this.parsingChannel) {
// the channel ends with the first item not with the channel close tag
theChannel.setChannel(item);
parsingChannel = false;
this.theChannel.setChannel(this.item);
this.parsingChannel = false;
}
item = new RSSMessage();
parsingItem = true;
} else if (parsingItem && this.type == Type.atom && "link".equals(tag)) {
String url = atts.getValue("href");
if (url != null && url.length() > 0) item.setValue("link", url);
this.item = new RSSMessage();
this.parsingItem = true;
} else if (this.parsingItem && this.type == Type.atom && "link".equals(tag)) {
final String url = atts.getValue("href");
if (url != null && url.length() > 0) this.item.setValue("link", url);
} else if ("image".equals(tag)) {
parsingImage = true;
this.parsingImage = true;
}
}
@ -182,37 +181,37 @@ public class RSSReader extends DefaultHandler {
public void endElement(final String uri, final String name, final String tag) {
if (tag == null) return;
if ("channel".equals(tag) || "feed".equals(tag)) {
if (parsingChannel) theChannel.setChannel(item);
parsingChannel = false;
if (this.parsingChannel) this.theChannel.setChannel(this.item);
this.parsingChannel = false;
} else if ("item".equals(tag) || "entry".equals(tag)) {
theChannel.addMessage(item);
parsingItem = false;
this.theChannel.addMessage(this.item);
this.parsingItem = false;
} else if ("image".equals(tag)) {
parsingImage = false;
} else if ((parsingImage) && (parsingChannel)) {
final String value = buffer.toString().trim();
buffer.setLength(0);
if ("url".equals(tag)) theChannel.setImage(value);
} else if (parsingItem) {
final String value = buffer.toString().trim();
buffer.setLength(0);
if (RSSMessage.tags.contains(tag) && value.length() > 0) item.setValue(tag, value);
} else if (parsingChannel) {
final String value = buffer.toString().trim();
buffer.setLength(0);
if (RSSMessage.tags.contains(tag)) item.setValue(tag, value);
this.parsingImage = false;
} else if ((this.parsingImage) && (this.parsingChannel)) {
final String value = this.buffer.toString().trim();
this.buffer.setLength(0);
if ("url".equals(tag)) this.theChannel.setImage(value);
} else if (this.parsingItem) {
final String value = this.buffer.toString().trim();
this.buffer.setLength(0);
if (RSSMessage.tags.contains(tag) && value.length() > 0) this.item.setValue(tag, value);
} else if (this.parsingChannel) {
final String value = this.buffer.toString().trim();
this.buffer.setLength(0);
if (RSSMessage.tags.contains(tag)) this.item.setValue(tag, value);
}
}
@Override
public void characters(final char ch[], final int start, final int length) {
if (parsingItem || parsingChannel) {
buffer.append(ch, start, length);
if (this.parsingItem || this.parsingChannel) {
this.buffer.append(ch, start, length);
}
}
public RSSFeed getFeed() {
return theChannel;
return this.theChannel;
}
}

@ -11,12 +11,12 @@
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
@ -39,10 +39,10 @@ import net.yacy.kelondro.order.Base64Order;
public class WordTokenizer implements Enumeration<String> {
// this enumeration removes all words that contain either wrong characters or are too short
private StringBuilder buffer = null;
private unsievedWordsEnum e;
private WordCache meaningLib;
private final unsievedWordsEnum e;
private final WordCache meaningLib;
public WordTokenizer(final InputStream is, final WordCache meaningLib) {
assert is != null;
@ -52,13 +52,13 @@ public class WordTokenizer implements Enumeration<String> {
}
public void pre(final boolean x) {
e.pre(x);
this.e.pre(x);
}
private StringBuilder nextElement0() {
StringBuilder s;
loop: while (e.hasMoreElements()) {
s = e.nextElement();
loop: while (this.e.hasMoreElements()) {
s = this.e.nextElement();
if ((s.length() == 1) && (SentenceReader.punctuation(s.charAt(0)))) return s;
for (int i = 0; i < s.length(); i++) {
if (SentenceReader.invisible(s.charAt(i))) continue loop;
@ -69,85 +69,85 @@ public class WordTokenizer implements Enumeration<String> {
}
public boolean hasMoreElements() {
return buffer != null;
return this.buffer != null;
}
public String nextElement() {
final String r = (buffer == null) ? null : buffer.toString();
buffer = nextElement0();
final String r = (this.buffer == null) ? null : this.buffer.toString();
this.buffer = nextElement0();
// put word to words statistics cache
if (meaningLib != null) meaningLib.learn(r);
if (this.meaningLib != null) WordCache.learn(r);
return r;
}
private static class unsievedWordsEnum implements Enumeration<StringBuilder> {
// returns an enumeration of StringBuilder Objects
private StringBuilder buffer = null;
private SentenceReader e;
private List<StringBuilder> s;
private final SentenceReader e;
private final List<StringBuilder> s;
private int sIndex;
public unsievedWordsEnum(final InputStream is) {
assert is != null;
e = new SentenceReader(is);
s = new ArrayList<StringBuilder>();
sIndex = 0;
buffer = nextElement0();
this.e = new SentenceReader(is);
this.s = new ArrayList<StringBuilder>();
this.sIndex = 0;
this.buffer = nextElement0();
}
public void pre(final boolean x) {
e.pre(x);
this.e.pre(x);
}
private StringBuilder nextElement0() {
StringBuilder r;
StringBuilder sb;
char c;
if (sIndex >= s.size()) {
sIndex = 0;
s.clear();
if (this.sIndex >= this.s.size()) {
this.sIndex = 0;
this.s.clear();
}
while (s.isEmpty()) {
if (!e.hasNext()) return null;
r = e.next();
while (this.s.isEmpty()) {
if (!this.e.hasNext()) return null;
r = this.e.next();
if (r == null) return null;
r = trim(r);
sb = new StringBuilder(20);
for (int i = 0; i < r.length(); i++) {
c = r.charAt(i);
if (SentenceReader.invisible(c)) {
if (sb.length() > 0) {s.add(sb); sb = new StringBuilder(20);}
if (sb.length() > 0) {this.s.add(sb); sb = new StringBuilder(20);}
} else if (SentenceReader.punctuation(c)) {
if (sb.length() > 0) {s.add(sb); sb = new StringBuilder(1);}
if (sb.length() > 0) {this.s.add(sb); sb = new StringBuilder(1);}
sb.append(c);
s.add(sb);
this.s.add(sb);
sb = new StringBuilder(20);
} else {
sb = sb.append(c);
}
}
if (sb.length() > 0) {
s.add(sb);
this.s.add(sb);
sb = null;
}
}
r = s.get(sIndex++);
r = this.s.get(this.sIndex++);
return r;
}
public boolean hasMoreElements() {
return buffer != null;
return this.buffer != null;
}
public StringBuilder nextElement() {
final StringBuilder r = buffer;
buffer = nextElement0();
final StringBuilder r = this.buffer;
this.buffer = nextElement0();
return r;
}
}
public static StringBuilder trim(StringBuilder sb) {
public static StringBuilder trim(final StringBuilder sb) {
int i = 0;
while (i < sb.length() && sb.charAt(i) <= ' ') {
i++;
@ -180,13 +180,13 @@ public class WordTokenizer implements Enumeration<String> {
while (words.hasMoreElements()) {
word = words.nextElement();
hash = Word.word2hash(word.toString());
// don't overwrite old values, that leads to too far word distances
oldpos = map.put(hash, LargeNumberCache.valueOf(pos));
if (oldpos != null) {
map.put(hash, oldpos);
}
pos += word.length() + 1;
}
return map;

Loading…
Cancel
Save