diff --git a/defaults/yacy.network.freeworld.unit b/defaults/yacy.network.freeworld.unit index 7a5247b4a..656af9cfa 100644 --- a/defaults/yacy.network.freeworld.unit +++ b/defaults/yacy.network.freeworld.unit @@ -82,7 +82,6 @@ network.unit.update.location0 = http://yacy.net/en/index.html network.unit.update.location1 = http://latest.yacy.de network.unit.update.location2 = http://www.yacystats.de/yacybuild/ network.unit.update.location2.key = MIIBtTCCASwGByqGSM44BAEwggEfAoGBAP1/U4EddRIpUt9KnC7s5Of2EbdSPO9EAMMeP4C2USZpRV1AIlH7WT2NWPq/xfW6MPbLm1Vs14E7gB00b/JmYLdrmVClpJ+f6AR7ECLCT7up1/63xhv4O1fnxqimFQ8E+4P208UewwI1VBNaFpEy9nXzrith1yrv8iIDGZ3RSAHHAhUAl2BQjxUjC8yykrmCouuEC/BYHPUCgYEA9+GghdabPd7LvKtcNrhXuXmUr7v6OuqC+VdMCz0HgmdRWVeOutRZT+ZxBxCBgLRJFnEj6EwoFhO3zwkyjMim4TwWeotUfI0o4KOuHiuzpnWRbqN/C/ohNWLx+2J6ASQ7zKTxvqhRkImog9/hWuWfBpKLZl6Ae1UlZAFMO/7PSSoDgYIAAn8bzmhk8EWOj9h+7zng1o3OlgI+LsG7lI7kVsDxyzMB/WYTpO+NKWoibFjQDFN92TKBQVAA8DQciHfolqcFfVAot9/3ipamXVGz29OAxz8i0Wty6KI6w50YrL2xAkWjx7jSBghJKlnKx3V0PaDCWqz37ogQvuxLKBFORyAjbv3O -network.unit.update.location3 = https://latestyacy.f1ori.de/ # properties for in-protocol response authentication: network.unit.protocol.control = uncontrolled diff --git a/source/de/anomic/http/server/ServerSideIncludes.java b/source/de/anomic/http/server/ServerSideIncludes.java index d6ff63d86..087c2e128 100644 --- a/source/de/anomic/http/server/ServerSideIncludes.java +++ b/source/de/anomic/http/server/ServerSideIncludes.java @@ -94,7 +94,7 @@ public class ServerSideIncludes { header.put(RequestHeader.AUTHORIZATION, authorization); if (requestHeader.containsKey(RequestHeader.COOKIE)) header.put(RequestHeader.COOKIE, requestHeader.get(RequestHeader.COOKIE)); - header.put(RequestHeader.REFERER, requestHeader.get(RequestHeader.CONNECTION_PROP_PATH)); + header.put(RequestHeader.REFERER, requestHeader.get(HeaderFramework.CONNECTION_PROP_PATH)); HTTPDFileHandler.doGet(conProp, header, out); } } diff --git a/source/de/anomic/search/AccessTracker.java b/source/de/anomic/search/AccessTracker.java index 0b9210a5e..4fdf49d9d 100644 --- a/source/de/anomic/search/AccessTracker.java +++ b/source/de/anomic/search/AccessTracker.java @@ -12,12 +12,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -36,7 +36,7 @@ import java.util.LinkedList; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.UTF8; -import net.yacy.document.LibraryProvider; +import net.yacy.document.WordCache; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.MemoryControl; @@ -45,22 +45,22 @@ public class AccessTracker { public static final int minSize = 100; public static final int maxSize = 1000; public static final int maxAge = 10 * 60 * 1000; - + public enum Location {local, remote} - + private static LinkedList localSearches = new LinkedList(); private static LinkedList remoteSearches = new LinkedList(); private static ArrayList log = new ArrayList(); - - public static void add(Location location, QueryParams query) { + + public static void add(final Location location, final QueryParams query) { if (location == Location.local) synchronized (localSearches) {add(localSearches, query);} if (location == Location.remote) synchronized (remoteSearches) {add(remoteSearches, query);} } - - private static void add(LinkedList list, QueryParams query) { + + private static void add(final LinkedList list, final QueryParams query) { // learn that this word can be a word completion for the DidYouMeanLibrary - if (query.resultcount > 10 && query.queryString != null && query.queryString.length() > 0) LibraryProvider.dymLib.learn(query.queryString); - + if (query.resultcount > 10 && query.queryString != null && query.queryString.length() > 0) WordCache.learn(query.queryString); + // add query to statistics list list.add(query); @@ -70,35 +70,35 @@ public class AccessTracker { if (list.size() > 0) addToDump(list.removeFirst()); else break; } } - + // if the list is small we can terminate if (list.size() <= minSize) return; - + // if the list is large we look for too old entries - long timeout = System.currentTimeMillis() - maxAge; + final long timeout = System.currentTimeMillis() - maxAge; while (list.size() > 0) { - QueryParams q = list.getFirst(); + final QueryParams q = list.getFirst(); if (q.time.longValue() > timeout) break; addToDump(list.removeFirst()); } } - - public static Iterator get(Location location) { + + public static Iterator get(final Location location) { if (location == Location.local) return localSearches.descendingIterator(); if (location == Location.remote) return remoteSearches.descendingIterator(); return null; } - - public static int size(Location location) { + + public static int size(final Location location) { if (location == Location.local) synchronized (localSearches) {return localSearches.size();} if (location == Location.remote) synchronized (remoteSearches) {return remoteSearches.size();} return 0; } - - private static void addToDump(QueryParams query) { + + private static void addToDump(final QueryParams query) { //if (query.resultcount == 0) return; if (query.queryString == null || query.queryString.length() == 0) return; - StringBuilder sb = new StringBuilder(40); + final StringBuilder sb = new StringBuilder(40); sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(query.time))); sb.append(' '); sb.append(Integer.toString(query.resultcount)); @@ -106,22 +106,22 @@ public class AccessTracker { sb.append(query.queryString); log.add(sb.toString()); } - - public static void dumpLog(File file) { + + public static void dumpLog(final File file) { while (localSearches.size() > 0) { addToDump(localSearches.removeFirst()); } try { - RandomAccessFile raf = new RandomAccessFile(file, "rw"); + final RandomAccessFile raf = new RandomAccessFile(file, "rw"); raf.seek(raf.length()); - for (String s: log) { + for (final String s: log) { raf.write(UTF8.getBytes(s)); raf.writeByte(10); } log.clear(); - } catch (FileNotFoundException e) { + } catch (final FileNotFoundException e) { Log.logException(e); - } catch (IOException e) { + } catch (final IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } diff --git a/source/de/anomic/search/DocumentIndex.java b/source/de/anomic/search/DocumentIndex.java index cade93eb6..2b5620f79 100644 --- a/source/de/anomic/search/DocumentIndex.java +++ b/source/de/anomic/search/DocumentIndex.java @@ -225,7 +225,7 @@ public class DocumentIndex extends Segment { @Override public void close() { // send termination signal to worker threads - for (final Worker element : this.worker) { + for (@SuppressWarnings("unused") final Worker element : this.worker) { try { this.queue.put(poison); } catch (final InterruptedException e) {} diff --git a/source/de/anomic/search/ResultFetcher.java b/source/de/anomic/search/ResultFetcher.java index 3eef2b6ba..9edf9439c 100644 --- a/source/de/anomic/search/ResultFetcher.java +++ b/source/de/anomic/search/ResultFetcher.java @@ -358,7 +358,6 @@ public class ResultFetcher { // start fetching urls and snippets URIMetadataRow page; ResultEntry resultEntry; - String rawLine; //final int fetchAhead = snippetMode == 0 ? 0 : 10; final boolean nav_topics = ResultFetcher.this.query.navigators.equals("all") || ResultFetcher.this.query.navigators.indexOf("topics") >= 0; try { @@ -405,9 +404,9 @@ public class ResultFetcher { loops++; resultEntry = fetchSnippet(page, solrContent, this.cacheStrategy); // does not fetch snippets if snippetMode == 0 if (resultEntry == null) continue; // the entry had some problems, cannot be used - rawLine = resultEntry.textSnippet() == null ? null : resultEntry.textSnippet().getLineRaw(); + //final String rawLine = resultEntry.textSnippet() == null ? null : resultEntry.textSnippet().getLineRaw(); //System.out.println("***SNIPPET*** raw='" + rawLine + "', pattern='" + this.snippetPattern.toString() + "'"); - if (rawLine != null && !this.snippetPattern.matcher(rawLine).matches()) continue; + //if (rawLine != null && !this.snippetPattern.matcher(rawLine).matches()) continue; //if (result.contains(resultEntry)) continue; ResultFetcher.this.urlRetrievalAllTime += resultEntry.dbRetrievalTime; diff --git a/source/net/yacy/cora/document/RSSReader.java b/source/net/yacy/cora/document/RSSReader.java index c3a06be83..643a2011d 100644 --- a/source/net/yacy/cora/document/RSSReader.java +++ b/source/net/yacy/cora/document/RSSReader.java @@ -7,12 +7,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -33,32 +33,31 @@ import org.xml.sax.Attributes; import org.xml.sax.EntityResolver; import org.xml.sax.InputSource; import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; import org.xml.sax.helpers.DefaultHandler; public class RSSReader extends DefaultHandler { - + // class variables private RSSMessage item; private final StringBuilder buffer; private boolean parsingChannel, parsingImage, parsingItem; private final RSSFeed theChannel; private Type type; - + public enum Type { rss, atom, rdf, none }; - - private RSSReader(int maxsize) { - theChannel = new RSSFeed(maxsize); - buffer = new StringBuilder(300); - item = null; - parsingChannel = false; - parsingImage = false; - parsingItem = false; - type = Type.none; + + private RSSReader(final int maxsize) { + this.theChannel = new RSSFeed(maxsize); + this.buffer = new StringBuilder(300); + this.item = null; + this.parsingChannel = false; + this.parsingImage = false; + this.parsingItem = false; + this.type = Type.none; } - - public RSSReader(int maxsize, final InputStream stream, Type type) throws IOException { + + public RSSReader(final int maxsize, final InputStream stream, final Type type) throws IOException { this(maxsize); this.type = type; final SAXParserFactory factory = SAXParserFactory.newInstance(); @@ -73,18 +72,18 @@ public class RSSReader extends DefaultHandler { } }); saxParser.parse(stream, this); - } catch (SAXException e) { + } catch (final SAXException e) { throw new IOException (e.getMessage()); - } catch (ParserConfigurationException e) { + } catch (final ParserConfigurationException e) { throw new IOException (e.getMessage()); } } - + public Type getType() { return this.type; } - - public static RSSReader parse(int maxsize, final byte[] a) throws IOException { + + public static RSSReader parse(final int maxsize, final byte[] a) throws IOException { // check integrity of array if ((a == null) || (a.length == 0)) { @@ -96,15 +95,15 @@ public class RSSReader extends DefaultHandler { if (!equals(a, UTF8.getBytes(" 0) item.setValue("link", url); + this.item = new RSSMessage(); + this.parsingItem = true; + } else if (this.parsingItem && this.type == Type.atom && "link".equals(tag)) { + final String url = atts.getValue("href"); + if (url != null && url.length() > 0) this.item.setValue("link", url); } else if ("image".equals(tag)) { - parsingImage = true; + this.parsingImage = true; } } @@ -182,37 +181,37 @@ public class RSSReader extends DefaultHandler { public void endElement(final String uri, final String name, final String tag) { if (tag == null) return; if ("channel".equals(tag) || "feed".equals(tag)) { - if (parsingChannel) theChannel.setChannel(item); - parsingChannel = false; + if (this.parsingChannel) this.theChannel.setChannel(this.item); + this.parsingChannel = false; } else if ("item".equals(tag) || "entry".equals(tag)) { - theChannel.addMessage(item); - parsingItem = false; + this.theChannel.addMessage(this.item); + this.parsingItem = false; } else if ("image".equals(tag)) { - parsingImage = false; - } else if ((parsingImage) && (parsingChannel)) { - final String value = buffer.toString().trim(); - buffer.setLength(0); - if ("url".equals(tag)) theChannel.setImage(value); - } else if (parsingItem) { - final String value = buffer.toString().trim(); - buffer.setLength(0); - if (RSSMessage.tags.contains(tag) && value.length() > 0) item.setValue(tag, value); - } else if (parsingChannel) { - final String value = buffer.toString().trim(); - buffer.setLength(0); - if (RSSMessage.tags.contains(tag)) item.setValue(tag, value); + this.parsingImage = false; + } else if ((this.parsingImage) && (this.parsingChannel)) { + final String value = this.buffer.toString().trim(); + this.buffer.setLength(0); + if ("url".equals(tag)) this.theChannel.setImage(value); + } else if (this.parsingItem) { + final String value = this.buffer.toString().trim(); + this.buffer.setLength(0); + if (RSSMessage.tags.contains(tag) && value.length() > 0) this.item.setValue(tag, value); + } else if (this.parsingChannel) { + final String value = this.buffer.toString().trim(); + this.buffer.setLength(0); + if (RSSMessage.tags.contains(tag)) this.item.setValue(tag, value); } } @Override public void characters(final char ch[], final int start, final int length) { - if (parsingItem || parsingChannel) { - buffer.append(ch, start, length); + if (this.parsingItem || this.parsingChannel) { + this.buffer.append(ch, start, length); } } - + public RSSFeed getFeed() { - return theChannel; + return this.theChannel; } } \ No newline at end of file diff --git a/source/net/yacy/document/WordTokenizer.java b/source/net/yacy/document/WordTokenizer.java index fb26699a2..e1ac29216 100644 --- a/source/net/yacy/document/WordTokenizer.java +++ b/source/net/yacy/document/WordTokenizer.java @@ -11,12 +11,12 @@ * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with this program in the file lgpl21.txt * If not, see . @@ -39,10 +39,10 @@ import net.yacy.kelondro.order.Base64Order; public class WordTokenizer implements Enumeration { // this enumeration removes all words that contain either wrong characters or are too short - + private StringBuilder buffer = null; - private unsievedWordsEnum e; - private WordCache meaningLib; + private final unsievedWordsEnum e; + private final WordCache meaningLib; public WordTokenizer(final InputStream is, final WordCache meaningLib) { assert is != null; @@ -52,13 +52,13 @@ public class WordTokenizer implements Enumeration { } public void pre(final boolean x) { - e.pre(x); + this.e.pre(x); } - + private StringBuilder nextElement0() { StringBuilder s; - loop: while (e.hasMoreElements()) { - s = e.nextElement(); + loop: while (this.e.hasMoreElements()) { + s = this.e.nextElement(); if ((s.length() == 1) && (SentenceReader.punctuation(s.charAt(0)))) return s; for (int i = 0; i < s.length(); i++) { if (SentenceReader.invisible(s.charAt(i))) continue loop; @@ -69,85 +69,85 @@ public class WordTokenizer implements Enumeration { } public boolean hasMoreElements() { - return buffer != null; + return this.buffer != null; } public String nextElement() { - final String r = (buffer == null) ? null : buffer.toString(); - buffer = nextElement0(); + final String r = (this.buffer == null) ? null : this.buffer.toString(); + this.buffer = nextElement0(); // put word to words statistics cache - if (meaningLib != null) meaningLib.learn(r); + if (this.meaningLib != null) WordCache.learn(r); return r; } - + private static class unsievedWordsEnum implements Enumeration { // returns an enumeration of StringBuilder Objects private StringBuilder buffer = null; - private SentenceReader e; - private List s; + private final SentenceReader e; + private final List s; private int sIndex; public unsievedWordsEnum(final InputStream is) { assert is != null; - e = new SentenceReader(is); - s = new ArrayList(); - sIndex = 0; - buffer = nextElement0(); + this.e = new SentenceReader(is); + this.s = new ArrayList(); + this.sIndex = 0; + this.buffer = nextElement0(); } public void pre(final boolean x) { - e.pre(x); + this.e.pre(x); } - + private StringBuilder nextElement0() { StringBuilder r; StringBuilder sb; char c; - if (sIndex >= s.size()) { - sIndex = 0; - s.clear(); + if (this.sIndex >= this.s.size()) { + this.sIndex = 0; + this.s.clear(); } - while (s.isEmpty()) { - if (!e.hasNext()) return null; - r = e.next(); + while (this.s.isEmpty()) { + if (!this.e.hasNext()) return null; + r = this.e.next(); if (r == null) return null; r = trim(r); sb = new StringBuilder(20); for (int i = 0; i < r.length(); i++) { c = r.charAt(i); if (SentenceReader.invisible(c)) { - if (sb.length() > 0) {s.add(sb); sb = new StringBuilder(20);} + if (sb.length() > 0) {this.s.add(sb); sb = new StringBuilder(20);} } else if (SentenceReader.punctuation(c)) { - if (sb.length() > 0) {s.add(sb); sb = new StringBuilder(1);} + if (sb.length() > 0) {this.s.add(sb); sb = new StringBuilder(1);} sb.append(c); - s.add(sb); + this.s.add(sb); sb = new StringBuilder(20); } else { sb = sb.append(c); } } if (sb.length() > 0) { - s.add(sb); + this.s.add(sb); sb = null; } } - r = s.get(sIndex++); + r = this.s.get(this.sIndex++); return r; } public boolean hasMoreElements() { - return buffer != null; + return this.buffer != null; } public StringBuilder nextElement() { - final StringBuilder r = buffer; - buffer = nextElement0(); + final StringBuilder r = this.buffer; + this.buffer = nextElement0(); return r; } } - - public static StringBuilder trim(StringBuilder sb) { + + public static StringBuilder trim(final StringBuilder sb) { int i = 0; while (i < sb.length() && sb.charAt(i) <= ' ') { i++; @@ -180,13 +180,13 @@ public class WordTokenizer implements Enumeration { while (words.hasMoreElements()) { word = words.nextElement(); hash = Word.word2hash(word.toString()); - + // don't overwrite old values, that leads to too far word distances oldpos = map.put(hash, LargeNumberCache.valueOf(pos)); if (oldpos != null) { map.put(hash, oldpos); } - + pos += word.length() + 1; } return map;