diff --git a/htroot/PerformanceSearch_p.java b/htroot/PerformanceSearch_p.java index 80a74a6cd..1974fdbee 100644 --- a/htroot/PerformanceSearch_p.java +++ b/htroot/PerformanceSearch_p.java @@ -28,8 +28,8 @@ import java.util.Date; import java.util.Iterator; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.kelondro.util.EventTracker; import net.yacy.peers.graphics.ProfilingGraph; +import net.yacy.search.EventTracker; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 91cc9c7bc..408bc844f 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -37,13 +37,13 @@ import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.EventTracker; import net.yacy.peers.Seed; import net.yacy.peers.Protocol; import net.yacy.peers.Network; import net.yacy.peers.dht.PeerSelection; import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.operation.yacyVersion; +import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; import de.anomic.server.serverCore; import de.anomic.server.serverObjects; diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index da33bc4ff..8c44f3260 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -55,7 +55,6 @@ import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.order.Bitfield; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.util.ByteBuffer; -import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.ISO639; import net.yacy.kelondro.util.MemoryControl; import net.yacy.peers.EventChannel; @@ -63,6 +62,7 @@ import net.yacy.peers.Network; import net.yacy.peers.Protocol; import net.yacy.peers.Seed; import net.yacy.peers.graphics.ProfilingGraph; +import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.Segment; diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index f75bdf4a7..461dd9608 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -61,7 +61,6 @@ import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Bitfield; -import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.ISO639; import net.yacy.kelondro.util.MemoryControl; @@ -69,6 +68,7 @@ import net.yacy.kelondro.util.SetTools; import net.yacy.peers.EventChannel; import net.yacy.peers.NewsPool; import net.yacy.peers.graphics.ProfilingGraph; +import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.Segment; diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index 94fe4e859..4e864831a 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -36,11 +36,11 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader.FileType; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.Formatter; import net.yacy.peers.NewsPool; import net.yacy.peers.Seed; import net.yacy.peers.graphics.ProfilingGraph; +import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.query.QueryParams; diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index fd5a2a2f9..934632d6e 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -32,9 +32,9 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.ScoreMap; import net.yacy.document.Autotagging; import net.yacy.document.LibraryProvider; -import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.Formatter; import net.yacy.peers.graphics.ProfilingGraph; +import net.yacy.search.EventTracker; import net.yacy.search.query.QueryParams; import net.yacy.search.query.SearchEvent; import net.yacy.search.query.SearchEventCache; diff --git a/source/de/anomic/crawler/RobotsTxt.java b/source/de/anomic/crawler/RobotsTxt.java index b71666907..e0381b51c 100644 --- a/source/de/anomic/crawler/RobotsTxt.java +++ b/source/de/anomic/crawler/RobotsTxt.java @@ -261,7 +261,7 @@ public class RobotsTxt { // methods that had been in robotsParser.java: private static final int DOWNLOAD_ACCESS_RESTRICTED = 0; - private static final int DOWNLOAD_ROBOTS_TXT = 1; + static final int DOWNLOAD_ROBOTS_TXT = 1; private static final int DOWNLOAD_ETAG = 2; private static final int DOWNLOAD_MODDATE = 3; @@ -287,7 +287,7 @@ public class RobotsTxt { return port; } - private static Object[] downloadRobotsTxt(final MultiProtocolURI robotsURL, int redirectionCount, final RobotsTxtEntry entry) throws Exception { + static Object[] downloadRobotsTxt(final MultiProtocolURI robotsURL, int redirectionCount, final RobotsTxtEntry entry) throws Exception { if (robotsURL == null || !robotsURL.getProtocol().startsWith("http")) return null; if (redirectionCount < 0) return new Object[]{Boolean.FALSE,null,null}; @@ -308,7 +308,7 @@ public class RobotsTxt { // adding referer reqHeaders.put(RequestHeader.REFERER, (MultiProtocolURI.newURL(robotsURL,"/")).toNormalform(true, true)); - reqHeaders.put(RequestHeader.ACCEPT, HTTPLoader.DEFAULT_ACCEPT); + reqHeaders.put(HeaderFramework.ACCEPT, HTTPLoader.DEFAULT_ACCEPT); if (entry != null) { oldEtag = entry.getETag(); reqHeaders = new RequestHeader(); diff --git a/source/de/anomic/crawler/RobotsTxtParser.java b/source/de/anomic/crawler/RobotsTxtParser.java index 986edfd3b..7af6a140a 100644 --- a/source/de/anomic/crawler/RobotsTxtParser.java +++ b/source/de/anomic/crawler/RobotsTxtParser.java @@ -35,11 +35,15 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.net.MalformedURLException; import java.util.ArrayList; +import java.util.HashSet; import java.util.Set; import java.util.regex.Pattern; +import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.UTF8; +import net.yacy.kelondro.data.meta.DigestURI; /* * A class for Parsing robots.txt files. @@ -96,6 +100,35 @@ public final class RobotsTxtParser { } } + + public static RobotsTxtParser getRobots(String homepage) { + DigestURI theURL; + try { + theURL = new DigestURI(homepage); + } catch (MalformedURLException e1) { + return null; + } + + final String urlHostPort = RobotsTxt.getHostPort(theURL); + MultiProtocolURI robotsURL = null; + try { + robotsURL = new MultiProtocolURI("http://" + urlHostPort + "/robots.txt"); + } catch (final MalformedURLException e) { + return null; + } + + Object[] result; + try { + result = RobotsTxt.downloadRobotsTxt(robotsURL, 0, null); + } catch (Exception e) { + return null; + } + + final byte[] robotsTxt = (byte[]) result[RobotsTxt.DOWNLOAD_ROBOTS_TXT]; + RobotsTxtParser parserResult = new RobotsTxtParser(new HashSet(), robotsTxt); + return parserResult; + } + private void parse(final BufferedReader reader) { final ArrayList deny4AllAgents = new ArrayList(); final ArrayList deny4ThisAgents = new ArrayList(); diff --git a/source/net/yacy/cora/services/federated/solr/SolrScheme.java b/source/net/yacy/cora/services/federated/solr/SolrScheme.java index cd428293c..8302b24e9 100644 --- a/source/net/yacy/cora/services/federated/solr/SolrScheme.java +++ b/source/net/yacy/cora/services/federated/solr/SolrScheme.java @@ -417,7 +417,7 @@ public class SolrScheme extends ConfigurationSet { if (isEmpty() || contains(Field.images_alt_txt.name())) addSolr(solrdoc, Field.images_alt_txt, imgalts); // style sheets - if (isEmpty() || contains("css_txt")) { + if (isEmpty() || contains(Field.css_tag_txt.name())) { final Map csss = html.getCSS(); final String[] css_tag = new String[csss.size()]; final String[] css_url = new String[csss.size()]; diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.java index 71b39613f..c3e878647 100644 --- a/source/net/yacy/document/parser/html/ContentScraper.java +++ b/source/net/yacy/document/parser/html/ContentScraper.java @@ -84,6 +84,7 @@ public class ContentScraper extends AbstractScraper implements Scraper { link(TagType.singleton), embed(TagType.singleton), //added by [MN] param(TagType.singleton), //added by [MN] + iframe(TagType.singleton), // scraped as singleton to get such iframes that have no closing tag a(TagType.pair), h1(TagType.pair), @@ -97,7 +98,6 @@ public class ContentScraper extends AbstractScraper implements Scraper { strong(TagType.pair), i(TagType.pair), li(TagType.pair), - iframe(TagType.pair), script(TagType.pair); public TagType type; @@ -351,7 +351,7 @@ public class ContentScraper extends AbstractScraper implements Scraper { final String content = tagopts.getProperty("content", EMPTY_STRING); if (name.length() > 0) { this.metas.put(name.toLowerCase(), CharacterCoding.html2unicode(content)); - if (name.equals("generator")) { + if (name.toLowerCase().equals("generator")) { this.evaluationScores.match(Element.metagenerator, content); } } else { @@ -420,6 +420,12 @@ public class ContentScraper extends AbstractScraper implements Scraper { tagopts.put("value", url.toNormalform(true, false)); mergeAnchors(url, tagopts /* with property "name" */); } + } else if (tagname.equalsIgnoreCase("iframe")) { + final MultiProtocolURI src = absolutePath(tagopts.getProperty("src", EMPTY_STRING)); + tagopts.put("src", src.toNormalform(true, false)); + mergeAnchors(src, tagopts /* with property "name" */); + this.iframes.add(src); + this.evaluationScores.match(Element.iframepath, src.toNormalform(true, false)); } // fire event @@ -482,12 +488,6 @@ public class ContentScraper extends AbstractScraper implements Scraper { } else if ((tagname.equalsIgnoreCase("li")) && (text.length < 1024)) { h = recursiveParse(text); if (h.length() > 0) this.li.add(h); - } else if (tagname.equalsIgnoreCase("iframe")) { - final MultiProtocolURI src = absolutePath(tagopts.getProperty("src", EMPTY_STRING)); - tagopts.put("src", src.toNormalform(true, false)); - mergeAnchors(src, tagopts /* with property "name" */); - this.iframes.add(src); - this.evaluationScores.match(Element.iframepath, src.toNormalform(true, false)); } else if (tagname.equalsIgnoreCase("script")) { final String src = tagopts.getProperty("src", EMPTY_STRING); if (src.length() > 0) { diff --git a/source/net/yacy/document/parser/html/Evaluation.java b/source/net/yacy/document/parser/html/Evaluation.java index 3c675a4b0..56fd29e9e 100644 --- a/source/net/yacy/document/parser/html/Evaluation.java +++ b/source/net/yacy/document/parser/html/Evaluation.java @@ -84,6 +84,11 @@ public class Evaluation { this.subject = subject; this.pattern = pattern; } + + @Override + public String toString() { + return this.subject + ":" + this.pattern.toString(); + } } private static class Model { @@ -153,11 +158,18 @@ public class Evaluation { final List patterns = this.elementMatcher.get(element); if (patterns == null) return subjects; for (final Attribute attribute: patterns) { - if (attribute.pattern.matcher(content).matches()) subjects.inc(attribute.subject); + if (attribute.pattern.matcher(content).matches()) { + subjects.inc(attribute.subject); + } } return subjects; } + @Override + public String toString() { + return this.modelName + ":" + this.elementMatcher.toString(); + } + } private final Map> modelMap; // a map from model names to attribute scores @@ -166,6 +178,11 @@ public class Evaluation { this.modelMap = new HashMap>(); } + @Override + public String toString() { + return this.modelMap.toString(); + } + /** * produce all model names * @return a set of model names @@ -221,4 +238,15 @@ public class Evaluation { } } + public static void main(String[] args) { + String t = + "// [CDATA[\n" + + "var gaJsH xost = ((\"https:\" == document.location.protocol) ? \"https://ssl.\" : \"http://www.\");\n" + + "document.write(unescape(\"%3Cscript xsrc='\" + gaJ sHost + \"google-analytics.com/ga.js'\n" + + "type='text/javascript'%3E%3C/script%3E\"));\"\n" + + "\"// ]]\""; + Pattern p = Pattern.compile("(?s).*gaJsHost.*|(?s).*_gat._anonymizeIp.*"); + if (p.matcher(t).matches()) System.out.println("1"); + } + } diff --git a/source/net/yacy/document/parser/html/TransformerWriter.java b/source/net/yacy/document/parser/html/TransformerWriter.java index 4901a2543..e2db4aa77 100644 --- a/source/net/yacy/document/parser/html/TransformerWriter.java +++ b/source/net/yacy/document/parser/html/TransformerWriter.java @@ -70,8 +70,6 @@ public final class TransformerWriter extends Writer { private boolean inSingleQuote; private boolean inDoubleQuote; private boolean inComment; - private boolean inScript; - private boolean inStyle; private boolean binaryUnsuspect; private final boolean passbyIfBinarySuspect; @@ -103,8 +101,6 @@ public final class TransformerWriter extends Writer { this.inSingleQuote = false; this.inDoubleQuote = false; this.inComment = false; - this.inScript = false; - this.inStyle = false; this.binaryUnsuspect = true; this.passbyIfBinarySuspect = passbyIfBinarySuspect; @@ -191,7 +187,7 @@ public final class TransformerWriter extends Writer { } private char[] filterTag(final String tag, final boolean opening, final char[] content, final char quotechar) { - //System.out.println("FILTER1: filterTag=" + ((this.filterTag == null) ? "null" : this.filterTag) + ", tag=" + tag + ", opening=" + ((opening) ? "true" : "false") + ", content=" + new String(content)); // debug + //System.out.println("filterTag: filterTag=" + ((this.filterTag == null) ? "null" : this.filterTag) + ", tag=" + tag + ", opening=" + ((opening) ? "true" : "false") + ", content=" + new String(content)); // debug // distinguish the following cases: // - (1) not collecting data for a tag and getting no tag (not opener and not close) // - (2) not collecting data for a tag and getting a tag opener @@ -224,7 +220,7 @@ public final class TransformerWriter extends Writer { } // we are collection tag text for the tag 'filterTag' -> case (4) - (7) - if (tag == null) { + if (tag == null || tag.equals("!")) { // case (4): getting no tag, go on collecting content if (this.scraper != null) this.scraper.scrapeText(content, this.filterTag); if (this.transformer != null) { @@ -324,7 +320,7 @@ public final class TransformerWriter extends Writer { private char[] filterSentence(final char[] in, final char quotechar) { if (in.length == 0) return in; - //System.out.println("FILTER0: " + new String(in)); // debug + //System.out.println("filterSentence, quotechar = \"" + quotechar + "\": " + new String(in)); // debug // scan the string and parse structure if (in.length > 2 && in[0] == lb) { @@ -367,7 +363,7 @@ public final class TransformerWriter extends Writer { @Override public void write(final int c) throws IOException { -// System.out.println((char) c); + //System.out.println((char) c); if ((this.binaryUnsuspect) && (binaryHint((char)c))) { this.binaryUnsuspect = false; if (this.passbyIfBinarySuspect) close(); @@ -412,41 +408,6 @@ public final class TransformerWriter extends Writer { // this.buffer = new serverByteBuffer(); this.buffer.reset(); } - } else if (this.inScript) { - this.buffer.append(c); - final int bufferLength = this.buffer.length(); - if ((c == rb) && (bufferLength > 14) && - (this.buffer.charAt(bufferLength - 9) == lb) && - (this.buffer.charAt(bufferLength - 8) == '/') && - (this.buffer.charAt(bufferLength - 7) == 's') && - (this.buffer.charAt(bufferLength - 6) == 'c') && - (this.buffer.charAt(bufferLength - 5) == 'r') && - (this.buffer.charAt(bufferLength - 4) == 'i') && - (this.buffer.charAt(bufferLength - 3) == 'p') && - (this.buffer.charAt(bufferLength - 2) == 't')) { - // script is at end - this.inScript = false; - if (this.out != null) this.out.write(this.buffer.getChars()); - // this.buffer = new serverByteBuffer(); - this.buffer.reset(); - } - } else if (this.inStyle) { - this.buffer.append(c); - final int bufferLength = this.buffer.length(); - if ((c == rb) && (bufferLength > 13) && - (this.buffer.charAt(bufferLength - 8) == lb) && - (this.buffer.charAt(bufferLength - 7) == '/') && - (this.buffer.charAt(bufferLength - 6) == 's') && - (this.buffer.charAt(bufferLength - 5) == 't') && - (this.buffer.charAt(bufferLength - 4) == 'y') && - (this.buffer.charAt(bufferLength - 3) == 'l') && - (this.buffer.charAt(bufferLength - 2) == 'e')) { - // style is at end - this.inStyle = false; - if (this.out != null) this.out.write(this.buffer.getChars()); - // this.buffer = new serverByteBuffer(); - this.buffer.reset(); - } } else { if (this.buffer.length() == 0) { if (c == rb) { @@ -464,25 +425,6 @@ public final class TransformerWriter extends Writer { // this is the start of a comment this.inComment = true; this.buffer.append(c); - } else if ((this.buffer.length() >= 6) && - (this.buffer.charAt(1) == 's') && - (this.buffer.charAt(2) == 'c') && - (this.buffer.charAt(3) == 'r') && - (this.buffer.charAt(4) == 'i') && - (this.buffer.charAt(5) == 'p') && - (c == 't')) { - // this is the start of a javascript - this.inScript = true; - this.buffer.append(c); - } else if ((this.buffer.length() >= 5) && - (this.buffer.charAt(1) == 's') && - (this.buffer.charAt(2) == 't') && - (this.buffer.charAt(3) == 'y') && - (this.buffer.charAt(4) == 'l') && - (c == 'e')) { - // this is the start of a css-style - this.inStyle = true; - this.buffer.append(c); } else if (c == rb) { this.buffer.append(c); // the tag ends here. after filtering: pass on diff --git a/source/net/yacy/kelondro/blob/Stack.java b/source/net/yacy/kelondro/blob/Stack.java index 003fe5659..5c1579bc5 100644 --- a/source/net/yacy/kelondro/blob/Stack.java +++ b/source/net/yacy/kelondro/blob/Stack.java @@ -7,7 +7,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -37,7 +37,7 @@ public class Stack { private final Heap stack; private long lastHandle; - + /** * create a new stack object. * a stack object is backed by a blob file that contains the stack entries. @@ -45,7 +45,7 @@ public class Stack { * represented as b256-encoded byte[] as key in the blob. * The handle is created using the current time. That means that the top * element on the stack has the maximum time as key handle and the element - * at the bottom of the stack has the minimum time as key handle + * at the bottom of the stack has the minimum time as key handle * @param stackFile * @throws IOException */ @@ -53,7 +53,15 @@ public class Stack { this.stack = new Heap(stackFile, 8, NaturalOrder.naturalOrder, 0); this.lastHandle = 0; } - + + /** + * clear the stack content + * @throws IOException + */ + public void clear() throws IOException { + this.stack.clear(); + } + /** * create a new time handle. In case that the method is called * within a single millisecond twice, a new handle is created using @@ -63,11 +71,11 @@ public class Stack { */ private long nextHandle() { long h = System.currentTimeMillis(); - if (h <= this.lastHandle) h = lastHandle + 1; - lastHandle = h; + if (h <= this.lastHandle) h = this.lastHandle + 1; + this.lastHandle = h; return h; } - + /** * Iterate all handles from the stack as Long numbers * @return an iterator of all handles of the stack @@ -76,7 +84,7 @@ public class Stack { public synchronized Iterator handles() throws IOException { return NaturalOrder.LongIterator(this.stack.keys(true, false)); } - + /** * get the size of a stack * @return the number of entries on the stack @@ -84,20 +92,20 @@ public class Stack { public synchronized int size() { return this.stack.size(); } - + /** * push a new element on the top of the stack * @param b the new stack element * @return the handle used to store the new element * @throws IOException - * @throws RowSpaceExceededException + * @throws RowSpaceExceededException */ public synchronized long push(final byte[] b) throws IOException, RowSpaceExceededException { long handle = nextHandle(); this.stack.insert(NaturalOrder.encodeLong(handle, 8), b); return handle; } - + /** * push a new element on the top of the stack using a entry object * this is only useful for internal processes where a special handle @@ -105,19 +113,19 @@ public class Stack { * @param b the new stack element * @return the handle used to store the new element * @throws IOException - * @throws RowSpaceExceededException + * @throws RowSpaceExceededException */ protected synchronized void push(final Entry e) throws IOException, RowSpaceExceededException { this.stack.insert(NaturalOrder.encodeLong(e.h, 8), e.b); } - + /** * get an element from the stack using the handle * @param handle * @return the object that belongs to the handle * or null if no such element exists * @throws IOException - * @throws RowSpaceExceededException + * @throws RowSpaceExceededException */ public synchronized byte[] get(final long handle) throws IOException, RowSpaceExceededException { byte[] k = NaturalOrder.encodeLong(handle, 8); @@ -125,13 +133,13 @@ public class Stack { if (b == null) return null; return b; } - + /** * remove an element from the stack using the entry handle * @param handle * @return the removed element * @throws IOException - * @throws RowSpaceExceededException + * @throws RowSpaceExceededException */ public synchronized byte[] remove(final long handle) throws IOException, RowSpaceExceededException { byte[] k = NaturalOrder.encodeLong(handle, 8); @@ -140,7 +148,7 @@ public class Stack { this.stack.delete(k); return b; } - + /** * remove the top element from the stack * @return the top element or null if the stack is empty @@ -149,7 +157,7 @@ public class Stack { public synchronized Entry pop() throws IOException { return po(this.stack.lastKey(), true); } - + /** * return the top element of the stack. * The element is not removed from the stack. @@ -160,7 +168,7 @@ public class Stack { public synchronized Entry top() throws IOException { return po(this.stack.lastKey(), false); } - + /** * remove the bottom element from the stack * @return the bottom element or null if the stack is empty @@ -169,7 +177,7 @@ public class Stack { public synchronized Entry pot() throws IOException { return po(this.stack.firstKey(), true); } - + /** * return the bottom element of the stack. * The element is not removed from the stack. @@ -180,7 +188,7 @@ public class Stack { public synchronized Entry bot() throws IOException { return po(this.stack.firstKey(), false); } - + private Entry po(final byte[] k, final boolean remove) throws IOException { if (k == null) return null; assert k.length == 8; @@ -196,12 +204,12 @@ public class Stack { if (remove) this.stack.delete(k); return new Entry(k, b); } - + public class Entry { - + long h; byte[] b; - + /** * create a new entry object using a long handle * @param h @@ -211,7 +219,7 @@ public class Stack { this.h = h; this.b = b; } - + /** * create a new entry object using the byte[] encoded handle * @param k @@ -221,31 +229,31 @@ public class Stack { this.h = NaturalOrder.decodeLong(k); this.b = b; } - + /** * get the handle * @return the handle */ public long handle() { - return h; + return this.h; } - + /** * get the blob entry * @return the blob */ public byte[] blob() { - return b; + return this.b; } } - + /** * close the stack file and write a handle index */ public synchronized void close() { this.stack.close(true); } - + @Override public void finalize() { this.close(); diff --git a/source/net/yacy/kelondro/rwi/IndexCell.java b/source/net/yacy/kelondro/rwi/IndexCell.java index c5a0c87e2..acbbc9775 100644 --- a/source/net/yacy/kelondro/rwi/IndexCell.java +++ b/source/net/yacy/kelondro/rwi/IndexCell.java @@ -42,8 +42,8 @@ import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.MergeIterator; -import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.MemoryControl; +import net.yacy.search.EventTracker; /* diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index c74b77f34..c9b6500a4 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -90,13 +90,13 @@ import net.yacy.kelondro.rwi.Reference; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainerCache; import net.yacy.kelondro.util.ByteBuffer; -import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.FileUtils; import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.peers.graphics.WebStructureGraph; import net.yacy.peers.graphics.WebStructureGraph.HostReference; import net.yacy.peers.operation.yacyVersion; import net.yacy.repository.Blacklist; +import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.Segment; diff --git a/source/net/yacy/peers/graphics/ProfilingGraph.java b/source/net/yacy/peers/graphics/ProfilingGraph.java index da0bc9c8a..b4b91c65c 100644 --- a/source/net/yacy/peers/graphics/ProfilingGraph.java +++ b/source/net/yacy/peers/graphics/ProfilingGraph.java @@ -29,8 +29,8 @@ package net.yacy.peers.graphics; import java.util.ConcurrentModificationException; import java.util.Iterator; -import net.yacy.kelondro.util.EventTracker; -import net.yacy.kelondro.util.EventTracker.Event; +import net.yacy.search.EventTracker; +import net.yacy.search.EventTracker.Event; import net.yacy.search.query.SearchEvent; import net.yacy.visualization.ChartPlotter; import net.yacy.visualization.RasterPlotter; diff --git a/source/net/yacy/kelondro/util/EventTracker.java b/source/net/yacy/search/EventTracker.java similarity index 96% rename from source/net/yacy/kelondro/util/EventTracker.java rename to source/net/yacy/search/EventTracker.java index af338b8bd..131205078 100644 --- a/source/net/yacy/kelondro/util/EventTracker.java +++ b/source/net/yacy/search/EventTracker.java @@ -24,7 +24,7 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -package net.yacy.kelondro.util; +package net.yacy.search; import java.util.Iterator; import java.util.Map; diff --git a/source/net/yacy/kelondro/util/MemoryTracker.java b/source/net/yacy/search/MemoryTracker.java similarity index 96% rename from source/net/yacy/kelondro/util/MemoryTracker.java rename to source/net/yacy/search/MemoryTracker.java index 7d6e87b33..de30394b9 100644 --- a/source/net/yacy/kelondro/util/MemoryTracker.java +++ b/source/net/yacy/search/MemoryTracker.java @@ -24,9 +24,10 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -package net.yacy.kelondro.util; +package net.yacy.search; import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.util.MemoryControl; public class MemoryTracker extends Thread { diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 050790c6a..d1246873b 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -116,10 +116,8 @@ import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.rwi.ReferenceContainer; -import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; -import net.yacy.kelondro.util.MemoryTracker; import net.yacy.kelondro.util.OS; import net.yacy.kelondro.util.SetTools; import net.yacy.kelondro.workflow.BusyThread; diff --git a/source/net/yacy/search/query/RWIProcess.java b/source/net/yacy/search/query/RWIProcess.java index 648bdedc2..145678cca 100644 --- a/source/net/yacy/search/query/RWIProcess.java +++ b/source/net/yacy/search/query/RWIProcess.java @@ -62,8 +62,8 @@ import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.TermSearch; -import net.yacy.kelondro.util.EventTracker; import net.yacy.peers.graphics.ProfilingGraph; +import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; import net.yacy.search.index.Segment; import net.yacy.search.ranking.ReferenceOrder; diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index e49f072e8..c335798d9 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -47,7 +47,6 @@ import net.yacy.kelondro.data.word.WordReferenceFactory; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.rwi.ReferenceContainer; -import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.SetTools; import net.yacy.peers.RemoteSearch; @@ -55,6 +54,7 @@ import net.yacy.peers.SeedDB; import net.yacy.peers.dht.FlatWordPartitionScheme; import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.repository.LoaderDispatcher; +import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; import net.yacy.search.query.SnippetProcess.Worker; import net.yacy.search.ranking.ReferenceOrder; diff --git a/source/net/yacy/search/query/SnippetProcess.java b/source/net/yacy/search/query/SnippetProcess.java index 37beee409..3d265a065 100644 --- a/source/net/yacy/search/query/SnippetProcess.java +++ b/source/net/yacy/search/query/SnippetProcess.java @@ -47,11 +47,11 @@ import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.MemoryControl; import net.yacy.peers.SeedDB; import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.repository.LoaderDispatcher; +import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; import net.yacy.search.snippet.MediaSnippet; import net.yacy.search.snippet.ResultEntry;