diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java index 2eb4231dc..24d017932 100644 --- a/source/de/anomic/crawler/CrawlQueues.java +++ b/source/de/anomic/crawler/CrawlQueues.java @@ -98,7 +98,7 @@ public class CrawlQueues { this.delegatedURL = new ZURL(this.sb.indexSegments.segment(PROCESS).getSolr(), newQueuePath, DELEGATED_DB_FILENAME, true, this.sb.useTailCache, this.sb.exceed134217727); } - public void close() { + public synchronized void close() { // wait for all workers to finish for (final Loader w: this.workers.values()) { w.interrupt(); diff --git a/source/de/anomic/crawler/CrawlStacker.java b/source/de/anomic/crawler/CrawlStacker.java index 6c5e96080..305781e02 100644 --- a/source/de/anomic/crawler/CrawlStacker.java +++ b/source/de/anomic/crawler/CrawlStacker.java @@ -121,7 +121,7 @@ public final class CrawlStacker { this.slowQueue.announceShutdown(); } - public void close() { + public synchronized void close() { this.log.logInfo("Shutdown. waiting for remaining " + size() + " crawl stacker job entries. please wait."); this.fastQueue.announceShutdown(); this.slowQueue.announceShutdown(); diff --git a/source/de/anomic/crawler/CrawlSwitchboard.java b/source/de/anomic/crawler/CrawlSwitchboard.java index 7f78fd13f..dbec1f6f0 100644 --- a/source/de/anomic/crawler/CrawlSwitchboard.java +++ b/source/de/anomic/crawler/CrawlSwitchboard.java @@ -541,7 +541,7 @@ public final class CrawlSwitchboard return hasDoneSomething; } - public void close() { + public synchronized void close() { this.profilesActiveCrawlsCache.clear(); this.profilesActiveCrawls.close(); this.profilesPassiveCrawls.close(); diff --git a/source/de/anomic/crawler/NoticedURL.java b/source/de/anomic/crawler/NoticedURL.java index d3fdf7174..9dc6e74ba 100644 --- a/source/de/anomic/crawler/NoticedURL.java +++ b/source/de/anomic/crawler/NoticedURL.java @@ -91,7 +91,7 @@ public class NoticedURL { this.noloadStack.clear(); } - public void close() { + public synchronized void close() { Log.logInfo("NoticedURL", "CLOSING ALL STACKS"); if (this.coreStack != null) { this.coreStack.close(); diff --git a/source/de/anomic/data/BlogBoard.java b/source/de/anomic/data/BlogBoard.java index a41484eba..13dd200e0 100644 --- a/source/de/anomic/data/BlogBoard.java +++ b/source/de/anomic/data/BlogBoard.java @@ -88,7 +88,7 @@ public class BlogBoard { return database.containsKey(UTF8.getBytes(key)); } - public void close() { + public synchronized void close() { database.close(); } diff --git a/source/de/anomic/data/BlogBoardComments.java b/source/de/anomic/data/BlogBoardComments.java index 836ed1448..690c053bd 100644 --- a/source/de/anomic/data/BlogBoardComments.java +++ b/source/de/anomic/data/BlogBoardComments.java @@ -81,7 +81,7 @@ public class BlogBoardComments { return this.database.size(); } - public void close() { + public synchronized void close() { this.database.close(); } diff --git a/source/de/anomic/data/BookmarkDate.java b/source/de/anomic/data/BookmarkDate.java index 82737d457..689873e0b 100644 --- a/source/de/anomic/data/BookmarkDate.java +++ b/source/de/anomic/data/BookmarkDate.java @@ -53,7 +53,7 @@ public class BookmarkDate { this.datesTable = new MapHeap(datesFile, 20, NaturalOrder.naturalOrder, 1024 * 64, 500, '_'); } - public void close() { + public synchronized void close() { this.datesTable.close(); } diff --git a/source/de/anomic/data/BookmarksDB.java b/source/de/anomic/data/BookmarksDB.java index d03603cd7..1ed5d15a8 100644 --- a/source/de/anomic/data/BookmarksDB.java +++ b/source/de/anomic/data/BookmarksDB.java @@ -113,7 +113,7 @@ public class BookmarksDB { // bookmarksDB's functions for 'destructing' the class // ----------------------------------------------------- - public void close(){ + public synchronized void close(){ this.bookmarks.close(); this.tags.clear(); this.dates.close(); @@ -764,4 +764,4 @@ public class BookmarksDB { public BookmarkDate.Entry getDate(final String date) { return this.dates.getDate(date); } -} \ No newline at end of file +} diff --git a/source/de/anomic/data/MessageBoard.java b/source/de/anomic/data/MessageBoard.java index 08e1530a1..bd72e09e3 100644 --- a/source/de/anomic/data/MessageBoard.java +++ b/source/de/anomic/data/MessageBoard.java @@ -67,7 +67,7 @@ public class MessageBoard { return database.size(); } - public void close() { + public synchronized void close() { database.close(); } diff --git a/source/de/anomic/data/UserDB.java b/source/de/anomic/data/UserDB.java index a794cc995..b37cf697a 100644 --- a/source/de/anomic/data/UserDB.java +++ b/source/de/anomic/data/UserDB.java @@ -78,7 +78,7 @@ public final class UserDB { } } - public void close() { + public synchronized void close() { userTable.close(); } diff --git a/source/de/anomic/data/wiki/WikiBoard.java b/source/de/anomic/data/wiki/WikiBoard.java index 31163ddfd..ffffc79ec 100644 --- a/source/de/anomic/data/wiki/WikiBoard.java +++ b/source/de/anomic/data/wiki/WikiBoard.java @@ -100,7 +100,7 @@ public class WikiBoard { /** * Closes database files. */ - public void close() { + public synchronized void close() { datbase.close(); bkpbase.close(); } diff --git a/source/de/anomic/http/server/ChunkedInputStream.java b/source/de/anomic/http/server/ChunkedInputStream.java index ba6661bd8..a5930f008 100644 --- a/source/de/anomic/http/server/ChunkedInputStream.java +++ b/source/de/anomic/http/server/ChunkedInputStream.java @@ -62,7 +62,7 @@ import java.io.UnsupportedEncodingException; */ public class ChunkedInputStream extends InputStream { /** The inputstream that we're wrapping */ - private InputStream in; + private final InputStream in; /** The chunk size */ private int chunkSize; @@ -87,7 +87,7 @@ public class ChunkedInputStream extends InputStream { * @throws IOException If an IO error occurs */ public ChunkedInputStream(final InputStream in) throws IOException { - + if (in == null) { throw new IllegalArgumentException("InputStream parameter may not be null"); } @@ -95,37 +95,38 @@ public class ChunkedInputStream extends InputStream { this.pos = 0; } - + /** *

Returns all the data in a chunked stream in coalesced form. A chunk * is followed by a CRLF. The method returns -1 as soon as a chunksize of 0 * is detected.

- * + * *

Trailer headers are read automcatically at the end of the stream and * can be obtained with the getResponseFooters() method.

* * @return -1 of the end of the stream has been reached or the next data * byte * @throws IOException If an IO problem occurs - * + * * @see HttpMethod#getResponseFooters() */ + @Override public int read() throws IOException { - if (closed) { + if (this.closed) { throw new IOException("Attempted read from closed stream."); } - if (eof) { + if (this.eof) { return -1; - } - if (pos >= chunkSize) { + } + if (this.pos >= this.chunkSize) { nextChunk(); - if (eof) { + if (this.eof) { return -1; } } - pos++; - return in.read(); + this.pos++; + return this.in.read(); } /** @@ -139,20 +140,21 @@ public class ChunkedInputStream extends InputStream { * @see java.io.InputStream#read(byte[], int, int) * @throws IOException if an IO problem occurs. */ + @Override public int read(byte[] b, int off, int len) throws IOException { - if (closed) throw new IOException("Attempted read from closed stream."); - if (eof) return -1; - - if (pos >= chunkSize) { + if (this.closed) throw new IOException("Attempted read from closed stream."); + if (this.eof) return -1; + + if (this.pos >= this.chunkSize) { nextChunk(); - if (eof) { + if (this.eof) { return -1; } } - len = Math.min(len, chunkSize - pos); - int count = in.read(b, off, len); - pos += count; + len = Math.min(len, this.chunkSize - this.pos); + int count = this.in.read(b, off, len); + this.pos += count; return count; } @@ -164,6 +166,7 @@ public class ChunkedInputStream extends InputStream { * @see java.io.InputStream#read(byte[]) * @throws IOException if an IO problem occurs. */ + @Override public int read(byte[] b) throws IOException { return read(b, 0, b.length); } @@ -173,9 +176,9 @@ public class ChunkedInputStream extends InputStream { * @throws IOException If an IO error occurs. */ private void readCRLF() throws IOException { - int cr = in.read(); + int cr = this.in.read(); if (cr != '\r') throw new IOException("CRLF expected at end of chunk: cr != " + cr); - int lf = in.read(); + int lf = this.in.read(); if (lf != '\n') throw new IOException("CRLF expected at end of chunk: lf != " + lf); } @@ -185,12 +188,12 @@ public class ChunkedInputStream extends InputStream { * @throws IOException If an IO error occurs. */ private void nextChunk() throws IOException { - if (!bof) readCRLF(); - chunkSize = getChunkSizeFromInputStream(in); - bof = false; - pos = 0; - if (chunkSize == 0) { - eof = true; + if (!this.bof) readCRLF(); + this.chunkSize = getChunkSizeFromInputStream(this.in); + this.bof = false; + this.pos = 0; + if (this.chunkSize == 0) { + this.eof = true; skipTrailerHeaders(); } } @@ -203,24 +206,24 @@ public class ChunkedInputStream extends InputStream { * @param in The new input stream. * @param required true if a valid chunk must be present, * false otherwise. - * + * * @return the chunk size as integer - * + * * @throws IOException when the chunk size could not be parsed */ - private static int getChunkSizeFromInputStream(final InputStream in) + private static int getChunkSizeFromInputStream(final InputStream in) throws IOException { - + ByteArrayOutputStream baos = new ByteArrayOutputStream(); // States: 0=normal, 1=\r was scanned, 2=inside quoted string, -1=end - int state = 0; + int state = 0; while (state != -1) { int b = in.read(); - if (b == -1) { + if (b == -1) { throw new IOException("chunked stream ended unexpectedly"); } switch (state) { - case 0: + case 0: switch (b) { case '\r': state = 1; @@ -286,7 +289,7 @@ public class ChunkedInputStream extends InputStream { * * @param data the byte array to be encoded * @return The string representation of the byte array - * + * * @since 3.0 */ private static String getAsciiString(final byte[] data) throws IOException { @@ -300,14 +303,14 @@ public class ChunkedInputStream extends InputStream { throw new IOException("HttpClient requires ASCII support"); } } - + /** * Reads and stores the Trailer headers. * @throws IOException If an IO problem occurs */ private void skipTrailerHeaders() throws IOException { for (; ;) { - String line = readLine(in, "US-ASCII"); + String line = readLine(this.in, "US-ASCII"); if ((line == null) || (line.trim().length() < 1)) break; } } @@ -324,7 +327,7 @@ public class ChunkedInputStream extends InputStream { * * @throws IOException if an I/O problem occurs * @return a line from the stream - * + * * @since 3.0 */ private static String readLine(InputStream inputStream, String charset) throws IOException { @@ -348,7 +351,7 @@ public class ChunkedInputStream extends InputStream { final String result = getString(rawdata, 0, len - offset, charset); return result; } - + /** * Converts the byte array of HTTP content characters to a string. If @@ -357,16 +360,16 @@ public class ChunkedInputStream extends InputStream { * * @param data the byte array to be encoded * @param offset the index of the first byte to encode - * @param length the number of bytes to encode + * @param length the number of bytes to encode * @param charset the desired character encoding * @return The result of the conversion. - * + * * @since 3.0 */ private static String getString( - final byte[] data, - int offset, - int length, + final byte[] data, + int offset, + int length, String charset ) { @@ -384,12 +387,12 @@ public class ChunkedInputStream extends InputStream { return new String(data, offset, length); } } - + /** * Return byte array from an (unchunked) input stream. - * Stop reading when "\n" terminator encountered + * Stop reading when "\n" terminator encountered * If the stream ends before the line terminator is found, - * the last part of the string will still be returned. + * the last part of the string will still be returned. * If no input data available, null is returned. * * @param inputStream the stream to read from @@ -412,22 +415,23 @@ public class ChunkedInputStream extends InputStream { } return buf.toByteArray(); } - + /** * Upon close, this reads the remainder of the chunked message, * leaving the underlying socket at a position to start reading the * next response without scanning. * @throws IOException If an IO problem occurs. */ - public void close() throws IOException { - if (!closed) { + @Override + public synchronized void close() throws IOException { + if (!this.closed) { try { - if (!eof) { + if (!this.eof) { exhaustInputStream(this); } } finally { - eof = true; - closed = true; + this.eof = true; + this.closed = true; } } } diff --git a/source/de/anomic/http/server/ChunkedOutputStream.java b/source/de/anomic/http/server/ChunkedOutputStream.java index cadba14da..6ce9022a2 100644 --- a/source/de/anomic/http/server/ChunkedOutputStream.java +++ b/source/de/anomic/http/server/ChunkedOutputStream.java @@ -1,4 +1,4 @@ -//httpChunkedOutputStream.java +//httpChunkedOutputStream.java //------------------------------------- //part of YACY //(C) by Michael Peter Christen; mc@yacy.net @@ -33,21 +33,21 @@ import net.yacy.cora.document.ASCII; import net.yacy.cora.document.UTF8; import net.yacy.kelondro.util.ByteBuffer; import net.yacy.kelondro.util.FileUtils; - import de.anomic.server.serverCore; public final class ChunkedOutputStream extends FilterOutputStream { - private boolean finished = false; - + private boolean finished = false; + public ChunkedOutputStream(final OutputStream out) { super(out); } - - public void close() throws IOException { + + @Override + public synchronized void close() throws IOException { if (!this.finished) this.finish(); this.out.close(); } - + public void finish() throws IOException { if (!this.finished) { this.out.write((byte) 48); @@ -57,55 +57,58 @@ public final class ChunkedOutputStream extends FilterOutputStream { this.finished = true; } } - + + @Override public void write(final byte[] b) throws IOException { - if (this.finished) throw new IOException("ChunkedOutputStream already finalized."); + if (this.finished) throw new IOException("ChunkedOutputStream already finalized."); if (b.length == 0) return; - + this.out.write(ASCII.getBytes(Integer.toHexString(b.length))); this.out.write(serverCore.CRLF); this.out.write(b); this.out.write(serverCore.CRLF); this.out.flush(); } - + + @Override public void write(final byte[] b, final int off, final int len) throws IOException { if (this.finished) throw new IOException("ChunkedOutputStream already finalized."); if (len == 0) return; - + this.out.write(ASCII.getBytes(Integer.toHexString(len))); this.out.write(serverCore.CRLF); this.out.write(b, off, len); this.out.write(serverCore.CRLF); this.out.flush(); } - + public void write(final ByteBuffer b, final int off, final int len) throws IOException { if (this.finished) throw new IOException("ChunkedOutputStream already finalized."); if (len == 0) return; - + this.out.write(ASCII.getBytes(Integer.toHexString(len))); this.out.write(serverCore.CRLF); this.out.write(b.getBytes(off, len)); this.out.write(serverCore.CRLF); this.out.flush(); } - + public void write(final InputStream b) throws IOException { if (this.finished) throw new IOException("ChunkedOutputStream already finalized."); final int len = b.available(); if (len == 0) return; - + this.out.write(ASCII.getBytes(Integer.toHexString(len))); this.out.write(serverCore.CRLF); - FileUtils.copy(b, out, len); + FileUtils.copy(b, this.out, len); this.out.write(serverCore.CRLF); this.out.flush(); } - + + @Override public void write(final int b) throws IOException { if (this.finished) throw new IOException("ChunkedOutputStream already finalized."); - + this.out.write(UTF8.getBytes("1")); this.out.write(serverCore.CRLF); this.out.write(b); diff --git a/source/de/anomic/http/server/ContentLengthInputStream.java b/source/de/anomic/http/server/ContentLengthInputStream.java index 3332065f9..d15f575ff 100644 --- a/source/de/anomic/http/server/ContentLengthInputStream.java +++ b/source/de/anomic/http/server/ContentLengthInputStream.java @@ -62,12 +62,12 @@ import java.io.InputStream; * @since 2.0 */ public class ContentLengthInputStream extends InputStream { - + /** * The maximum number of bytes that can be read from the stream. Subsequent * read operations will return -1. */ - private long contentLength; + private final long contentLength; /** The current position */ private long pos = 0; @@ -86,7 +86,7 @@ public class ContentLengthInputStream extends InputStream { * @param in The stream to wrap * @param contentLength The maximum number of bytes that can be read from * the stream. Subsequent read operations will return -1. - * + * * @since 3.0 */ public ContentLengthInputStream(InputStream in, long contentLength) { @@ -102,14 +102,15 @@ public class ContentLengthInputStream extends InputStream { * primed to parse the next response.

* @throws IOException If an IO problem occurs. */ - public void close() throws IOException { - if (!closed) { + @Override + public synchronized void close() throws IOException { + if (!this.closed) { try { ChunkedInputStream.exhaustInputStream(this); } finally { // close after above so that we don't throw an exception trying // to read after closed! - closed = true; + this.closed = true; } } } @@ -121,15 +122,16 @@ public class ContentLengthInputStream extends InputStream { * @throws IOException If an IO problem occurs * @see java.io.InputStream#read() */ + @Override public int read() throws IOException { - if (closed) { + if (this.closed) { throw new IOException("Attempted read from closed stream."); } - if (pos >= contentLength) { + if (this.pos >= this.contentLength) { return -1; } - pos++; + this.pos++; return this.wrappedStream.read(); } @@ -145,20 +147,21 @@ public class ContentLengthInputStream extends InputStream { * * @throws java.io.IOException Should an error occur on the wrapped stream. */ + @Override public int read (byte[] b, int off, int len) throws java.io.IOException { - if (closed) { + if (this.closed) { throw new IOException("Attempted read from closed stream."); } - if (pos >= contentLength) { + if (this.pos >= this.contentLength) { return -1; } - if (pos + len > contentLength) { - len = (int) (contentLength - pos); + if (this.pos + len > this.contentLength) { + len = (int) (this.contentLength - this.pos); } int count = this.wrappedStream.read(b, off, len); - pos += count; + this.pos += count; return count; } @@ -170,6 +173,7 @@ public class ContentLengthInputStream extends InputStream { * @throws IOException If an IO problem occurs * @see java.io.InputStream#read(byte[]) */ + @Override public int read(byte[] b) throws IOException { return read(b, 0, b.length); } @@ -182,20 +186,22 @@ public class ContentLengthInputStream extends InputStream { * @throws IOException If an error occurs while skipping bytes. * @see InputStream#skip(long) */ + @Override public long skip(long n) throws IOException { - // make sure we don't skip more bytes than are + // make sure we don't skip more bytes than are // still available - long length = Math.min(n, contentLength - pos); + long length = Math.min(n, this.contentLength - this.pos); // skip and keep track of the bytes actually skipped length = this.wrappedStream.skip(length); // only add the skipped bytes to the current position // if bytes were actually skipped if (length > 0) { - pos += length; + this.pos += length; } return length; } + @Override public int available() throws IOException { if (this.closed) { return 0; @@ -204,7 +210,7 @@ public class ContentLengthInputStream extends InputStream { if (this.pos + avail > this.contentLength ) { avail = (int)(this.contentLength - this.pos); } - return avail; + return avail; } - + } diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index 557ceb53e..da4a13241 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -560,7 +560,7 @@ public final class serverCore extends AbstractBusyThread implements BusyThread { return this.stopped; } - public void close() { + public synchronized void close() { // closing the socket to the client if (this.controlSocket != null) try { this.controlSocket.close(); diff --git a/source/net/yacy/cora/lod/TripleStore.java b/source/net/yacy/cora/lod/TripleStore.java index f39bcd4c1..ec48ee894 100644 --- a/source/net/yacy/cora/lod/TripleStore.java +++ b/source/net/yacy/cora/lod/TripleStore.java @@ -117,7 +117,7 @@ public class TripleStore { return this.store.keyIterator(); } - public void close() { + public synchronized void close() { this.store.close(); } diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index 57d5f2b15..1bf1321e3 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -50,6 +50,7 @@ import net.yacy.cora.plugin.ClassProvider; import net.yacy.cora.storage.ARC; import net.yacy.cora.storage.ConcurrentARC; import net.yacy.cora.storage.KeyList; +import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.MemoryControl; public class Domains { @@ -468,8 +469,8 @@ public class Domains { noLocalCheck = v; } - public static void close() { - if (globalHosts != null) try {globalHosts.close();} catch (final IOException e) {} + public static synchronized void close() { + if (globalHosts != null) try {globalHosts.close();} catch (final IOException e) {Log.logException(e);} } /** diff --git a/source/net/yacy/cora/services/federated/solr/SolrRetryConnector.java b/source/net/yacy/cora/services/federated/solr/SolrRetryConnector.java index fe9b1fce7..607bea828 100644 --- a/source/net/yacy/cora/services/federated/solr/SolrRetryConnector.java +++ b/source/net/yacy/cora/services/federated/solr/SolrRetryConnector.java @@ -44,7 +44,7 @@ public class SolrRetryConnector implements SolrConnector { } @Override - public void close() { + public synchronized void close() { this.solrConnector.close(); } diff --git a/source/net/yacy/cora/services/federated/solr/SolrShardingConnector.java b/source/net/yacy/cora/services/federated/solr/SolrShardingConnector.java index df4f82f1c..7e4103b44 100644 --- a/source/net/yacy/cora/services/federated/solr/SolrShardingConnector.java +++ b/source/net/yacy/cora/services/federated/solr/SolrShardingConnector.java @@ -57,7 +57,7 @@ public class SolrShardingConnector implements SolrConnector { } @Override - public void close() { + public synchronized void close() { for (final SolrConnector connector: this.connectors) connector.close(); } diff --git a/source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java b/source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java index d2f746c15..6a2261514 100644 --- a/source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java +++ b/source/net/yacy/cora/services/federated/solr/SolrSingleConnector.java @@ -115,7 +115,7 @@ public class SolrSingleConnector implements SolrConnector { } @Override - public void close() { + public synchronized void close() { try { this.server.commit(); } catch (SolrServerException e) { @@ -205,6 +205,7 @@ public class SolrSingleConnector implements SolrConnector { } } + @Override public void add(final Collection solrdocs) throws IOException, SolrException { ArrayList l = new ArrayList(); for (SolrDoc d: solrdocs) l.add(d); diff --git a/source/net/yacy/cora/storage/KeyList.java b/source/net/yacy/cora/storage/KeyList.java index 5dadf30b1..9fd0b6588 100644 --- a/source/net/yacy/cora/storage/KeyList.java +++ b/source/net/yacy/cora/storage/KeyList.java @@ -91,7 +91,7 @@ public class KeyList implements Iterable { } } - public void close() throws IOException { + public synchronized void close() throws IOException { synchronized (this.raf) { this.raf.close(); } diff --git a/source/net/yacy/cora/storage/ZIPReader.java b/source/net/yacy/cora/storage/ZIPReader.java new file mode 100644 index 000000000..ae31752ed --- /dev/null +++ b/source/net/yacy/cora/storage/ZIPReader.java @@ -0,0 +1,91 @@ +package net.yacy.cora.storage; + +import java.io.File; +import java.io.IOException; +import java.util.AbstractMap; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +public class ZIPReader extends AbstractMap implements Map, Iterable> { + + private final Set filenames; + private final ZipFile zipFile; + + public ZIPReader(File file) throws IOException { + super(); + if (!file.exists()) throw new IOException("ZIPWriter can only be used for existing files"); + this.zipFile = new ZipFile(file); + + // read all entries + this.filenames = new HashSet(); + final Enumeration e = this.zipFile.entries(); + while (e.hasMoreElements()) { + ZipEntry z = e.nextElement(); + this.filenames.add(z.getName()); + } + } + + @Override + public Iterator> iterator() { + final Enumeration e = this.zipFile.entries(); + return new Iterator>() { + + @Override + public boolean hasNext() { + return e.hasMoreElements(); + } + + @Override + public java.util.Map.Entry next() { + ZipEntry z = e.nextElement(); + return new AbstractMap.SimpleImmutableEntry(z.getName(), z); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + }; + } + + @Override + public int size() { + return this.zipFile.size(); + } + + @Override + public boolean isEmpty() { + return this.zipFile.size() == 0; + } + + @Override + public boolean containsKey(Object key) { + return this.filenames.contains(key); + } + + @Override + public ZipEntry get(Object key) { + return this.zipFile.getEntry((String) key); + } + + @Override + public Set keySet() { + return this.filenames; + } + + @Override + public Set> entrySet() { + throw new UnsupportedOperationException(); + } + + public void close() throws IOException { + this.zipFile.close(); + } + +} diff --git a/source/net/yacy/cora/storage/ZIPWriter.java b/source/net/yacy/cora/storage/ZIPWriter.java new file mode 100644 index 000000000..6164e97ca --- /dev/null +++ b/source/net/yacy/cora/storage/ZIPWriter.java @@ -0,0 +1,62 @@ +package net.yacy.cora.storage; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class ZIPWriter extends AbstractMap implements Map, Iterable> { + + private final HashMap backup; + private final ZipOutputStream zos; + + public ZIPWriter(File file) throws IOException { + super(); + if (file.exists()) throw new IOException("ZIPWriter can only be used for new files"); + this.backup = new HashMap(); + this.zos = new ZipOutputStream(new FileOutputStream(file)); + } + + @Override + public ZipEntry put(String key, ZipEntry value) { + assert !this.backup.containsKey(key); + try { + this.zos.putNextEntry(value); + this.backup.put(key, value); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public ZipEntry get(Object key) { + return this.backup.get(key); + } + + @Override + public Iterator> iterator() { + return this.backup.entrySet().iterator(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + @Override + public Set> entrySet() { + return this.backup.entrySet(); + } + + public void close() throws IOException { + this.zos.close(); + } + +} diff --git a/source/net/yacy/document/Document.java b/source/net/yacy/document/Document.java index 8b519f903..c323a8be9 100644 --- a/source/net/yacy/document/Document.java +++ b/source/net/yacy/document/Document.java @@ -702,7 +702,7 @@ dc_rights } } - public void close() { + public synchronized void close() { if (this.text == null) return; // try close the output stream diff --git a/source/net/yacy/document/SentenceReader.java b/source/net/yacy/document/SentenceReader.java index 8f8a2fb37..9878a9fb7 100644 --- a/source/net/yacy/document/SentenceReader.java +++ b/source/net/yacy/document/SentenceReader.java @@ -142,7 +142,7 @@ public class SentenceReader implements Iterator { throw new UnsupportedOperationException(); } - public void close() { + public synchronized void close() { try { raf.close(); } catch(IOException ioe) { diff --git a/source/net/yacy/document/WordTokenizer.java b/source/net/yacy/document/WordTokenizer.java index 6dc30bb75..4f5db3833 100644 --- a/source/net/yacy/document/WordTokenizer.java +++ b/source/net/yacy/document/WordTokenizer.java @@ -82,7 +82,7 @@ public class WordTokenizer implements Enumeration { return r; } - public void close() { + public synchronized void close() { this.e.close(); } @@ -153,7 +153,7 @@ public class WordTokenizer implements Enumeration { return r; } - public void close() { + public synchronized void close() { this.e.close(); } } diff --git a/source/net/yacy/document/content/dao/DatabaseConnection.java b/source/net/yacy/document/content/dao/DatabaseConnection.java index 6cfadca26..dddb392f6 100644 --- a/source/net/yacy/document/content/dao/DatabaseConnection.java +++ b/source/net/yacy/document/content/dao/DatabaseConnection.java @@ -85,7 +85,7 @@ public class DatabaseConnection { } } - public void close() { + public synchronized void close() { if (connection != null) { try { connection.close(); diff --git a/source/net/yacy/document/content/dao/ImportDump.java b/source/net/yacy/document/content/dao/ImportDump.java index 1b22248d4..4bb67575d 100644 --- a/source/net/yacy/document/content/dao/ImportDump.java +++ b/source/net/yacy/document/content/dao/ImportDump.java @@ -100,7 +100,7 @@ public class ImportDump { close(); } - public void close() { + public synchronized void close() { this.conn.close(); } diff --git a/source/net/yacy/document/content/dao/PhpBB3Dao.java b/source/net/yacy/document/content/dao/PhpBB3Dao.java index 33fe846ef..3c5fbbcf7 100644 --- a/source/net/yacy/document/content/dao/PhpBB3Dao.java +++ b/source/net/yacy/document/content/dao/PhpBB3Dao.java @@ -7,7 +7,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -65,17 +65,19 @@ public class PhpBB3Dao implements Dao { this.prefix = prefix; this.users = new HashMap(); } - + + @Override protected void finalize() throws Throwable { close(); } - + + @Override public Date first() { Statement stmt = null; ResultSet rs = null; try { - stmt = conn.statement(); - rs = stmt.executeQuery("select min(post_time) from " + prefix + "posts"); + stmt = this.conn.statement(); + rs = stmt.executeQuery("select min(post_time) from " + this.prefix + "posts"); if (rs.next()) { return new Date(rs.getLong(1) * 1000L); } @@ -89,12 +91,13 @@ public class PhpBB3Dao implements Dao { } } + @Override public Date latest() { Statement stmt = null; ResultSet rs = null; try { - stmt = conn.statement(); - rs = stmt.executeQuery("select max(post_time) from " + prefix + "posts"); + stmt = this.conn.statement(); + rs = stmt.executeQuery("select max(post_time) from " + this.prefix + "posts"); if (rs.next()) { return new Date(rs.getLong(1) * 1000L); } @@ -108,18 +111,21 @@ public class PhpBB3Dao implements Dao { } } + @Override public int size() throws SQLException { - return this.conn.count(prefix + "posts"); + return this.conn.count(this.prefix + "posts"); } + @Override public DCEntry get(int item) { - return getOne("select * from " + prefix + "posts where post_id = " + item); + return getOne("select * from " + this.prefix + "posts where post_id = " + item); } - + + @Override public BlockingQueue query(int from, int until, int queueSize) { // define the sql query final StringBuilder sql = new StringBuilder(256); - sql.append("select * from " + prefix + "posts where post_id >= "); + sql.append("select * from " + this.prefix + "posts where post_id >= "); sql.append(from); if (until > from) { sql.append(" and post_id < "); @@ -130,24 +136,25 @@ public class PhpBB3Dao implements Dao { // execute the query and push entries to a queue concurrently return toQueue(sql, queueSize); } - + + @Override public BlockingQueue query(Date from, int queueSize) { // define the sql query final StringBuilder sql = new StringBuilder(256); - sql.append("select * from " + prefix + "posts where post_time >= "); + sql.append("select * from " + this.prefix + "posts where post_time >= "); sql.append(from.getTime() / 1000); sql.append(" order by post_id"); // execute the query and push entries to a queue concurrently return toQueue(sql, queueSize); } - - + + private DCEntry getOne(String sql) { Statement stmt = null; ResultSet rs = null; try { - stmt = conn.statement(); + stmt = this.conn.statement(); rs = stmt.executeQuery(sql); if (rs.next()) { try { @@ -165,16 +172,17 @@ public class PhpBB3Dao implements Dao { if (stmt != null) try {stmt.close();} catch (SQLException e) {} } } - + private BlockingQueue toQueue(final StringBuilder sql, int queueSize) { // execute the query and push entries to a queue concurrently final BlockingQueue queue = new ArrayBlockingQueue(queueSize); Thread dbreader = new Thread() { + @Override public void run() { Statement stmt = null; ResultSet rs = null; try { - stmt = conn.statement(); + stmt = PhpBB3Dao.this.conn.statement(); rs = stmt.executeQuery(sql.toString()); while (rs.next()) { try { @@ -197,7 +205,7 @@ public class PhpBB3Dao implements Dao { dbreader.start(); return queue; } - + protected DCEntry parseResultSet(ResultSet rs) throws SQLException, MalformedURLException { DigestURI url; int item = rs.getInt("post_id"); @@ -208,7 +216,7 @@ public class PhpBB3Dao implements Dao { Date date = new Date(rs.getLong("post_time") * 1000L); return new DCEntry(url, date, subject, user, text, 0.0f, 0.0f); } - + public static String xmlCleaner(String s) { if (s == null) return null; @@ -217,10 +225,10 @@ public class PhpBB3Dao implements Dao { for (int i = 0; i < s.length(); i++ ) { c = s.charAt(i); - if ((c >= 0x0020 && c <= 0xD7FF) || + if ((c >= 0x0020 && c <= 0xD7FF) || (c >= 0xE000 && c <= 0xFFFD) || c == 0x0009 || - c == 0x000A || + c == 0x000A || c == 0x000D ) { sbOutput.append(c); } @@ -231,14 +239,14 @@ public class PhpBB3Dao implements Dao { private String getUser(int poster_id) { String nick = this.users.get(poster_id); if (nick != null) return nick; - + StringBuilder sql = new StringBuilder(256); - sql.append("select * from " + prefix + "users where user_id = "); + sql.append("select * from " + this.prefix + "users where user_id = "); sql.append(poster_id); Statement stmt = null; ResultSet rs = null; try { - stmt = conn.statement(); + stmt = this.conn.statement(); rs = stmt.executeQuery(sql.toString()); if (rs.next()) nick = rs.getString("username"); if (nick == null) nick = ""; @@ -252,7 +260,8 @@ public class PhpBB3Dao implements Dao { if (stmt != null) try {stmt.close();} catch (SQLException e) {} } } - + + @Override public int writeSurrogates( BlockingQueue queue, File targetdir, @@ -264,7 +273,7 @@ public class PhpBB3Dao implements Dao { String targethost = new DigestURI(this.urlstub).getHost(); int fc = 0; File outputfiletmp = null, outputfile = null; - + // write the result from the query concurrently in a file OutputStreamWriter osw = null; DCEntry e; @@ -304,11 +313,12 @@ public class PhpBB3Dao implements Dao { } return 0; } - - public void close() { + + @Override + public synchronized void close() { this.conn.close(); } - + public static void main(String[] args) { PhpBB3Dao db; try { @@ -331,5 +341,5 @@ public class PhpBB3Dao implements Dao { Log.logException(e); } } - + } diff --git a/source/net/yacy/document/importer/MediawikiImporter.java b/source/net/yacy/document/importer/MediawikiImporter.java index d45657a19..1cf6f2962 100644 --- a/source/net/yacy/document/importer/MediawikiImporter.java +++ b/source/net/yacy/document/importer/MediawikiImporter.java @@ -564,7 +564,7 @@ public class MediawikiImporter extends Thread implements Importer { return this.bb.getBytes(); } - public void close() { + public synchronized void close() { try { this.is.close(); } catch (final IOException e) { diff --git a/source/net/yacy/document/parser/dwgParser.java b/source/net/yacy/document/parser/dwgParser.java new file mode 100644 index 000000000..f5d58ce42 --- /dev/null +++ b/source/net/yacy/document/parser/dwgParser.java @@ -0,0 +1,390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * this parser was copied and modified to fit into YaCy from the apache tika project + */ + + +package net.yacy.document.parser; + + +import java.io.InputStream; + +import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.document.AbstractParser; +import net.yacy.document.Document; +import net.yacy.document.Parser; +import net.yacy.kelondro.util.MemoryControl; + +import org.apache.poi.util.StringUtil; + + +public class dwgParser extends AbstractParser implements Parser { + + + private static final String HEADER_2000_PROPERTIES_MARKER_STR = "DWGPROPS COOKIE"; + private static final byte[] HEADER_2000_PROPERTIES_MARKER = new byte[HEADER_2000_PROPERTIES_MARKER_STR.length()]; + + static { + StringUtil.putCompressedUnicode( + HEADER_2000_PROPERTIES_MARKER_STR, + HEADER_2000_PROPERTIES_MARKER, 0); + } + + /** + * How far to skip after the last standard property, before + * we find any custom properties that might be there. + */ + private static final int CUSTOM_PROPERTIES_SKIP = 20; + + public dwgParser() { + super("DWG (CAD Drawing) parser (very basic)"); + this.SUPPORTED_EXTENSIONS.add("dwg"); + this.SUPPORTED_MIME_TYPES.add("application/dwg"); + this.SUPPORTED_MIME_TYPES.add("applications/vnd.dwg"); + } + + @Override + public Document[] parse(final MultiProtocolURI location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { + + // check memory for parser + if (!MemoryControl.request(200 * 1024 * 1024, true)) + throw new Parser.Failure("Not enough Memory available for pdf parser: " + MemoryControl.available(), location); + return null; + // First up, which version of the format are we handling? + /* + byte[] header = new byte[128]; + IOUtils.readFully(source, header); + String version = new String(header, 0, 6, "US-ASCII"); + XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); + xhtml.startDocument(); + + if (version.equals("AC1015")) { + metadata.set(Metadata.CONTENT_TYPE, TYPE.toString()); + if (skipTo2000PropertyInfoSection(stream, header)) { + get2000Props(stream,metadata,xhtml); + } + } else if (version.equals("AC1018")) { + metadata.set(Metadata.CONTENT_TYPE, TYPE.toString()); + if (skipToPropertyInfoSection(stream, header)) { + get2004Props(stream,metadata,xhtml); + } + } else if (version.equals("AC1021") || version.equals("AC1024")) { + metadata.set(Metadata.CONTENT_TYPE, TYPE.toString()); + if (skipToPropertyInfoSection(stream, header)) { + get2007and2010Props(stream,metadata,xhtml); + } + } else { + throw new TikaException( + "Unsupported AutoCAD drawing version: " + version); + } + + xhtml.endDocument(); + + + String docTitle = null, docSubject = null, docAuthor = null, docPublisher = null, docKeywordStr = null; + if (info != null) { + docTitle = info.getTitle(); + docSubject = info.getSubject(); + docAuthor = info.getAuthor(); + docPublisher = info.getProducer(); + if (docPublisher == null || docPublisher.length() == 0) docPublisher = info.getCreator(); + docKeywordStr = info.getKeywords(); + } + + if (docTitle == null || docTitle.length() == 0) { + docTitle = MultiProtocolURI.unescape(location.getFileName()); + } + + String[] docKeywords = null; + if (docKeywordStr != null) { + docKeywords = docKeywordStr.split(" |,"); + } + if (docTitle == null) { + docTitle = docSubject; + } + + byte[] contentBytes; + + return new Document[]{new Document( + location, + mimeType, + "UTF-8", + this, + null, + docKeywords, + docTitle, + docAuthor, + docPublisher, + null, + null, + 0.0f, 0.0f, + contentBytes, + null, + null, + null, + false)}; + */ + } + + /* + private void get2004Props( + InputStream stream, Metadata metadata, XHTMLContentHandler xhtml) + throws IOException, TikaException, SAXException { + // Standard properties + for (int i = 0; i < HEADER_PROPERTIES_ENTRIES.length; i++) { + String headerValue = read2004String(stream); + handleHeader(i, headerValue, metadata, xhtml); + } + + // Custom properties + int customCount = skipToCustomProperties(stream); + for (int i = 0; i < customCount; i++) { + String propName = read2004String(stream); + String propValue = read2004String(stream); + if(propName.length() > 0 && propValue.length() > 0) { + metadata.add(propName, propValue); + } + } + } + + private String read2004String(InputStream stream) throws IOException, TikaException { + int stringLen = EndianUtils.readUShortLE(stream); + + byte[] stringData = new byte[stringLen]; + IOUtils.readFully(stream, stringData); + + // Often but not always null terminated + if (stringData[stringLen-1] == 0) { + stringLen--; + } + String value = StringUtil.getFromCompressedUnicode(stringData, 0, stringLen); + return value; + } + + // Stored as UCS2, so 16 bit "unicode" + private void get2007and2010Props( + InputStream stream, Metadata metadata, XHTMLContentHandler xhtml) + throws IOException, TikaException, SAXException { + // Standard properties + for (int i = 0; i < HEADER_PROPERTIES_ENTRIES.length; i++) { + String headerValue = read2007and2010String(stream); + handleHeader(i, headerValue, metadata, xhtml); + } + + // Custom properties + int customCount = skipToCustomProperties(stream); + for (int i = 0; i < customCount; i++) { + String propName = read2007and2010String(stream); + String propValue = read2007and2010String(stream); + if(propName.length() > 0 && propValue.length() > 0) { + metadata.add(propName, propValue); + } + } + } + + private String read2007and2010String(InputStream stream) throws IOException, TikaException { + int stringLen = EndianUtils.readUShortLE(stream); + + byte[] stringData = new byte[stringLen * 2]; + IOUtils.readFully(stream, stringData); + String value = StringUtil.getFromUnicodeLE(stringData); + + // Some strings are null terminated + if(value.charAt(value.length()-1) == 0) { + value = value.substring(0, value.length()-1); + } + + return value; + } + + private void get2000Props( + InputStream stream, Metadata metadata, XHTMLContentHandler xhtml) + throws IOException, TikaException, SAXException { + int propCount = 0; + while(propCount < 30) { + int propIdx = EndianUtils.readUShortLE(stream); + int length = EndianUtils.readUShortLE(stream); + int valueType = stream.read(); + + if(propIdx == 0x28) { + // This one seems not to follow the pattern + length = 0x19; + } else if(propIdx == 90) { + // We think this means the end of properties + break; + } + + byte[] value = new byte[length]; + IOUtils.readFully(stream, value); + if(valueType == 0x1e) { + // Normal string, good + String val = StringUtil.getFromCompressedUnicode(value, 0, length); + + // Is it one we can look up by index? + if(propIdx < HEADER_2000_PROPERTIES_ENTRIES.length) { + metadata.add(HEADER_2000_PROPERTIES_ENTRIES[propIdx], val); + xhtml.element("p", val); + } else if(propIdx == 0x012c) { + int splitAt = val.indexOf('='); + if(splitAt > -1) { + String propName = val.substring(0, splitAt); + String propVal = val.substring(splitAt+1); + metadata.add(propName, propVal); + } + } + } else { + // No idea... + } + + propCount++; + } + } + + private void handleHeader( + int headerNumber, String value, Metadata metadata, + XHTMLContentHandler xhtml) throws SAXException { + if(value == null || value.length() == 0) { + return; + } + + String headerProp = HEADER_PROPERTIES_ENTRIES[headerNumber]; + if(headerProp != null) { + metadata.set(headerProp, value); + } + + xhtml.element("p", value); + } + + // Grab the offset, then skip there + private boolean skipToPropertyInfoSection(InputStream stream, byte[] header) + throws IOException, TikaException { + // The offset is stored in the header from 0x20 onwards + long offsetToSection = EndianUtils.getLongLE(header, 0x20); + long toSkip = offsetToSection - header.length; + if(offsetToSection == 0){ + return false; + } + while (toSkip > 0) { + byte[] skip = new byte[Math.min((int) toSkip, 0x4000)]; + IOUtils.readFully(stream, skip); + toSkip -= skip.length; + } + return true; + } + + //We think it can be anywhere... + private boolean skipTo2000PropertyInfoSection(InputStream stream, byte[] header) + throws IOException { + int val = 0; + while(val != -1) { + val = stream.read(); + if(val == HEADER_2000_PROPERTIES_MARKER[0]) { + boolean going = true; + for(int i=1; i 0 && count < 0x7f) { + // Looks plausible + return count; + } else { + // No properties / count is too high to trust + return 0; + } + } else { + // No padding. That probably means no custom props + return 0; + } + } + + public static void main(final String[] args) { + if (args.length > 0 && args[0].length() > 0) { + // file + final File dwgFile = new File(args[0]); + if(dwgFile.canRead()) { + + System.out.println(dwgFile.getAbsolutePath()); + final long startTime = System.currentTimeMillis(); + + // parse + final AbstractParser parser = new dwgParser(); + Document document = null; + try { + document = Document.mergeDocuments(null, "application/dwg", parser.parse(null, "application/dwg", null, new FileInputStream(dwgFile))); + } catch (final Parser.Failure e) { + System.err.println("Cannot parse file " + dwgFile.getAbsolutePath()); + Log.logException(e); + } catch (final InterruptedException e) { + System.err.println("Interrupted while parsing!"); + Log.logException(e); + } catch (final NoClassDefFoundError e) { + System.err.println("class not found: " + e.getMessage()); + } catch (final FileNotFoundException e) { + Log.logException(e); + } + + // statistics + System.out.println("\ttime elapsed: " + (System.currentTimeMillis() - startTime) + " ms"); + + // output + if (document == null) { + System.out.println("\t!!!Parsing without result!!!"); + } else { + System.out.println("\tParsed text with " + document.getTextLength() + " chars of text and " + document.getAnchors().size() + " anchors"); + try { + // write file + FileUtils.copy(document.getText(), new File("parsedPdf.txt")); + } catch (final IOException e) { + System.err.println("error saving parsed document"); + Log.logException(e); + } + } + } else { + System.err.println("Cannot read file "+ dwgFile.getAbsolutePath()); + } + } else { + System.out.println("Please give a filename as first argument."); + } + } +*/ +} diff --git a/source/net/yacy/document/parser/html/AbstractTransformer.java b/source/net/yacy/document/parser/html/AbstractTransformer.java index 20fefbdb8..d812606c9 100644 --- a/source/net/yacy/document/parser/html/AbstractTransformer.java +++ b/source/net/yacy/document/parser/html/AbstractTransformer.java @@ -1,4 +1,4 @@ -// AbstractTransformer.java +// AbstractTransformer.java // ---------------------------------- // (C) by Michael Peter Christen; mc@yacy.net // first published on http://www.anomic.de @@ -37,15 +37,18 @@ public abstract class AbstractTransformer implements Transformer { this.tags1 = tags1; } + @Override public boolean isTag0(final String tag) { - return tags0.contains(tag); + return this.tags0.contains(tag); } + @Override public boolean isTag1(final String tag) { - return tags1.contains(tag); + return this.tags1.contains(tag); } //the 'missing' method that shall be implemented: + @Override public abstract char[] transformText(char[] text); /* could be easily implemented as: { @@ -54,18 +57,21 @@ public abstract class AbstractTransformer implements Transformer { */ // the other methods must take into account to construct the return value correctly + @Override public char[] transformTag0(final String tagname, final Properties tagopts, final char quotechar) { return TransformerWriter.genTag0(tagname, tagopts, quotechar); } + @Override public char[] transformTag1(final String tagname, final Properties tagopts, final char[] text, final char quotechar) { return TransformerWriter.genTag1(tagname, tagopts, text, quotechar); } - - public void close() { + + @Override + public synchronized void close() { // free resources - tags0 = null; - tags1 = null; + this.tags0 = null; + this.tags1 = null; } - + } diff --git a/source/net/yacy/document/parser/html/ContentTransformer.java b/source/net/yacy/document/parser/html/ContentTransformer.java index ce4679676..957202eb6 100644 --- a/source/net/yacy/document/parser/html/ContentTransformer.java +++ b/source/net/yacy/document/parser/html/ContentTransformer.java @@ -139,9 +139,9 @@ public class ContentTransformer extends AbstractTransformer implements Transform } @Override - public void close() { + public synchronized void close() { // free resources super.close(); } -} \ No newline at end of file +} diff --git a/source/net/yacy/document/parser/html/ScraperInputStream.java b/source/net/yacy/document/parser/html/ScraperInputStream.java index 8c3fa454d..3a3d4f9ab 100644 --- a/source/net/yacy/document/parser/html/ScraperInputStream.java +++ b/source/net/yacy/document/parser/html/ScraperInputStream.java @@ -170,7 +170,7 @@ public class ScraperInputStream extends InputStream implements ScraperListener { } @Override - public void close() throws IOException { + public synchronized void close() throws IOException { if (this.writer != null) this.writer.close(); } diff --git a/source/net/yacy/kelondro/blob/BEncodedHeap.java b/source/net/yacy/kelondro/blob/BEncodedHeap.java index 31e4fa6db..b9cd2986c 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeap.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeap.java @@ -62,7 +62,7 @@ public class BEncodedHeap implements MapStore { /** * produce or open a properties table - * + * * @param location the file * @param keylength length of access keys * @param ordering ordering on the keys @@ -80,7 +80,7 @@ public class BEncodedHeap implements MapStore { /** * convenience method to open a properies table - * + * * @param location the file * @param keylength length of access keys */ @@ -120,11 +120,11 @@ public class BEncodedHeap implements MapStore { public CloneableIterator clone(Object modifier) { return this; } - + }; } } - + public byte[] encodedKey(final String key) { return Base64Order.enhancedCoder.encodeSubstring(Digest.encodeMD5Raw(key), this.table.keylength); } @@ -207,7 +207,7 @@ public class BEncodedHeap implements MapStore { /** * the map is stored inside a file; this method may return the file - * + * * @return the file where the map is stored */ public File getFile() { @@ -216,7 +216,7 @@ public class BEncodedHeap implements MapStore { /** * Retur the number of key-value mappings in this map. - * + * * @return the number of entries mappings in this map */ @Override @@ -234,7 +234,7 @@ public class BEncodedHeap implements MapStore { /** * check if a row with given key exists in the table - * + * * @param name * @return true if the row exists */ @@ -244,7 +244,7 @@ public class BEncodedHeap implements MapStore { /** * check if a row with given key exists in the table This method is here to implement the Map interface - * + * * @param name * @return true if the row exists */ @@ -267,7 +267,7 @@ public class BEncodedHeap implements MapStore { /** * get a map from the table - * + * * @param name * @return the map if one found or NULL if no entry exists or the entry is corrupt * @throws RowSpaceExceededException @@ -283,7 +283,7 @@ public class BEncodedHeap implements MapStore { /** * get a map from the table this method is here to implement the Map interface - * + * * @param name * @return the map if one found or NULL if no entry exists or the entry is corrupt */ @@ -305,7 +305,7 @@ public class BEncodedHeap implements MapStore { /** * convenience method to get a value from a map - * + * * @param pk * @param key * @return the value @@ -324,7 +324,7 @@ public class BEncodedHeap implements MapStore { /** * select all rows from a table where a given matcher matches with elements in a given row this method * makes a full-table scan of the whole table - * + * * @param columnName the name of the column where the matcher shall match * @param columnMatcher the matcher for the elements of the column * @return a set of primary keys where the matcher matched @@ -351,7 +351,7 @@ public class BEncodedHeap implements MapStore { /** * select one row from a table where a given matcher matches with elements in a given row this method * stops the full-table scan as soon as a first matcher was found - * + * * @param columnName the name of the column where the matcher shall match * @param columnMatcher the matcher for the elements of the column * @return the row where the matcher matched the given column @@ -379,7 +379,7 @@ public class BEncodedHeap implements MapStore { /** * insert a map into the table this method shall be used in exchange of the get method if the previous * entry value is not needed. - * + * * @param name * @param map * @throws RowSpaceExceededException @@ -427,7 +427,7 @@ public class BEncodedHeap implements MapStore { /** * insert a map into the table - * + * * @param name * @param map */ @@ -450,7 +450,7 @@ public class BEncodedHeap implements MapStore { /** * delete a map from the table - * + * * @param name * @throws IOException */ @@ -460,7 +460,7 @@ public class BEncodedHeap implements MapStore { /** * delete a map from the table - * + * * @param name * @throws RowSpaceExceededException * @throws IOException @@ -489,7 +489,7 @@ public class BEncodedHeap implements MapStore { /** * Copy all the mappings from the specified map to this map. - * + * * @param m mappings to be stored in this map */ @Override @@ -522,7 +522,8 @@ public class BEncodedHeap implements MapStore { * close the backen-file. Should be called explicitely to ensure that all data waiting in IO write buffers * are flushed */ - public void close() { + @Override + public synchronized void close() { int s = this.size(); File f = this.table.heapFile; this.table.close(); @@ -532,7 +533,7 @@ public class BEncodedHeap implements MapStore { /** * Return a Set of the keys contained in this map. This may not be a useful method, if possible use the * keys() method instead to iterate all keys from the backend-file - * + * * @return a set view of the keys contained in this map */ @Override @@ -550,7 +551,7 @@ public class BEncodedHeap implements MapStore { /** * iterate all keys of the table - * + * * @return an iterator of byte[] * @throws IOException */ @@ -562,7 +563,7 @@ public class BEncodedHeap implements MapStore { * the values() method is not implemented in this class because it does not make sense to use such a * method for file-based data structures. To get a collection view of all the entries, just use a entry * iterator instead. - * + * * @return nothing. The method throws always a UnsupportedOperationException */ @Override @@ -613,7 +614,7 @@ public class BEncodedHeap implements MapStore { /** * iterate all rows of the table. this is a static method that expects that the given file is not opened * by any other application - * + * * @param location * @param keylen * @return @@ -637,7 +638,7 @@ public class BEncodedHeap implements MapStore { * Produce a list of column names from this table This method may be useful if the table shall be * displayed as a table in GUIs. To show the first line of the table, the table header, a list of all * column names is required. This can be generated with this method - * + * * @return a list of column names */ public ArrayList columns() { diff --git a/source/net/yacy/kelondro/blob/BEncodedHeapShard.java b/source/net/yacy/kelondro/blob/BEncodedHeapShard.java index df0961f0a..40e1672cb 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeapShard.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeapShard.java @@ -154,7 +154,7 @@ public class BEncodedHeapShard extends AbstractMapStore implements MapStore { } @Override - public void close() { + public synchronized void close() { if (this.shard == null) return; final Iterator i = this.shard.values().iterator(); diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index 037b58c5d..2be59f361 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -803,8 +803,8 @@ public class HeapReader { } } - public void close() { - if (this.is != null) try { this.is.close(); } catch (final IOException e) {} + public synchronized void close() { + if (this.is != null) try { this.is.close(); } catch (final IOException e) {Log.logException(e);} this.is = null; } diff --git a/source/net/yacy/kelondro/blob/Tables.java b/source/net/yacy/kelondro/blob/Tables.java index 32db61977..aa459bbe3 100644 --- a/source/net/yacy/kelondro/blob/Tables.java +++ b/source/net/yacy/kelondro/blob/Tables.java @@ -99,7 +99,7 @@ public class Tables implements Iterable { heap.close(); } - public void close() { + public synchronized void close() { for (final BEncodedHeap heap: this.tables.values()) heap.close(); this.tables.clear(); } diff --git a/source/net/yacy/kelondro/index/BufferedObjectIndex.java b/source/net/yacy/kelondro/index/BufferedObjectIndex.java index 67a1f7ca1..38b418b1b 100644 --- a/source/net/yacy/kelondro/index/BufferedObjectIndex.java +++ b/source/net/yacy/kelondro/index/BufferedObjectIndex.java @@ -111,7 +111,7 @@ public class BufferedObjectIndex implements Index, Iterable { } @Override - public void close() { + public synchronized void close() { synchronized (this.backend) { try { flushBuffer(); diff --git a/source/net/yacy/kelondro/index/HandleMap.java b/source/net/yacy/kelondro/index/HandleMap.java index 8d4dd0863..42590aaee 100644 --- a/source/net/yacy/kelondro/index/HandleMap.java +++ b/source/net/yacy/kelondro/index/HandleMap.java @@ -434,7 +434,7 @@ public final class HandleMap implements Iterable { return this.map; } - public void close() { + public synchronized void close() { this.map.close(); } } diff --git a/source/net/yacy/kelondro/index/RAMIndexCluster.java b/source/net/yacy/kelondro/index/RAMIndexCluster.java index ea5d01eea..a70a00057 100644 --- a/source/net/yacy/kelondro/index/RAMIndexCluster.java +++ b/source/net/yacy/kelondro/index/RAMIndexCluster.java @@ -82,6 +82,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return (int) ((this.rowdef.objectOrder.cardinal(row.bytes(), 0, row.getPrimaryKeyLength()) / 17) % (this.cluster.length)); } + @Override public final byte[] smallestKey() { final HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.cluster.length); synchronized (this.cluster) { @@ -94,6 +95,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return keysort.smallestKey(); } + @Override public final byte[] largestKey() { final HandleSet keysort = new HandleSet(this.rowdef.primaryKeyLength, this.rowdef.objectOrder, this.cluster.length); synchronized (this.cluster) { @@ -115,6 +117,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return r; } + @Override public final void addUnique(final Entry row) throws RowSpaceExceededException { final int i = indexFor(row); assert i >= 0 : "i = " + i; @@ -126,28 +129,37 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea for (final Entry row: rows) addUnique(row); } + @Override public final void clear() { synchronized (this.cluster) { for (final RAMIndex c: this.cluster) if (c != null) c.clear(); } } - public final void close() { - clear(); + @Override + public final void close() { synchronized (this.cluster) { - for (final RAMIndex c: this.cluster) if (c != null) c.close(); + for (final RAMIndex c: this.cluster) { + if (c != null) { + //Log.logInfo("RAMIndexCluster", "Closing RAM index at " + c.getName() + " with " + c.size() + " entries ..."); + c.close(); + } + } } } + @Override public final void deleteOnExit() { // no nothing here } + @Override public final String filename() { // we don't have a file name return null; } + @Override public final Entry get(final byte[] key, final boolean forcecopy) { final int i = indexFor(key); if (i < 0) return null; @@ -156,8 +168,10 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return r.get(key, forcecopy); } + @Override public Map get(final Collection keys, final boolean forcecopy) throws IOException, InterruptedException { final Map map = new TreeMap(row().objectOrder); + Row.Entry entry; for (final byte[] key: keys) { entry = get(key, forcecopy); @@ -166,6 +180,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return map; } + @Override public final boolean has(final byte[] key) { final int i = indexFor(key); if (i < 0) return false; @@ -174,6 +189,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return r.has(key); } + @Override public final CloneableIterator keys(final boolean up, final byte[] firstKey) { synchronized (this.cluster) { final Collection> col = new ArrayList>(); @@ -193,6 +209,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea * @throws IOException * @throws RowSpaceExceededException */ + @Override public final boolean put(final Entry row) throws RowSpaceExceededException { final int i = indexFor(row); assert i >= 0 : "i = " + i; @@ -200,18 +217,21 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return accessArray(i).put(row); } + @Override public final boolean delete(final byte[] key) { final int i = indexFor(key); if (i < 0) return false; return accessArray(i).delete(key); } + @Override public final Entry remove(final byte[] key) { final int i = indexFor(key); if (i < 0) return null; return accessArray(i).remove(key); } + @Override public final ArrayList removeDoubles() throws RowSpaceExceededException { final ArrayList col = new ArrayList(); synchronized (this.cluster) { @@ -225,6 +245,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return col; } + @Override public final Entry removeOne() { synchronized (this.cluster) { for (int i = 0; i < this.cluster.length; i++) { @@ -238,6 +259,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return null; } + @Override public List top(final int count) { final List list = new ArrayList(); synchronized (this.cluster) { @@ -256,6 +278,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return list; } + @Override public final Entry replace(final Entry row) throws RowSpaceExceededException { final int i = indexFor(row); assert i >= 0 : "i = " + i; @@ -263,10 +286,12 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return accessArray(i).replace(row); } + @Override public final Row row() { return this.rowdef; } + @Override @SuppressWarnings("unchecked") public final CloneableIterator rows(final boolean up, final byte[] firstKey) { synchronized (this.cluster) { @@ -282,10 +307,12 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea } } + @Override public final CloneableIterator rows() { return rows(true, null); } + @Override public final int size() { int c = 0; synchronized (this.cluster) { @@ -294,6 +321,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return c; } + @Override public long mem() { long m = 0; synchronized (this.cluster) { @@ -302,6 +330,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return m; } + @Override public final boolean isEmpty() { synchronized (this.cluster) { for (final RAMIndex i: this.cluster) if (i != null && !i.isEmpty()) return false; @@ -309,6 +338,7 @@ public final class RAMIndexCluster implements Index, Iterable, Clonea return true; } + @Override public final Iterator iterator() { return this.rows(true, null); } diff --git a/source/net/yacy/kelondro/io/ByteCountInputStream.java b/source/net/yacy/kelondro/io/ByteCountInputStream.java index faa2e0aba..89629d7ef 100644 --- a/source/net/yacy/kelondro/io/ByteCountInputStream.java +++ b/source/net/yacy/kelondro/io/ByteCountInputStream.java @@ -1,4 +1,4 @@ -//httpByteCountinputStream.java +//httpByteCountinputStream.java //----------------------- //(C) by Michael Peter Christen; mc@yacy.net //first published on http://www.anomic.de @@ -29,24 +29,23 @@ package net.yacy.kelondro.io; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; -//import java.util.HashMap; import net.yacy.kelondro.logging.Log; public final class ByteCountInputStream extends FilterInputStream { - + // private final static Object syncObject = new Object(); // private final static HashMap byteCountInfo = new HashMap(2); // private static long globalByteCount = 0; - + private boolean finished = false; protected long byteCount; - private String byteCountAccountName = null; + private String byteCountAccountName = null; protected ByteCountInputStream(final InputStream inputStream) { this(inputStream, null); } - + /** * Constructor of this class * @param inputStream the {@link InputStream} to read from @@ -54,7 +53,7 @@ public final class ByteCountInputStream extends FilterInputStream { public ByteCountInputStream(final InputStream inputStream, final String accountName) { this(inputStream,0,accountName); } - + /** * Constructor of this class * @param inputStream the {@link InputStream} to read from @@ -64,14 +63,16 @@ public final class ByteCountInputStream extends FilterInputStream { super(inputStream); this.byteCount = initByteCount; this.byteCountAccountName = accountName; - } - + } + + @Override public final int read(final byte[] b) throws IOException { final int readCount = super.read(b); if (readCount > 0) this.byteCount += readCount; return readCount; } + @Override public final int read(final byte[] b, final int off, final int len) throws IOException { try { final int readCount = super.read(b, off, len); @@ -82,31 +83,33 @@ public final class ByteCountInputStream extends FilterInputStream { } } + @Override public final int read() throws IOException { this.byteCount++; return super.read(); } - + + @Override public final long skip(final long len) throws IOException { final long skipCount = super.skip(len); - if (skipCount > 0) this.byteCount += skipCount; + if (skipCount > 0) this.byteCount += skipCount; return skipCount; } public final long getCount() { return this.byteCount; } - + public final String getAccountName() { return this.byteCountAccountName; } - + // public final static long getGlobalCount() { // synchronized (syncObject) { // return globalByteCount; // } // } - + // public final static long getAccountCount(final String accountName) { // synchronized (syncObject) { // if (byteCountInfo.containsKey(accountName)) { @@ -115,8 +118,9 @@ public final class ByteCountInputStream extends FilterInputStream { // return 0; // } // } - - public final void close() throws IOException { + + @Override + public final synchronized void close() throws IOException { try { super.close(); } catch (OutOfMemoryError e) { @@ -124,10 +128,10 @@ public final class ByteCountInputStream extends FilterInputStream { } this.finish(); } - + public final void finish() { if (this.finished) return; - + this.finished = true; ByteCount.addAccountCount(this.byteCountAccountName, this.byteCount); // synchronized (syncObject) { @@ -140,10 +144,10 @@ public final class ByteCountInputStream extends FilterInputStream { // lastByteCount += this.byteCount; // byteCountInfo.put(this.byteCountAccountName, Long.valueOf(lastByteCount)); // } -// -// } +// +// } } - + // public final static void resetCount() { // synchronized (syncObject) { // globalByteCount = 0; diff --git a/source/net/yacy/kelondro/io/CharBuffer.java b/source/net/yacy/kelondro/io/CharBuffer.java index 4c2e698aa..128581cfa 100644 --- a/source/net/yacy/kelondro/io/CharBuffer.java +++ b/source/net/yacy/kelondro/io/CharBuffer.java @@ -473,7 +473,7 @@ public final class CharBuffer extends Writer { } @Override - public void close() { + public synchronized void close() { this.length = 0; this.offset = 0; this.buffer = null; // assist with garbage collection @@ -484,4 +484,4 @@ public final class CharBuffer extends Writer { trimToSize(); } -} \ No newline at end of file +} diff --git a/source/net/yacy/kelondro/logging/ConsoleOutErrHandler.java b/source/net/yacy/kelondro/logging/ConsoleOutErrHandler.java index 17906ef2b..b2cfd9f98 100644 --- a/source/net/yacy/kelondro/logging/ConsoleOutErrHandler.java +++ b/source/net/yacy/kelondro/logging/ConsoleOutErrHandler.java @@ -1,4 +1,4 @@ -//ConsoleOutErrHandler.java +//ConsoleOutErrHandler.java //------------------------------------- //part of YACY //(C) by Michael Peter Christen; mc@yacy.net @@ -42,37 +42,37 @@ public final class ConsoleOutErrHandler extends Handler { private boolean ignoreCtrlChr = false; private Level splitLevel = Level.WARNING; private final Handler stdOutHandler; - private final Handler stdErrHandler; - + private final Handler stdErrHandler; + public ConsoleOutErrHandler() { this.stdOutHandler = new ConsoleOutHandler(); - this.stdErrHandler = new ConsoleHandler(); + this.stdErrHandler = new ConsoleHandler(); this.stdOutHandler.setLevel(Level.FINEST); this.stdErrHandler.setLevel(Level.WARNING); configure(); } - + /** * Get any configuration properties set */ private void configure() { final LogManager manager = LogManager.getLogManager(); final String className = getClass().getName(); - + final String level = manager.getProperty(className + ".level"); setLevel((level == null) ? Level.INFO : Level.parse(level)); - + final Level levelStdOut = parseLevel(manager.getProperty(className + ".levelStdOut")); final Level levelSplit = parseLevel(manager.getProperty(className + ".levelSplit")); final Level levelStdErr = parseLevel(manager.getProperty(className + ".levelStdErr")); setLevels(levelStdOut,levelSplit,levelStdErr); - + final String filter = manager.getProperty(className + ".filter"); setFilter(makeFilter(filter)); - + final String formatter = manager.getProperty(className + ".formatter"); setFormatter(makeFormatter(formatter)); - + final String encoding = manager.getProperty(className + ".encoding"); try { this.stdOutHandler.setEncoding(encoding); @@ -80,12 +80,12 @@ public final class ConsoleOutErrHandler extends Handler { } catch (final UnsupportedEncodingException e) { Log.logException(e); } - + final String ignoreCtrlChrStr = manager.getProperty(className + ".ignoreCtrlChr"); this.ignoreCtrlChr = (ignoreCtrlChrStr==null) ? false : "true".equalsIgnoreCase(ignoreCtrlChrStr); - - } - + + } + private Level parseLevel(final String levelName) { try { return (levelName == null) ? Level.INFO : Level.parse(levelName); @@ -93,10 +93,10 @@ public final class ConsoleOutErrHandler extends Handler { return Level.ALL; } } - + private Filter makeFilter(final String name) { if (name == null) return null; - + Filter f = null; try { final Class c = Class.forName(name); @@ -107,11 +107,11 @@ public final class ConsoleOutErrHandler extends Handler { } } return f; - } - + } + private Formatter makeFormatter(final String name) { if (name == null) return null; - + Formatter f = null; try { final Class c = Class.forName(name); @@ -120,12 +120,13 @@ public final class ConsoleOutErrHandler extends Handler { f = new SimpleFormatter(); } return f; - } - - + } + + + @Override public final void publish(final LogRecord record) { if (!isLoggable(record)) return; - + if (this.ignoreCtrlChr) { String msg = record.getMessage(); if (msg != null) { @@ -133,8 +134,8 @@ public final class ConsoleOutErrHandler extends Handler { } record.setMessage(msg); } - - if (record.getLevel().intValue() >= splitLevel.intValue()) { + + if (record.getLevel().intValue() >= this.splitLevel.intValue()) { this.stdErrHandler.publish(record); } else { this.stdOutHandler.publish(record); @@ -142,27 +143,29 @@ public final class ConsoleOutErrHandler extends Handler { flush(); } + @Override public void flush() { this.stdOutHandler.flush(); this.stdErrHandler.flush(); } - - public void close() throws SecurityException { - this.stdOutHandler.close(); + + @Override + public synchronized void close() throws SecurityException { + this.stdOutHandler.close(); this.stdErrHandler.close(); } - + @Override public synchronized void setLevel(final Level newLevel) throws SecurityException { super.setLevel(newLevel); } - + public void setLevels(final Level stdOutLevel, final Level splitLevel, final Level stdErrLevel) throws SecurityException { this.stdOutHandler.setLevel(stdOutLevel); this.splitLevel = splitLevel; this.stdErrHandler.setLevel(stdErrLevel); } - + @Override public void setFormatter(final Formatter newFormatter) throws SecurityException { super.setFormatter(newFormatter); diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index 174b02cf2..829db45b1 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -75,7 +75,7 @@ public final class ReferenceContainerArray { true); } - public void close() { + public synchronized void close() { this.array.close(true); } diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java index 5c2583520..b3f2978c5 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java @@ -101,7 +101,7 @@ public final class ReferenceContainerCache exte } @Override - public void close() { + public synchronized void close() { this.cache = null; } diff --git a/source/net/yacy/kelondro/rwi/ReferenceIterator.java b/source/net/yacy/kelondro/rwi/ReferenceIterator.java index fb54e6c95..5bb7fa6ad 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceIterator.java +++ b/source/net/yacy/kelondro/rwi/ReferenceIterator.java @@ -83,7 +83,7 @@ public class ReferenceIterator extends LookAhe return null; } - public void close() { + public synchronized void close() { if (this.blobs != null) this.blobs.close(); this.blobs = null; } diff --git a/source/net/yacy/kelondro/table/SQLTable.java b/source/net/yacy/kelondro/table/SQLTable.java index 9ee56e70e..56b9a89e5 100644 --- a/source/net/yacy/kelondro/table/SQLTable.java +++ b/source/net/yacy/kelondro/table/SQLTable.java @@ -108,23 +108,28 @@ public class SQLTable implements Index, Iterable { } + @Override public long mem() { return 0; } + @Override public byte[] smallestKey() { return null; } + @Override public byte[] largestKey() { return null; } + @Override public String filename() { return "dbtest." + this.theDBConnection.hashCode(); } - public void close() { + @Override + public synchronized void close() { if (this.theDBConnection != null) try { this.theDBConnection.close(); } catch (final SQLException e) { @@ -133,6 +138,7 @@ public class SQLTable implements Index, Iterable { this.theDBConnection = null; } + @Override public int size() { int size = -1; try { @@ -155,14 +161,17 @@ public class SQLTable implements Index, Iterable { } } + @Override public boolean isEmpty() { return size() == 0; } + @Override public Row row() { return this.rowdef; } + @Override public boolean has(final byte[] key) { try { return (get(key, false) != null); @@ -171,10 +180,12 @@ public class SQLTable implements Index, Iterable { } } + @Override public ArrayList removeDoubles() { return new ArrayList(); } + @Override public Row.Entry get(final byte[] key, final boolean forcecopy) throws IOException { try { final String sqlQuery = "SELECT value from test where hash = ?"; @@ -199,6 +210,7 @@ public class SQLTable implements Index, Iterable { } } + @Override public Map get(final Collection keys, final boolean forcecopy) throws IOException, InterruptedException { final Map map = new TreeMap(row().objectOrder); Row.Entry entry; @@ -209,6 +221,7 @@ public class SQLTable implements Index, Iterable { return map; } + @Override public Row.Entry replace(final Row.Entry row) throws IOException { try { final Row.Entry oldEntry = remove(row.getPrimaryKeyBytes()); @@ -231,6 +244,7 @@ public class SQLTable implements Index, Iterable { } } + @Override public boolean put(final Row.Entry row) throws IOException { try { final String sqlQuery = "INSERT INTO test (" + @@ -252,6 +266,7 @@ public class SQLTable implements Index, Iterable { } } + @Override public synchronized void addUnique(final Row.Entry row) throws IOException { throw new UnsupportedOperationException(); } @@ -264,6 +279,7 @@ public class SQLTable implements Index, Iterable { throw new UnsupportedOperationException(); } + @Override public Row.Entry remove(final byte[] key) throws IOException { PreparedStatement sqlStatement = null; try { @@ -292,23 +308,28 @@ public class SQLTable implements Index, Iterable { } } + @Override public boolean delete(final byte[] key) throws IOException { return remove(key) != null; } + @Override public Row.Entry removeOne() { return null; } + @Override public List top(final int count) throws IOException { return null; } + @Override public CloneableIterator rows(final boolean up, final byte[] startKey) throws IOException { // Objects are of type kelondroRow.Entry return null; } + @Override public Iterator iterator() { try { return rows(); @@ -317,10 +338,12 @@ public class SQLTable implements Index, Iterable { } } + @Override public CloneableIterator rows() throws IOException { return null; } + @Override public CloneableIterator keys(final boolean up, final byte[] startKey) { // Objects are of type byte[] return null; @@ -362,10 +385,12 @@ public class SQLTable implements Index, Iterable { return new int[]{0,0,0,0,0,0,0,0,0,0}; } + @Override public void clear() { // do nothing } + @Override public void deleteOnExit() { // do nothing } diff --git a/source/net/yacy/kelondro/util/XMLTables.java b/source/net/yacy/kelondro/util/XMLTables.java index 01f584cc7..fdfcb2926 100644 --- a/source/net/yacy/kelondro/util/XMLTables.java +++ b/source/net/yacy/kelondro/util/XMLTables.java @@ -161,7 +161,7 @@ public class XMLTables { return null; } - public void close() throws IOException { + public synchronized void close() throws IOException { commit(true); } diff --git a/source/net/yacy/peers/NewsDB.java b/source/net/yacy/peers/NewsDB.java index 0f5ff7aec..82502db3a 100644 --- a/source/net/yacy/peers/NewsDB.java +++ b/source/net/yacy/peers/NewsDB.java @@ -122,7 +122,7 @@ public class NewsDB { } } - public void close() { + public synchronized void close() { if (this.news != null) this.news.close(); this.news = null; } @@ -329,4 +329,4 @@ public class NewsDB { } } -} \ No newline at end of file +} diff --git a/source/net/yacy/peers/NewsQueue.java b/source/net/yacy/peers/NewsQueue.java index 1b6750a0c..20f52b12c 100644 --- a/source/net/yacy/peers/NewsQueue.java +++ b/source/net/yacy/peers/NewsQueue.java @@ -98,7 +98,7 @@ public class NewsQueue implements Iterable { } } - public void close() { + public synchronized void close() { if (this.queueStack != null) this.queueStack.close(); this.queueStack = null; } @@ -224,4 +224,4 @@ public class NewsQueue implements Iterable { } -} \ No newline at end of file +} diff --git a/source/net/yacy/peers/graphics/WebStructureGraph.java b/source/net/yacy/peers/graphics/WebStructureGraph.java index ef466eee6..9530f9bd5 100644 --- a/source/net/yacy/peers/graphics/WebStructureGraph.java +++ b/source/net/yacy/peers/graphics/WebStructureGraph.java @@ -749,7 +749,7 @@ public class WebStructureGraph } } - public void close() { + public synchronized void close() { // finish dns resolving queue if ( this.publicRefDNSResolvingWorker.isAlive() ) { log.logInfo("Waiting for the DNS Resolving Queue to terminate"); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 9dfeead92..207453187 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -1551,7 +1551,7 @@ public final class Switchboard extends serverSwitch return this.crawler.clear(); } - public void close() { + public synchronized void close() { this.log.logConfig("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:"); MemoryTracker.stopSystemProfiling(); terminateAllThreads(true); diff --git a/source/net/yacy/search/index/DocumentIndex.java b/source/net/yacy/search/index/DocumentIndex.java index e29397e4f..edb2df025 100644 --- a/source/net/yacy/search/index/DocumentIndex.java +++ b/source/net/yacy/search/index/DocumentIndex.java @@ -243,7 +243,7 @@ public class DocumentIndex extends Segment * close the index. This terminates all worker threads and then closes the segment. */ @Override - public void close() { + public synchronized void close() { // send termination signal to worker threads for ( @SuppressWarnings("unused") final Worker element : this.worker ) { diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java index 3700452ab..d4ae0a852 100644 --- a/source/net/yacy/search/index/Segment.java +++ b/source/net/yacy/search/index/Segment.java @@ -303,7 +303,7 @@ public class Segment { return refCount; } - public void close() { + public synchronized void close() { this.termIndex.close(); this.urlMetadata.close(); this.urlCitationIndex.close(); diff --git a/source/net/yacy/search/index/Segments.java b/source/net/yacy/search/index/Segments.java index 2f7e3fb57..2d546f990 100644 --- a/source/net/yacy/search/index/Segments.java +++ b/source/net/yacy/search/index/Segments.java @@ -176,7 +176,7 @@ public class Segments implements Iterable { segment(this.process_assignment.get(process)).close(); } - public void close() { + public synchronized void close() { if (this.segments != null) for (final Segment s: this.segments.values()) s.close(); this.segments = null; } diff --git a/source/org/apache/tools/tar/TarBuffer.java b/source/org/apache/tools/tar/TarBuffer.java index a5e3288fb..f33882bab 100644 --- a/source/org/apache/tools/tar/TarBuffer.java +++ b/source/org/apache/tools/tar/TarBuffer.java @@ -436,7 +436,7 @@ public class TarBuffer { * current block before closing. * @throws IOException on error */ - public void close() throws IOException { + public synchronized void close() throws IOException { if (this.debug) { System.err.println("TarBuffer.closeBuffer()."); } diff --git a/source/org/apache/tools/tar/TarInputStream.java b/source/org/apache/tools/tar/TarInputStream.java index 74fe5e88b..011652184 100644 --- a/source/org/apache/tools/tar/TarInputStream.java +++ b/source/org/apache/tools/tar/TarInputStream.java @@ -111,7 +111,7 @@ public class TarInputStream extends FilterInputStream { * @throws IOException on error */ @Override - public void close() throws IOException { + public synchronized void close() throws IOException { this.buffer.close(); } diff --git a/source/org/apache/tools/tar/TarOutputStream.java b/source/org/apache/tools/tar/TarOutputStream.java index 75f9c53d3..c7afda383 100644 --- a/source/org/apache/tools/tar/TarOutputStream.java +++ b/source/org/apache/tools/tar/TarOutputStream.java @@ -141,7 +141,7 @@ public class TarOutputStream extends FilterOutputStream { * @throws IOException on error */ @Override - public void close() throws IOException { + public synchronized void close() throws IOException { if (!closed) { this.finish(); this.buffer.close();