diff --git a/source/net/yacy/cora/federate/solr/connector/ShardSelection.java b/source/net/yacy/cora/federate/solr/connector/ShardSelection.java index 527d84de3..402e31d03 100644 --- a/source/net/yacy/cora/federate/solr/connector/ShardSelection.java +++ b/source/net/yacy/cora/federate/solr/connector/ShardSelection.java @@ -28,17 +28,13 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.concurrent.atomic.AtomicLong; +import net.yacy.cora.document.ASCII; import net.yacy.cora.federate.solr.YaCySchema; import org.apache.solr.common.SolrInputDocument; public class ShardSelection { - public final static Charset charsetUTF8; - static { - charsetUTF8 = Charset.forName("UTF-8"); - } - private final Method method; // the sharding method private final AtomicLong chardID; // the next id that shall be given away private final int dimension; // the number of chards @@ -74,7 +70,7 @@ public class ShardSelection { if (host == null) throw new IOException("sharding - bad url, host empty: " + sku); try { final MessageDigest digest = MessageDigest.getInstance("MD5"); - digest.update(host.getBytes(charsetUTF8)); + digest.update(ASCII.getBytes(host)); final byte[] md5 = digest.digest(); return (0xff & md5[0]) % this.dimension; } catch (final NoSuchAlgorithmException e) { diff --git a/source/net/yacy/kelondro/rwi/AbstractIndex.java b/source/net/yacy/kelondro/rwi/AbstractIndex.java index 56e0404ad..d5e495800 100644 --- a/source/net/yacy/kelondro/rwi/AbstractIndex.java +++ b/source/net/yacy/kelondro/rwi/AbstractIndex.java @@ -72,16 +72,6 @@ public abstract class AbstractIndex implements } } - @Override - public void removeDelayed(final HandleSet termHashes, final byte[] urlHashBytes) throws IOException { - // remove the same url hashes for multiple words - // this is mainly used when correcting a index after a search - final Iterator i = termHashes.iterator(); - while (i.hasNext()) { - removeDelayed(i.next(), urlHashBytes); - } - } - @Override public int remove(final HandleSet termHashes, final byte[] urlHashBytes) throws IOException { // remove the same url hashes for multiple words @@ -94,32 +84,6 @@ public abstract class AbstractIndex implements return c; } - @Override - public synchronized TreeSet> referenceContainer(final byte[] startHash, final boolean rot, final boolean excludePrivate, int count) throws IOException { - // creates a set of indexContainers - // this does not use the cache - final Order> containerOrder = new ReferenceContainerOrder(this.factory, termKeyOrdering().clone()); - final ReferenceContainer emptyContainer = ReferenceContainer.emptyContainer(this.factory, startHash); - containerOrder.rotate(emptyContainer); - final TreeSet> containers = new TreeSet>(containerOrder); - final Iterator> i = referenceContainerIterator(startHash, rot, excludePrivate); - //if (ram) count = Math.min(size(), count); - ReferenceContainer container; - // this loop does not terminate using the i.hasNex() predicate when rot == true - // because then the underlying iterator is a rotating iterator without termination - // in this case a termination must be ensured with a counter - // It must also be ensured that the counter is in/decreased every loop - while ((count > 0) && (i.hasNext())) { - container = i.next(); - if (container != null && !container.isEmpty()) { - containers.add(container); - } - count--; // decrease counter even if the container was null or empty to ensure termination - } - return containers; // this may return less containers as demanded - } - - // methods to search in the index /** @@ -165,48 +129,6 @@ public abstract class AbstractIndex implements return containers; } - /** - * collect containers for given word hashes and join them as they are retrieved. - * This collection stops if a single container does not contain any references - * or the current result of the container join results in an empty container. - * In any fail case only a empty result container is returned. - * @param wordHashes - * @param urlselection - * @param maxDistance the maximum distance that the words in the result may have - * @return ReferenceContainer the join result - * @throws SpaceExceededException - */ - public ReferenceContainer searchJoin(final HandleSet wordHashes, final HandleSet urlselection, final int maxDistance) throws SpaceExceededException { - // first check if there is any entry that has no match; - // this uses only operations in ram - for (final byte[] wordHash: wordHashes) { - if (!has(wordHash)) return ReferenceContainer.emptyContainer(this.factory, null, 0); - } - - // retrieve entities that belong to the hashes - ReferenceContainer resultContainer = null; - ReferenceContainer singleContainer; - for (final byte[] wordHash: wordHashes) { - // retrieve index - try { - singleContainer = get(wordHash, urlselection); - } catch (final IOException e) { - Log.logException(e); - continue; - } - - // check result - if ((singleContainer == null || singleContainer.isEmpty())) return ReferenceContainer.emptyContainer(this.factory, null, 0); - if (resultContainer == null) resultContainer = singleContainer; else { - resultContainer = ReferenceContainer.joinConstructive(this.factory, resultContainer, singleContainer, maxDistance); - } - - // finish if the result is empty - if (resultContainer.isEmpty()) return resultContainer; - } - return resultContainer; - } - public TermSearch query( final HandleSet queryHashes, final HandleSet excludeHashes, diff --git a/source/net/yacy/kelondro/rwi/IODispatcher.java b/source/net/yacy/kelondro/rwi/IODispatcher.java index b46b1a022..d3e925a07 100644 --- a/source/net/yacy/kelondro/rwi/IODispatcher.java +++ b/source/net/yacy/kelondro/rwi/IODispatcher.java @@ -48,7 +48,7 @@ import net.yacy.kelondro.util.MemoryControl; */ public class IODispatcher extends Thread { - public static final Log log = new Log("IODispatcher"); + private static final Log log = new Log("IODispatcher"); private Semaphore controlQueue; private final Semaphore termination; diff --git a/source/net/yacy/kelondro/rwi/Index.java b/source/net/yacy/kelondro/rwi/Index.java index fe375f2b9..baabe936c 100644 --- a/source/net/yacy/kelondro/rwi/Index.java +++ b/source/net/yacy/kelondro/rwi/Index.java @@ -138,9 +138,7 @@ public interface Index extends Iterable extends Iterable> referenceContainer( - byte[] startHash, - boolean rot, - boolean excludePrivate, - int count - ) throws IOException; - /** * collect containers for given word hashes. This collection stops if a single container does not contain any references. * In that case only a empty result is returned. diff --git a/source/net/yacy/kelondro/rwi/IndexCell.java b/source/net/yacy/kelondro/rwi/IndexCell.java index c41f6c7b7..05a6ea6af 100644 --- a/source/net/yacy/kelondro/rwi/IndexCell.java +++ b/source/net/yacy/kelondro/rwi/IndexCell.java @@ -174,7 +174,7 @@ public final class IndexCell extends AbstractBu } - public boolean shrink(final long targetFileSize, final long maxFileSize) { + private boolean shrink(final long targetFileSize, final long maxFileSize) { if (this.array.entries() < 2) return false; boolean donesomething = false; @@ -391,26 +391,6 @@ public final class IndexCell extends AbstractBu return; } - @Override - public void removeDelayed(final byte[] termHash, final HandleSet urlHashes) { - HandleSet r; - synchronized (this.removeDelayedURLs) { - r = this.removeDelayedURLs.get(termHash); - } - if (r == null) { - r = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 0); - } - try { - r.putAll(urlHashes); - } catch (final SpaceExceededException e) { - try {remove(termHash, urlHashes);} catch (final IOException e1) {} - return; - } - synchronized (this.removeDelayedURLs) { - this.removeDelayedURLs.put(termHash, r); - } - } - @Override public void removeDelayed(final byte[] termHash, final byte[] urlHashBytes) { HandleSet r; @@ -582,7 +562,7 @@ public final class IndexCell extends AbstractBu throw new UnsupportedOperationException("an accumulated size of index entries would not reflect the real number of words, which cannot be computed easily"); } - public int[] sizes() { + private int[] sizes() { final int[] as = this.array.sizes(); final int[] asr = new int[as.length + 1]; System.arraycopy(as, 0, asr, 0, as.length); @@ -608,16 +588,6 @@ public final class IndexCell extends AbstractBu return this.array.ordering(); } - public File newContainerBLOBFile() { - // for migration of cache files - return this.array.newContainerBLOBFile(); - } - - public void mountBLOBFile(final File blobFile) throws IOException { - // for migration of cache files - this.array.mountBLOBFile(blobFile); - } - @Override public long getBufferMaxAge() { return System.currentTimeMillis(); diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index 82cbe8e63..deed77ef5 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -32,24 +32,21 @@ import java.util.Iterator; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.sorting.Rating; -import net.yacy.cora.storage.HandleMap; import net.yacy.cora.util.SpaceExceededException; import net.yacy.kelondro.blob.ArrayStack; import net.yacy.kelondro.blob.BLOB; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.Row; -import net.yacy.kelondro.index.RowHandleMap; import net.yacy.kelondro.index.RowSet; import net.yacy.kelondro.logging.Log; -import net.yacy.search.index.Segment; public final class ReferenceContainerArray { private final static long METHOD_MAXRUNTIME = 5000L; - protected final ReferenceFactory factory; - protected final ArrayStack array; + private final ReferenceFactory factory; + private final ArrayStack array; /** * open a index container array based on BLOB dumps. The content of the BLOBs will not be read @@ -85,10 +82,6 @@ public final class ReferenceContainerArray { this.array.clear(); } - public long mem() { - return this.array.mem(); - } - public int[] sizes() { return (this.array == null) ? new int[0] : this.array.sizes(); } @@ -355,18 +348,6 @@ public final class ReferenceContainerArray { return c; } - /** - * calculate an upper limit for a ranking number of the container size - * the returned number is not a counter. It can only be used to compare the - * ReferenceContainer, that may be produced as a result of get() - * @param termHash - * @return a ranking number - * @throws IOException - */ - public long lenghtRankingUpperLimit(final byte[] termHash) throws IOException { - return this.array.lengthAdd(termHash); - } - /** * delete a indexContainer from the heap cache. This can only be used for write-enabled heaps * @param wordHash @@ -444,52 +425,4 @@ public final class ReferenceContainerArray { merger.merge(ff, null, this.factory, this.array, newContainerBLOBFile()); return true; } - - public static HandleMap referenceHashes( - final File heapLocation, - final ReferenceFactory factory, - final ByteOrder termOrder, - final Row payloadrow) throws IOException, SpaceExceededException { - - System.out.println("CELL REFERENCE COLLECTION startup"); - final HandleMap references = new RowHandleMap(payloadrow.primaryKeyLength, termOrder, 4, 1000000, heapLocation.getAbsolutePath()); - final String[] files = heapLocation.list(); - for (final String f: files) { - if (f.length() < 22 || !f.startsWith(Segment.termIndexName) || !f.endsWith(".blob")) continue; - final File fl = new File(heapLocation, f); - System.out.println("CELL REFERENCE COLLECTION opening blob " + fl); - final CloneableIterator> ei = new ReferenceIterator(fl, factory); - - ReferenceContainer container; - final long start = System.currentTimeMillis(); - long lastlog = start - 27000; - int count = 0; - ReferenceType reference; - byte[] mh; - while (ei.hasNext()) { - container = ei.next(); - if (container == null) continue; - final Iterator refi = container.entries(); - while (refi.hasNext()) { - reference = refi.next(); - if (reference == null) continue; - mh = reference.urlhash(); - if (mh == null) continue; - references.inc(mh); - } - count++; - // write a log - if (System.currentTimeMillis() - lastlog > 30000) { - System.out.println("CELL REFERENCE COLLECTION scanned " + count + " RWI index entries. "); - lastlog = System.currentTimeMillis(); - } - } - ei.close(); - } - references.trim(); - System.out.println("CELL REFERENCE COLLECTION finished"); - return references; - } - - } diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java index 70bed64a3..93d207d01 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerCache.java @@ -59,7 +59,7 @@ import net.yacy.kelondro.util.FileUtils; */ public final class ReferenceContainerCache extends AbstractIndex implements Index, IndexReader, Iterable> { - public static final Log log = new Log("ReferenceContainerCache"); + private static final Log log = new Log("ReferenceContainerCache"); private final int termSize; private final ByteOrder termOrder; @@ -176,7 +176,7 @@ public final class ReferenceContainerCache exte * create a clone of the cache content that is sorted using the this.containerOrder * @return the sorted ReferenceContainer[] */ - protected List> sortedClone() { + private List> sortedClone() { final List> cachecopy = new ArrayList>(this.cache.size()); synchronized (this.cache) { for (final Map.Entry> entry: this.cache.entrySet()) { @@ -187,7 +187,7 @@ public final class ReferenceContainerCache exte return cachecopy; } - protected List> ratingList() { + private List> ratingList() { final List> list = new ArrayList>(this.cache.size()); synchronized (this.cache) { for (final Map.Entry> entry: this.cache.entrySet()) { @@ -510,11 +510,6 @@ public final class ReferenceContainerCache exte return false; } - @Override - public void removeDelayed(final byte[] termHash, final HandleSet urlHashes) { - remove(termHash, urlHashes); - } - @Override public int remove(final byte[] termHash, final HandleSet urlHashes) { assert this.cache != null; diff --git a/source/net/yacy/kelondro/rwi/ReferenceIterator.java b/source/net/yacy/kelondro/rwi/ReferenceIterator.java index fef6509a0..216f7fdf3 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceIterator.java +++ b/source/net/yacy/kelondro/rwi/ReferenceIterator.java @@ -42,9 +42,9 @@ import net.yacy.kelondro.logging.Log; * iterator of BLOBHeap files: is used to import heap dumps into a write-enabled index heap */ public class ReferenceIterator extends LookAheadIterator> implements CloneableIterator>, Iterable> { - HeapReader.entries blobs; - File blobFile; - ReferenceFactory factory; + private HeapReader.entries blobs; + private File blobFile; + private ReferenceFactory factory; public ReferenceIterator(final File blobFile, final ReferenceFactory factory) throws IOException { this.blobs = new HeapReader.entries(blobFile, factory.getRow().primaryKeyLength); diff --git a/source/net/yacy/kelondro/rwi/TermSearch.java b/source/net/yacy/kelondro/rwi/TermSearch.java index b62f12691..1074601dd 100644 --- a/source/net/yacy/kelondro/rwi/TermSearch.java +++ b/source/net/yacy/kelondro/rwi/TermSearch.java @@ -37,7 +37,7 @@ import net.yacy.cora.util.SpaceExceededException; public class TermSearch { private final ReferenceContainer joinResult; - TreeMap> inclusionContainers; + private TreeMap> inclusionContainers; public TermSearch( Index base, diff --git a/source/net/yacy/kelondro/table/SQLTable.java b/source/net/yacy/kelondro/table/SQLTable.java index 22356d4f0..b7961f7fe 100644 --- a/source/net/yacy/kelondro/table/SQLTable.java +++ b/source/net/yacy/kelondro/table/SQLTable.java @@ -340,37 +340,6 @@ public class SQLTable implements Index, Iterable { return null; } - public int columns() { - // TODO Auto-generated method stub - return 0; - } - - public ByteOrder order() { - return order; - } - - public int primarykey() { - return 0; - } - - public final int cacheObjectChunkSize() { - // dummy method - return -1; - } - - public long[] cacheObjectStatus() { - // dummy method - return null; - } - - public final int cacheNodeChunkSize() { - return -1; - } - - public final int[] cacheNodeStatus() { - return new int[]{0,0,0,0,0,0,0,0,0,0}; - } - @Override public void clear() { // do nothing diff --git a/source/net/yacy/kelondro/util/BDecoder.java b/source/net/yacy/kelondro/util/BDecoder.java index c68cfb7f2..bb949fab8 100644 --- a/source/net/yacy/kelondro/util/BDecoder.java +++ b/source/net/yacy/kelondro/util/BDecoder.java @@ -41,11 +41,11 @@ import net.yacy.cora.document.UTF8; public class BDecoder { - final static byte[] _e = "e".getBytes(); - final static byte[] _i = "i".getBytes(); - final static byte[] _d = "d".getBytes(); - final static byte[] _l = "l".getBytes(); - final static byte[] _p = ":".getBytes(); + private final static byte[] _e = "e".getBytes(); + private final static byte[] _i = "i".getBytes(); + private final static byte[] _d = "d".getBytes(); + private final static byte[] _l = "l".getBytes(); + private final static byte[] _p = ":".getBytes(); private final byte[] b; private int pos; @@ -70,7 +70,7 @@ public class BDecoder { public void toStream(OutputStream os) throws IOException; } - public static abstract class BDfltObject implements BObject { + private static abstract class BDfltObject implements BObject { public long getInteger() { throw new UnsupportedOperationException(); @@ -121,12 +121,12 @@ public class BDecoder { os.write(_p); os.write(this.b); } - public static void toStream(OutputStream os, byte[] b) throws IOException { + private static void toStream(OutputStream os, byte[] b) throws IOException { os.write(ASCII.getBytes(Integer.toString(b.length))); os.write(_p); os.write(b); } - public static void toStream(OutputStream os, String s) throws IOException { + private static void toStream(OutputStream os, String s) throws IOException { final byte[] b = UTF8.getBytes(s); os.write(ASCII.getBytes(Integer.toString(b.length))); os.write(_p); @@ -203,7 +203,7 @@ public class BDecoder { public static class BIntegerObject extends BDfltObject implements BObject { private long i; - public BIntegerObject(long i) { + private BIntegerObject(long i) { this.i = i; } @Override @@ -285,7 +285,7 @@ public class BDecoder { } } - public static void print(BObject bo, int t) { + private static void print(BObject bo, int t) { for (int i = 0; i < t; i++) System.out.print(" "); if (bo.getType() == BType.integer) System.out.println(bo.getInteger()); if (bo.getType() == BType.string) System.out.println(bo.getString()); diff --git a/source/net/yacy/kelondro/util/BEncoder.java b/source/net/yacy/kelondro/util/BEncoder.java index 362370c5d..205738d56 100644 --- a/source/net/yacy/kelondro/util/BEncoder.java +++ b/source/net/yacy/kelondro/util/BEncoder.java @@ -26,34 +26,13 @@ package net.yacy.kelondro.util; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import net.yacy.cora.document.UTF8; import net.yacy.kelondro.util.BDecoder.BObject; public class BEncoder { - - // lists - public static List transcode(List list) { - ArrayList l = new ArrayList(list.size()); - for (byte[] entry: list) l.add(new BDecoder.BStringObject(entry)); - return l; - } - - public static byte[] encode(List list) { - BDecoder.BListObject l = new BDecoder.BListObject(list); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try { - l.toStream(baos); - baos.close(); - return baos.toByteArray(); - } catch (IOException e) { - } - return null; - } // maps public static Map transcode(Map map) { diff --git a/source/net/yacy/kelondro/util/Bitfield.java b/source/net/yacy/kelondro/util/Bitfield.java index 434782d04..fa033c58d 100644 --- a/source/net/yacy/kelondro/util/Bitfield.java +++ b/source/net/yacy/kelondro/util/Bitfield.java @@ -127,42 +127,6 @@ public class Bitfield implements Cloneable, Serializable { return this.toString().hashCode(); } - public void and(final Bitfield x) { - final int c = Math.min(x.length(), this.length()); - for (int i = 0; i < c; i++) set(i, this.get(i) && x.get(i)); - } - - public void or(final Bitfield x) { - final int c = Math.min(x.length(), this.length()); - for (int i = 0; i < c; i++) set(i, this.get(i) || x.get(i)); - if (x.length() > c) { - for (int i = c; i < x.length(); i++) set(i, x.get(i)); - } - } - - public void xor(final Bitfield x) { - final int c = Math.min(x.length(), this.length()); - for (int i = 0; i < c; i++) set(i, this.get(i) != x.get(i)); - if (x.length() > c) { - for (int i = c; i < x.length(); i++) set(i, x.get(i)); - } - } - - public boolean anyOf(final Bitfield x) { - final int c = Math.min(x.length(), this.length()); - for (int i = 0; i < c; i++) if ((x.get(i)) && (this.get(i))) return true; - return false; - } - - public boolean allOf(final Bitfield x) { - final int c = Math.min(x.length(), this.length()); - for (int i = 0; i < c; i++) if ((x.get(i)) && (!(this.get(i)))) return false; - if (x.length() > c) { - for (int i = c; i < x.length(); i++) if (x.get(i)) return false; - } - return true; - } - public static void main(final String[] args) { Bitfield test = new Bitfield(4); final int l = test.length(); diff --git a/source/net/yacy/kelondro/util/ByteArray.java b/source/net/yacy/kelondro/util/ByteArray.java index 1aab46737..0265a9599 100644 --- a/source/net/yacy/kelondro/util/ByteArray.java +++ b/source/net/yacy/kelondro/util/ByteArray.java @@ -27,7 +27,6 @@ package net.yacy.kelondro.util; import java.util.HashMap; import net.yacy.cora.document.UTF8; -import net.yacy.cora.order.ByteOrder; /** @@ -50,18 +49,10 @@ public class ByteArray { this.buffer = bb; } - public int length() { - return this.buffer.length; - } - public byte[] asBytes() { return this.buffer; } - public byte readByte(final int pos) { - return this.buffer[pos]; - } - public static boolean startsWith(final byte[] buffer, final byte[] pattern) { // compares two byte arrays: true, if pattern appears completely at offset position if (buffer == null && pattern == null) return true; @@ -71,16 +62,6 @@ public class ByteArray { return true; } - public int compareTo(final ByteArray b, final ByteOrder order) { - assert this.buffer.length == b.buffer.length; - return order.compare(this.buffer, b.buffer); - } - - public int compareTo(final int aoffset, final int alength, final ByteArray b, final int boffset, final int blength, final ByteOrder order) { - assert alength == blength; - return order.compare(this.buffer, aoffset, b.buffer, boffset, blength); - } - private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful @Override diff --git a/source/net/yacy/kelondro/util/ByteBuffer.java b/source/net/yacy/kelondro/util/ByteBuffer.java index bc6630619..ed58b119c 100644 --- a/source/net/yacy/kelondro/util/ByteBuffer.java +++ b/source/net/yacy/kelondro/util/ByteBuffer.java @@ -24,25 +24,20 @@ package net.yacy.kelondro.util; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; import java.util.List; -import java.util.Properties; import net.yacy.cora.document.UTF8; public final class ByteBuffer extends OutputStream { - public static final byte singlequote = (byte) 39; - public static final byte doublequote = (byte) 34; - public static final byte equal = (byte) '='; + private static final byte singlequote = (byte) 39; + private static final byte doublequote = (byte) 34; + private static final byte equal = (byte) '='; private byte[] buffer; private int offset; @@ -61,67 +56,12 @@ public final class ByteBuffer extends OutputStream { this.offset = 0; } - public ByteBuffer(final byte[] bb) { - this.buffer = bb; - this.length = bb.length; - this.offset = 0; - } - public ByteBuffer(final String s) { this.buffer = UTF8.getBytes(s); this.length = this.buffer.length; this.offset = 0; } - public ByteBuffer(final byte[] bb, final int initLength) { - this.buffer = new byte[initLength]; - System.arraycopy(bb, 0, this.buffer, 0, bb.length); - this.length = bb.length; - this.offset = 0; - } - - public ByteBuffer(final byte[] bb, final int of, final int le) { - if (of * 2 > bb.length) { - this.buffer = new byte[le]; - System.arraycopy(bb, of, this.buffer, 0, le); - this.length = le; - this.offset = 0; - } else { - this.buffer = bb; - this.length = le; - this.offset = of; - } - } - - public ByteBuffer(final ByteBuffer bb) { - this.buffer = bb.buffer; - this.length = bb.length; - this.offset = bb.offset; - } - - public ByteBuffer(final File f) throws IOException { - // initially fill the byte buffer with the content of a file - if (f.length() > Integer.MAX_VALUE) throw new IOException("file is too large for buffering"); - - this.length = (int) f.length(); - this.buffer = new byte[this.length]; - this.offset = 0; - - try { - final FileInputStream fis = new FileInputStream(f); -// byte[] buf = new byte[512]; -// int p = 0; -// while ((l = fis.read(buf)) > 0) { -// System.arraycopy(buf, 0, buffer, p, l); -// p += l; - /*int l =*/ fis.read(this.buffer); -// } - fis.close(); - } catch (final FileNotFoundException e) { - throw new IOException("File not found: " + f.toString() + "; " + e.getMessage()); - } - } - public void clear() { // we keep the byte[] and just set the pointer to write positions to zero this.length = 0; @@ -154,7 +94,7 @@ public final class ByteBuffer extends OutputStream { write((byte) b); } - public void write(final byte b) { + private void write(final byte b) { if (this.offset + this.length + 1 > this.buffer.length) grow(); this.buffer[this.offset + this.length++] = b; } @@ -171,28 +111,6 @@ public final class ByteBuffer extends OutputStream { this.length += le; } - // overwrite does not increase the 'length' write position pointer! - - public void overwrite(final int pos, final int b) { - overwrite(pos, (byte) (b & 0xff)); - } - - public void overwrite(final int pos, final byte b) { - if (this.offset + pos + 1 > this.buffer.length) grow(); - this.buffer[this.offset + pos] = b; - if (pos >= this.length) this.length = pos + 1; - } - - public void overwrite(final int pos, final byte[] bb) { - overwrite(pos, bb, 0, bb.length); - } - - public void overwrite(final int pos, final byte[] bb, final int of, final int le) { - while (this.offset + pos + le > this.buffer.length) grow(); - System.arraycopy(bb, of, this.buffer, this.offset + pos, le); - if (pos + le > this.length) this.length = pos + le; - } - public ByteBuffer append(final byte b) { write(b); return this; @@ -222,49 +140,11 @@ public final class ByteBuffer extends OutputStream { return append(UTF8.getBytes(s)); } - public ByteBuffer append(final String s, final String charset) throws UnsupportedEncodingException { - return append(s.getBytes(charset)); - } - - public ByteBuffer append(final ByteBuffer bb) { - return append(bb.buffer, bb.offset, bb.length); - } - - public ByteBuffer append(final Object o) { - if (o instanceof String) return append((String) o); - if (o instanceof byte[]) return append((byte[]) o); - return null; - } - public byte byteAt(final int pos) { if (pos > this.length) return -1; return this.buffer[this.offset + pos]; } - public void deleteByteAt(final int pos) { - if (pos < 0) return; - if (pos >= this.length) return; - if (pos == this.length - 1) { - this.length--; - } else { - System.arraycopy(this.buffer, this.offset + pos + 1, this.buffer, this.offset + pos, this.length - pos - 1); - } - } - - public int indexOf(final byte b) { - return indexOf(b, 0); - } - - public int indexOf(final byte[] bs) { - return indexOf(bs, 0); - } - - public int indexOf(final byte b, final int start) { - if (start >= this.length) return -1; - for (int i = start; i < this.length; i++) if (this.buffer[this.offset + i] == b) return i; - return -1; - } - public int indexOf(final byte[] bs, final int start) { if (start + bs.length > this.length) return -1; loop: for (int i = start; i <= this.length - bs.length; i++) { @@ -282,15 +162,6 @@ public final class ByteBuffer extends OutputStream { return -1; } - public int lastIndexOf(final byte b) { - for (int i = this.length - 1; i >= 0; i--) if (this.buffer[this.offset + i] == b) return i; - return -1; - } - - public boolean startsWith(final byte[] bs) { - return startsWith(bs, 0); - } - public boolean startsWith(final byte[] bs, final int start) { if (this.length - start < bs.length) return false; for (int i = 0; i < bs.length; i++) { @@ -303,7 +174,7 @@ public final class ByteBuffer extends OutputStream { return getBytes(0); } - public byte[] getBytes(final int start) { + private byte[] getBytes(final int start) { return getBytes(start, this.length); } @@ -330,83 +201,6 @@ public final class ByteBuffer extends OutputStream { return this; } - public ByteBuffer trim() { - int l = 0; - while ((l < this.length) && (this.buffer[this.offset + l] <= 32)) { - l++; - } - int r = this.length - 1; - while ((r > l) && (this.buffer[this.offset + r] <= 32)) r--; - return trim(l, r - l + 1); - } - - public int isUTF8char(final int start) { - // a sequence of bytes is a utf-8 character, if one of the following 4 conditions is true: - // - ASCII equivalence range; (first) byte begins with zero - // - first byte begins with 110, the following byte begins with 10 - // - first byte begins with 1110, the following two bytes begin with 10 - // - First byte begins with 11110, the following three bytes begin with 10 - // if an utf-8 sequence is detected, the length of the sequence is returned. -1 otherwise - if ((start < this.length) && - ((this.buffer[this.offset + start] & 0x80) != 0)) return 1; - if ((start < this.length - 1) && - ((this.buffer[this.offset + start ] & 0xE0) == 0xC0) && - ((this.buffer[this.offset + start + 1] & 0xC0) == 0x80)) return 2; - if ((start < this.length - 2) && - ((this.buffer[this.offset + start ] & 0xF0) == 0xE0) && - ((this.buffer[this.offset + start + 1] & 0xC0) == 0x80) && - ((this.buffer[this.offset + start + 2] & 0xC0) == 0x80)) return 3; - if ((start < this.length - 3) && - ((this.buffer[this.offset + start ] & 0xF8) == 0xF0) && - ((this.buffer[this.offset + start + 1] & 0xC0) == 0x80) && - ((this.buffer[this.offset + start + 2] & 0xC0) == 0x80) && - ((this.buffer[this.offset + start + 3] & 0xC0) == 0x80)) return 4; - return -1; - } - - public boolean isWhitespace(final boolean includeNonLetterBytes) { - // returns true, if trim() would result in an empty serverByteBuffer - if (includeNonLetterBytes) { - byte b; - for (int i = 0; i < this.length; i++) { - b = this.buffer[this.offset + i]; - if (((b >= '0') && (b <= '9')) || ((b >= 'A') && (b <= 'Z')) || ((b >= 'a') && (b <= 'z'))) return false; - } - } else { - for (int i = 0; i < this.length; i++) if (this.buffer[this.offset + i] > 32) return false; - } - return true; - } - - public int whitespaceStart(final boolean includeNonLetterBytes) { - // returns number of whitespace bytes at the beginning of text - if (includeNonLetterBytes) { - byte b; - for (int i = 0; i < this.length; i++) { - b = this.buffer[this.offset + i]; - if (((b >= '0') && (b <= '9')) || ((b >= 'A') && (b <= 'Z')) || ((b >= 'a') && (b <= 'z'))) return i; - } - } else { - for (int i = 0; i < this.length; i++) if (this.buffer[this.offset + i] > 32) return i; - } - return this.length; - } - - public int whitespaceEnd(final boolean includeNonLetterBytes) { - // returns position of whitespace at the end of text - if (includeNonLetterBytes) { - byte b; - for (int i = this.length - 1; i >= 0; i--) { - b = this.buffer[this.offset + i]; - if (((b >= '0') && (b <= '9')) || ((b >= 'A') && (b <= 'Z')) || ((b >= 'a') && (b <= 'z'))) return i + 1; - } - } else { - for (int i = this.length - 1; i >= 0; i--) if (this.buffer[this.offset + i] > 32) return i + 1; - } - return 0; - } - - @Override public String toString() { return UTF8.String(this.buffer, this.offset, this.length); @@ -425,74 +219,6 @@ public final class ByteBuffer extends OutputStream { return sb; } - public Properties propParser(final String charset) { - // extract a=b or a="b" - relations from the buffer - int pos = this.offset; - int start; - String key; - final Properties p = new Properties(); - // eat up spaces at beginning - while ((pos < this.length) && (this.buffer[pos] <= 32)) pos++; - while (pos < this.length) { - // pos is at start of next key - start = pos; - while ((pos < this.length) && (this.buffer[pos] != equal)) pos++; - if (pos >= this.length) break; // this is the case if we found no equal - try { - key = new String(this.buffer, start, pos - start, charset).trim().toLowerCase(); - } catch (final UnsupportedEncodingException e1) { - key = UTF8.String(this.buffer, start, pos - start).trim().toLowerCase(); - } - // we have a key - pos++; - // find start of value - while ((pos < this.length) && (this.buffer[pos] <= 32)) pos++; - // doublequotes are obligatory. However, we want to be fuzzy if they - // are ommittet - if (pos >= this.length) { - // error case: input ended too early - break; - } else if (this.buffer[pos] == doublequote) { - // search next doublequote - pos++; - start = pos; - while ((pos < this.length) && (this.buffer[pos] != doublequote)) pos++; - if (pos >= this.length) break; // this is the case if we found no parent doublequote - try { - p.setProperty(key, new String(this.buffer, start, pos - start,charset).trim()); - } catch (final UnsupportedEncodingException e) { - p.setProperty(key, UTF8.String(this.buffer, start, pos - start).trim()); - } - pos++; - } else if (this.buffer[pos] == singlequote) { - // search next singlequote - pos++; - start = pos; - while ((pos < this.length) && (this.buffer[pos] != singlequote)) pos++; - if (pos >= this.length) break; // this is the case if we found no parent singlequote - try { - p.setProperty(key, new String(this.buffer, start, pos - start,charset).trim()); - } catch (final UnsupportedEncodingException e) { - p.setProperty(key, UTF8.String(this.buffer, start, pos - start).trim()); - } - pos++; - } else { - // search next whitespace - start = pos; - while ((pos < this.length) && (this.buffer[pos] > 32)) pos++; - try { - p.setProperty(key, new String(this.buffer, start, pos - start,charset).trim()); - } catch (final UnsupportedEncodingException e) { - p.setProperty(key, UTF8.String(this.buffer, start, pos - start).trim()); - } - } - // pos should point now to a whitespace: eat up spaces - while ((pos < this.length) && (this.buffer[pos] <= 32)) pos++; - // go on with next loop - } - return p; - } - public static boolean equals(final byte[] buffer, final byte[] pattern) { // compares two byte arrays: true, if pattern appears completely at offset position if (buffer.length < pattern.length) return false; @@ -500,18 +226,6 @@ public final class ByteBuffer extends OutputStream { return true; } - public static boolean equals(final byte[] buffer, final int offset, final byte[] pattern) { - // compares two byte arrays: true, if pattern appears completely at offset position - if (buffer.length < offset + pattern.length) return false; - for (int i = 0; i < pattern.length; i++) if (buffer[offset + i] != pattern[i]) return false; - return true; - } - - public void reset() { - this.length = 0; - this.offset = 0; - } - public void writeTo(final OutputStream dest) throws IOException { dest.write(this.buffer, this.offset, this.length); dest.flush(); diff --git a/source/net/yacy/kelondro/util/CompressedHashMap.java b/source/net/yacy/kelondro/util/CompressedHashMap.java deleted file mode 100644 index 5252099d4..000000000 --- a/source/net/yacy/kelondro/util/CompressedHashMap.java +++ /dev/null @@ -1,322 +0,0 @@ -// CompressedHashMap.java -// ----------------------- -// part of YaCy -// (C) by Michael Peter Christen; mc@yacy.net -// first published on http://www.anomic.de -// Frankfurt, Germany, 2005 -// Created 08.12.2005 -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.kelondro.util; - -import java.util.Iterator; - -import net.yacy.cora.document.UTF8; - -public class CompressedHashMap { - - int keylen; - int valuelen; - int reclen, count; - byte[] mem; - private final byte[] emptykey; - - public CompressedHashMap(final int valuelen) { - // initializes a hash map with integer access key - this(4, valuelen); - } - - public CompressedHashMap(final int keylen, final int valuelen) { - this.keylen = keylen; - this.valuelen = valuelen; - this.reclen = keylen + valuelen; - this.mem = new byte[1 * reclen]; - this.count = 0; - this.emptykey = new byte[keylen]; - for (int i = 0; i < keylen; i++) emptykey[i] = 0; - for (int i = 0; i < (mem.length / reclen); i++) System.arraycopy(emptykey, 0, mem, i * reclen, keylen); - } - - private boolean equals(final int posKeyInMem, final byte[] otherkey) { - assert (otherkey.length == keylen); - int pos = posKeyInMem * reclen; - int i = 0; - while (i < keylen) if (mem[pos++] != otherkey[i++]) return false; - return true; - } - - private int rehashtry() { - return 1 + mem.length / 2; - } - - private int capacity() { - return mem.length / reclen; - } - - private static int hashkey(final byte[] key, final int capacity) { - int h = 0; - for (int i = 0; i < key.length; i++) h = h * 15 + (0xff & key[i]); - return h % capacity; - } - - private static int rehash(int previousKey, final int capacity) { - if (previousKey == 0) previousKey = capacity; - return previousKey - 1; - } - - public int size() { - return count; - } - - private int findExisting(final byte[] key) { - // returns an index position if found; -1 otherwise - int hash = hashkey(key, capacity()); - int testcount = 0; - while (testcount++ < rehashtry()) { - if (mem[hash * reclen] == 0) return -1; - if (equals(hash, key)) return hash; - hash = rehash(hash, capacity()); - } - return -1; - } - - private int findSpace(final byte[] key) { - // returns an new index position which is empty - // if there is no space left -1 is returned - int hash = hashkey(key, capacity()); - int testcount = 0; - while (testcount++ < rehashtry()) { - if (mem[hash * reclen] == 0) return hash; - hash = rehash(hash, capacity()); - } - return -1; - } - - private static int findSpace(final byte[] m, final byte[] key, final int rl, final int trycount, final int capacity) { - // returns an new index position which is empty - // if there is no space left -1 is returned - int hash = hashkey(key, capacity); - int testcount = 0; - while (testcount++ < trycount) { - if (m[hash * rl] == 0) return hash; - hash = rehash(hash, capacity); - } - return -1; - } - - private static byte[] toByteKey(int v) { - assert (v >= 0); - v = v | 0x80000000; // set bit in first byte - return new byte[]{(byte) ((v >>> 24) & 0xFF), (byte) ((v >>> 16) & 0xFF), - (byte) ((v >>> 8) & 0xFF), (byte) ((v >>> 0) & 0xFF)}; - } - - public void put(final int key, final byte[] value) { - put(toByteKey(key), value); - } - - public void put(final byte[] key, final byte[] value) { - // inserts a new value or overwrites existing - // if the hash is full, a RuntimeException is thrown - // this method does not return the old value to avoid generation of objects - assert (key.length == keylen); - assert (value.length == valuelen); - - int hash = findExisting(key); - if (hash < 0) { - // insert new entry - hash = findSpace(key); - if (hash < 0) { - // increase space of hashtable - // create temporary bigger hashtable and insert all - synchronized (mem) { - System.out.println("increasing space to " + mem.length * 2); - final int newspace = mem.length * 2; - final int newcapacity = capacity() * 2; - byte[] newmem = new byte[newspace]; - final Iterator i = entries(); - CompressedHashMap.entry e; - int mempos; - while (i.hasNext()) { - e = i.next(); - hash = findSpace(newmem, e.key, reclen, newspace, newcapacity); - mempos = hash * reclen; - System.arraycopy(e.key, 0, newmem, mempos, keylen); - System.arraycopy(e.value, 0, newmem, mempos + keylen, valuelen); - } - // finally insert new value - hash = findSpace(newmem, key, reclen, newspace, newcapacity); - mempos = hash * reclen; - System.arraycopy(key, 0, newmem, mempos, keylen); - System.arraycopy(value, 0, newmem, mempos + keylen, valuelen); - // move newmem to mem - mem = newmem; - } - } else { - // there is enough space - final int mempos = hash * reclen; - System.arraycopy(key, 0, mem, mempos, keylen); - System.arraycopy(value, 0, mem, mempos + keylen, valuelen); - } - count++; - } else { - // overwrite old entry - final int mempos = hash * reclen; - System.arraycopy(key, 0, mem, mempos, keylen); - System.arraycopy(value, 0, mem, mempos + keylen, valuelen); - } - } - - public byte[] get(final int key) { - return get(toByteKey(key)); - } - - public byte[] get(final byte[] key) { - assert (key.length == keylen); - - final int hash = findExisting(key); - if (hash < 0) { - return null; - } - // read old entry - final byte[] value = new byte[valuelen]; - System.arraycopy(mem, hash * reclen + keylen, value, 0, valuelen); - return value; - } - - public void remove(final int key) { - remove(toByteKey(key)); - } - - public void remove(final byte[] key) { - assert (key.length == keylen); - - System.out.println("REMOVE!"); - final int hash = findExisting(key); - if (hash >= 0) { - // overwrite old key - System.arraycopy(emptykey, 0, mem, hash * reclen, keylen); - count--; - } - } - - Iterator entries() { - return new entryIterator(); - } - - public class entryIterator implements Iterator { - - int hashkey; - - public entryIterator() { - hashkey = anyhashpos(0); - } - - public boolean hasNext() { - return hashkey >= 0; - } - - public entry next() { - final int i = hashkey; - hashkey = anyhashpos(hashkey + 1); - return new entry(i); - } - - public void remove() { - mem[hashkey * reclen] = 0; - } - - } - - public void removeany() { - //System.out.println("CALLED REMOVEANY"); - int start = 0; - while (start < capacity()) { - if (mem[start * reclen] != 0) { - System.arraycopy(emptykey,0, mem, start * reclen, keylen); - count--; - return; - } - start++; - } - return; - } - - protected int anyhashpos(int start) { - while (start < capacity()) { - if (mem[start * reclen] != 0) return start; - start++; - } - return -1; - } - - public byte[] anykey() { - int hash = 0; - int mempos; - while (hash < capacity()) { - mempos = hash * reclen; - if (mem[mempos] != 0) { - final byte[] key = new byte[keylen]; - System.arraycopy(mem, mempos, key, 0, keylen); - return key; - } - hash++; - } - return null; - } - - public class entry { - public byte[] key, value; - - public entry(final int index) { - this.key = new byte[keylen]; - this.value = new byte[valuelen]; - final int mempos = index * (reclen); - System.arraycopy(mem, mempos, key, 0, keylen); - System.arraycopy(mem, mempos + keylen, value, 0, valuelen); - } - - public entry(final byte[] key, final byte[] value) { - this.key = key; - this.value = value; - } - } - - - public static void main(final String[] args) { - final long start = System.currentTimeMillis(); - final CompressedHashMap map = new CompressedHashMap(4); - for (int i = 0; i < 100; i++) map.put(3333 + i, Integer.toString(1000 + i).getBytes()); - final Iterator i = map.entries(); - CompressedHashMap.entry e; - System.out.println("Enumeration of elements: count=" + map.size()); - int c = 0; - while (i.hasNext()) { - e = i.next(); - System.out.println("key=" + UTF8.String(e.key) + ", value=" + UTF8.String(e.value) + ", retrieved=" + UTF8.String(map.get(e.key))); - c++; - } - System.out.println("c = " + c + "; re-catch:"); - for (int j = 0; j < 100; j++) { - System.out.println("key=" + j + ", retrieved=" + UTF8.String(map.get(3333 + j))); - } - System.out.println("runtime = " + (System.currentTimeMillis() - start)); - } -} diff --git a/source/net/yacy/kelondro/util/ConsoleInterface.java b/source/net/yacy/kelondro/util/ConsoleInterface.java index 932715b7b..0eb84aba4 100644 --- a/source/net/yacy/kelondro/util/ConsoleInterface.java +++ b/source/net/yacy/kelondro/util/ConsoleInterface.java @@ -42,7 +42,7 @@ public class ConsoleInterface extends Thread { private final Log log; - public ConsoleInterface(final InputStream stream, final Log log) { + private ConsoleInterface(final InputStream stream, final Log log) { this.log = log; this.stream = stream; // block reading {@see getOutput()} @@ -106,7 +106,7 @@ public class ConsoleInterface extends Thread { * @return console output * @throws IOException */ - public static List getConsoleOutput(final List processArgs, Log log) throws IOException { + private static List getConsoleOutput(final List processArgs, Log log) throws IOException { final ProcessBuilder processBuilder = new ProcessBuilder(processArgs); Process process = null; ConsoleInterface inputStream = null; diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index d31d2fd02..612ef8a75 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -24,8 +24,6 @@ package net.yacy.kelondro.util; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.ByteArrayInputStream; @@ -41,36 +39,26 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintWriter; import java.io.Reader; -import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.io.Writer; import java.nio.charset.Charset; import java.util.ArrayList; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.StringTokenizer; -import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import net.yacy.cora.document.UTF8; import net.yacy.cora.storage.Files; -import net.yacy.kelondro.index.Row; -import net.yacy.kelondro.index.RowSet; import net.yacy.kelondro.logging.Log; -public final class FileUtils -{ +public final class FileUtils { private static final int DEFAULT_BUFFER_SIZE = 1024; // this is also the maximum chunk size @@ -119,22 +107,6 @@ public final class FileUtils return total; } - public static int copy(final File source, final Charset inputCharset, final Writer dest) - throws IOException { - InputStream fis = null; - try { - fis = new FileInputStream(source); - return copy(fis, dest, inputCharset); - } finally { - if ( fis != null ) { - try { - fis.close(); - } catch (Exception e ) { - } - } - } - } - public static int copy(final InputStream source, final Writer dest) throws IOException { final InputStreamReader reader = new InputStreamReader(source); return copy(reader, dest); @@ -333,62 +305,6 @@ public final class FileUtils return buffer; } - public static byte[] readAndZip(final File source) throws IOException { - ByteArrayOutputStream byteOut = null; - GZIPOutputStream zipOut = null; - try { - byteOut = new ByteArrayOutputStream((int) (source.length() / 2)); - zipOut = new GZIPOutputStream(byteOut); - copy(source, zipOut); - zipOut.close(); - return byteOut.toByteArray(); - } finally { - if ( zipOut != null ) { - try { - zipOut.close(); - } catch ( final Exception e ) { - } - } - if ( byteOut != null ) { - try { - byteOut.close(); - } catch ( final Exception e ) { - } - } - } - } - - public static void writeAndGZip(final byte[] source, final File dest) throws IOException { - FileOutputStream fos = null; - try { - fos = new FileOutputStream(dest); - writeAndGZip(source, fos); - } finally { - if ( fos != null ) { - try { - fos.close(); - } catch ( final Exception e ) { - } - } - } - } - - public static void writeAndGZip(final byte[] source, final OutputStream dest) throws IOException { - GZIPOutputStream zipOut = null; - try { - zipOut = new GZIPOutputStream(dest); - copy(source, zipOut); - zipOut.close(); - } finally { - if ( zipOut != null ) { - try { - zipOut.close(); - } catch ( final Exception e ) { - } - } - } - } - /** * This function determines if a byte array is gzip compressed and uncompress it * @@ -524,102 +440,6 @@ public final class FileUtils saveMap(file, m, comment); } - public static Set loadSet(final File file, final int chunksize, final boolean tree) - throws IOException { - final Set set = - (tree) ? (Set) new TreeSet() : (Set) new HashSet(); - final byte[] b = read(file); - for ( int i = 0; (i + chunksize) <= b.length; i++ ) { - set.add(UTF8.String(b, i, chunksize)); - } - return set; - } - - public static Set loadSet(final File file, final String sep, final boolean tree) - throws IOException { - final Set set = - (tree) ? (Set) new TreeSet() : (Set) new HashSet(); - final byte[] b = read(file); - final StringTokenizer st = new StringTokenizer(UTF8.String(b), sep); - while ( st.hasMoreTokens() ) { - set.add(st.nextToken()); - } - return set; - } - - public static void saveSet(final File file, final String format, final Set set, final String sep) throws IOException { - final File tf = new File(file.toString() + ".prt" + (System.currentTimeMillis() % 1000)); - OutputStream os = null; - if ( (format == null) || (format.equals("plain")) ) { - os = new BufferedOutputStream(new FileOutputStream(tf)); - } else if ( format.equals("gzip") ) { - os = new GZIPOutputStream(new FileOutputStream(tf)); - } else if ( format.equals("zip") ) { - @SuppressWarnings("resource") - final ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(file)); - String name = file.getName(); - if ( name.endsWith(".zip") ) { - name = name.substring(0, name.length() - 4); - } - zos.putNextEntry(new ZipEntry(name + ".txt")); - os = zos; - } - IOException ex = null; - if ( os != null ) try { - for ( final byte[] b : set ) { - os.write(b); - if ( sep != null ) { - os.write(UTF8.getBytes(sep)); - } - } - os.close(); - } catch (IOException e) { - ex = e; - } finally { - os.close(); - } - if (ex != null) throw ex; - forceMove(tf, file); - } - - public static void saveSet(final File file, final String format, final RowSet set, final String sep) throws IOException { - final File tf = new File(file.toString() + ".prt" + (System.currentTimeMillis() % 1000)); - OutputStream os = null; - if ( (format == null) || (format.equals("plain")) ) { - os = new BufferedOutputStream(new FileOutputStream(tf)); - } else if ( format.equals("gzip") ) { - os = new GZIPOutputStream(new FileOutputStream(tf)); - } else if ( format.equals("zip") ) { - @SuppressWarnings("resource") - final ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(file)); - String name = file.getName(); - if ( name.endsWith(".zip") ) { - name = name.substring(0, name.length() - 4); - } - zos.putNextEntry(new ZipEntry(name + ".txt")); - os = zos; - } - IOException ex = null; - if ( os != null ) try { - final Iterator i = set.iterator(); - if ( i.hasNext() ) { - os.write(i.next().getPrimaryKeyBytes()); - } - while ( i.hasNext() ) { - if ( sep != null ) { - os.write(UTF8.getBytes(sep)); - } - os.write(i.next().getPrimaryKeyBytes()); - } - } catch (IOException e) { - ex = e; - } finally { - os.close(); - } - if (ex != null) throw ex; - forceMove(tf, file); - } - public static ConcurrentHashMap table(final Reader r) { final BufferedReader br = new BufferedReader(r); return table(new StringsIterator(br)); @@ -711,7 +531,7 @@ public final class FileUtils * @param out the String to write * @return returns true if successful, false otherwise */ - public static boolean writeList(final File listFile, final String out) { + private static boolean writeList(final File listFile, final String out) { BufferedWriter bw = null; try { bw = new BufferedWriter(new PrintWriter(new FileWriter(listFile))); @@ -730,8 +550,8 @@ public final class FileUtils } } - public static final char LF = (char) 10; - public static final char CR = (char) 13; + private static final char LF = (char) 10; + private static final char CR = (char) 13; /** * Read lines of a text file into a String, optionally ignoring comments. @@ -839,7 +659,7 @@ public final class FileUtils /** * Returns a List of all dirs and subdirs as File Objects Warning: untested */ - public static ArrayList getDirsRecursive( + private static ArrayList getDirsRecursive( final File dir, final String notdir, final boolean excludeDotfiles) { @@ -877,12 +697,11 @@ public final class FileUtils return FileUtils.writeList(listFile, new String(out)); //(File, String) } - public static class StringsIterator implements Iterator - { + private static class StringsIterator implements Iterator { private BufferedReader reader; private String nextLine; - public StringsIterator(final BufferedReader reader) { + private StringsIterator(final BufferedReader reader) { this.reader = reader; this.nextLine = null; next(); @@ -939,52 +758,6 @@ public final class FileUtils } } - /** - * Moves all files from a directory to another. - * - * @param from_dir Directory which contents will be moved. - * @param to_dir Directory to move into. It must exist already. - */ - public static void moveAll(final File from_dir, final File to_dir) { - if ( !(from_dir.isDirectory()) ) { - return; - } - if ( !(to_dir.isDirectory()) ) { - return; - } - final String[] list = from_dir.list(); - for ( int i = 0; i < list.length; i++ ) { - if ( !new File(from_dir, list[i]).renameTo(new File(to_dir, list[i])) ) { - Log.logWarning("serverFileUtils", "moveAll(): could not move from " - + from_dir - + list[i] - + " to " - + to_dir - + list[i]); - } - } - } - - public static class dirlistComparator implements Comparator, Serializable - { - - /** - * generated serial - */ - private static final long serialVersionUID = -5196490300039230135L; - - @Override - public int compare(final File file1, final File file2) { - if ( file1.isDirectory() && !file2.isDirectory() ) { - return -1; - } else if ( !file1.isDirectory() && file2.isDirectory() ) { - return 1; - } else { - return file1.getName().compareToIgnoreCase(file2.getName()); - } - } - } - public static final File createTempFile(final Class classObj, final String name) throws IOException { String parserClassName = classObj.getName(); int idx = parserClassName.lastIndexOf('.'); @@ -1006,107 +779,7 @@ public final class FileUtils (!fileExt.isEmpty()) ? "." + fileExt : fileExt); return tempFile; } - - /** - * copies the input stream to one output stream (byte per byte) - * - * @param in - * @param out - * @return number of copies bytes - * @throws IOException - */ - public static int copyToStream(final BufferedInputStream in, final BufferedOutputStream out) - throws IOException { - int count = 0; - // copy bytes - int b; - while ( (b = in.read()) != -1 ) { - count++; - out.write(b); - } - out.flush(); - return count; - } - - /** - * copies the input stream to both output streams (byte per byte) - * - * @param in - * @param out0 - * @param out1 - * @return number of copies bytes - * @throws IOException - */ - public static int copyToStreams( - final BufferedInputStream in, - final BufferedOutputStream out0, - final BufferedOutputStream out1) throws IOException { - assert out0 != null; - assert out1 != null; - - int count = 0; - // copy bytes - int b; - while ( (b = in.read()) != -1 ) { - count++; - out0.write(b); - out1.write(b); - } - out0.flush(); - out1.flush(); - return count; - } - - /** - * copies the input stream to all writers (byte per byte) - * - * @param data - * @param writer - * @param charSet - * @return - * @throws IOException - */ - public static int copyToWriter( - final BufferedInputStream data, - final BufferedWriter writer, - final Charset charSet) throws IOException { - // the docs say: "For top efficiency, consider wrapping an InputStreamReader within a BufferedReader." - final Reader sourceReader = new InputStreamReader(data, charSet); - - int count = 0; - // copy bytes - int b; - while ( (b = sourceReader.read()) != -1 ) { - count++; - writer.write(b); - } - writer.flush(); - return count; - } - - public static int copyToWriters( - final BufferedInputStream data, - final BufferedWriter writer0, - final BufferedWriter writer1, - final Charset charSet) throws IOException { - // the docs say: "For top efficiency, consider wrapping an InputStreamReader within a BufferedReader." - assert writer0 != null; - assert writer1 != null; - final Reader sourceReader = new InputStreamReader(data, charSet); - - int count = 0; - // copy bytes - int b; - while ( (b = sourceReader.read()) != -1 ) { - count++; - writer0.write(b); - writer1.write(b); - } - writer0.flush(); - writer1.flush(); - return count; - } - + /** * delete files and directories if a directory is not empty, delete also everything inside because * deletion sometimes fails on windows, there is also a windows exec included @@ -1174,12 +847,4 @@ public final class FileUtils } } - public static void main(final String[] args) { - try { - writeAndGZip("ein zwei drei, Zauberei".getBytes(), new File("zauberei.txt.gz")); - } catch ( final IOException e ) { - Log.logException(e); - } - } - } diff --git a/source/net/yacy/kelondro/util/Formatter.java b/source/net/yacy/kelondro/util/Formatter.java index 5277904aa..52432b951 100644 --- a/source/net/yacy/kelondro/util/Formatter.java +++ b/source/net/yacy/kelondro/util/Formatter.java @@ -69,7 +69,7 @@ public final class Formatter { * @param locale the {@link Locale} to set or null to set the special * empty locale to create unformatted numbers */ - public static void setLocale(final Locale l) { + private static void setLocale(final Locale l) { locale = l; } @@ -86,7 +86,7 @@ public final class Formatter { return (localized ? numForm.get().format(d) : cleanNumForm.get().format(d)); } - public static String number(final double d) { + private static String number(final double d) { return numForm.get().format(d); } diff --git a/source/net/yacy/kelondro/util/ISO639.java b/source/net/yacy/kelondro/util/ISO639.java index c66f3176e..16ba36653 100644 --- a/source/net/yacy/kelondro/util/ISO639.java +++ b/source/net/yacy/kelondro/util/ISO639.java @@ -31,7 +31,7 @@ import java.util.concurrent.ConcurrentHashMap; public class ISO639 { - static final String[] codes = { + private static final String[] codes = { "aa-Afar", "ab-Abkhazian", "af-Afrikaans", @@ -169,7 +169,7 @@ public class ISO639 { "zh-Chinese", "zu-Zulu"}; - static Map mapping = new ConcurrentHashMap(codes.length); + private static Map mapping = new ConcurrentHashMap(codes.length); static { for (int i = 0; i < codes.length; i++) { diff --git a/source/net/yacy/kelondro/util/OS.java b/source/net/yacy/kelondro/util/OS.java index 8d8ad407d..09d82aae0 100644 --- a/source/net/yacy/kelondro/util/OS.java +++ b/source/net/yacy/kelondro/util/OS.java @@ -54,9 +54,9 @@ public final class OS { public static final String blankTypeString = "____"; // system-identification statics - public static final System systemOS; + private static final System systemOS; public static final boolean isMacArchitecture; - public static final boolean isUnixFS; + private static final boolean isUnixFS; public static final boolean canExecUnix; public static final boolean isWindows; public static final boolean isWin32; @@ -95,7 +95,7 @@ public final class OS { * @return heap in -Xmx[heap]m * @author [DW], 07.02.2009 */ - public static int getWin32MaxHeap() { + private static int getWin32MaxHeap() { int maxmem = 1000; while(checkWin32Heap(maxmem)) maxmem += 100; while(!checkWin32Heap(maxmem)) maxmem -= 10; @@ -108,7 +108,7 @@ public final class OS { * @return true if possible * @author [DW], 07.02.2009 */ - public static boolean checkWin32Heap(final int mem){ + private static boolean checkWin32Heap(final int mem){ String line = ""; final List processArgs = new ArrayList(); processArgs.add("java"); diff --git a/source/net/yacy/kelondro/util/ReverseMapIterator.java b/source/net/yacy/kelondro/util/ReverseMapIterator.java index 509d7a68a..c8cab3bd8 100644 --- a/source/net/yacy/kelondro/util/ReverseMapIterator.java +++ b/source/net/yacy/kelondro/util/ReverseMapIterator.java @@ -32,8 +32,8 @@ import java.util.Iterator; import java.util.Map; public class ReverseMapIterator implements Iterator> { - ArrayList a; - Map map; + private ArrayList a; + private Map map; E last; public ReverseMapIterator(Map map) { @@ -66,7 +66,7 @@ public class ReverseMapIterator implements Iterator> { this.map.remove(this.last); } - public class Entry0 implements Map.Entry { + private class Entry0 implements Map.Entry { E e; F f; public Entry0(final E e, final F f) { diff --git a/source/net/yacy/kelondro/util/RotateIterator.java b/source/net/yacy/kelondro/util/RotateIterator.java index fc419cd2d..c8a429a7f 100644 --- a/source/net/yacy/kelondro/util/RotateIterator.java +++ b/source/net/yacy/kelondro/util/RotateIterator.java @@ -31,9 +31,9 @@ import net.yacy.cora.order.CloneableIterator; public class RotateIterator implements CloneableIterator { - CloneableIterator a, clone; + private CloneableIterator a, clone; Object modifier; - boolean nempty; + private boolean nempty; int terminationCount; public RotateIterator(final CloneableIterator a, final Object modifier, final int terminationCount) { diff --git a/source/net/yacy/kelondro/workflow/AbstractBusyThread.java b/source/net/yacy/kelondro/workflow/AbstractBusyThread.java index 9f26d0be2..aa97b5a70 100644 --- a/source/net/yacy/kelondro/workflow/AbstractBusyThread.java +++ b/source/net/yacy/kelondro/workflow/AbstractBusyThread.java @@ -53,10 +53,6 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT this.minBusySleep = minBusySleep; this.maxBusySleep = maxBusySleep; } - - protected final void announceMoreSleepTime(final long millis) { - this.idletime += millis; - } public final void setStartupSleep(final long milliseconds) { // sets a sleep time before execution of the job-loop diff --git a/source/net/yacy/kelondro/workflow/AbstractThread.java b/source/net/yacy/kelondro/workflow/AbstractThread.java index b2945d3ca..b52e246ef 100644 --- a/source/net/yacy/kelondro/workflow/AbstractThread.java +++ b/source/net/yacy/kelondro/workflow/AbstractThread.java @@ -40,18 +40,14 @@ import net.yacy.kelondro.logging.Log; public abstract class AbstractThread extends Thread implements WorkflowThread { private static Log log = new Log("WorkflowThread"); - protected boolean running = true, announcedShutdown = false; + protected boolean running = true; + private boolean announcedShutdown = false; protected long busytime = 0, memuse = 0; private long blockPause = 0; private String shortDescr = "", longDescr = ""; private String monitorURL = null; private long threadBlockTimestamp = System.currentTimeMillis(); - - protected final boolean announceShutdown() { - return this.announcedShutdown; - } - protected final void announceThreadBlockApply() { // shall only be used, if a thread blocks for an important reason // like a socket connect and must renew the timestamp to correct @@ -68,10 +64,6 @@ public abstract class AbstractThread extends Thread implements WorkflowThread { this.busytime -= thisBlockTime; } - protected final void announceMoreExecTime(final long millis) { - this.busytime += millis; - } - @Override public final void setDescription(final String shortText, final String longText, final String monitorURL) { // sets a visible description string diff --git a/source/net/yacy/kelondro/workflow/BusyThread.java b/source/net/yacy/kelondro/workflow/BusyThread.java index c9577e4e5..787c62e34 100644 --- a/source/net/yacy/kelondro/workflow/BusyThread.java +++ b/source/net/yacy/kelondro/workflow/BusyThread.java @@ -99,5 +99,4 @@ public interface BusyThread extends WorkflowThread { */ public void freemem(); - public void notifyThread(); } diff --git a/source/net/yacy/kelondro/workflow/InstantBlockingThread.java b/source/net/yacy/kelondro/workflow/InstantBlockingThread.java index 0be898322..20870f90f 100644 --- a/source/net/yacy/kelondro/workflow/InstantBlockingThread.java +++ b/source/net/yacy/kelondro/workflow/InstantBlockingThread.java @@ -39,8 +39,8 @@ public class InstantBlockingThread extends AbstractBlocki private final Object environment; private final Long handle; private static int handleCounter = 0; - public static int instantThreadCounter = 0; - public static final ConcurrentMap jobs = new ConcurrentHashMap(); + private static int instantThreadCounter = 0; + private static final ConcurrentMap jobs = new ConcurrentHashMap(); public InstantBlockingThread(final Object env, final String jobExec, final WorkflowProcessor manager) { // jobExec is the name of a method of the object 'env' that executes the one-step-run @@ -56,20 +56,6 @@ public class InstantBlockingThread extends AbstractBlocki this.handle = Long.valueOf(System.currentTimeMillis() + getName().hashCode()); } - public InstantBlockingThread(final Object env, final Method jobExecMethod, final WorkflowProcessor manager) { - // jobExec is the name of a method of the object 'env' that executes the one-step-run - // jobCount is the name of a method that returns the size of the job - - // set the manager of blocking queues for input and output - setManager(manager); - - // define execution class - this.jobExecMethod = jobExecMethod; - this.environment = (env instanceof Class) ? null : env; - setName(jobExecMethod.getClass().getName() + "." + jobExecMethod.getName() + "." + handleCounter++); - this.handle = Long.valueOf(System.currentTimeMillis() + getName().hashCode()); - } - protected static Method execMethod(final Object env, final String jobExec) { final Class theClass = (env instanceof Class) ? (Class) env : env.getClass(); try { diff --git a/source/net/yacy/kelondro/workflow/InstantBusyThread.java b/source/net/yacy/kelondro/workflow/InstantBusyThread.java index e91b6cf7f..d0ae39290 100644 --- a/source/net/yacy/kelondro/workflow/InstantBusyThread.java +++ b/source/net/yacy/kelondro/workflow/InstantBusyThread.java @@ -37,8 +37,8 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT private final Object environment; private final Long handle; - public static int instantThreadCounter = 0; - public static final TreeMap jobs = new TreeMap(); + private static int instantThreadCounter = 0; + private static final TreeMap jobs = new TreeMap(); public InstantBusyThread( final Object env, @@ -175,12 +175,7 @@ public final class InstantBusyThread extends AbstractBusyThread implements BusyT public static WorkflowThread oneTimeJob(final Runnable thread, final long startupDelay) { return oneTimeJob(thread, "run", startupDelay); } - - public static WorkflowThread oneTimeJob(final Runnable thread, final long startupDelay, final int maxJobs) { - while (instantThreadCounter >= maxJobs) try {Thread.sleep(100);} catch (final InterruptedException e) {break;} - return oneTimeJob( thread, startupDelay); - } - + @Override public void open() { // Not implemented in this thread diff --git a/source/net/yacy/kelondro/workflow/WorkflowJob.java b/source/net/yacy/kelondro/workflow/WorkflowJob.java index 7f9b538c7..d2302f4ca 100644 --- a/source/net/yacy/kelondro/workflow/WorkflowJob.java +++ b/source/net/yacy/kelondro/workflow/WorkflowJob.java @@ -38,7 +38,7 @@ public class WorkflowJob { this.status = STATUS_INITIATED; } - public WorkflowJob(final int status) { + private WorkflowJob(final int status) { this.status = status; } diff --git a/source/net/yacy/kelondro/workflow/WorkflowProcessor.java b/source/net/yacy/kelondro/workflow/WorkflowProcessor.java index 926840f97..b4ec8261f 100644 --- a/source/net/yacy/kelondro/workflow/WorkflowProcessor.java +++ b/source/net/yacy/kelondro/workflow/WorkflowProcessor.java @@ -125,7 +125,7 @@ public class WorkflowProcessor { } } - public synchronized void relaxCapacity() { + private synchronized void relaxCapacity() { if (this.input.isEmpty()) { return; }