diff --git a/source/de/anomic/yacy/yacyAccessible.java b/source/de/anomic/yacy/yacyAccessible.java index aa666697d..74a20c1ad 100644 --- a/source/de/anomic/yacy/yacyAccessible.java +++ b/source/de/anomic/yacy/yacyAccessible.java @@ -5,6 +5,7 @@ // Frankfurt, Germany, 2004 // // this file is contributed by Stephan Hermens +// // $LastChangedDate$ // $LastChangedRevision$ // $LastChangedBy$ diff --git a/source/de/anomic/yacy/yacyDHTAction.java b/source/de/anomic/yacy/yacyDHTAction.java index 5a54ca649..460eb992c 100644 --- a/source/de/anomic/yacy/yacyDHTAction.java +++ b/source/de/anomic/yacy/yacyDHTAction.java @@ -3,7 +3,10 @@ // (C) by Michael Peter Christen; mc@anomic.de // first published on http://www.anomic.de // Frankfurt, Germany, 2005 -// last major change: 23.02.2005 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/source/de/anomic/yacy/yacyNewsAction.java b/source/de/anomic/yacy/yacyNewsAction.java index e392c150a..ad7b0a5d8 100644 --- a/source/de/anomic/yacy/yacyNewsAction.java +++ b/source/de/anomic/yacy/yacyNewsAction.java @@ -1,84 +1,87 @@ -// yacyNewsAction.java -// ----------------------- -// part of YaCy -// (C) by Michael Peter Christen; mc@anomic.de -// first published on http://www.anomic.de -// Frankfurt, Germany, 2005 -// last major change: 29.07.2005 -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// Using this software in any meaning (reading, learning, copying, compiling, -// running) means that you agree that the Author(s) is (are) not responsible -// for cost, loss of data or any harm that may be caused directly or indirectly -// by usage of this softare or this documentation. The usage of this software -// is on your own risk. The installation and usage (starting/running) of this -// software may allow other people or application to access your computer and -// any attached devices and is highly dependent on the configuration of the -// software which must be done by the user of the software; the author(s) is -// (are) also not responsible for proper configuration and usage of the -// software, even if provoked by documentation provided together with -// the software. -// -// Any changes to this file according to the GPL as documented in the file -// gpl.txt aside this file in the shipment you received can be done to the -// lines that follows this copyright notice here, but changes must not be -// done inside the copyright notice above. A re-distribution must contain -// the intact and unchanged copyright notice. -// Contributions and changes to the program code must be marked as such. - -package de.anomic.yacy; - -import java.io.IOException; - -import de.anomic.server.serverCodings; -import de.anomic.server.logging.serverLog; - -public class yacyNewsAction implements yacyPeerAction { - - yacyNewsPool pool; - - public yacyNewsAction(yacyNewsPool pool) { - this.pool = pool; - } - - public void processPeerArrival(yacySeed peer, boolean direct) { - String recordString = peer.get("news", null); - //System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached")); - if ((recordString == null) || (recordString.length() == 0)) return; - String decodedString = de.anomic.tools.crypt.simpleDecode(recordString, ""); - yacyNewsRecord record = new yacyNewsRecord(decodedString); - //System.out.println("### news arrival from peer " + peer.getName() + ", decoded=" + decodedString + ", record=" + recordString + ", news=" + record.toString()); - String cre1 = (String) serverCodings.string2map(decodedString).get("cre"); - String cre2 = (String) serverCodings.string2map(record.toString()).get("cre"); - if ((cre1 == null) || (cre2 == null) || (!(cre1.equals(cre2)))) { - System.out.println("### ERROR - cre are not equal: cre1=" + cre1 + ", cre2=" + cre2); - return; - } - try { - synchronized (pool) {this.pool.enqueueIncomingNews(record);} - } catch (IOException e) { - serverLog.logSevere("YACY", "processPeerArrival", e); - } - } - - public void processPeerDeparture(yacySeed peer) { - } - - public void processPeerPing(yacySeed peer) { - processPeerArrival(peer, true); - } - -} +// yacyNewsAction.java +// ----------------------- +// part of YaCy +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2005 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// Any changes to this file according to the GPL as documented in the file +// gpl.txt aside this file in the shipment you received can be done to the +// lines that follows this copyright notice here, but changes must not be +// done inside the copyright notice above. A re-distribution must contain +// the intact and unchanged copyright notice. +// Contributions and changes to the program code must be marked as such. + +package de.anomic.yacy; + +import java.io.IOException; + +import de.anomic.server.serverCodings; +import de.anomic.server.logging.serverLog; + +public class yacyNewsAction implements yacyPeerAction { + + yacyNewsPool pool; + + public yacyNewsAction(yacyNewsPool pool) { + this.pool = pool; + } + + public void processPeerArrival(yacySeed peer, boolean direct) { + String recordString = peer.get("news", null); + //System.out.println("### triggered news arrival from peer " + peer.getName() + ", news " + ((recordString == null) ? "empty" : "attached")); + if ((recordString == null) || (recordString.length() == 0)) return; + String decodedString = de.anomic.tools.crypt.simpleDecode(recordString, ""); + yacyNewsRecord record = new yacyNewsRecord(decodedString); + //System.out.println("### news arrival from peer " + peer.getName() + ", decoded=" + decodedString + ", record=" + recordString + ", news=" + record.toString()); + String cre1 = (String) serverCodings.string2map(decodedString).get("cre"); + String cre2 = (String) serverCodings.string2map(record.toString()).get("cre"); + if ((cre1 == null) || (cre2 == null) || (!(cre1.equals(cre2)))) { + System.out.println("### ERROR - cre are not equal: cre1=" + cre1 + ", cre2=" + cre2); + return; + } + try { + synchronized (pool) {this.pool.enqueueIncomingNews(record);} + } catch (IOException e) { + serverLog.logSevere("YACY", "processPeerArrival", e); + } + } + + public void processPeerDeparture(yacySeed peer) { + } + + public void processPeerPing(yacySeed peer) { + processPeerArrival(peer, true); + } + +} \ No newline at end of file diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index 5abbc1bcc..81e92104a 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -1,189 +1,192 @@ -// yacyNewsQueue.java -// ----------------------- -// part of YaCy -// (C) by Michael Peter Christen; mc@anomic.de -// first published on http://www.anomic.de -// Frankfurt, Germany, 2005 -// last major change: 13.07.2005 -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// Using this software in any meaning (reading, learning, copying, compiling, -// running) means that you agree that the Author(s) is (are) not responsible -// for cost, loss of data or any harm that may be caused directly or indirectly -// by usage of this softare or this documentation. The usage of this software -// is on your own risk. The installation and usage (starting/running) of this -// software may allow other people or application to access your computer and -// any attached devices and is highly dependent on the configuration of the -// software which must be done by the user of the software; the author(s) is -// (are) also not responsible for proper configuration and usage of the -// software, even if provoked by documentation provided together with -// the software. -// -// Any changes to this file according to the GPL as documented in the file -// gpl.txt aside this file in the shipment you received can be done to the -// lines that follows this copyright notice here, but changes must not be -// done inside the copyright notice above. A re-distribution must contain -// the intact and unchanged copyright notice. -// Contributions and changes to the program code must be marked as such. - -package de.anomic.yacy; - -import java.io.File; -import java.io.IOException; -import java.util.Iterator; - -import de.anomic.yacy.yacyCore; -import de.anomic.kelondro.kelondroBase64Order; -import de.anomic.kelondro.kelondroTree; -import de.anomic.kelondro.kelondroException; -import de.anomic.server.serverCodings; -import de.anomic.server.serverDate; - -public class yacyNewsDB { - - private File path; - private int bufferkb; - private kelondroTree news; - - public static final int attributesMaxLength = yacyNewsRecord.maxNewsRecordLength - - yacyNewsRecord.idLength() - - yacyNewsRecord.categoryStringLength - - yacyCore.universalDateShortPattern.length() - - 2; - - public yacyNewsDB(File path, int bufferkb) { - this.path = path; - this.bufferkb = bufferkb; - - if (path.exists()) try { - news = new kelondroTree(path, bufferkb * 0x400); - } catch (IOException e) { - news = createDB(path, bufferkb); - } else { - news = createDB(path, bufferkb); - } - } - - private static kelondroTree createDB(File path, int bufferkb) { - return new kelondroTree(path, bufferkb * 0x400, new int[] { - yacyNewsRecord.idLength(), // id = created + originator - yacyNewsRecord.categoryStringLength, // category - yacyCore.universalDateShortPattern.length(), // received - 2, - attributesMaxLength - }, true); - } - - private void resetDB() { - try {close();} catch (Exception e) {} - if (path.exists()) path.delete(); - news = createDB(path, bufferkb); - } - - public int[] dbCacheChunkSize() { - return news.cacheChunkSize(); - } - - public int[] dbCacheFillStatus() { - return news.cacheFillStatus(); - } - - public void close() { - if (news != null) try {news.close();} catch (IOException e) {} - news = null; - } - - public void finalize() { - close(); - } - - public int size() { - return news.size(); - } - - public void remove(String id) throws IOException { - news.remove(id.getBytes()); - } - - public synchronized yacyNewsRecord put(yacyNewsRecord record) throws IOException { - try { - return b2r(news.put(r2b(record))); - } catch (kelondroException e) { - resetDB(); - return b2r(news.put(r2b(record))); - } - } - - public synchronized Iterator news() throws IOException { - // the iteration iterates yacyNewsRecord - type objects - return new recordIterator(); - } - - public class recordIterator implements Iterator { - - Iterator nodeIterator; - - public recordIterator() throws IOException { - nodeIterator = news.rows(true, false); - } - - public boolean hasNext() { - return nodeIterator.hasNext(); - } - - public Object next() { - return b2r((byte[][]) nodeIterator.next()); - } - - public void remove() { - } - - } - - public synchronized yacyNewsRecord get(String id) throws IOException { - try { - return b2r(news.get(id.getBytes())); - } catch (kelondroException e) { - resetDB(); - return null; - } - } - - private static yacyNewsRecord b2r(byte[][] b) { - if (b == null) return null; - return new yacyNewsRecord( - new String(b[0]), - new String(b[1]), - (b[2] == null) ? null : yacyCore.parseUniversalDate(new String(b[2]), serverDate.UTCDiffString()), - (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(b[3])), - serverCodings.string2map(new String(b[4])) - ); - } - - private static byte[][] r2b(yacyNewsRecord r) { - if (r == null) return null; - String attributes = r.attributes().toString(); - if (attributes.length() > attributesMaxLength) throw new IllegalArgumentException("attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength); - byte[][] b = new byte[5][]; - b[0] = r.id().getBytes(); - b[1] = r.category().getBytes(); - b[2] = (r.received() == null) ? null : yacyCore.universalDateShortString(r.received()).getBytes(); - b[3] = kelondroBase64Order.enhancedCoder.encodeLong(r.distributed(), 2).getBytes(); - b[4] = attributes.getBytes(); - return b; - } - -} +// yacyNewsQueue.java +// ----------------------- +// part of YaCy +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2005 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// Any changes to this file according to the GPL as documented in the file +// gpl.txt aside this file in the shipment you received can be done to the +// lines that follows this copyright notice here, but changes must not be +// done inside the copyright notice above. A re-distribution must contain +// the intact and unchanged copyright notice. +// Contributions and changes to the program code must be marked as such. + +package de.anomic.yacy; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import de.anomic.yacy.yacyCore; +import de.anomic.kelondro.kelondroBase64Order; +import de.anomic.kelondro.kelondroTree; +import de.anomic.kelondro.kelondroException; +import de.anomic.server.serverCodings; +import de.anomic.server.serverDate; + +public class yacyNewsDB { + + private File path; + private int bufferkb; + private kelondroTree news; + + public static final int attributesMaxLength = yacyNewsRecord.maxNewsRecordLength + - yacyNewsRecord.idLength() + - yacyNewsRecord.categoryStringLength + - yacyCore.universalDateShortPattern.length() + - 2; + + public yacyNewsDB(File path, int bufferkb) { + this.path = path; + this.bufferkb = bufferkb; + + if (path.exists()) try { + news = new kelondroTree(path, bufferkb * 0x400); + } catch (IOException e) { + news = createDB(path, bufferkb); + } else { + news = createDB(path, bufferkb); + } + } + + private static kelondroTree createDB(File path, int bufferkb) { + return new kelondroTree(path, bufferkb * 0x400, new int[] { + yacyNewsRecord.idLength(), // id = created + originator + yacyNewsRecord.categoryStringLength, // category + yacyCore.universalDateShortPattern.length(), // received + 2, + attributesMaxLength + }, true); + } + + private void resetDB() { + try {close();} catch (Exception e) {} + if (path.exists()) path.delete(); + news = createDB(path, bufferkb); + } + + public int[] dbCacheChunkSize() { + return news.cacheChunkSize(); + } + + public int[] dbCacheFillStatus() { + return news.cacheFillStatus(); + } + + public void close() { + if (news != null) try {news.close();} catch (IOException e) {} + news = null; + } + + public void finalize() { + close(); + } + + public int size() { + return news.size(); + } + + public void remove(String id) throws IOException { + news.remove(id.getBytes()); + } + + public synchronized yacyNewsRecord put(yacyNewsRecord record) throws IOException { + try { + return b2r(news.put(r2b(record))); + } catch (kelondroException e) { + resetDB(); + return b2r(news.put(r2b(record))); + } + } + + public synchronized Iterator news() throws IOException { + // the iteration iterates yacyNewsRecord - type objects + return new recordIterator(); + } + + public class recordIterator implements Iterator { + + Iterator nodeIterator; + + public recordIterator() throws IOException { + nodeIterator = news.rows(true, false); + } + + public boolean hasNext() { + return nodeIterator.hasNext(); + } + + public Object next() { + return b2r((byte[][]) nodeIterator.next()); + } + + public void remove() { + } + + } + + public synchronized yacyNewsRecord get(String id) throws IOException { + try { + return b2r(news.get(id.getBytes())); + } catch (kelondroException e) { + resetDB(); + return null; + } + } + + private static yacyNewsRecord b2r(byte[][] b) { + if (b == null) return null; + return new yacyNewsRecord( + new String(b[0]), + new String(b[1]), + (b[2] == null) ? null : yacyCore.parseUniversalDate(new String(b[2]), serverDate.UTCDiffString()), + (int) kelondroBase64Order.enhancedCoder.decodeLong(new String(b[3])), + serverCodings.string2map(new String(b[4])) + ); + } + + private static byte[][] r2b(yacyNewsRecord r) { + if (r == null) return null; + String attributes = r.attributes().toString(); + if (attributes.length() > attributesMaxLength) throw new IllegalArgumentException("attribute length=" + attributes.length() + " exceeds maximum size=" + attributesMaxLength); + byte[][] b = new byte[5][]; + b[0] = r.id().getBytes(); + b[1] = r.category().getBytes(); + b[2] = (r.received() == null) ? null : yacyCore.universalDateShortString(r.received()).getBytes(); + b[3] = kelondroBase64Order.enhancedCoder.encodeLong(r.distributed(), 2).getBytes(); + b[4] = attributes.getBytes(); + return b; + } + +} \ No newline at end of file diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java index 3d76479ea..e278f014a 100644 --- a/source/de/anomic/yacy/yacyNewsQueue.java +++ b/source/de/anomic/yacy/yacyNewsQueue.java @@ -1,173 +1,176 @@ -// yacyNewsQueue.java -// ----------------------- -// part of YaCy -// (C) by Michael Peter Christen; mc@anomic.de -// first published on http://www.anomic.de -// Frankfurt, Germany, 2005 -// last major change: 13.07.2005 -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// Using this software in any meaning (reading, learning, copying, compiling, -// running) means that you agree that the Author(s) is (are) not responsible -// for cost, loss of data or any harm that may be caused directly or indirectly -// by usage of this softare or this documentation. The usage of this software -// is on your own risk. The installation and usage (starting/running) of this -// software may allow other people or application to access your computer and -// any attached devices and is highly dependent on the configuration of the -// software which must be done by the user of the software; the author(s) is -// (are) also not responsible for proper configuration and usage of the -// software, even if provoked by documentation provided together with -// the software. -// -// Any changes to this file according to the GPL as documented in the file -// gpl.txt aside this file in the shipment you received can be done to the -// lines that follows this copyright notice here, but changes must not be -// done inside the copyright notice above. A re-distribution must contain -// the intact and unchanged copyright notice. -// Contributions and changes to the program code must be marked as such. - -package de.anomic.yacy; - -import java.io.File; -import java.io.IOException; -import java.util.Date; - -import de.anomic.kelondro.kelondroStack; -import de.anomic.kelondro.kelondroException; - -public class yacyNewsQueue { - - private File path; - private kelondroStack queueStack; - private yacyNewsDB newsDB; - - public yacyNewsQueue(File path, yacyNewsDB newsDB) { - this.path = path; - this.newsDB = newsDB; - - if (path.exists()) try { - queueStack = new kelondroStack(path, 0); - } catch (kelondroException e) { - path.delete(); - queueStack = createStack(path); - } catch (IOException e) { - path.delete(); - queueStack = createStack(path); - } else { - queueStack = createStack(path); - } - } - - private static kelondroStack createStack(File path) { - return new kelondroStack(path, 0, new int[] { - yacyNewsRecord.idLength(), // id = created + originator - yacyCore.universalDateShortPattern.length() // last touched - }, true); - } - - private void resetDB() { - try {close();} catch (Exception e) {} - if (path.exists()) path.delete(); - queueStack = createStack(path); - } - - public void clear() { - resetDB(); - } - - public void close() { - if (queueStack != null) try {queueStack.close();} catch (IOException e) {} - queueStack = null; - } - - public void finalize() { - close(); - } - - public int size() { - return queueStack.size(); - } - - public synchronized void push(yacyNewsRecord entry) throws IOException { - queueStack.push(r2b(entry, true)); - } - - public synchronized yacyNewsRecord pop(int dist) throws IOException { - if (queueStack.size() == 0) return null; - return b2r(queueStack.pop(dist)); - } - - public synchronized yacyNewsRecord top(int dist) throws IOException { - if (queueStack.size() == 0) return null; - return b2r(queueStack.top(dist)); - } - - public synchronized yacyNewsRecord topInc() throws IOException { - if (queueStack.size() == 0) return null; - yacyNewsRecord entry = pop(0); - if (entry != null) { - entry.incDistribution(); - push(entry); - } - return entry; - } - - public synchronized yacyNewsRecord remove(String id) throws IOException { - yacyNewsRecord record; - for (int i = 0; i < size(); i++) { - record = top(i); - if ((record != null) && (record.id().equals(id))) { - pop(i); - return record; - } - } - return null; - } - - - /* - public synchronized void incDistributedCounter(yacyNewsRecord entry) throws IOException { - // this works only if the entry element lies ontop of the stack - yacyNewsRecord topEntry = top(); - if (!(topEntry.id().equals(entry.id()))) throw new IllegalArgumentException("entry is not ontop of the stack"); - pop(); - entry.incDistribution(); - push(entry); - } - */ - - private yacyNewsRecord b2r(byte[][] b) throws IOException { - if (b == null) return null; - String id = new String(b[0]); - //Date touched = yacyCore.parseUniversalDate(new String(b[1])); - return newsDB.get(id); - } - - private byte[][] r2b(yacyNewsRecord r, boolean updateDB) throws IOException { - if (r == null) return null; - if (updateDB) { - newsDB.put(r); - } else { - yacyNewsRecord r1 = newsDB.get(r.id()); - if (r1 == null) newsDB.put(r); - } - byte[][] b = new byte[2][]; - b[0] = r.id().getBytes(); - b[1] = yacyCore.universalDateShortString(new Date()).getBytes(); - return b; - } - -} +// yacyNewsQueue.java +// ----------------------- +// part of YaCy +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2005 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// Any changes to this file according to the GPL as documented in the file +// gpl.txt aside this file in the shipment you received can be done to the +// lines that follows this copyright notice here, but changes must not be +// done inside the copyright notice above. A re-distribution must contain +// the intact and unchanged copyright notice. +// Contributions and changes to the program code must be marked as such. + +package de.anomic.yacy; + +import java.io.File; +import java.io.IOException; +import java.util.Date; + +import de.anomic.kelondro.kelondroStack; +import de.anomic.kelondro.kelondroException; + +public class yacyNewsQueue { + + private File path; + private kelondroStack queueStack; + private yacyNewsDB newsDB; + + public yacyNewsQueue(File path, yacyNewsDB newsDB) { + this.path = path; + this.newsDB = newsDB; + + if (path.exists()) try { + queueStack = new kelondroStack(path, 0); + } catch (kelondroException e) { + path.delete(); + queueStack = createStack(path); + } catch (IOException e) { + path.delete(); + queueStack = createStack(path); + } else { + queueStack = createStack(path); + } + } + + private static kelondroStack createStack(File path) { + return new kelondroStack(path, 0, new int[] { + yacyNewsRecord.idLength(), // id = created + originator + yacyCore.universalDateShortPattern.length() // last touched + }, true); + } + + private void resetDB() { + try {close();} catch (Exception e) {} + if (path.exists()) path.delete(); + queueStack = createStack(path); + } + + public void clear() { + resetDB(); + } + + public void close() { + if (queueStack != null) try {queueStack.close();} catch (IOException e) {} + queueStack = null; + } + + public void finalize() { + close(); + } + + public int size() { + return queueStack.size(); + } + + public synchronized void push(yacyNewsRecord entry) throws IOException { + queueStack.push(r2b(entry, true)); + } + + public synchronized yacyNewsRecord pop(int dist) throws IOException { + if (queueStack.size() == 0) return null; + return b2r(queueStack.pop(dist)); + } + + public synchronized yacyNewsRecord top(int dist) throws IOException { + if (queueStack.size() == 0) return null; + return b2r(queueStack.top(dist)); + } + + public synchronized yacyNewsRecord topInc() throws IOException { + if (queueStack.size() == 0) return null; + yacyNewsRecord entry = pop(0); + if (entry != null) { + entry.incDistribution(); + push(entry); + } + return entry; + } + + public synchronized yacyNewsRecord remove(String id) throws IOException { + yacyNewsRecord record; + for (int i = 0; i < size(); i++) { + record = top(i); + if ((record != null) && (record.id().equals(id))) { + pop(i); + return record; + } + } + return null; + } + + + /* + public synchronized void incDistributedCounter(yacyNewsRecord entry) throws IOException { + // this works only if the entry element lies ontop of the stack + yacyNewsRecord topEntry = top(); + if (!(topEntry.id().equals(entry.id()))) throw new IllegalArgumentException("entry is not ontop of the stack"); + pop(); + entry.incDistribution(); + push(entry); + } + */ + + private yacyNewsRecord b2r(byte[][] b) throws IOException { + if (b == null) return null; + String id = new String(b[0]); + //Date touched = yacyCore.parseUniversalDate(new String(b[1])); + return newsDB.get(id); + } + + private byte[][] r2b(yacyNewsRecord r, boolean updateDB) throws IOException { + if (r == null) return null; + if (updateDB) { + newsDB.put(r); + } else { + yacyNewsRecord r1 = newsDB.get(r.id()); + if (r1 == null) newsDB.put(r); + } + byte[][] b = new byte[2][]; + b[0] = r.id().getBytes(); + b[1] = yacyCore.universalDateShortString(new Date()).getBytes(); + return b; + } + +} \ No newline at end of file diff --git a/source/de/anomic/yacy/yacyNewsRecord.java b/source/de/anomic/yacy/yacyNewsRecord.java index 29490e211..b4af7092d 100644 --- a/source/de/anomic/yacy/yacyNewsRecord.java +++ b/source/de/anomic/yacy/yacyNewsRecord.java @@ -1,153 +1,156 @@ -// yacyNewsRecord.java -// ----------------------- -// part of YaCy -// (C) by Michael Peter Christen; mc@anomic.de -// first published on http://www.anomic.de -// Frankfurt, Germany, 2005 -// last major change: 13.07.2005 -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// -// Using this software in any meaning (reading, learning, copying, compiling, -// running) means that you agree that the Author(s) is (are) not responsible -// for cost, loss of data or any harm that may be caused directly or indirectly -// by usage of this softare or this documentation. The usage of this software -// is on your own risk. The installation and usage (starting/running) of this -// software may allow other people or application to access your computer and -// any attached devices and is highly dependent on the configuration of the -// software which must be done by the user of the software; the author(s) is -// (are) also not responsible for proper configuration and usage of the -// software, even if provoked by documentation provided together with -// the software. -// -// Any changes to this file according to the GPL as documented in the file -// gpl.txt aside this file in the shipment you received can be done to the -// lines that follows this copyright notice here, but changes must not be -// done inside the copyright notice above. A re-distribution must contain -// the intact and unchanged copyright notice. -// Contributions and changes to the program code must be marked as such. - -package de.anomic.yacy; - -import de.anomic.server.serverCodings; -import de.anomic.server.serverDate; - -import java.util.Date; -import java.util.Map; - -public class yacyNewsRecord { - - public static final int maxNewsRecordLength = 512; - public static final int categoryStringLength = 8; - - private String originator; // hash of originating peer - private Date created; // Date when news was created by originator - private Date received; // Date when news was received here at this peer - private String category; // keyword that adresses possible actions - private int distributed; // counter that counts number of distributions of this news record - private Map attributes; // elemets of the news for a special category - - public yacyNewsRecord(String newsString) { - this.attributes = serverCodings.string2map(newsString); - this.received = (attributes.containsKey("rec")) ? yacyCore.parseUniversalDate((String) attributes.get("rec"), serverDate.UTCDiffString()) : new Date(); - this.created = (attributes.containsKey("cre")) ? yacyCore.parseUniversalDate((String) attributes.get("cre"), serverDate.UTCDiffString()) : new Date(); - this.category = (attributes.containsKey("cat")) ? (String) attributes.get("cat") : null; - this.distributed = (attributes.containsKey("dis")) ? Integer.parseInt((String) attributes.get("dis")) : 0; - this.originator = (attributes.containsKey("ori")) ? (String) attributes.get("ori") : null; - removeStandards(); - } - - public yacyNewsRecord(String category, Map attributes) { - if (category.length() > categoryStringLength) throw new IllegalArgumentException("category length exceeds maximum"); - this.attributes = attributes; - this.received = null; - this.created = new Date(); - this.category = category; - this.distributed = 0; - this.originator = yacyCore.seedDB.mySeed.hash; - removeStandards(); - } - - protected yacyNewsRecord(String id, String category, Date received, int distributed, Map attributes) { - this.attributes = attributes; - this.received = received; - this.created = yacyCore.parseUniversalDate(id.substring(0, yacyCore.universalDateShortPattern.length()), serverDate.UTCDiffString()); - this.category = category; - this.distributed = distributed; - this.originator = id.substring(yacyCore.universalDateShortPattern.length()); - removeStandards(); - } - - private void removeStandards() { - attributes.remove("ori"); - attributes.remove("cat"); - attributes.remove("cre"); - attributes.remove("rec"); - attributes.remove("dis"); - } - - public String toString() { - // this creates the string that shall be distributed - // attention: this has no additional encoding - if (this.originator != null) attributes.put("ori", this.originator); - if (this.category != null) attributes.put("cat", this.category); - if (this.created != null) attributes.put("cre", yacyCore.universalDateShortString(this.created)); - if (this.received != null) attributes.put("rec", yacyCore.universalDateShortString(this.received)); - attributes.put("dis", Integer.toString(this.distributed)); - String theString = attributes.toString(); - removeStandards(); - return theString; - } - - public String id() { - return yacyCore.universalDateShortString(created) + originator; - } - - public static int idLength() { - return yacyCore.universalDateShortPattern.length() + yacySeedDB.commonHashLength; - } - - public String originator() { - return originator; - } - - public Date created() { - return created; - } - - public Date received() { - return received; - } - - public String category() { - return category; - } - - public int distributed() { - return distributed; - } - - public void incDistribution() { - distributed++; - } - - public Map attributes() { - return attributes; - } - - public static void main(String[] args) { - System.out.println((new yacyNewsRecord(args[0])).toString()); - } -} +// yacyNewsRecord.java +// ----------------------- +// part of YaCy +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2005 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// Any changes to this file according to the GPL as documented in the file +// gpl.txt aside this file in the shipment you received can be done to the +// lines that follows this copyright notice here, but changes must not be +// done inside the copyright notice above. A re-distribution must contain +// the intact and unchanged copyright notice. +// Contributions and changes to the program code must be marked as such. + +package de.anomic.yacy; + +import de.anomic.server.serverCodings; +import de.anomic.server.serverDate; + +import java.util.Date; +import java.util.Map; + +public class yacyNewsRecord { + + public static final int maxNewsRecordLength = 512; + public static final int categoryStringLength = 8; + + private String originator; // hash of originating peer + private Date created; // Date when news was created by originator + private Date received; // Date when news was received here at this peer + private String category; // keyword that adresses possible actions + private int distributed; // counter that counts number of distributions of this news record + private Map attributes; // elemets of the news for a special category + + public yacyNewsRecord(String newsString) { + this.attributes = serverCodings.string2map(newsString); + this.received = (attributes.containsKey("rec")) ? yacyCore.parseUniversalDate((String) attributes.get("rec"), serverDate.UTCDiffString()) : new Date(); + this.created = (attributes.containsKey("cre")) ? yacyCore.parseUniversalDate((String) attributes.get("cre"), serverDate.UTCDiffString()) : new Date(); + this.category = (attributes.containsKey("cat")) ? (String) attributes.get("cat") : null; + this.distributed = (attributes.containsKey("dis")) ? Integer.parseInt((String) attributes.get("dis")) : 0; + this.originator = (attributes.containsKey("ori")) ? (String) attributes.get("ori") : null; + removeStandards(); + } + + public yacyNewsRecord(String category, Map attributes) { + if (category.length() > categoryStringLength) throw new IllegalArgumentException("category length exceeds maximum"); + this.attributes = attributes; + this.received = null; + this.created = new Date(); + this.category = category; + this.distributed = 0; + this.originator = yacyCore.seedDB.mySeed.hash; + removeStandards(); + } + + protected yacyNewsRecord(String id, String category, Date received, int distributed, Map attributes) { + this.attributes = attributes; + this.received = received; + this.created = yacyCore.parseUniversalDate(id.substring(0, yacyCore.universalDateShortPattern.length()), serverDate.UTCDiffString()); + this.category = category; + this.distributed = distributed; + this.originator = id.substring(yacyCore.universalDateShortPattern.length()); + removeStandards(); + } + + private void removeStandards() { + attributes.remove("ori"); + attributes.remove("cat"); + attributes.remove("cre"); + attributes.remove("rec"); + attributes.remove("dis"); + } + + public String toString() { + // this creates the string that shall be distributed + // attention: this has no additional encoding + if (this.originator != null) attributes.put("ori", this.originator); + if (this.category != null) attributes.put("cat", this.category); + if (this.created != null) attributes.put("cre", yacyCore.universalDateShortString(this.created)); + if (this.received != null) attributes.put("rec", yacyCore.universalDateShortString(this.received)); + attributes.put("dis", Integer.toString(this.distributed)); + String theString = attributes.toString(); + removeStandards(); + return theString; + } + + public String id() { + return yacyCore.universalDateShortString(created) + originator; + } + + public static int idLength() { + return yacyCore.universalDateShortPattern.length() + yacySeedDB.commonHashLength; + } + + public String originator() { + return originator; + } + + public Date created() { + return created; + } + + public Date received() { + return received; + } + + public String category() { + return category; + } + + public int distributed() { + return distributed; + } + + public void incDistribution() { + distributed++; + } + + public Map attributes() { + return attributes; + } + + public static void main(String[] args) { + System.out.println((new yacyNewsRecord(args[0])).toString()); + } +} \ No newline at end of file diff --git a/source/de/anomic/yacy/yacyPeerAction.java b/source/de/anomic/yacy/yacyPeerAction.java index 7fbfaa319..25ce26a01 100644 --- a/source/de/anomic/yacy/yacyPeerAction.java +++ b/source/de/anomic/yacy/yacyPeerAction.java @@ -3,7 +3,10 @@ // (C) by Michael Peter Christen; mc@anomic.de // first published on http://www.anomic.de // Frankfurt, Germany, 2005 -// last major change: 23.02.2005 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by diff --git a/source/de/anomic/yacy/yacySeedUploader.java b/source/de/anomic/yacy/yacySeedUploader.java index c4cf8506c..0047b113a 100644 --- a/source/de/anomic/yacy/yacySeedUploader.java +++ b/source/de/anomic/yacy/yacySeedUploader.java @@ -1,3 +1,46 @@ +// yacySeedUploader.java +// ------------------------------------- +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004, 2005 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// Any changes to this file according to the GPL as documented in the file +// gpl.txt aside this file in the shipment you received can be done to the +// lines that follows this copyright notice here, but changes must not be +// done inside the copyright notive above. A re-distribution must contain +// the intact and unchanged copyright notice. +// Contributions and changes to the program code must be marked as such. + package de.anomic.yacy; import java.io.File; diff --git a/source/de/anomic/yacy/yacyVersion.java b/source/de/anomic/yacy/yacyVersion.java index b4f671743..5b40fd4a8 100644 --- a/source/de/anomic/yacy/yacyVersion.java +++ b/source/de/anomic/yacy/yacyVersion.java @@ -1,3 +1,46 @@ +// yacyVersion.java +// ------------------------------------- +// (C) by Michael Peter Christen; mc@anomic.de +// first published on http://www.anomic.de +// Frankfurt, Germany, 2004, 2005 +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +// +// Using this software in any meaning (reading, learning, copying, compiling, +// running) means that you agree that the Author(s) is (are) not responsible +// for cost, loss of data or any harm that may be caused directly or indirectly +// by usage of this softare or this documentation. The usage of this software +// is on your own risk. The installation and usage (starting/running) of this +// software may allow other people or application to access your computer and +// any attached devices and is highly dependent on the configuration of the +// software which must be done by the user of the software; the author(s) is +// (are) also not responsible for proper configuration and usage of the +// software, even if provoked by documentation provided together with +// the software. +// +// Any changes to this file according to the GPL as documented in the file +// gpl.txt aside this file in the shipment you received can be done to the +// lines that follows this copyright notice here, but changes must not be +// done inside the copyright notive above. A re-distribution must contain +// the intact and unchanged copyright notice. +// Contributions and changes to the program code must be marked as such. + package de.anomic.yacy; public final class yacyVersion {