diff --git a/htroot/Bookmarks.java b/htroot/Bookmarks.java index 9d7d6e605..b195d9937 100644 --- a/htroot/Bookmarks.java +++ b/htroot/Bookmarks.java @@ -37,6 +37,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Set; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.Document; @@ -44,7 +45,6 @@ import net.yacy.document.Parser; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; import de.anomic.crawler.CrawlProfile; import de.anomic.data.BookmarkHelper; @@ -316,7 +316,7 @@ public class Bookmarks { } prop.putHTML("display_bookmarks_"+count+"_title", bookmark.getTitle()); prop.putHTML("display_bookmarks_"+count+"_description", bookmark.getDescription()); - prop.put("display_bookmarks_"+count+"_date", DateFormatter.formatISO8601(new Date(bookmark.getTimeStamp()))); + prop.put("display_bookmarks_"+count+"_date", ISO8601Formatter.FORMATTER.format(new Date(bookmark.getTimeStamp()))); prop.put("display_bookmarks_"+count+"_rfc822date", HeaderFramework.formatRFC1123(new Date(bookmark.getTimeStamp()))); prop.put("display_bookmarks_"+count+"_public", (bookmark.getPublic() ? "1" : "0")); diff --git a/htroot/Connections_p.java b/htroot/Connections_p.java index dc1f86866..9002c3cd2 100644 --- a/htroot/Connections_p.java +++ b/htroot/Connections_p.java @@ -35,7 +35,6 @@ import java.util.Set; import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.workflow.WorkflowThread; //import de.anomic.http.client.ConnectionInfo; @@ -45,6 +44,7 @@ import de.anomic.server.serverCore; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.serverCore.Session; +import de.anomic.yacy.yacyPeerActions; import de.anomic.yacy.yacySeed; public final class Connections_p { @@ -117,7 +117,7 @@ public final class Connections_p { prop.put("list_" + idx + "_proto", prot); if (sessionTime > 1000*60) { prop.put("list_" + idx + "_ms", "0"); - prop.put("list_" + idx + "_ms_duration",DateFormatter.formatInterval(sessionTime)); + prop.put("list_" + idx + "_ms_duration",yacyPeerActions.formatInterval(sessionTime)); } else { prop.put("list_" + idx + "_ms", "1"); prop.putNum("list_" + idx + "_ms_duration", sessionTime); diff --git a/htroot/ContentIntegrationPHPBB3_p.java b/htroot/ContentIntegrationPHPBB3_p.java index 93710af46..2ba20051a 100644 --- a/htroot/ContentIntegrationPHPBB3_p.java +++ b/htroot/ContentIntegrationPHPBB3_p.java @@ -23,14 +23,14 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import java.io.File; +import java.util.Date; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.content.dao.Dao; import net.yacy.document.content.dao.ImportDump; import net.yacy.document.content.dao.PhpBB3Dao; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; - import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -107,7 +107,7 @@ public class ContentIntegrationPHPBB3_p { dbpw ); - int files = db.writeSurrogates(db.query(0, -1, 100), sb.surrogatesInPath, "fullexport-" + DateFormatter.formatShortSecond(), ppf); + int files = db.writeSurrogates(db.query(0, -1, 100), sb.surrogatesInPath, "fullexport-" + GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()), ppf); prop.put("export", 1); prop.put("export_files", files); db.close(); diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index 88448a11c..a6e1c5f4e 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -164,7 +164,7 @@ public class CrawlStartScanner_p { // execute the scan results if (Scanner.scancacheSize() > 0) { // make a comment cache - Map apiCommentCache = Scanner.commentCache(sb); + Map apiCommentCache = WorkTables.commentCache(sb); String urlString; DigestURI u; @@ -196,7 +196,7 @@ public class CrawlStartScanner_p { // write scan table if (Scanner.scancacheSize() > 0) { // make a comment cache - Map apiCommentCache = Scanner.commentCache(sb); + Map apiCommentCache = WorkTables.commentCache(sb); // show scancache table prop.put("servertable", 1); diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index c61b13af2..2ea39ce29 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -34,6 +34,7 @@ import java.util.Iterator; import java.util.List; import java.util.TreeMap; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.Condenser; import net.yacy.kelondro.data.meta.DigestURI; @@ -49,7 +50,6 @@ import net.yacy.kelondro.order.Bitfield; import net.yacy.kelondro.rwi.Reference; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainerCache; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist; @@ -432,7 +432,7 @@ public class IndexControlRWIs_p { prop.putNum("genUrlList_urlList_"+i+"_urlExists_ybr", RankingProcess.ybr(entry.hash())); prop.putNum("genUrlList_urlList_"+i+"_urlExists_tf", 1000.0 * entry.word().termFrequency()); prop.putNum("genUrlList_urlList_"+i+"_urlExists_authority", (ranked.getOrder() == null) ? -1 : ranked.getOrder().authority(entry.hash())); - prop.put("genUrlList_urlList_"+i+"_urlExists_date", DateFormatter.formatShortDay(new Date(entry.word().lastModified()))); + prop.put("genUrlList_urlList_"+i+"_urlExists_date", GenericFormatter.SHORT_DAY_FORMATTER.format(new Date(entry.word().lastModified()))); prop.putNum("genUrlList_urlList_"+i+"_urlExists_wordsintitle", entry.word().wordsintitle()); prop.putNum("genUrlList_urlList_"+i+"_urlExists_wordsintext", entry.word().wordsintext()); prop.putNum("genUrlList_urlList_"+i+"_urlExists_phrasesintext", entry.word().phrasesintext()); diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java index 59106d1ed..ba16f592e 100644 --- a/htroot/IndexControlURLs_p.java +++ b/htroot/IndexControlURLs_p.java @@ -28,8 +28,10 @@ import java.io.File; import java.io.IOException; import java.net.MalformedURLException; +import java.util.Date; import java.util.Iterator; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; @@ -37,8 +39,6 @@ import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.RotateIterator; -import net.yacy.kelondro.util.DateFormatter; - import de.anomic.crawler.CrawlProfile; import de.anomic.search.MetadataRepository; import de.anomic.search.Segment; @@ -102,7 +102,7 @@ public class IndexControlURLs_p { prop.put("reload", 1); } else { prop.put("lurlexport", 1); - prop.put("lurlexport_exportfile", sb.getDataPath() + "/DATA/EXPORT/" + DateFormatter.formatShortSecond()); + prop.put("lurlexport_exportfile", sb.getDataPath() + "/DATA/EXPORT/" + GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date())); if (export == null) { // there has never been an export prop.put("lurlexportfinished", 0); diff --git a/htroot/Network.java b/htroot/Network.java index 6d01fa207..03caf3ca5 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -39,7 +39,6 @@ import java.util.regex.PatternSyntaxException; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.MapTools; //import de.anomic.http.client.Client; @@ -50,6 +49,7 @@ import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyNewsDB; import de.anomic.yacy.yacyNewsPool; +import de.anomic.yacy.yacyPeerActions; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyVersion; @@ -131,7 +131,7 @@ public class Network { myqph = 60d * sb.averageQPM(); prop.put("table_my-version", seed.get(yacySeed.VERSION, "-")); prop.put("table_my-utc", seed.get(yacySeed.UTC, "-")); - prop.put("table_my-uptime", DateFormatter.formatInterval(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "")))); + prop.put("table_my-uptime", yacyPeerActions.formatInterval(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "")))); prop.putNum("table_my-LCount", LCount); prop.putNum("table_my-ICount", ICount); prop.putNum("table_my-RCount", RCount); @@ -437,7 +437,7 @@ public class Network { prop.putHTML(STR_TABLE_LIST + conCount + "_version", yacyVersion.combined2prettyVersion(seed.get(yacySeed.VERSION, "0.1"), shortname)); prop.putNum(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen); prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-")); - prop.putHTML(STR_TABLE_LIST + conCount + "_uptime", DateFormatter.formatInterval(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0")))); + prop.putHTML(STR_TABLE_LIST + conCount + "_uptime", yacyPeerActions.formatInterval(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0")))); prop.putNum(STR_TABLE_LIST + conCount + "_LCount", seed.getLinkCount()); prop.putNum(STR_TABLE_LIST + conCount + "_ICount", seed.getWordCount()); prop.putNum(STR_TABLE_LIST + conCount + "_RCount", seed.getLong(yacySeed.RCOUNT, 0)); diff --git a/htroot/News.java b/htroot/News.java index 7d0093bc4..662c2376d 100644 --- a/htroot/News.java +++ b/htroot/News.java @@ -28,11 +28,10 @@ import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; - import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -117,10 +116,10 @@ public class News { final String category = record.category(); prop.put("table_list_" + i + "_id", record.id()); prop.putHTML("table_list_" + i + "_ori", (seed == null) ? record.originator() : seed.getName()); - prop.put("table_list_" + i + "_cre", DateFormatter.formatShortSecond(record.created())); + prop.put("table_list_" + i + "_cre", GenericFormatter.SHORT_SECOND_FORMATTER.format(record.created())); prop.put("table_list_" + i + "_crerfcdate", HeaderFramework.formatRFC1123(record.created())); prop.putHTML("table_list_" + i + "_cat", category); - prop.put("table_list_" + i + "_rec", (record.received() == null) ? "-" : DateFormatter.formatShortSecond(record.received())); + prop.put("table_list_" + i + "_rec", (record.received() == null) ? "-" : GenericFormatter.SHORT_SECOND_FORMATTER.format(record.received())); prop.put("table_list_" + i + "_dis", record.distributed()); final Map attributeMap = record.attributes(); diff --git a/htroot/RemoteCrawl_p.java b/htroot/RemoteCrawl_p.java index 19433f02c..2bc0a741c 100644 --- a/htroot/RemoteCrawl_p.java +++ b/htroot/RemoteCrawl_p.java @@ -31,13 +31,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.kelondro.util.DateFormatter; import de.anomic.data.WorkTables; import de.anomic.search.Switchboard; import de.anomic.search.SwitchboardConstants; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; +import de.anomic.yacy.yacyPeerActions; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyVersion; @@ -119,7 +119,7 @@ public class RemoteCrawl_p { prop.putHTML(STR_TABLE_LIST + conCount + "_version", yacyVersion.combined2prettyVersion(seed.get(yacySeed.VERSION, "0.1"), shortname)); prop.putNum(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen); prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-")); - prop.putHTML(STR_TABLE_LIST + conCount + "_uptime", DateFormatter.formatInterval(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0")))); + prop.putHTML(STR_TABLE_LIST + conCount + "_uptime", yacyPeerActions.formatInterval(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0")))); prop.putNum(STR_TABLE_LIST + conCount + "_LCount", seed.getLinkCount()); prop.putNum(STR_TABLE_LIST + conCount + "_ICount", seed.getWordCount()); prop.putNum(STR_TABLE_LIST + conCount + "_RCount", rcount); diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index 0210df0e0..5aeeee6f6 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -40,7 +40,6 @@ import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Digest; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.Formatter; import de.anomic.http.server.HTTPDemon; @@ -50,6 +49,7 @@ import de.anomic.server.serverCore; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyCore; +import de.anomic.yacy.yacyPeerActions; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeedUploader; @@ -524,7 +524,7 @@ public class SettingsAck_p { } // everything is ok - prop.put("info_crawler.clientTimeout",(crawlerTimeout==0) ? "0" :DateFormatter.formatInterval(crawlerTimeout)); + prop.put("info_crawler.clientTimeout",(crawlerTimeout==0) ? "0" :yacyPeerActions.formatInterval(crawlerTimeout)); prop.put("info_crawler.http.maxFileSize",(maxHttpSize==-1)? "-1":Formatter.bytesToString(maxHttpSize)); prop.put("info_crawler.ftp.maxFileSize", (maxFtpSize==-1) ? "-1":Formatter.bytesToString(maxFtpSize)); prop.put("info_crawler.smb.maxFileSize", (maxSmbSize==-1) ? "-1":Formatter.bytesToString(maxSmbSize)); diff --git a/htroot/Status.java b/htroot/Status.java index e6f1c48f4..4f87c6e61 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -33,9 +33,6 @@ import java.util.Date; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.io.ByteCount; -//import net.yacy.kelondro.io.ByteCountInputStream; -//import net.yacy.kelondro.io.ByteCountOutputStream; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.OS; @@ -48,6 +45,7 @@ import de.anomic.server.serverCore; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyBuildProperties; +import de.anomic.yacy.yacyPeerActions; import de.anomic.yacy.yacySeed; public class Status { @@ -203,7 +201,7 @@ public class Status { } else { final long uptime = 60000 * Long.parseLong(sb.peers.mySeed().get(yacySeed.UPTIME, "0")); prop.put("peerStatistics", "1"); - prop.put("peerStatistics_uptime", DateFormatter.formatInterval(uptime)); + prop.put("peerStatistics_uptime", yacyPeerActions.formatInterval(uptime)); prop.putNum("peerStatistics_pagesperminute", sb.peers.mySeed().getPPM()); prop.putNum("peerStatistics_queriesperhour", Math.round(6000d * sb.peers.mySeed().getQPM()) / 100d); prop.putNum("peerStatistics_links", sb.peers.mySeed().getLinkCount()); @@ -262,7 +260,7 @@ public class Status { prop.putHTML("seedServer_seedFile", sb.getConfig("seedFilePath", "")); } prop.put("seedServer_lastUpload", - DateFormatter.formatInterval(System.currentTimeMillis() - sb.peers.lastSeedUpload_timeStamp)); + yacyPeerActions.formatInterval(System.currentTimeMillis() - sb.peers.lastSeedUpload_timeStamp)); } else { prop.put(SEEDSERVER, "0"); // disabled } diff --git a/htroot/Supporter.java b/htroot/Supporter.java index 0f5a33f77..b6ae9ce26 100644 --- a/htroot/Supporter.java +++ b/htroot/Supporter.java @@ -31,6 +31,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Iterator; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.storage.DynamicScore; import net.yacy.cora.storage.ScoreCluster; @@ -38,7 +39,6 @@ import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.order.NaturalOrder; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.repository.Blacklist; import de.anomic.search.Switchboard; @@ -106,7 +106,7 @@ public class Supporter { //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); final DynamicScore ranking = new ScoreCluster(); // score cluster for url hashes - final Row rowdef = new Row("String url-255, String title-120, String description-120, String refid-" + (DateFormatter.PATTERN_SHORT_SECOND.length() + 12), NaturalOrder.naturalOrder); + final Row rowdef = new Row("String url-255, String title-120, String description-120, String refid-" + (GenericFormatter.PATTERN_SHORT_SECOND.length() + 12), NaturalOrder.naturalOrder); final HashMap Supporter = new HashMap(); // a mapping from an url hash to a kelondroRow.Entry with display properties accumulateSupporter(sb, Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); //accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); diff --git a/htroot/Surftips.java b/htroot/Surftips.java index 6557f1e62..63b32ec81 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -31,6 +31,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Iterator; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.storage.DynamicScore; import net.yacy.cora.storage.ScoreCluster; @@ -38,7 +39,6 @@ import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.order.NaturalOrder; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.repository.Blacklist; import de.anomic.search.Switchboard; @@ -114,7 +114,7 @@ public class Surftips { //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); final DynamicScore ranking = new ScoreCluster(); // score cluster for url hashes - final Row rowdef = new Row("String url-255, String title-120, String description-120, String refid-" + (DateFormatter.PATTERN_SHORT_SECOND.length() + 12), NaturalOrder.naturalOrder); + final Row rowdef = new Row("String url-255, String title-120, String description-120, String refid-" + (GenericFormatter.PATTERN_SHORT_SECOND.length() + 12), NaturalOrder.naturalOrder); final HashMap surftips = new HashMap(); // a mapping from an url hash to a kelondroRow.Entry with display properties accumulateSurftips(sb, surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); //accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); diff --git a/htroot/api/bookmarks/get_bookmarks.java b/htroot/api/bookmarks/get_bookmarks.java index 22d8c416d..d337733f1 100644 --- a/htroot/api/bookmarks/get_bookmarks.java +++ b/htroot/api/bookmarks/get_bookmarks.java @@ -3,10 +3,10 @@ import java.util.Date; import java.util.Iterator; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.parser.html.CharacterCoding; -import net.yacy.kelondro.util.DateFormatter; import de.anomic.data.BookmarkHelper; import de.anomic.data.BookmarksDB; @@ -160,7 +160,7 @@ public class get_bookmarks { if (bookmark != null) { prop.put("display_bookmarks_"+count+"_id",count); prop.put("display_bookmarks_"+count+"_link",bookmark.getUrl()); - prop.put("display_bookmarks_"+count+"_date", DateFormatter.formatISO8601(new Date(bookmark.getTimeStamp()))); + prop.put("display_bookmarks_"+count+"_date", ISO8601Formatter.FORMATTER.format(new Date(bookmark.getTimeStamp()))); prop.put("display_bookmarks_"+count+"_rfc822date", HeaderFramework.formatRFC1123(new Date(bookmark.getTimeStamp()))); prop.put("display_bookmarks_"+count+"_public", (bookmark.getPublic() ? "0" : "1")); prop.put("display_bookmarks_"+count+"_hash", bookmark.getUrlHash()); @@ -253,7 +253,7 @@ public class get_bookmarks { date = new Date(bookmark.getTimeStamp()); prop.put("display_xbel_"+count+"_elements", ""); + + "\" added=\"" + CharacterCoding.unicode2xml(ISO8601Formatter.FORMATTER.format(date), true)+"\">"); count++; prop.put("display_xbel_"+count+"_elements", ""); count++; diff --git a/htroot/api/bookmarks/posts/all.java b/htroot/api/bookmarks/posts/all.java index 1576710a5..12bb2038b 100755 --- a/htroot/api/bookmarks/posts/all.java +++ b/htroot/api/bookmarks/posts/all.java @@ -2,9 +2,9 @@ import java.util.Date; import java.util.Iterator; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.order.Digest; -import net.yacy.kelondro.util.DateFormatter; import de.anomic.data.BookmarksDB; import de.anomic.search.Switchboard; @@ -37,8 +37,8 @@ public class all { prop.putXML("posts_"+count+"_title", bookmark.getTitle()); prop.putXML("posts_"+count+"_description", bookmark.getDescription()); prop.putXML("posts_"+count+"_md5", Digest.encodeMD5Hex(bookmark.getUrl())); - date=new Date(bookmark.getTimeStamp()); - prop.putXML("posts_"+count+"_time", DateFormatter.formatISO8601(date)); + date = new Date(bookmark.getTimeStamp()); + prop.putXML("posts_"+count+"_time", ISO8601Formatter.FORMATTER.format(date)); prop.putXML("posts_"+count+"_tags", bookmark.getTagsString().replaceAll(","," ")); // additional XML tags diff --git a/htroot/api/bookmarks/posts/get.java b/htroot/api/bookmarks/posts/get.java index da2fb28b0..1d6a22de1 100755 --- a/htroot/api/bookmarks/posts/get.java +++ b/htroot/api/bookmarks/posts/get.java @@ -4,10 +4,9 @@ import java.text.ParseException; import java.util.Date; import java.util.List; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.order.Digest; -import net.yacy.kelondro.util.DateFormatter; - import de.anomic.data.BookmarksDB; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; @@ -24,12 +23,12 @@ public class get { //String url=""; //urlfilter not yet implemented if (post != null && post.containsKey("tag")) { - tag=post.get("tag"); + tag = post.get("tag"); } if (post != null && post.containsKey("date")) { - date=post.get("date"); + date = post.get("date"); } else { - date=DateFormatter.formatISO8601(new Date(System.currentTimeMillis())); + date = ISO8601Formatter.FORMATTER.format(new Date(System.currentTimeMillis())); } // if an extended xml should be used @@ -39,7 +38,7 @@ public class get { Date parsedDate = null; try { - parsedDate = DateFormatter.parseISO8601(date); + parsedDate = ISO8601Formatter.FORMATTER.parse(date); } catch (final ParseException e) { parsedDate = new Date(); } @@ -48,7 +47,7 @@ public class get { BookmarksDB.Bookmark bookmark = null; for (final String bookmark_hash : bookmark_hashes){ bookmark=switchboard.bookmarksDB.getBookmark(bookmark_hash); - if(DateFormatter.formatISO8601(new Date(bookmark.getTimeStamp())).equals(date) && + if (ISO8601Formatter.FORMATTER.format(new Date(bookmark.getTimeStamp())).equals(date) && tag==null || bookmark.getTags().contains(tag) && isAdmin || bookmark.getPublic()){ prop.putHTML("posts_"+count+"_url", bookmark.getUrl()); diff --git a/htroot/api/bookmarks/xbel/xbel.java b/htroot/api/bookmarks/xbel/xbel.java index 6688f6979..12c23d44b 100755 --- a/htroot/api/bookmarks/xbel/xbel.java +++ b/htroot/api/bookmarks/xbel/xbel.java @@ -3,10 +3,9 @@ import java.util.Date; import java.util.Iterator; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.parser.html.CharacterCoding; -import net.yacy.kelondro.util.DateFormatter; - import de.anomic.data.BookmarkHelper; import de.anomic.data.BookmarksDB; import de.anomic.search.Switchboard; @@ -111,7 +110,7 @@ public class xbel { date=new Date(bookmark.getTimeStamp()); prop.put("xbel_"+count+"_elements", "<bookmark id=\"" + bookmark.getUrlHash() + "\" href=\"" + CharacterCoding.unicode2xml(bookmark.getUrl(), true) - + "\" added=\"" + CharacterCoding.unicode2xml(DateFormatter.formatISO8601(date), true)+"\">"); + + "\" added=\"" + CharacterCoding.unicode2xml(ISO8601Formatter.FORMATTER.format(date), true)+"\">"); count++; prop.put("xbel_"+count+"_elements", "<title>"); count++; diff --git a/htroot/api/latency_p.java b/htroot/api/latency_p.java index e331ccee8..f7ea70938 100644 --- a/htroot/api/latency_p.java +++ b/htroot/api/latency_p.java @@ -25,9 +25,8 @@ import java.util.Date; import java.util.Iterator; import java.util.Map; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.kelondro.util.DateFormatter; - import de.anomic.crawler.Latency; import de.anomic.crawler.NoticedURL; import de.anomic.crawler.Latency.Host; @@ -49,7 +48,7 @@ public class latency_p { host = e.getValue(); prop.putXML("domains_" + c + "_hosthash", e.getKey()); prop.putXML("domains_" + c + "_host", host.host()); - prop.putXML("domains_" + c + "_lastaccess", DateFormatter.formatShortSecond(new Date(host.lastacc()))); + prop.putXML("domains_" + c + "_lastaccess", GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(host.lastacc()))); prop.put("domains_" + c + "_count", host.count()); prop.put("domains_" + c + "_average", host.average()); prop.put("domains_" + c + "_robots", host.robotsDelay()); diff --git a/htroot/api/timeline.java b/htroot/api/timeline.java index 98161a275..c02441f1e 100644 --- a/htroot/api/timeline.java +++ b/htroot/api/timeline.java @@ -28,6 +28,7 @@ import java.util.Date; import java.util.Iterator; import java.util.TreeSet; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.WordReference; @@ -36,7 +37,6 @@ import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.TermSearch; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.ISO639; import de.anomic.search.QueryParams; @@ -108,7 +108,7 @@ public final class timeline { while (i.hasNext() && c < count) { entry = i.next(); lm = new Date(entry.lastModified()); - lms = DateFormatter.formatANSIC(lm); + lms = GenericFormatter.ANSIC_FORMATTER.format(lm); prop.put("event_" + c + "_start", lms); // like "Wed May 01 1963 00:00:00 GMT-0600" prop.put("event_" + c + "_end", lms); // like "Sat Jun 01 1963 00:00:00 GMT-0600" prop.put("event_" + c + "_isDuration", 0); // 0 (only a point) or 1 (period of time) diff --git a/htroot/api/ymarks/get_treeview.java b/htroot/api/ymarks/get_treeview.java index 0e6040fd3..73170df67 100644 --- a/htroot/api/ymarks/get_treeview.java +++ b/htroot/api/ymarks/get_treeview.java @@ -6,6 +6,7 @@ import java.util.EnumMap; import java.util.Iterator; import java.util.TreeMap; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.Document; import net.yacy.document.Parser.Failure; @@ -14,7 +15,6 @@ import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.retrieval.Response; import de.anomic.data.UserDB; @@ -147,7 +147,7 @@ public class get_treeview { if(key.startsWith("date")) { final String d = new String(bmk_row.get(key)); if(!d.isEmpty()) { - final String date = DateFormatter.formatISO8601(new Date(Long.parseLong(d))); + final String date = ISO8601Formatter.FORMATTER.format(new Date(Long.parseLong(d))); prop.put("folders_"+count+"_foldername","<small><b>"+key+":</b> " + date + "</small>"); putProp(count, "date"); count++; diff --git a/htroot/cytag.java b/htroot/cytag.java index 8ed00ba2b..ab2e93c52 100644 --- a/htroot/cytag.java +++ b/htroot/cytag.java @@ -30,12 +30,12 @@ import java.io.File; import java.io.IOException; import java.util.Date; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.ImageParser; import net.yacy.document.parser.html.CharacterCoding; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; import de.anomic.search.Switchboard; @@ -52,7 +52,7 @@ public class cytag { // harvest request information StringBuilder connect = new StringBuilder(); connect.append('{'); - appendJSON(connect, "time", DateFormatter.formatShortMilliSecond(new Date())); + appendJSON(connect, "time", GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date())); appendJSON(connect, "trail", (referer == null) ? "" : referer.toNormalform(false, false)); appendJSON(connect, "nick", (post == null) ? "" : post.get("nick", "")); appendJSON(connect, "tag", (post == null) ? "" : post.get("tag", "")); diff --git a/htroot/yacy/query.java b/htroot/yacy/query.java index cf5866b86..f2694283a 100644 --- a/htroot/yacy/query.java +++ b/htroot/yacy/query.java @@ -28,12 +28,12 @@ // if the shell's current path is HTROOT import java.io.IOException; +import java.util.Date; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; - import de.anomic.search.Segments; import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; @@ -75,7 +75,7 @@ public final class query { final String obj = post.get("object", ""); // keyword for query subject final String env = post.get("env", ""); // argument to query - prop.put("mytime", DateFormatter.formatShortSecond()); + prop.put("mytime", GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date())); // check if we are the right target and requester has correct information about this peer if (sb.peers.mySeed() == null || !sb.peers.mySeed().hash.equals(youare)) { diff --git a/htroot/yacy/transferURL.java b/htroot/yacy/transferURL.java index 75d7375d8..5e5fd4e50 100644 --- a/htroot/yacy/transferURL.java +++ b/htroot/yacy/transferURL.java @@ -29,11 +29,11 @@ import java.io.IOException; import java.text.ParseException; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.RSSMessage; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.repository.Blacklist; import de.anomic.crawler.retrieval.EventOrigin; @@ -52,7 +52,7 @@ public final class transferURL { public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) throws InterruptedException { final long start = System.currentTimeMillis(); long freshdate = 0; - try {freshdate = DateFormatter.parseShortDay("20061101").getTime();} catch (final ParseException e1) {} + try {freshdate = GenericFormatter.SHORT_DAY_FORMATTER.parse("20061101").getTime();} catch (final ParseException e1) {} // return variable that accumulates replacements final Switchboard sb = (Switchboard) env; diff --git a/htroot/yacy/urls.java b/htroot/yacy/urls.java index 962b4fe8c..e512d0bc6 100644 --- a/htroot/yacy/urls.java +++ b/htroot/yacy/urls.java @@ -27,11 +27,10 @@ import java.io.IOException; import java.util.Date; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; -import net.yacy.kelondro.util.DateFormatter; - import de.anomic.crawler.NoticedURL; import de.anomic.crawler.retrieval.Request; import de.anomic.search.Segments; @@ -93,7 +92,7 @@ public class urls { prop.putXML("item_" + c + "_referrer", (referrer == null) ? "" : referrer.toNormalform(true, false)); prop.putXML("item_" + c + "_description", entry.name()); prop.put("item_" + c + "_author", ""); - prop.put("item_" + c + "_pubDate", DateFormatter.formatShortSecond(entry.appdate())); + prop.put("item_" + c + "_pubDate", GenericFormatter.SHORT_SECOND_FORMATTER.format(entry.appdate())); prop.put("item_" + c + "_guid", entry.url().hash()); c++; maxCount--; @@ -123,7 +122,7 @@ public class urls { prop.putXML("item_" + c + "_referrer", (referrer == null) ? "" : referrer.toNormalform(true, false)); prop.putXML("item_" + c + "_description", metadata.dc_title()); prop.put("item_" + c + "_author", metadata.dc_creator()); - prop.put("item_" + c + "_pubDate", DateFormatter.formatShortSecond(entry.moddate())); + prop.put("item_" + c + "_pubDate", GenericFormatter.SHORT_SECOND_FORMATTER.format(entry.moddate())); prop.put("item_" + c + "_guid", new String(entry.hash())); c++; } diff --git a/htroot/yacyinteractive.html b/htroot/yacyinteractive.html index bd0074a25..b7e13b9d4 100644 --- a/htroot/yacyinteractive.html +++ b/htroot/yacyinteractive.html @@ -68,25 +68,7 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de </div> <div id="downloadscript" style="clear:both;"></div> <div id="searchresults" style="clear:both;"> -<h3>DECENTRALIZE - run this search portal yourself (if you like). Just follow these steps:</h3> -<ul> -<li>download the software from yacy.net and run it. (you need OpenJDK6). A http server is started and serves the administration pages.</li> -<li>open http://localhost:8080/ConfigBasic.html and select 'Search portal for your own web pages' ('Intranet Indexing' if you wanto to do this for your / other network) -> Set Configuration</li> -<li>click on 'System Update' and use 'Automatic Update' to get the latest features which are needed for this. The Peer re-starts automatically. Wait and open http://localhost:8080 again.</li> -<li>click 'Index Creation' -> 'Network Scanner' ( http://localhost:8080/CrawlStartScanner_p.html )</li> -<li>in 'Scan Range' enter some IPs (comma-separated) that shall be used as prototype for a 255.255.255.0 subrange. I use '81.163.150.96,81.163.2.42,81.163.18.81,81.163.62.38,81.163.22.118, 81.163.112.96' (without quotes)</li> -<li>at 'Scheduler' select 'scan and add all sites with granted access automatically' -> Scan. Wait. This makes a portscan and ftp/smb access test on the given range. Wait at least one minute. Keep calm.</li> -<li>You should now see a list of hosts that are crawled. Your CPU load should increase. All IPs are scanned every 10 minutes, but crawls are done only for new visible servers.</li> -<li>You can monitor the crawler at 'Crawler Monitor' ( http://localhost:8080/Crawler_p.html ).</li> -<li>You may want to increase the memory at 'Admin Console' -> 'Performance' ( http://localhost:8080/Performance_p.html ) (you must restart then but wait until the FTP site-listing is done it is lost otherwards :-(</li> -<li>Please set a very low download limit at 'Advanced Settings' -> 'Crawler Settings' ( http://localhost:8080/Settings_p.html?page=crawler ) .. if documents are not loaded, just their document name is indexed.</li> -<li>Ready - search at 'Interactive local search' ( http://localhost:8080/yacyinteractive.html )</li> -<li>You may want to set a peer password at http://localhost:8080/ConfigAccounts_p.html</li> -<li>Get the right look at http://localhost:8080/ConfigAppearance_p.html and select the 27c3 Skin</li> -<li>Set search page descriptions and images at http://localhost:8080/ConfigPortal.html</li> -<li>Visit our <a href="http://forum.yacy-websuche.de/viewtopic.php?p=21332#p21332">forum</a> if there are any questions.</li> -</ul> -<div><img src="PerformanceGraph.png" id="graph" alt="PerformanceGraph" width="660" height="240"/></div> +<!-- <div><img src="PerformanceGraph.png" id="graph" alt="PerformanceGraph" width="660" height="240"/></div> --> </div> <script type="text/javascript"> //<![CDATA[ diff --git a/htroot/yacysearch_location.java b/htroot/yacysearch_location.java index 9c467bed3..5820f6243 100644 --- a/htroot/yacysearch_location.java +++ b/htroot/yacysearch_location.java @@ -20,6 +20,7 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import net.yacy.cora.document.RSSMessage; @@ -91,7 +92,8 @@ public class yacysearch_location { if (search_title || search_publisher || search_creator || search_subject) try { // get a queue of search results String rssSearchServiceURL = "http://localhost:" + sb.getConfig("port", "8080") + "/yacysearch.rss"; - BlockingQueue<RSSMessage> results = Search.search(rssSearchServiceURL, query, false, false, maximumTime, Integer.MAX_VALUE); + BlockingQueue<RSSMessage> results = new LinkedBlockingQueue<RSSMessage>(); + Search.searchSRURSS(rssSearchServiceURL, query, maximumTime, Integer.MAX_VALUE, false, false, results); // take the results and compute some locations RSSMessage message; diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index 16082d332..0647212be 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -28,11 +28,11 @@ import java.net.MalformedURLException; import java.util.List; import java.util.TreeSet; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.Formatter; @@ -116,7 +116,7 @@ public class yacysearchitem { prop.put("content", 1); // switch on specific content prop.put("content_authorized", authenticated ? "1" : "0"); - prop.put("content_authorized_recommend", "0" /*(sb.peers.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? "1" : "0"*/); // emergency fix! please re-design net.yacy.kelondro.table.ChunkIterator.<init>(ChunkIterator.java:63) + prop.put("content_authorized_recommend", (sb.peers.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? "1" : "0"); prop.putHTML("content_authorized_recommend_deletelink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&deleteref=" + new String(result.hash()) + "&urlmaskfilter=.*"); prop.putHTML("content_authorized_recommend_recommendlink", "/yacysearch.html?query=" + theQuery.queryString.replace(' ', '+') + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&recommendref=" + new String(result.hash()) + "&urlmaskfilter=.*"); prop.put("content_authorized_urlhash", new String(result.hash())); @@ -131,7 +131,7 @@ public class yacysearchitem { prop.put("content_urlhash", resulthashString); prop.put("content_urlhexhash", yacySeed.b64Hash2hexHash(resulthashString)); prop.putHTML("content_urlname", nxTools.shortenURLString(result.urlname(), urllength)); - prop.put("content_date", DateFormatter.formatRFC1123_short(result.modified())); + prop.put("content_date", GenericFormatter.RFC1123_SHORT_FORMATTER.format(result.modified())); prop.put("content_date822", HeaderFramework.formatRFC1123(result.modified())); //prop.put("content_ybr", RankingProcess.ybr(result.hash())); prop.putHTML("content_size", Integer.toString(result.filesize())); // we don't use putNUM here because that number shall be usable as sorting key. To print the size, use 'sizename' diff --git a/source/de/anomic/crawler/retrieval/Response.java b/source/de/anomic/crawler/retrieval/Response.java index d3f29fda6..f634727e2 100755 --- a/source/de/anomic/crawler/retrieval/Response.java +++ b/source/de/anomic/crawler/retrieval/Response.java @@ -29,6 +29,7 @@ package de.anomic.crawler.retrieval; import java.io.ByteArrayInputStream; import java.util.Date; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; @@ -37,8 +38,6 @@ import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.TextParser; import net.yacy.kelondro.data.meta.DigestURI; -import net.yacy.kelondro.util.DateFormatter; - import de.anomic.crawler.CrawlProfile; public class Response { @@ -217,7 +216,7 @@ public class Response { if (docDate == null) docDate = responseHeader.date(); } if (docDate == null && request != null) docDate = request.appdate(); - if (docDate == null) docDate = new Date(DateFormatter.correctedUTCTime()); + if (docDate == null) docDate = new Date(GenericFormatter.correctedUTCTime()); return docDate; } @@ -332,7 +331,7 @@ public class Response { if (date == null) return "stale_no_date_given_in_response"; try { final long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live - if (DateFormatter.correctedUTCTime() - date.getTime() > ttl) { + if (GenericFormatter.correctedUTCTime() - date.getTime() > ttl) { //System.out.println("***not indexed because cache-control"); return "stale_expired"; } @@ -419,8 +418,8 @@ public class Response { if (!responseHeader.containsKey(HeaderFramework.LAST_MODIFIED)) { return false; } // parse date Date d1, d2; - d2 = responseHeader.lastModified(); if (d2 == null) { d2 = new Date(DateFormatter.correctedUTCTime()); } - d1 = requestHeader.ifModifiedSince(); if (d1 == null) { d1 = new Date(DateFormatter.correctedUTCTime()); } + d2 = responseHeader.lastModified(); if (d2 == null) { d2 = new Date(GenericFormatter.correctedUTCTime()); } + d1 = requestHeader.ifModifiedSince(); if (d1 == null) { d1 = new Date(GenericFormatter.correctedUTCTime()); } // finally, we shall treat the cache as stale if the modification time is after the if-.. time if (d2.after(d1)) { return false; } } @@ -461,7 +460,7 @@ public class Response { final Date expires = responseHeader.expires(); if (expires != null) { // System.out.println("EXPIRES-TEST: expires=" + expires + ", NOW=" + serverDate.correctedGMTDate() + ", url=" + url); - if (expires.before(new Date(DateFormatter.correctedUTCTime()))) { return false; } + if (expires.before(new Date(GenericFormatter.correctedUTCTime()))) { return false; } } final Date lastModified = responseHeader.lastModified(); cacheControl = responseHeader.get(HeaderFramework.CACHE_CONTROL); @@ -475,13 +474,13 @@ public class Response { // file may only be treated as fresh for one more month, not more. Date date = responseHeader.date(); if (lastModified != null) { - if (date == null) { date = new Date(DateFormatter.correctedUTCTime()); } + if (date == null) { date = new Date(GenericFormatter.correctedUTCTime()); } final long age = date.getTime() - lastModified.getTime(); if (age < 0) { return false; } // TTL (Time-To-Live) is age/10 = (d2.getTime() - d1.getTime()) / 10 // the actual living-time is serverDate.correctedGMTDate().getTime() - d2.getTime() // therefore the cache is stale, if serverDate.correctedGMTDate().getTime() - d2.getTime() > age/10 - if (DateFormatter.correctedUTCTime() - date.getTime() > age / 10) { return false; } + if (GenericFormatter.correctedUTCTime() - date.getTime() > age / 10) { return false; } } // -cache-control in cached response @@ -500,7 +499,7 @@ public class Response { if (date == null) { return false; } try { final long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live - if (DateFormatter.correctedUTCTime() - date.getTime() > ttl) { + if (GenericFormatter.correctedUTCTime() - date.getTime() > ttl) { return false; } } catch (final Exception e) { @@ -588,7 +587,7 @@ public class Response { // parse date Date d = responseHeader.lastModified(); if (d == null) { - d = new Date(DateFormatter.correctedUTCTime()); + d = new Date(GenericFormatter.correctedUTCTime()); } // finally, we shall treat the cache as stale if the modification time is after the if-.. time if (d.after(ifModifiedSince)) { @@ -613,7 +612,7 @@ public class Response { // sometimes, the expires date is set to the past to prevent that a page is cached // we use that information to see if we should index it final Date expires = responseHeader.expires(); - if (expires != null && expires.before(new Date(DateFormatter.correctedUTCTime()))) { + if (expires != null && expires.before(new Date(GenericFormatter.correctedUTCTime()))) { return "Stale_(Expired)"; } @@ -646,7 +645,7 @@ public class Response { } try { final long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live - if (DateFormatter.correctedUTCTime() - date.getTime() > ttl) { + if (GenericFormatter.correctedUTCTime() - date.getTime() > ttl) { //System.out.println("***not indexed because cache-control"); return "Stale_(expired_by_cache-control)"; } diff --git a/source/de/anomic/data/BlogBoard.java b/source/de/anomic/data/BlogBoard.java index 70af1d7fe..2b96de55a 100644 --- a/source/de/anomic/data/BlogBoard.java +++ b/source/de/anomic/data/BlogBoard.java @@ -46,12 +46,12 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import net.yacy.cora.date.GenericFormatter; import net.yacy.kelondro.blob.MapHeap; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.NaturalOrder; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.kelondroException; import org.w3c.dom.Document; @@ -209,7 +209,7 @@ public class BlogBoard { } try { - date = DateFormatter.parseShortSecond(StrDate); + date = GenericFormatter.SHORT_SECOND_FORMATTER.parse(StrDate); } catch (final ParseException e1) { date = new Date(); } @@ -414,7 +414,7 @@ public class BlogBoard { } return new Date(); } - return DateFormatter.parseShortSecond(date); + return GenericFormatter.SHORT_SECOND_FORMATTER.parse(date); } catch (final ParseException ex) { return new Date(); } @@ -425,7 +425,7 @@ public class BlogBoard { if (ret == null) { ret = new Date(); } - record.put("date", DateFormatter.formatShortSecond(ret)); + record.put("date", GenericFormatter.SHORT_SECOND_FORMATTER.format(ret)); } public String getTimestamp() { @@ -434,7 +434,7 @@ public class BlogBoard { if (Log.isFinest("Blog")) { Log.logFinest("Blog", "ERROR: date field missing in blogBoard"); } - return DateFormatter.formatShortSecond(); + return GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()); } return timestamp; } diff --git a/source/de/anomic/data/BookmarkHelper.java b/source/de/anomic/data/BookmarkHelper.java index 0764e233a..411c4faea 100644 --- a/source/de/anomic/data/BookmarkHelper.java +++ b/source/de/anomic/data/BookmarkHelper.java @@ -52,13 +52,13 @@ import org.xml.sax.SAXException; import de.anomic.data.BookmarksDB.Bookmark; import de.anomic.data.BookmarksDB.Tag; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.document.parser.html.ContentScraper; import net.yacy.document.parser.html.TransformerWriter; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; public class BookmarkHelper { @@ -229,7 +229,7 @@ public class BookmarkHelper { Date parsedDate = null; try { - parsedDate = DateFormatter.parseISO8601(time); + parsedDate = ISO8601Formatter.FORMATTER.parse(time); } catch (final ParseException e) { parsedDate = new Date(); } diff --git a/source/de/anomic/data/WorkTables.java b/source/de/anomic/data/WorkTables.java index 3814140f2..c63982a49 100644 --- a/source/de/anomic/data/WorkTables.java +++ b/source/de/anomic/data/WorkTables.java @@ -31,9 +31,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Date; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.TreeMap; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.data.meta.DigestURI; @@ -41,8 +44,9 @@ import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; +import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.rwi.IndexCell; -import net.yacy.kelondro.util.DateFormatter; +import de.anomic.search.Switchboard; import de.anomic.server.serverObjects; public class WorkTables extends Tables { @@ -120,7 +124,7 @@ public class WorkTables extends Tables { Data data = new Data(); data.put(TABLE_API_COL_TYPE, type.getBytes()); data.put(TABLE_API_COL_COMMENT, comment.getBytes()); - byte[] date = DateFormatter.formatShortMilliSecond(new Date()).getBytes(); + byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()).getBytes(); data.put(TABLE_API_COL_DATE_RECORDING, date); data.put(TABLE_API_COL_DATE_LAST_EXEC, date); data.put(TABLE_API_COL_URL, apiurl.getBytes()); @@ -132,7 +136,7 @@ public class WorkTables extends Tables { // modify and update existing entry // modify date attributes and patch old values - row.put(TABLE_API_COL_DATE_LAST_EXEC, DateFormatter.formatShortMilliSecond(new Date()).getBytes()); + row.put(TABLE_API_COL_DATE_LAST_EXEC, GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()).getBytes()); if (!row.containsKey(TABLE_API_COL_DATE_RECORDING)) row.put(TABLE_API_COL_DATE_RECORDING, row.get(TABLE_API_COL_DATE)); row.remove(TABLE_API_COL_DATE); @@ -181,7 +185,7 @@ public class WorkTables extends Tables { Data data = new Data(); data.put(TABLE_API_COL_TYPE, type.getBytes()); data.put(TABLE_API_COL_COMMENT, comment.getBytes()); - byte[] date = DateFormatter.formatShortMilliSecond(new Date()).getBytes(); + byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()).getBytes(); data.put(TABLE_API_COL_DATE_RECORDING, date); data.put(TABLE_API_COL_DATE_LAST_EXEC, date); data.put(TABLE_API_COL_URL, apiurl.getBytes()); @@ -305,7 +309,7 @@ public class WorkTables extends Tables { try { // create and insert new entry Data data = new Data(); - byte[] date = DateFormatter.formatShortMilliSecond(new Date()).getBytes(); + byte[] date = GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()).getBytes(); data.put(TABLE_SEARCH_FAILURE_COL_URL, url.toNormalform(true, false)); data.put(TABLE_SEARCH_FAILURE_COL_DATE, date); data.put(TABLE_SEARCH_FAILURE_COL_WORDS, queryHashes.export()); @@ -324,4 +328,20 @@ public class WorkTables extends Tables { return false; } } + + public static Map<byte[], String> commentCache(Switchboard sb) { + Map<byte[], String> comments = new TreeMap<byte[], String>(Base64Order.enhancedCoder); + Iterator<Tables.Row> i; + try { + i = sb.tables.iterator(WorkTables.TABLE_API_NAME); + Tables.Row row; + while (i.hasNext()) { + row = i.next(); + comments.put(row.getPK(), new String(row.get(WorkTables.TABLE_API_COL_COMMENT))); + } + } catch (IOException e) { + Log.logException(e); + } + return comments; + } } diff --git a/source/de/anomic/data/YMarkTables.java b/source/de/anomic/data/YMarkTables.java index 870312e74..5aa7345a5 100644 --- a/source/de/anomic/data/YMarkTables.java +++ b/source/de/anomic/data/YMarkTables.java @@ -16,6 +16,7 @@ import java.util.Iterator; import java.util.Map; import java.util.TreeMap; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.document.Condenser; import net.yacy.document.Document; import net.yacy.kelondro.blob.Tables; @@ -25,7 +26,6 @@ import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; import de.anomic.search.Segment; public class YMarkTables { @@ -202,7 +202,7 @@ public class YMarkTables { if(date != null) { final String s = new String(date); if(s != null && s.length() > 0) - return DateFormatter.formatISO8601(new Date(Long.parseLong(s))); + return ISO8601Formatter.FORMATTER.format(new Date(Long.parseLong(s))); } return ""; } @@ -419,9 +419,9 @@ public class YMarkTables { if (urlEntry != null) { metadata.put(METADATA.IN_URLDB, "true"); metadata.put(METADATA.SIZE, String.valueOf(urlEntry.size())); - metadata.put(METADATA.FRESHDATE, DateFormatter.formatISO8601(urlEntry.freshdate())); - metadata.put(METADATA.LOADDATE, DateFormatter.formatISO8601(urlEntry.loaddate())); - metadata.put(METADATA.MODDATE, DateFormatter.formatISO8601(urlEntry.moddate())); + metadata.put(METADATA.FRESHDATE, ISO8601Formatter.FORMATTER.format(urlEntry.freshdate())); + metadata.put(METADATA.LOADDATE, ISO8601Formatter.FORMATTER.format(urlEntry.loaddate())); + metadata.put(METADATA.MODDATE, ISO8601Formatter.FORMATTER.format(urlEntry.moddate())); metadata.put(METADATA.SNIPPET, String.valueOf(urlEntry.snippet())); metadata.put(METADATA.WORDCOUNT, String.valueOf(urlEntry.wordCount())); metadata.put(METADATA.MIMETYPE, String.valueOf(urlEntry.doctype())); diff --git a/source/de/anomic/http/server/AlternativeDomainNames.java b/source/de/anomic/http/server/AlternativeDomainNames.java index 83e2a3103..a6f5c87f7 100644 --- a/source/de/anomic/http/server/AlternativeDomainNames.java +++ b/source/de/anomic/http/server/AlternativeDomainNames.java @@ -1,26 +1,26 @@ -// httpdAlternativeDomainNames.java -// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany -// first published 05.05.2008 on http://yacy.net -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +/** + * AlternativeDomainNames + * (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany + * first published 05.05.2008 on http://yacy.net + * + * $LastChangedDate$ + * $LastChangedRevision$ + * $LastChangedBy$ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ package de.anomic.http.server; diff --git a/source/de/anomic/http/server/HTTPDFileHandler.java b/source/de/anomic/http/server/HTTPDFileHandler.java index a108b5178..7fe0eb570 100644 --- a/source/de/anomic/http/server/HTTPDFileHandler.java +++ b/source/de/anomic/http/server/HTTPDFileHandler.java @@ -79,6 +79,7 @@ import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.zip.GZIPOutputStream; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; @@ -90,7 +91,6 @@ import net.yacy.document.parser.html.ScraperInputStream; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.ByteBuffer; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; import net.yacy.visualization.RasterPlotter; @@ -490,7 +490,7 @@ public final class HTTPDFileHandler { if (author != null && author.length() > 0) aBuffer.append("Author: " + author + "<br/>"); if (publisher != null && publisher.length() > 0) aBuffer.append("Publisher: " + publisher + "<br/>"); if (description != null && description.length() > 0) aBuffer.append("Description: " + description + "<br/>"); - aBuffer.append(DateFormatter.formatShortDay(new Date(f.lastModified())) + ", " + size + ((images > 0) ? ", " + images + " images" : "") + ((links > 0) ? ", " + links + " links" : "") + "<br/></li>\n"); + aBuffer.append(GenericFormatter.SHORT_DAY_FORMATTER.format(new Date(f.lastModified())) + ", " + size + ((images > 0) ? ", " + images + " images" : "") + ((links > 0) ? ", " + links + " links" : "") + "<br/></li>\n"); } } aBuffer.append(" </ul>\n</body>\n</html>\n"); @@ -823,7 +823,7 @@ public final class HTTPDFileHandler { templatePatterns.put(servletProperties.PEER_STAT_UPTIME, ((System.currentTimeMillis() - serverCore.startupTime) / 1000) / 60); // uptime in minutes templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTNAME, switchboard.getConfig("peerName", "anomic")); templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTID, ((Switchboard) switchboard).peers.myID()); - templatePatterns.put(servletProperties.PEER_STAT_MYTIME, DateFormatter.formatShortSecond()); + templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date())); yacySeed myPeer = sb.peers.mySeed(); templatePatterns.put("newpeer", myPeer.getAge() >= 1 ? 0 : 1); templatePatterns.putHTML("newpeer_peerhash", myPeer.hash); diff --git a/source/de/anomic/http/server/HTTPDemon.java b/source/de/anomic/http/server/HTTPDemon.java index c5befd437..30c181791 100644 --- a/source/de/anomic/http/server/HTTPDemon.java +++ b/source/de/anomic/http/server/HTTPDemon.java @@ -48,6 +48,7 @@ import java.util.Properties; import java.util.StringTokenizer; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; import net.yacy.cora.protocol.Domains; @@ -398,7 +399,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { : readHeader(prop, session); // handling transparent proxy support - HeaderFramework.handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy); + handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy); // determines if the connection should be kept alive handlePersistentConnection(header, prop); @@ -464,7 +465,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { else header = readHeader(prop,session); // handle transparent proxy support - HeaderFramework.handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy); + handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy); // determines if the connection should be kept alive handlePersistentConnection(header, prop); @@ -530,7 +531,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { } // handle transparent proxy support - HeaderFramework.handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy); + handleTransparentProxySupport(header, prop, virtualHost, HTTPDProxyHandler.isTransparentProxy); // determines if the connection should be kept alive handlePersistentConnection(header, prop); @@ -564,7 +565,22 @@ public final class HTTPDemon implements serverHandler, Cloneable { return serverCore.TERMINATE_CONNECTION; } } - + + public static void handleTransparentProxySupport(final RequestHeader header, final Properties prop, final String virtualHost, final boolean isTransparentProxy) { + // transparent proxy support is only available for http 1.0 and above connections + if (prop.getProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, "HTTP/0.9").equals("HTTP/0.9")) return; + + // if the transparent proxy support was disabled, we have nothing todo here ... + if (!(isTransparentProxy && header.containsKey(HeaderFramework.HOST))) return; + + // we only need to do the transparent proxy support if the request URL didn't contain the hostname + // and therefor was set to virtualHost by function parseQuery() + if (!prop.getProperty(HeaderFramework.CONNECTION_PROP_HOST).equals(virtualHost)) return; + + // TODO: we could have problems with connections from extern here ... + final String dstHostSocket = header.get(HeaderFramework.HOST); + prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST,(HTTPDemon.isThisHostName(dstHostSocket)?virtualHost:dstHostSocket)); + } public Boolean CONNECT(String arg, Session session) throws IOException { // establish a ssh-tunneled http connection @@ -628,7 +644,7 @@ public final class HTTPDemon implements serverHandler, Cloneable { private final Properties parseRequestLine(final String cmd, final String s, Session session) { // parsing the header - Properties p = RequestHeader.parseRequestLine(cmd, s, virtualHost); + Properties p = parseRequestLine(cmd, s, virtualHost); // track the request final String path = p.getProperty(HeaderFramework.CONNECTION_PROP_URL); @@ -1425,5 +1441,126 @@ public final class HTTPDemon implements serverHandler, Cloneable { return header; } + + private static final Pattern P_20 = Pattern.compile(" ", Pattern.LITERAL); + private static final Pattern P_7B = Pattern.compile("{", Pattern.LITERAL); + private static final Pattern P_7D = Pattern.compile("}", Pattern.LITERAL); + private static final Pattern P_7C = Pattern.compile("|", Pattern.LITERAL); + private static final Pattern P_5C = Pattern.compile("\\", Pattern.LITERAL); + private static final Pattern P_5E = Pattern.compile("^", Pattern.LITERAL); + private static final Pattern P_5B = Pattern.compile("[", Pattern.LITERAL); + private static final Pattern P_5D = Pattern.compile("]", Pattern.LITERAL); + private static final Pattern P_60 = Pattern.compile("`", Pattern.LITERAL); + + public static Properties parseRequestLine(final String cmd, String args, final String virtualHost) { + + final Properties prop = new Properties(); + + // storing informations about the request + prop.setProperty(HeaderFramework.CONNECTION_PROP_METHOD, cmd); + prop.setProperty(HeaderFramework.CONNECTION_PROP_REQUESTLINE, cmd + " " + args); + + // this parses a whole URL + if (args.length() == 0) { + prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST, virtualHost); + prop.setProperty(HeaderFramework.CONNECTION_PROP_PATH, "/"); + prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9); + prop.setProperty(HeaderFramework.CONNECTION_PROP_EXT, ""); + return prop; + } + + // store the version propery "HTTP" and cut the query at both ends + int sep = args.lastIndexOf(' '); + if ((sep >= 0)&&(args.substring(sep + 1).toLowerCase().startsWith("http/"))) { + // HTTP version is given + prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, args.substring(sep + 1).trim()); + args = args.substring(0, sep).trim(); // cut off HTTP version mark + } else { + // HTTP version is not given, it will be treated as ver 0.9 + prop.setProperty(HeaderFramework.CONNECTION_PROP_HTTP_VER, HeaderFramework.HTTP_VERSION_0_9); + } + + // replacing spaces in the url string correctly + args = P_20.matcher(args).replaceAll("%20"); + // replace unwise characters (see RFC 2396, 2.4.3), which may not be escaped + args = P_7B.matcher(args).replaceAll("%7B"); + args = P_7D.matcher(args).replaceAll("%7D"); + args = P_7C.matcher(args).replaceAll("%7C"); + args = P_5C.matcher(args).replaceAll("%5C"); + args = P_5E.matcher(args).replaceAll("%5E"); + args = P_5B.matcher(args).replaceAll("%5B"); + args = P_5D.matcher(args).replaceAll("%5D"); + args = P_60.matcher(args).replaceAll("%60"); + + // properties of the query are stored with the prefix "&" + // additionally, the values URL and ARGC are computed + + String argsString = ""; + sep = args.indexOf('?'); + if (sep >= 0) { + // there are values attached to the query string + argsString = args.substring(sep + 1); // cut head from tail of query + args = args.substring(0, sep); + } + prop.setProperty(HeaderFramework.CONNECTION_PROP_URL, args); // store URL + //System.out.println("HTTPD: ARGS=" + argsString); + if (argsString.length() != 0) prop.setProperty(HeaderFramework.CONNECTION_PROP_ARGS, argsString); // store arguments in original form + + // finally find host string + String path; + if (args.toUpperCase().startsWith("HTTP://")) { + // a host was given. extract it and set path + args = args.substring(7); + sep = args.indexOf('/'); + if (sep < 0) { + // this is a malformed url, something like + // http://index.html + // we are lazy and guess that it means + // /index.html + // which is a localhost access to the file servlet + prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST, args); + path = "/"; + } else { + // THIS IS THE "GOOD" CASE + // a perfect formulated url + final String dstHostSocket = args.substring(0, sep); + prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST, (HTTPDemon.isThisHostName(dstHostSocket) ? virtualHost : dstHostSocket)); + path = args.substring(sep); // yes, including beginning "/" + } + } else { + // no host in url. set path + if (args.length() > 0 && args.charAt(0) == '/') { + // thats also fine, its a perfect localhost access + // in this case, we simulate a + // http://localhost/s + // access by setting a virtual host + prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST, virtualHost); + path = args; + } else { + // the client 'forgot' to set a leading '/' + // this is the same case as above, with some lazyness + prop.setProperty(HeaderFramework.CONNECTION_PROP_HOST, virtualHost); + path = "/" + args; + } + } + prop.setProperty(HeaderFramework.CONNECTION_PROP_PATH, path); + + // find out file extension (we already stripped ?-parameters from args) + String ext = ""; // default when no file extension + sep = path.lastIndexOf('.'); + if (sep >= 0) { + final int ancpos = path.indexOf("#", sep + 1); + if (ancpos >= sep) { + // ex: /foo/bar.html#xy => html + ext = path.substring(sep + 1, ancpos).toLowerCase(); + } else { + // ex: /foo/bar.php => php + ext = path.substring(sep + 1).toLowerCase(); + } + } + prop.setProperty(HeaderFramework.CONNECTION_PROP_EXT, ext); + + return prop; + } } diff --git a/source/de/anomic/search/AccessTracker.java b/source/de/anomic/search/AccessTracker.java index 10ac39a61..3d27b6728 100644 --- a/source/de/anomic/search/AccessTracker.java +++ b/source/de/anomic/search/AccessTracker.java @@ -26,12 +26,12 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.util.ArrayList; +import java.util.Date; import java.util.Iterator; import java.util.LinkedList; +import net.yacy.cora.date.GenericFormatter; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; - import de.anomic.data.LibraryProvider; public class AccessTracker { @@ -85,7 +85,7 @@ public class AccessTracker { //if (query.resultcount == 0) return; if (query.queryString == null || query.queryString.length() == 0) return; StringBuilder sb = new StringBuilder(40); - sb.append(DateFormatter.formatShortSecond()); + sb.append(GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date())); sb.append(' '); sb.append(Integer.toString(query.resultcount)); sb.append(' '); diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index aa1dc5155..9d4721e66 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -75,6 +75,7 @@ import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.RSSFeed; import net.yacy.cora.document.RSSMessage; @@ -104,7 +105,6 @@ import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.order.NaturalOrder; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.EventTracker; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; @@ -2457,7 +2457,7 @@ public final class Switchboard extends serverSwitch { peers.mySeed().put(yacySeed.VERSION, yacyBuildProperties.getLongVersion()); peers.mySeed().setFlagDirectConnect(true); peers.mySeed().setLastSeenUTC(); - peers.mySeed().put(yacySeed.UTC, DateFormatter.UTCDiffString()); + peers.mySeed().put(yacySeed.UTC, GenericFormatter.UTCDiffString()); peers.mySeed().setFlagAcceptRemoteCrawl(getConfig("crawlResponse", "").equals("true")); peers.mySeed().setFlagAcceptRemoteIndex(getConfig("allowReceiveIndex", "").equals("true")); //mySeed.setFlagAcceptRemoteIndex(true); diff --git a/source/de/anomic/server/serverObjects.java b/source/de/anomic/server/serverObjects.java index f9b799546..f0cb692e2 100644 --- a/source/de/anomic/server/serverObjects.java +++ b/source/de/anomic/server/serverObjects.java @@ -56,7 +56,6 @@ import java.util.regex.Pattern; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.document.parser.html.CharacterCoding; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.Formatter; import de.anomic.search.Switchboard; @@ -160,10 +159,6 @@ public class serverObjects extends HashMap<String, String> implements Cloneable return this.put(key, value.toString()); } - public String put(final String key, final DateFormatter value) { - return this.put(key, value.toString()); - } - public String put(final String key, final InetAddress value) { return this.put(key, value.toString()); } diff --git a/source/de/anomic/yacy/dht/PeerSelection.java b/source/de/anomic/yacy/dht/PeerSelection.java index 6aa08930e..7b13063bf 100755 --- a/source/de/anomic/yacy/dht/PeerSelection.java +++ b/source/de/anomic/yacy/dht/PeerSelection.java @@ -30,6 +30,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import net.yacy.cora.date.AbstractFormatter; import net.yacy.cora.storage.DynamicScore; import net.yacy.cora.storage.ScoreCluster; import net.yacy.kelondro.data.word.Word; @@ -38,7 +39,6 @@ import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Digest; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.kelondroException; import de.anomic.yacy.yacyCore; @@ -351,7 +351,7 @@ public class PeerSelection { while ((s.hasNext()) && (searchcount-- > 0)) { ys = s.next(); if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try { - absage = Math.abs(System.currentTimeMillis() + DateFormatter.dayMillis - ys.getLastSeenUTC()) / 1000 / 60; + absage = Math.abs(System.currentTimeMillis() + AbstractFormatter.dayMillis - ys.getLastSeenUTC()) / 1000 / 60; if (absage > Integer.MAX_VALUE) absage = Integer.MAX_VALUE; seedScore.inc(ys.hash, (int) absage); // the higher absage, the older is the peer } catch (final Exception e) {} diff --git a/source/de/anomic/yacy/graphics/WebStructureGraph.java b/source/de/anomic/yacy/graphics/WebStructureGraph.java index ab6ded34d..f679136f4 100644 --- a/source/de/anomic/yacy/graphics/WebStructureGraph.java +++ b/source/de/anomic/yacy/graphics/WebStructureGraph.java @@ -37,12 +37,12 @@ import java.util.SortedMap; import java.util.TreeMap; import java.util.TreeSet; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.document.Condenser; import net.yacy.document.Document; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.LookAheadIterator; @@ -150,7 +150,7 @@ public class WebStructureGraph { private static String map2refstr(final Map<String, Integer> map) { final StringBuilder s = new StringBuilder(map.size() * 10); - s.append(DateFormatter.formatShortDay(new Date())); + s.append(GenericFormatter.SHORT_DAY_FORMATTER.format(new Date())); String h; for (final Map.Entry<String, Integer> entry : map.entrySet()) { s.append(entry.getKey()); @@ -229,7 +229,7 @@ public class WebStructureGraph { return new structureEntry( domhash, host, - DateFormatter.formatShortDay(new Date()), + GenericFormatter.SHORT_DAY_FORMATTER.format(new Date()), domhashes); } diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 0ecf44ee6..30ad547d7 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -372,7 +372,7 @@ public final class yacyClient { public static RSSFeed search(final yacySeed targetSeed, String query, boolean verify, boolean global, long timeout, int startRecord, int maximumRecords) throws IOException { String address = (targetSeed == null || targetSeed == Switchboard.getSwitchboard().peers.mySeed()) ? "localhost:" + Switchboard.getSwitchboard().getConfig("port", "8080") : targetSeed.getClusterAddress(); String urlBase = "http://" + address + "/yacysearch.rss"; - return Search.search(urlBase, query, verify, global, timeout, startRecord, maximumRecords); + return Search.loadSRURSS(urlBase, query, timeout, startRecord, maximumRecords, verify, global); } @SuppressWarnings("unchecked") diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index bf73d785d..0e082e682 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -49,12 +49,11 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.RSSFeed; import net.yacy.cora.document.RSSMessage; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; - import de.anomic.search.Switchboard; import de.anomic.server.serverCore; import de.anomic.yacy.dht.PeerSelection; @@ -256,14 +255,14 @@ public class yacyCore { if (newSeed.getLastSeenUTC() >= this.seed.getLastSeenUTC()) { if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " with old LastSeen: '" + - DateFormatter.formatShortSecond(new Date(newSeed.getLastSeenUTC())) + "'"); + GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "'"); newSeed.setLastSeenUTC(); sb.peers.peerActions.peerArrival(newSeed, true); } else { if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " with old LastSeen: '" + - DateFormatter.formatShortSecond(new Date(newSeed.getLastSeenUTC())) + "', this is more recent: '" + - DateFormatter.formatShortSecond(new Date(this.seed.getLastSeenUTC())) + "'"); + GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "', this is more recent: '" + + GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(this.seed.getLastSeenUTC())) + "'"); this.seed.setLastSeenUTC(); sb.peers.peerActions.peerArrival(this.seed, true); } diff --git a/source/de/anomic/yacy/yacyNetwork.java b/source/de/anomic/yacy/yacyNetwork.java index a60b96de2..f97754083 100644 --- a/source/de/anomic/yacy/yacyNetwork.java +++ b/source/de/anomic/yacy/yacyNetwork.java @@ -33,9 +33,8 @@ import java.util.Date; import java.util.LinkedHashMap; //import java.util.List; +import net.yacy.cora.date.GenericFormatter; import net.yacy.kelondro.order.Digest; -import net.yacy.kelondro.util.DateFormatter; - //import org.apache.commons.httpclient.methods.multipart.Part; import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.StringBody; @@ -115,7 +114,7 @@ public class yacyNetwork { if (targetHash != null) parts.put("youare", new StringBody(targetHash)); // time information for synchronization - parts.put("mytime", new StringBody(DateFormatter.formatShortSecond(new Date()))); + parts.put("mytime", new StringBody(GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()))); parts.put("myUTC", new StringBody(Long.toString(System.currentTimeMillis()))); // network identification diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index 114e157d4..1b197b917 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -54,6 +54,7 @@ import java.util.Map; import java.util.Properties; import java.util.Map.Entry; +import net.yacy.cora.date.GenericFormatter; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.Index; import net.yacy.kelondro.index.Row; @@ -62,7 +63,6 @@ import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.table.Table; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.kelondroException; import net.yacy.kelondro.util.MapTools; @@ -76,7 +76,7 @@ public class yacyNewsDB { protected Index news; private static final int categoryStringLength = 8; - public static final int idLength = DateFormatter.PATTERN_SHORT_SECOND.length() + Word.commonHashLength; + public static final int idLength = GenericFormatter.PATTERN_SHORT_SECOND.length() + Word.commonHashLength; public yacyNewsDB( final File path, @@ -87,12 +87,12 @@ public class yacyNewsDB { this.attributesMaxLength = maxNewsRecordLength - idLength - categoryStringLength - - DateFormatter.PATTERN_SHORT_SECOND.length() + - GenericFormatter.PATTERN_SHORT_SECOND.length() - 2; this.rowdef = new Row( "String idx-" + idLength + " \"id = created + originator\"," + "String cat-" + categoryStringLength + "," + - "String rec-" + DateFormatter.PATTERN_SHORT_SECOND.length() + "," + + "String rec-" + GenericFormatter.PATTERN_SHORT_SECOND.length() + "," + "short dis-2 {b64e}," + "String att-" + attributesMaxLength, NaturalOrder.naturalOrder @@ -189,7 +189,7 @@ public class yacyNewsDB { return new yacyNewsDB.Record( b.getColString(0, null), b.getColString(1, "UTF-8"), - (b.empty(2)) ? null : DateFormatter.parseShortSecond(b.getColString(2, null), DateFormatter.UTCDiffString()), + (b.empty(2)) ? null : GenericFormatter.SHORT_SECOND_FORMATTER.parse(b.getColString(2, null), GenericFormatter.UTCDiffString()), (int) b.getColLong(3), MapTools.string2map(b.getColString(4, "UTF-8"), ",") ); @@ -206,7 +206,7 @@ public class yacyNewsDB { final Row.Entry entry = this.news.row().newEntry(); entry.setCol(0, r.id().getBytes()); entry.setCol(1, r.category().getBytes("UTF-8")); - entry.setCol(2, (r.received() == null) ? null : DateFormatter.formatShortSecond(r.received()).getBytes()); + entry.setCol(2, (r.received() == null) ? null : GenericFormatter.SHORT_SECOND_FORMATTER.format(r.received()).getBytes()); entry.setCol(3, Base64Order.enhancedCoder.encodeLong(r.distributed(), 2).getBytes()); entry.setCol(4, attributes.getBytes("UTF-8")); return entry; @@ -265,8 +265,8 @@ public class yacyNewsDB { if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")"); this.category = (attributes.containsKey("cat")) ? attributes.get("cat") : ""; if (category.length() > yacyNewsDB.categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + yacyNewsDB.categoryStringLength + ")"); - this.received = (attributes.containsKey("rec")) ? DateFormatter.parseShortSecond(attributes.get("rec"), DateFormatter.UTCDiffString()) : new Date(); - this.created = (attributes.containsKey("cre")) ? DateFormatter.parseShortSecond(attributes.get("cre"), DateFormatter.UTCDiffString()) : new Date(); + this.received = (attributes.containsKey("rec")) ? GenericFormatter.SHORT_SECOND_FORMATTER.parse(attributes.get("rec"), GenericFormatter.UTCDiffString()) : new Date(); + this.created = (attributes.containsKey("cre")) ? GenericFormatter.SHORT_SECOND_FORMATTER.parse(attributes.get("cre"), GenericFormatter.UTCDiffString()) : new Date(); this.distributed = (attributes.containsKey("dis")) ? Integer.parseInt(attributes.get("dis")) : 0; this.originator = (attributes.containsKey("ori")) ? attributes.get("ori") : ""; removeStandards(); @@ -289,10 +289,10 @@ public class yacyNewsDB { if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")"); this.attributes = attributes; this.received = received; - this.created = DateFormatter.parseShortSecond(id.substring(0, DateFormatter.PATTERN_SHORT_SECOND.length()), DateFormatter.UTCDiffString()); + this.created = GenericFormatter.SHORT_SECOND_FORMATTER.parse(id.substring(0, GenericFormatter.PATTERN_SHORT_SECOND.length()), GenericFormatter.UTCDiffString()); this.category = category; this.distributed = distributed; - this.originator = id.substring(DateFormatter.PATTERN_SHORT_SECOND.length()); + this.originator = id.substring(GenericFormatter.PATTERN_SHORT_SECOND.length()); removeStandards(); } @@ -309,8 +309,8 @@ public class yacyNewsDB { // attention: this has no additional encoding if (this.originator != null) attributes.put("ori", this.originator); if (this.category != null) attributes.put("cat", this.category); - if (this.created != null) attributes.put("cre", DateFormatter.formatShortSecond(this.created)); - if (this.received != null) attributes.put("rec", DateFormatter.formatShortSecond(this.received)); + if (this.created != null) attributes.put("cre", GenericFormatter.SHORT_SECOND_FORMATTER.format(this.created)); + if (this.received != null) attributes.put("rec", GenericFormatter.SHORT_SECOND_FORMATTER.format(this.received)); attributes.put("dis", Integer.toString(this.distributed)); final String theString = attributes.toString(); removeStandards(); @@ -318,7 +318,7 @@ public class yacyNewsDB { } public String id() { - return DateFormatter.formatShortSecond(created) + originator; + return GenericFormatter.SHORT_SECOND_FORMATTER.format(created) + originator; } public String originator() { diff --git a/source/de/anomic/yacy/yacyNewsQueue.java b/source/de/anomic/yacy/yacyNewsQueue.java index 6e59bdb13..ca2c43058 100644 --- a/source/de/anomic/yacy/yacyNewsQueue.java +++ b/source/de/anomic/yacy/yacyNewsQueue.java @@ -50,13 +50,13 @@ import java.util.Date; import java.util.HashSet; import java.util.Iterator; +import net.yacy.cora.date.GenericFormatter; import net.yacy.kelondro.index.Column; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.table.Table; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; @@ -68,7 +68,7 @@ public class yacyNewsQueue { public static final Row rowdef = new Row(new Column[]{ new Column("newsid", Column.celltype_string, Column.encoder_bytes, yacyNewsDB.idLength, "id = created + originator"), - new Column("last touched", Column.celltype_string, Column.encoder_bytes, DateFormatter.PATTERN_SHORT_SECOND.length(), "") + new Column("last touched", Column.celltype_string, Column.encoder_bytes, GenericFormatter.PATTERN_SHORT_SECOND.length(), "") }, NaturalOrder.naturalOrder ); @@ -173,7 +173,7 @@ public class yacyNewsQueue { } final Row.Entry b = queueStack.row().newEntry(new byte[][]{ r.id().getBytes(), - DateFormatter.formatShortSecond(new Date()).getBytes()}); + GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()).getBytes()}); return b; } diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index 88d23420b..c6eddd8cc 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -29,7 +29,6 @@ import java.util.concurrent.ConcurrentHashMap; import net.yacy.cora.document.RSSMessage; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.MapTools; @@ -107,7 +106,7 @@ public class yacyPeerActions { } if (Math.abs(nowUTC0Time - ctimeUTC0) / 1000 / 60 > 60 * 6 ) { // the new connection is out-of-age, we reject the connection - if (yacyCore.log.isFine()) yacyCore.log.logFine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getPublicAddress() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + DateFormatter.formatInterval(Math.abs(nowUTC0Time - ctimeUTC0))); + if (yacyCore.log.isFine()) yacyCore.log.logFine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getPublicAddress() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + formatInterval(Math.abs(nowUTC0Time - ctimeUTC0))); return false; } @@ -254,4 +253,31 @@ public class yacyPeerActions { return (userAgent == null) ? "" : userAgent; } + /** + * Format a time inteval in milliseconds into a String of the form + * X 'day'['s'] HH':'mm + */ + public static String formatInterval(final long millis) { + try { + final long mins = millis / 60000; + + final StringBuilder uptime = new StringBuilder(); + + final int uptimeDays = (int) (Math.floor(mins/1440.0)); + final int uptimeHours = (int) (Math.floor(mins/60.0)%24); + final int uptimeMins = (int) mins%60; + + uptime.append(uptimeDays) + .append(((uptimeDays == 1)?" day ":" days ")) + .append((uptimeHours < 10)?"0":"") + .append(uptimeHours) + .append(":") + .append((uptimeMins < 10)?"0":"") + .append(uptimeMins); + + return uptime.toString(); + } catch (final Exception e) { + return "unknown"; + } + } } diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index 3bfb36cf6..10d763674 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -57,12 +57,13 @@ import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; +import net.yacy.cora.date.AbstractFormatter; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.protocol.Domains; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.HandleSet; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Digest; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.MapTools; import de.anomic.tools.bitfield; @@ -209,7 +210,7 @@ public class yacySeed implements Cloneable { this.dna.put(yacySeed.IPTYPE, "∅"); // settings that can only be computed by visiting peer - this.dna.put(yacySeed.LASTSEEN, DateFormatter.formatShortSecond(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/))); // for last-seen date + this.dna.put(yacySeed.LASTSEEN, GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/))); // for last-seen date this.dna.put(yacySeed.USPEED, yacySeed.ZERO); // the computated uplink speed of the peer // settings that are needed to organize the seed round-trip @@ -493,7 +494,7 @@ public class yacySeed implements Cloneable { // because java thinks it must apply the UTC offset to the current time, // to create a string that looks like our current time, it adds the local UTC offset to the // time. To create a corrected UTC Date string, we first subtract the local UTC offset. - String ls = DateFormatter.formatShortSecond(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/)); + String ls = GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/)); //System.out.println("SETTING LAST-SEEN of " + this.getName() + " to " + ls); dna.put(yacySeed.LASTSEEN, ls ); } @@ -503,7 +504,7 @@ public class yacySeed implements Cloneable { */ public final long getLastSeenUTC() { try { - final long t = DateFormatter.parseShortSecond(get(yacySeed.LASTSEEN, "20040101000000")).getTime(); + final long t = GenericFormatter.SHORT_SECOND_FORMATTER.parse(get(yacySeed.LASTSEEN, "20040101000000")).getTime(); // getTime creates a UTC time number. But in this case java thinks, that the given // time string is a local time, which has a local UTC offset applied. // Therefore java subtracts the local UTC offset, to get a UTC number. @@ -512,9 +513,9 @@ public class yacySeed implements Cloneable { // offset again. return t /*+ DateFormatter.UTCDiff()*/; } catch (final java.text.ParseException e) { // in case of an error make seed look old!!! - return System.currentTimeMillis() - DateFormatter.dayMillis; + return System.currentTimeMillis() - AbstractFormatter.dayMillis; } catch (final java.lang.NumberFormatException e) { - return System.currentTimeMillis() - DateFormatter.dayMillis; + return System.currentTimeMillis() - AbstractFormatter.dayMillis; } } @@ -530,7 +531,7 @@ public class yacySeed implements Cloneable { /** @return the age of the seed in number of days */ public final int getAge() { try { - final long t = DateFormatter.parseShortSecond(get(yacySeed.BDATE, "20040101000000")).getTime(); + final long t = GenericFormatter.SHORT_SECOND_FORMATTER.parse(get(yacySeed.BDATE, "20040101000000")).getTime(); return (int) ((System.currentTimeMillis() - (t /*- getUTCDiff() + DateFormatter.UTCDiff()*/)) / 1000 / 60 / 60 / 24); } catch (final java.text.ParseException e) { return -1; @@ -740,9 +741,9 @@ public class yacySeed implements Cloneable { // now calculate other information about the host newSeed.dna.put(yacySeed.NAME, (name) == null ? "anonymous" : name); newSeed.dna.put(yacySeed.PORT, Integer.toString((port <= 0) ? 8080 : port)); - newSeed.dna.put(yacySeed.BDATE, DateFormatter.formatShortSecond(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/)) ); + newSeed.dna.put(yacySeed.BDATE, GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/)) ); newSeed.dna.put(yacySeed.LASTSEEN, newSeed.dna.get(yacySeed.BDATE)); // just as initial setting - newSeed.dna.put(yacySeed.UTC, DateFormatter.UTCDiffString()); + newSeed.dna.put(yacySeed.UTC, GenericFormatter.UTCDiffString()); newSeed.dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); return newSeed; diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index 6726cfdf1..641f5f6ea 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -54,8 +54,8 @@ import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.kelondroException; //import de.anomic.http.client.Client; -import de.anomic.http.server.HTTPDemon; import de.anomic.http.server.AlternativeDomainNames; +import de.anomic.http.server.HTTPDemon; //import de.anomic.http.server.ResponseContainer; import de.anomic.search.Switchboard; import de.anomic.server.serverCore; @@ -849,26 +849,6 @@ public final class yacySeedDB implements AlternativeDomainNames { reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache"); // httpc uses HTTP/1.0 is this necessary? reqHeader.put(HeaderFramework.USER_AGENT, MultiProtocolURI.yacybotUserAgent); - // init http-client -// final Client client = new Client(10000, reqHeader); -// byte[] content = null; -// ResponseContainer res = null; -// try { -// // send request -// res = client.GET(seedURL.toString()); -// -// // check response code -// if (res.getStatusCode() != 200) { -// throw new IOException("Server returned status: " + res.getStatusLine()); -// } -// -// // read byte array -// content = res.getData(); -// } finally { -// if(res != null) { -// res.closeStream(); -// } -// } final HTTPClient client = new HTTPClient(); client.setHeader(reqHeader.entrySet()); byte[] content = null; diff --git a/source/net/yacy/cora/date/AbstractFormatter.java b/source/net/yacy/cora/date/AbstractFormatter.java new file mode 100644 index 000000000..c5805ebda --- /dev/null +++ b/source/net/yacy/cora/date/AbstractFormatter.java @@ -0,0 +1,45 @@ +/** + * AbstractFormatter + * Copyright 2011 by Michael Peter Christen + * First released 2.1.2011 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.cora.date; + +import java.text.ParseException; +import java.util.Date; +import java.util.TimeZone; + +public abstract class AbstractFormatter implements DateFormatter { + + protected static final TimeZone TZ_GMT = TimeZone.getTimeZone("GMT"); + + // statics + public final static long secondMillis = 1000; + public final static long minuteMillis = 60 * secondMillis; + public final static long hourMillis = 60 * minuteMillis; + public final static long dayMillis = 24 * hourMillis; + public final static long normalyearMillis = 365 * dayMillis; + public final static long leapyearMillis = 366 * dayMillis; + + protected long last_time; + protected String last_format; + + public abstract Date parse(String s) throws ParseException; + public abstract String format(final Date date); + +} diff --git a/source/net/yacy/cora/date/DateFormatter.java b/source/net/yacy/cora/date/DateFormatter.java new file mode 100644 index 000000000..e61ee81cc --- /dev/null +++ b/source/net/yacy/cora/date/DateFormatter.java @@ -0,0 +1,31 @@ +/** + * DateFormatter + * Copyright 2011 by Michael Peter Christen + * First released 2.1.2011 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.cora.date; + +import java.text.ParseException; +import java.util.Date; + +public interface DateFormatter { + + public Date parse(String s) throws ParseException; + public String format(final Date date); + +} diff --git a/source/net/yacy/cora/date/GenericFormatter.java b/source/net/yacy/cora/date/GenericFormatter.java new file mode 100644 index 000000000..183214c53 --- /dev/null +++ b/source/net/yacy/cora/date/GenericFormatter.java @@ -0,0 +1,168 @@ +/** + * GenericFormatter + * Copyright 2011 by Michael Peter Christen + * First released 2.1.2011 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.cora.date; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Locale; + +public class GenericFormatter extends AbstractFormatter implements DateFormatter { + + public static final String PATTERN_SHORT_DAY = "yyyyMMdd"; + public static final String PATTERN_SHORT_SECOND = "yyyyMMddHHmmss"; + public static final String PATTERN_SHORT_MILSEC = "yyyyMMddHHmmssSSS"; + private static final String PATTERN_RFC1123_SHORT = "EEE, dd MMM yyyy"; + private static final String PATTERN_ANSIC = "EEE MMM d HH:mm:ss yyyy"; + + private static final SimpleDateFormat FORMAT_SHORT_DAY = new SimpleDateFormat(PATTERN_SHORT_DAY, Locale.US); + private static final SimpleDateFormat FORMAT_SHORT_SECOND = new SimpleDateFormat(PATTERN_SHORT_SECOND, Locale.US); + private static final SimpleDateFormat FORMAT_SHORT_MILSEC = new SimpleDateFormat(PATTERN_SHORT_MILSEC, Locale.US); + private static final SimpleDateFormat FORMAT_ANSIC = new SimpleDateFormat(PATTERN_ANSIC, Locale.US); + private static final SimpleDateFormat FORMAT_RFC1123_SHORT = new SimpleDateFormat(PATTERN_RFC1123_SHORT, Locale.US); + + // find out time zone and DST offset + private static Calendar thisCalendar = Calendar.getInstance(); + + static { + // we want GMT times on the formats as well as they don't support any timezone + FORMAT_SHORT_DAY.setTimeZone(TZ_GMT); + FORMAT_SHORT_SECOND.setTimeZone(TZ_GMT); + FORMAT_SHORT_MILSEC.setTimeZone(TZ_GMT); + } + + public static final GenericFormatter SHORT_DAY_FORMATTER = new GenericFormatter(FORMAT_SHORT_DAY); + public static final GenericFormatter SHORT_SECOND_FORMATTER = new GenericFormatter(FORMAT_SHORT_SECOND); + public static final GenericFormatter SHORT_MILSEC_FORMATTER = new GenericFormatter(FORMAT_SHORT_MILSEC); + public static final GenericFormatter ANSIC_FORMATTER = new GenericFormatter(FORMAT_ANSIC); + public static final GenericFormatter RFC1123_SHORT_FORMATTER = new GenericFormatter(FORMAT_RFC1123_SHORT); + + private SimpleDateFormat dateFormat; + + public GenericFormatter(SimpleDateFormat dateFormat) { + this.dateFormat = dateFormat; + this.last_time = 0; + this.last_format = ""; + } + + /** + * Note: The short day format doesn't include any timezone information. This method + * transforms the date into the GMT/UTC timezone. Example: If the local system time is, + * 2007-12-18 01:15:00 +0200, then the resulting String will be "2007-12-17". + * In case you need a format with a timezon offset, use {@link #formatShortDay(TimeZone)} + * @return a String representation of the current system date in GMT using the + * short day format, e.g. "20071218". + */ + @Override + public String format(final Date date) { + if (date == null) return ""; + if (Math.abs(date.getTime() - last_time) < 1000) return last_format; + synchronized (this.dateFormat) { + last_format = this.dateFormat.format(date); + last_time = date.getTime(); + } + return last_format; + } + + /** + * Parse a String representation of a Date in short day format assuming the date + * is aligned to the GMT/UTC timezone. An example for such a date string is "20071218". + * @see #formatShortDay() + * @throws ParseException The exception is thrown if an error occured during while parsing + * the String. + */ + @Override + public Date parse(final String timeString) throws ParseException { + synchronized (this.dateFormat) { + return this.dateFormat.parse(timeString); + } + } + + /** + * Like {@link #parseShortSecond(String)} using additional timezone information provided in an + * offset String, like "+0100" for CET. + */ + public Date parse(final String timeString, final String UTCOffset) { + // FIXME: This method returns an incorrect date, check callers! + // ex: de.anomic.server.serverDate.parseShortSecond("20070101120000", "+0200").toGMTString() + // => 1 Jan 2007 13:00:00 GMT + if (timeString == null || timeString.length() == 0) { return new Date(); } + if (UTCOffset == null || UTCOffset.length() == 0) { return new Date(); } + try { + return new Date(this.dateFormat.parse(timeString).getTime() - UTCDiff() + UTCDiff(UTCOffset)); + } catch (final java.text.ParseException e) { + //serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]"); + return new Date(); + } catch (final java.lang.NumberFormatException e) { + //serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]"); + return new Date(); + } + } + + private static long UTCDiff(final String diffString) { + if (diffString.length() != 5) throw new IllegalArgumentException("UTC String malformed (wrong size):" + diffString); + boolean ahead = true; + if (diffString.length() > 0 && diffString.charAt(0) == '+') ahead = true; + else if (diffString.length() > 0 && diffString.charAt(0) == '-') ahead = false; + else throw new IllegalArgumentException("UTC String malformed (wrong sign):" + diffString); + final long oh = Long.parseLong(diffString.substring(1, 3)); + final long om = Long.parseLong(diffString.substring(3)); + return ((ahead) ? (long) 1 : (long) -1) * (oh * AbstractFormatter.hourMillis + om * AbstractFormatter.minuteMillis); + } + + private static long UTCDiff() { + // DST_OFFSET is dependent on the time of the Calendar, so it has to be updated + // to get the correct current offset + synchronized (thisCalendar) { + thisCalendar.setTimeInMillis(System.currentTimeMillis()); + final long zoneOffsetHours = thisCalendar.get(Calendar.ZONE_OFFSET); + final long DSTOffsetHours = thisCalendar.get(Calendar.DST_OFFSET); + return zoneOffsetHours + DSTOffsetHours; + } + } + + public static String UTCDiffString() { + // we express the UTC Difference in 5 digits: + // SHHMM + // S ::= '+'|'-' + // HH ::= '00'|'01'|'02'|'03'|'04'|'05'|'06'|'07'|'08'|'09'|'10'|'11'|'12' + // MM ::= '00'|'15'|'30'|'45' + // since there are some places on earth where there is a time shift of half an hour + // we need too show also the minutes of the time shift + // Examples: http://www.timeanddate.com/library/abbreviations/timezones/ + final long offsetHours = UTCDiff(); + final int om = Math.abs((int) (offsetHours / AbstractFormatter.minuteMillis)) % 60; + final int oh = Math.abs((int) (offsetHours / AbstractFormatter.hourMillis)); + String diff = Integer.toString(om); + if (diff.length() < 2) diff = "0" + diff; + diff = Integer.toString(oh) + diff; + if (diff.length() < 4) diff = "0" + diff; + if (offsetHours < 0) { + return "-" + diff; + } + return "+" + diff; + } + + public static long correctedUTCTime() { + return System.currentTimeMillis() - UTCDiff(); + } +} diff --git a/source/net/yacy/cora/date/ISO8601Formatter.java b/source/net/yacy/cora/date/ISO8601Formatter.java new file mode 100644 index 000000000..4feea85ad --- /dev/null +++ b/source/net/yacy/cora/date/ISO8601Formatter.java @@ -0,0 +1,191 @@ +/** + * ISO8601 + * Copyright 2011 by Michael Peter Christen + * First released 2.1.2011 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.cora.date; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Locale; +import java.util.NoSuchElementException; +import java.util.StringTokenizer; + +public class ISO8601Formatter extends AbstractFormatter implements DateFormatter { + + /** pattern for a W3C datetime variant of a non-localized ISO8601 date */ + private static final String PATTERN_ISO8601 = "yyyy-MM-dd'T'HH:mm:ss'Z'"; + + /** Date formatter/non-sloppy parser for W3C datetime (ISO8601) in GMT/UTC */ + private static final SimpleDateFormat FORMAT_ISO8601 = new SimpleDateFormat(PATTERN_ISO8601, Locale.US); + + static { + FORMAT_ISO8601.setTimeZone(TZ_GMT); + } + + public static final ISO8601Formatter FORMATTER = new ISO8601Formatter(); + + public ISO8601Formatter() { + last_time = 0; + last_format = ""; + } + + /** + * Parse dates as defined in {@linkplain http://www.w3.org/TR/NOTE-datetime}. + * This format (also specified in ISO8601) allows different "precisions". + * The following lower precision versions for the complete date + * "2007-12-19T10:20:30.567+0300" are allowed:<br> + * "2007"<br> + * "2007-12"<br> + * "2007-12-19"<br> + * "2007-12-19T10:20+0300<br> + * "2007-12-19T10:20:30+0300<br> + * "2007-12-19T10:20:30.567+0300<br> + * Additionally a timezone offset of "+0000" can be substituted as "Z".<br> + * Parsing is done in a fuzzy way. If there is an illegal character somewhere in + * the String, the date parsed so far will be returned, e.g. the input + * "2007-12-19FOO" would return a date that represents "2007-12-19". + * + * @param s + * @return + * @throws ParseException + */ + @Override + public Date parse(String s) throws ParseException { + // do some lazy checks here + s = s.trim(); + while (s.length() > 0 && s.endsWith("?")) s = s.substring(0, s.length() - 1); // sometimes used if write is not sure about date + if (s.startsWith("{")) s = s.substring(1); + if (s.endsWith("}")) s = s.substring(0, s.length() - 1); + if (s.startsWith("[")) s = s.substring(1); + if (s.endsWith("]")) s = s.substring(0, s.length() - 1); + while (s.length() > 0 && (s.charAt(0) > '9' || s.charAt(0) < '0')) s = s.substring(1); + if (s.endsWith("--")) s = s.substring(0, s.length() - 2) + "00"; + int p = s.indexOf(';'); if (p >= 0) s = s.substring(0, p); // a semicolon may be used to separate two dates from each other; then we take the first + p = s.indexOf(','); if (p >= 0) s = s.substring(0, p); // a comma may be used to separate two dates from each other; then we take the first + while (s.length() > 0 && s.endsWith("?")) s = s.substring(0, s.length() - 1); // sometimes used if write is not sure about date + + // no go for exact parsing + final Calendar cal = Calendar.getInstance(TZ_GMT, Locale.US); + cal.clear(); + + // split 2007-12-19T10:20:30.789+0500 into its parts + // correct: yyyy['-'MM['-'dd['T'HH':'MM[':'ss['.'SSS]]('Z'|ZZZZZ)]]] + final StringTokenizer t = new StringTokenizer(s, "-T:.Z+", true); + if (s == null || t.countTokens() == 0) + throw new ParseException("parseISO8601: Cannot parse '" + s + "'", 0); + + try { + // year + cal.set(Calendar.YEAR, Integer.parseInt(t.nextToken())); + // month + if (t.nextToken().equals("-")) { + cal.set(Calendar.MONTH, Integer.parseInt(t.nextToken()) - 1); + } else { + return cal.getTime(); + } + // day + if (t.nextToken().equals("-")) { + cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(t.nextToken())); + } else { + return cal.getTime(); + } + // The standard says: + // if there is an hour there has to be a minute and a timezone token, too. + if (t.nextToken().equals("T")) { + final int hour = Integer.parseInt(t.nextToken()); + // no error, got hours + int min = 0; + int sec = 0; + int msec = 0; + if (t.nextToken().equals(":")) { + min = Integer.parseInt(t.nextToken()); + // no error, got minutes + // need TZ or seconds + String token = t.nextToken(); + if (token.equals(":")) { + sec = Integer.parseInt(t.nextToken()); + // need millisecs or TZ + token = t.nextToken(); + if (token.equals(".")) { + msec = Integer.parseInt(t.nextToken()); + // need TZ + token = t.nextToken(); + } + } + + // check for TZ data + int offset; + if (token.equals("Z")) { + offset = 0; + } else { + int sign = 0; + if (token.equals("+")) { + sign = 1; + } else if (token.equals("-")) { + sign = -1; + } else { + // no legal TZ offset found + return cal.getTime(); + } + offset = sign * Integer.parseInt(t.nextToken()) * 10 * 3600; + } + cal.set(Calendar.ZONE_OFFSET, offset); + } + cal.set(Calendar.HOUR_OF_DAY, hour); + cal.set(Calendar.MINUTE, min); + cal.set(Calendar.SECOND, sec); + cal.set(Calendar.MILLISECOND, msec); + } + } catch (final NoSuchElementException e) { + // ignore this as it is perfectly fine to have non-complete date in this format + } catch (final Exception e) { + // catch all Exceptions and return what we parsed so far + //serverLog.logInfo("SERVER", "parseISO8601: DATE ERROR with: '" + s + "' got so far: '" + cal.toString()); + } + + // in case we couldn't even parse a year + if (!cal.isSet(Calendar.YEAR)) + throw new ParseException("parseISO8601: Cannot parse '" + s + "'", 0); + Date d = cal.getTime(); + return d; + } + + + /** + * Creates a String representation of a Date using the format defined + * in ISO8601/W3C datetime + * The result will be in UTC/GMT, e.g. "2007-12-19T10:20:30Z". + * + * @param date The Date instance to transform. + * @return A fixed width (20 chars) ISO8601 date String. + */ + @Override + public final String format(final Date date) { + if (date == null) return ""; + if (Math.abs(date.getTime() - last_time) < 1000) return last_format; + synchronized (FORMAT_ISO8601) { + last_format = FORMAT_ISO8601.format(date); + last_time = date.getTime(); + } + return last_format; + } + +} diff --git a/source/net/yacy/cora/document/MultiProtocolURI.java b/source/net/yacy/cora/document/MultiProtocolURI.java index bf0348e21..e0b6a4974 100644 --- a/source/net/yacy/cora/document/MultiProtocolURI.java +++ b/source/net/yacy/cora/document/MultiProtocolURI.java @@ -54,6 +54,7 @@ import net.yacy.cora.protocol.http.HTTPClient; */ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolURI> { + public static final MultiProtocolURI POISON = new MultiProtocolURI(); // poison pill for concurrent link generators private static final long serialVersionUID = -1173233022912141884L; private static final long SMB_TIMEOUT = 5000; diff --git a/source/net/yacy/cora/document/RSSFeed.java b/source/net/yacy/cora/document/RSSFeed.java index bc5cead92..9a9017969 100644 --- a/source/net/yacy/cora/document/RSSFeed.java +++ b/source/net/yacy/cora/document/RSSFeed.java @@ -20,11 +20,14 @@ package net.yacy.cora.document; +import java.net.MalformedURLException; import java.util.Collections; import java.util.ConcurrentModificationException; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; public class RSSFeed implements Iterable<RSSMessage> { @@ -58,6 +61,14 @@ public class RSSFeed implements Iterable<RSSMessage> { return this.imageURL; } + public Set<MultiProtocolURI> getLinks() { + Set<MultiProtocolURI> links = new HashSet<MultiProtocolURI>(); + for (RSSMessage message: messages.values()) { + try {links.add(new MultiProtocolURI(message.getLink()));} catch (MalformedURLException e) {} + } + return links; + } + public void addMessage(final RSSMessage item) { final String guid = item.getGuid(); messages.put(guid, item); diff --git a/source/net/yacy/cora/document/RSSMessage.java b/source/net/yacy/cora/document/RSSMessage.java index 659ca99eb..df751ea1b 100644 --- a/source/net/yacy/cora/document/RSSMessage.java +++ b/source/net/yacy/cora/document/RSSMessage.java @@ -27,9 +27,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; - +import net.yacy.cora.date.GenericFormatter; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.protocol.HeaderFramework; -import net.yacy.kelondro.util.DateFormatter; public class RSSMessage implements Hit { @@ -89,7 +89,7 @@ public class RSSMessage implements Hit { map.put("title", title); map.put("description", description); map.put("link", link); - map.put("pubDate", DateFormatter.formatShortSecond(new Date())); + map.put("pubDate", ISO8601Formatter.FORMATTER.format(new Date())); map.put("guid", artificialGuidPrefix + Integer.toHexString((title + description + link).hashCode())); } @@ -150,10 +150,10 @@ public class RSSMessage implements Hit { String dateString = Token.pubDate.valueFrom(this.map); Date date; try { - date = DateFormatter.parseShortSecond(dateString); + date = ISO8601Formatter.FORMATTER.parse(dateString); } catch (ParseException e) { try { - date = DateFormatter.parseISO8601(dateString); + date = GenericFormatter.SHORT_SECOND_FORMATTER.parse(dateString); } catch (ParseException e1) { date = HeaderFramework.parseHTTPDate(dateString); } @@ -228,7 +228,7 @@ public class RSSMessage implements Hit { } public void setPubDate(Date pubdate) { - setValue("pubDate", DateFormatter.formatISO8601(new Date())); + setValue("pubDate", ISO8601Formatter.FORMATTER.format(pubdate)); } public void setReferrer(String referrer) { diff --git a/source/net/yacy/cora/document/RSSReader.java b/source/net/yacy/cora/document/RSSReader.java index e4555cda4..b1ead3459 100644 --- a/source/net/yacy/cora/document/RSSReader.java +++ b/source/net/yacy/cora/document/RSSReader.java @@ -54,19 +54,6 @@ public class RSSReader extends DefaultHandler { type = Type.none; } - public RSSReader(int maxsize, final String path) throws IOException { - this(maxsize); - final SAXParserFactory factory = SAXParserFactory.newInstance(); - try { - final SAXParser saxParser = factory.newSAXParser(); - saxParser.parse(path, this); - } catch (SAXException e) { - throw new IOException (e.getMessage()); - } catch (ParserConfigurationException e) { - throw new IOException (e.getMessage()); - } - } - public RSSReader(int maxsize, final InputStream stream, Type type) throws IOException { this(maxsize); this.type = type; diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index 3ad6198e0..ff82561fb 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -38,7 +38,6 @@ import java.util.regex.Pattern; import net.yacy.cora.storage.ARC; import net.yacy.cora.storage.ConcurrentARC; -import net.yacy.kelondro.logging.Log; public class Domains { @@ -603,7 +602,6 @@ public class Domains { } } } catch (SocketException e) { - Log.logException(e); } // now look up the host name @@ -617,7 +615,6 @@ public class Domains { InetAddress[] moreAddresses = InetAddress.getAllByName(localHostName); if (moreAddresses != null) for (InetAddress a: moreAddresses) localHostAddresses.add(a); } catch (UnknownHostException e) { - Log.logException(e); } // fill a cache of local host names diff --git a/source/net/yacy/cora/protocol/HeaderFramework.java b/source/net/yacy/cora/protocol/HeaderFramework.java index 266aa5979..c5973d1f7 100644 --- a/source/net/yacy/cora/protocol/HeaderFramework.java +++ b/source/net/yacy/cora/protocol/HeaderFramework.java @@ -39,8 +39,6 @@ import java.util.TreeMap; import java.util.Vector; import java.util.concurrent.ConcurrentHashMap; -import de.anomic.http.server.HTTPDemon; - import net.yacy.cora.document.MultiProtocolURI; @@ -564,23 +562,6 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri return url; } - public static void handleTransparentProxySupport(final RequestHeader header, final Properties prop, final String virtualHost, final boolean isTransparentProxy) { - // transparent proxy support is only available for http 1.0 and above connections - if (prop.getProperty(CONNECTION_PROP_HTTP_VER, "HTTP/0.9").equals("HTTP/0.9")) return; - - // if the transparent proxy support was disabled, we have nothing todo here ... - if (!(isTransparentProxy && header.containsKey(HOST))) return; - - // we only need to do the transparent proxy support if the request URL didn't contain the hostname - // and therefor was set to virtualHost by function parseQuery() - if (!prop.getProperty(CONNECTION_PROP_HOST).equals(virtualHost)) return; - - // TODO: we could have problems with connections from extern here ... - final String dstHostSocket = header.get(HeaderFramework.HOST); - prop.setProperty(CONNECTION_PROP_HOST,(HTTPDemon.isThisHostName(dstHostSocket)?virtualHost:dstHostSocket)); - } - - /** * Reading http headers from a reader class and building up a httpHeader object * @param reader the {@link BufferedReader} that is used to read the http header lines diff --git a/source/net/yacy/cora/protocol/RequestHeader.java b/source/net/yacy/cora/protocol/RequestHeader.java index 3de365164..1846c29f6 100755 --- a/source/net/yacy/cora/protocol/RequestHeader.java +++ b/source/net/yacy/cora/protocol/RequestHeader.java @@ -23,14 +23,9 @@ package net.yacy.cora.protocol; import java.net.MalformedURLException; import java.util.Date; import java.util.Map; -import java.util.Properties; -import java.util.regex.Pattern; - -import de.anomic.http.server.HTTPDemon; import net.yacy.cora.document.MultiProtocolURI; - public class RequestHeader extends HeaderFramework { // request header properties @@ -59,16 +54,6 @@ public class RequestHeader extends HeaderFramework { private static final long serialVersionUID = 0L; - private static final Pattern P_20 = Pattern.compile(" ", Pattern.LITERAL); - private static final Pattern P_7B = Pattern.compile("{", Pattern.LITERAL); - private static final Pattern P_7D = Pattern.compile("}", Pattern.LITERAL); - private static final Pattern P_7C = Pattern.compile("|", Pattern.LITERAL); - private static final Pattern P_5C = Pattern.compile("\\", Pattern.LITERAL); - private static final Pattern P_5E = Pattern.compile("^", Pattern.LITERAL); - private static final Pattern P_5B = Pattern.compile("[", Pattern.LITERAL); - private static final Pattern P_5D = Pattern.compile("]", Pattern.LITERAL); - private static final Pattern P_60 = Pattern.compile("`", Pattern.LITERAL); - public RequestHeader() { super(); } @@ -120,116 +105,4 @@ public class RequestHeader extends HeaderFramework { return ((containsKey(ACCEPT_ENCODING)) && ((get(ACCEPT_ENCODING)).toUpperCase().indexOf("GZIP")) != -1); } - - public static Properties parseRequestLine(final String cmd, String args, final String virtualHost) { - - final Properties prop = new Properties(); - - // storing informations about the request - prop.setProperty(CONNECTION_PROP_METHOD, cmd); - prop.setProperty(CONNECTION_PROP_REQUESTLINE, cmd + " " + args); - - // this parses a whole URL - if (args.length() == 0) { - prop.setProperty(CONNECTION_PROP_HOST, virtualHost); - prop.setProperty(CONNECTION_PROP_PATH, "/"); - prop.setProperty(CONNECTION_PROP_HTTP_VER, HTTP_VERSION_0_9); - prop.setProperty(CONNECTION_PROP_EXT, ""); - return prop; - } - - // store the version propery "HTTP" and cut the query at both ends - int sep = args.lastIndexOf(' '); - if ((sep >= 0)&&(args.substring(sep + 1).toLowerCase().startsWith("http/"))) { - // HTTP version is given - prop.setProperty(CONNECTION_PROP_HTTP_VER, args.substring(sep + 1).trim()); - args = args.substring(0, sep).trim(); // cut off HTTP version mark - } else { - // HTTP version is not given, it will be treated as ver 0.9 - prop.setProperty(CONNECTION_PROP_HTTP_VER, HTTP_VERSION_0_9); - } - - // replacing spaces in the url string correctly - args = P_20.matcher(args).replaceAll("%20"); - // replace unwise characters (see RFC 2396, 2.4.3), which may not be escaped - args = P_7B.matcher(args).replaceAll("%7B"); - args = P_7D.matcher(args).replaceAll("%7D"); - args = P_7C.matcher(args).replaceAll("%7C"); - args = P_5C.matcher(args).replaceAll("%5C"); - args = P_5E.matcher(args).replaceAll("%5E"); - args = P_5B.matcher(args).replaceAll("%5B"); - args = P_5D.matcher(args).replaceAll("%5D"); - args = P_60.matcher(args).replaceAll("%60"); - - // properties of the query are stored with the prefix "&" - // additionally, the values URL and ARGC are computed - - String argsString = ""; - sep = args.indexOf('?'); - if (sep >= 0) { - // there are values attached to the query string - argsString = args.substring(sep + 1); // cut head from tail of query - args = args.substring(0, sep); - } - prop.setProperty(CONNECTION_PROP_URL, args); // store URL - //System.out.println("HTTPD: ARGS=" + argsString); - if (argsString.length() != 0) prop.setProperty(CONNECTION_PROP_ARGS, argsString); // store arguments in original form - - // finally find host string - String path; - if (args.toUpperCase().startsWith("HTTP://")) { - // a host was given. extract it and set path - args = args.substring(7); - sep = args.indexOf('/'); - if (sep < 0) { - // this is a malformed url, something like - // http://index.html - // we are lazy and guess that it means - // /index.html - // which is a localhost access to the file servlet - prop.setProperty(CONNECTION_PROP_HOST, args); - path = "/"; - } else { - // THIS IS THE "GOOD" CASE - // a perfect formulated url - final String dstHostSocket = args.substring(0, sep); - prop.setProperty(CONNECTION_PROP_HOST, (HTTPDemon.isThisHostName(dstHostSocket)?virtualHost:dstHostSocket)); - path = args.substring(sep); // yes, including beginning "/" - } - } else { - // no host in url. set path - if (args.length() > 0 && args.charAt(0) == '/') { - // thats also fine, its a perfect localhost access - // in this case, we simulate a - // http://localhost/s - // access by setting a virtual host - prop.setProperty(CONNECTION_PROP_HOST, virtualHost); - path = args; - } else { - // the client 'forgot' to set a leading '/' - // this is the same case as above, with some lazyness - prop.setProperty(CONNECTION_PROP_HOST, virtualHost); - path = "/" + args; - } - } - prop.setProperty(CONNECTION_PROP_PATH, path); - - // find out file extension (we already stripped ?-parameters from args) - String ext = ""; // default when no file extension - sep = path.lastIndexOf('.'); - if (sep >= 0) { - final int ancpos = path.indexOf("#", sep + 1); - if (ancpos >= sep) { - // ex: /foo/bar.html#xy => html - ext = path.substring(sep + 1, ancpos).toLowerCase(); - } else { - // ex: /foo/bar.php => php - ext = path.substring(sep + 1).toLowerCase(); - } - } - prop.setProperty(CONNECTION_PROP_EXT, ext); - - return prop; - } - } diff --git a/source/net/yacy/cora/protocol/Scanner.java b/source/net/yacy/cora/protocol/Scanner.java index 5c17e0da0..2291b1bf2 100644 --- a/source/net/yacy/cora/protocol/Scanner.java +++ b/source/net/yacy/cora/protocol/Scanner.java @@ -32,21 +32,14 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.LinkedBlockingQueue; -import de.anomic.data.WorkTables; -import de.anomic.search.Switchboard; - import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.ftp.FTPClient; import net.yacy.cora.protocol.http.HTTPClient; -import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.order.Base64Order; /** * a protocol scanner @@ -208,7 +201,6 @@ public class Scanner extends Thread { runner.start(); } } catch (InterruptedException e) { - Log.logException(e); } } @@ -302,7 +294,6 @@ public class Scanner extends Thread { try { this.scanqueue.put(new Service(protocol, i)); } catch (InterruptedException e) { - Log.logException(e); } } } @@ -336,22 +327,6 @@ public class Scanner extends Thread { return null; } - public static Map<byte[], String> commentCache(Switchboard sb) { - Map<byte[], String> comments = new TreeMap<byte[], String>(Base64Order.enhancedCoder); - Iterator<Tables.Row> i; - try { - i = sb.tables.iterator(WorkTables.TABLE_API_NAME); - Tables.Row row; - while (i.hasNext()) { - row = i.next(); - comments.put(row.getPK(), new String(row.get(WorkTables.TABLE_API_COL_COMMENT))); - } - } catch (IOException e) { - Log.logException(e); - } - return comments; - } - public static void main(String[] args) { //try {System.out.println("192.168.1.91: " + ping(new MultiProtocolURI("smb://192.168.1.91/"), 1000));} catch (MalformedURLException e) {} Scanner scanner = new Scanner(100, 10); diff --git a/source/net/yacy/cora/protocol/TimeoutRequest.java b/source/net/yacy/cora/protocol/TimeoutRequest.java index 11e87eb36..15489892a 100644 --- a/source/net/yacy/cora/protocol/TimeoutRequest.java +++ b/source/net/yacy/cora/protocol/TimeoutRequest.java @@ -37,8 +37,6 @@ import java.util.concurrent.TimeoutException; import jcifs.smb.SmbException; import jcifs.smb.SmbFile; -import net.yacy.kelondro.logging.Log; - /** * TimeoutRequest is a class that can apply a timeout on method calls that may block * for undefined time. Some network operations can only be accessed without a given @@ -316,7 +314,7 @@ public class TimeoutRequest<E> { public String[] call() { try { return file.list(); } catch (SmbException e) { - Log.logWarning("TimeoutRequest:list", file.toString() + " - no list", e); + //Log.logWarning("TimeoutRequest:list", file.toString() + " - no list", e); return null; } } }).call(timeout); diff --git a/source/net/yacy/cora/protocol/http/LinkExtractor.java b/source/net/yacy/cora/protocol/http/LinkExtractor.java new file mode 100644 index 000000000..1ba832d1f --- /dev/null +++ b/source/net/yacy/cora/protocol/http/LinkExtractor.java @@ -0,0 +1,74 @@ +/** + * LinkExtractor + * Copyright 2011 by Michael Peter Christen + * First released 2.01.2011 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.cora.protocol.http; + +import java.net.MalformedURLException; +import java.util.WeakHashMap; +import java.util.regex.Pattern; + +import net.yacy.cora.document.MultiProtocolURI; + +public class LinkExtractor { + + private static final char lb = '<', rb = '>', dquotes = '"', space = ' '; + private static final Object PRESENT = new Object(); + + private WeakHashMap<MultiProtocolURI, Object> links; + private Pattern blackpattern; + + public LinkExtractor(Pattern blackpattern) { + this.links = new WeakHashMap<MultiProtocolURI, Object>(); + this.blackpattern = blackpattern; + } + + public void scrape(String text) { + text = text.replace(lb, space).replace(rb, space).replace(dquotes, space); + int p, q, s = 0; + String u; + while (s < text.length()) { + p = Math.min(find(text, "smb://", s), Math.min(find(text, "ftp://", s), Math.min(find(text, "http://", s), find(text, "https://", s)))); + if (p == Integer.MAX_VALUE) break; + q = text.indexOf(" ", p + 1); + u = text.substring(p, q < 0 ? text.length() : q); + if (u.endsWith(".")) u = u.substring(0, u.length() - 1); // remove the '.' that was appended above + s = p + 1; + if (this.blackpattern.matcher(u).matches()) continue; + try {links.put(new MultiProtocolURI(u), PRESENT);} catch (MalformedURLException e) {} + } + } + + /** + * return the links in the text in the order as they appear + * @return a list of urls + */ + public MultiProtocolURI[] getLinks() { + MultiProtocolURI[] urls = new MultiProtocolURI[this.links.size()]; + int i = 0; + for (MultiProtocolURI uri: this.links.keySet()) urls[i++] = uri; + return urls; + } + + private static final int find(final String s, final String m, final int start) { + final int p = s.indexOf(m, start); + return (p < 0) ? Integer.MAX_VALUE : p; + } + +} diff --git a/source/net/yacy/cora/protocol/http/ProxySettings.java b/source/net/yacy/cora/protocol/http/ProxySettings.java index 493ec95cb..eb5f1b66f 100644 --- a/source/net/yacy/cora/protocol/http/ProxySettings.java +++ b/source/net/yacy/cora/protocol/http/ProxySettings.java @@ -23,8 +23,6 @@ package net.yacy.cora.protocol.http; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -//import org.apache.commons.httpclient.HostConfiguration; -//import org.apache.commons.httpclient.HttpClient; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; diff --git a/source/net/yacy/cora/services/Search.java b/source/net/yacy/cora/services/Search.java index 354fe3840..83291d343 100644 --- a/source/net/yacy/cora/services/Search.java +++ b/source/net/yacy/cora/services/Search.java @@ -18,85 +18,132 @@ * If not, see <http://www.gnu.org/licenses/>. */ - package net.yacy.cora.services; import java.io.IOException; import java.net.MalformedURLException; -//import java.nio.charset.Charset; -//import java.util.ArrayList; +import java.util.ArrayList; +import java.util.Iterator; import java.util.LinkedHashMap; -//import java.util.List; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.RSSFeed; import net.yacy.cora.document.RSSMessage; import net.yacy.cora.document.RSSReader; +import net.yacy.cora.protocol.HeaderFramework; +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.protocol.http.HTTPConnector; +import net.yacy.cora.protocol.http.LinkExtractor; +import net.yacy.cora.storage.ScoreMap; -//import org.apache.commons.httpclient.methods.multipart.Part; -//import org.apache.commons.httpclient.methods.multipart.StringPart; import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.entity.mime.content.StringBody; -public class Search { - - public static BlockingQueue<RSSMessage> search(String rssSearchServiceURL, String query, boolean verify, boolean global, long timeout, int maximumRecords) { - BlockingQueue<RSSMessage> queue = new LinkedBlockingQueue<RSSMessage>(); - searchJob job = new searchJob(rssSearchServiceURL, query, verify, global, timeout, maximumRecords, queue); - job.start(); - return queue; - } - - private final static int recordsPerSession = 10; - - public static class searchJob extends Thread { +public class Search extends Thread { - String urlBase, query; - boolean verify, global; - long timeout; - int startRecord, maximumRecords; - BlockingQueue<RSSMessage> queue; + private final static int recordsPerSession = 10; - public searchJob(String urlBase, String query, boolean verify, boolean global, long timeout, int maximumRecords, BlockingQueue<RSSMessage> queue) { - this.urlBase = urlBase; - this.query = query; - this.verify = verify; - this.global = global; - this.timeout = timeout; - this.startRecord = 0; - this.maximumRecords = maximumRecords; - this.queue = queue; - } + public static final String[] SRURSSServicesList = { + "http://yacy.dyndns.org:8000/yacysearch.rss", + "http://yacy.caloulinux.net:8085/yacysearch.rss", + "http://algire.dyndns.org:8085/yacysearch.rss", + "http://breyvogel.dyndns.org:8002/yacysearch.rss" + }; + + public static final String[] genericServicesList = { + "http://www.scroogle.org/cgi-bin/nbbw.cgi?Gw=$&n=2", + "http://blekko.com/ws/$+/rss", + "http://www.bing.com/search?q=$&format=rss", + "http://search.twitter.com/search.atom?q=$" + }; - public void run() { - RSSMessage message; - mainloop: while (timeout > 0 && maximumRecords > 0) { - long st = System.currentTimeMillis(); - RSSFeed feed; + public static Thread accumulateSRURSS( + final String urlBase, + final String query, + final long timeoutInit, + final int maximumRecordsInit, + final boolean verify, + final boolean global, + final Map<MultiProtocolURI, List<Integer>> result) { + Thread t = new Thread() { + BlockingQueue<RSSMessage> results = new LinkedBlockingQueue<RSSMessage>(); + public void run() { + searchSRURSS(urlBase, query, timeoutInit, maximumRecordsInit, verify, global, results); + int p = 1; + RSSMessage message; try { - feed = search(urlBase, query, verify, global, timeout, startRecord, recordsPerSession); - } catch (IOException e1) { - break mainloop; + while ((message = results.poll(timeoutInit, TimeUnit.MILLISECONDS)) != RSSMessage.POISON) { + MultiProtocolURI uri; + if (message == null) break; + try { + uri = new MultiProtocolURI(message.getLink()); + List<Integer> m = result.get(uri); + if (m == null) m = new ArrayList<Integer>(); + m.add(new Integer(p++)); + result.put(uri, m); + } catch (MalformedURLException e) { + e.printStackTrace(); + } + } + } catch (InterruptedException e) { + e.printStackTrace(); } - if (feed == null || feed.isEmpty()) break mainloop; - maximumRecords -= feed.size(); - innerloop: while (!feed.isEmpty()) { - message = feed.pollMessage(); - if (message == null) break innerloop; + } + }; + t.start(); + return t; + } + + public static Thread searchSRURSS( + final String urlBase, + final String query, + final long timeoutInit, + final int maximumRecordsInit, + final boolean verify, + final boolean global, + final BlockingQueue<RSSMessage> queue) { + Thread job = new Thread() { + public void run() { + int startRecord = 0; + RSSMessage message; + int maximumRecords = maximumRecordsInit; + long timeout = timeoutInit; + mainloop: while (timeout > 0 && maximumRecords > 0) { + long st = System.currentTimeMillis(); + RSSFeed feed; try { - queue.put(message); - } catch (InterruptedException e) { - break innerloop; + feed = loadSRURSS(urlBase, query, timeout, startRecord, recordsPerSession, verify, global); + } catch (IOException e1) { + break mainloop; } + if (feed == null || feed.isEmpty()) break mainloop; + maximumRecords -= feed.size(); + innerloop: while (!feed.isEmpty()) { + message = feed.pollMessage(); + if (message == null) break innerloop; + try { + queue.put(message); + } catch (InterruptedException e) { + break innerloop; + } + } + startRecord += recordsPerSession; + timeout -= System.currentTimeMillis() - st; } - startRecord += recordsPerSession; - timeout -= System.currentTimeMillis() - st; + try { queue.put(RSSMessage.POISON); } catch (InterruptedException e) {} } - try { queue.put(RSSMessage.POISON); } catch (InterruptedException e) {} - } + }; + job.start(); + return job; } /** @@ -110,7 +157,14 @@ public class Search { * @param timeout milliseconds that are waited at maximum for a search result * @return */ - public static RSSFeed search(String rssSearchServiceURL, String query, boolean verify, boolean global, long timeout, int startRecord, int maximumRecords) throws IOException { + public static RSSFeed loadSRURSS( + String rssSearchServiceURL, + String query, + long timeout, + int startRecord, + int maximumRecords, + boolean verify, + boolean global) throws IOException { MultiProtocolURI uri = null; try { uri = new MultiProtocolURI(rssSearchServiceURL); @@ -142,5 +196,118 @@ public class Search { throw new IOException("cora.Search error asking peer '" + uri.getHost() + "':" + e.toString()); } } + + public static Thread accumulateGeneric( + String query, + String service, + final Map<MultiProtocolURI, List<Integer>> result, + final int timeout) { + query = query.replace(' ', '+'); + final String servicePatched = service.replaceAll("\\$", query); + Thread t = new Thread() { + public void run() { + try { + MultiProtocolURI[] sr = loadGeneric(new MultiProtocolURI(servicePatched), timeout); + int p = 1; + for (MultiProtocolURI u: sr) { + List<Integer> m = result.get(u); + if (m == null) m = new ArrayList<Integer>(); + m.add(new Integer(p++)); + result.put(u, m); + } + } catch (MalformedURLException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + }; + t.start(); + return t; + } + + private static MultiProtocolURI[] loadGeneric(MultiProtocolURI uri, long timeout) throws IOException { + final RequestHeader requestHeader = new RequestHeader(); + requestHeader.put(HeaderFramework.USER_AGENT, MultiProtocolURI.yacybotUserAgent); + final HTTPClient client = new HTTPClient(); + client.setTimout((int) timeout); + client.setHeader(requestHeader.entrySet()); + byte[] result = client.GETbytes(uri.toString()); + client.finish(); + if (client.getStatusCode() != 200) { + throw new IOException("Server returned status: " + client.getHttpResponse().getStatusLine()); + } + if (result == null) throw new IOException("cora.Search error asking peer '" + uri.getHost() + "': null"); + LinkExtractor le = new LinkExtractor(Pattern.compile(".*" + uri.getHost() + ".*")); + le.scrape(new String(result)); + MultiProtocolURI[] links = le.getLinks(); + return links; + } + + public static RSSFeed links2feed(Set<MultiProtocolURI> links, String source) { + RSSFeed feed = new RSSFeed(Integer.MAX_VALUE); + String u; + RSSMessage message; + for (MultiProtocolURI uri: links) { + u = uri.toNormalform(true, false); + message = new RSSMessage(u, "", u); + message.setAuthor(source); + feed.addMessage(message); + } + return feed; + } + + private Map<MultiProtocolURI, List<Integer>> result; + private String query; + private int count; + private String[] yacyServices, rssServices, genericServices; + private List<Thread> threads; + + public Search(String query, int count, String[] rssServices, String[] genericServices) { + this.result = new ConcurrentHashMap<MultiProtocolURI, List<Integer>>(); + this.query = query; + this.count = count; + this.yacyServices = yacyServices; + this.rssServices = rssServices; + this.genericServices = genericServices; + this.threads = new ArrayList<Thread>(); + } + + public void run() { + for (String service: this.rssServices) threads.add(accumulateSRURSS(service, this.query, 10000, this.count, false, true, this.result)); + for (String service: this.genericServices) threads.add(accumulateGeneric(this.query, service, this.result, 10000)); + } + + public ScoreMap<MultiProtocolURI> getResults() { + ScoreMap<MultiProtocolURI> scores = new ScoreMap<MultiProtocolURI>(); + int m = this.rssServices.length + this.genericServices.length; + for (Map.Entry<MultiProtocolURI, List<Integer>> entry: this.result.entrySet()) { + int a = 0; + for (Integer i : entry.getValue()) a += i.intValue(); + scores.inc(entry.getKey(), a * m / entry.getValue().size()); + } + return scores; + } + + public void waitTermination() { + for (Thread t: threads) try {t.join();} catch (InterruptedException e) {} + } + public static void main(String[] args) { + StringBuilder sb = new StringBuilder(); + for (String s: args) sb.append(s).append(' '); + String query = sb.toString().trim(); + Search search = new Search(query, 100, SRURSSServicesList, genericServicesList); + search.start(); + try {Thread.sleep(100);} catch (InterruptedException e1) {} + search.waitTermination(); + ScoreMap<MultiProtocolURI> result = search.getResults(); + Iterator<MultiProtocolURI> i = result.keys(true); + MultiProtocolURI u; + while (i.hasNext()) { + u = i.next(); + System.out.println("[" + result.get(u) + "] " + u.toNormalform(true, false)); + } + try {HTTPClient.closeConnectionManager();} catch (InterruptedException e) {} + } } diff --git a/source/net/yacy/cora/storage/IntScore.java b/source/net/yacy/cora/storage/IntScore.java index c2b9ec9b8..72954f596 100644 --- a/source/net/yacy/cora/storage/IntScore.java +++ b/source/net/yacy/cora/storage/IntScore.java @@ -94,4 +94,8 @@ public class IntScore implements Comparable<IntScore>, Comparator<IntScore> { public int compare(IntScore o1, IntScore o2) { return o1.compareTo(o2); } + + public String toString() { + return Integer.toString(this.value); + } } diff --git a/source/net/yacy/cora/storage/OutOfLimitsException.java b/source/net/yacy/cora/storage/OutOfLimitsException.java new file mode 100644 index 000000000..d818972de --- /dev/null +++ b/source/net/yacy/cora/storage/OutOfLimitsException.java @@ -0,0 +1,39 @@ +/** + * OutOfLimitsException + * Copyright 2006 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany + * First released 17.01.2006 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.cora.storage; + +public class OutOfLimitsException extends java.lang.RuntimeException { + + private static final long serialVersionUID = 1L; + + public OutOfLimitsException() { + super("unspecific-error"); + } + + public OutOfLimitsException(final int expectedLimit, final int actualSize) { + super("Object size is " + actualSize + "; it exceeds the size limit " + expectedLimit); + } + + public OutOfLimitsException(final int actualSize) { + super("Object size is " + actualSize + "; must not be negative"); + } + +} diff --git a/source/net/yacy/cora/storage/ScoreCluster.java b/source/net/yacy/cora/storage/ScoreCluster.java index b94689c6f..59a83bbcf 100644 --- a/source/net/yacy/cora/storage/ScoreCluster.java +++ b/source/net/yacy/cora/storage/ScoreCluster.java @@ -30,9 +30,6 @@ import java.util.Random; import java.util.SortedMap; import java.util.TreeMap; -import net.yacy.kelondro.util.kelondroOutOfLimitsException; - - public final class ScoreCluster<E> implements DynamicScore<E> { protected final Map<E, Long> map; // a mapping from a reference to the cluster key @@ -198,7 +195,7 @@ public final class ScoreCluster<E> implements DynamicScore<E> { if (obj == null) return; synchronized (this) { Long usk = map.remove(obj); // get unique score key, old entry is not needed any more - if (newScore < 0) throw new kelondroOutOfLimitsException(newScore); + if (newScore < 0) throw new OutOfLimitsException(newScore); if (usk == null) { // set new value @@ -235,7 +232,7 @@ public final class ScoreCluster<E> implements DynamicScore<E> { if (usk == null) { // set new value - if (incrementScore < 0) throw new kelondroOutOfLimitsException(incrementScore); + if (incrementScore < 0) throw new OutOfLimitsException(incrementScore); usk = Long.valueOf(scoreKey(encnt++, incrementScore)); // put new value into cluster @@ -253,7 +250,7 @@ public final class ScoreCluster<E> implements DynamicScore<E> { // set new value final int newValue = oldScore + incrementScore; - if (newValue < 0) throw new kelondroOutOfLimitsException(newValue); + if (newValue < 0) throw new OutOfLimitsException(newValue); usk = Long.valueOf(scoreKey(oldHandle, newValue)); // generates an unique key for a specific score map.put(obj, usk); pam.put(usk, obj); diff --git a/source/net/yacy/document/Document.java b/source/net/yacy/document/Document.java index c59e8db56..ecacf5bf5 100644 --- a/source/net/yacy/document/Document.java +++ b/source/net/yacy/document/Document.java @@ -50,12 +50,12 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.document.parser.html.ContentScraper; import net.yacy.document.parser.html.ImageEntry; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.ByteBuffer; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; @@ -593,7 +593,7 @@ dc_rights } String language = this.dc_language(); if (language != null && language.length() > 0) os.write("<dc:language>" + this.dc_language() + "</dc:language>\n"); - os.write("<dc:date>" + DateFormatter.formatISO8601(date) + "</dc:date>\n"); + os.write("<dc:date>" + ISO8601Formatter.FORMATTER.format(date) + "</dc:date>\n"); os.write("</record>\n"); } diff --git a/source/net/yacy/document/content/DCEntry.java b/source/net/yacy/document/content/DCEntry.java index 039a111de..6bda11cad 100644 --- a/source/net/yacy/document/content/DCEntry.java +++ b/source/net/yacy/document/content/DCEntry.java @@ -36,10 +36,10 @@ import java.util.HashSet; import java.util.Locale; import java.util.TreeMap; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.document.Document; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; public class DCEntry extends TreeMap<String, String> { @@ -67,7 +67,7 @@ public class DCEntry extends TreeMap<String, String> { ) { super((Collator) insensitiveCollator.clone()); this.put("dc:identifier", url.toNormalform(true, false)); - this.put("dc:date", DateFormatter.formatISO8601(date)); + this.put("dc:date", ISO8601Formatter.FORMATTER.format(date)); this.put("dc:title", title); this.put("dc:creator", author); this.put("dc:description", body); @@ -98,7 +98,7 @@ public class DCEntry extends TreeMap<String, String> { if (d == null) return null; if (d.length() == 0) return null; try { - return DateFormatter.parseISO8601(d); + return ISO8601Formatter.FORMATTER.parse(d); } catch (ParseException e) { Log.logException(e); return new Date(); diff --git a/source/net/yacy/document/importer/OAIPMHImporter.java b/source/net/yacy/document/importer/OAIPMHImporter.java index 8d3a43781..c80a4bca3 100644 --- a/source/net/yacy/document/importer/OAIPMHImporter.java +++ b/source/net/yacy/document/importer/OAIPMHImporter.java @@ -32,9 +32,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import net.yacy.cora.date.GenericFormatter; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.repository.LoaderDispatcher; import de.anomic.search.Switchboard; @@ -202,7 +202,7 @@ public class OAIPMHImporter extends Thread implements Importer, Comparable<OAIPM for (String s: surrogates.list()) { if (s.startsWith(filenamePrefix) && s.endsWith(".xml") && s.charAt(s.length() - 22) == filenameSeparationChar) { try { - Date fd = DateFormatter.parseShortMilliSecond(s.substring(s.length() - 21, s.length() - 4)); + Date fd = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(s.substring(s.length() - 21, s.length() - 4)); String hostID = s.substring(7, s.length() - 22); Date md = map.get(hostID); if (md == null || fd.after(md)) map.put(hostID, fd); @@ -242,6 +242,6 @@ public class OAIPMHImporter extends Thread implements Importer, Comparable<OAIPM public static final String filename4Source(DigestURI source) { return filenamePrefix + OAIPMHImporter.filenameSeparationChar + OAIPMHImporter.hostID(source) + OAIPMHImporter.filenameSeparationChar + - DateFormatter.formatShortMilliSecond(new Date()) + ".xml"; + GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()) + ".xml"; } } \ No newline at end of file diff --git a/source/net/yacy/document/importer/ResumptionToken.java b/source/net/yacy/document/importer/ResumptionToken.java index 0929d04c4..191c98c23 100644 --- a/source/net/yacy/document/importer/ResumptionToken.java +++ b/source/net/yacy/document/importer/ResumptionToken.java @@ -39,9 +39,9 @@ import org.xml.sax.Attributes; import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.logging.Log; -import net.yacy.kelondro.util.DateFormatter; public class ResumptionToken extends TreeMap<String, String> { @@ -142,7 +142,7 @@ public class ResumptionToken extends TreeMap<String, String> { // can be detected with given expiration date Date expiration = getExpirationDate(); if (expiration != null) { - if (expiration.before(new Date())) throw new IOException("the resumption is expired at " + DateFormatter.formatISO8601(expiration) + " (now: " + DateFormatter.formatISO8601(new Date())); + if (expiration.before(new Date())) throw new IOException("the resumption is expired at " + ISO8601Formatter.FORMATTER.format(expiration) + " (now: " + ISO8601Formatter.FORMATTER.format(new Date())); // the resumption token is still fresh } String u = url + "verb=ListRecords&resumptionToken=" + escape(token); @@ -191,7 +191,7 @@ public class ResumptionToken extends TreeMap<String, String> { String d = this.get("expirationDate"); if (d == null) return null; try { - return DateFormatter.parseISO8601(d); + return ISO8601Formatter.FORMATTER.parse(d); } catch (ParseException e) { Log.logException(e); return new Date(); @@ -241,7 +241,7 @@ public class ResumptionToken extends TreeMap<String, String> { } public String toString() { - return "source = " + this.source + ", expirationDate=" + DateFormatter.formatISO8601(this.getExpirationDate()) + ", completeListSize=" + getCompleteListSize() + + return "source = " + this.source + ", expirationDate=" + ISO8601Formatter.FORMATTER.format(this.getExpirationDate()) + ", completeListSize=" + getCompleteListSize() + ", cursor=" + this.getCursor() + ", token=" + this.getToken(); } diff --git a/source/net/yacy/document/parser/sitemapParser.java b/source/net/yacy/document/parser/sitemapParser.java index dcf3a6184..d88f53122 100644 --- a/source/net/yacy/document/parser/sitemapParser.java +++ b/source/net/yacy/document/parser/sitemapParser.java @@ -45,6 +45,7 @@ import org.w3c.dom.NodeList; import org.xml.sax.SAXException; import org.xml.sax.SAXParseException; +import net.yacy.cora.date.ISO8601Formatter; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; @@ -57,7 +58,6 @@ import net.yacy.document.TextParser; import net.yacy.document.parser.html.ImageEntry; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.io.ByteCountInputStream; -import net.yacy.kelondro.util.DateFormatter; public class sitemapParser extends AbstractParser implements Parser { @@ -198,7 +198,7 @@ public class sitemapParser extends AbstractParser implements Parser { public Date lastmod(final Date dflt) { try { - return DateFormatter.parseISO8601(lastmod); + return ISO8601Formatter.FORMATTER.parse(lastmod); } catch (final ParseException e) { return dflt; } @@ -219,7 +219,7 @@ public class sitemapParser extends AbstractParser implements Parser { public Date lastmod(final Date dflt) { try { - return DateFormatter.parseISO8601(lastmod); + return ISO8601Formatter.FORMATTER.parse(lastmod); } catch (final ParseException e) { return dflt; } diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index 1949f2e6e..87fa765c4 100755 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -47,6 +47,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import net.yacy.cora.date.GenericFormatter; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; @@ -58,7 +59,6 @@ import net.yacy.kelondro.rwi.Reference; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceFactory; import net.yacy.kelondro.rwi.ReferenceIterator; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.LookAheadIterator; import net.yacy.kelondro.util.NamePrefixThreadFactory; @@ -161,7 +161,7 @@ public class ArrayStack implements BLOB { f.delete(); deletions = true; } else try { - d = DateFormatter.parseShortSecond(files[i].substring(0, 14)); + d = GenericFormatter.SHORT_SECOND_FORMATTER.parse(files[i].substring(0, 14)); f.renameTo(newBLOB(d)); deletions = true; } catch (ParseException e) {continue;} @@ -177,7 +177,7 @@ public class ArrayStack implements BLOB { for (int i = 0; i < files.length; i++) { if (files[i].length() >= 22 && files[i].startsWith(prefix) && files[i].endsWith(".blob")) { try { - d = DateFormatter.parseShortMilliSecond(files[i].substring(prefix.length() + 1, prefix.length() + 18)); + d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(files[i].substring(prefix.length() + 1, prefix.length() + 18)); time = d.getTime(); if (time > maxtime) maxtime = time; } catch (ParseException e) {continue;} @@ -188,7 +188,7 @@ public class ArrayStack implements BLOB { for (int i = 0; i < files.length; i++) { if (files[i].length() >= 22 && files[i].startsWith(prefix) && files[i].endsWith(".blob")) { try { - d = DateFormatter.parseShortMilliSecond(files[i].substring(prefix.length() + 1, prefix.length() + 18)); + d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(files[i].substring(prefix.length() + 1, prefix.length() + 18)); f = new File(heapLocation, files[i]); time = d.getTime(); if (time == maxtime && !trimall) { @@ -231,7 +231,7 @@ public class ArrayStack implements BLOB { public synchronized void mountBLOB(File location, boolean full) throws IOException { Date d; try { - d = DateFormatter.parseShortMilliSecond(location.getName().substring(prefix.length() + 1, prefix.length() + 18)); + d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(location.getName().substring(prefix.length() + 1, prefix.length() + 18)); } catch (ParseException e) { throw new IOException("date parse problem with file " + location.toString() + ": " + e.getMessage()); } @@ -365,7 +365,7 @@ public class ArrayStack implements BLOB { */ public synchronized File newBLOB(Date creation) { //return new File(heapLocation, DateFormatter.formatShortSecond(creation) + "." + blobSalt + ".blob"); - return new File(heapLocation, prefix + "." + DateFormatter.formatShortMilliSecond(creation) + ".blob"); + return new File(heapLocation, prefix + "." + GenericFormatter.SHORT_MILSEC_FORMATTER.format(creation) + ".blob"); } public String name() { diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index fe813e0c5..401207639 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -33,6 +33,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.util.Collection; +import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -40,6 +41,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.storage.ARC; import net.yacy.cora.storage.ConcurrentARC; import net.yacy.kelondro.index.RowSpaceExceededException; @@ -48,7 +50,6 @@ import net.yacy.kelondro.order.ByteOrder; import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.order.RotateIterator; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.kelondroException; @@ -139,7 +140,7 @@ public class MapHeap implements Map<byte[], Map<String, String>> { assert key.length > 0; assert newMap != null; key = normalizeKey(key); - String s = map2string(newMap, "W" + DateFormatter.formatShortSecond() + " "); + String s = map2string(newMap, "W" + GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()) + " "); assert s != null; byte[] sb = s.getBytes(); synchronized (this) { diff --git a/source/net/yacy/kelondro/blob/Tables.java b/source/net/yacy/kelondro/blob/Tables.java index 4b1dc36d7..bc865054c 100644 --- a/source/net/yacy/kelondro/blob/Tables.java +++ b/source/net/yacy/kelondro/blob/Tables.java @@ -41,11 +41,11 @@ import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; +import net.yacy.cora.date.GenericFormatter; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.util.ByteArray; import net.yacy.kelondro.util.ByteBuffer; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.LookAheadIterator; @@ -408,7 +408,7 @@ public class Tables { } public void put(String colname, Date value) { - super.put(colname, DateFormatter.formatShortMilliSecond(value).getBytes()); + super.put(colname, GenericFormatter.SHORT_MILSEC_FORMATTER.format(value).getBytes()); } public byte[] get(String colname, byte[] dflt) { @@ -447,7 +447,7 @@ public class Tables { byte[] r = this.get(colname); if (r == null) return dflt; try { - return DateFormatter.parseShortMilliSecond(new String(r)); + return GenericFormatter.SHORT_MILSEC_FORMATTER.parse(new String(r)); } catch (ParseException e) { return dflt; } diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java index 9d864e05a..39e297530 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Properties; import java.util.regex.Pattern; +import net.yacy.cora.date.GenericFormatter; import net.yacy.kelondro.data.word.WordReferenceRow; import net.yacy.kelondro.data.word.WordReferenceVars; import net.yacy.kelondro.index.Row; @@ -44,7 +45,6 @@ import net.yacy.kelondro.order.Bitfield; import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.util.ByteBuffer; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.kelondroException; import net.yacy.kelondro.util.MapTools; @@ -228,17 +228,17 @@ public class URIMetadataRow implements URIMetadata { this.entry.setCol(col_hash, url.hash()); // FIXME potential null pointer access this.entry.setCol(col_comp, encodeComp(url, descr, dc_creator, tags, dc_publisher)); try { - encodeDate(col_mod, DateFormatter.parseShortDay(prop.getProperty("mod", "20000101"))); + encodeDate(col_mod, GenericFormatter.SHORT_DAY_FORMATTER.parse(prop.getProperty("mod", "20000101"))); } catch (final ParseException e) { encodeDate(col_mod, new Date()); } try { - encodeDate(col_load, DateFormatter.parseShortDay(prop.getProperty("load", "20000101"))); + encodeDate(col_load, GenericFormatter.SHORT_DAY_FORMATTER.parse(prop.getProperty("load", "20000101"))); } catch (final ParseException e) { encodeDate(col_load, new Date()); } try { - encodeDate(col_fresh, DateFormatter.parseShortDay(prop.getProperty("fresh", "20000101"))); + encodeDate(col_fresh, GenericFormatter.SHORT_DAY_FORMATTER.parse(prop.getProperty("fresh", "20000101"))); } catch (final ParseException e) { encodeDate(col_fresh, new Date()); } @@ -305,11 +305,11 @@ public class URIMetadataRow implements URIMetadata { assert (s.toString().indexOf(0) < 0); s.append(",publisher=").append(crypt.simpleEncode(metadata.dc_publisher())); assert (s.toString().indexOf(0) < 0); - s.append(",mod=").append(DateFormatter.formatShortDay(moddate())); + s.append(",mod=").append(GenericFormatter.SHORT_DAY_FORMATTER.format(moddate())); assert (s.toString().indexOf(0) < 0); - s.append(",load=").append(DateFormatter.formatShortDay(loaddate())); + s.append(",load=").append(GenericFormatter.SHORT_DAY_FORMATTER.format(loaddate())); assert (s.toString().indexOf(0) < 0); - s.append(",fresh=").append(DateFormatter.formatShortDay(freshdate())); + s.append(",fresh=").append(GenericFormatter.SHORT_DAY_FORMATTER.format(freshdate())); assert (s.toString().indexOf(0) < 0); s.append(",referrer=").append(referrerHash() == null ? "" : new String(referrerHash())); assert (s.toString().indexOf(0) < 0); diff --git a/source/net/yacy/kelondro/table/SplitTable.java b/source/net/yacy/kelondro/table/SplitTable.java index 540681cb8..67e5bbf06 100644 --- a/source/net/yacy/kelondro/table/SplitTable.java +++ b/source/net/yacy/kelondro/table/SplitTable.java @@ -41,6 +41,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import net.yacy.cora.date.GenericFormatter; import net.yacy.kelondro.blob.ArrayStack; import net.yacy.kelondro.index.Cache; import net.yacy.kelondro.index.HandleSet; @@ -54,7 +55,6 @@ import net.yacy.kelondro.order.CloneableIterator; import net.yacy.kelondro.order.MergeIterator; import net.yacy.kelondro.order.Order; import net.yacy.kelondro.order.StackIterator; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.NamePrefixThreadFactory; @@ -136,7 +136,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> { } private String newFilename() { - return prefix + "." + DateFormatter.formatShortMilliSecond(new Date()) + ".table"; + return prefix + "." + GenericFormatter.SHORT_MILSEC_FORMATTER.format(new Date()) + ".table"; } private void init() throws RowSpaceExceededException { @@ -172,7 +172,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> { (tablefile[i].length() == prefix.length() + 24)) { f = new File(path, tablefile[i]); try { - d = DateFormatter.parseShortMilliSecond(tablefile[i].substring(prefix.length() + 1, prefix.length() + 18)); + d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(tablefile[i].substring(prefix.length() + 1, prefix.length() + 18)); } catch (ParseException e) { Log.logSevere("SplitTable", "", e); continue; @@ -331,7 +331,7 @@ public class SplitTable implements Index, Iterable<Row.Entry> { String name = new File(table.filename()).getName(); long d; try { - d = DateFormatter.parseShortMilliSecond(name.substring(prefix.length() + 1, prefix.length() + 18)).getTime(); + d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(name.substring(prefix.length() + 1, prefix.length() + 18)).getTime(); } catch (ParseException e) { Log.logSevere("SplitTable", "", e); d = 0; diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index d7e3fd305..ae49b229f 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -828,19 +828,14 @@ public class Table implements Index, Iterable<Row.Entry> { int idx; byte[] key; - public rowIteratorNoOrder() throws IOException { + public rowIteratorNoOrder() { // don't use the ChunkIterator here because it may create too many open files during string load //ri = new ChunkIterator(tablefile, rowdef.objectsize, rowdef.objectsize); i = index.iterator(); } public CloneableIterator<Entry> clone(Object modifier) { - try { - return new rowIteratorNoOrder(); - } catch (IOException e) { - Log.logSevere("Table", "", e); - return null; - } + return new rowIteratorNoOrder(); } public boolean hasNext() { diff --git a/source/net/yacy/kelondro/util/DateFormatter.java b/source/net/yacy/kelondro/util/DateFormatter.java deleted file mode 100644 index 5b7a5967e..000000000 --- a/source/net/yacy/kelondro/util/DateFormatter.java +++ /dev/null @@ -1,473 +0,0 @@ -// serverDate.java -// ------------------------------------------- -// (C) by Michael Peter Christen; mc@yacy.net -// (C) by by Bjoern 'Fuchs' Krombholz; fox.box@gmail.com -// first published on http://www.anomic.de -// Frankfurt, Germany, 2005, 2007 -// last major change: 14.03.2005 -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -// this class is needed to replace the slow java built-in date method by a faster version - -package net.yacy.kelondro.util; - -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.Locale; -import java.util.NoSuchElementException; -import java.util.StringTokenizer; -import java.util.TimeZone; - -public final class DateFormatter { - - /** minimal date format without time information (fixed width: 8) */ - public static final String PATTERN_SHORT_DAY = "yyyyMMdd"; - /** minimal date format (fixed width: 14) */ - public static final String PATTERN_SHORT_SECOND = "yyyyMMddHHmmss"; - /** minimal date format including milliseconds (fixed width: 17) */ - public static final String PATTERN_SHORT_MILSEC = "yyyyMMddHHmmssSSS"; - - /** default HTTP 1.1 header date format pattern */ - public static final String PATTERN_RFC1123 = "EEE, dd MMM yyyy HH:mm:ss Z"; // with numeric time zone indicator as defined in RFC5322 - public static final String PATTERN_RFC1123_SHORT = "EEE, dd MMM yyyy"; - - /** date pattern used in older HTTP implementations */ - public static final String PATTERN_ANSIC = "EEE MMM d HH:mm:ss yyyy"; - /** date pattern used in older HTTP implementations */ - public static final String PATTERN_RFC1036 = "EEEE, dd-MMM-yy HH:mm:ss zzz"; - - /** pattern for a W3C datetime variant of a non-localized ISO8601 date */ - public static final String PATTERN_ISO8601 = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - - /** predefined GMT TimeZone object */ - private static final TimeZone TZ_GMT = TimeZone.getTimeZone("GMT"); - - /** predefined non-localized Calendar object for generic GMT dates */ - private static final Calendar CAL_GMT = Calendar.getInstance(TZ_GMT, Locale.US); - - /** Date formatter/parser for minimal yyyyMMdd pattern */ - private static final SimpleDateFormat FORMAT_SHORT_DAY = new SimpleDateFormat(PATTERN_SHORT_DAY, Locale.US); - /** Date formatter/parser for minimal yyyyMMddHHmmss pattern */ - private static final SimpleDateFormat FORMAT_SHORT_SECOND = new SimpleDateFormat(PATTERN_SHORT_SECOND, Locale.US); - /** Date formatter/parser for minimal yyyyMMddHHmmssSSS pattern */ - private static final SimpleDateFormat FORMAT_SHORT_MILSEC = new SimpleDateFormat(PATTERN_SHORT_MILSEC, Locale.US); - - /** Date formatter/non-sloppy parser for W3C datetime (ISO8601) in GMT/UTC */ - private static final SimpleDateFormat FORMAT_ISO8601 = new SimpleDateFormat(PATTERN_ISO8601, Locale.US); - - /** Date formatter/parser for standard compliant HTTP header dates (RFC 1123) */ - private static final SimpleDateFormat FORMAT_ANSIC = new SimpleDateFormat(PATTERN_ANSIC, Locale.US); - - private static final SimpleDateFormat FORMAT_RFC1123_SHORT = new SimpleDateFormat(PATTERN_RFC1123_SHORT, Locale.US); - - - /** Initialization of static formats */ - static { - // 2-digit dates are automatically parsed by SimpleDateFormat, - // we need to detect the real year by adding 1900 or 2000 to - // the year value starting with 1970 - CAL_GMT.setTimeInMillis(0); - - // we want GMT times on the SHORT formats as well as they don't support any timezone - FORMAT_SHORT_DAY.setTimeZone(TZ_GMT); - FORMAT_SHORT_SECOND.setTimeZone(TZ_GMT); - FORMAT_SHORT_MILSEC.setTimeZone(TZ_GMT); - FORMAT_ISO8601.setTimeZone(TZ_GMT); - } - - - /** - * Creates a String representation of a Date using the format defined - * in ISO8601/W3C datetime - * The result will be in UTC/GMT, e.g. "2007-12-19T10:20:30Z". - * - * @param date The Date instance to transform. - * @return A fixed width (20 chars) ISO8601 date String. - */ - private static long FORMAT_ISO8601_last_time = 0; - private static String FORMAT_ISO8601_last_format = ""; - public static final String formatISO8601(final Date date) { - if (date == null) return ""; - if (Math.abs(date.getTime() - FORMAT_ISO8601_last_time) < 1000) return FORMAT_ISO8601_last_format; - FORMAT_ISO8601_last_format = format(FORMAT_ISO8601, date); - FORMAT_ISO8601_last_time = date.getTime(); - return FORMAT_ISO8601_last_format; - } - public static final String formatANSIC(final Date date) { - if (date == null) return ""; - return format(FORMAT_ANSIC, date); - } - - public static String formatRFC1123_short(final Date date) { - if (date == null) return ""; - return FORMAT_RFC1123_SHORT.format(date); - } - - /** - * Identical to {@link #formatShortDay(Date)}, but for short second format. - */ - private static long FORMAT_SHORT_SECOND_last_time = 0; - private static String FORMAT_SHORT_SECOND_last_format = ""; - public static String formatShortSecond(final Date date) { - if (date == null) return ""; - if (Math.abs(date.getTime() - FORMAT_SHORT_SECOND_last_time) < 1000) return FORMAT_SHORT_SECOND_last_format; - FORMAT_SHORT_SECOND_last_format = format(FORMAT_SHORT_SECOND, date); - FORMAT_SHORT_SECOND_last_time = date.getTime(); - return FORMAT_SHORT_SECOND_last_format; - } - - public static String formatShortMilliSecond(final Date date) { - return format(FORMAT_SHORT_MILSEC, date); - } - - /** - * Note: The short day format doesn't include any timezone information. This method - * transforms the date into the GMT/UTC timezone. Example: If the local system time is, - * 2007-12-18 01:15:00 +0200, then the resulting String will be "2007-12-17". - * In case you need a format with a timezon offset, use {@link #formatShortDay(TimeZone)} - * @return a String representation of the current system date in GMT using the - * short day format, e.g. "20071218". - */ - public static String formatShortDay() { - return format(FORMAT_SHORT_DAY, new Date()); - } - - /** - * @see #formatShortDay() - * @param date the Date to transform - */ - private static long FORMAT_SHORT_DAY_last_time = 0; - private static String FORMAT_SHORT_DAY_last_format = ""; - public static String formatShortDay(final Date date) { - if (date == null) return ""; - if (Math.abs(date.getTime() - FORMAT_SHORT_DAY_last_time) < 1000) return FORMAT_SHORT_DAY_last_format; - FORMAT_SHORT_DAY_last_format = format(FORMAT_SHORT_DAY, date); - FORMAT_SHORT_DAY_last_time = date.getTime(); - return FORMAT_SHORT_DAY_last_format; - } - - /** - * Parse dates as defined in {@linkplain http://www.w3.org/TR/NOTE-datetime}. - * This format (also specified in ISO8601) allows different "precisions". - * The following lower precision versions for the complete date - * "2007-12-19T10:20:30.567+0300" are allowed:<br> - * "2007"<br> - * "2007-12"<br> - * "2007-12-19"<br> - * "2007-12-19T10:20+0300<br> - * "2007-12-19T10:20:30+0300<br> - * "2007-12-19T10:20:30.567+0300<br> - * Additionally a timezone offset of "+0000" can be substituted as "Z".<br> - * Parsing is done in a fuzzy way. If there is an illegal character somewhere in - * the String, the date parsed so far will be returned, e.g. the input - * "2007-12-19FOO" would return a date that represents "2007-12-19". - * - * @param s - * @return - * @throws ParseException - */ - public static Date parseISO8601(String s) throws ParseException { - // do some lazy checks here - s = s.trim(); - while (s.length() > 0 && s.endsWith("?")) s = s.substring(0, s.length() - 1); // sometimes used if write is not sure about date - if (s.startsWith("{")) s = s.substring(1); - if (s.endsWith("}")) s = s.substring(0, s.length() - 1); - if (s.startsWith("[")) s = s.substring(1); - if (s.endsWith("]")) s = s.substring(0, s.length() - 1); - while (s.length() > 0 && (s.charAt(0) > '9' || s.charAt(0) < '0')) s = s.substring(1); - if (s.endsWith("--")) s = s.substring(0, s.length() - 2) + "00"; - int p = s.indexOf(';'); if (p >= 0) s = s.substring(0, p); // a semicolon may be used to separate two dates from each other; then we take the first - p = s.indexOf(','); if (p >= 0) s = s.substring(0, p); // a comma may be used to separate two dates from each other; then we take the first - while (s.length() > 0 && s.endsWith("?")) s = s.substring(0, s.length() - 1); // sometimes used if write is not sure about date - - // no go for exact parsing - final Calendar cal = Calendar.getInstance(TZ_GMT, Locale.US); - cal.clear(); - - // split 2007-12-19T10:20:30.789+0500 into its parts - // correct: yyyy['-'MM['-'dd['T'HH':'MM[':'ss['.'SSS]]('Z'|ZZZZZ)]]] - final StringTokenizer t = new StringTokenizer(s, "-T:.Z+", true); - if (s == null || t.countTokens() == 0) - throw new ParseException("parseISO8601: Cannot parse '" + s + "'", 0); - - try { - // year - cal.set(Calendar.YEAR, Integer.parseInt(t.nextToken())); - // month - if (t.nextToken().equals("-")) { - cal.set(Calendar.MONTH, Integer.parseInt(t.nextToken()) - 1); - } else { - return cal.getTime(); - } - // day - if (t.nextToken().equals("-")) { - cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(t.nextToken())); - } else { - return cal.getTime(); - } - // The standard says: if there is an hour there has to be a minute and a - // timezone token, too. - // hour - if (t.nextToken().equals("T")) { - final int hour = Integer.parseInt(t.nextToken()); - // no error, got hours - int min = 0; - int sec = 0; - int msec = 0; - if (t.nextToken().equals(":")) { - min = Integer.parseInt(t.nextToken()); - // no error, got minutes - // need TZ or seconds - String token = t.nextToken(); - if (token.equals(":")) { - sec = Integer.parseInt(t.nextToken()); - // need millisecs or TZ - token = t.nextToken(); - if (token.equals(".")) { - msec = Integer.parseInt(t.nextToken()); - // need TZ - token = t.nextToken(); - } - } - - // check for TZ data - int offset; - if (token.equals("Z")) { - offset = 0; - } else { - int sign = 0; - if (token.equals("+")) { - sign = 1; - } else if (token.equals("-")) { - sign = -1; - } else { - // no legal TZ offset found - return cal.getTime(); - } - offset = sign * Integer.parseInt(t.nextToken()) * 10 * 3600; - } - cal.set(Calendar.ZONE_OFFSET, offset); - } - cal.set(Calendar.HOUR_OF_DAY, hour); - cal.set(Calendar.MINUTE, min); - cal.set(Calendar.SECOND, sec); - cal.set(Calendar.MILLISECOND, msec); - } - } catch (final NoSuchElementException e) { - // ignore this as it is perfectly fine to have non-complete date in this format - } catch (final Exception e) { - // catch all Exceptions and return what we parsed so far - //serverLog.logInfo("SERVER", "parseISO8601: DATE ERROR with: '" + s + "' got so far: '" + cal.toString()); - } - - // in case we couldn't even parse a year - if (!cal.isSet(Calendar.YEAR)) - throw new ParseException("parseISO8601: Cannot parse '" + s + "'", 0); - Date d = cal.getTime(); - return d; - } - - /** - * Parse a String representation of a Date in short day format assuming the date - * is aligned to the GMT/UTC timezone. An example for such a date string is "20071218". - * @see #formatShortDay() - * @throws ParseException The exception is thrown if an error occured during while parsing - * the String. - */ - public static Date parseShortDay(final String timeString) throws ParseException { - return parse(FORMAT_SHORT_DAY, timeString); - } - - /** - * Returns the current date in short second format which is a fixed width (14 chars) - * String including the date and the time like "20071218233510". The result is in GMT/UTC. - * @see #formatShortDay() - */ - public static String formatShortSecond() { - return formatShortSecond(new Date()); - } - - //TODO check the following 2 parse methods for correct use (GMT vs. different timezone) - /** - * Like {@link #parseShortDay(String)}, but for the "short second" format which is short date - * plus a 6 digit day time value, like "20071218233510". The String should be in GMT/UTC to - * get a correct Date. - */ - public static Date parseShortSecond(final String timeString) throws ParseException { - return parse(FORMAT_SHORT_SECOND, timeString); - } - public static Date parseShortMilliSecond(final String timeString) throws ParseException { - return parse(FORMAT_SHORT_MILSEC, timeString); -} - - /** - * Like {@link #parseShortSecond(String)} using additional timezone information provided in an - * offset String, like "+0100" for CET. - */ - public static Date parseShortSecond(final String remoteTimeString, final String remoteUTCOffset) { - // FIXME: This method returns an incorrect date, check callers! - // ex: de.anomic.server.serverDate.parseShortSecond("20070101120000", "+0200").toGMTString() - // => 1 Jan 2007 13:00:00 GMT - if (remoteTimeString == null || remoteTimeString.length() == 0) { return new Date(); } - if (remoteUTCOffset == null || remoteUTCOffset.length() == 0) { return new Date(); } - try { - return new Date(parse(FORMAT_SHORT_SECOND, remoteTimeString).getTime() - DateFormatter.UTCDiff() + DateFormatter.UTCDiff(remoteUTCOffset)); - } catch (final java.text.ParseException e) { - //serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]"); - return new Date(); - } catch (final java.lang.NumberFormatException e) { - //serverLog.logFinest("parseUniversalDate", e.getMessage() + ", remoteTimeString=[" + remoteTimeString + "]"); - return new Date(); - } - } - - - /** - * Format a time inteval in milliseconds into a String of the form - * X 'day'['s'] HH':'mm - */ - public static String formatInterval(final long millis) { - try { - final long mins = millis / 60000; - - final StringBuilder uptime = new StringBuilder(); - - final int uptimeDays = (int) (Math.floor(mins/1440.0)); - final int uptimeHours = (int) (Math.floor(mins/60.0)%24); - final int uptimeMins = (int) mins%60; - - uptime.append(uptimeDays) - .append(((uptimeDays == 1)?" day ":" days ")) - .append((uptimeHours < 10)?"0":"") - .append(uptimeHours) - .append(":") - .append((uptimeMins < 10)?"0":"") - .append(uptimeMins); - - return uptime.toString(); - } catch (final Exception e) { - return "unknown"; - } - } - - /** called by all public format...(...) methods */ - private static String format(final SimpleDateFormat format, final Date date) { - synchronized (format) { - return format.format(date); - } - } - - /** calles by all public parse...(...) methods */ - private static Date parse(final SimpleDateFormat format, final String dateString) throws ParseException { - synchronized (format) { - return format.parse(dateString); - } - } - - // statics - public final static long secondMillis = 1000; - public final static long minuteMillis = 60 * secondMillis; - public final static long hourMillis = 60 * minuteMillis; - public final static long dayMillis = 24 * hourMillis; - public final static long normalyearMillis = 365 * dayMillis; - public final static long leapyearMillis = 366 * dayMillis; - public final static int january = 31, normalfebruary = 28, leapfebruary = 29, march = 31, - april = 30, may = 31, june = 30, july = 31, august = 31, - september = 30, october = 31, november = 30, december = 31; - public final static int[] dimnormal = {january, normalfebruary, march, april, may, june, july, august, september, october, november, december}; - public final static int[] dimleap = {january, leapfebruary, march, april, may, june, july, august, september, october, november, december}; - public final static String[] wkday = {"Mon","Tue","Wed","Thu","Fri","Sat","Sun"}; - //private final static String[] month = {"Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"}; - - // find out time zone and DST offset - private static Calendar thisCalendar = Calendar.getInstance(); - - public static String UTCDiffString() { - // we express the UTC Difference in 5 digits: - // SHHMM - // S ::= '+'|'-' - // HH ::= '00'|'01'|'02'|'03'|'04'|'05'|'06'|'07'|'08'|'09'|'10'|'11'|'12' - // MM ::= '00'|'15'|'30'|'45' - // since there are some places on earth where there is a time shift of half an hour - // we need too show also the minutes of the time shift - // Examples: http://www.timeanddate.com/library/abbreviations/timezones/ - final long offsetHours = UTCDiff(); - final int om = Math.abs((int) (offsetHours / minuteMillis)) % 60; - final int oh = Math.abs((int) (offsetHours / hourMillis)); - String diff = Integer.toString(om); - if (diff.length() < 2) diff = "0" + diff; - diff = Integer.toString(oh) + diff; - if (diff.length() < 4) diff = "0" + diff; - if (offsetHours < 0) { - return "-" + diff; - } - return "+" + diff; - } - - private static long UTCDiff() { - // DST_OFFSET is dependent on the time of the Calendar, so it has to be updated - // to get the correct current offset - synchronized(thisCalendar) { - thisCalendar.setTimeInMillis(System.currentTimeMillis()); - final long zoneOffsetHours = thisCalendar.get(Calendar.ZONE_OFFSET); - final long DSTOffsetHours = thisCalendar.get(Calendar.DST_OFFSET); - return zoneOffsetHours + DSTOffsetHours; - } - } - - private static long UTCDiff(final String diffString) { - if (diffString.length() != 5) throw new IllegalArgumentException("UTC String malformed (wrong size):" + diffString); - boolean ahead = true; - if (diffString.length() > 0 && diffString.charAt(0) == '+') ahead = true; - else if (diffString.length() > 0 && diffString.charAt(0) == '-') ahead = false; - else throw new IllegalArgumentException("UTC String malformed (wrong sign):" + diffString); - final long oh = Long.parseLong(diffString.substring(1, 3)); - final long om = Long.parseLong(diffString.substring(3)); - return ((ahead) ? (long) 1 : (long) -1) * (oh * hourMillis + om * minuteMillis); - } - - public static long correctedUTCTime() { - return System.currentTimeMillis() - UTCDiff(); - } - - public static long remainingTime(final long start, final long due, final long minimum) { - if (due < 0) return -1; - final long r = due + start - System.currentTimeMillis(); - if (r <= 0) return minimum; - return r; - } - - public static void main(final String[] args) { - //System.out.println("kelondroDate is (" + new kelondroDate().toString() + ")"); - System.out.println("offset is " + (UTCDiff()/1000/60/60) + " hours, javaDate is " + new Date() + ", correctedDate is " + new Date(correctedUTCTime())); - System.out.println(" javaDate : " + formatShortSecond()); - System.out.println("serverDate : " + new DateFormatter().toString()); - System.out.println(" JavaDate : " + DateFormat.getDateInstance().format(new Date())); - System.out.println(" JavaDate0: " + format(FORMAT_SHORT_SECOND, new Date(0))); - System.out.println(" JavaDate0: " + DateFormat.getDateInstance().format(new Date(0))); - //String testresult; - final int cycles = 10000; - long start; - - final String[] testresult = new String[10000]; - start = System.currentTimeMillis(); - for (int i = 0; i < cycles; i++) testresult[i] = format(FORMAT_SHORT_SECOND, new Date()); - System.out.println("time for " + cycles + " calls to javaDate:" + (System.currentTimeMillis() - start) + " milliseconds"); - } -} diff --git a/source/net/yacy/kelondro/util/kelondroOutOfLimitsException.java b/source/net/yacy/kelondro/util/kelondroOutOfLimitsException.java deleted file mode 100644 index c76ecc8b7..000000000 --- a/source/net/yacy/kelondro/util/kelondroOutOfLimitsException.java +++ /dev/null @@ -1,41 +0,0 @@ -// kelondroOutOfLimitsException.java -// --------------------------------- -// part of The Kelondro Database -// (C) by Michael Peter Christen; mc@yacy.net -// first published on http://www.anomic.de -// Frankfurt, Germany, 2006 -// created: 17.01.2006 -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.kelondro.util; - -public class kelondroOutOfLimitsException extends java.lang.RuntimeException { - - private static final long serialVersionUID = 1L; - - public kelondroOutOfLimitsException() { - super("unspecific-error"); - } - - public kelondroOutOfLimitsException(final int expectedLimit, final int actualSize) { - super("Object size is " + actualSize + "; it exceeds the size limit " + expectedLimit); - } - - public kelondroOutOfLimitsException(final int actualSize) { - super("Object size is " + actualSize + "; must not be negative"); - } - -} diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index 806e9afff..e2f71c99b 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -46,6 +46,7 @@ import java.util.concurrent.Semaphore; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.http.HTTPClient; @@ -63,7 +64,6 @@ import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.rwi.Reference; import net.yacy.kelondro.rwi.ReferenceContainer; -import net.yacy.kelondro.util.DateFormatter; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.MemoryControl; @@ -192,7 +192,7 @@ public final class yacy { Log.logConfig("STARTUP", "Operation system: " + System.getProperty("os.name","unknown")); Log.logConfig("STARTUP", "Application root-path: " + appHome); Log.logConfig("STARTUP", "Data root-path: " + dataHome); - Log.logConfig("STARTUP", "Time zone: UTC" + DateFormatter.UTCDiffString() + "; UTC+0000 is " + System.currentTimeMillis()); + Log.logConfig("STARTUP", "Time zone: UTC" + GenericFormatter.UTCDiffString() + "; UTC+0000 is " + System.currentTimeMillis()); Log.logConfig("STARTUP", "Maximum file system path length: " + OS.maxPathLength); f = new File(dataHome, "DATA/yacy.running");