From e004da48d392aa61fd74244bff8f0258a4636133 Mon Sep 17 00:00:00 2001 From: orbiter Date: Mon, 29 Dec 2008 12:22:13 +0000 Subject: [PATCH] - added fast fingerprint computation for files (any). Will be used in new index dump method - refactoring git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5415 6c8d7289-2bf4-0310-a012-ef5d649a1542 --- htroot/ConfigAccounts_p.java | 10 +- htroot/SettingsAck_p.java | 4 +- htroot/User.java | 8 +- htroot/xml/bookmarks/posts/all.java | 4 +- htroot/xml/bookmarks/posts/get.java | 4 +- htroot/yacy/transfer.java | 8 +- source/de/anomic/crawler/CrawlProfile.java | 4 +- source/de/anomic/data/userDB.java | 4 +- source/de/anomic/http/httpd.java | 5 +- source/de/anomic/http/httpdProxyHandler.java | 6 - source/de/anomic/index/indexURLReference.java | 5 +- source/de/anomic/index/indexWord.java | 4 +- .../anomic/kelondro/kelondroBytesIntMap.java | 7 - .../kelondro/kelondroCollectionIndex.java | 3 +- source/de/anomic/kelondro/kelondroDigest.java | 333 ++++++++++++++++++ .../de/anomic/kelondro/kelondroEcoTable.java | 6 +- .../de/anomic/kelondro/kelondroRAMIndex.java | 2 + source/de/anomic/plasma/plasmaDHTChunk.java | 4 +- source/de/anomic/plasma/plasmaDHTFlush.java | 4 +- .../plasma/plasmaRankingRCIEvaluation.java | 4 +- .../plasma/plasmaSearchRankingProcess.java | 4 +- .../de/anomic/plasma/plasmaSwitchboard.java | 4 +- source/de/anomic/server/serverCodings.java | 254 +------------ source/de/anomic/tools/cryptbig.java | 4 +- source/de/anomic/yacy/yacyClient.java | 6 +- source/de/anomic/yacy/yacyNetwork.java | 6 +- source/de/anomic/yacy/yacySeed.java | 11 +- source/de/anomic/yacy/yacyURL.java | 8 +- source/genpw.java | 4 +- source/migration.java | 15 +- 30 files changed, 420 insertions(+), 325 deletions(-) create mode 100644 source/de/anomic/kelondro/kelondroDigest.java diff --git a/htroot/ConfigAccounts_p.java b/htroot/ConfigAccounts_p.java index 35af08fb7..ba1549395 100644 --- a/htroot/ConfigAccounts_p.java +++ b/htroot/ConfigAccounts_p.java @@ -34,8 +34,8 @@ import de.anomic.data.userDB; import de.anomic.http.httpRequestHeader; import de.anomic.http.httpd; import de.anomic.kelondro.kelondroBase64Order; +import de.anomic.kelondro.kelondroDigest; import de.anomic.plasma.plasmaSwitchboard; -import de.anomic.server.serverCodings; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -57,7 +57,7 @@ public class ConfigAccounts_p { // may be overwritten if new password is given if ((user.length() > 0) && (pw1.length() > 3) && (pw1.equals(pw2))) { // check passed. set account: - env.setConfig(httpd.ADMIN_ACCOUNT_B64MD5, serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(user + ":" + pw1))); + env.setConfig(httpd.ADMIN_ACCOUNT_B64MD5, kelondroDigest.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(user + ":" + pw1))); env.setConfig("adminAccount", ""); } @@ -73,7 +73,7 @@ public class ConfigAccounts_p { // if not, set a random password if (post != null && env.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() == 0) { // make a 'random' password - env.setConfig(httpd.ADMIN_ACCOUNT_B64MD5, "0000" + serverCodings.encodeMD5Hex(System.getProperties().toString() + System.currentTimeMillis())); + env.setConfig(httpd.ADMIN_ACCOUNT_B64MD5, "0000" + kelondroDigest.encodeMD5Hex(System.getProperties().toString() + System.currentTimeMillis())); env.setConfig("adminAccount", ""); } } @@ -171,7 +171,7 @@ public class ConfigAccounts_p { if( post.get("current_user").equals("newuser")){ //new user if(!pw1.equals("")){ //change only if set - mem.put(userDB.Entry.MD5ENCODED_USERPWD_STRING, serverCodings.encodeMD5Hex(username+":"+pw1)); + mem.put(userDB.Entry.MD5ENCODED_USERPWD_STRING, kelondroDigest.encodeMD5Hex(username+":"+pw1)); } mem.put(userDB.Entry.USER_FIRSTNAME, firstName); mem.put(userDB.Entry.USER_LASTNAME, lastName); @@ -196,7 +196,7 @@ public class ConfigAccounts_p { if(entry != null){ try{ if(! pw1.equals("")){ - entry.setProperty(userDB.Entry.MD5ENCODED_USERPWD_STRING, serverCodings.encodeMD5Hex(username+":"+pw1)); + entry.setProperty(userDB.Entry.MD5ENCODED_USERPWD_STRING, kelondroDigest.encodeMD5Hex(username+":"+pw1)); } entry.setProperty(userDB.Entry.USER_FIRSTNAME, firstName); entry.setProperty(userDB.Entry.USER_LASTNAME, lastName); diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index dd3fb4827..6ecaf3d25 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -43,9 +43,9 @@ import de.anomic.http.httpRemoteProxyConfig; import de.anomic.http.httpd; import de.anomic.http.httpdProxyHandler; import de.anomic.kelondro.kelondroBase64Order; +import de.anomic.kelondro.kelondroDigest; import de.anomic.plasma.plasmaParser; import de.anomic.plasma.plasmaSwitchboard; -import de.anomic.server.serverCodings; import de.anomic.server.serverCore; import de.anomic.server.serverDate; import de.anomic.server.serverMemory; @@ -95,7 +95,7 @@ public class SettingsAck_p { return prop; } // check passed. set account: - env.setConfig(httpd.ADMIN_ACCOUNT_B64MD5, serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(user + ":" + pw1))); + env.setConfig(httpd.ADMIN_ACCOUNT_B64MD5, kelondroDigest.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(user + ":" + pw1))); env.setConfig("adminAccount", ""); prop.put("info", "5");//admin account changed prop.putHTML("info_user", user); diff --git a/htroot/User.java b/htroot/User.java index 3875f4078..958dc098e 100644 --- a/htroot/User.java +++ b/htroot/User.java @@ -34,8 +34,8 @@ import de.anomic.http.httpRequestHeader; import de.anomic.http.httpResponseHeader; import de.anomic.http.httpd; import de.anomic.kelondro.kelondroBase64Order; +import de.anomic.kelondro.kelondroDigest; import de.anomic.plasma.plasmaSwitchboard; -import de.anomic.server.serverCodings; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.servletProperties; @@ -95,7 +95,7 @@ public class User{ entry=sb.userDB.passwordAuth(username, password); final boolean staticAdmin = sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").equals( - serverCodings.encodeMD5Hex( + kelondroDigest.encodeMD5Hex( kelondroBase64Order.standardCoder.encodeString(username + ":" + password) ) ); @@ -123,11 +123,11 @@ public class User{ if(post!= null && entry != null){ if(post.containsKey("changepass")){ prop.put("status", "1"); //password - if(entry.getMD5EncodedUserPwd().equals(serverCodings.encodeMD5Hex(entry.getUserName()+":"+post.get("oldpass", "")))){ + if(entry.getMD5EncodedUserPwd().equals(kelondroDigest.encodeMD5Hex(entry.getUserName()+":"+post.get("oldpass", "")))){ if(post.get("newpass").equals(post.get("newpass2"))){ if(!post.get("newpass", "").equals("")){ try { - entry.setProperty(userDB.Entry.MD5ENCODED_USERPWD_STRING, serverCodings.encodeMD5Hex(entry.getUserName()+":"+post.get("newpass", ""))); + entry.setProperty(userDB.Entry.MD5ENCODED_USERPWD_STRING, kelondroDigest.encodeMD5Hex(entry.getUserName()+":"+post.get("newpass", ""))); prop.put("status_password", "0"); //changes } catch (final IOException e) {} }else{ diff --git a/htroot/xml/bookmarks/posts/all.java b/htroot/xml/bookmarks/posts/all.java index 3ca5ac4c3..6985111a7 100644 --- a/htroot/xml/bookmarks/posts/all.java +++ b/htroot/xml/bookmarks/posts/all.java @@ -31,8 +31,8 @@ import java.util.Iterator; import de.anomic.data.bookmarksDB; import de.anomic.http.httpRequestHeader; +import de.anomic.kelondro.kelondroDigest; import de.anomic.plasma.plasmaSwitchboard; -import de.anomic.server.serverCodings; import de.anomic.server.serverDate; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -62,7 +62,7 @@ public class all { prop.putXML("posts_"+count+"_url", bookmark.getUrl()); prop.putXML("posts_"+count+"_title", bookmark.getTitle()); prop.putXML("posts_"+count+"_description", bookmark.getDescription()); - prop.putXML("posts_"+count+"_md5", serverCodings.encodeMD5Hex(bookmark.getUrl())); + prop.putXML("posts_"+count+"_md5", kelondroDigest.encodeMD5Hex(bookmark.getUrl())); date=new Date(bookmark.getTimeStamp()); prop.putXML("posts_"+count+"_time", serverDate.formatISO8601(date)); prop.putXML("posts_"+count+"_tags", bookmark.getTagsString().replaceAll(","," ")); diff --git a/htroot/xml/bookmarks/posts/get.java b/htroot/xml/bookmarks/posts/get.java index fd893a001..756face1a 100644 --- a/htroot/xml/bookmarks/posts/get.java +++ b/htroot/xml/bookmarks/posts/get.java @@ -26,8 +26,8 @@ import java.util.Iterator; import de.anomic.data.bookmarksDB; import de.anomic.http.httpRequestHeader; +import de.anomic.kelondro.kelondroDigest; import de.anomic.plasma.plasmaSwitchboard; -import de.anomic.server.serverCodings; import de.anomic.server.serverDate; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -74,7 +74,7 @@ public class get { prop.putHTML("posts_"+count+"_url", bookmark.getUrl()); prop.putHTML("posts_"+count+"_title", bookmark.getTitle()); prop.putHTML("posts_"+count+"_description", bookmark.getDescription()); - prop.put("posts_"+count+"_md5", serverCodings.encodeMD5Hex(bookmark.getUrl())); + prop.put("posts_"+count+"_md5", kelondroDigest.encodeMD5Hex(bookmark.getUrl())); prop.put("posts_"+count+"_time", date); prop.putHTML("posts_"+count+"_tags", bookmark.getTagsString().replaceAll(","," ")); diff --git a/htroot/yacy/transfer.java b/htroot/yacy/transfer.java index 039420c7c..03b4f8527 100644 --- a/htroot/yacy/transfer.java +++ b/htroot/yacy/transfer.java @@ -29,9 +29,9 @@ import java.io.IOException; import de.anomic.http.httpRequestHeader; import de.anomic.kelondro.kelondroBase64Order; +import de.anomic.kelondro.kelondroDigest; import de.anomic.plasma.plasmaRankingDistribution; import de.anomic.plasma.plasmaSwitchboard; -import de.anomic.server.serverCodings; import de.anomic.server.serverFileUtils; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -85,14 +85,14 @@ public final class transfer { // consolidation of cr files //System.out.println("yacy/transfer:post=" + post.toString()); //String cansendprotocol = (String) post.get("can-send-protocol", "http"); - final String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(otherpeer + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis())); + final String access = kelondroBase64Order.enhancedCoder.encode(kelondroDigest.encodeMD5Raw(otherpeer + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(kelondroDigest.encodeMD5Raw("" + System.currentTimeMillis())); prop.put("response", "ok"); prop.put("process_access", access); prop.put("process_address", sb.webIndex.seedDB.mySeed().getPublicAddress()); prop.put("process_protocol", "http"); prop.put("process_path", ""); // currently empty; the store process will find a path prop.put("process_maxsize", "-1"); // if response is too big we return the size of the file - sb.rankingPermissions.put(serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)), filename); + sb.rankingPermissions.put(kelondroDigest.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)), filename); if (sb.getLog().isFine()) sb.getLog().logFine("RankingTransmission: granted peer " + otherpeerName + " to send CR file " + filename); } return prop; @@ -120,7 +120,7 @@ public final class transfer { try { if (file.getCanonicalPath().startsWith(path.getCanonicalPath())){ serverFileUtils.copy(fileString.getBytes(), file); - final String md5t = serverCodings.encodeMD5Hex(file); + final String md5t = kelondroDigest.encodeMD5Hex(file); if (md5t.equals(md5)) { prop.put("response", "ok"); if (sb.getLog().isFine()) sb.getLog().logFine("RankingTransmission: received from peer " + otherpeerName + " CR file " + filename); diff --git a/source/de/anomic/crawler/CrawlProfile.java b/source/de/anomic/crawler/CrawlProfile.java index d14e6d2bd..37dbfe137 100644 --- a/source/de/anomic/crawler/CrawlProfile.java +++ b/source/de/anomic/crawler/CrawlProfile.java @@ -35,10 +35,10 @@ import de.anomic.kelondro.kelondroBLOB; import de.anomic.kelondro.kelondroBLOBHeap; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroCloneableIterator; +import de.anomic.kelondro.kelondroDigest; import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroMap; import de.anomic.kelondro.kelondroNaturalOrder; -import de.anomic.server.serverCodings; import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.yacyURL; @@ -295,7 +295,7 @@ public class CrawlProfile { final boolean remoteIndexing, final boolean xsstopw, final boolean xdstopw, final boolean xpstopw) { if (name == null || name.length() == 0) throw new NullPointerException("name must not be null"); - final String handle = (startURL == null) ? kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(0, yacySeedDB.commonHashLength) : startURL.hash(); + final String handle = (startURL == null) ? kelondroBase64Order.enhancedCoder.encode(kelondroDigest.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(0, yacySeedDB.commonHashLength) : startURL.hash(); mem = new HashMap(); mem.put(HANDLE, handle); mem.put(NAME, name); diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index 92b8dfbf4..9e3cf8028 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -39,10 +39,10 @@ import de.anomic.http.httpRequestHeader; import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroCloneableIterator; +import de.anomic.kelondro.kelondroDigest; import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroMap; import de.anomic.kelondro.kelondroNaturalOrder; -import de.anomic.server.serverCodings; public final class userDB { @@ -197,7 +197,7 @@ public final class userDB { } public Entry passwordAuth(final String user, final String password){ final Entry entry=this.getEntry(user); - if( entry != null && entry.getMD5EncodedUserPwd().equals(serverCodings.encodeMD5Hex(user+":"+password)) ){ + if( entry != null && entry.getMD5EncodedUserPwd().equals(kelondroDigest.encodeMD5Hex(user+":"+password)) ){ if(entry.isLoggedOut()){ try{ entry.setProperty(Entry.LOGGED_OUT, "false"); diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java index 21f3ceef2..b1a5b122e 100644 --- a/source/de/anomic/http/httpd.java +++ b/source/de/anomic/http/httpd.java @@ -63,6 +63,7 @@ import org.apache.commons.httpclient.ContentLengthInputStream; import de.anomic.data.userDB; import de.anomic.htmlFilter.htmlFilterCharacterCoding; import de.anomic.kelondro.kelondroBase64Order; +import de.anomic.kelondro.kelondroDigest; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverByteBuffer; import de.anomic.server.serverCodings; @@ -295,7 +296,7 @@ public final class httpd implements serverHandler, Cloneable { //if (authorization.length() < 6) return 1; // no authentication information given final String adminAccountBase64MD5 = sw.getConfig(ADMIN_ACCOUNT_B64MD5, ""); if (adminAccountBase64MD5.length() == 0) return 2; // no password stored - if (adminAccountBase64MD5.equals(serverCodings.encodeMD5Hex(authorization))) return 4; // hard-authenticated, all ok + if (adminAccountBase64MD5.equals(kelondroDigest.encodeMD5Hex(authorization))) return 4; // hard-authenticated, all ok return 1; } @@ -317,7 +318,7 @@ public final class httpd implements serverHandler, Cloneable { this.session.out.write((httpResponseHeader.CONTENT_LENGTH + ": 0\r\n").getBytes()); this.session.out.write("\r\n".getBytes()); return false; - } else if (!this.serverAccountBase64MD5.equals(serverCodings.encodeMD5Hex(auth.trim().substring(6)))) { + } else if (!this.serverAccountBase64MD5.equals(kelondroDigest.encodeMD5Hex(auth.trim().substring(6)))) { // wrong password given: ask for authenticate again log.logInfo("Wrong log-in for account 'server' in HTTPD.GET " + this.prop.getProperty("PATH") + " from IP " + this.clientIP); this.session.out.write((httpVersion + " 401 log-in required" + serverCore.CRLF_STRING + diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index d70125d6a..825f34108 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -42,9 +42,7 @@ package de.anomic.http; -import java.io.BufferedInputStream; import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -52,9 +50,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; -import java.io.OutputStreamWriter; import java.io.PrintWriter; -import java.io.Writer; import java.net.BindException; import java.net.ConnectException; import java.net.InetAddress; @@ -64,13 +60,11 @@ import java.net.Socket; import java.net.SocketException; import java.net.SocketTimeoutException; import java.net.UnknownHostException; -import java.nio.charset.Charset; import java.util.Arrays; import java.util.Date; import java.util.HashSet; import java.util.Iterator; import java.util.Properties; -import java.util.Set; import java.util.logging.FileHandler; import java.util.logging.Level; import java.util.logging.LogManager; diff --git a/source/de/anomic/index/indexURLReference.java b/source/de/anomic/index/indexURLReference.java index f767637c0..d7b1c5cb4 100644 --- a/source/de/anomic/index/indexURLReference.java +++ b/source/de/anomic/index/indexURLReference.java @@ -39,6 +39,7 @@ import de.anomic.kelondro.kelondroBitfield; import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroRow; +import de.anomic.kelondro.kelondroDigest; import de.anomic.plasma.plasmaSearchQuery; import de.anomic.server.serverCharBuffer; import de.anomic.server.serverCodings; @@ -242,7 +243,7 @@ public class indexURLReference { } catch (UnsupportedEncodingException e1) { this.entry.setCol(col_referrer, prop.getProperty("referrer", "").getBytes()); } - this.entry.setCol(col_md5, serverCodings.decodeHex(prop.getProperty("md5", ""))); + this.entry.setCol(col_md5, kelondroDigest.decodeHex(prop.getProperty("md5", ""))); this.entry.setCol(col_size, Integer.parseInt(prop.getProperty("size", "0"))); this.entry.setCol(col_wc, Integer.parseInt(prop.getProperty("wc", "0"))); this.entry.setCol(col_dt, new byte[]{(byte) prop.getProperty("dt", "t").charAt(0)}); @@ -370,7 +371,7 @@ public class indexURLReference { public String md5() { // returns the md5 in hex representation - return serverCodings.encodeHex(entry.getColBytes(col_md5)); + return kelondroDigest.encodeHex(entry.getColBytes(col_md5)); } public char doctype() { diff --git a/source/de/anomic/index/indexWord.java b/source/de/anomic/index/indexWord.java index 185e54d61..be0f8f607 100644 --- a/source/de/anomic/index/indexWord.java +++ b/source/de/anomic/index/indexWord.java @@ -35,7 +35,7 @@ import java.util.TreeSet; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBitfield; -import de.anomic.server.serverCodings; +import de.anomic.kelondro.kelondroDigest; import de.anomic.yacy.yacySeedDB; public class indexWord { @@ -82,7 +82,7 @@ public class indexWord { public static final String word2hash(final String word) { String h = hashCache.get(word); if (h != null) return h; - h = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(word.toLowerCase(Locale.ENGLISH))).substring(0, yacySeedDB.commonHashLength); + h = kelondroBase64Order.enhancedCoder.encode(kelondroDigest.encodeMD5Raw(word.toLowerCase(Locale.ENGLISH))).substring(0, yacySeedDB.commonHashLength); hashCache.put(word, h); // prevent expensive MD5 computation and encoding if (hashCache.size() > 100000) hashCache.clear(); // prevent memory laeak return h; diff --git a/source/de/anomic/kelondro/kelondroBytesIntMap.java b/source/de/anomic/kelondro/kelondroBytesIntMap.java index 3966e898e..d5a10ccfd 100644 --- a/source/de/anomic/kelondro/kelondroBytesIntMap.java +++ b/source/de/anomic/kelondro/kelondroBytesIntMap.java @@ -40,13 +40,6 @@ public class kelondroBytesIntMap { private final kelondroRow rowdef; private kelondroIndex index; - public kelondroBytesIntMap(final kelondroIndex ki) { - assert (ki.row().columns() == 2); // must be a key/index relation - assert (ki.row().width(1) == 4); // the value must be a b256-encoded int, 4 bytes long - this.index = ki; - this.rowdef = ki.row(); - } - public kelondroBytesIntMap(final int keylength, final kelondroByteOrder objectOrder, final int space) { this.rowdef = new kelondroRow(new kelondroColumn[]{new kelondroColumn("key", kelondroColumn.celltype_binary, kelondroColumn.encoder_bytes, keylength, "key"), new kelondroColumn("int c-4 {b256}")}, objectOrder, 0); this.index = new kelondroRAMIndex(rowdef, space); diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index fab7cd712..a040b907c 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -42,7 +42,6 @@ import java.util.TimeZone; import de.anomic.index.indexContainer; import de.anomic.kelondro.kelondroRow.EntryIndex; -import de.anomic.server.serverCodings; import de.anomic.server.serverFileUtils; import de.anomic.server.serverMemory; import de.anomic.server.logging.serverLog; @@ -627,7 +626,7 @@ public class kelondroCollectionIndex { newcommon.sort(); final SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss"); formatter.setTimeZone(TimeZone.getTimeZone("GMT")); - final String filename = serverCodings.encodeHex(kelondroBase64Order.enhancedCoder.decode(new String(key), "de.anomic.kelondro.kelondroCollectionIndex.shrinkCollection(...)")) + "_" + formatter.format(new Date()) + ".collection"; + final String filename = kelondroDigest.encodeHex(kelondroBase64Order.enhancedCoder.decode(new String(key), "de.anomic.kelondro.kelondroCollectionIndex.shrinkCollection(...)")) + "_" + formatter.format(new Date()) + ".collection"; final File storagePath = new File(commonsPath, filename.substring(0, 2)); // make a subpath storagePath.mkdirs(); final File file = new File(storagePath, filename); diff --git a/source/de/anomic/kelondro/kelondroDigest.java b/source/de/anomic/kelondro/kelondroDigest.java new file mode 100644 index 000000000..30307b0e9 --- /dev/null +++ b/source/de/anomic/kelondro/kelondroDigest.java @@ -0,0 +1,333 @@ +// kelondroDigest.java +// ----------------------- +// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 28.12.2008 on http://yacy.net +// this uses methods that had been implemented in serverCodings +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.kelondro; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.io.UnsupportedEncodingException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; + +public class kelondroDigest { + + public static String encodeHex(final long in, final int length) { + String s = Long.toHexString(in); + while (s.length() < length) s = "0" + s; + return s; + } + + public static String encodeOctal(final byte[] in) { + if (in == null) return ""; + final StringBuilder result = new StringBuilder(in.length * 8 / 3); + for (int i = 0; i < in.length; i++) { + if ((0Xff & in[i]) < 8) result.append('0'); + result.append(Integer.toOctalString(0Xff & in[i])); + } + return new String(result); + } + + public static String encodeHex(final byte[] in) { + if (in == null) return ""; + final StringBuilder result = new StringBuilder(in.length * 2); + for (int i = 0; i < in.length; i++) { + if ((0Xff & in[i]) < 16) result.append('0'); + result.append(Integer.toHexString(0Xff & in[i])); + } + return new String(result); + } + + public static byte[] decodeHex(final String hex) { + final byte[] result = new byte[hex.length() / 2]; + for (int i = 0; i < result.length; i++) { + result[i] = (byte) (16 * Integer.parseInt(hex.charAt(i * 2) + "", 16) + Integer.parseInt(hex.charAt(i * 2 + 1) + "", 16)); + } + return result; + } + + public static String encodeMD5Hex(final String key) { + // generate a hex representation from the md5 of a string + return encodeHex(encodeMD5Raw(key)); + } + + public static String encodeMD5Hex(final File file) { + // generate a hex representation from the md5 of a file + return encodeHex(encodeMD5Raw(file)); + } + + public static String encodeMD5Hex(final byte[] b) { + // generate a hex representation from the md5 of a byte-array + return encodeHex(encodeMD5Raw(b)); + } + + public static byte[] encodeMD5Raw(final String key) { + try { + final MessageDigest digest = MessageDigest.getInstance("MD5"); + digest.reset(); + byte[] keyBytes; + try { + keyBytes = key.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + keyBytes = key.getBytes(); + } + digest.update(keyBytes); + return digest.digest(); + } catch (final java.security.NoSuchAlgorithmException e) { + System.out.println("Internal Error at md5:" + e.getMessage()); + } + return null; + } + + public static byte[] encodeMD5Raw(final File file) { + FileInputStream in; + try { + in = new FileInputStream(file); + } catch (final java.io.FileNotFoundException e) { + System.out.println("file not found:" + file.toString()); + e.printStackTrace(); + return null; + } + + // create a concurrent thread that consumes data as it is read + // and computes the md5 while doing IO + md5FilechunkConsumer md5consumer = new md5FilechunkConsumer(1024 * 64, 8); + ExecutorService service = Executors.newSingleThreadExecutor(); + Future md5result = service.submit(md5consumer); + service.shutdown(); + + filechunk c; + try { + while (true) { + c = md5consumer.nextFree(); + c.n = in.read(c.b); + if (c.n <= 0) break; + md5consumer.consume(c); + } + in.close(); + } catch (final IOException e) { + System.out.println("file error with " + file.toString() + ": " + e.getMessage()); + md5consumer.consume(md5FilechunkConsumer.poison); + return null; + } + // put in poison into queue to tell the consumer to stop + md5consumer.consume(md5FilechunkConsumer.poison); + + // return the md5 digest from future task + try { + return md5result.get().digest(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + return null; + } + + private static class filechunk { + public byte[] b; + public int n; + public filechunk(int len) { + b = new byte[len]; + n = 0; + } + } + + private static class md5FilechunkConsumer implements Callable { + + private BlockingQueue empty; + private BlockingQueue filed; + private static filechunk poison = new filechunk(0); + private MessageDigest digest; + + public md5FilechunkConsumer(int bufferSize, int bufferCount) { + empty = new ArrayBlockingQueue(bufferCount); + filed = new LinkedBlockingQueue(); + // fill the empty queue + for (int i = 0; i < bufferCount; i++) empty.add(new filechunk(bufferSize)); + // init digest + try { + digest = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + System.out.println("Internal Error at md5:" + e.getMessage()); + } + digest.reset(); + } + + public void consume(filechunk c) { + try { + filed.put(c); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + public filechunk nextFree() { + try { + return empty.take(); + } catch (InterruptedException e) { + e.printStackTrace(); + return null; + } + } + + public MessageDigest call() { + try { + filechunk c; + while(true) { + c = filed.take(); + if (c == poison) break; + digest.update(c.b, 0, c.n); + empty.put(c); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + return digest; + } + + } + + public static String fastFingerprintHex(final File file, boolean includeDate) { + try { + return encodeHex(fastFingerprintRaw(file, includeDate)); + } catch (IOException e) { + return null; + } + } + + public static String fastFingerprintB64(final File file, boolean includeDate) { + try { + return kelondroBase64Order.enhancedCoder.encode(fastFingerprintRaw(file, includeDate)); + } catch (IOException e) { + return null; + } + } + + + /** + * the fast fingerprint computes a md5-like hash from a given file, + * which is different from a md5 because it does not read the complete file + * but reads only the first and last megabyte of it. In case that the + * file is less or equal of one megabyte, the fast fingerprint is equal + * to the md5. In other cases the fingerprint is computed from a + * array = byte[32k + 8] array, which consists of: + * array[0 .. 16k - 1] = first MB of file + * array[16k .. 32k - 1] = last MB of file + * array[32k .. 32k + 7] = length of file as long + * if the date flag is set, the array is extended to + * array[32k + 8 .. 32k + 15] = lastModified of file as long + * @param file + * @return fingerprint in md5 raw format + * @throws IOException + */ + public static byte[] fastFingerprintRaw(final File file, boolean includeDate) throws IOException { + final int mb = 16 * 1024; + final long fl = file.length(); + if (fl <= 2 * mb) return encodeMD5Raw(file); + RandomAccessFile raf = new RandomAccessFile(file, "r"); + MessageDigest digest; + try { + digest = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + return null; + } + raf.seek(0); + byte[] a = new byte[mb]; + raf.readFully(a, 0, mb - 1); + digest.update(a, 0, mb); + raf.seek(fl - mb); + raf.readFully(a, 0, mb - 1); + digest.update(a, 0, mb); + digest.update(kelondroNaturalOrder.encodeLong(fl, 8), 0, 8); + if (includeDate) digest.update(kelondroNaturalOrder.encodeLong(file.lastModified(), 8), 0, 8); + return digest.digest(); + } + + private static byte[] encodeMD5Raw(final byte[] b) { + try { + final MessageDigest digest = MessageDigest.getInstance("MD5"); + digest.reset(); + final InputStream in = new ByteArrayInputStream(b); + final byte[] buf = new byte[2048]; + int n; + while ((n = in.read(buf)) > 0) digest.update(buf, 0, n); + in.close(); + // now compute the hex-representation of the md5 digest + return digest.digest(); + } catch (final java.security.NoSuchAlgorithmException e) { + System.out.println("Internal Error at md5:" + e.getMessage()); + } catch (final java.io.IOException e) { + System.out.println("byte[] error: " + e.getMessage()); + } + return null; + } + + public static void main(final String[] s) { + // usage example: + // java -classpath classes de.anomic.kelondro.kelondroDigest -md5 DATA/HTCACHE/mediawiki/wikipedia.de.xml + // java -classpath classes de.anomic.kelondro.kelondroDigest -md5 readme.txt + // java -classpath classes de.anomic.kelondro.kelondroDigest -fb64 DATA/HTCACHE/responseHeader.heap + // compare with: + // md5 readme.txt + long start = System.currentTimeMillis(); + + if (s.length == 0) { + System.out.println("usage: -[md5|fingerprint] "); + System.exit(0); + } + + if (s[0].equals("-md5")) { + // generate a md5 from a given file + File f = new File(s[1]); + System.out.println("MD5 (" + f.getName() + ") = " + encodeMD5Hex(f)); + } + + if (s[0].equals("-fhex")) { + // generate a fast fingerprint from a given file + File f = new File(s[1]); + System.out.println("fingerprint hex (" + f.getName() + ") = " + fastFingerprintHex(f, true)); + } + if (s[0].equals("-fb64")) { + // generate a fast fingerprint from a given file + File f = new File(s[1]); + System.out.println("fingerprint b64 (" + f.getName() + ") = " + fastFingerprintB64(f, true)); + } + System.out.println("time: " + (System.currentTimeMillis() - start) + " ms"); + } +} diff --git a/source/de/anomic/kelondro/kelondroEcoTable.java b/source/de/anomic/kelondro/kelondroEcoTable.java index dbf667469..df32085bb 100644 --- a/source/de/anomic/kelondro/kelondroEcoTable.java +++ b/source/de/anomic/kelondro/kelondroEcoTable.java @@ -313,8 +313,10 @@ public class kelondroEcoTable implements kelondroIndex { } public void close() { - file.close(); - file = null; + this.file.close(); + this.file = null; + this.table = null; + this.index = null; } protected void finalize() { diff --git a/source/de/anomic/kelondro/kelondroRAMIndex.java b/source/de/anomic/kelondro/kelondroRAMIndex.java index cbb0b4cad..8a8b1a570 100644 --- a/source/de/anomic/kelondro/kelondroRAMIndex.java +++ b/source/de/anomic/kelondro/kelondroRAMIndex.java @@ -43,6 +43,8 @@ public class kelondroRAMIndex implements kelondroIndex { reset(initialspace); } + // TODO: import and export method to write index completely as-is to file and restore it again + public void clear() { reset(0); } diff --git a/source/de/anomic/plasma/plasmaDHTChunk.java b/source/de/anomic/plasma/plasmaDHTChunk.java index 55434faf5..59d149779 100644 --- a/source/de/anomic/plasma/plasmaDHTChunk.java +++ b/source/de/anomic/plasma/plasmaDHTChunk.java @@ -36,8 +36,8 @@ import de.anomic.index.indexRWIEntry; import de.anomic.index.indexRWIRowEntry; import de.anomic.index.indexURLReference; import de.anomic.kelondro.kelondroBase64Order; +import de.anomic.kelondro.kelondroDigest; import de.anomic.kelondro.kelondroException; -import de.anomic.server.serverCodings; import de.anomic.server.logging.serverLog; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeedDB; @@ -125,7 +125,7 @@ public class plasmaDHTChunk { } public static String selectTransferStart() { - return kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(2, 2 + yacySeedDB.commonHashLength); + return kelondroBase64Order.enhancedCoder.encode(kelondroDigest.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(2, 2 + yacySeedDB.commonHashLength); } private void selectTransferContainers(final String hash, final int mincount, final int maxcount, final int maxtime) throws InterruptedException { diff --git a/source/de/anomic/plasma/plasmaDHTFlush.java b/source/de/anomic/plasma/plasmaDHTFlush.java index 8ccf73d57..248890db5 100644 --- a/source/de/anomic/plasma/plasmaDHTFlush.java +++ b/source/de/anomic/plasma/plasmaDHTFlush.java @@ -24,7 +24,7 @@ package de.anomic.plasma; import de.anomic.kelondro.kelondroBase64Order; -import de.anomic.server.serverCodings; +import de.anomic.kelondro.kelondroDigest; import de.anomic.server.logging.serverLog; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacySeedDB; @@ -163,7 +163,7 @@ public class plasmaDHTFlush extends Thread { if (this.sb.webIndex.size() > 0 && this.delete) { // if there are still words in the index we try it again now if((iteration % 10L) == 0) { // seems to be blocked, try another startpoint - this.startPointHash = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(2, 2 + yacySeedDB.commonHashLength); + this.startPointHash = kelondroBase64Order.enhancedCoder.encode(kelondroDigest.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(2, 2 + yacySeedDB.commonHashLength); } else { this.startPointHash = "AAAAAAAAAAAA"; } diff --git a/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java b/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java index b3be3ec13..727370ac4 100644 --- a/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java +++ b/source/de/anomic/plasma/plasmaRankingRCIEvaluation.java @@ -36,7 +36,7 @@ import java.util.TreeSet; import de.anomic.kelondro.kelondroAttrSeq; import de.anomic.kelondro.kelondroBase64Order; -import de.anomic.server.serverCodings; +import de.anomic.kelondro.kelondroDigest; import de.anomic.server.serverFileUtils; import de.anomic.yacy.yacyURL; @@ -169,7 +169,7 @@ public class plasmaRankingRCIEvaluation { String filename; if (!(tablePath.exists())) tablePath.mkdirs(); for (int i = 0; i < ranking.length - 1; i++) { - filename = "YBR-4-" + serverCodings.encodeHex(i, 2) + ".idx"; + filename = "YBR-4-" + kelondroDigest.encodeHex(i, 2) + ".idx"; serverFileUtils.saveSet(new File(tablePath, filename), "plain", ranking[i], ""); } } diff --git a/source/de/anomic/plasma/plasmaSearchRankingProcess.java b/source/de/anomic/plasma/plasmaSearchRankingProcess.java index 676c7b6da..ca882ad0f 100644 --- a/source/de/anomic/plasma/plasmaSearchRankingProcess.java +++ b/source/de/anomic/plasma/plasmaSearchRankingProcess.java @@ -44,9 +44,9 @@ import de.anomic.index.indexRWIVarEntry; import de.anomic.index.indexURLReference; import de.anomic.index.indexWord; import de.anomic.kelondro.kelondroBinSearch; +import de.anomic.kelondro.kelondroDigest; import de.anomic.kelondro.kelondroMScoreCluster; import de.anomic.kelondro.kelondroSortStack; -import de.anomic.server.serverCodings; import de.anomic.server.serverFileUtils; import de.anomic.server.serverProfiling; import de.anomic.yacy.yacyURL; @@ -400,7 +400,7 @@ public final class plasmaSearchRankingProcess { File f; try { for (int i = 0; i < count; i++) { - ybrName = "YBR-4-" + serverCodings.encodeHex(i, 2) + ".idx"; + ybrName = "YBR-4-" + kelondroDigest.encodeHex(i, 2) + ".idx"; f = new File(rankingPath, ybrName); if (f.exists()) { ybrTables[i] = new kelondroBinSearch(serverFileUtils.read(f), 6); diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index ee507cc1c..308c3d78d 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -143,12 +143,12 @@ import de.anomic.index.indexReferenceBlacklist; import de.anomic.index.indexURLReference; import de.anomic.kelondro.kelondroCache; import de.anomic.kelondro.kelondroCachedRecords; +import de.anomic.kelondro.kelondroDigest; import de.anomic.kelondro.kelondroMSetTools; import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.plasma.parser.ParserException; import de.anomic.server.serverAbstractSwitch; import de.anomic.server.serverBusyThread; -import de.anomic.server.serverCodings; import de.anomic.server.serverCore; import de.anomic.server.serverDate; import de.anomic.server.serverDomains; @@ -1292,7 +1292,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch md5result = service.submit(md5consumer); - service.shutdown(); - - filechunk c; - try { - while (true) { - c = md5consumer.nextFree(); - c.n = in.read(c.b); - if (c.n <= 0) break; - md5consumer.consume(c); - } - in.close(); - } catch (final IOException e) { - System.out.println("file error with " + file.toString() + ": " + e.getMessage()); - md5consumer.consume(md5DataConsumer.poison); - return null; - } - // put in poison into queue to tell the consumer to stop - md5consumer.consume(md5DataConsumer.poison); - - // return the md5 digest from future task - try { - return md5result.get().digest(); - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ExecutionException e) { - e.printStackTrace(); - } - return null; - } - - private static class filechunk { - public byte[] b; - public int n; - public filechunk(int len) { - b = new byte[len]; - n = 0; - } - } - - private static class md5DataConsumer implements Callable { - - private BlockingQueue empty; - private BlockingQueue filed; - private static filechunk poison = new filechunk(0); - private MessageDigest digest; - - public md5DataConsumer(int bufferSize, int bufferCount) { - empty = new ArrayBlockingQueue(bufferCount); - filed = new ArrayBlockingQueue(bufferCount); - // fill the empty queue - for (int i = 0; i < bufferCount; i++) empty.add(new filechunk(bufferSize)); - // init digest - try { - digest = MessageDigest.getInstance("MD5"); - } catch (NoSuchAlgorithmException e) { - System.out.println("Internal Error at md5:" + e.getMessage()); - } - digest.reset(); - } - - public void consume(filechunk c) { - try { - filed.put(c); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - - public filechunk nextFree() { - try { - return empty.take(); - } catch (InterruptedException e) { - e.printStackTrace(); - return null; - } - } - - public MessageDigest call() { - try { - filechunk c; - while(true) { - c = filed.take(); - if (c == poison) break; - digest.update(c.b, 0, c.n); - empty.put(c); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - return digest; - } - - } - - private static byte[] encodeMD5Raw(final byte[] b) { - try { - final MessageDigest digest = MessageDigest.getInstance("MD5"); - digest.reset(); - final InputStream in = new ByteArrayInputStream(b); - final byte[] buf = new byte[2048]; - int n; - while ((n = in.read(buf)) > 0) digest.update(buf, 0, n); - in.close(); - // now compute the hex-representation of the md5 digest - return digest.digest(); - } catch (final java.security.NoSuchAlgorithmException e) { - System.out.println("Internal Error at md5:" + e.getMessage()); - } catch (final java.io.IOException e) { - System.out.println("byte[] error: " + e.getMessage()); - } - return null; - } - public static Properties s2p(final String s) { - final Properties p = new Properties(); - int pos; - final StringTokenizer st = new StringTokenizer(s, ","); - String token; - while (st.hasMoreTokens()) { - token = st.nextToken().trim(); - pos = token.indexOf("="); - if (pos > 0) p.setProperty(token.substring(0, pos).trim(), token.substring(pos + 1).trim()); - } - return p; + final Properties p = new Properties(); + int pos; + final StringTokenizer st = new StringTokenizer(s, ","); + String token; + while (st.hasMoreTokens()) { + token = st.nextToken().trim(); + pos = token.indexOf("="); + if (pos > 0) p.setProperty(token.substring(0, pos).trim(), token.substring(pos + 1).trim()); + } + return p; } public static HashMap string2map(String string, final String separator) { @@ -304,7 +94,7 @@ public final class serverCodings { return new String(buf); } } - + public static Set string2set(String string, final String separator) { // this can be used to parse a Map.toString() into a Map again if (string == null) return null; @@ -332,28 +122,6 @@ public final class serverCodings { if (braces) buf.append("}"); return new String(buf); } - - public static void main(final String[] s) { - if (s.length == 0) { - System.out.println("usage: -[md5|s2m] "); - System.exit(0); - } - if (s[0].equals("-s2m")) { - // generate a b64 decoding from a given string - System.out.println(string2map(s[1], ",").toString()); - } - - // usage example: - // java -classpath classes de.anomic.server.serverCodings -md5 DATA/HTCACHE/mediawiki/wikipedia.de.xml - // java -classpath classes de.anomic.server.serverCodings -md5 readme.txt - // compare with: - // md5 readme.txt - if (s[0].equals("-md5")) { - // generate a b64 decoding from a given string - File f = new File(s[1]); - System.out.println("MD5 (" + f.getName() + ") = " + encodeMD5Hex(f)); - } - } } diff --git a/source/de/anomic/tools/cryptbig.java b/source/de/anomic/tools/cryptbig.java index 46ea635b8..9b2b79da8 100644 --- a/source/de/anomic/tools/cryptbig.java +++ b/source/de/anomic/tools/cryptbig.java @@ -53,7 +53,7 @@ import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.PBEParameterSpec; import de.anomic.kelondro.kelondroBase64Order; -import de.anomic.server.serverCodings; +import de.anomic.kelondro.kelondroDigest; public class cryptbig { @@ -634,7 +634,7 @@ public class cryptbig { if (s[0].equals("-md5")) { // generate a public key from a password that can be used for encryption if (s.length != 2) {help(); System.exit(-1);} - final String md5s = serverCodings.encodeMD5Hex(new File(s[1])); + final String md5s = kelondroDigest.encodeMD5Hex(new File(s[1])); System.out.println(md5s); System.exit(0); } diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 841f59013..870756fc5 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -75,6 +75,7 @@ import de.anomic.index.indexURLReference; import de.anomic.index.indexWord; import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBitfield; +import de.anomic.kelondro.kelondroDigest; import de.anomic.plasma.plasmaSearchRankingProcess; import de.anomic.plasma.plasmaSearchRankingProfile; import de.anomic.plasma.plasmaSnippetCache; @@ -82,7 +83,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboardConstants; import de.anomic.plasma.plasmaWordIndex; import de.anomic.server.serverByteBuffer; -import de.anomic.server.serverCodings; import de.anomic.server.serverCore; import de.anomic.server.serverDomains; import de.anomic.tools.crypt; @@ -757,7 +757,7 @@ public final class yacyClient { post.add(new DefaultCharsetStringPart("process", "store")); post.add(new DefaultCharsetStringPart("purpose", "crcon")); post.add(new DefaultCharsetStringPart("filesize", Long.toString(file.length))); - post.add(new DefaultCharsetStringPart("md5", serverCodings.encodeMD5Hex(file))); + post.add(new DefaultCharsetStringPart("md5", kelondroDigest.encodeMD5Hex(file))); post.add(new DefaultCharsetStringPart("access", access)); post.add(new DefaultCharsetFilePart("filename", new ByteArrayPartSource(filename, file))); @@ -783,7 +783,7 @@ public final class yacyClient { String response = phase1.get("response"); if ((response == null) || (protocol == null) || (access == null)) return "wrong return values from other peer; phase 1"; if (!(response.equals("ok"))) return "remote peer rejected transfer: " + response; - final String accesscode = serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)); + final String accesscode = kelondroDigest.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(access)); if (protocol.equals("http")) { final HashMap phase2 = transferStore(nextaddress, accesscode, filename, file); if (phase2 == null) return "no connection to remote address " + targetAddress + "; phase 2"; diff --git a/source/de/anomic/yacy/yacyNetwork.java b/source/de/anomic/yacy/yacyNetwork.java index 1023b105c..d1c3c449a 100644 --- a/source/de/anomic/yacy/yacyNetwork.java +++ b/source/de/anomic/yacy/yacyNetwork.java @@ -34,9 +34,9 @@ import java.util.List; import org.apache.commons.httpclient.methods.multipart.Part; import de.anomic.http.DefaultCharsetStringPart; +import de.anomic.kelondro.kelondroDigest; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboardConstants; -import de.anomic.server.serverCodings; import de.anomic.server.serverDate; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -64,7 +64,7 @@ public class yacyNetwork { final String salt = post.get("key", ""); final String iam = post.get("iam", ""); final String magic = env.getConfig("network.unit.protocol.request.authentification.essentials", ""); - final String md5 = serverCodings.encodeMD5Hex(salt + iam + magic); + final String md5 = kelondroDigest.encodeMD5Hex(salt + iam + magic); return post.get("magicmd5", "").equals(md5); } @@ -96,7 +96,7 @@ public class yacyNetwork { if (authentificationMethod.equals("salted-magic-sim")) { // generate an authentification essential using the salt, the iam-hash and the network magic final String magic = sb.getConfig("network.unit.protocol.request.authentification.essentials", ""); - final String md5 = serverCodings.encodeMD5Hex(salt + sb.webIndex.seedDB.mySeed().hash + magic); + final String md5 = kelondroDigest.encodeMD5Hex(salt + sb.webIndex.seedDB.mySeed().hash + magic); post.add(new DefaultCharsetStringPart("magicmd5", md5)); } } diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index 8f818f4d4..d18636b4c 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -57,6 +57,7 @@ import java.util.TreeMap; import de.anomic.index.indexWord; import de.anomic.kelondro.kelondroBase64Order; +import de.anomic.kelondro.kelondroDigest; import de.anomic.net.natLib; import de.anomic.server.serverCodings; import de.anomic.server.serverDate; @@ -401,7 +402,7 @@ public class yacySeed implements Cloneable { * @return the octal representation of the given base64 hash */ public static String b64Hash2octalHash(final String b64Hash) { - return serverCodings.encodeOctal(kelondroBase64Order.enhancedCoder.decode(b64Hash, "de.anomic.yacy.yacySeed.b64Hash2octalHash()")); + return kelondroDigest.encodeOctal(kelondroBase64Order.enhancedCoder.decode(b64Hash, "de.anomic.yacy.yacySeed.b64Hash2octalHash()")); } /** @@ -411,7 +412,7 @@ public class yacySeed implements Cloneable { */ public static String b64Hash2hexHash(final String b64Hash) { // the hash string represents 12 * 6 bit = 72 bits. This is too much for a long integer. - return serverCodings.encodeHex(kelondroBase64Order.enhancedCoder.decode(b64Hash, "de.anomic.yacy.yacySeed.b64Hash2hexHash()")); + return kelondroDigest.encodeHex(kelondroBase64Order.enhancedCoder.decode(b64Hash, "de.anomic.yacy.yacySeed.b64Hash2hexHash()")); } /** @@ -419,7 +420,7 @@ public class yacySeed implements Cloneable { * @return the base64 representation of the given hex hash */ public static String hexHash2b64Hash(final String hexHash) { - return kelondroBase64Order.enhancedCoder.encode(serverCodings.decodeHex(hexHash)); + return kelondroBase64Order.enhancedCoder.encode(kelondroDigest.decodeHex(hexHash)); } /** @@ -885,8 +886,8 @@ public class yacySeed implements Cloneable { public static String randomHash() { final String hash = - kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(Long.toString(random.nextLong()))).substring(0, 6) + - kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(Long.toString(random.nextLong()))).substring(0, 6); + kelondroBase64Order.enhancedCoder.encode(kelondroDigest.encodeMD5Raw(Long.toString(random.nextLong()))).substring(0, 6) + + kelondroBase64Order.enhancedCoder.encode(kelondroDigest.encodeMD5Raw(Long.toString(random.nextLong()))).substring(0, 6); return hash; } diff --git a/source/de/anomic/yacy/yacyURL.java b/source/de/anomic/yacy/yacyURL.java index 0752d1ced..994abdf53 100644 --- a/source/de/anomic/yacy/yacyURL.java +++ b/source/de/anomic/yacy/yacyURL.java @@ -36,7 +36,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import de.anomic.kelondro.kelondroBase64Order; -import de.anomic.server.serverCodings; +import de.anomic.kelondro.kelondroDigest; import de.anomic.server.serverDomains; import de.anomic.tools.Punycode; import de.anomic.tools.Punycode.PunycodeException; @@ -711,7 +711,7 @@ public class yacyURL implements Serializable { // combine the attributes final StringBuilder hash = new StringBuilder(12); // form the 'local' part of the hash - hash.append(kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(toNormalform(true, true))).substring(0, 5)); // 5 chars + hash.append(kelondroBase64Order.enhancedCoder.encode(kelondroDigest.encodeMD5Raw(toNormalform(true, true))).substring(0, 5)); // 5 chars hash.append(subdomPortPath(subdom, port, rootpath)); // 1 char // form the 'global' part of the hash hash.append(hosthash5(this.protocol, host, port)); // 5 chars @@ -722,7 +722,7 @@ public class yacyURL implements Serializable { } private static char subdomPortPath(final String subdom, final int port, final String rootpath) { - return kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(subdom + ":" + port + ":" + rootpath)).charAt(0); + return kelondroBase64Order.enhancedCoder.encode(kelondroDigest.encodeMD5Raw(subdom + ":" + port + ":" + rootpath)).charAt(0); } private static final char rootURLFlag0 = subdomPortPath("", 80, ""); @@ -733,7 +733,7 @@ public class yacyURL implements Serializable { } private static final String hosthash5(final String protocol, final String host, final int port) { - return kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(protocol + ":" + host + ":" + port)).substring(0, 5); + return kelondroBase64Order.enhancedCoder.encode(kelondroDigest.encodeMD5Raw(protocol + ":" + host + ":" + port)).substring(0, 5); } /** diff --git a/source/genpw.java b/source/genpw.java index 591da05f0..c93cdae55 100644 --- a/source/genpw.java +++ b/source/genpw.java @@ -3,7 +3,7 @@ import java.io.IOException; import java.io.InputStreamReader; import de.anomic.kelondro.kelondroBase64Order; -import de.anomic.server.serverCodings; +import de.anomic.kelondro.kelondroDigest; // migration.java // ----------------------- @@ -50,6 +50,6 @@ public class genpw { } } - System.out.println(serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(username+":"+password))); + System.out.println(kelondroDigest.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(username+":"+password))); } } diff --git a/source/migration.java b/source/migration.java index 30b077ad5..9b7ea62cf 100644 --- a/source/migration.java +++ b/source/migration.java @@ -25,6 +25,7 @@ import java.util.List; import de.anomic.data.listManager; import de.anomic.http.httpd; import de.anomic.kelondro.kelondroBase64Order; +import de.anomic.kelondro.kelondroDigest; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboardConstants; import de.anomic.server.serverFileUtils; @@ -185,33 +186,33 @@ public class migration { // set preset accounts/passwords String acc; if ((acc = sb.getConfig("serverAccount", "")).length() > 0) { - sb.setConfig("serverAccountBase64MD5", de.anomic.server.serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(acc))); + sb.setConfig("serverAccountBase64MD5", kelondroDigest.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(acc))); sb.setConfig("serverAccount", ""); } if ((acc = sb.getConfig("adminAccount", "")).length() > 0) { - sb.setConfig(httpd.ADMIN_ACCOUNT_B64MD5, de.anomic.server.serverCodings.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(acc))); + sb.setConfig(httpd.ADMIN_ACCOUNT_B64MD5, kelondroDigest.encodeMD5Hex(kelondroBase64Order.standardCoder.encodeString(acc))); sb.setConfig("adminAccount", ""); } // fix unsafe old passwords if ((acc = sb.getConfig("proxyAccountBase64", "")).length() > 0) { - sb.setConfig("proxyAccountBase64MD5", de.anomic.server.serverCodings.encodeMD5Hex(acc)); + sb.setConfig("proxyAccountBase64MD5", kelondroDigest.encodeMD5Hex(acc)); sb.setConfig("proxyAccountBase64", ""); } if ((acc = sb.getConfig("serverAccountBase64", "")).length() > 0) { - sb.setConfig("serverAccountBase64MD5", de.anomic.server.serverCodings.encodeMD5Hex(acc)); + sb.setConfig("serverAccountBase64MD5", kelondroDigest.encodeMD5Hex(acc)); sb.setConfig("serverAccountBase64", ""); } if ((acc = sb.getConfig("adminAccountBase64", "")).length() > 0) { - sb.setConfig(httpd.ADMIN_ACCOUNT_B64MD5, de.anomic.server.serverCodings.encodeMD5Hex(acc)); + sb.setConfig(httpd.ADMIN_ACCOUNT_B64MD5, kelondroDigest.encodeMD5Hex(acc)); sb.setConfig("adminAccountBase64", ""); } if ((acc = sb.getConfig("uploadAccountBase64", "")).length() > 0) { - sb.setConfig("uploadAccountBase64MD5", de.anomic.server.serverCodings.encodeMD5Hex(acc)); + sb.setConfig("uploadAccountBase64MD5", kelondroDigest.encodeMD5Hex(acc)); sb.setConfig("uploadAccountBase64", ""); } if ((acc = sb.getConfig("downloadAccountBase64", "")).length() > 0) { - sb.setConfig("downloadAccountBase64MD5", de.anomic.server.serverCodings.encodeMD5Hex(acc)); + sb.setConfig("downloadAccountBase64MD5", kelondroDigest.encodeMD5Hex(acc)); sb.setConfig("downloadAccountBase64", ""); } }