boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Enhanced Base64Order transformation: less overhead (transformation between StringBuilder and byte[])

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7523 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent f95e50ec3d
commit ad7fcb9d61
9 changed files with 46 additions and 47 deletions
Show Stats Download Patch File Download Diff File

2
source/de/anomic/tools/crypt.java
Unescape View File

@ -47,7 +47,7 @@ public class crypt {
saltcounter++;
// we generate 48-bit salt values, that are represented as 8-character
// b64-encoded strings
return Base64Order.standardCoder.encodeLong(salt & 0XffffffffffffL, 8);
return Base64Order.standardCoder.encodeLongSB(salt & 0XffffffffffffL, 8).toString();
}
// --------------------------------------------------------

10
source/de/anomic/tools/cryptbig.java
Unescape View File

@ -72,7 +72,7 @@ public class cryptbig {
saltcounter++;
// we generate 48-bit salt values, that are represented as 8-character
// b64-encoded strings
return Base64Order.standardCoder.encodeLong(salt & 0XffffffffffffL, 8);
return Base64Order.standardCoder.encodeLongSB(salt & 0XffffffffffffL, 8).toString();
}
// --------------------------------------------------------
@ -234,7 +234,7 @@ public class cryptbig {
final File inFile = new File(inFileName);
final String inFileDate = dateFormatter.format(new Date(inFile.lastModified())); // 17 byte
final String encryptionDate = dateFormatter.format(new Date()); // 17 byte
final String inFileSize = Base64Order.standardCoder.encodeLong(inFile.length(), 11); // 64 / 6 = 11; 11 byte
final String inFileSize = Base64Order.standardCoder.encodeLongSB(inFile.length(), 11).toString(); // 64 / 6 = 11; 11 byte
final String flag = "1"; // 1 byte
//int inFileNameLength = inFileName.length(); // 256
final String X = inFileDate + encryptionDate + inFileSize + flag + inFileName;
@ -259,8 +259,8 @@ public class cryptbig {
// - after the magic String we write C, B and A
try {
final String A = new String(ecipher.doFinal(X.getBytes("UTF8")));
final String B = new String(ecipher.doFinal(Base64Order.standardCoder.encodeLong(A.length(), 2).getBytes("UTF8"))); // most probable not longer than 4
final String C = Base64Order.standardCoder.encodeLong(B.length(), 1); // fixed length 1 (6 bits, that should be enough)
final String B = new String(ecipher.doFinal(Base64Order.standardCoder.encodeLongSB(A.length(), 2).toString().getBytes("UTF8"))); // most probable not longer than 4
final String C = Base64Order.standardCoder.encodeLongSB(B.length(), 1).toString(); // fixed length 1 (6 bits, that should be enough)
fout.write(magicString.getBytes()); // the magic string, used to identify a 'crypt'-file
fout.write(C.getBytes());
fout.write(B.getBytes());
@ -581,7 +581,7 @@ public class cryptbig {
if (s[0].equals("-ec64")) {
// generate a b64 encoding from a given cardinal
if (s.length != 2) {help(); System.exit(-1);}
System.out.println(Base64Order.standardCoder.encodeLong(Long.parseLong(s[1]), 0));
System.out.println(Base64Order.standardCoder.encodeLongSB(Long.parseLong(s[1]), 0).toString());
System.exit(0);
}
if (s[0].equals("-dc64")) {

2
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -181,7 +181,7 @@ public class yacyNewsDB {
entry.setCol(0, r.id().getBytes());
entry.setCol(1, r.category().getBytes("UTF-8"));
entry.setCol(2, (r.received() == null) ? null : GenericFormatter.SHORT_SECOND_FORMATTER.format(r.received()).getBytes());
entry.setCol(3, Base64Order.enhancedCoder.encodeLong(r.distributed(), 2).getBytes());
entry.setCol(3, Base64Order.enhancedCoder.encodeLongBA(r.distributed(), 2));
entry.setCol(4, attributes.getBytes("UTF-8"));
return entry;
} catch(final UnsupportedEncodingException e) {

4
source/net/yacy/dbtest.java
Unescape View File

@ -41,8 +41,8 @@ public class dbtest {
public static byte[] randomHash(final long r0, final long r1) {
// a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits
// so we construct a generic Hash using two long values
final String s = (Base64Order.enhancedCoder.encodeLong(Math.abs(r0), 6) +
Base64Order.enhancedCoder.encodeLong(Math.abs(r1), 6));
final String s = (Base64Order.enhancedCoder.encodeLongSB(Math.abs(r0), 6).toString() +
Base64Order.enhancedCoder.encodeLongSB(Math.abs(r1), 6).toString());
return s.getBytes();
}

4
source/net/yacy/kelondro/index/IndexTest.java
Unescape View File

@ -46,8 +46,8 @@ public class IndexTest {
public static byte[] randomHash(final long r0, final long r1) {
// a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits
// so we construct a generic Hash using two long values
final String s = (Base64Order.enhancedCoder.encodeLong(Math.abs(r0), 6) +
Base64Order.enhancedCoder.encodeLong(Math.abs(r1), 6));
final String s = (Base64Order.enhancedCoder.encodeLongSB(Math.abs(r0), 6).toString() +
Base64Order.enhancedCoder.encodeLongSB(Math.abs(r1), 6).toString());
return s.getBytes();
}

6
source/net/yacy/kelondro/index/RowCollection.java
Unescape View File

@ -1090,9 +1090,9 @@ public class RowCollection implements Iterable<Row.Entry>, Cloneable {
private static Random random = null;
private static String randomHash() {
return
Base64Order.enhancedCoder.encodeLong(random.nextLong(), 4) +
Base64Order.enhancedCoder.encodeLong(random.nextLong(), 4) +
Base64Order.enhancedCoder.encodeLong(random.nextLong(), 4);
Base64Order.enhancedCoder.encodeLongSB(random.nextLong(), 4).toString() +
Base64Order.enhancedCoder.encodeLongSB(random.nextLong(), 4).toString() +
Base64Order.enhancedCoder.encodeLongSB(random.nextLong(), 4).toString();
}
public static void test(final int testsize) throws RowSpaceExceededException {

4
source/net/yacy/kelondro/index/RowSet.java
Unescape View File

@ -659,8 +659,8 @@ public class RowSet extends RowCollection implements Index, Iterable<Row.Entry>
public static byte[] randomHash(final long r0, final long r1) {
// a long can have 64 bit, but a 12-byte hash can have 6 * 12 = 72 bits
// so we construct a generic Hash using two long values
return (Base64Order.enhancedCoder.encodeLong(Math.abs(r0), 11).substring(5) +
Base64Order.enhancedCoder.encodeLong(Math.abs(r1), 11).substring(5)).getBytes();
return (Base64Order.enhancedCoder.encodeLongSB(Math.abs(r0), 11).substring(5) +
Base64Order.enhancedCoder.encodeLongSB(Math.abs(r1), 11).substring(5)).getBytes();
}
public static byte[] randomHash(final Random r) {
return randomHash(r.nextLong(), r.nextLong());

59
source/net/yacy/kelondro/order/Base64Order.java
Unescape View File

@ -37,8 +37,8 @@ import net.yacy.kelondro.logging.Log;
public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Comparator<byte[]>, Cloneable {
protected static final char[] alpha_standard = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray();
protected static final char[] alpha_enhanced = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".toCharArray();
protected static final byte[] alpha_standard = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".getBytes();
protected static final byte[] alpha_enhanced = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".getBytes();
protected static final byte[] ahpla_standard = new byte[128];
protected static final byte[] ahpla_enhanced = new byte[128];
@ -59,7 +59,7 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
public static final Base64Order enhancedCoder = new Base64Order(true, false);
private final boolean rfc1113compliant;
private final char[] alpha;
private final byte[] alpha;
private final byte[] ahpla;
private final byte[] ab; // decision table for comparisments
@ -74,8 +74,8 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
byte acc, bcc;
byte c;
// pre-compute comparisment results: this omits one single ahpla lookup during comparisment
for (final char ac: alpha) {
for (final char bc: alpha) {
for (final byte ac: alpha) {
for (final byte bc: alpha) {
acc = ahpla[ac];
bcc = ahpla[bc];
c = 0;
@ -95,7 +95,7 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
public static byte[] zero(int length) {
final byte[] z = new byte[length];
while (length > 0) {
length--; z[length] = (byte) alpha_standard[0];
length--; z[length] = alpha_standard[0];
}
return z;
}
@ -137,7 +137,7 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
}
public final char encodeByte(final byte b) {
return alpha[b];
return (char) alpha[b];
}
public final byte decodeByte(final byte b) {
@ -148,30 +148,29 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
return ahpla[b];
}
public final String encodeLongSmart(final long c, int length) {
if (c >= max(length)) {
final StringBuilder s = new StringBuilder(length);
s.setLength(length);
while (length > 0) s.setCharAt(--length, alpha[63]);
return new String(s);
}
return encodeLong(c, length);
}
public final String encodeLong(long c, int length) {
public final StringBuilder encodeLongSB(long c, int length) {
final StringBuilder s = new StringBuilder(length);
s.setLength(length);
while (length > 0) {
s.setCharAt(--length, alpha[(byte) (c & 0x3F)]);
s.setCharAt(--length, (char) alpha[(byte) (c & 0x3F)]);
c >>= 6;
}
return s;
}
public final byte[] encodeLongBA(long c, int length) {
byte[] s = new byte[length];
while (length > 0) {
s[--length] = alpha[(byte) (c & 0x3F)];
c >>= 6;
}
return new String(s);
return s;
}
public final byte[] encodeLongSubstr(long c, int length) {
final byte[] s = new byte[length];
while (length > 0) {
s[--length] = (byte) alpha[(byte) (c & 0x3F)];
s[--length] = alpha[(byte) (c & 0x3F)];
c >>= 6;
}
return s;
@ -180,7 +179,7 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
public final void encodeLong(long c, final byte[] b, final int offset, int length) {
assert offset + length <= b.length;
while (length > 0) {
b[--length + offset] = (byte) alpha[(byte) (c & 0x3F)];
b[--length + offset] = alpha[(byte) (c & 0x3F)];
c >>= 6;
}
}
@ -227,10 +226,10 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
while (in.length - pos >= 3) {
l = ((((0XffL & in[pos]) << 8) + (0XffL & in[pos + 1])) << 8) + (0XffL & in[pos + 2]);
pos += 3;
out = out.append(encodeLong(l, 4));
out = out.append(encodeLongSB(l, 4));
}
// now there may be remaining bytes
if (in.length % 3 != 0) out = out.append((in.length % 3 == 2) ? encodeLong((((0XffL & in[pos]) << 8) + (0XffL & in[pos + 1])) << 8, 4).substring(0, 3) : encodeLong((((0XffL & in[pos])) << 8) << 8, 4).substring(0, 2));
if (in.length % 3 != 0) out = out.append((in.length % 3 == 2) ? encodeLongSB((((0XffL & in[pos]) << 8) + (0XffL & in[pos + 1])) << 8, 4).substring(0, 3) : encodeLongSB((((0XffL & in[pos])) << 8) << 8, 4).substring(0, 2));
if (rfc1113compliant) while (out.length() % 4 > 0) out.append("=");
// return result
//assert lene == out.length() : "lene = " + lene + ", out.len = " + out.length();
@ -252,10 +251,10 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
// now there may be remaining bytes
if (in.length % 3 != 0 && writepos < sublen) {
if (in.length % 3 == 2) {
System.arraycopy(encodeLong((((0XffL & in[pos]) << 8) + (0XffL & in[pos + 1])) << 8, 4).getBytes(), 0, out, writepos, 3);
System.arraycopy(encodeLongBA((((0XffL & in[pos]) << 8) + (0XffL & in[pos + 1])) << 8, 4), 0, out, writepos, 3);
writepos += 3;
} else {
System.arraycopy(encodeLong((((0XffL & in[pos])) << 8) << 8, 4).substring(0, 2).getBytes(), 0, out, writepos, 2);
System.arraycopy(encodeLongBA((((0XffL & in[pos])) << 8) << 8, 4), 0, out, writepos, 2);
writepos += 2;
}
}
@ -357,11 +356,11 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
c = c >> 3;
byte[] b = new byte[12];
for (int p = 9; p >= 0; p--) {
b[p] = (byte) alpha[(int) (c & 0x3fL)];
b[p] = alpha[(int) (c & 0x3fL)];
c = c >> 6;
}
b[10] = (byte) alpha[0x3f];
b[11] = (byte) alpha[0x3f];
b[10] = alpha[0x3f];
b[11] = alpha[0x3f];
return b;
}
@ -537,7 +536,7 @@ public class Base64Order extends AbstractOrder<byte[]> implements ByteOrder, Com
}
if ("-ec".equals(s[0])) {
// generate a b64 encoding from a given cardinal
System.out.println(b64.encodeLong(Long.parseLong(s[1]), 4));
System.out.println(b64.encodeLongSB(Long.parseLong(s[1]), 4));
}
if ("-dc".equals(s[0])) {
// generate a b64 decoding from a given cardinal

2
source/net/yacy/kelondro/order/MicroDate.java
Unescape View File

@ -48,7 +48,7 @@ public class MicroDate {
}
public static String microDateHoursStr(final long time) {
return Base64Order.enhancedCoder.encodeLong(microDateHoursInt(time), 3);
return Base64Order.enhancedCoder.encodeLongSB(microDateHoursInt(time), 3).toString();
}
public static int microDateHoursInt(final long time) {

Write Preview
Loading…
Cancel
Save