performance hack to parse numbers inside of substrings without actually

generating a substring. This avoids the allocation of a String object
ech time a substring is parsed. Should affect CPU load during RWI
transmission.
pull/1/head
Michael Peter Christen 13 years ago
parent 7e0ddbd275
commit cf47d94888

@ -37,6 +37,7 @@ import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.services.federated.yacy.CacheStrategy;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.util.NumberTools;
import net.yacy.search.Switchboard;
import net.yacy.search.index.Segment;
import net.yacy.search.index.Segments;
@ -83,7 +84,7 @@ public class QuickCrawlLink_p {
int port = 80;
final int pos = hostSocket.indexOf(':',0);
if (pos != -1) {
port = Integer.parseInt(hostSocket.substring(pos + 1));
port = NumberTools.parseIntDecSubstring(hostSocket, pos + 1);
//host = hostSocket.substring(0, pos);
}

@ -29,6 +29,7 @@
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.util.NumberTools;
import net.yacy.search.Switchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -52,7 +53,7 @@ public class YaCySearchPluginFF {
int port = 80;
final int pos = hostSocket.indexOf(':',0);
if (pos != -1) {
port = Integer.parseInt(hostSocket.substring(pos + 1));
port = NumberTools.parseIntDecSubstring(hostSocket, pos + 1);
host = hostSocket.substring(0, pos);
}

@ -29,6 +29,7 @@
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.kelondro.util.NumberTools;
import net.yacy.search.SwitchboardConstants;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -57,7 +58,7 @@ public class autoconfig {
int port = 80;
final int pos = hostSocket.indexOf(':',0);
if (pos != -1) {
port = Integer.parseInt(hostSocket.substring(pos + 1));
port = NumberTools.parseIntDecSubstring(hostSocket, pos + 1);
host = hostSocket.substring(0, pos);
}

@ -40,6 +40,7 @@ import net.yacy.document.Document;
import net.yacy.document.Parser;
import net.yacy.document.TextParser;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.util.NumberTools;
import de.anomic.crawler.CrawlProfile;
import de.anomic.crawler.ResultURLs.EventOrigin;
@ -349,7 +350,7 @@ public class Response {
final Date date = this.responseHeader.date();
if (date == null) return "stale_no_date_given_in_response";
try {
final long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live
final long ttl = 1000 * NumberTools.parseLongDecSubstring(cacheControl, 8); // milliseconds to live
if (GenericFormatter.correctedUTCTime() - date.getTime() > ttl) {
//System.out.println("***not indexed because cache-control");
return "stale_expired";
@ -517,7 +518,7 @@ public class Response {
// we need also the load date
if (date == null) { return false; }
try {
final long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live
final long ttl = 1000 * NumberTools.parseLongDecSubstring(cacheControl, 8); // milliseconds to live
if (GenericFormatter.correctedUTCTime() - date.getTime() > ttl) {
return false;
}
@ -663,7 +664,7 @@ public class Response {
return "Stale_(no_date_given_in_response)";
}
try {
final long ttl = 1000 * Long.parseLong(cacheControl.substring(8)); // milliseconds to live
final long ttl = 1000 * NumberTools.parseLongDecSubstring(cacheControl,8); // milliseconds to live
if (GenericFormatter.correctedUTCTime() - date.getTime() > ttl) {
//System.out.println("***not indexed because cache-control");
return "Stale_(expired_by_cache-control)";

@ -84,6 +84,7 @@ import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.io.ByteCountOutputStream;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.NumberTools;
import net.yacy.repository.Blacklist;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@ -341,7 +342,7 @@ public final class HTTPDProxyHandler {
if ((pos = host.indexOf(':')) < 0) {
port = 80;
} else {
port = Integer.parseInt(host.substring(pos + 1));
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
}
@ -457,7 +458,7 @@ public final class HTTPDProxyHandler {
if ((pos = host.indexOf(':')) < 0) {
port = 80;
} else {
port = Integer.parseInt(host.substring(pos + 1));
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
}
@ -753,7 +754,7 @@ public final class HTTPDProxyHandler {
if ((pos = host.indexOf(':')) < 0) {
port = 80;
} else {
port = Integer.parseInt(host.substring(pos + 1));
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
}
@ -870,7 +871,7 @@ public final class HTTPDProxyHandler {
if ((pos = host.indexOf(':')) < 0) {
port = 80;
} else {
port = Integer.parseInt(host.substring(pos + 1));
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
}
@ -1196,7 +1197,7 @@ public final class HTTPDProxyHandler {
if ((pos = host.indexOf(':')) < 0) {
port = 80;
} else {
port = Integer.parseInt(host.substring(pos + 1));
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
}

@ -65,6 +65,7 @@ import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.NumberTools;
import net.yacy.search.Switchboard;
import org.apache.commons.fileupload.FileItem;
@ -269,7 +270,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
// default port 80
return false; // not allowed
}
if (Integer.parseInt(host.substring(pos + 1)) == 80) return false;
if (NumberTools.parseIntDecSubstring(host, pos + 1) == 80) return false;
// the access path must be into the yacy protocol path; it must start with 'yacy'
if (!(prop.containsKey(HeaderFramework.CONNECTION_PROP_PATH) && ((String) prop.get(HeaderFramework.CONNECTION_PROP_PATH)).startsWith("/yacy/"))) return false;
@ -596,7 +597,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
pos = arg.indexOf(':');
int port = 443;
if (pos >= 0) {
port = Integer.parseInt(arg.substring(pos + 1));
port = NumberTools.parseIntDecSubstring(arg, pos + 1);
//the offcut: arg = arg.substring(0, pos);
}
@ -755,8 +756,8 @@ public final class HTTPDemon implements serverHandler, Cloneable {
try {
if (s.length() >= pos + 6 && (s.charAt(pos + 1) == 'u' || s.charAt(pos + 1) == 'U')) {
// non-standard encoding of IE for unicode-chars
final int bh = Integer.parseInt(s.substring(pos + 2, pos + 4), 16);
final int bl = Integer.parseInt(s.substring(pos + 4, pos + 6), 16);
final int bh = NumberTools.parseIntDecSubstring(s.substring(pos + 2, pos + 4), 16);
final int bl = NumberTools.parseIntDecSubstring(s.substring(pos + 4, pos + 6), 16);
// TODO: needs conversion from UTF-16 to UTF-8
baos.write(bh);
baos.write(bl);
@ -1054,7 +1055,7 @@ public final class HTTPDemon implements serverHandler, Cloneable {
final int port;
final int pos = host.indexOf(':');
if (pos != -1) {
port = Integer.parseInt(host.substring(pos + 1));
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
} else {
port = 80;

@ -31,6 +31,8 @@ import java.util.Calendar;
import java.util.Date;
import java.util.Locale;
import net.yacy.kelondro.util.NumberTools;
public class GenericFormatter extends AbstractFormatter implements DateFormatter {
public static final String PATTERN_SHORT_DAY = "yyyyMMdd";
@ -93,6 +95,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
}
}
@Override
public String format() {
if (Math.abs(System.currentTimeMillis() - this.last_time) < this.maxCacheDiff) return this.last_format;
// threads that had been waiting here may use the cache now instead of calculating the date again
@ -146,7 +149,7 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
else if (diffString.length() > 0 && diffString.charAt(0) == '-') ahead = false;
else throw new IllegalArgumentException("UTC String malformed (wrong sign):" + diffString);
final long oh = Long.parseLong(diffString.substring(1, 3));
final long om = Long.parseLong(diffString.substring(3));
final long om = NumberTools.parseLongDecSubstring(diffString, 3);
return ((ahead) ? (long) 1 : (long) -1) * (oh * AbstractFormatter.hourMillis + om * AbstractFormatter.minuteMillis);
}

@ -42,6 +42,7 @@ import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.util.NumberTools;
/**
@ -554,7 +555,7 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
if ((pos = host.indexOf(':')) < 0) {
port = 80;
} else {
port = Integer.parseInt(host.substring(pos + 1));
port = NumberTools.parseIntDecSubstring(host, pos + 1);
host = host.substring(0, pos);
}

@ -29,6 +29,7 @@ package net.yacy.kelondro.index;
import java.io.Serializable;
import net.yacy.kelondro.util.NumberTools;
import net.yacy.kelondro.util.kelondroException;
public final class Column implements Cloneable, Serializable {
@ -60,7 +61,7 @@ public final class Column implements Cloneable, Serializable {
this.nickname = nickname;
this.description = description;
}
public Column(String celldef) {
// define column with column syntax
// example: <UDate-3>
@ -134,7 +135,7 @@ public final class Column implements Cloneable, Serializable {
final int q = celldef.indexOf(' ');
if (q < 0) {
try {
this.cellwidth = Integer.parseInt(celldef.substring(p + 1));
this.cellwidth = NumberTools.parseIntDecSubstring(celldef, p + 1);
} catch (final NumberFormatException e) {
throw new kelondroException("kelondroColumn - cellwidth description wrong:" + celldef.substring(p + 1));
}
@ -200,6 +201,7 @@ public final class Column implements Cloneable, Serializable {
* th clone method is useful to produce a similiar column with a different cell width
* @return the cloned Column
*/
@Override
public Object clone() {
return new Column(this.nickname, this.celltype, this.encoder, this.cellwidth, this.description);
}

@ -46,6 +46,7 @@ import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.Bitfield;
import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.util.ByteBuffer;
import net.yacy.kelondro.util.NumberTools;
import net.yacy.kelondro.util.kelondroException;
@ -316,7 +317,7 @@ public final class Row implements Serializable {
} else {
if ((decimalCardinal) && (col.celltype == Column.celltype_cardinal)) {
try {
setCol(col.encoder, this.offset + clstrt, col.cellwidth, Long.parseLong(token.substring(p + 1).trim()));
setCol(col.encoder, this.offset + clstrt, col.cellwidth, NumberTools.parseLongDecSubstring(token, p + 1));
} catch (final NumberFormatException e) {
Log.logSevere("kelondroRow", "NumberFormatException for celltype_cardinal, celltype = " + col.celltype + ", encoder = " + col.encoder + ", value = '" + token.substring(p + 1).trim() + "'");
setCol(col.encoder, this.offset + clstrt, col.cellwidth, 0);
@ -324,7 +325,7 @@ public final class Row implements Serializable {
} else if ((decimalCardinal) && (col.celltype == Column.celltype_binary)) {
assert col.cellwidth == 1;
try {
setCol(clstrt, col.cellwidth, new byte[]{(byte) Integer.parseInt(token.substring(p + 1).trim())});
setCol(clstrt, col.cellwidth, new byte[]{(byte) NumberTools.parseIntDecSubstring(token, p + 1)});
} catch (final NumberFormatException e) {
Log.logSevere("kelondroRow", "NumberFormatException for celltype_binary, celltype = " + col.celltype + ", encoder = " + col.encoder + ", value = '" + token.substring(p + 1).trim() + "'");
setCol(clstrt, col.cellwidth, new byte[]{0});

@ -36,6 +36,7 @@ import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.NaturalOrder;
import net.yacy.kelondro.util.NumberTools;
public class Relations {
@ -59,11 +60,11 @@ public class Relations {
if (p >= 0) filename = filename.substring(0, p);
p = filename.lastIndexOf('-');
assert p >= 0;
final int payloadsize = Integer.parseInt(filename.substring(p + 1));
final int payloadsize = NumberTools.parseIntDecSubstring(filename, p + 1);
filename = filename.substring(0, p);
p = filename.lastIndexOf('-');
assert p >= 0;
final int keysize = Integer.parseInt(filename.substring(p + 1));
final int keysize = NumberTools.parseIntDecSubstring(filename, p + 1);
return rowdef(keysize, payloadsize);
}

@ -0,0 +1,118 @@
/**
* NumberTools
* Copyright 2012 by Sebastian Gaebel
* First released 21.05.2012 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.kelondro.util;
public class NumberTools {
/**
* this method replaces Long.parseLong/2 where a substring of decimal numbers shall be parsed
* Strings are also auto-trimmed, that means parsing stops at spaces without throwing a NumberFormatException
* @param s
* @param startPos
* @return the number
* @throws NumberFormatException
*/
public static final long parseLongDecSubstring(String s, int startPos) throws NumberFormatException {
if (s == null) {
throw new NumberFormatException("null");
}
final int len = s.length();
if (len <= startPos) {
throw new NumberFormatException(s);
}
long result = 0;
boolean negative = false;
int i = startPos;
long limit = -Long.MAX_VALUE;
final long multmin;
int digit;
char c;
char firstChar = s.charAt(i);
if (firstChar < '0') {
if (firstChar == '-') {
negative = true;
limit = Long.MIN_VALUE;
} else if (firstChar != '+') throw new NumberFormatException(s);
if (len == 1) throw new NumberFormatException(s);
i++;
}
multmin = limit / 10;
while (i < len) {
c = s.charAt(i++);
if (c == ' ') break;
digit = c - 48;
if (digit < 0 || digit > 9 || result < multmin) throw new NumberFormatException(s);
result *= 10;
if (result < limit + digit) throw new NumberFormatException(s);
result -= digit;
}
return negative ? result : -result;
}
public static final int parseIntDecSubstring(String s, int startPos) throws NumberFormatException {
if (s == null) {
throw new NumberFormatException("null");
}
final int len = s.length();
if (len <= startPos) {
throw new NumberFormatException(s);
}
int result = 0;
boolean negative = false;
int i = startPos;
int limit = -Integer.MAX_VALUE;
final int multmin;
int digit;
char c;
char firstChar = s.charAt(i);
if (firstChar < '0') {
if (firstChar == '-') {
negative = true;
limit = Integer.MIN_VALUE;
} else if (firstChar != '+') throw new NumberFormatException(s);
if (len == 1) throw new NumberFormatException(s);
i++;
}
multmin = limit / 10;
while (i < len) {
c = s.charAt(i++);
if (c == ' ') break;
digit = c - 48;
if (digit < 0 || digit > 9 || result < multmin) throw new NumberFormatException(s);
result *= 10;
if (result < limit + digit) throw new NumberFormatException(s);
result -= digit;
}
return negative ? result : -result;
}
public static void main(String[] args) {
System.out.println("the number is " + parseLongDecSubstring("number=78 ", 7));
}
}

@ -33,6 +33,7 @@ import java.util.Map;
import net.yacy.cora.document.Classification;
import net.yacy.cora.document.Classification.ContentDomain;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.util.NumberTools;
public class RankingProfile {
@ -145,7 +146,7 @@ public class RankingProfile {
p = e.indexOf('=');
if (p < 0) System.out.println("DEBUG: bug in plasmaSearchRankingProfile: e = " + e);
if ((p > 0) && (e.length() > p + 1)) try {
coeff.put(e.substring(s, p), Integer.valueOf(Integer.parseInt(e.substring(p + 1))));
coeff.put(e.substring(s, p), Integer.valueOf(NumberTools.parseIntDecSubstring(e, p + 1)));
} catch (final NumberFormatException e1) {
System.out.println("wrong parameter: " + e.substring(s, p) + "=" + e.substring(p + 1));
Log.logException(e1);

@ -50,6 +50,7 @@ import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.util.ByteArray;
import net.yacy.kelondro.util.NumberTools;
import net.yacy.repository.Blacklist;
import net.yacy.search.Switchboard;
import de.anomic.crawler.ZURL.FailCategory;
@ -77,7 +78,7 @@ public class MediaSnippet implements Comparable<MediaSnippet>, Comparator<MediaS
int p = 0;
if (attr != null && (p = attr.indexOf(" x ",0)) > 0) {
this.width = Integer.parseInt(attr.substring(0, p).trim());
this.height = Integer.parseInt(attr.substring(p + 3).trim());
this.height = NumberTools.parseIntDecSubstring(attr, p + 3);
}
this.ranking = ranking; // the smaller the better! small values should be shown first
if ((this.name == null) || (this.name.length() == 0)) this.name = "_";

Loading…
Cancel
Save