boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- enhanced date formatter cache

Browse Source 
- added more instances of formatter objects to different classes to make them independent in case of lockings that may applay during synchronization of the date formatter object (date formatting is not thread-safe and must be synchronized therefore)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7528 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent 48a61c39a3
commit 19b2a50578
9 changed files with 66 additions and 38 deletions
Show Stats Download Patch File Download Diff File

9
source/de/anomic/yacy/yacyCore.java
Unescape View File

@ -210,6 +210,9 @@ public class yacyCore {
/*final int newSeeds =*/ publishMySeed(true);
return (oldAddress != null && oldAddress.equals(sb.peers.mySeed().getPublicAddress()));
}
// use our own formatter to prevent concurrency locks with other processes
private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND);
protected class publishThread extends Thread {
int added;
@ -255,14 +258,14 @@ public class yacyCore {
if (newSeed.getLastSeenUTC() >= this.seed.getLastSeenUTC()) {
if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) +
" peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " with old LastSeen: '" +
GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "'");
my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "'");
newSeed.setLastSeenUTC();
sb.peers.peerActions.peerArrival(newSeed, true);
} else {
if (log.isFine()) log.logFine("publish: recently handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) +
" peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress() + " with old LastSeen: '" +
GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "', this is more recent: '" +
GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(this.seed.getLastSeenUTC())) + "'");
my_SHORT_SECOND_FORMATTER.format(new Date(newSeed.getLastSeenUTC())) + "', this is more recent: '" +
my_SHORT_SECOND_FORMATTER.format(new Date(this.seed.getLastSeenUTC())) + "'");
this.seed.setLastSeenUTC();
sb.peers.peerActions.peerArrival(this.seed, true);
}

4
source/de/anomic/yacy/yacyNetwork.java
Unescape View File

@ -101,6 +101,8 @@ public class yacyNetwork {
return parts;
}
// use our own formatter to prevent concurrency locks with other processes
private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND);
public static final LinkedHashMap<String,ContentBody> basicRequestParts(String myHash, String targetHash, String networkName) {
// put in all the essentials for routing and network authentication
@ -114,7 +116,7 @@ public class yacyNetwork {
if (targetHash != null) parts.put("youare", new StringBody(targetHash));
// time information for synchronization
parts.put("mytime", new StringBody(GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date())));
parts.put("mytime", new StringBody(my_SHORT_SECOND_FORMATTER.format(new Date())));
parts.put("myUTC", new StringBody(Long.toString(System.currentTimeMillis())));
// network identification

19
source/de/anomic/yacy/yacyNewsDB.java
Unescape View File

@ -157,13 +157,16 @@ public class yacyNewsDB {
return null;
}
}
// use our own formatter to prevent concurrency locks with other processes
private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND);
private Record b2r(final Row.Entry b) {
if (b == null) return null;
return new yacyNewsDB.Record(
b.getColString(0, null),
b.getColString(1, "UTF-8"),
(b.empty(2)) ? null : GenericFormatter.SHORT_SECOND_FORMATTER.parse(b.getColString(2, null), GenericFormatter.UTCDiffString()),
(b.empty(2)) ? null : my_SHORT_SECOND_FORMATTER.parse(b.getColString(2, null), GenericFormatter.UTCDiffString()),
(int) b.getColLong(3),
MapTools.string2map(b.getColString(4, "UTF-8"), ",")
);
@ -180,7 +183,7 @@ public class yacyNewsDB {
final Row.Entry entry = this.news.row().newEntry();
entry.setCol(0, r.id().getBytes());
entry.setCol(1, r.category().getBytes("UTF-8"));
entry.setCol(2, (r.received() == null) ? null : GenericFormatter.SHORT_SECOND_FORMATTER.format(r.received()).getBytes());
entry.setCol(2, (r.received() == null) ? null : my_SHORT_SECOND_FORMATTER.format(r.received()).getBytes());
entry.setCol(3, Base64Order.enhancedCoder.encodeLongBA(r.distributed(), 2));
entry.setCol(4, attributes.getBytes("UTF-8"));
return entry;
@ -239,8 +242,8 @@ public class yacyNewsDB {
if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")");
this.category = (attributes.containsKey("cat")) ? attributes.get("cat") : "";
if (category.length() > yacyNewsDB.categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + yacyNewsDB.categoryStringLength + ")");
this.received = (attributes.containsKey("rec")) ? GenericFormatter.SHORT_SECOND_FORMATTER.parse(attributes.get("rec"), GenericFormatter.UTCDiffString()) : new Date();
this.created = (attributes.containsKey("cre")) ? GenericFormatter.SHORT_SECOND_FORMATTER.parse(attributes.get("cre"), GenericFormatter.UTCDiffString()) : new Date();
this.received = (attributes.containsKey("rec")) ? my_SHORT_SECOND_FORMATTER.parse(attributes.get("rec"), GenericFormatter.UTCDiffString()) : new Date();
this.created = (attributes.containsKey("cre")) ? my_SHORT_SECOND_FORMATTER.parse(attributes.get("cre"), GenericFormatter.UTCDiffString()) : new Date();
this.distributed = (attributes.containsKey("dis")) ? Integer.parseInt(attributes.get("dis")) : 0;
this.originator = (attributes.containsKey("ori")) ? attributes.get("ori") : "";
removeStandards();
@ -263,7 +266,7 @@ public class yacyNewsDB {
if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")");
this.attributes = attributes;
this.received = received;
this.created = GenericFormatter.SHORT_SECOND_FORMATTER.parse(id.substring(0, GenericFormatter.PATTERN_SHORT_SECOND.length()), GenericFormatter.UTCDiffString());
this.created = my_SHORT_SECOND_FORMATTER.parse(id.substring(0, GenericFormatter.PATTERN_SHORT_SECOND.length()), GenericFormatter.UTCDiffString());
this.category = category;
this.distributed = distributed;
this.originator = id.substring(GenericFormatter.PATTERN_SHORT_SECOND.length());
@ -284,8 +287,8 @@ public class yacyNewsDB {
// attention: this has no additional encoding
if (this.originator != null) attributes.put("ori", this.originator);
if (this.category != null) attributes.put("cat", this.category);
if (this.created != null) attributes.put("cre", GenericFormatter.SHORT_SECOND_FORMATTER.format(this.created));
if (this.received != null) attributes.put("rec", GenericFormatter.SHORT_SECOND_FORMATTER.format(this.received));
if (this.created != null) attributes.put("cre", my_SHORT_SECOND_FORMATTER.format(this.created));
if (this.received != null) attributes.put("rec", my_SHORT_SECOND_FORMATTER.format(this.received));
attributes.put("dis", Integer.toString(this.distributed));
final String theString = attributes.toString();
removeStandards();
@ -293,7 +296,7 @@ public class yacyNewsDB {
}
public String id() {
return GenericFormatter.SHORT_SECOND_FORMATTER.format(created) + originator;
return my_SHORT_SECOND_FORMATTER.format(created) + originator;
}
public String originator() {

13
source/de/anomic/yacy/yacySeed.java
Unescape View File

@ -183,6 +183,9 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
if ((flags == null) || (flags.length() != 4)) { this.dna.put(yacySeed.FLAGS, yacySeed.FLAGSZERO); }
this.dna.put(yacySeed.NAME, checkPeerName(get(yacySeed.NAME, "&empty;")));
}
// use our own formatter to prevent concurrency locks with other processes
private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND);
private yacySeed(final String theHash) {
this.dna = new ConcurrentHashMap<String, String>();
@ -212,7 +215,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
this.dna.put(yacySeed.IPTYPE, "&empty;");
// settings that can only be computed by visiting peer
this.dna.put(yacySeed.LASTSEEN, GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/))); // for last-seen date
this.dna.put(yacySeed.LASTSEEN, my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/))); // for last-seen date
this.dna.put(yacySeed.USPEED, yacySeed.ZERO); // the computated uplink speed of the peer
// settings that are needed to organize the seed round-trip
@ -491,7 +494,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
// because java thinks it must apply the UTC offset to the current time,
// to create a string that looks like our current time, it adds the local UTC offset to the
// time. To create a corrected UTC Date string, we first subtract the local UTC offset.
String ls = GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/));
String ls = my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/));
//System.out.println("SETTING LAST-SEEN of " + this.getName() + " to " + ls);
dna.put(yacySeed.LASTSEEN, ls );
}
@ -501,7 +504,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
*/
public final long getLastSeenUTC() {
try {
final long t = GenericFormatter.SHORT_SECOND_FORMATTER.parse(get(yacySeed.LASTSEEN, "20040101000000")).getTime();
final long t = my_SHORT_SECOND_FORMATTER.parse(get(yacySeed.LASTSEEN, "20040101000000")).getTime();
// getTime creates a UTC time number. But in this case java thinks, that the given
// time string is a local time, which has a local UTC offset applied.
// Therefore java subtracts the local UTC offset, to get a UTC number.
@ -519,7 +522,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
/** @return the age of the seed in number of days */
public final int getAge() {
try {
final long t = GenericFormatter.SHORT_SECOND_FORMATTER.parse(get(yacySeed.BDATE, "20040101000000")).getTime();
final long t = my_SHORT_SECOND_FORMATTER.parse(get(yacySeed.BDATE, "20040101000000")).getTime();
return (int) ((System.currentTimeMillis() - (t /*- getUTCDiff() + DateFormatter.UTCDiff()*/)) / 1000 / 60 / 60 / 24);
} catch (final java.text.ParseException e) {
return -1;
@ -725,7 +728,7 @@ public class yacySeed implements Cloneable, Comparable<yacySeed>, Comparator<yac
// now calculate other information about the host
newSeed.dna.put(yacySeed.NAME, (name) == null ? defaultPeerName() : name);
newSeed.dna.put(yacySeed.PORT, Integer.toString((port <= 0) ? 8090 : port));
newSeed.dna.put(yacySeed.BDATE, GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/)) );
newSeed.dna.put(yacySeed.BDATE, my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/)) );
newSeed.dna.put(yacySeed.LASTSEEN, newSeed.dna.get(yacySeed.BDATE)); // just as initial setting
newSeed.dna.put(yacySeed.UTC, GenericFormatter.UTCDiffString());
newSeed.dna.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);

18
source/net/yacy/cora/date/GenericFormatter.java
Unescape View File

@ -31,14 +31,14 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
public static final String PATTERN_SHORT_DAY = "yyyyMMdd";
public static final String PATTERN_SHORT_SECOND = "yyyyMMddHHmmss";
public static final String PATTERN_SHORT_MILSEC = "yyyyMMddHHmmssSSS";
private static final String PATTERN_RFC1123_SHORT = "EEE, dd MMM yyyy";
private static final String PATTERN_ANSIC = "EEE MMM d HH:mm:ss yyyy";
public static final String PATTERN_RFC1123_SHORT = "EEE, dd MMM yyyy";
public static final String PATTERN_ANSIC = "EEE MMM d HH:mm:ss yyyy";
private static final SimpleDateFormat FORMAT_SHORT_DAY = new SimpleDateFormat(PATTERN_SHORT_DAY, Locale.US);
private static final SimpleDateFormat FORMAT_SHORT_SECOND = new SimpleDateFormat(PATTERN_SHORT_SECOND, Locale.US);
private static final SimpleDateFormat FORMAT_SHORT_MILSEC = new SimpleDateFormat(PATTERN_SHORT_MILSEC, Locale.US);
private static final SimpleDateFormat FORMAT_ANSIC = new SimpleDateFormat(PATTERN_ANSIC, Locale.US);
private static final SimpleDateFormat FORMAT_RFC1123_SHORT = new SimpleDateFormat(PATTERN_RFC1123_SHORT, Locale.US);
public static final SimpleDateFormat FORMAT_SHORT_DAY = new SimpleDateFormat(PATTERN_SHORT_DAY, Locale.US);
public static final SimpleDateFormat FORMAT_SHORT_SECOND = new SimpleDateFormat(PATTERN_SHORT_SECOND, Locale.US);
public static final SimpleDateFormat FORMAT_SHORT_MILSEC = new SimpleDateFormat(PATTERN_SHORT_MILSEC, Locale.US);
public static final SimpleDateFormat FORMAT_ANSIC = new SimpleDateFormat(PATTERN_ANSIC, Locale.US);
public static final SimpleDateFormat FORMAT_RFC1123_SHORT = new SimpleDateFormat(PATTERN_RFC1123_SHORT, Locale.US);
// find out time zone and DST offset
private static Calendar thisCalendar = Calendar.getInstance();
@ -86,6 +86,10 @@ public class GenericFormatter extends AbstractFormatter implements DateFormatter
if (date == null) return "";
if (Math.abs(date.getTime() - last_time) < maxCacheDiff) return last_format;
synchronized (this.dateFormat) {
// threads that had been waiting here may use the cache now instead of calculating the date again
if (Math.abs(date.getTime() - last_time) < maxCacheDiff) return last_format;
// if the cache is not fresh, calculate the date
last_format = this.dateFormat.format(date);
last_time = date.getTime();
}

12
source/net/yacy/kelondro/blob/ArrayStack.java
Unescape View File

@ -97,6 +97,10 @@ public class ArrayStack implements BLOB {
// the thread pool for the keeperOf executor service
private final ExecutorService executor;
// use our own formatter to prevent concurrency locks with other processes
private final static GenericFormatter my_SHORT_MILSEC_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_MILSEC);
public ArrayStack(
final File heapLocation,
final String prefix,
@ -177,7 +181,7 @@ public class ArrayStack implements BLOB {
for (int i = 0; i < files.length; i++) {
if (files[i].length() >= 22 && files[i].startsWith(prefix) && files[i].endsWith(".blob")) {
try {
d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(files[i].substring(prefix.length() + 1, prefix.length() + 18));
d = my_SHORT_MILSEC_FORMATTER.parse(files[i].substring(prefix.length() + 1, prefix.length() + 18));
time = d.getTime();
if (time > maxtime) maxtime = time;
} catch (ParseException e) {continue;}
@ -188,7 +192,7 @@ public class ArrayStack implements BLOB {
for (int i = 0; i < files.length; i++) {
if (files[i].length() >= 22 && files[i].startsWith(prefix) && files[i].endsWith(".blob")) {
try {
d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(files[i].substring(prefix.length() + 1, prefix.length() + 18));
d = my_SHORT_MILSEC_FORMATTER.parse(files[i].substring(prefix.length() + 1, prefix.length() + 18));
f = new File(heapLocation, files[i]);
time = d.getTime();
if (time == maxtime && !trimall) {
@ -231,7 +235,7 @@ public class ArrayStack implements BLOB {
public synchronized void mountBLOB(File location, boolean full) throws IOException {
Date d;
try {
d = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(location.getName().substring(prefix.length() + 1, prefix.length() + 18));
d = my_SHORT_MILSEC_FORMATTER.parse(location.getName().substring(prefix.length() + 1, prefix.length() + 18));
} catch (ParseException e) {
throw new IOException("date parse problem with file " + location.toString() + ": " + e.getMessage());
}
@ -365,7 +369,7 @@ public class ArrayStack implements BLOB {
*/
public synchronized File newBLOB(Date creation) {
//return new File(heapLocation, DateFormatter.formatShortSecond(creation) + "." + blobSalt + ".blob");
return new File(heapLocation, prefix + "." + GenericFormatter.SHORT_MILSEC_FORMATTER.format(creation) + ".blob");
return new File(heapLocation, prefix + "." + my_SHORT_MILSEC_FORMATTER.format(creation) + ".blob");
}
public String name() {

6
source/net/yacy/kelondro/blob/MapHeap.java
Unescape View File

@ -127,7 +127,11 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
}
return map;
}
// use our own formatter to prevent concurrency locks with other processes
private final static GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND);
/**
* write a whole byte array as Map to the table
* @param key the primary key
@ -140,7 +144,7 @@ public class MapHeap implements Map<byte[], Map<String, String>> {
assert key.length > 0;
assert newMap != null;
key = normalizeKey(key);
String s = map2string(newMap, "W" + GenericFormatter.SHORT_SECOND_FORMATTER.format(new Date()) + " ");
String s = map2string(newMap, "W" + my_SHORT_SECOND_FORMATTER.format(new Date()) + " ");
assert s != null;
byte[] sb = s.getBytes();
synchronized (this) {

9
source/net/yacy/kelondro/blob/Tables.java
Unescape View File

@ -59,7 +59,10 @@ public class Tables {
private File location;
private ConcurrentHashMap<String, BEncodedHeap> tables;
int keymaxlen;
// use our own formatter to prevent concurrency locks with other processes
private final static GenericFormatter my_SHORT_MILSEC_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_MILSEC);
public Tables(final File location, final int keymaxlen) {
this.location = new File(location.getAbsolutePath());
if (!this.location.exists()) this.location.mkdirs();
@ -408,7 +411,7 @@ public class Tables {
}
public void put(String colname, Date value) {
super.put(colname, GenericFormatter.SHORT_MILSEC_FORMATTER.format(value).getBytes());
super.put(colname, my_SHORT_MILSEC_FORMATTER.format(value).getBytes());
}
public byte[] get(String colname, byte[] dflt) {
@ -447,7 +450,7 @@ public class Tables {
byte[] r = this.get(colname);
if (r == null) return dflt;
try {
return GenericFormatter.SHORT_MILSEC_FORMATTER.parse(new String(r));
return my_SHORT_MILSEC_FORMATTER.parse(new String(r));
} catch (ParseException e) {
return dflt;
}

14
source/net/yacy/kelondro/data/meta/URIMetadataRow.java
Unescape View File

@ -106,6 +106,8 @@ public class URIMetadataRow implements URIMetadata {
private final long ranking; // during generation of a search result this value is set
private Components comp;
private static final GenericFormatter mySHORT_DAY_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY);
public URIMetadataRow() {
// create a dummy entry, good to produce poison objects
this.entry = rowdef.newEntry();
@ -228,17 +230,17 @@ public class URIMetadataRow implements URIMetadata {
this.entry.setCol(col_hash, url.hash()); // FIXME potential null pointer access
this.entry.setCol(col_comp, encodeComp(url, descr, dc_creator, tags, dc_publisher));
try {
encodeDate(col_mod, GenericFormatter.SHORT_DAY_FORMATTER.parse(prop.getProperty("mod", "20000101")));
encodeDate(col_mod, mySHORT_DAY_FORMATTER.parse(prop.getProperty("mod", "20000101")));
} catch (final ParseException e) {
encodeDate(col_mod, new Date());
}
try {
encodeDate(col_load, GenericFormatter.SHORT_DAY_FORMATTER.parse(prop.getProperty("load", "20000101")));
encodeDate(col_load, mySHORT_DAY_FORMATTER.parse(prop.getProperty("load", "20000101")));
} catch (final ParseException e) {
encodeDate(col_load, new Date());
}
try {
encodeDate(col_fresh, GenericFormatter.SHORT_DAY_FORMATTER.parse(prop.getProperty("fresh", "20000101")));
encodeDate(col_fresh, mySHORT_DAY_FORMATTER.parse(prop.getProperty("fresh", "20000101")));
} catch (final ParseException e) {
encodeDate(col_fresh, new Date());
}
@ -306,11 +308,11 @@ public class URIMetadataRow implements URIMetadata {
assert (s.toString().indexOf(0) < 0);
s.append(",publisher=").append(crypt.simpleEncode(metadata.dc_publisher()));
assert (s.toString().indexOf(0) < 0);
s.append(",mod=").append(GenericFormatter.SHORT_DAY_FORMATTER.format(moddate()));
s.append(",mod=").append(mySHORT_DAY_FORMATTER.format(moddate()));
assert (s.toString().indexOf(0) < 0);
s.append(",load=").append(GenericFormatter.SHORT_DAY_FORMATTER.format(loaddate()));
s.append(",load=").append(mySHORT_DAY_FORMATTER.format(loaddate()));
assert (s.toString().indexOf(0) < 0);
s.append(",fresh=").append(GenericFormatter.SHORT_DAY_FORMATTER.format(freshdate()));
s.append(",fresh=").append(mySHORT_DAY_FORMATTER.format(freshdate()));
assert (s.toString().indexOf(0) < 0);
s.append(",referrer=").append(referrerHash() == null ? "" : new String(referrerHash()));
assert (s.toString().indexOf(0) < 0);

Write Preview
Loading…
Cancel
Save