boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

performance enhancements using an alternative to a insensitive collator (a complex string compare):

Browse Source 
- less synchronizations
- better speed
..at most important and commonly used classes: http headers, url parsing and html parsing

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7526 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 14 years ago
parent f2e8ffd768
commit a92d80a545
6 changed files with 108 additions and 50 deletions
Show Stats Download Patch File Download Diff File

12
htroot/AccessPicture_p.java
Unescape View File

@ -88,13 +88,13 @@ public class AccessPicture_p {
int centerx = (picture.gridWidth() >> 1) - 1;
int centery = picture.gridHeight() >> 1;
picture.setColor(color_dot);
picture.gridDot(centerx, centery, 5);
picture.gridDot(centerx, centery, 5, true, 100);
if (corona) {
for (int i = 0; i < 6; i++) {
picture.gridDot(centerx, centery, 50, i * 60 + coronaangle / 6, i * 60 + 30 + coronaangle / 6);
}
} else {
picture.gridDot(centerx, centery, 50);
picture.gridDot(centerx, centery, 50, false, 100);
}
//picture.gridDot(centerx, centery, 31, false);
picture.setColor(color_text);
@ -129,8 +129,8 @@ public class AccessPicture_p {
for (int i = 0; i < hosts.length; i++) {
if (hosts[i] != null) {
picture.setColor(color_dot);
picture.gridDot(gridLeft, i * 2 + 1, 7);
picture.gridDot(gridLeft, i * 2 + 1, 8);
picture.gridDot(gridLeft, i * 2 + 1, 7, false, 100);
picture.gridDot(gridLeft, i * 2 + 1, 8, false, 100);
picture.setColor(color_text);
picture.gridPrint(gridLeft, i * 2 + 1, 8, hosts[i].toUpperCase(), "COUNT = " + count[i] + ", TIME > " + ((time[i] >= 60000) ? ((time[i] / 60000) + " MINUTES") : ((time[i] / 1000) + " SECONDS")), -1);
if (corona) {
@ -164,8 +164,8 @@ public class AccessPicture_p {
for (int i = 0; i < hosts.length; i++) {
if (hosts[i] != null) {
picture.setColor(color_dot);
picture.gridDot(gridRight, i * 2 + 1, 7);
picture.gridDot(gridRight, i * 2 + 1, 8);
picture.gridDot(gridRight, i * 2 + 1, 7, false, 100);
picture.gridDot(gridRight, i * 2 + 1, 8, false, 100);
picture.setColor(color_text);
picture.gridPrint(gridRight, i * 2 + 1, 8, hosts[i].toUpperCase(), count[i] + " BYTES, " + time[i] + " MS DUE", 1);
if (corona) {

78
source/net/yacy/cora/document/ASCIIComparator.java
Unescape View File

@ -0,0 +1,78 @@
/**
* ASCIIComparator
* Copyright 2010 by Michael Peter Christen
* First released 25.2.2011 at http://yacy.net
*
* This file is part of YaCy Content Integration
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.document;
import java.util.Comparator;
/**
* this is a replacement of an insensitive collator object, produced by a RuleBasedCollator Class
* The RuleBasedCollator is a very inefficient class if it is used only for insensitive ASCII comparisments
* This class is a very simple comparator for Strings which can be used to compare also Strings with upper/lowercase
* Strings without applying .toUpperCase or .toLowerCase
* Strings must contain no other than ASCII code.
*/
public class ASCIIComparator implements Comparator<String> {
public static final ASCIIComparator insensitiveASCIIComparator = new ASCIIComparator(true);
public boolean insensitive;
public ASCIIComparator(boolean insensitive) {
this.insensitive = insensitive;
}
public Object clone() {
return this; // because we do not have any class variables that changes
}
public int compare(String s0, String s1) {
if (s0 == null && s1 == null) return 0;
if (s0 == null) return -1;
if (s1 == null) return 1;
int i = 0;
int l0 = s0.length(), l1 = s1.length();
int lm = Math.min(l0, l1);
char c0, c1;
while (i < lm) {
c0 = s0.charAt(i);
c1 = s1.charAt(i);
if (this.insensitive && c0 >= 'A' && c0 <='Z') c0 = (char) ((byte) c0 + 32);
if (this.insensitive && c1 >= 'A' && c1 <='Z') c1 = (char) ((byte) c1 + 32);
if (c0 > c1) return 1;
if (c1 > c0) return -1;
i++;
}
if (l0 > l1) return 1;
if (l1 > l0) return -1;
return 0;
}
public boolean equals(Object obj) {
return (obj == this);
}
public int hashCode() {
return System.identityHashCode(this);
}
}

19
source/net/yacy/cora/document/MultiProtocolURI.java
Unescape View File

@ -29,12 +29,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.net.MalformedURLException;
import java.text.Collator;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -69,19 +67,14 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
//private static final Pattern patternSpace = Pattern.compile("%20");
// session id handling
private static final Collator insensitiveCollator = Collator.getInstance(Locale.US);
private static final TreeSet<String> sessionIDnames;
static {
insensitiveCollator.setStrength(Collator.SECONDARY);
insensitiveCollator.setDecomposition(Collator.NO_DECOMPOSITION);
sessionIDnames = new TreeSet<String>(insensitiveCollator);
}
private static final Object PRESENT = new Object();
private static final ConcurrentHashMap<String, Object> sessionIDnames = new ConcurrentHashMap<String, Object>();
public static final void initSessionIDNames(Set<String> idNames) {
for (String s: idNames) {
if (s == null) continue;
s = s.trim();
if (s.length() > 0) sessionIDnames.add(s);
if (s.length() > 0) sessionIDnames.put(s, PRESENT);
}
}
@ -669,7 +662,7 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
}
String q = quest;
if (removeSessionID) {
for (String sid: sessionIDnames) {
for (String sid: sessionIDnames.keySet()) {
if (q.toLowerCase().startsWith(sid.toLowerCase() + "=")) {
int p = q.indexOf('&');
if (p < 0) {
@ -936,7 +929,7 @@ public class MultiProtocolURI implements Serializable, Comparable<MultiProtocolU
public final boolean isIndividual() {
final String q = unescape(path.toLowerCase());
for (String sid: sessionIDnames) {
for (String sid: sessionIDnames.keySet()) {
if (q.startsWith(sid.toLowerCase() + "=")) return true;
int p = q.indexOf("&" + sid.toLowerCase() + "=");
if (p >= 0) return true;

18
source/net/yacy/cora/protocol/HeaderFramework.java
Unescape View File

@ -25,7 +25,6 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
import java.text.Collator;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
@ -39,6 +38,7 @@ import java.util.TreeMap;
import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.document.ASCIIComparator;
import net.yacy.cora.document.MultiProtocolURI;
@ -212,13 +212,6 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
private final Map<String, String> reverseMappingCache;
private static final Collator insensitiveCollator = Collator.getInstance(Locale.US);
static {
insensitiveCollator.setStrength(Collator.SECONDARY);
insensitiveCollator.setDecomposition(Collator.NO_DECOMPOSITION);
}
public HeaderFramework() {
this(null);
}
@ -229,13 +222,13 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
// 'proper' appearance, a translation cache is needed.
// upon instantiation, such a mapping cache can be handed over
// If the reverseMappingCache is null, none is used
super((Collator) insensitiveCollator.clone());
super(ASCIIComparator.insensitiveASCIIComparator);
this.reverseMappingCache = reverseMappingCache;
}
public HeaderFramework(final Map<String, String> reverseMappingCache, final Map<String, String> othermap) {
// creates a case insensitive map from another map
super((Collator) insensitiveCollator.clone());
super(ASCIIComparator.insensitiveASCIIComparator);
this.reverseMappingCache = reverseMappingCache;
// load with data
@ -334,13 +327,14 @@ public class HeaderFramework extends TreeMap<String, String> implements Map<Stri
public String add(final String key, final String value) {
final int c = keyCount(key);
if (c == 0) return put(key, value);
return put("*" + key + "-" + c, value);
return put("*" + key + "-" + Integer.toString(c), value);
}
public int keyCount(final String key) {
if (!(containsKey(key))) return 0;
int c = 1;
while (containsKey("*" + key + "-" + c)) c++;
String h = "*" + key + "-";
while (containsKey(h + Integer.toString(c))) c++;
return c;
}

13
source/net/yacy/document/parser/html/ContentTransformer.java
Unescape View File

@ -28,26 +28,19 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.text.Collator;
import java.util.ArrayList;
import java.util.Locale;
import java.util.Properties;
import java.util.TreeSet;
import net.yacy.cora.document.ASCIIComparator;
import net.yacy.kelondro.io.CharBuffer;
import net.yacy.kelondro.logging.Log;
public class ContentTransformer extends AbstractTransformer implements Transformer {
// statics: for initialization of the HTMLFilterAbstractTransformer
private static final Collator insensitiveCollator = Collator.getInstance(Locale.US);
private static final TreeSet<String> linkTags0 = new TreeSet<String>(insensitiveCollator);;
private static final TreeSet<String> linkTags1 = new TreeSet<String>(insensitiveCollator);;
static {
insensitiveCollator.setStrength(Collator.SECONDARY);
insensitiveCollator.setDecomposition(Collator.NO_DECOMPOSITION);
}
private static final TreeSet<String> linkTags0 = new TreeSet<String>(ASCIIComparator.insensitiveASCIIComparator);
private static final TreeSet<String> linkTags1 = new TreeSet<String>(ASCIIComparator.insensitiveASCIIComparator);
static {
linkTags0.add("img");

18
source/net/yacy/visualization/HexGridPlotter.java
Unescape View File

@ -98,8 +98,8 @@ public class HexGridPlotter extends RasterPlotter {
}
}
public void gridDot(final int x, final int y, final int radius) {
dot(projectionX(x, y), projectionY(y), radius, true, 100);
public void gridDot(final int x, final int y, final int radius, boolean filled, int intensity) {
dot(projectionX(x, y), projectionY(y), radius, filled, intensity);
}
public void gridDot(final int x, final int y, final int radius, int fromArc, int toArc) {
@ -156,26 +156,26 @@ public class HexGridPlotter extends RasterPlotter {
final HexGridPlotter picture = new HexGridPlotter(640, 480, DrawMode.MODE_SUB, "FFFFFF", 18);
picture.drawGrid("555555");
picture.setColor("33ff33");
picture.gridDot(0, 0, 5); picture.gridPrint(0, 0, 5, "", "0,0", -1);
picture.gridDot(0, 0, 5, true, 100); picture.gridPrint(0, 0, 5, "", "0,0", -1);
for (int i = 1; i < picture.gridHeight() -1; i++) {
picture.setColor("33ff33");picture.gridDot(0, i, 3);
picture.setColor("33ff33");picture.gridDot(0, i, 3, true, 100);
picture.setColor("334433");picture.gridPrint(0, i, 3, "", "0," + i, -1);
}
for (int i = 1; i < picture.gridWidth() -1; i++) {
picture.setColor("33ff33");picture.gridDot(i, 0, 3);
picture.setColor("33ff33");picture.gridDot(i, 0, 3, true, 100);
picture.setColor("334433");picture.gridPrint315(i, 0, 3, i + ",0");
}
picture.setColor("33ff33");
picture.gridDot(0, picture.gheight - 1, 5); picture.gridPrint(0, picture.gheight - 1, 5, "0, grid.gheight - 1", "", -1);
picture.gridDot(picture.gwidth - 1, 0, 5); picture.gridPrint(picture.gwidth - 1, 0, 5, "", "grid.gwidth - 1, 0", -1);
picture.gridDot(picture.gwidth - 1, picture.gheight - 1, 5); picture.gridPrint(picture.gwidth - 1, picture.gheight - 1, 5, "grid.gwidth - 1, grid.gheight - 1", "", 1);
picture.gridDot(0, picture.gheight - 1, 5, true, 100); picture.gridPrint(0, picture.gheight - 1, 5, "0, grid.gheight - 1", "", -1);
picture.gridDot(picture.gwidth - 1, 0, 5, true, 100); picture.gridPrint(picture.gwidth - 1, 0, 5, "", "grid.gwidth - 1, 0", -1);
picture.gridDot(picture.gwidth - 1, picture.gheight - 1, 5, true, 100); picture.gridPrint(picture.gwidth - 1, picture.gheight - 1, 5, "grid.gwidth - 1, grid.gheight - 1", "", 1);
picture.gridDot(3, 3, 20, 0, 360);
picture.gridDot(7, 5, 5, 0, 360);
picture.gridPrint(7, 5, 8, "COMMUNICATION TEST", "TRANSFER 64KBIT", -1);
picture.gridDot(14, 8, 5);
picture.gridDot(14, 8, 5, true, 100);
picture.gridLine(7, 5, 14, 8);
picture.gridPrint315(14, 8, 8, "NULL");

Write Preview
Loading…
Cancel
Save