boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

discovered and removed a (possibly large) memory leak:

Browse Source 
many classes used the kelondroMapDataMining (was: kelondroMapObjects) which adds statistical
functions to the kelondroMap (was: kelondroObjects), but these functions were not used by these
classes. Especially the HTCACHE and robots.txt database allocate a very large number of objects
for statistical use, but never used them. By replacing the kelondroMapDataMining with the
kelondroMap object for these classes now less memory is allocated.


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4986 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 0f5fe8cc53
commit 7b1c9e6aee
10 changed files with 36 additions and 41 deletions
Show Stats Download Patch File Download Diff File

8
source/de/anomic/crawler/CrawlProfile.java
Unescape View File

@ -53,7 +53,7 @@ import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCloneableIterator; import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapDataMining; import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverCodings; import de.anomic.server.serverCodings;
import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.yacySeedDB;
@ -63,14 +63,14 @@ public class CrawlProfile {
static HashMap<String, Map<String, DomProfile>> domsCache = new HashMap<String, Map<String, DomProfile>>(); static HashMap<String, Map<String, DomProfile>> domsCache = new HashMap<String, Map<String, DomProfile>>();
kelondroMapDataMining profileTable; kelondroMap profileTable;
private File profileTableFile; private File profileTableFile;
public CrawlProfile(File file) { public CrawlProfile(File file) {
this.profileTableFile = file; this.profileTableFile = file;
profileTableFile.getParentFile().mkdirs(); profileTableFile.getParentFile().mkdirs();
kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true); kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
profileTable = new kelondroMapDataMining(dyn, 500); profileTable = new kelondroMap(dyn, 500);
} }
public void clear() { public void clear() {
@ -79,7 +79,7 @@ public class CrawlProfile {
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database"); if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs(); profileTableFile.getParentFile().mkdirs();
kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true); kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
profileTable = new kelondroMapDataMining(dyn, 500); profileTable = new kelondroMap(dyn, 500);
} }
public void close() { public void close() {

8
source/de/anomic/crawler/RobotsTxt.java
Unescape View File

@ -65,7 +65,7 @@ import de.anomic.kelondro.kelondroBLOB;
import de.anomic.kelondro.kelondroBLOBHeap; import de.anomic.kelondro.kelondroBLOBHeap;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapDataMining; import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverByteBuffer; import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverFileUtils; import de.anomic.server.serverFileUtils;
@ -76,7 +76,7 @@ public class RobotsTxt {
public static final String ROBOTS_DB_PATH_SEPARATOR = ";"; public static final String ROBOTS_DB_PATH_SEPARATOR = ";";
kelondroMapDataMining robotsTable; kelondroMap robotsTable;
private final File robotsTableFile; private final File robotsTableFile;
public RobotsTxt(File robotsTableFile) { public RobotsTxt(File robotsTableFile) {
@ -92,7 +92,7 @@ public class RobotsTxt {
} else { } else {
blob = new kelondroBLOBTree(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true); blob = new kelondroBLOBTree(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true);
} }
robotsTable = new kelondroMapDataMining(blob, 100); robotsTable = new kelondroMap(blob, 100);
} }
private void resetDatabase() { private void resetDatabase() {
@ -100,7 +100,7 @@ public class RobotsTxt {
if (robotsTable != null) robotsTable.close(); if (robotsTable != null) robotsTable.close();
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database"); if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs(); robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMapDataMining(new kelondroBLOBTree(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100); robotsTable = new kelondroMap(new kelondroBLOBTree(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100);
} }
public void clear() throws IOException { public void clear() throws IOException {

8
source/de/anomic/data/blogBoard.java
Unescape View File

@ -68,7 +68,7 @@ import org.xml.sax.SAXException;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapDataMining; import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverDate; import de.anomic.server.serverDate;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
@ -78,12 +78,12 @@ public class blogBoard {
public static final int keyLength = 64; public static final int keyLength = 64;
private static final int recordSize = 512; private static final int recordSize = 512;
kelondroMapDataMining database = null; kelondroMap database = null;
public blogBoard(File actpath) { public blogBoard(File actpath) {
new File(actpath.getParent()).mkdir(); new File(actpath.getParent()).mkdir();
if (database == null) { if (database == null) {
database = new kelondroMapDataMining(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); database = new kelondroMap(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
} }
} }
public int size() { public int size() {
@ -137,7 +137,7 @@ public class blogBoard {
public BlogEntry readBlogEntry(String key) { public BlogEntry readBlogEntry(String key) {
return readBlogEntry(key, database); return readBlogEntry(key, database);
} }
private BlogEntry readBlogEntry(String key, kelondroMapDataMining base) { private BlogEntry readBlogEntry(String key, kelondroMap base) {
key = normalize(key); key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength); if (key.length() > keyLength) key = key.substring(0, keyLength);
HashMap<String, String> record; HashMap<String, String> record;

8
source/de/anomic/data/blogBoardComments.java
Unescape View File

@ -66,7 +66,7 @@ import org.xml.sax.SAXException;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapDataMining; import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.logging.serverLog; import de.anomic.server.logging.serverLog;
@ -81,11 +81,11 @@ public class blogBoardComments {
static { static {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT")); SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
} }
private kelondroMapDataMining database = null; private kelondroMap database = null;
public blogBoardComments(File actpath) { public blogBoardComments(File actpath) {
new File(actpath.getParent()).mkdir(); new File(actpath.getParent()).mkdir();
if (database == null) { if (database == null) {
database = new kelondroMapDataMining(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, false, false, false), 500); database = new kelondroMap(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, false, false, false), 500);
} }
} }
public int size() { public int size() {
@ -130,7 +130,7 @@ public class blogBoardComments {
//System.out.println("DEBUG: read from blogBoardComments"); //System.out.println("DEBUG: read from blogBoardComments");
return read(key, database); return read(key, database);
} }
private CommentEntry read(String key, kelondroMapDataMining base) { private CommentEntry read(String key, kelondroMap base) {
key = normalize(key); key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength); if (key.length() > keyLength) key = key.substring(0, keyLength);
HashMap<String, String> record; HashMap<String, String> record;

9
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -78,7 +78,6 @@ import de.anomic.index.indexWord;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroCloneableIterator; import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroMap; import de.anomic.kelondro.kelondroMap;
import de.anomic.server.serverDate; import de.anomic.server.serverDate;
@ -99,11 +98,11 @@ public class bookmarksDB {
kelondroMap bookmarksTable; // kelondroMap bookmarksTable; kelondroMap bookmarksTable; // kelondroMap bookmarksTable;
// tags // tags
kelondroMapDataMining tagsTable; kelondroMap tagsTable;
HashMap<String, Tag> tagCache; HashMap<String, Tag> tagCache;
// dates // dates
kelondroMapDataMining datesTable; kelondroMap datesTable;
// ------------------------------------ // ------------------------------------
@ -120,12 +119,12 @@ public class bookmarksDB {
// tags // tags
tagsFile.getParentFile().mkdirs(); tagsFile.getParentFile().mkdirs();
boolean tagsFileExisted = tagsFile.exists(); boolean tagsFileExisted = tagsFile.exists();
this.tagsTable = new kelondroMapDataMining(new kelondroBLOBTree(tagsFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); this.tagsTable = new kelondroMap(new kelondroBLOBTree(tagsFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
if (!tagsFileExisted) rebuildTags(); if (!tagsFileExisted) rebuildTags();
// dates // dates
boolean datesExisted = datesFile.exists(); boolean datesExisted = datesFile.exists();
this.datesTable = new kelondroMapDataMining(new kelondroBLOBTree(datesFile, true, true, 20, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); this.datesTable = new kelondroMap(new kelondroBLOBTree(datesFile, true, true, 20, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
if (!datesExisted) rebuildDates(); if (!datesExisted) rebuildDates();
} }

6
source/de/anomic/data/messageBoard.java
Unescape View File

@ -51,7 +51,7 @@ import java.util.TimeZone;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapDataMining; import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;
public class messageBoard { public class messageBoard {
@ -66,13 +66,13 @@ public class messageBoard {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT")); SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
} }
kelondroMapDataMining database = null; kelondroMap database = null;
private int sn = 0; private int sn = 0;
public messageBoard(File path) { public messageBoard(File path) {
new File(path.getParent()).mkdir(); new File(path.getParent()).mkdir();
if (database == null) { if (database == null) {
database = new kelondroMapDataMining(new kelondroBLOBTree(path, true, true, categoryLength + dateFormat.length() + 2, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); database = new kelondroMap(new kelondroBLOBTree(path, true, true, categoryLength + dateFormat.length() + 2, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
} }
sn = 0; sn = 0;
} }

8
source/de/anomic/data/userDB.java
Unescape View File

@ -59,7 +59,7 @@ import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCloneableIterator; import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapDataMining; import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverCodings; import de.anomic.server.serverCodings;
@ -68,7 +68,7 @@ public final class userDB {
public static final int USERNAME_MAX_LENGTH = 128; public static final int USERNAME_MAX_LENGTH = 128;
public static final int USERNAME_MIN_LENGTH = 4; public static final int USERNAME_MIN_LENGTH = 4;
kelondroMapDataMining userTable; kelondroMap userTable;
private final File userTableFile; private final File userTableFile;
HashMap<String, String> ipUsers = new HashMap<String, String>(); HashMap<String, String> ipUsers = new HashMap<String, String>();
HashMap<String, Object> cookieUsers = new HashMap<String, Object>(); HashMap<String, Object> cookieUsers = new HashMap<String, Object>();
@ -76,7 +76,7 @@ public final class userDB {
public userDB(File userTableFile) { public userDB(File userTableFile) {
this.userTableFile = userTableFile; this.userTableFile = userTableFile;
userTableFile.getParentFile().mkdirs(); userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMapDataMining(new kelondroBLOBTree(userTableFile, true, true, 128, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10); this.userTable = new kelondroMap(new kelondroBLOBTree(userTableFile, true, true, 128, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
} }
void resetDatabase() { void resetDatabase() {
@ -84,7 +84,7 @@ public final class userDB {
if (userTable != null) userTable.close(); if (userTable != null) userTable.close();
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database"); if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs(); userTableFile.getParentFile().mkdirs();
userTable = new kelondroMapDataMining(new kelondroBLOBTree(userTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10); userTable = new kelondroMap(new kelondroBLOBTree(userTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
} }
public void close() { public void close() {

12
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -51,7 +51,7 @@ import java.util.TimeZone;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapDataMining; import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroNaturalOrder;
public class wikiBoard { public class wikiBoard {
@ -66,18 +66,18 @@ public class wikiBoard {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT")); SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
} }
kelondroMapDataMining datbase = null; kelondroMap datbase = null;
kelondroMapDataMining bkpbase = null; kelondroMap bkpbase = null;
static HashMap<String, String> authors = new HashMap<String, String>(); static HashMap<String, String> authors = new HashMap<String, String>();
public wikiBoard(File actpath, File bkppath) { public wikiBoard(File actpath, File bkppath) {
new File(actpath.getParent()).mkdirs(); new File(actpath.getParent()).mkdirs();
if (datbase == null) { if (datbase == null) {
datbase = new kelondroMapDataMining(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); datbase = new kelondroMap(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
} }
new File(bkppath.getParent()).mkdirs(); new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) { if (bkpbase == null) {
bkpbase = new kelondroMapDataMining(new kelondroBLOBTree(bkppath, true, true, keyLength + dateFormat.length(), recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500); bkpbase = new kelondroMap(new kelondroBLOBTree(bkppath, true, true, keyLength + dateFormat.length(), recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
} }
} }
@ -290,7 +290,7 @@ public class wikiBoard {
return read(key, datbase); return read(key, datbase);
} }
entry read(String key, kelondroMapDataMining base) { entry read(String key, kelondroMap base) {
try { try {
key = normalize(key); key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength); if (key.length() > keyLength) key = key.substring(0, keyLength);

4
source/de/anomic/kelondro/kelondroMapDataMining.java
Unescape View File

@ -41,10 +41,6 @@ public class kelondroMapDataMining extends kelondroMap {
private HashMap<String, Object> accMap; // to store accumulations of specific fields private HashMap<String, Object> accMap; // to store accumulations of specific fields
private int elementCount; private int elementCount;
public kelondroMapDataMining(kelondroBLOB dyn, int cachesize) {
this(dyn, cachesize, null, null, null, null, null);
}
@SuppressWarnings({ "unchecked", "null" }) @SuppressWarnings({ "unchecked", "null" })
public kelondroMapDataMining(kelondroBLOB dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) { public kelondroMapDataMining(kelondroBLOB dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) {
super(dyn, cachesize); super(dyn, cachesize);

6
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -57,7 +57,7 @@ import de.anomic.kelondro.kelondroBLOBHeap;
import de.anomic.kelondro.kelondroBLOBTree; import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMScoreCluster; import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapDataMining; import de.anomic.kelondro.kelondroMap;
import de.anomic.plasma.cache.IResourceInfo; import de.anomic.plasma.cache.IResourceInfo;
import de.anomic.plasma.cache.ResourceInfoFactory; import de.anomic.plasma.cache.ResourceInfoFactory;
import de.anomic.plasma.cache.UnsupportedProtocolException; import de.anomic.plasma.cache.UnsupportedProtocolException;
@ -80,7 +80,7 @@ public final class plasmaHTCache {
private static final int stackLimit = 150; // if we exceed that limit, we do not check idle private static final int stackLimit = 150; // if we exceed that limit, we do not check idle
public static final long oneday = 1000 * 60 * 60 * 24; // milliseconds of a day public static final long oneday = 1000 * 60 * 60 * 24; // milliseconds of a day
private static kelondroMapDataMining responseHeaderDB = null; private static kelondroMap responseHeaderDB = null;
private static final ConcurrentLinkedQueue<Entry> cacheStack = new ConcurrentLinkedQueue<Entry>(); private static final ConcurrentLinkedQueue<Entry> cacheStack = new ConcurrentLinkedQueue<Entry>();
private static final ConcurrentHashMap<String, File> cacheAge = new ConcurrentHashMap<String, File>(); // a <date+hash, cache-path> - relation private static final ConcurrentHashMap<String, File> cacheAge = new ConcurrentHashMap<String, File>(); // a <date+hash, cache-path> - relation
public static long curCacheSize = 0; public static long curCacheSize = 0;
@ -272,7 +272,7 @@ public final class plasmaHTCache {
} else { } else {
blob = new kelondroBLOBTree(dbfile, true, true, yacySeedDB.commonHashLength, 150, '#', kelondroBase64Order.enhancedCoder, false, false, true); blob = new kelondroBLOBTree(dbfile, true, true, yacySeedDB.commonHashLength, 150, '#', kelondroBase64Order.enhancedCoder, false, false, true);
} }
responseHeaderDB = new kelondroMapDataMining(blob, 500); responseHeaderDB = new kelondroMap(blob, 500);
} }
private static void deleteOldHTCache(File directory) { private static void deleteOldHTCache(File directory) {

Write Preview
Loading…
Cancel
Save