boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

discovered and removed a (possibly large) memory leak:

Browse Source 
many classes used the kelondroMapDataMining (was: kelondroMapObjects) which adds statistical
functions to the kelondroMap (was: kelondroObjects), but these functions were not used by these
classes. Especially the HTCACHE and robots.txt database allocate a very large number of objects
for statistical use, but never used them. By replacing the kelondroMapDataMining with the
kelondroMap object for these classes now less memory is allocated.


git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4986 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 0f5fe8cc53
commit 7b1c9e6aee
10 changed files with 36 additions and 41 deletions
Show Stats Download Patch File Download Diff File

8
source/de/anomic/crawler/CrawlProfile.java
Unescape View File

@ -53,7 +53,7 @@ import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverCodings;
import de.anomic.yacy.yacySeedDB;
@ -63,14 +63,14 @@ public class CrawlProfile {
static HashMap<String, Map<String, DomProfile>> domsCache = new HashMap<String, Map<String, DomProfile>>();
kelondroMapDataMining profileTable;
kelondroMap profileTable;
private File profileTableFile;
public CrawlProfile(File file) {
this.profileTableFile = file;
profileTableFile.getParentFile().mkdirs();
kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
profileTable = new kelondroMapDataMining(dyn, 500);
profileTable = new kelondroMap(dyn, 500);
}
public void clear() {
@ -79,7 +79,7 @@ public class CrawlProfile {
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs();
kelondroBLOB dyn = new kelondroBLOBTree(profileTableFile, true, true, yacySeedDB.commonHashLength, 2000, '#', kelondroNaturalOrder.naturalOrder, false, false, true);
profileTable = new kelondroMapDataMining(dyn, 500);
profileTable = new kelondroMap(dyn, 500);
}
public void close() {

8
source/de/anomic/crawler/RobotsTxt.java
Unescape View File

@ -65,7 +65,7 @@ import de.anomic.kelondro.kelondroBLOB;
import de.anomic.kelondro.kelondroBLOBHeap;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverByteBuffer;
import de.anomic.server.serverFileUtils;
@ -76,7 +76,7 @@ public class RobotsTxt {
public static final String ROBOTS_DB_PATH_SEPARATOR = ";";
kelondroMapDataMining robotsTable;
kelondroMap robotsTable;
private final File robotsTableFile;
public RobotsTxt(File robotsTableFile) {
@ -92,7 +92,7 @@ public class RobotsTxt {
} else {
blob = new kelondroBLOBTree(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true);
}
robotsTable = new kelondroMapDataMining(blob, 100);
robotsTable = new kelondroMap(blob, 100);
}
private void resetDatabase() {
@ -100,7 +100,7 @@ public class RobotsTxt {
if (robotsTable != null) robotsTable.close();
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs();
robotsTable = new kelondroMapDataMining(new kelondroBLOBTree(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100);
robotsTable = new kelondroMap(new kelondroBLOBTree(robotsTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, false, false, true), 100);
}
public void clear() throws IOException {

8
source/de/anomic/data/blogBoard.java
Unescape View File

@ -68,7 +68,7 @@ import org.xml.sax.SAXException;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverDate;
import de.anomic.server.logging.serverLog;
@ -78,12 +78,12 @@ public class blogBoard {
public static final int keyLength = 64;
private static final int recordSize = 512;
kelondroMapDataMining database = null;
kelondroMap database = null;
public blogBoard(File actpath) {
new File(actpath.getParent()).mkdir();
if (database == null) {
database = new kelondroMapDataMining(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
database = new kelondroMap(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
}
public int size() {
@ -137,7 +137,7 @@ public class blogBoard {
public BlogEntry readBlogEntry(String key) {
return readBlogEntry(key, database);
}
private BlogEntry readBlogEntry(String key, kelondroMapDataMining base) {
private BlogEntry readBlogEntry(String key, kelondroMap base) {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
HashMap<String, String> record;

8
source/de/anomic/data/blogBoardComments.java
Unescape View File

@ -66,7 +66,7 @@ import org.xml.sax.SAXException;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.logging.serverLog;
@ -81,11 +81,11 @@ public class blogBoardComments {
static {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
}
private kelondroMapDataMining database = null;
private kelondroMap database = null;
public blogBoardComments(File actpath) {
new File(actpath.getParent()).mkdir();
if (database == null) {
database = new kelondroMapDataMining(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, false, false, false), 500);
database = new kelondroMap(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, false, false, false), 500);
}
}
public int size() {
@ -130,7 +130,7 @@ public class blogBoardComments {
//System.out.println("DEBUG: read from blogBoardComments");
return read(key, database);
}
private CommentEntry read(String key, kelondroMapDataMining base) {
private CommentEntry read(String key, kelondroMap base) {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
HashMap<String, String> record;

9
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -78,7 +78,6 @@ import de.anomic.index.indexWord;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.kelondro.kelondroMap;
import de.anomic.server.serverDate;
@ -99,11 +98,11 @@ public class bookmarksDB {
kelondroMap bookmarksTable; // kelondroMap bookmarksTable;
// tags
kelondroMapDataMining tagsTable;
kelondroMap tagsTable;
HashMap<String, Tag> tagCache;
// dates
kelondroMapDataMining datesTable;
kelondroMap datesTable;
// ------------------------------------
@ -120,12 +119,12 @@ public class bookmarksDB {
// tags
tagsFile.getParentFile().mkdirs();
boolean tagsFileExisted = tagsFile.exists();
this.tagsTable = new kelondroMapDataMining(new kelondroBLOBTree(tagsFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
this.tagsTable = new kelondroMap(new kelondroBLOBTree(tagsFile, true, true, 12, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
if (!tagsFileExisted) rebuildTags();
// dates
boolean datesExisted = datesFile.exists();
this.datesTable = new kelondroMapDataMining(new kelondroBLOBTree(datesFile, true, true, 20, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
this.datesTable = new kelondroMap(new kelondroBLOBTree(datesFile, true, true, 20, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
if (!datesExisted) rebuildDates();
}

6
source/de/anomic/data/messageBoard.java
Unescape View File

@ -51,7 +51,7 @@ import java.util.TimeZone;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder;
public class messageBoard {
@ -66,13 +66,13 @@ public class messageBoard {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
}
kelondroMapDataMining database = null;
kelondroMap database = null;
private int sn = 0;
public messageBoard(File path) {
new File(path.getParent()).mkdir();
if (database == null) {
database = new kelondroMapDataMining(new kelondroBLOBTree(path, true, true, categoryLength + dateFormat.length() + 2, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
database = new kelondroMap(new kelondroBLOBTree(path, true, true, categoryLength + dateFormat.length() + 2, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
sn = 0;
}

8
source/de/anomic/data/userDB.java
Unescape View File

@ -59,7 +59,7 @@ import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroCloneableIterator;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder;
import de.anomic.server.serverCodings;
@ -68,7 +68,7 @@ public final class userDB {
public static final int USERNAME_MAX_LENGTH = 128;
public static final int USERNAME_MIN_LENGTH = 4;
kelondroMapDataMining userTable;
kelondroMap userTable;
private final File userTableFile;
HashMap<String, String> ipUsers = new HashMap<String, String>();
HashMap<String, Object> cookieUsers = new HashMap<String, Object>();
@ -76,7 +76,7 @@ public final class userDB {
public userDB(File userTableFile) {
this.userTableFile = userTableFile;
userTableFile.getParentFile().mkdirs();
this.userTable = new kelondroMapDataMining(new kelondroBLOBTree(userTableFile, true, true, 128, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
this.userTable = new kelondroMap(new kelondroBLOBTree(userTableFile, true, true, 128, 256, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
}
void resetDatabase() {
@ -84,7 +84,7 @@ public final class userDB {
if (userTable != null) userTable.close();
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs();
userTable = new kelondroMapDataMining(new kelondroBLOBTree(userTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
userTable = new kelondroMap(new kelondroBLOBTree(userTableFile, true, true, 256, 512, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 10);
}
public void close() {

12
source/de/anomic/data/wikiBoard.java
Unescape View File

@ -51,7 +51,7 @@ import java.util.TimeZone;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroMap;
import de.anomic.kelondro.kelondroNaturalOrder;
public class wikiBoard {
@ -66,18 +66,18 @@ public class wikiBoard {
SimpleFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
}
kelondroMapDataMining datbase = null;
kelondroMapDataMining bkpbase = null;
kelondroMap datbase = null;
kelondroMap bkpbase = null;
static HashMap<String, String> authors = new HashMap<String, String>();
public wikiBoard(File actpath, File bkppath) {
new File(actpath.getParent()).mkdirs();
if (datbase == null) {
datbase = new kelondroMapDataMining(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
datbase = new kelondroMap(new kelondroBLOBTree(actpath, true, true, keyLength, recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) {
bkpbase = new kelondroMapDataMining(new kelondroBLOBTree(bkppath, true, true, keyLength + dateFormat.length(), recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
bkpbase = new kelondroMap(new kelondroBLOBTree(bkppath, true, true, keyLength + dateFormat.length(), recordSize, '_', kelondroNaturalOrder.naturalOrder, true, false, false), 500);
}
}
@ -290,7 +290,7 @@ public class wikiBoard {
return read(key, datbase);
}
entry read(String key, kelondroMapDataMining base) {
entry read(String key, kelondroMap base) {
try {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);

4
source/de/anomic/kelondro/kelondroMapDataMining.java
Unescape View File

@ -41,10 +41,6 @@ public class kelondroMapDataMining extends kelondroMap {
private HashMap<String, Object> accMap; // to store accumulations of specific fields
private int elementCount;
public kelondroMapDataMining(kelondroBLOB dyn, int cachesize) {
this(dyn, cachesize, null, null, null, null, null);
}
@SuppressWarnings({ "unchecked", "null" })
public kelondroMapDataMining(kelondroBLOB dyn, int cachesize, String[] sortfields, String[] longaccfields, String[] doubleaccfields, Method externalInitializer, Object externalHandler) {
super(dyn, cachesize);

6
source/de/anomic/plasma/plasmaHTCache.java
Unescape View File

@ -57,7 +57,7 @@ import de.anomic.kelondro.kelondroBLOBHeap;
import de.anomic.kelondro.kelondroBLOBTree;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.kelondro.kelondroMap;
import de.anomic.plasma.cache.IResourceInfo;
import de.anomic.plasma.cache.ResourceInfoFactory;
import de.anomic.plasma.cache.UnsupportedProtocolException;
@ -80,7 +80,7 @@ public final class plasmaHTCache {
private static final int stackLimit = 150; // if we exceed that limit, we do not check idle
public static final long oneday = 1000 * 60 * 60 * 24; // milliseconds of a day
private static kelondroMapDataMining responseHeaderDB = null;
private static kelondroMap responseHeaderDB = null;
private static final ConcurrentLinkedQueue<Entry> cacheStack = new ConcurrentLinkedQueue<Entry>();
private static final ConcurrentHashMap<String, File> cacheAge = new ConcurrentHashMap<String, File>(); // a <date+hash, cache-path> - relation
public static long curCacheSize = 0;
@ -272,7 +272,7 @@ public final class plasmaHTCache {
} else {
blob = new kelondroBLOBTree(dbfile, true, true, yacySeedDB.commonHashLength, 150, '#', kelondroBase64Order.enhancedCoder, false, false, true);
}
responseHeaderDB = new kelondroMapDataMining(blob, 500);
responseHeaderDB = new kelondroMap(blob, 500);
}
private static void deleteOldHTCache(File directory) {

Write Preview
Loading…
Cancel
Save