boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

- fix for caching of OSM tiles. if you want that this fix applies to your peer, please delete the crawl profiles

Browse Source 
- fix for initial generation of crawl profiles (one more reason to remove your crawl profiles)
- more String -> byte[] migration
- more logging for cache store/hit

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6874 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 15 years ago
parent 2126c03a62
commit 2a8f70f0ca
20 changed files with 114 additions and 212 deletions
Show Stats Download Patch File Download Diff File

10
htroot/CacheResource_p.java
Unescape View File

@ -22,8 +22,6 @@
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import java.io.IOException;
import java.net.MalformedURLException;
import net.yacy.kelondro.data.meta.DigestURI;
@ -52,13 +50,7 @@ public class CacheResource_p {
}
byte[] resource = null;
// trying to load the resource body
try {
resource = Cache.getContent(url);
} catch (IOException e) {
Log.logException(e);
return prop;
}
resource = Cache.getContent(url);
if (resource == null) return prop;
//ResponseHeader responseHeader = Cache.getResponseHeader(url);
//String resMime = responseHeader.mime();

9
htroot/ViewFile.java
Unescape View File

@ -167,16 +167,9 @@ public class ViewFile {
// loading the resource content as byte array
prop.put("error_incache", Cache.has(url) ? 1 : 0);
byte[] resource = null;
ResponseHeader responseHeader = null;
String resMime = null;
// trying to load the resource body
try {
resource = Cache.getContent(url);
} catch (IOException e) {
Log.logException(e);
resource = null;
}
byte[] resource = Cache.getContent(url);
if (resource == null && authorized) {
// load resource from net

21
source/de/anomic/crawler/CrawlProfile.java
Unescape View File

@ -147,11 +147,11 @@ public class CrawlProfile {
public entry newEntry(final Map<String, String> mem) {
final entry ne = new entry(mem);
try {
profileTable.put(ne.handle(), ne.map());
profileTable.put(ne.handle().getBytes(), ne.map());
} catch (final Exception e) {
clear();
try {
profileTable.put(ne.handle(), ne.map());
profileTable.put(ne.handle().getBytes(), ne.map());
} catch (final Exception ee) {
Log.logException(e);
System.exit(0);
@ -184,11 +184,11 @@ public class CrawlProfile {
xsstopw, xdstopw, xpstopw,
cacheStrategy);
try {
profileTable.put(ne.handle(), ne.map());
profileTable.put(ne.handle().getBytes(), ne.map());
} catch (final Exception e) {
clear();
try {
profileTable.put(ne.handle(), ne.map());
profileTable.put(ne.handle().getBytes(), ne.map());
} catch (final Exception ee) {
Log.logException(e);
System.exit(0);
@ -198,19 +198,14 @@ public class CrawlProfile {
}
public boolean hasEntry(final String handle) {
try {
return profileTable.has(handle);
} catch (final IOException e) {
Log.logException(e);
return false;
}
return profileTable.has(handle.getBytes());
}
public entry getEntry(final String handle) {
if (profileTable == null) return null;
Map<String, String> m;
try {
m = profileTable.get(handle);
m = profileTable.get(handle.getBytes());
} catch (final IOException e) {
Log.logException(e);
return null;
@ -222,7 +217,7 @@ public class CrawlProfile {
public void changeEntry(final entry e, final String propName, final String newValue) throws IOException, RowSpaceExceededException {
e.mem.put(propName, newValue);
assert e.handle() != null;
profileTable.put(e.handle(), e.mem);
profileTable.put(e.handle().getBytes(), e.mem);
}
public long getRecrawlDate(final long oldTimeMinutes) {
@ -305,7 +300,7 @@ public class CrawlProfile {
final boolean xsstopw, final boolean xdstopw, final boolean xpstopw,
final CacheStrategy cacheStrategy) {
if (name == null || name.length() == 0) throw new NullPointerException("name must not be null");
final String handle = (startURL == null) ? Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(0, Word.commonHashLength) : new String(startURL.hash());
final String handle = (startURL == null) ? Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(name)).substring(0, Word.commonHashLength) : new String(startURL.hash());
mem = new HashMap<String, String>(40);
mem.put(HANDLE, handle);
mem.put(NAME, name);

2
source/de/anomic/crawler/CrawlSwitchboard.java
Unescape View File

@ -188,7 +188,7 @@ public final class CrawlSwitchboard {
if (this.defaultMediaSnippetLocalProfile == null) {
// generate new default entry for snippet fetch and optional crawling
defaultMediaSnippetLocalProfile = this.profilesActiveCrawls.newEntry(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA, null, CrawlProfile.MATCH_ALL, CrawlProfile.MATCH_BAD_URL, 0,
this.profilesActiveCrawls.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE), -1, -1, true, false, false, false, false, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST);
this.profilesActiveCrawls.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE), -1, -1, true, false, false, true, false, false, true, true, false, CrawlProfile.CacheStrategy.IFEXIST);
}
if (this.defaultMediaSnippetGlobalProfile == null) {
// generate new default entry for snippet fetch and optional crawling

6
source/de/anomic/data/BookmarkDate.java
Unescape View File

@ -54,7 +54,7 @@ public class BookmarkDate {
public Entry getDate(final String date) {
Map<String, String> map;
try {
map = datesTable.get(date);
map = datesTable.get(date.getBytes());
} catch (final IOException e) {
map = null;
}
@ -141,13 +141,13 @@ public class BookmarkDate {
public void setDatesTable() {
if (this.size() >0) {
try {
datesTable.put(getDateString(), mem);
datesTable.put(getDateString().getBytes(), mem);
} catch (Exception e) {
Log.logException(e);
}
} else {
try {
datesTable.remove(getDateString());
datesTable.remove(getDateString().getBytes());
} catch (IOException e) {
Log.logException(e);
}

12
source/de/anomic/data/blogBoard.java
Unescape View File

@ -81,11 +81,7 @@ public class blogBoard {
* @return true if the database contains the element, else false
*/
public boolean contains(final String key) {
try {
return database.has(key);
} catch (IOException ex) {
return false;
}
return database.has(key.getBytes());
}
public void close() {
@ -126,7 +122,7 @@ public class blogBoard {
public String writeBlogEntry(final BlogEntry page) {
String ret = null;
try {
database.put(page.key, page.record);
database.put(page.key.getBytes(), page.record);
ret = page.key;
} catch (IOException ex) {
Log.logException(ex);
@ -144,7 +140,7 @@ public class blogBoard {
final String normalized = normalize(key);
Map<String, String> record;
try {
record = base.get(normalized.substring(0, Math.min(normalized.length(), KEY_LENGTH)));
record = base.get(normalized.substring(0, Math.min(normalized.length(), KEY_LENGTH)).getBytes());
} catch (final IOException e) {
record = null;
}
@ -240,7 +236,7 @@ public class blogBoard {
public void deleteBlogEntry(final String key) {
try {
database.remove(normalize(key));
database.remove(normalize(key).getBytes());
} catch (final IOException e) { }
}

6
source/de/anomic/data/blogBoardComments.java
Unescape View File

@ -111,7 +111,7 @@ public class blogBoardComments {
public String write(final CommentEntry page) {
// writes a new page and returns key
try {
database.put(page.key, page.record);
database.put(page.key.getBytes(), page.record);
return page.key;
} catch (final Exception e) {
Log.logException(e);
@ -128,7 +128,7 @@ public class blogBoardComments {
copyOfKey = copyOfKey.substring(0, Math.min(copyOfKey.length(), KEY_LENGTH));
Map<String, String> record;
try {
record = base.get(copyOfKey);
record = base.get(copyOfKey.getBytes());
} catch (final IOException e) {
record = null;
}
@ -226,7 +226,7 @@ public class blogBoardComments {
public void delete(final String key) {
try {
database.remove(normalize(key));
database.remove(normalize(key).getBytes());
}
catch (final IOException e) { }
}

6
source/de/anomic/data/bookmarksDB.java
Unescape View File

@ -317,7 +317,7 @@ public class bookmarksDB {
// adding a bookmark to the bookmarksDB
public void saveBookmark(final Bookmark bookmark){
try {
bookmarks.put(bookmark.getUrlHash(), bookmark.entry);
bookmarks.put(bookmark.getUrlHash().getBytes(), bookmark.entry);
} catch (final Exception e) {
Log.logException(e);
}
@ -330,7 +330,7 @@ public class bookmarksDB {
public Bookmark getBookmark(final String urlHash){
try {
final Map<String, String> map = bookmarks.get(urlHash);
final Map<String, String> map = bookmarks.get(urlHash.getBytes());
return (map == null) ? null : new Bookmark(map);
} catch (final IOException e) {
return null;
@ -353,7 +353,7 @@ public class bookmarksDB {
Bookmark b;
try {
b = getBookmark(urlHash);
bookmarks.remove(urlHash);
bookmarks.remove(urlHash.getBytes());
} catch (final IOException e) {
b = null;
}

6
source/de/anomic/data/messageBoard.java
Unescape View File

@ -182,7 +182,7 @@ public class messageBoard {
public String write(final entry message) {
// writes a message and returns key
try {
database.put(message.key, message.record);
database.put(message.key.getBytes(), message.record);
return message.key;
} catch (final Exception e) {
Log.logException(e);
@ -193,7 +193,7 @@ public class messageBoard {
public entry read(final String key) {
Map<String, String> record;
try {
record = database.get(key);
record = database.get(key.getBytes());
} catch (final IOException e) {
return null;
}
@ -202,7 +202,7 @@ public class messageBoard {
public void remove(final String key) {
try {
database.remove(key);
database.remove(key.getBytes());
} catch (final IOException e) {
}
}

10
source/de/anomic/data/userDB.java
Unescape View File

@ -85,7 +85,7 @@ public final class userDB {
public void removeEntry(final String hostName) {
try {
userTable.remove(hostName.toLowerCase());
userTable.remove(hostName.toLowerCase().getBytes());
} catch (final IOException e) {}
}
@ -95,7 +95,7 @@ public final class userDB {
}
Map<String, String> record;
try {
record = userTable.get(userName);
record = userTable.get(userName.getBytes());
} catch (final IOException e) {
return null;
}
@ -110,7 +110,7 @@ public final class userDB {
public String addEntry(final Entry entry) {
try {
userTable.put(entry.userName,entry.mem);
userTable.put(entry.userName.getBytes(), entry.mem);
return entry.userName;
} catch (final Exception e) {
Log.logException(e);
@ -484,7 +484,7 @@ public final class userDB {
}
try {
userDB.this.userTable.put(getUserName(), this.mem);
userDB.this.userTable.put(getUserName().getBytes(), this.mem);
} catch(final Exception e){
Log.logException(e);
}
@ -501,7 +501,7 @@ public final class userDB {
public void setProperty(final String propName, final String newValue) throws IOException, RowSpaceExceededException {
this.mem.put(propName, newValue);
userDB.this.userTable.put(getUserName(), this.mem);
userDB.this.userTable.put(getUserName().getBytes(), this.mem);
}
public String getProperty(final String propName, final String defaultValue) {

6
source/de/anomic/data/wiki/wikiBoard.java
Unescape View File

@ -265,9 +265,9 @@ public class wikiBoard {
//System.out.println("key = " + page.key);
//System.out.println("oldDate = " + oldDate);
//System.out.println("record = " + oldEntry.record.toString());
bkpbase.put(page.key + dateString(oldDate), oldEntry.record);
bkpbase.put((page.key + dateString(oldDate)).getBytes(), oldEntry.record);
// write the new page
datbase.put(page.key, page.record);
datbase.put(page.key.getBytes(), page.record);
return page.key;
} catch (final Exception e) {
Log.logException(e);
@ -283,7 +283,7 @@ public class wikiBoard {
try {
key = normalize(key);
if (key.length() > keyLength) key = key.substring(0, keyLength);
final Map<String, String> record = base.get(key);
final Map<String, String> record = base.get(key.getBytes());
if (record == null) return newEntry(key, "anonymous", "127.0.0.1", "New Page", "".getBytes());
return new entry(key, record);
} catch (final IOException e) {

36
source/de/anomic/http/client/Cache.java
Unescape View File

@ -59,7 +59,7 @@ public final class Cache {
private static Compressor fileDB = null;
private static ArrayStack fileDBunbuffered = null;
private static long maxCacheSize = 0l;
private static long maxCacheSize = Long.MAX_VALUE;
private static File cachePath = null;
private static String prefix;
public static final Log log = new Log("HTCACHE");
@ -111,13 +111,14 @@ public final class Cache {
public static void store(final DigestURI url, final ResponseHeader responseHeader, final byte[] file) throws IOException {
if (responseHeader == null) throw new IOException("Cache.store of url " + url.toString() + " not possible: responseHeader == null");
if (file == null) throw new IOException("Cache.store of url " + url.toString() + " not possible: file == null");
log.logInfo("storing content of url " + url.toString() + ", " + file.length + " bytes");
// store the response header into the header database
final HashMap<String, String> hm = new HashMap<String, String>();
hm.putAll(responseHeader);
hm.put("@@URL", url.toNormalform(true, false));
try {
responseHeaderDB.put(new String(url.hash()), hm);
responseHeaderDB.put(url.hash(), hm);
} catch (Exception e) {
throw new IOException("Cache.store: cannot write to headerDB: " + e.getMessage());
}
@ -140,20 +141,17 @@ public final class Cache {
*/
public static boolean has(final DigestURI url) {
boolean headerExists;
try {
headerExists = responseHeaderDB.has(new String(url.hash()));
} catch (IOException e1) {
try {
fileDB.remove(url.hash());
} catch (IOException e) {}
return false;
}
headerExists = responseHeaderDB.has(url.hash());
boolean fileExists = fileDB.has(url.hash());
if (headerExists && fileExists) return true;
if (!headerExists && !fileExists) return false;
// if not both is there then we do a clean-up
if (headerExists) try {
responseHeaderDB.remove(new String(url.hash()));
log.logWarning("header but not content of url " + url.toString() + " in cache; cleaned up");
responseHeaderDB.remove(url.hash());
} catch (IOException e) {}
if (fileExists) try {
log.logWarning("content but not header of url " + url.toString() + " in cache; cleaned up");
fileDB.remove(url.hash());
} catch (IOException e) {}
return false;
@ -173,8 +171,9 @@ public final class Cache {
// loading data from database
Map<String, String> hdb;
try {
hdb = responseHeaderDB.get(new String(url.hash()));
hdb = responseHeaderDB.get(url.hash());
} catch (final IOException e) {
Log.logException(e);
return null;
}
if (hdb == null) return null;
@ -191,24 +190,29 @@ public final class Cache {
* is returned.
* @throws IOException
*/
public static byte[] getContent(final DigestURI url) throws IOException {
public static byte[] getContent(final DigestURI url) {
// load the url as resource from the cache
try {
return fileDB.get(url.hash());
byte[] b = fileDB.get(url.hash());
if (b == null) return null;
log.logInfo("cache hit for url " + url.toString() + ", " + b.length + " bytes");
return b;
} catch (UnsupportedEncodingException e) {
Log.logException(e);
return null;
} catch (IOException e) {
Log.logException(e);
return null;
}
}
/**
* removed response header and cached content from the database
* @param url
* @throws IOException
*/
public static void delete(final DigestURI url) throws IOException {
responseHeaderDB.remove(new String(url.hash()));
responseHeaderDB.remove(url.hash());
fileDB.remove(url.hash());
}
}

6
source/de/anomic/search/Switchboard.java
Unescape View File

@ -1700,7 +1700,7 @@ public final class Switchboard extends serverSwitch {
final long parsingStartTime = System.currentTimeMillis();
// fetch the document from the response
byte[] b = response.getContent();
if (b == null) try {
if (b == null) {
// fetch the document from cache
b = Cache.getContent(response.url());
if (b == null) {
@ -1708,10 +1708,6 @@ public final class Switchboard extends serverSwitch {
addURLtoErrorDB(response.url(), response.referrerHash(), response.initiator(), response.name(), "missing");
return null;
}
} catch (IOException e) {
this.log.logWarning("Unable fetch the resource '" + response.url() + "'. from the cache: " + e.getMessage());
addURLtoErrorDB(response.url(), response.referrerHash(), response.initiator(), response.name(), e.getMessage());
return null;
}
try {

7
source/de/anomic/yacy/graphics/OSMTile.java
Unescape View File

@ -80,12 +80,7 @@ public class OSMTile {
return null;
}
//System.out.println("*** DEBUG: fetching OSM tile: " + tileURL.toNormalform(true, true));
byte[] tileb = null;
try {
tileb = Cache.getContent(tileURL);
} catch (IOException e1) {
Log.logException(e1);
}
byte[] tileb = Cache.getContent(tileURL);
if (tileb == null) {
// download resource using the crawler and keep resource in memory if possible
Response entry = null;

40
source/de/anomic/yacy/yacySeedDB.java
Unescape View File

@ -268,9 +268,9 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (seedActiveDB.isEmpty() && seedPassiveDB.isEmpty() && seedPotentialDB.isEmpty()) return; // avoid that the own seed is initialized too early
if (this.mySeed == null) initMySeed();
try {
seedActiveDB.remove(mySeed.hash);
seedPassiveDB.remove(mySeed.hash);
seedPotentialDB.remove(mySeed.hash);
seedActiveDB.remove(mySeed.hash.getBytes());
seedPassiveDB.remove(mySeed.hash.getBytes());
seedPotentialDB.remove(mySeed.hash.getBytes());
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
}
@ -292,12 +292,12 @@ public final class yacySeedDB implements AlternativeDomainNames {
Log.logWarning("yacySeedDB", "could not create directories for "+ seedDBFile.getParent());
}
try {
return new MapDataMining(seedDBFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, sortFields, longaccFields, doubleaccFields, null, this);
return new MapDataMining(seedDBFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, sortFields, longaccFields, doubleaccFields, this);
} catch (final Exception e) {
// try again
FileUtils.deletedelete(seedDBFile);
try {
return new MapDataMining(seedDBFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, sortFields, longaccFields, doubleaccFields, null, this);
return new MapDataMining(seedDBFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, sortFields, longaccFields, doubleaccFields, this);
} catch (IOException e1) {
Log.logException(e1);
System.exit(-1);
@ -442,10 +442,10 @@ public final class yacySeedDB implements AlternativeDomainNames {
nameLookupCache.put(seed.getName(), seed.hash);
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedActiveDB.put(seed.hash, seedPropMap);
seedActiveDB.put(seed.hash.getBytes(), seedPropMap);
}
seedPassiveDB.remove(seed.hash);
seedPotentialDB.remove(seed.hash);
seedPassiveDB.remove(seed.hash.getBytes());
seedPotentialDB.remove(seed.hash.getBytes());
} catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
resetActiveTable();
@ -456,14 +456,14 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (seed.isProper(false) != null) return;
try {
nameLookupCache.remove(seed.getName());
seedActiveDB.remove(seed.hash);
seedPotentialDB.remove(seed.hash);
seedActiveDB.remove(seed.hash.getBytes());
seedPotentialDB.remove(seed.hash.getBytes());
} catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedPassiveDB.put(seed.hash, seedPropMap);
seedPassiveDB.put(seed.hash.getBytes(), seedPropMap);
}
} catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
@ -475,14 +475,14 @@ public final class yacySeedDB implements AlternativeDomainNames {
if (seed.isProper(false) != null) return;
try {
nameLookupCache.remove(seed.getName());
seedActiveDB.remove(seed.hash);
seedPassiveDB.remove(seed.hash);
seedActiveDB.remove(seed.hash.getBytes());
seedPassiveDB.remove(seed.hash.getBytes());
} catch (final Exception e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
//seed.put(yacySeed.LASTSEEN, yacyCore.shortFormatter.format(new Date(yacyCore.universalTime())));
try {
final Map<String, String> seedPropMap = seed.getMap();
synchronized (seedPropMap) {
seedPotentialDB.put(seed.hash, seedPropMap);
seedPotentialDB.put(seed.hash.getBytes(), seedPropMap);
}
} catch (final Exception e) {
yacyCore.log.logSevere("ERROR add: seed.db corrupt (" + e.getMessage() + "); resetting seed.db", e);
@ -493,14 +493,14 @@ public final class yacySeedDB implements AlternativeDomainNames {
public synchronized void removeDisconnected(final String peerHash) {
if(peerHash == null) return;
try {
seedPassiveDB.remove(peerHash);
seedPassiveDB.remove(peerHash.getBytes());
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
}
public synchronized void removePotential(final String peerHash) {
if(peerHash == null) return;
try {
seedPotentialDB.remove(peerHash);
seedPotentialDB.remove(peerHash.getBytes());
} catch (final IOException e) { Log.logWarning("yacySeedDB", "could not remove hash ("+ e.getClass() +"): "+ e.getMessage()); }
}
@ -521,7 +521,7 @@ public final class yacySeedDB implements AlternativeDomainNames {
if ((this.mySeed != null) && (hash.equals(mySeed.hash))) return mySeed;
ConcurrentHashMap<String, String> entry = new ConcurrentHashMap<String, String>();
try {
Map<String, String> map = database.get(hash);
Map<String, String> map = database.get(hash.getBytes());
if (map == null) return null;
entry.putAll(map);
} catch (final IOException e) {
@ -558,13 +558,13 @@ public final class yacySeedDB implements AlternativeDomainNames {
}
yacySeed s = get(hash, seedActiveDB);
if (s != null) try { seedActiveDB.put(hash, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
if (s != null) try { seedActiveDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
s = get(hash, seedPassiveDB);
if (s != null) try { seedPassiveDB.put(hash, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
if (s != null) try { seedPassiveDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
s = get(hash, seedPotentialDB);
if (s != null) try { seedPotentialDB.put(hash, seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
if (s != null) try { seedPotentialDB.put(hash.getBytes(), seed.getMap()); return;} catch (final Exception e) {Log.logException(e);}
}
public yacySeed lookupByName(String peerName) {

14
source/net/yacy/kelondro/blob/Heap.java
Unescape View File

@ -516,13 +516,13 @@ public final class Heap extends HeapModifier implements BLOB {
try {
//f.delete();
final MapHeap heap = new MapHeap(f, 12, NaturalOrder.naturalOrder, 1024 * 512, 500, '_');
heap.put("aaaaaaaaaaaa", map("aaaaaaaaaaaa", "eins zwei drei"));
heap.put("aaaaaaaaaaab", map("aaaaaaaaaaab", "vier fuenf sechs"));
heap.put("aaaaaaaaaaac", map("aaaaaaaaaaac", "sieben acht neun"));
heap.put("aaaaaaaaaaad", map("aaaaaaaaaaad", "zehn elf zwoelf"));
heap.remove("aaaaaaaaaaab");
heap.remove("aaaaaaaaaaac");
heap.put("aaaaaaaaaaaX", map("aaaaaaaaaaad", "WXYZ"));
heap.put("aaaaaaaaaaaa".getBytes(), map("aaaaaaaaaaaa", "eins zwei drei"));
heap.put("aaaaaaaaaaab".getBytes(), map("aaaaaaaaaaab", "vier fuenf sechs"));
heap.put("aaaaaaaaaaac".getBytes(), map("aaaaaaaaaaac", "sieben acht neun"));
heap.put("aaaaaaaaaaad".getBytes(), map("aaaaaaaaaaad", "zehn elf zwoelf"));
heap.remove("aaaaaaaaaaab".getBytes());
heap.remove("aaaaaaaaaaac".getBytes());
heap.put("aaaaaaaaaaaX".getBytes(), map("aaaaaaaaaaad", "WXYZ"));
heap.close();
} catch (final IOException e) {
Log.logException(e);

40
source/net/yacy/kelondro/blob/MapDataMining.java
Unescape View File

@ -29,14 +29,11 @@ package net.yacy.kelondro.blob;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.ByteOrder;
import net.yacy.kelondro.order.CloneableIterator;
import net.yacy.kelondro.util.ScoreCluster;
@ -61,7 +58,6 @@ public class MapDataMining extends MapHeap {
final String[] sortfields,
final String[] longaccfields,
final String[] doubleaccfields,
final Method externalInitializer,
final Object externalHandler) throws IOException {
super(heapFile, keylength, ordering, buffermax, cachesize, '_');
@ -103,19 +99,19 @@ public class MapDataMining extends MapHeap {
// fill cluster and accumulator with values
if ((sortfields != null) || (longaccfields != null) || (doubleaccfields != null)) try {
final CloneableIterator<byte[]> it = super.keys(true, false);
String mapname;
byte[] mapnameb;
String cell;
long valuel;
double valued;
Map<String, String> map;
while (it.hasNext()) {
mapname = new String(it.next());
map = super.get(mapname);
mapnameb = it.next();
map = super.get(mapnameb);
if (map == null) break;
if (sortfields != null && cluster != null) for (int i = 0; i < sortfields.length; i++) {
cell = map.get(sortfields[i]);
if (cell != null) cluster[i].setScore(mapname, ScoreCluster.object2score(cell));
if (cell != null) cluster[i].setScore(new String(mapnameb), ScoreCluster.object2score(cell));
}
if (longaccfields != null && longaccumulator != null) for (int i = 0; i < longaccfields.length; i++) {
@ -135,18 +131,6 @@ public class MapDataMining extends MapHeap {
doubleaccumulator[i] = new Double(doubleaccumulator[i].doubleValue() + valued);
} catch (final NumberFormatException e) {}
}
if ((externalHandler != null) && (externalInitializer != null)) {
try {
externalInitializer.invoke(externalHandler, new Object[]{mapname, map});
} catch (final IllegalArgumentException e) {
Log.logException(e);
} catch (final IllegalAccessException e) {
Log.logException(e);
} catch (final InvocationTargetException e) {
Log.logException(e);
}
}
}
} catch (final IOException e) {}
@ -187,9 +171,9 @@ public class MapDataMining extends MapHeap {
}
@Override
public synchronized void put(final String key, final Map<String, String> newMap) throws IOException, RowSpaceExceededException {
public synchronized void put(final byte[] key, final Map<String, String> newMap) throws IOException, RowSpaceExceededException {
assert (key != null);
assert (key.length() > 0);
assert (key.length > 0);
assert (newMap != null);
// update elementCount
@ -207,7 +191,7 @@ public class MapDataMining extends MapHeap {
super.put(key, newMap);
// update sortCluster
if (sortClusterMap != null) updateSortCluster(key, newMap);
if (sortClusterMap != null) updateSortCluster(new String(key), newMap);
}
private void updateAcc(final Map<String, String> map, final boolean add) {
@ -260,7 +244,7 @@ public class MapDataMining extends MapHeap {
}
@Override
public synchronized void remove(final String key) throws IOException {
public synchronized void remove(final byte[] key) throws IOException {
if (key == null) return;
// update elementCount
@ -272,7 +256,7 @@ public class MapDataMining extends MapHeap {
if ((longaccfields != null) || (doubleaccfields != null)) updateAcc(map, false);
// remove from sortCluster
if (sortfields != null) deleteSortCluster(key);
if (sortfields != null) deleteSortCluster(new String(key));
}
}
super.remove(key);
@ -390,10 +374,10 @@ public class MapDataMining extends MapHeap {
private Map<String, String> next0() {
if (keyIterator == null) return null;
String nextKey;
byte[] nextKey;
Map<String, String> map;
while (keyIterator.hasNext()) {
nextKey = new String(keyIterator.next());
nextKey = keyIterator.next();
try {
map = get(nextKey);
} catch (final IOException e) {
@ -401,7 +385,7 @@ public class MapDataMining extends MapHeap {
}
assert map != null;
if (map == null) continue; // circumvention of a modified exception
map.put("key", nextKey);
map.put("key", new String(nextKey));
return map;
}
return null;

79
source/net/yacy/kelondro/blob/MapHeap.java
Unescape View File

@ -128,21 +128,20 @@ public class MapHeap {
* @throws IOException
* @throws RowSpaceExceededException
*/
public void put(String key, final Map<String, String> newMap) throws IOException, RowSpaceExceededException {
public void put(byte[] key, final Map<String, String> newMap) throws IOException, RowSpaceExceededException {
assert key != null;
assert key.length() > 0;
assert key.length > 0;
assert newMap != null;
key = normalizeKey(key);
byte[] keyb = key.getBytes();
String s = map2string(newMap, "W" + DateFormatter.formatShortSecond() + " ");
assert s != null;
byte[] sb = s.getBytes();
synchronized (this) {
// write entry
if (blob != null) blob.put(keyb, sb);
if (blob != null) blob.put(key, sb);
// write map to cache
if (cache != null) cache.put(key, newMap);
if (cache != null) cache.put(new String(key), newMap);
}
}
@ -164,26 +163,6 @@ public class MapHeap {
blob.remove(key);
}
}
/**
* remove a Map
* @param key the primary key
* @throws IOException
*/
public void remove(String key) throws IOException {
// update elementCount
if (key == null) return;
key = normalizeKey(key);
byte[] keyb = key.getBytes();
synchronized (this) {
// remove from cache
cache.remove(key);
// remove from file
blob.remove(keyb);
}
}
/**
* check if a specific key is in the database
@ -191,17 +170,6 @@ public class MapHeap {
* @return
* @throws IOException
*/
public boolean has(String key) throws IOException {
assert key != null;
if (cache == null) return false; // case may appear during shutdown
key = normalizeKey(key);
byte[] keyb = key.getBytes();
boolean h;
synchronized (this) {
h = this.cache.containsKey(key) || this.blob.has(keyb);
}
return h;
}
public boolean has(byte[] key) {
assert key != null;
@ -220,25 +188,10 @@ public class MapHeap {
* @return
* @throws IOException
*/
public Map<String, String> get(final String key) throws IOException {
public Map<String, String> get(final byte[] key) throws IOException {
if (key == null) return null;
return get(key, true);
}
private String normalizeKey(String key) {
if (blob == null || key == null) return key;
if (key.length() > blob.keylength()) {
return key.substring(0, blob.keylength());
}
if (key.length() < blob.keylength()) {
byte[] k = key.getBytes();
byte[] b = new byte[blob.keylength()];
System.arraycopy(k, 0, b, 0, k.length);
for (int i = k.length; i < b.length; i++) b[i] = (byte) fillchar;
return new String(b);
}
return key;
}
private byte[] normalizeKey(byte[] key) {
if (blob == null || key == null) return key;
@ -256,21 +209,21 @@ public class MapHeap {
return key;
}
protected Map<String, String> get(String key, final boolean storeCache) throws IOException {
protected Map<String, String> get(byte[] key, final boolean storeCache) throws IOException {
// load map from cache
assert key != null;
if (cache == null) return null; // case may appear during shutdown
key = normalizeKey(key);
byte[] keyb = key.getBytes();
Map<String, String> map;
if (storeCache) {
synchronized (this) {
map = cache.get(key);
String keys = new String(key);
map = cache.get(keys);
if (map != null) return map;
// read object
final byte[] b = blob.get(keyb);
final byte[] b = blob.get(key);
if (b == null) return null;
try {
map = bytes2map(b);
@ -279,7 +232,7 @@ public class MapHeap {
}
// write map to cache
cache.put(key, map);
cache.put(keys, map);
}
// return value
@ -287,9 +240,9 @@ public class MapHeap {
} else {
byte[] b;
synchronized (this) {
map = cache.get(key);
map = cache.get(new String(key));
if (map != null) return map;
b = blob.get(keyb);
b = blob.get(key);
}
if (b == null) return null;
try {
@ -390,7 +343,7 @@ public class MapHeap {
return null;
}
try {
final Map<String, String> obj = get(new String(nextKey));
final Map<String, String> obj = get(nextKey);
if (obj == null) throw new kelondroException("no more elements available");
return obj;
} catch (final IOException e) {
@ -413,9 +366,9 @@ public class MapHeap {
MapHeap map = new MapHeap(f, 12, NaturalOrder.naturalOrder, 1024 * 1024, 1024, '_');
// put some values into the map
Map<String, String> m = new HashMap<String, String>();
m.put("k", "000"); map.put("123", m);
m.put("k", "111"); map.put("456", m);
m.put("k", "222"); map.put("789", m);
m.put("k", "000"); map.put("123".getBytes(), m);
m.put("k", "111"); map.put("456".getBytes(), m);
m.put("k", "222"); map.put("789".getBytes(), m);
// iterate over keys
Iterator<byte[]> i = map.keys(true, false);
while (i.hasNext()) {

8
source/net/yacy/repository/LoaderDispatcher.java
Unescape View File

@ -190,13 +190,7 @@ public final class LoaderDispatcher {
// now see if there is a cache entry
ResponseHeader cachedResponse = (request.url().isLocal()) ? null : Cache.getResponseHeader(request.url());
byte[] content = null;
try {
content = (cachedResponse == null) ? null : Cache.getContent(request.url());
} catch (IOException e) {
Log.logException(e);
content = null;
}
byte[] content = (cachedResponse == null) ? null : Cache.getContent(request.url());
if (cachedResponse != null && content != null) {
// yes we have the content

2
source/net/yacy/yacy.java
Unescape View File

@ -917,7 +917,7 @@ public final class yacy {
final String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
final File dbFile = new File(yacyDBPath,dbFileNames[i]);
final MapDataMining db = new MapDataMining(dbFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null);
final MapDataMining db = new MapDataMining(dbFile, Word.commonHashLength, Base64Order.enhancedCoder, 1024 * 512, 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null);
MapDataMining.mapIterator it;
it = db.maps(true, false);

Write Preview
Loading…
Cancel
Save