boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

fixed a bug in kelondroBLOBHeap. The following files are probably inconsistent and should be deleted:

Browse Source 
DATA/HTCACHE/responseHeader.heap
DATA/PLASMADB/crawlRobotsTxt.heap

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4988 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
orbiter 17 years ago
parent 28d5703f8a
commit 38eb5bd1ee
2 changed files with 73 additions and 16 deletions
Show Stats Download Patch File Download Diff File

88
source/de/anomic/kelondro/kelondroBLOBHeap.java
Unescape View File

@ -38,11 +38,20 @@ import de.anomic.server.logging.serverLog;
public final class kelondroBLOBHeap implements kelondroBLOB { public final class kelondroBLOBHeap implements kelondroBLOB {
private kelondroBytesLongMap index; // key/seek relation for used records private kelondroBytesLongMap index; // key/seek relation for used records
private ArrayList<Long[]> free; // list of {size, seek} pairs denoting space and position of free records private ArrayList<gap> free; // list of {size, seek} pairs denoting space and position of free records
private File heapFile; // the file of the heap private File heapFile; // the file of the heap
private kelondroByteOrder ordering; // the ordering on keys private kelondroByteOrder ordering; // the ordering on keys
private RandomAccessFile file; // a random access to the file private RandomAccessFile file; // a random access to the file
public class gap {
public long seek;
public int size;
public gap(long seek, int size) {
this.seek = seek;
this.size = size;
}
}
/* /*
* This class implements a BLOB management based on a sequence of records in a random access file * This class implements a BLOB management based on a sequence of records in a random access file
* The data structure is: * The data structure is:
@ -78,7 +87,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
this.heapFile = heapFile; this.heapFile = heapFile;
this.index = new kelondroBytesLongMap(keylength, this.ordering, 0); this.index = new kelondroBytesLongMap(keylength, this.ordering, 0);
this.free = new ArrayList<Long[]>(); this.free = new ArrayList<gap>();
this.file = new RandomAccessFile(heapFile, "rw"); this.file = new RandomAccessFile(heapFile, "rw");
byte[] key = new byte[keylength]; byte[] key = new byte[keylength];
int reclen; int reclen;
@ -93,7 +102,11 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
// read length of the following record without the length of the record size bytes // read length of the following record without the length of the record size bytes
reclen = file.readInt(); reclen = file.readInt();
assert reclen > 0; assert reclen > 0;
if (reclen == 0) break loop; // very bad file inconsistency if (reclen == 0) {
// very bad file inconsistency
this.file.setLength(seek); // delete everything else at the remaining of the file :-(
break loop;
}
// read key // read key
file.readFully(key); file.readFully(key);
@ -106,7 +119,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
// check if this record is empty // check if this record is empty
if (key == null || key[0] == 0) { if (key == null || key[0] == 0) {
// it is an empty record, store to free list // it is an empty record, store to free list
if (reclen > 0) free.add(new Long[]{new Long(seek), new Long(reclen)}); if (reclen > 0) free.add(new gap(seek, reclen));
} else { } else {
// store key and access address of entry in index // store key and access address of entry in index
try { try {
@ -124,6 +137,32 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
seek += 4L + reclen; seek += 4L + reclen;
} }
// try to merge free entries
if (this.free.size() > 1) {
int merged = 0;
gap lastFree, nextFree;
Iterator<gap> i = this.free.iterator();
lastFree = i.next();
while (i.hasNext()) {
nextFree = i.next();
System.out.println("*** DEBUG BLOB: free-seek = " + nextFree.seek + ", size = " + nextFree.size);
// check if they follow directly
if (lastFree.seek + lastFree.size + 4 == nextFree.seek) {
// merge those records
file.seek(lastFree.seek);
lastFree.size = lastFree.size + nextFree.size + 4; // this updates also the free array
file.writeInt(lastFree.size);
file.seek(nextFree.seek);
file.write(0);file.write(0);file.write(0);file.write(0);
i.remove();
merged++;
} else {
lastFree = nextFree;
}
}
serverLog.logInfo("kelondroBLOBHeap", "BLOB " + heapFile.getName() + ": merged " + merged + " free records");
}
// DEBUG // DEBUG
/* /*
Iterator<byte[]> i = index.keys(true, null); Iterator<byte[]> i = index.keys(true, null);
@ -286,14 +325,14 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
long lseek = -1; long lseek = -1;
int lsize = 0; int lsize = 0;
int reclen = b.length + index.row().primaryKeyLength; int reclen = b.length + index.row().primaryKeyLength;
Long[] entry; gap entry;
Iterator<Long[]> i = this.free.iterator(); Iterator<gap> i = this.free.iterator();
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
if (entry[0].longValue() == (long) reclen) { if (entry.size == reclen) {
// we found an entry that has exactly the size that we need! // we found an entry that has exactly the size that we need!
// we use that entry and stop looking for a larger entry // we use that entry and stop looking for a larger entry
file.seek(entry[1].longValue()); file.seek(entry.seek);
int reclenf = file.readInt(); int reclenf = file.readInt();
assert reclenf == reclen; assert reclenf == reclen;
file.write(key); file.write(key);
@ -303,17 +342,17 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
i.remove(); i.remove();
// add the entry to the index // add the entry to the index
this.index.putl(key, entry[1].longValue()); this.index.putl(key, entry.seek);
System.out.println("*** DEBUG BLOB: replaced-fit record at " + entry[1].longValue() + ", reclen=" + reclen + ", key=" + new String(key)); System.out.println("*** DEBUG BLOB: replaced-fit record at " + entry.seek + ", reclen=" + reclen + ", key=" + new String(key));
// finished! // finished!
return; return;
} }
// look for the biggest size // look for the biggest size
if (entry[0].longValue() > lsize) { if (entry.size > lsize) {
lsize = (int) entry[0].longValue(); lseek = entry.seek;
lseek = entry[1].longValue(); lsize = entry.size;
} }
} }
@ -342,7 +381,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
i = this.free.iterator(); i = this.free.iterator();
while (i.hasNext()) { while (i.hasNext()) {
entry = i.next(); entry = i.next();
if (entry[0].longValue() == (long) lsize && entry[1].longValue() == lseek) { if (entry.size == (long) lsize && entry.seek == lseek) {
// remove the entry from the free list // remove the entry from the free list
i.remove(); i.remove();
break; break;
@ -350,7 +389,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
} }
// add a new free entry // add a new free entry
free.add(new Long[]{new Long(newfreereclen), new Long(lseek + 4 + reclen)}); free.add(new gap(lseek + 4 + reclen, newfreereclen));
System.out.println("*** DEBUG BLOB: replaced-split record at " + lseek + ", reclen=" + reclen + ", new reclen=" + newfreereclen + ", key=" + new String(key)); System.out.println("*** DEBUG BLOB: replaced-split record at " + lseek + ", reclen=" + reclen + ", new reclen=" + newfreereclen + ", key=" + new String(key));
@ -380,7 +419,7 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
int len = file.readInt(); int len = file.readInt();
// add entry to free array // add entry to free array
this.free.add(new Long[]{new Long(len), new Long(pos)}); this.free.add(new gap(pos, len));
// fill zeros to the content // fill zeros to the content
while (len-- > 0) file.write(0); while (len-- > 0) file.write(0);
@ -411,4 +450,21 @@ public final class kelondroBLOBHeap implements kelondroBLOB {
return this.index.keys(up, firstKey); return this.index.keys(up, firstKey);
} }
public static void main(String[] args) {
File f = new File("/Users/admin/blobtest.heap");
try {
//f.delete();
kelondroBLOBHeap heap = new kelondroBLOBHeap(f, 12, kelondroNaturalOrder.naturalOrder);
heap.put("aaaaaaaaaaaa".getBytes(), "eins zwei drei".getBytes());
heap.put("aaaaaaaaaaab".getBytes(), "vier fuenf sechs".getBytes());
heap.put("aaaaaaaaaaac".getBytes(), "sieben acht neun".getBytes());
heap.put("aaaaaaaaaaad".getBytes(), "zehn elf zwoelf".getBytes());
heap.remove("aaaaaaaaaaab".getBytes());
heap.remove("aaaaaaaaaaac".getBytes());
heap.close();
} catch (IOException e) {
e.printStackTrace();
}
}
} }

1
source/de/anomic/yacy/yacyPeerActions.java
Unescape View File

@ -267,6 +267,7 @@ public class yacyPeerActions {
} }
public void setUserAgent(String IP, String userAgent) { public void setUserAgent(String IP, String userAgent) {
if (userAgents == null) return; // case can happen during shutdown
userAgents.put(IP, userAgent); userAgents.put(IP, userAgent);
} }

Write Preview
Loading…
Cancel
Save