boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

using less memory on merging and rewriting blobs

Browse Source 
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7317 6c8d7289-2bf4-0310-a012-ef5d649a1542
pull/1/head
sixcooler 15 years ago
parent 94a9be18a4
commit b87bf88ac8
1 changed files with 22 additions and 19 deletions
Show Stats Download Patch File Download Diff File

41
source/net/yacy/kelondro/blob/ArrayStack.java
Unescape View File

@ -940,7 +940,8 @@ public class ArrayStack implements BLOB {
ByteOrder ordering, HeapWriter writer) throws IOException, RowSpaceExceededException { ByteOrder ordering, HeapWriter writer) throws IOException, RowSpaceExceededException {
assert i1.hasNext(); assert i1.hasNext();
assert i2.hasNext(); assert i2.hasNext();
ReferenceContainer<ReferenceType> c1, c2, c1o, c2o; byte[] c1lh, c2lh;
ReferenceContainer<ReferenceType> c1, c2;
c1 = i1.next(); c1 = i1.next();
c2 = i2.next(); c2 = i2.next();
int e; int e;
@ -951,9 +952,9 @@ public class ArrayStack implements BLOB {
if (e < 0) { if (e < 0) {
writer.add(c1.getTermHash(), c1.exportCollection()); writer.add(c1.getTermHash(), c1.exportCollection());
if (i1.hasNext()) { if (i1.hasNext()) {
c1o = c1; c1lh = c1.getTermHash();
c1 = i1.next(); c1 = i1.next();
assert ordering.compare(c1.getTermHash(), c1o.getTermHash()) > 0; assert ordering.compare(c1.getTermHash(), c1lh) > 0;
continue; continue;
} }
c1 = null; c1 = null;
@ -962,9 +963,9 @@ public class ArrayStack implements BLOB {
if (e > 0) { if (e > 0) {
writer.add(c2.getTermHash(), c2.exportCollection()); writer.add(c2.getTermHash(), c2.exportCollection());
if (i2.hasNext()) { if (i2.hasNext()) {
c2o = c2; c2lh = c2.getTermHash();
c2 = i2.next(); c2 = i2.next();
assert ordering.compare(c2.getTermHash(), c2o.getTermHash()) > 0; assert ordering.compare(c2.getTermHash(), c2lh) > 0;
continue; continue;
} }
c2 = null; c2 = null;
@ -972,25 +973,26 @@ public class ArrayStack implements BLOB {
} }
assert e == 0; assert e == 0;
// merge the entries // merge the entries
writer.add(c1.getTermHash(), (c1.merge(c2)).exportCollection()); c1 = c1.merge(c2);
c1o = c1; writer.add(c1.getTermHash(), c1.exportCollection());
c2o = c2; c1lh = c1.getTermHash();
c2lh = c2.getTermHash();
if (i1.hasNext() && i2.hasNext()) { if (i1.hasNext() && i2.hasNext()) {
c1 = i1.next(); c1 = i1.next();
assert ordering.compare(c1.getTermHash(), c1o.getTermHash()) > 0; assert ordering.compare(c1.getTermHash(), c1lh) > 0;
c2 = i2.next(); c2 = i2.next();
assert ordering.compare(c2.getTermHash(), c2o.getTermHash()) > 0; assert ordering.compare(c2.getTermHash(), c2lh) > 0;
continue; continue;
} }
c1 = null; c1 = null;
c2 = null; c2 = null;
if (i1.hasNext()) { if (i1.hasNext()) {
c1 = i1.next(); c1 = i1.next();
assert ordering.compare(c1.getTermHash(), c1o.getTermHash()) > 0; assert ordering.compare(c1.getTermHash(), c1lh) > 0;
} }
if (i2.hasNext()) { if (i2.hasNext()) {
c2 = i2.next(); c2 = i2.next();
assert ordering.compare(c2.getTermHash(), c2o.getTermHash()) > 0; assert ordering.compare(c2.getTermHash(), c2lh) > 0;
} }
break; break;
@ -1002,9 +1004,9 @@ public class ArrayStack implements BLOB {
//System.out.println("FLUSH REMAINING 1: " + c1.getWordHash()); //System.out.println("FLUSH REMAINING 1: " + c1.getWordHash());
writer.add(c1.getTermHash(), c1.exportCollection()); writer.add(c1.getTermHash(), c1.exportCollection());
if (i1.hasNext()) { if (i1.hasNext()) {
c1o = c1; c1lh = c1.getTermHash();
c1 = i1.next(); c1 = i1.next();
assert ordering.compare(c1.getTermHash(), c1o.getTermHash()) > 0; assert ordering.compare(c1.getTermHash(), c1lh) > 0;
} else { } else {
c1 = null; c1 = null;
} }
@ -1013,9 +1015,9 @@ public class ArrayStack implements BLOB {
//System.out.println("FLUSH REMAINING 2: " + c2.getWordHash()); //System.out.println("FLUSH REMAINING 2: " + c2.getWordHash());
writer.add(c2.getTermHash(), c2.exportCollection()); writer.add(c2.getTermHash(), c2.exportCollection());
if (i2.hasNext()) { if (i2.hasNext()) {
c2o = c2; c2lh = c2.getTermHash();
c2 = i2.next(); c2 = i2.next();
assert ordering.compare(c2.getTermHash(), c2o.getTermHash()) > 0; assert ordering.compare(c2.getTermHash(), c2lh) > 0;
} else { } else {
c2 = null; c2 = null;
} }
@ -1027,15 +1029,16 @@ public class ArrayStack implements BLOB {
CloneableIterator<ReferenceContainer<ReferenceType>> i, CloneableIterator<ReferenceContainer<ReferenceType>> i,
ByteOrder ordering, HeapWriter writer) throws IOException, RowSpaceExceededException { ByteOrder ordering, HeapWriter writer) throws IOException, RowSpaceExceededException {
assert i.hasNext(); assert i.hasNext();
ReferenceContainer<ReferenceType> c, co; byte[] clh;
ReferenceContainer<ReferenceType> c;
c = i.next(); c = i.next();
while (true) { while (true) {
assert c != null; assert c != null;
writer.add(c.getTermHash(), c.exportCollection()); writer.add(c.getTermHash(), c.exportCollection());
if (i.hasNext()) { if (i.hasNext()) {
co = c; clh = c.getTermHash();
c = i.next(); c = i.next();
assert ordering.compare(c.getTermHash(), co.getTermHash()) > 0; assert ordering.compare(c.getTermHash(), clh) > 0;
continue; continue;
} }
break; break;

Write Preview
Loading…
Cancel
Save