boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added ramaining iteration methods for solr in fulltext class

Browse Source
pull/1/head
orbiter 12 years ago
parent acd2dc3575
commit 780f8974e7
3 changed files with 84 additions and 95 deletions
Show Stats Download Patch File Download Diff File

44
htroot/IndexControlURLs_p.java
Unescape View File

@ -198,30 +198,26 @@ public class IndexControlURLs_p {
// generate list
if (post.containsKey("urlhashsimilar")) {
try {
final Iterator<URIMetadata> entryIt = new RotateIterator<URIMetadata>(segment.fulltext().entries(true, urlhash), ASCII.String(Base64Order.zero((urlhash == null ? 0 : urlhash.length()))), segment.termIndex().sizesMax());
final StringBuilder result = new StringBuilder("Sequential List of URL-Hashes:<br />");
URIMetadata entry;
int i = 0, rows = 0, cols = 0;
prop.put("urlhashsimilar", "1");
while (entryIt.hasNext() && i < 256) {
entry = entryIt.next();
if (entry == null) break;
prop.put("urlhashsimilar_rows_"+rows+"_cols_"+cols+"_urlHash", ASCII.String(entry.hash()));
cols++;
if (cols==8) {
prop.put("urlhashsimilar_rows_"+rows+"_cols", cols);
cols = 0;
rows++;
}
i++;
}
prop.put("statistics", 0);
prop.put("urlhashsimilar_rows", rows);
prop.put("result", result.toString());
} catch (final IOException e) {
prop.putHTML("result", "No Entries for URL hash " + urlhash);
}
final Iterator<URIMetadata> entryIt = new RotateIterator<URIMetadata>(segment.fulltext().entries(), ASCII.String(Base64Order.zero((urlhash == null ? 0 : urlhash.length()))), segment.termIndex().sizesMax());
final StringBuilder result = new StringBuilder("Sequential List of URL-Hashes:<br />");
URIMetadata entry;
int i = 0, rows = 0, cols = 0;
prop.put("urlhashsimilar", "1");
while (entryIt.hasNext() && i < 256) {
entry = entryIt.next();
if (entry == null) break;
prop.put("urlhashsimilar_rows_"+rows+"_cols_"+cols+"_urlHash", ASCII.String(entry.hash()));
cols++;
if (cols==8) {
prop.put("urlhashsimilar_rows_"+rows+"_cols", cols);
cols = 0;
rows++;
}
i++;
}
prop.put("statistics", 0);
prop.put("urlhashsimilar_rows", rows);
prop.put("result", result.toString());
prop.put("lurlexport", 0);
prop.put("reload", 0);
}

2
source/net/yacy/cora/order/CloneableIterator.java
Unescape View File

@ -26,7 +26,7 @@ package net.yacy.cora.order;
import java.util.Iterator;
public interface CloneableIterator<E> extends Iterator<E> {
public interface CloneableIterator<E> extends Iterator<E>, Cloneable {
/**
* clone the iterator using a modifier

133
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -52,6 +52,7 @@ import net.yacy.kelondro.index.Cache;
import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.MergeIterator;
import net.yacy.kelondro.table.SplitTable;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.Switchboard;
@ -364,78 +365,66 @@ public final class Fulltext implements Iterable<byte[]> {
@Override
public Iterator<byte[]> iterator() {
try {
return this.urlIndexFile.keys(true, null);
} catch (final IOException e) {
Log.logException(e);
return null;
}
}
public CloneableIterator<URIMetadata> entries() throws IOException {
// enumerates entry elements
return new kiter();
}
public CloneableIterator<URIMetadata> entries(final boolean up, final String firstHash) throws IOException {
// enumerates entry elements
return new kiter(up, firstHash);
}
public class kiter implements CloneableIterator<URIMetadata> {
CloneableIterator<byte[]> a = null;
if (this.urlIndexFile != null) try {a = this.urlIndexFile.keys(true, null);} catch (IOException e) {}
final Iterator<String> idi = this.solr.iterator();
CloneableIterator<byte[]> b = new CloneableIterator<byte[]>() {
@Override
public boolean hasNext() {
return idi.hasNext();
}
@Override
public byte[] next() {
String s = idi.next();
return s == null ? null : ASCII.getBytes(s);
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
@Override
public CloneableIterator<byte[]> clone(Object modifier) {
return this;
}
@Override
public void close() {
}
};
if (a == null) return b;
return new MergeIterator<byte[]>(a, b,
URIMetadataRow.rowdef.objectOrder,
MergeIterator.simpleMerge,
true);
}
public CloneableIterator<URIMetadata> entries() {
// enumerates entry elements
private final CloneableIterator<Row.Entry> iter;
private final boolean error;
boolean up;
public kiter() throws IOException {
this.up = true;
this.iter = Fulltext.this.urlIndexFile.rows();
this.error = false;
}
public kiter(final boolean up, final String firstHash) throws IOException {
this.up = up;
this.iter = Fulltext.this.urlIndexFile.rows(up, (firstHash == null) ? null : ASCII.getBytes(firstHash));
this.error = false;
}
@Override
public kiter clone(final Object secondHash) {
try {
return new kiter(this.up, (String) secondHash);
} catch (final IOException e) {
return null;
final Iterator<byte[]> ids = iterator();
return new CloneableIterator<URIMetadata>() {
@Override
public CloneableIterator<URIMetadata> clone(final Object secondHash) {
return this;
}
}
@Override
public final boolean hasNext() {
if (this.error) return false;
if (this.iter == null) return false;
return this.iter.hasNext();
}
@Override
public final URIMetadata next() {
Row.Entry e = null;
if (this.iter == null) { return null; }
if (this.iter.hasNext()) { e = this.iter.next(); }
if (e == null) { return null; }
return new URIMetadataRow(e, null, 0);
}
@Override
public final void remove() {
this.iter.remove();
}
@Override
public void close() {
this.iter.close();
}
@Override
public final boolean hasNext() {
return ids.hasNext();
}
@Override
public final URIMetadata next() {
byte[] id = ids.next();
if (id == null) return null;
return getMetadata(id);
}
@Override
public final void remove() {
ids.remove();
}
@Override
public void close() {
}
};
}
// export methods
public Export export(final File f, final String filter, final HandleSet set, final int format, final boolean dom) {
if ((this.exportthread != null) && (this.exportthread.isAlive())) {
@ -570,7 +559,7 @@ public final class Fulltext implements Iterable<byte[]> {
final Map<String, URLHashCounter> map = new HashMap<String, URLHashCounter>();
// first collect all domains and calculate statistics about it
synchronized (this) {
final CloneableIterator<byte[]> i = this.urlIndexFile.keys(true, null);
final Iterator<byte[]> i = this.iterator();
String hosthash;
byte[] urlhashb;
URLHashCounter ds;
@ -710,6 +699,10 @@ public final class Fulltext implements Iterable<byte[]> {
public int deleteDomain(final String hosthash) throws IOException {
// first collect all url hashes that belong to the domain
assert hosthash.length() == 6;
// delete in solr
this.solr.deleteByQuery(YaCySchema.host_id_s.name() + ":\"" + hosthash + "\"");
// delete in old metadata structure
final ArrayList<String> l = new ArrayList<String>();
synchronized (this) {
final CloneableIterator<byte[]> i = this.urlIndexFile.keys(true, null);

Write Preview
Loading…
Cancel
Save