boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added ramaining iteration methods for solr in fulltext class

Browse Source
pull/1/head
orbiter 13 years ago
parent acd2dc3575
commit 780f8974e7
3 changed files with 84 additions and 95 deletions
Show Stats Download Patch File Download Diff File

44
htroot/IndexControlURLs_p.java
Unescape View File

@ -198,30 +198,26 @@ public class IndexControlURLs_p {
// generate list // generate list
if (post.containsKey("urlhashsimilar")) { if (post.containsKey("urlhashsimilar")) {
try { final Iterator<URIMetadata> entryIt = new RotateIterator<URIMetadata>(segment.fulltext().entries(), ASCII.String(Base64Order.zero((urlhash == null ? 0 : urlhash.length()))), segment.termIndex().sizesMax());
final Iterator<URIMetadata> entryIt = new RotateIterator<URIMetadata>(segment.fulltext().entries(true, urlhash), ASCII.String(Base64Order.zero((urlhash == null ? 0 : urlhash.length()))), segment.termIndex().sizesMax()); final StringBuilder result = new StringBuilder("Sequential List of URL-Hashes:<br />");
final StringBuilder result = new StringBuilder("Sequential List of URL-Hashes:<br />"); URIMetadata entry;
URIMetadata entry; int i = 0, rows = 0, cols = 0;
int i = 0, rows = 0, cols = 0; prop.put("urlhashsimilar", "1");
prop.put("urlhashsimilar", "1"); while (entryIt.hasNext() && i < 256) {
while (entryIt.hasNext() && i < 256) { entry = entryIt.next();
entry = entryIt.next(); if (entry == null) break;
if (entry == null) break; prop.put("urlhashsimilar_rows_"+rows+"_cols_"+cols+"_urlHash", ASCII.String(entry.hash()));
prop.put("urlhashsimilar_rows_"+rows+"_cols_"+cols+"_urlHash", ASCII.String(entry.hash())); cols++;
cols++; if (cols==8) {
if (cols==8) { prop.put("urlhashsimilar_rows_"+rows+"_cols", cols);
prop.put("urlhashsimilar_rows_"+rows+"_cols", cols); cols = 0;
cols = 0; rows++;
rows++; }
} i++;
i++; }
} prop.put("statistics", 0);
prop.put("statistics", 0); prop.put("urlhashsimilar_rows", rows);
prop.put("urlhashsimilar_rows", rows); prop.put("result", result.toString());
prop.put("result", result.toString());
} catch (final IOException e) {
prop.putHTML("result", "No Entries for URL hash " + urlhash);
}
prop.put("lurlexport", 0); prop.put("lurlexport", 0);
prop.put("reload", 0); prop.put("reload", 0);
} }

2
source/net/yacy/cora/order/CloneableIterator.java
Unescape View File

@ -26,7 +26,7 @@ package net.yacy.cora.order;
import java.util.Iterator; import java.util.Iterator;
public interface CloneableIterator<E> extends Iterator<E> { public interface CloneableIterator<E> extends Iterator<E>, Cloneable {
/** /**
* clone the iterator using a modifier * clone the iterator using a modifier

133
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -52,6 +52,7 @@ import net.yacy.kelondro.index.Cache;
import net.yacy.kelondro.index.Index; import net.yacy.kelondro.index.Index;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.MergeIterator;
import net.yacy.kelondro.table.SplitTable; import net.yacy.kelondro.table.SplitTable;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
@ -364,78 +365,66 @@ public final class Fulltext implements Iterable<byte[]> {
@Override @Override
public Iterator<byte[]> iterator() { public Iterator<byte[]> iterator() {
try { CloneableIterator<byte[]> a = null;
return this.urlIndexFile.keys(true, null); if (this.urlIndexFile != null) try {a = this.urlIndexFile.keys(true, null);} catch (IOException e) {}
} catch (final IOException e) { final Iterator<String> idi = this.solr.iterator();
Log.logException(e); CloneableIterator<byte[]> b = new CloneableIterator<byte[]>() {
return null; @Override
} public boolean hasNext() {
} return idi.hasNext();
}
public CloneableIterator<URIMetadata> entries() throws IOException { @Override
// enumerates entry elements public byte[] next() {
return new kiter(); String s = idi.next();
} return s == null ? null : ASCII.getBytes(s);
}
public CloneableIterator<URIMetadata> entries(final boolean up, final String firstHash) throws IOException { @Override
// enumerates entry elements public void remove() {
return new kiter(up, firstHash); throw new UnsupportedOperationException();
} }
@Override
public class kiter implements CloneableIterator<URIMetadata> { public CloneableIterator<byte[]> clone(Object modifier) {
return this;
}
@Override
public void close() {
}
};
if (a == null) return b;
return new MergeIterator<byte[]>(a, b,
URIMetadataRow.rowdef.objectOrder,
MergeIterator.simpleMerge,
true);
}
public CloneableIterator<URIMetadata> entries() {
// enumerates entry elements // enumerates entry elements
private final CloneableIterator<Row.Entry> iter; final Iterator<byte[]> ids = iterator();
private final boolean error; return new CloneableIterator<URIMetadata>() {
boolean up; @Override
public CloneableIterator<URIMetadata> clone(final Object secondHash) {
public kiter() throws IOException { return this;
this.up = true;
this.iter = Fulltext.this.urlIndexFile.rows();
this.error = false;
}
public kiter(final boolean up, final String firstHash) throws IOException {
this.up = up;
this.iter = Fulltext.this.urlIndexFile.rows(up, (firstHash == null) ? null : ASCII.getBytes(firstHash));
this.error = false;
}
@Override
public kiter clone(final Object secondHash) {
try {
return new kiter(this.up, (String) secondHash);
} catch (final IOException e) {
return null;
} }
} @Override
public final boolean hasNext() {
@Override return ids.hasNext();
public final boolean hasNext() { }
if (this.error) return false; @Override
if (this.iter == null) return false; public final URIMetadata next() {
return this.iter.hasNext(); byte[] id = ids.next();
} if (id == null) return null;
return getMetadata(id);
@Override }
public final URIMetadata next() { @Override
Row.Entry e = null; public final void remove() {
if (this.iter == null) { return null; } ids.remove();
if (this.iter.hasNext()) { e = this.iter.next(); } }
if (e == null) { return null; } @Override
return new URIMetadataRow(e, null, 0); public void close() {
} }
};
@Override
public final void remove() {
this.iter.remove();
}
@Override
public void close() {
this.iter.close();
}
} }
// export methods // export methods
public Export export(final File f, final String filter, final HandleSet set, final int format, final boolean dom) { public Export export(final File f, final String filter, final HandleSet set, final int format, final boolean dom) {
if ((this.exportthread != null) && (this.exportthread.isAlive())) { if ((this.exportthread != null) && (this.exportthread.isAlive())) {
@ -570,7 +559,7 @@ public final class Fulltext implements Iterable<byte[]> {
final Map<String, URLHashCounter> map = new HashMap<String, URLHashCounter>(); final Map<String, URLHashCounter> map = new HashMap<String, URLHashCounter>();
// first collect all domains and calculate statistics about it // first collect all domains and calculate statistics about it
synchronized (this) { synchronized (this) {
final CloneableIterator<byte[]> i = this.urlIndexFile.keys(true, null); final Iterator<byte[]> i = this.iterator();
String hosthash; String hosthash;
byte[] urlhashb; byte[] urlhashb;
URLHashCounter ds; URLHashCounter ds;
@ -710,6 +699,10 @@ public final class Fulltext implements Iterable<byte[]> {
public int deleteDomain(final String hosthash) throws IOException { public int deleteDomain(final String hosthash) throws IOException {
// first collect all url hashes that belong to the domain // first collect all url hashes that belong to the domain
assert hosthash.length() == 6; assert hosthash.length() == 6;
// delete in solr
this.solr.deleteByQuery(YaCySchema.host_id_s.name() + ":\"" + hosthash + "\"");
// delete in old metadata structure
final ArrayList<String> l = new ArrayList<String>(); final ArrayList<String> l = new ArrayList<String>();
synchronized (this) { synchronized (this) {
final CloneableIterator<byte[]> i = this.urlIndexFile.keys(true, null); final CloneableIterator<byte[]> i = this.urlIndexFile.keys(true, null);

Write Preview
Loading…
Cancel
Save