less overhead calling exist() with only one hash

pull/1/head
Michael Peter Christen 12 years ago
parent 5a02d650ee
commit e1c1e57877

@ -329,6 +329,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
public Set<String> existsByIds(Set<String> ids) throws IOException { public Set<String> existsByIds(Set<String> ids) throws IOException {
HashSet<String> e = new HashSet<String>(); HashSet<String> e = new HashSet<String>();
if (ids == null || ids.size() == 0) return e; if (ids == null || ids.size() == 0) return e;
if (ids.size() == 1) return existsById(ids.iterator().next()) ? ids : e;
Set<String> idsC = new HashSet<String>(); Set<String> idsC = new HashSet<String>();
for (String id: ids) { for (String id: ids) {
if (this.idCache.has(ASCII.getBytes(id))) {cacheSuccessSign(); e.add(id); continue;} if (this.idCache.has(ASCII.getBytes(id))) {cacheSuccessSign(); e.add(id); continue;}

@ -348,6 +348,8 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
@Override @Override
public Set<String> existsByIds(Set<String> ids) throws IOException { public Set<String> existsByIds(Set<String> ids) throws IOException {
if (ids == null || ids.size() == 0) return new HashSet<String>();
if (ids.size() == 1) return existsById(ids.iterator().next()) ? ids : new HashSet<String>();
if (this.solr0 != null && this.solr1 == null) return this.solr0.existsByIds(ids); if (this.solr0 != null && this.solr1 == null) return this.solr0.existsByIds(ids);
if (this.solr0 == null && this.solr1 != null) return this.solr1.existsByIds(ids); if (this.solr0 == null && this.solr1 != null) return this.solr1.existsByIds(ids);
Set<String> s = new HashSet<String>(); Set<String> s = new HashSet<String>();

@ -25,7 +25,6 @@
package net.yacy.peers; package net.yacy.peers;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
@ -164,7 +163,7 @@ public class Transmission {
final ReferenceContainer<WordReference> c = (remaining >= container.size()) ? container : trimContainer(container, remaining); final ReferenceContainer<WordReference> c = (remaining >= container.size()) ? container : trimContainer(container, remaining);
// iterate through the entries in the container and check if the reference is in the repository // iterate through the entries in the container and check if the reference is in the repository
final List<byte[]> notFoundx = new ArrayList<byte[]>(); final List<byte[]> notFoundx = new ArrayList<byte[]>();
Collection<String> testids = new HashSet<String>(); Set<String> testids = new HashSet<String>();
Iterator<WordReference> i = c.entries(); Iterator<WordReference> i = c.entries();
while (i.hasNext()) { while (i.hasNext()) {
final WordReference e = i.next(); final WordReference e = i.next();

@ -1585,7 +1585,7 @@ public final class Switchboard extends serverSwitch {
* @param ids a collection of url hashes * @param ids a collection of url hashes
* @return a map from the hash id to: if it exists, the name of the database, otherwise null * @return a map from the hash id to: if it exists, the name of the database, otherwise null
*/ */
public Map<String, HarvestProcess> urlExists(final Collection<String> ids) { public Map<String, HarvestProcess> urlExists(final Set<String> ids) {
Set<String> e = this.index.exists(ids); Set<String> e = this.index.exists(ids);
Map<String, HarvestProcess> m = new HashMap<String, HarvestProcess>(); Map<String, HarvestProcess> m = new HashMap<String, HarvestProcess>();
for (String id: ids) { for (String id: ids) {
@ -2905,7 +2905,7 @@ public final class Switchboard extends serverSwitch {
// stacking may fail because of double occurrences of that url. Therefore // stacking may fail because of double occurrences of that url. Therefore
// we must wait here until the url has actually disappeared // we must wait here until the url has actually disappeared
int t = 100; int t = 100;
Collection<String> ids = new ArrayList<String>(1); ids.add(ASCII.String(urlhash)); Set<String> ids = new HashSet<String>(1); ids.add(ASCII.String(urlhash));
while (t-- > 0 && this.index.exists(ids).size() > 0) { while (t-- > 0 && this.index.exists(ids).size() > 0) {
try {Thread.sleep(100);} catch (final InterruptedException e) {} try {Thread.sleep(100);} catch (final InterruptedException e) {}
ConcurrentLog.fine("Switchboard", "STACKURL: waiting for deletion, t=" + t); ConcurrentLog.fine("Switchboard", "STACKURL: waiting for deletion, t=" + t);

@ -617,9 +617,10 @@ public final class Fulltext {
* @param ids * @param ids
* @return a set of ids which exist in the database * @return a set of ids which exist in the database
*/ */
public Set<String> exists(Collection<String> ids) { public Set<String> exists(Set<String> ids) {
HashSet<String> e = new HashSet<String>(); HashSet<String> e = new HashSet<String>();
if (ids == null || ids.size() == 0) return e; if (ids == null || ids.size() == 0) return e;
if (ids.size() == 1) return exists(ids.iterator().next()) ? ids : e;
Set<String> idsC = new HashSet<String>(); Set<String> idsC = new HashSet<String>();
idsC.addAll(ids); idsC.addAll(ids);
if (this.urlIndexFile != null) { if (this.urlIndexFile != null) {

@ -29,7 +29,6 @@ package net.yacy.search.index;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.util.Collection;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
@ -443,7 +442,7 @@ public class Segment {
* @param ids * @param ids
* @return a set of ids which exist in the database * @return a set of ids which exist in the database
*/ */
public Set<String> exists(final Collection<String> ids) { public Set<String> exists(final Set<String> ids) {
return this.fulltext.exists(ids); return this.fulltext.exists(ids);
} }

Loading…
Cancel
Save