boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring (change Metadata name of load time data structure to avoid

Browse Source 
confusion with Node data which is also called metadata)
pull/1/head
Michael Peter Christen 11 years ago
parent c95ba52cf0
commit 4eec1a7452
9 changed files with 35 additions and 35 deletions
Show Stats Download Patch File Download Diff File

8
source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java
Unescape View File

@ -85,7 +85,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
}
protected final static int pagesize = 100;
protected static Metadata getMetadata(final Object doc) {
protected static LoadTimeURL getLoadTimeURL(final Object doc) {
if (doc == null) return null;
Object d = null;
String url = null;
@ -110,7 +110,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
long date = -1;
if (d instanceof Long) date = ((Long) d).longValue();
if (d instanceof Date) date = ((Date) d).getTime();
return new Metadata(url, date);
return new LoadTimeURL(url, date);
}
/**
@ -306,7 +306,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
* @throws IOException
*/
@Override
public Metadata getMetadata(String id) throws IOException {
public LoadTimeURL getLoadTimeURL(String id) throws IOException {
// construct raw query
final SolrQuery params = new SolrQuery();
//params.setQuery(CollectionSchema.id.getSolrFieldName() + ":\"" + id + "\"");
@ -326,7 +326,7 @@ public abstract class AbstractSolrConnector implements SolrConnector {
final SolrDocumentList sdl = getDocumentListByParams(params);
if (sdl == null || sdl.getNumFound() <= 0) return null;
SolrDocument doc = sdl.iterator().next();
Metadata md = getMetadata(doc);
LoadTimeURL md = getLoadTimeURL(doc);
return md;
}

20
source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java
Unescape View File

@ -72,7 +72,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
}
private SolrConnector connector;
private ARC<String, Metadata> metadataCache;
private ARC<String, LoadTimeURL> metadataCache;
private final ARH<String> missCache;
private final LinkedHashMap<String, SolrInputDocument> docBuffer;
private CommitHandler processHandler;
@ -119,7 +119,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
}
// move documents to metadata cache
for (Map.Entry<String, SolrInputDocument> entry: this.docBuffer.entrySet()) {
updateCache(entry.getKey(), AbstractSolrConnector.getMetadata(entry.getValue()));
updateCache(entry.getKey(), AbstractSolrConnector.getLoadTimeURL(entry.getValue()));
}
this.docBuffer.clear();
}
@ -137,7 +137,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
this.missCache.clear();
}
private void updateCache(final String id, final Metadata md) {
private void updateCache(final String id, final LoadTimeURL md) {
if (id == null) return;
if (MemoryControl.shortStatus()) {
this.metadataCache.clear();
@ -243,9 +243,9 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
}
@Override
public Metadata getMetadata(String id) throws IOException {
public LoadTimeURL getLoadTimeURL(String id) throws IOException {
if (this.missCache.contains(id)) return null;
Metadata md = this.metadataCache.get(id);
LoadTimeURL md = this.metadataCache.get(id);
if (md != null) {
//System.out.println("*** metadata cache hit; metadataCache.size() = " + metadataCache.size());
//Thread.dumpStack();
@ -255,9 +255,9 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
if (doc != null) {
//System.out.println("*** docBuffer cache hit; docBuffer.size() = " + docBuffer.size());
//Thread.dumpStack();
return AbstractSolrConnector.getMetadata(doc);
return AbstractSolrConnector.getLoadTimeURL(doc);
}
md = this.connector.getMetadata(id);
md = this.connector.getLoadTimeURL(id);
if (md == null) {this.missCache.add(id); return null;}
updateCache(id, md);
return md;
@ -272,7 +272,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
synchronized (this.docBuffer) {this.docBuffer.put(id, solrdoc);}
} else {
this.connector.add(solrdoc);
updateCache(id, AbstractSolrConnector.getMetadata(solrdoc));
updateCache(id, AbstractSolrConnector.getLoadTimeURL(solrdoc));
}
if (MemoryControl.shortStatus() || this.docBuffer.size() > this.updateCapacity) {
commitDocBuffer();
@ -312,7 +312,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
this.missCache.add(id);
this.metadataCache.remove(id);
} else {
updateCache(id, AbstractSolrConnector.getMetadata(solrdoc));
updateCache(id, AbstractSolrConnector.getLoadTimeURL(solrdoc));
}
return solrdoc;
}
@ -329,7 +329,7 @@ public class ConcurrentUpdateSolrConnector implements SolrConnector {
SolrDocumentList sdl = this.connector.getDocumentListByParams(params);
for (SolrDocument doc: sdl) {
String id = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName());
updateCache(id, AbstractSolrConnector.getMetadata(doc));
updateCache(id, AbstractSolrConnector.getLoadTimeURL(doc));
}
return sdl;
}

4
source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
Unescape View File

@ -405,7 +405,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
* @throws IOException
*/
@Override
public Metadata getMetadata(String id) {
public LoadTimeURL getLoadTimeURL(String id) {
int responseCount = 0;
DocListSearcher docListSearcher = null;
try {
@ -417,7 +417,7 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
//for (int i = 0; i < responseCount; i++) {
Document doc = searcher.doc(iterator.nextDoc(), AbstractSolrConnector.SOLR_ID_and_LOAD_DATE_FIELDS);
if (doc == null) return null;
return AbstractSolrConnector.getMetadata(doc);
return AbstractSolrConnector.getLoadTimeURL(doc);
//}
} catch (Throwable e) {
ConcurrentLog.logException(e);

12
source/net/yacy/cora/federate/solr/connector/MirrorSolrConnector.java
Unescape View File

@ -413,17 +413,17 @@ public class MirrorSolrConnector extends AbstractSolrConnector implements SolrCo
}
@Override
public Metadata getMetadata(String id) throws IOException {
if (this.solr0 != null && this.solr1 == null) return this.solr0.getMetadata(id);
if (this.solr0 == null && this.solr1 != null) return this.solr1.getMetadata(id);
public LoadTimeURL getLoadTimeURL(String id) throws IOException {
if (this.solr0 != null && this.solr1 == null) return this.solr0.getLoadTimeURL(id);
if (this.solr0 == null && this.solr1 != null) return this.solr1.getLoadTimeURL(id);
if (this.solr0 == null && this.solr1 == null) return null;
Metadata md0 = this.solr0.getMetadata(id);
Metadata md1 = this.solr1.getMetadata(id);
LoadTimeURL md0 = this.solr0.getLoadTimeURL(id);
LoadTimeURL md1 = this.solr1.getLoadTimeURL(id);
if (md0 == null) return md1;
if (md1 == null) return md0;
long date = Math.max(md0.date, md1.date);
assert md0.url.equals(md1.url);
return new Metadata(md0.url, date);
return new LoadTimeURL(md0.url, date);
}
@Override

8
source/net/yacy/cora/federate/solr/connector/SolrConnector.java
Unescape View File

@ -36,10 +36,10 @@ import org.apache.solr.common.params.ModifiableSolrParams;
public interface SolrConnector extends Iterable<String> /* Iterable of document IDs */ {
public static class Metadata {
public static class LoadTimeURL {
public long date;
public String url;
public Metadata(final String url, final long date) {
public LoadTimeURL(final String url, final long date) {
this.url = url;
this.date = date;
}
@ -119,10 +119,10 @@ public interface SolrConnector extends Iterable<String> /* Iterable of document
/**
* check if a given document, identified by url hash as document id exists
* @param id the url hash and document id
* @return the metadata (url and load data) if any entry in solr exists, null otherwise
* @return the load time metadata (url and load data) if any entry in solr exists, null otherwise
* @throws IOException
*/
public Metadata getMetadata(final String id) throws IOException;
public LoadTimeURL getLoadTimeURL(final String id) throws IOException;
/**
* add a solr input document

6
source/net/yacy/crawler/CrawlStacker.java
Unescape View File

@ -43,7 +43,7 @@ import net.yacy.cora.document.id.AnchorURL;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.solr.FailCategory;
import net.yacy.cora.federate.solr.connector.SolrConnector.Metadata;
import net.yacy.cora.federate.solr.connector.SolrConnector.LoadTimeURL;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.ftp.FTPClient;
@ -384,9 +384,9 @@ public final class CrawlStacker {
return "double in: " + dbocc.name();
}
String urlhash = ASCII.String(url.hash());
Metadata oldEntry = null;
LoadTimeURL oldEntry = null;
try {
oldEntry = this.indexSegment.fulltext().getDefaultConnector().getMetadata(urlhash);
oldEntry = this.indexSegment.fulltext().getDefaultConnector().getLoadTimeURL(urlhash);
} catch (IOException e) {
ConcurrentLog.logException(e);
}

4
source/net/yacy/search/Switchboard.java
Unescape View File

@ -99,7 +99,7 @@ import net.yacy.cora.federate.solr.FailCategory;
import net.yacy.cora.federate.solr.Ranking;
import net.yacy.cora.federate.solr.SchemaConfiguration;
import net.yacy.cora.federate.solr.connector.ShardSelection;
import net.yacy.cora.federate.solr.connector.SolrConnector.Metadata;
import net.yacy.cora.federate.solr.connector.SolrConnector.LoadTimeURL;
import net.yacy.cora.federate.solr.instance.RemoteInstance;
import net.yacy.cora.federate.yacy.CacheStrategy;
import net.yacy.cora.order.Base64Order;
@ -1625,7 +1625,7 @@ public final class Switchboard extends serverSwitch {
HarvestProcess hp = this.crawlQueues.exists(ASCII.getBytes(hash));
if (hp != null) return hp;
try {
Metadata md = this.index.fulltext().getDefaultConnector().getMetadata(hash);
LoadTimeURL md = this.index.fulltext().getDefaultConnector().getLoadTimeURL(hash);
if (md == null) return null;
return HarvestProcess.LOADED; // todo: can also be in error
} catch (IOException e) {

4
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -477,7 +477,7 @@ public final class Fulltext {
if (urlHash == null || this.getDefaultConnector() == null) return null;
try {
SolrConnector.Metadata md = this.getDefaultConnector().getMetadata(urlHash);
SolrConnector.LoadTimeURL md = this.getDefaultConnector().getLoadTimeURL(urlHash);
if (md == null) return null;
return new DigestURL(md.url, ASCII.getBytes(urlHash));
} catch (final IOException e) {
@ -493,7 +493,7 @@ public final class Fulltext {
public long getLoadTime(final String urlHash) {
if (urlHash == null) return -1l;
try {
SolrConnector.Metadata md = this.getDefaultConnector().getMetadata(urlHash);
SolrConnector.LoadTimeURL md = this.getDefaultConnector().getLoadTimeURL(urlHash);
if (md == null) return -1l;
return md.date;
} catch (final Throwable e) {

4
source/net/yacy/search/schema/CollectionConfiguration.java
Unescape View File

@ -59,7 +59,7 @@ import net.yacy.cora.federate.solr.SchemaConfiguration;
import net.yacy.cora.federate.solr.SchemaDeclaration;
import net.yacy.cora.federate.solr.connector.AbstractSolrConnector;
import net.yacy.cora.federate.solr.connector.SolrConnector;
import net.yacy.cora.federate.solr.connector.SolrConnector.Metadata;
import net.yacy.cora.federate.solr.connector.SolrConnector.LoadTimeURL;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
@ -1583,7 +1583,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri
for (Map.Entry<byte[], CRV> entry: rm.entrySet()) {
if (entry == null || entry.getValue() == null) continue;
try {
Metadata md = connector.getMetadata(ASCII.String(entry.getKey()));
LoadTimeURL md = connector.getLoadTimeURL(ASCII.String(entry.getKey()));
ConcurrentLog.info("CollectionConfiguration", "CR for " + md.url);
ConcurrentLog.info("CollectionConfiguration", ">> " + entry.getValue().toString());
} catch (final IOException e) {

Write Preview
Loading…
Cancel
Save