boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

We assert that no other metadata storage than solr is used now.

Browse Source 
Therefore a property like solrConnected() must be true all the time.
Removal of this method causes removal of all write operations to the old
metadata index.
pull/1/head
Michael Peter Christen 13 years ago
parent abab291162
commit a4214694df
4 changed files with 63 additions and 135 deletions
Show Stats Download Patch File Download Diff File

2
htroot/IndexControlRWIs_p.java
Unescape View File

@ -159,7 +159,7 @@ public class IndexControlRWIs_p {
if ( post.get("deleteIndex", "").equals("on") ) { if ( post.get("deleteIndex", "").equals("on") ) {
segment.clear(); segment.clear();
} }
if ( post.get("deleteRemoteSolr", "").equals("on") && sb.index.fulltext().connectedSolr()) { if ( post.get("deleteRemoteSolr", "").equals("on")) {
try { try {
sb.index.fulltext().getSolr().clear(); sb.index.fulltext().getSolr().clear();
} catch ( final Exception e ) { } catch ( final Exception e ) {

59
source/net/yacy/search/Switchboard.java
Unescape View File

@ -2541,45 +2541,28 @@ public final class Switchboard extends serverSwitch
// STORE WORD INDEX // STORE WORD INDEX
URIMetadata newEntry = null; URIMetadata newEntry = null;
try { newEntry =
newEntry = this.index.storeDocument(
this.index.storeDocument(
url,
referrerURL,
queueEntry.lastModified(),
new Date(),
queueEntry.size(),
queueEntry.profile(),
queueEntry.getResponseHeader(),
document,
condenser,
searchEvent,
sourceName);
final RSSFeed feed =
EventChannel.channels(queueEntry.initiator() == null
? EventChannel.PROXY
: Base64Order.enhancedCoder.equal(
queueEntry.initiator(),
ASCII.getBytes(this.peers.mySeed().hash))
? EventChannel.LOCALINDEXING
: EventChannel.REMOTEINDEXING);
feed.addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url(), ASCII.String(queueEntry.url().hash())));
} catch ( final IOException e ) {
//if (this.log.isFine()) log.logFine("Not Indexed Resource '" + queueEntry.url().toNormalform(false, true) + "': process case=" + processCase);
addURLtoErrorDB(
url, url,
(referrerURL == null) ? null : referrerURL.hash(), referrerURL,
queueEntry.initiator(), queueEntry.lastModified(),
dc_title, new Date(),
FailCategory.FINAL_LOAD_CONTEXT, queueEntry.size(),
"error storing url: " queueEntry.profile(),
+ url.toNormalform(false, true) queueEntry.getResponseHeader(),
+ "': process case=" document,
+ processCase condenser,
+ ", error = " searchEvent,
+ e.getMessage()); sourceName);
return; final RSSFeed feed =
} EventChannel.channels(queueEntry.initiator() == null
? EventChannel.PROXY
: Base64Order.enhancedCoder.equal(
queueEntry.initiator(),
ASCII.getBytes(this.peers.mySeed().hash))
? EventChannel.LOCALINDEXING
: EventChannel.REMOTEINDEXING);
feed.addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url(), ASCII.String(queueEntry.url().hash())));
// store rss feeds in document into rss table // store rss feeds in document into rss table
for ( final Map.Entry<MultiProtocolURI, String> rssEntry : document.getRSS().entrySet() ) { for ( final Map.Entry<MultiProtocolURI, String> rssEntry : document.getRSS().entrySet() ) {

121
source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -47,7 +47,6 @@ import net.yacy.cora.sorting.ScoreMap;
import net.yacy.cora.storage.HandleSet; import net.yacy.cora.storage.HandleSet;
import net.yacy.cora.storage.ZIPReader; import net.yacy.cora.storage.ZIPReader;
import net.yacy.cora.storage.ZIPWriter; import net.yacy.cora.storage.ZIPWriter;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.document.parser.html.CharacterCoding; import net.yacy.document.parser.html.CharacterCoding;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadata; import net.yacy.kelondro.data.meta.URIMetadata;
@ -110,10 +109,6 @@ public final class Fulltext implements Iterable<byte[]> {
return this.solrScheme; return this.solrScheme;
} }
public boolean connectedSolr() {
return this.solr.isConnected0() || this.solr.isConnected1();
}
public boolean connectedLocalSolr() { public boolean connectedLocalSolr() {
return this.solr.isConnected0(); return this.solr.isConnected0();
} }
@ -227,16 +222,12 @@ public final class Fulltext implements Iterable<byte[]> {
// get the metadata from the old metadata index // get the metadata from the old metadata index
if (this.urlIndexFile != null) try { if (this.urlIndexFile != null) try {
if (this.connectedSolr()) { // slow migration to solr
// slow migration to solr final Row.Entry entry = this.urlIndexFile.remove(urlHash);
final Row.Entry entry = this.urlIndexFile.remove(urlHash); if (entry == null) return null;
if (entry == null) return null; URIMetadataRow row = new URIMetadataRow(entry, wre, weight);
URIMetadataRow row = new URIMetadataRow(entry, wre, weight); this.putDocument(this.solrScheme.metadata2solr(row));
this.putDocument(this.solrScheme.metadata2solr(row)); return row;
return row;
}
final Row.Entry entry = this.urlIndexFile.get(urlHash, false);
if (entry != null) return new URIMetadataRow(entry, wre, weight);
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} }
@ -269,17 +260,12 @@ public final class Fulltext implements Iterable<byte[]> {
// get the document from the old metadata index // get the document from the old metadata index
if (this.urlIndexFile != null) try { if (this.urlIndexFile != null) try {
if (this.connectedSolr()) { // slow migration to solr
// slow migration to solr final Row.Entry entry = this.urlIndexFile.remove(urlHash);
final Row.Entry entry = this.urlIndexFile.remove(urlHash);
if (entry == null) return null;
URIMetadataRow row = new URIMetadataRow(entry, wre, weight);
this.putDocument(this.solrScheme.metadata2solr(row));
return ClientUtils.toSolrDocument(getSolrScheme().metadata2solr(row));
}
final Row.Entry entry = this.urlIndexFile.get(urlHash, false);
if (entry == null) return null; if (entry == null) return null;
return ClientUtils.toSolrDocument(getSolrScheme().metadata2solr(new URIMetadataRow(entry, wre, weight))); URIMetadataRow row = new URIMetadataRow(entry, wre, weight);
this.putDocument(this.solrScheme.metadata2solr(row));
return ClientUtils.toSolrDocument(getSolrScheme().metadata2solr(row));
} catch (final IOException e) { } catch (final IOException e) {
Log.logException(e); Log.logException(e);
} }
@ -290,38 +276,19 @@ public final class Fulltext implements Iterable<byte[]> {
public void putDocument(final SolrInputDocument doc) throws IOException { public void putDocument(final SolrInputDocument doc) throws IOException {
String id = (String) doc.getFieldValue(YaCySchema.id.name()); String id = (String) doc.getFieldValue(YaCySchema.id.name());
byte[] idb = ASCII.getBytes(id); byte[] idb = ASCII.getBytes(id);
if (this.connectedSolr()) { try {
try { if (this.urlIndexFile != null) this.urlIndexFile.remove(idb);
if (this.urlIndexFile != null) this.urlIndexFile.remove(idb); SolrDocument sd = this.solr.get(id);
SolrDocument sd = this.solr.get(id); if (sd == null || this.solrScheme.getDate(sd, YaCySchema.last_modified).before(this.solrScheme.getDate(doc, YaCySchema.last_modified))) {
if (sd == null || this.solrScheme.getDate(sd, YaCySchema.last_modified).before(this.solrScheme.getDate(doc, YaCySchema.last_modified))) { if (this.solrScheme.contains(YaCySchema.ip_s)) {
if (this.solrScheme.contains(YaCySchema.ip_s)) { // ip_s needs a dns lookup which causes blockings during search here
// ip_s needs a dns lookup which causes blockings during search here this.solr.add(doc);
this.solr.add(doc); } else synchronized (this.solr) {
} else synchronized (this.solr) { this.solr.add(doc);
this.solr.add(doc); }
}
}
} catch (SolrException e) {
throw new IOException(e.getMessage(), e);
}
} else if (this.urlIndexFile != null) {
URIMetadata oldEntry = null;
try {
final Row.Entry oe = this.urlIndexFile.get(idb, false);
oldEntry = (oe == null) ? null : new URIMetadataRow(oe, null, 0);
} catch (final Throwable e) {
Log.logException(e);
oldEntry = null;
}
URIMetadataNode entry = new URIMetadataNode(ClientUtils.toSolrDocument(doc));
if (oldEntry == null || oldEntry.isOlder(entry)) {
try {
this.urlIndexFile.put(entry.toRow().toRowEntry());
} catch (final SpaceExceededException e) {
throw new IOException("RowSpaceExceededException in " + this.urlIndexFile.filename() + ": " + e.getMessage());
}
} }
} catch (SolrException e) {
throw new IOException(e.getMessage(), e);
} }
this.statsDump = null; this.statsDump = null;
if (MemoryControl.shortStatus()) clearCache(); if (MemoryControl.shortStatus()) clearCache();
@ -336,37 +303,19 @@ public final class Fulltext implements Iterable<byte[]> {
byte[] idb = row.hash(); byte[] idb = row.hash();
String id = ASCII.String(idb); String id = ASCII.String(idb);
if (this.connectedSolr()) { try {
try { if (this.urlIndexFile != null) this.urlIndexFile.remove(idb);
if (this.urlIndexFile != null) this.urlIndexFile.remove(idb); SolrDocument sd = this.solr.get(id);
SolrDocument sd = this.solr.get(id); if (sd == null || (new URIMetadataNode(sd)).isOlder(row)) {
if (sd == null || (new URIMetadataNode(sd)).isOlder(row)) { if (this.solrScheme.contains(YaCySchema.ip_s)) {
if (this.solrScheme.contains(YaCySchema.ip_s)) { // ip_s needs a dns lookup which causes blockings during search here
// ip_s needs a dns lookup which causes blockings during search here this.solr.add(getSolrScheme().metadata2solr(row));
this.solr.add(getSolrScheme().metadata2solr(row)); } else synchronized (this.solr) {
} else synchronized (this.solr) { this.solr.add(getSolrScheme().metadata2solr(row));
this.solr.add(getSolrScheme().metadata2solr(row));
}
} }
} catch (SolrException e) { }
throw new IOException(e.getMessage(), e); } catch (SolrException e) {
} throw new IOException(e.getMessage(), e);
} else if (this.urlIndexFile != null) {
URIMetadata oldEntry = null;
try {
final Row.Entry oe = this.urlIndexFile.get(idb, false);
oldEntry = (oe == null) ? null : new URIMetadataRow(oe, null, 0);
} catch (final Throwable e) {
Log.logException(e);
oldEntry = null;
}
if (oldEntry == null || oldEntry.isOlder(row)) {
try {
this.urlIndexFile.put(row.toRowEntry());
} catch (final SpaceExceededException e) {
throw new IOException("RowSpaceExceededException in " + this.urlIndexFile.filename() + ": " + e.getMessage());
}
}
} }
this.statsDump = null; this.statsDump = null;
if (MemoryControl.shortStatus()) clearCache(); if (MemoryControl.shortStatus()) clearCache();

16
source/net/yacy/search/index/Segment.java
Unescape View File

@ -352,7 +352,7 @@ public class Segment {
final Condenser condenser, final Condenser condenser,
final SearchEvent searchEvent, final SearchEvent searchEvent,
final String sourceName final String sourceName
) throws IOException { ) {
final long startTime = System.currentTimeMillis(); final long startTime = System.currentTimeMillis();
// CREATE INDEX // CREATE INDEX
@ -394,16 +394,12 @@ public class Segment {
); );
// STORE TO SOLR // STORE TO SOLR
// we do not store the data in metadatadb any more if a solr is connected try {
if (this.fulltext.connectedSolr()) { this.fulltext.putDocument(this.fulltext.getSolrScheme().yacy2solr(id, profile, responseHeader, document, metadata));
try { } catch ( final IOException e ) {
this.fulltext.putDocument(this.fulltext.getSolrScheme().yacy2solr(id, profile, responseHeader, document, metadata)); Log.logWarning("SOLR", "failed to send " + urlNormalform + " to solr: " + e.getMessage());
} catch ( final IOException e ) {
Log.logWarning("SOLR", "failed to send " + urlNormalform + " to solr: " + e.getMessage());
}
} else {
this.fulltext.putMetadata(metadata);
} }
final long storageEndTime = System.currentTimeMillis(); final long storageEndTime = System.currentTimeMillis();
// STORE PAGE INDEX INTO WORD INDEX DB // STORE PAGE INDEX INTO WORD INDEX DB

Write Preview
Loading…
Cancel
Save