boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring

Browse Source
pull/1/head
Michael Peter Christen 12 years ago
parent 2ccf1dba71
commit 40c0856489
8 changed files with 27 additions and 30 deletions
Show Stats Download Patch File Download Diff File

12
htroot/IndexControlURLs_p.java
Unescape View File

@ -41,7 +41,7 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.RotateIterator;
import net.yacy.search.Switchboard;
import net.yacy.search.index.MetadataRepository;
import net.yacy.search.index.Fulltext;
import net.yacy.search.index.Segment;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@ -69,7 +69,7 @@ public class IndexControlURLs_p {
prop.put("reload", 0);
// show export messages
final MetadataRepository.Export export = segment.urlMetadata().export();
final Fulltext.Export export = segment.urlMetadata().export();
if ((export != null) && (export.isAlive())) {
// there is currently a running export
prop.put("lurlexport", 2);
@ -245,7 +245,7 @@ public class IndexControlURLs_p {
final File f = new File(s);
f.getParentFile().mkdirs();
final String filter = post.get("exportfilter", ".*");
final MetadataRepository.Export running = segment.urlMetadata().export(f, filter, null, format, dom);
final Fulltext.Export running = segment.urlMetadata().export(f, filter, null, format, dom);
prop.put("lurlexport_exportfile", s);
prop.put("lurlexport_urlcount", running.count());
@ -270,14 +270,14 @@ public class IndexControlURLs_p {
if (post.containsKey("statistics")) {
final int count = post.getInt("lines", 100);
Iterator<MetadataRepository.HostStat> statsiter;
Iterator<Fulltext.HostStat> statsiter;
prop.put("statistics_lines", count);
int cnt = 0;
try {
final MetadataRepository metadata = segment.urlMetadata();
final Fulltext metadata = segment.urlMetadata();
statsiter = metadata.statistics(count, metadata.urlSampleScores(metadata.domainSampleCollector()));
boolean dark = true;
MetadataRepository.HostStat hs;
Fulltext.HostStat hs;
while (statsiter.hasNext() && cnt < count) {
hs = statsiter.next();
prop.put("statisticslines_domains_" + cnt + "_dark", (dark) ? "1" : "0");

6
htroot/YBRFetch_p.java
Unescape View File

@ -9,8 +9,8 @@ import net.yacy.kelondro.rwi.ReferenceContainerCache;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.peers.graphics.WebStructureGraph.HostReference;
import net.yacy.search.Switchboard;
import net.yacy.search.index.MetadataRepository;
import net.yacy.search.index.MetadataRepository.HostStat;
import net.yacy.search.index.Fulltext;
import net.yacy.search.index.Fulltext.HostStat;
import net.yacy.search.index.Segment;
import net.yacy.search.ranking.BlockRank;
import de.anomic.server.serverObjects;
@ -42,7 +42,7 @@ public class YBRFetch_p
// use an index segment to find hosts for given host hashes
final Segment segment = sb.index;
final MetadataRepository metadata = segment.urlMetadata();
final Fulltext metadata = segment.urlMetadata();
Map<String, HostStat> hostHashResolver;
try {
hostHashResolver = metadata.domainHashResolver(metadata.domainSampleCollector());

6
source/net/yacy/kelondro/data/word/WordReferenceVars.java
Unescape View File

@ -36,7 +36,6 @@ import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.data.meta.URIMetadata;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.Row.Entry;
import net.yacy.kelondro.logging.Log;
@ -48,8 +47,6 @@ import net.yacy.kelondro.rwi.Reference;
import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.util.ByteArray;
import org.apache.solr.common.SolrDocument;
public class WordReferenceVars extends AbstractReference implements WordReference, Reference, Cloneable, Comparable<WordReferenceVars>, Comparator<WordReferenceVars> {
@ -74,8 +71,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
private final Queue<Integer> positions;
public double termFrequency;
public WordReferenceVars(final SolrDocument doc) {
URIMetadata md = new URIMetadataNode(doc);
public WordReferenceVars(final URIMetadata md) {
this.language = md.language();
this.flags = md.flags();
this.lastModified = md.moddate().getTime();

14
source/net/yacy/search/index/MetadataRepository.java → source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -1,13 +1,9 @@
// indexRepositoryReference.java
// Fulltext.java
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 2006 as part of 'plasmaCrawlLURL.java' on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE
//
// This program is free software; you can redistribute it and/or modify
@ -66,7 +62,7 @@ import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
public final class MetadataRepository implements Iterable<byte[]> {
public final class Fulltext implements Iterable<byte[]> {
// class objects
private final File location;
@ -77,7 +73,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
private final MirrorSolrConnector solr;
private final SolrConfiguration solrScheme;
public MetadataRepository(final File path, final SolrConfiguration solrScheme) {
public Fulltext(final File path, final SolrConfiguration solrScheme) {
this.location = path;
this.tablename = null;
this.urlIndexFile = null;
@ -348,13 +344,13 @@ public final class MetadataRepository implements Iterable<byte[]> {
public kiter() throws IOException {
this.up = true;
this.iter = MetadataRepository.this.urlIndexFile.rows();
this.iter = Fulltext.this.urlIndexFile.rows();
this.error = false;
}
public kiter(final boolean up, final String firstHash) throws IOException {
this.up = up;
this.iter = MetadataRepository.this.urlIndexFile.rows(up, (firstHash == null) ? null : ASCII.getBytes(firstHash));
this.iter = Fulltext.this.urlIndexFile.rows(up, (firstHash == null) ? null : ASCII.getBytes(firstHash));
this.error = false;
}

6
source/net/yacy/search/index/Segment.java
Unescape View File

@ -100,7 +100,7 @@ public class Segment {
private final Log log;
private final File segmentPath;
protected final MetadataRepository urlMetadata;
protected final Fulltext urlMetadata;
protected IndexCell<WordReference> termIndex;
protected IndexCell<CitationReference> urlCitationIndex;
@ -110,7 +110,7 @@ public class Segment {
this.segmentPath = segmentPath;
// create LURL-db
this.urlMetadata = new MetadataRepository(segmentPath, solrScheme);
this.urlMetadata = new Fulltext(segmentPath, solrScheme);
}
public boolean connectedRWI() {
@ -165,7 +165,7 @@ public class Segment {
this.urlMetadata.connectUrlDb(UrlDbName, useTailCache, exceed134217727);
}
public MetadataRepository urlMetadata() {
public Fulltext urlMetadata() {
return this.urlMetadata;
}

7
source/net/yacy/search/query/RWIProcess.java
Unescape View File

@ -59,6 +59,7 @@ import net.yacy.document.Condenser;
import net.yacy.document.LibraryProvider;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadata;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.data.word.WordReference;
@ -210,7 +211,11 @@ public final class RWIProcess extends Thread
ReferenceContainer<WordReference> wr = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, null);
SolrDocumentList sdl = RWIProcess.this.query.getSegment().urlMetadata().getSolr().query(solrQuery, 0, 20);
for (SolrDocument d : sdl) {
try {wr.add(new WordReferenceVars(d));} catch (SpaceExceededException e) {}
try {
URIMetadataNode md = new URIMetadataNode(d);
WordReferenceVars v = new WordReferenceVars(md);
wr.add(v);
} catch (SpaceExceededException e) {}
}
Log.logInfo("SearchEvent", "added " + wr.size() + " hits from solr to ranking process");
RWIProcess.this.add(wr, true, "embedded solr", sdl.size(), 60000);

4
source/net/yacy/search/query/SnippetProcess.java
Unescape View File

@ -54,7 +54,7 @@ import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.repository.LoaderDispatcher;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard;
import net.yacy.search.index.MetadataRepository;
import net.yacy.search.index.Fulltext;
import net.yacy.search.index.Segment;
import net.yacy.search.snippet.MediaSnippet;
import net.yacy.search.snippet.ResultEntry;
@ -440,7 +440,7 @@ public class SnippetProcess {
private final CacheStrategy cacheStrategy;
private final int neededResults;
private boolean shallrun;
private final MetadataRepository metadata;
private final Fulltext metadata;
public Worker(final long maxlifetime, final CacheStrategy cacheStrategy, final int neededResults) {
this.cacheStrategy = cacheStrategy;

2
source/net/yacy/search/ranking/BlockRank.java
Unescape View File

@ -49,7 +49,7 @@ import net.yacy.peers.Seed;
import net.yacy.peers.SeedDB;
import net.yacy.peers.graphics.WebStructureGraph;
import net.yacy.peers.graphics.WebStructureGraph.HostReference;
import net.yacy.search.index.MetadataRepository.HostStat;
import net.yacy.search.index.Fulltext.HostStat;
import net.yacy.search.index.Segment;

Write Preview
Loading…
Cancel
Save