boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactoring

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent 2ccf1dba71
commit 40c0856489
8 changed files with 27 additions and 30 deletions
Show Stats Download Patch File Download Diff File

12
htroot/IndexControlURLs_p.java
Unescape View File

@ -41,7 +41,7 @@ import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order; import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.RotateIterator; import net.yacy.kelondro.order.RotateIterator;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.index.MetadataRepository; import net.yacy.search.index.Fulltext;
import net.yacy.search.index.Segment; import net.yacy.search.index.Segment;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch; import de.anomic.server.serverSwitch;
@ -69,7 +69,7 @@ public class IndexControlURLs_p {
prop.put("reload", 0); prop.put("reload", 0);
// show export messages // show export messages
final MetadataRepository.Export export = segment.urlMetadata().export(); final Fulltext.Export export = segment.urlMetadata().export();
if ((export != null) && (export.isAlive())) { if ((export != null) && (export.isAlive())) {
// there is currently a running export // there is currently a running export
prop.put("lurlexport", 2); prop.put("lurlexport", 2);
@ -245,7 +245,7 @@ public class IndexControlURLs_p {
final File f = new File(s); final File f = new File(s);
f.getParentFile().mkdirs(); f.getParentFile().mkdirs();
final String filter = post.get("exportfilter", ".*"); final String filter = post.get("exportfilter", ".*");
final MetadataRepository.Export running = segment.urlMetadata().export(f, filter, null, format, dom); final Fulltext.Export running = segment.urlMetadata().export(f, filter, null, format, dom);
prop.put("lurlexport_exportfile", s); prop.put("lurlexport_exportfile", s);
prop.put("lurlexport_urlcount", running.count()); prop.put("lurlexport_urlcount", running.count());
@ -270,14 +270,14 @@ public class IndexControlURLs_p {
if (post.containsKey("statistics")) { if (post.containsKey("statistics")) {
final int count = post.getInt("lines", 100); final int count = post.getInt("lines", 100);
Iterator<MetadataRepository.HostStat> statsiter; Iterator<Fulltext.HostStat> statsiter;
prop.put("statistics_lines", count); prop.put("statistics_lines", count);
int cnt = 0; int cnt = 0;
try { try {
final MetadataRepository metadata = segment.urlMetadata(); final Fulltext metadata = segment.urlMetadata();
statsiter = metadata.statistics(count, metadata.urlSampleScores(metadata.domainSampleCollector())); statsiter = metadata.statistics(count, metadata.urlSampleScores(metadata.domainSampleCollector()));
boolean dark = true; boolean dark = true;
MetadataRepository.HostStat hs; Fulltext.HostStat hs;
while (statsiter.hasNext() && cnt < count) { while (statsiter.hasNext() && cnt < count) {
hs = statsiter.next(); hs = statsiter.next();
prop.put("statisticslines_domains_" + cnt + "_dark", (dark) ? "1" : "0"); prop.put("statisticslines_domains_" + cnt + "_dark", (dark) ? "1" : "0");

6
htroot/YBRFetch_p.java
Unescape View File

@ -9,8 +9,8 @@ import net.yacy.kelondro.rwi.ReferenceContainerCache;
import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.MemoryControl;
import net.yacy.peers.graphics.WebStructureGraph.HostReference; import net.yacy.peers.graphics.WebStructureGraph.HostReference;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.index.MetadataRepository; import net.yacy.search.index.Fulltext;
import net.yacy.search.index.MetadataRepository.HostStat; import net.yacy.search.index.Fulltext.HostStat;
import net.yacy.search.index.Segment; import net.yacy.search.index.Segment;
import net.yacy.search.ranking.BlockRank; import net.yacy.search.ranking.BlockRank;
import de.anomic.server.serverObjects; import de.anomic.server.serverObjects;
@ -42,7 +42,7 @@ public class YBRFetch_p
// use an index segment to find hosts for given host hashes // use an index segment to find hosts for given host hashes
final Segment segment = sb.index; final Segment segment = sb.index;
final MetadataRepository metadata = segment.urlMetadata(); final Fulltext metadata = segment.urlMetadata();
Map<String, HostStat> hostHashResolver; Map<String, HostStat> hostHashResolver;
try { try {
hostHashResolver = metadata.domainHashResolver(metadata.domainSampleCollector()); hostHashResolver = metadata.domainHashResolver(metadata.domainSampleCollector());

6
source/net/yacy/kelondro/data/word/WordReferenceVars.java
Unescape View File

@ -36,7 +36,6 @@ import net.yacy.cora.document.ASCII;
import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.MultiProtocolURI;
import net.yacy.cora.document.UTF8; import net.yacy.cora.document.UTF8;
import net.yacy.kelondro.data.meta.URIMetadata; import net.yacy.kelondro.data.meta.URIMetadata;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row;
import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.index.Row.Entry;
import net.yacy.kelondro.logging.Log; import net.yacy.kelondro.logging.Log;
@ -48,8 +47,6 @@ import net.yacy.kelondro.rwi.Reference;
import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.util.ByteArray; import net.yacy.kelondro.util.ByteArray;
import org.apache.solr.common.SolrDocument;
public class WordReferenceVars extends AbstractReference implements WordReference, Reference, Cloneable, Comparable<WordReferenceVars>, Comparator<WordReferenceVars> { public class WordReferenceVars extends AbstractReference implements WordReference, Reference, Cloneable, Comparable<WordReferenceVars>, Comparator<WordReferenceVars> {
@ -74,8 +71,7 @@ public class WordReferenceVars extends AbstractReference implements WordReferenc
private final Queue<Integer> positions; private final Queue<Integer> positions;
public double termFrequency; public double termFrequency;
public WordReferenceVars(final SolrDocument doc) { public WordReferenceVars(final URIMetadata md) {
URIMetadata md = new URIMetadataNode(doc);
this.language = md.language(); this.language = md.language();
this.flags = md.flags(); this.flags = md.flags();
this.lastModified = md.moddate().getTime(); this.lastModified = md.moddate().getTime();

14
source/net/yacy/search/index/MetadataRepository.java → source/net/yacy/search/index/Fulltext.java
Unescape View File

@ -1,13 +1,9 @@
// indexRepositoryReference.java // Fulltext.java
// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany // (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 2006 as part of 'plasmaCrawlLURL.java' on http://yacy.net // first published 2006 as part of 'plasmaCrawlLURL.java' on http://yacy.net
// //
// This is a part of YaCy, a peer-to-peer based web search engine // This is a part of YaCy, a peer-to-peer based web search engine
// //
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// LICENSE // LICENSE
// //
// This program is free software; you can redistribute it and/or modify // This program is free software; you can redistribute it and/or modify
@ -66,7 +62,7 @@ import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
public final class MetadataRepository implements Iterable<byte[]> { public final class Fulltext implements Iterable<byte[]> {
// class objects // class objects
private final File location; private final File location;
@ -77,7 +73,7 @@ public final class MetadataRepository implements Iterable<byte[]> {
private final MirrorSolrConnector solr; private final MirrorSolrConnector solr;
private final SolrConfiguration solrScheme; private final SolrConfiguration solrScheme;
public MetadataRepository(final File path, final SolrConfiguration solrScheme) { public Fulltext(final File path, final SolrConfiguration solrScheme) {
this.location = path; this.location = path;
this.tablename = null; this.tablename = null;
this.urlIndexFile = null; this.urlIndexFile = null;
@ -348,13 +344,13 @@ public final class MetadataRepository implements Iterable<byte[]> {
public kiter() throws IOException { public kiter() throws IOException {
this.up = true; this.up = true;
this.iter = MetadataRepository.this.urlIndexFile.rows(); this.iter = Fulltext.this.urlIndexFile.rows();
this.error = false; this.error = false;
} }
public kiter(final boolean up, final String firstHash) throws IOException { public kiter(final boolean up, final String firstHash) throws IOException {
this.up = up; this.up = up;
this.iter = MetadataRepository.this.urlIndexFile.rows(up, (firstHash == null) ? null : ASCII.getBytes(firstHash)); this.iter = Fulltext.this.urlIndexFile.rows(up, (firstHash == null) ? null : ASCII.getBytes(firstHash));
this.error = false; this.error = false;
} }

6
source/net/yacy/search/index/Segment.java
Unescape View File

@ -100,7 +100,7 @@ public class Segment {
private final Log log; private final Log log;
private final File segmentPath; private final File segmentPath;
protected final MetadataRepository urlMetadata; protected final Fulltext urlMetadata;
protected IndexCell<WordReference> termIndex; protected IndexCell<WordReference> termIndex;
protected IndexCell<CitationReference> urlCitationIndex; protected IndexCell<CitationReference> urlCitationIndex;
@ -110,7 +110,7 @@ public class Segment {
this.segmentPath = segmentPath; this.segmentPath = segmentPath;
// create LURL-db // create LURL-db
this.urlMetadata = new MetadataRepository(segmentPath, solrScheme); this.urlMetadata = new Fulltext(segmentPath, solrScheme);
} }
public boolean connectedRWI() { public boolean connectedRWI() {
@ -165,7 +165,7 @@ public class Segment {
this.urlMetadata.connectUrlDb(UrlDbName, useTailCache, exceed134217727); this.urlMetadata.connectUrlDb(UrlDbName, useTailCache, exceed134217727);
} }
public MetadataRepository urlMetadata() { public Fulltext urlMetadata() {
return this.urlMetadata; return this.urlMetadata;
} }

7
source/net/yacy/search/query/RWIProcess.java
Unescape View File

@ -59,6 +59,7 @@ import net.yacy.document.Condenser;
import net.yacy.document.LibraryProvider; import net.yacy.document.LibraryProvider;
import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.data.meta.URIMetadata; import net.yacy.kelondro.data.meta.URIMetadata;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.meta.URIMetadataRow;
import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.data.word.WordReference;
@ -210,7 +211,11 @@ public final class RWIProcess extends Thread
ReferenceContainer<WordReference> wr = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, null); ReferenceContainer<WordReference> wr = ReferenceContainer.emptyContainer(Segment.wordReferenceFactory, null);
SolrDocumentList sdl = RWIProcess.this.query.getSegment().urlMetadata().getSolr().query(solrQuery, 0, 20); SolrDocumentList sdl = RWIProcess.this.query.getSegment().urlMetadata().getSolr().query(solrQuery, 0, 20);
for (SolrDocument d : sdl) { for (SolrDocument d : sdl) {
try {wr.add(new WordReferenceVars(d));} catch (SpaceExceededException e) {} try {
URIMetadataNode md = new URIMetadataNode(d);
WordReferenceVars v = new WordReferenceVars(md);
wr.add(v);
} catch (SpaceExceededException e) {}
} }
Log.logInfo("SearchEvent", "added " + wr.size() + " hits from solr to ranking process"); Log.logInfo("SearchEvent", "added " + wr.size() + " hits from solr to ranking process");
RWIProcess.this.add(wr, true, "embedded solr", sdl.size(), 60000); RWIProcess.this.add(wr, true, "embedded solr", sdl.size(), 60000);

4
source/net/yacy/search/query/SnippetProcess.java
Unescape View File

@ -54,7 +54,7 @@ import net.yacy.peers.graphics.ProfilingGraph;
import net.yacy.repository.LoaderDispatcher; import net.yacy.repository.LoaderDispatcher;
import net.yacy.search.EventTracker; import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard; import net.yacy.search.Switchboard;
import net.yacy.search.index.MetadataRepository; import net.yacy.search.index.Fulltext;
import net.yacy.search.index.Segment; import net.yacy.search.index.Segment;
import net.yacy.search.snippet.MediaSnippet; import net.yacy.search.snippet.MediaSnippet;
import net.yacy.search.snippet.ResultEntry; import net.yacy.search.snippet.ResultEntry;
@ -440,7 +440,7 @@ public class SnippetProcess {
private final CacheStrategy cacheStrategy; private final CacheStrategy cacheStrategy;
private final int neededResults; private final int neededResults;
private boolean shallrun; private boolean shallrun;
private final MetadataRepository metadata; private final Fulltext metadata;
public Worker(final long maxlifetime, final CacheStrategy cacheStrategy, final int neededResults) { public Worker(final long maxlifetime, final CacheStrategy cacheStrategy, final int neededResults) {
this.cacheStrategy = cacheStrategy; this.cacheStrategy = cacheStrategy;

2
source/net/yacy/search/ranking/BlockRank.java
Unescape View File

@ -49,7 +49,7 @@ import net.yacy.peers.Seed;
import net.yacy.peers.SeedDB; import net.yacy.peers.SeedDB;
import net.yacy.peers.graphics.WebStructureGraph; import net.yacy.peers.graphics.WebStructureGraph;
import net.yacy.peers.graphics.WebStructureGraph.HostReference; import net.yacy.peers.graphics.WebStructureGraph.HostReference;
import net.yacy.search.index.MetadataRepository.HostStat; import net.yacy.search.index.Fulltext.HostStat;
import net.yacy.search.index.Segment; import net.yacy.search.index.Segment;

Write Preview
Loading…
Cancel
Save