the segments had been there to create a tenant-infrastructure but were never be used since that was all much too complex. There will be a replacement using a solr navigation using a segment field in the search index.pull/1/head
Michael Peter Christen
13 years ago
parent
508a81b86c
commit
03280fb161
@ -1,196 +0,0 @@
|
|||||||
// Segments.java
|
|
||||||
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
|
|
||||||
// first published 30.07.2009 on http://yacy.net
|
|
||||||
//
|
|
||||||
// This is a part of YaCy, a peer-to-peer based web search engine
|
|
||||||
//
|
|
||||||
// $LastChangedDate$
|
|
||||||
// $LastChangedRevision$
|
|
||||||
// $LastChangedBy$
|
|
||||||
//
|
|
||||||
// LICENSE
|
|
||||||
//
|
|
||||||
// This program is free software; you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU General Public License as published by
|
|
||||||
// the Free Software Foundation; either version 2 of the License, or
|
|
||||||
// (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU General Public License
|
|
||||||
// along with this program; if not, write to the Free Software
|
|
||||||
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
||||||
|
|
||||||
package net.yacy.search.index;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Iterator;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import net.yacy.kelondro.data.word.WordReference;
|
|
||||||
import net.yacy.kelondro.logging.Log;
|
|
||||||
import net.yacy.kelondro.rwi.IndexCell;
|
|
||||||
|
|
||||||
|
|
||||||
public class Segments implements Iterable<Segment> {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* process enumeration type
|
|
||||||
* defines constants that can be used to assign process-related segment names
|
|
||||||
*/
|
|
||||||
public enum Process {
|
|
||||||
|
|
||||||
RECEIPTS,
|
|
||||||
QUERIES,
|
|
||||||
DHTIN,
|
|
||||||
DHTOUT, // the only segment that is used for reading-only
|
|
||||||
PROXY,
|
|
||||||
LOCALCRAWLING,
|
|
||||||
REMOTECRAWLING,
|
|
||||||
PUBLIC,
|
|
||||||
SURROGATES; // includes the index that can be retrieved by the yacy p2p api
|
|
||||||
|
|
||||||
public String toString() {
|
|
||||||
throw new UnsupportedOperationException("toString not allowed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private final Log log;
|
|
||||||
private final File segmentsPath;
|
|
||||||
private final int entityCacheMaxSize;
|
|
||||||
private final long maxFileSize;
|
|
||||||
private Map<String, Segment> segments;
|
|
||||||
private final HashMap<Process, String> process_assignment;
|
|
||||||
private final boolean useTailCache;
|
|
||||||
private final boolean exceed134217727;
|
|
||||||
|
|
||||||
public Segments(
|
|
||||||
final Log log,
|
|
||||||
final File segmentsPath,
|
|
||||||
final int entityCacheMaxSize,
|
|
||||||
final long maxFileSize,
|
|
||||||
final boolean useTailCache,
|
|
||||||
final boolean exceed134217727) {
|
|
||||||
this.log = log;
|
|
||||||
this.segmentsPath = segmentsPath;
|
|
||||||
this.entityCacheMaxSize = entityCacheMaxSize;
|
|
||||||
this.maxFileSize = maxFileSize;
|
|
||||||
this.useTailCache = useTailCache;
|
|
||||||
this.exceed134217727 = exceed134217727;
|
|
||||||
this.segments = new HashMap<String, Segment>();
|
|
||||||
this.process_assignment = new HashMap<Process, String>();
|
|
||||||
|
|
||||||
// assign default segment names for the processes
|
|
||||||
this.process_assignment.put(Process.RECEIPTS, "default");
|
|
||||||
this.process_assignment.put(Process.QUERIES, "default");
|
|
||||||
this.process_assignment.put(Process.DHTIN, "default");
|
|
||||||
this.process_assignment.put(Process.DHTOUT, "default");
|
|
||||||
this.process_assignment.put(Process.PROXY, "default");
|
|
||||||
this.process_assignment.put(Process.LOCALCRAWLING, "default");
|
|
||||||
this.process_assignment.put(Process.REMOTECRAWLING, "default");
|
|
||||||
this.process_assignment.put(Process.PUBLIC, "default");
|
|
||||||
this.process_assignment.put(Process.SURROGATES, "default");
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setSegment(final Process process, final String segmentName) {
|
|
||||||
this.process_assignment.put(process, segmentName);
|
|
||||||
}
|
|
||||||
|
|
||||||
public String[] segmentNames() {
|
|
||||||
return this.segments.keySet().toArray(new String[this.segments.size()]);
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean segmentExist(final String segmentName) {
|
|
||||||
return this.segments.containsKey(segmentName);
|
|
||||||
}
|
|
||||||
|
|
||||||
public Segment segment(final Process process) {
|
|
||||||
return segment(this.process_assignment.get(process));
|
|
||||||
}
|
|
||||||
|
|
||||||
public Segment segment(final String segmentName) {
|
|
||||||
if (this.segments == null) return null;
|
|
||||||
Segment segment = this.segments.get(segmentName);
|
|
||||||
if (segment == null) {
|
|
||||||
// generate the segment
|
|
||||||
try {
|
|
||||||
segment = new Segment(
|
|
||||||
this.log,
|
|
||||||
new File(this.segmentsPath, segmentName),
|
|
||||||
this.entityCacheMaxSize,
|
|
||||||
this.maxFileSize,
|
|
||||||
this.useTailCache,
|
|
||||||
this.exceed134217727);
|
|
||||||
} catch (final IOException e) {
|
|
||||||
Log.logException(e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
this.segments.put(segmentName, segment);
|
|
||||||
}
|
|
||||||
return segment;
|
|
||||||
}
|
|
||||||
|
|
||||||
public long URLCount() {
|
|
||||||
if (this.segments == null) return 0;
|
|
||||||
long c = 0;
|
|
||||||
for (final Segment s: this.segments.values()) c += s.urlMetadata().size();
|
|
||||||
return c;
|
|
||||||
}
|
|
||||||
|
|
||||||
public long RWICount() {
|
|
||||||
if (this.segments == null) return 0;
|
|
||||||
long c = 0;
|
|
||||||
for (final Segment s: this.segments.values()) c += s.termIndex().sizesMax();
|
|
||||||
return c;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int RWIBufferCount() {
|
|
||||||
if (this.segments == null) return 0;
|
|
||||||
int c = 0;
|
|
||||||
for (final Segment s: this.segments.values()) c += s.termIndex().getBufferSize();
|
|
||||||
return c;
|
|
||||||
}
|
|
||||||
|
|
||||||
public MetadataRepository urlMetadata(final Process process) {
|
|
||||||
return segment(this.process_assignment.get(process)).urlMetadata();
|
|
||||||
}
|
|
||||||
|
|
||||||
public IndexCell<WordReference> termIndex(final Process process) {
|
|
||||||
return segment(this.process_assignment.get(process)).termIndex();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void clear(final Process process) {
|
|
||||||
segment(this.process_assignment.get(process)).clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
public File getLocation(final Process process) {
|
|
||||||
return segment(this.process_assignment.get(process)).getLocation();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void close(final Process process) {
|
|
||||||
segment(this.process_assignment.get(process)).close();
|
|
||||||
}
|
|
||||||
|
|
||||||
public synchronized void close() {
|
|
||||||
if (this.segments != null) for (final Segment s: this.segments.values()) s.close();
|
|
||||||
this.segments = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void finalize() {
|
|
||||||
this.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
public synchronized Segment.ReferenceCleaner getReferenceCleaner(final String segmentName, final byte[] startHash) {
|
|
||||||
return segment(segmentName).getReferenceCleaner(startHash);
|
|
||||||
}
|
|
||||||
|
|
||||||
public Iterator<Segment> iterator() {
|
|
||||||
return this.segments.values().iterator();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Loading…
Reference in new issue