boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

added collections to crawl monitor

Browse Source
pull/1/head
Michael Peter Christen 13 years ago
parent ee23fc7a32
commit d8425e6809
9 changed files with 40 additions and 4 deletions
Show Stats Download Patch File Download Diff File

2
htroot/CrawlResults.html
Unescape View File

@ -111,6 +111,7 @@
</div>
</form>
</td>
#(showCollection)#::<td><strong>Collection</strong></td>#(/showCollection)#
#(showInit)#::<td><strong>Initiator</strong></td>#(/showInit)#
#(showExec)#::<td><strong>Executor</strong></td>#(/showExec)#
#(showDate)#::<td><strong>Modified</strong></td>#(/showDate)#
@ -131,6 +132,7 @@
</div>
</form>
</td>
#(showCollection)#::<td>#[collection]#</td>#(/showCollection)#
#(showInit)#::<td>#[initiatorSeed]#</td>#(/showInit)#
#(showExec)#::<td>#[executorSeed]#</td>#(/showExec)#

10
htroot/CrawlResults.java
Unescape View File

@ -26,6 +26,7 @@
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.Locale;
@ -39,6 +40,7 @@ import net.yacy.kelondro.data.meta.URIMetadata;
import net.yacy.kelondro.logging.Log;
import net.yacy.peers.Seed;
import net.yacy.search.Switchboard;
import net.yacy.search.index.YaCySchema;
import de.anomic.crawler.ResultURLs;
import de.anomic.crawler.ResultURLs.EventOrigin;
import de.anomic.crawler.ResultURLs.InitExecEntry;
@ -54,6 +56,7 @@ public class CrawlResults {
final serverObjects prop = new serverObjects();
int lines = 500;
boolean showCollection = sb.index.fulltext().getSolrScheme().isEmpty() || sb.index.fulltext().getSolrScheme().contains(YaCySchema.collection_sxt);
boolean showInit = env.getConfigBool("IndexMonitorInit", false);
boolean showExec = env.getConfigBool("IndexMonitorExec", false);
boolean showDate = env.getConfigBool("IndexMonitorDate", true);
@ -166,6 +169,7 @@ public class CrawlResults {
prop.putHTML("table_feedbackpage", "CrawlResults.html");
prop.put("table_tabletype", tabletype.getCode());
prop.put("table_showCollection", (showCollection) ? "1" : "0");
prop.put("table_showInit", (showInit) ? "1" : "0");
prop.put("table_showExec", (showExec) ? "1" : "0");
prop.put("table_showDate", (showDate) ? "1" : "0");
@ -204,6 +208,12 @@ public class CrawlResults {
prop.put("table_indexed_" + cnt + "_tabletype", tabletype.getCode());
prop.put("table_indexed_" + cnt + "_urlhash", entry.getKey());
if (showCollection) {
prop.put("table_indexed_" + cnt + "_showCollection", "1");
prop.put("table_indexed_" + cnt + "_showCollection_collection", Arrays.toString(urle.collections()));
} else
prop.put("table_indexed_" + cnt + "_showCollection", "0");
if (showInit) {
prop.put("table_indexed_" + cnt + "_showInit", "1");
prop.put("table_indexed_" + cnt + "_showInit_initiatorSeed", (initiatorSeed == null) ? "unknown" : initiatorSeed.getName());

1
htroot/ViewFile.html
Unescape View File

@ -63,6 +63,7 @@
<dt>Audio Links:</dt><dd>#[laudio]#</dd>
<dt>Video Links:</dt><dd>#[lvideo]#</dd>
<dt>App Links:</dt><dd>#[lapp]#</dd>
<dt>Collections:</dt><dd>#[collections]#</dd>
<dt>Triplestore:</dt><dd><pre>#[triples]#</pre></dd>
<dt><label for="viewMode">View as</label>:</dt>
<dd>

3
htroot/ViewFile.java
Unescape View File

@ -29,6 +29,7 @@
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Enumeration;
import java.util.Iterator;
@ -371,6 +372,7 @@ public class ViewFile {
prop.put("error_laudio", "");
prop.put("error_lvideo", "");
prop.put("error_lapp", "");
prop.put("error_collections", "");
} else {
prop.put("error_referrerHash", urlEntry.referrerHash());
prop.put("error_moddate", urlEntry.moddate());
@ -393,6 +395,7 @@ public class ViewFile {
prop.put("error_laudio", urlEntry.laudio());
prop.put("error_lvideo", urlEntry.lvideo());
prop.put("error_lapp", urlEntry.lapp());
prop.put("error_collections", Arrays.toString(urlEntry.collections()));
}
return prop;

2
source/de/anomic/crawler/ResultURLs.java
Unescape View File

@ -223,7 +223,7 @@ public final class ResultURLs {
public static void main(final String[] args) {
try {
final DigestURI url = new DigestURI("http", "www.yacy.net", 80, "/");
final URIMetadata urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0d, 0.0d, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), UTF8.getBytes("de"), 0, 0, 0, 0, 0, 0);
final URIMetadata urlRef = new URIMetadataRow(url, "YaCy Homepage", "", "", "", 0.0d, 0.0d, new Date(), new Date(), new Date(), "", new byte[] {}, 123, 42, '?', new Bitfield(), UTF8.getBytes("de"), 0, 0, 0, 0, 0, 0, new String[0]);
final EventOrigin stackNo = EventOrigin.LOCAL_CRAWLING;
System.out.println("valid test:\n=======");
// add

4
source/net/yacy/kelondro/data/meta/URIMetadata.java
Unescape View File

@ -72,7 +72,7 @@ public interface URIMetadata {
*/
@Override
public String toString();
public String dc_title();
public String dc_creator();
@ -117,6 +117,8 @@ public interface URIMetadata {
public String snippet();
public String[] collections();
public WordReference word();
public boolean isOlder(final URIMetadata other);

6
source/net/yacy/kelondro/data/meta/URIMetadataNode.java
Unescape View File

@ -310,6 +310,12 @@ public class URIMetadataNode implements URIMetadata {
return this.snippet;
}
@Override
public String[] collections() {
ArrayList<String> a = getArrayList(YaCySchema.collection_sxt);
return a.toArray(new String[a.size()]);
}
@Override
public WordReference word() {
return this.word;

13
source/net/yacy/kelondro/data/meta/URIMetadataRow.java
Unescape View File

@ -104,6 +104,7 @@ public class URIMetadataRow implements URIMetadata {
private final Row.Entry entry;
private final String snippet;
private final String[] collections;
private WordReference word; // this is only used if the url is transported via remote search requests
private final long ranking; // during generation of a search result this value is set
private Components comp;
@ -112,6 +113,7 @@ public class URIMetadataRow implements URIMetadata {
// create a dummy entry, good to produce poison objects
this.entry = rowdef.newEntry();
this.snippet = "";
this.collections = new String[0];
this.word = null;
this.ranking = 0;
this.comp = null;
@ -139,7 +141,8 @@ public class URIMetadataRow implements URIMetadata {
final int laudio,
final int limage,
final int lvideo,
final int lapp) {
final int lapp,
final String[] collections) {
// create new entry
this.entry = rowdef.newEntry();
this.entry.setCol(col_hash, url.hash());
@ -162,6 +165,7 @@ public class URIMetadataRow implements URIMetadata {
this.entry.setCol(col_lapp, lapp);
//System.out.println("===DEBUG=== " + load.toString() + ", " + decodeDate(col_load).toString());
this.snippet = "";
this.collections = collections;
this.word = null;
this.ranking = 0;
this.comp = null;
@ -211,6 +215,7 @@ public class URIMetadataRow implements URIMetadata {
this.word = searchedWord;
this.ranking = ranking;
this.comp = null;
this.collections = new String[0];
}
public URIMetadataRow(final Properties prop) throws kelondroException {
@ -277,6 +282,7 @@ public class URIMetadataRow implements URIMetadata {
}
this.ranking = 0;
this.comp = null;
this.collections = new String[0];
}
public static URIMetadataRow importEntry(final String propStr) {
@ -478,6 +484,11 @@ public class URIMetadataRow implements URIMetadata {
return this.snippet;
}
@Override
public String[] collections() {
return this.collections;
}
@Override
public WordReference word() {
return this.word;

3
source/net/yacy/search/index/Segment.java
Unescape View File

@ -389,7 +389,8 @@ public class Segment {
document.getAudiolinks().size(), // laudio
document.getImages().size(), // limage
document.getVideolinks().size(), // lvideo
document.getApplinks().size() // lapp
document.getApplinks().size(), // lapp
profile.collections() // collections
);
// STORE TO SOLR

Write Preview
Loading…
Cancel
Save