boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of git@gitorious.org:yacy/rc1.git

Browse Source
pull/1/head
orbiter 11 years ago
parent c600d5d144 f98ccf952f
commit 89f76da24b
15 changed files with 96 additions and 90 deletions
Show Stats Download Patch File Download Diff File

4
htroot/api/blacklists/add_entry_p.java
Unescape View File

@ -13,14 +13,14 @@ public class add_entry_p {
private static final String RESULT_FAILURE = "0";
private static final String RESULT_SUCCESS = "1";
private static final String XML_ITEM_STATUS = "status";
private static final String KEY_NEW_ENTRY = "entry";
private static final String KEY_NEW_ENTRY = "item";
private static final String KEY_CURRENT_BLACKLIST = "list";
public static serverObjects respond(final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
final serverObjects prop = new serverObjects();
if (post.containsKey(KEY_CURRENT_BLACKLIST) && post.containsKey(KEY_NEW_ENTRY)) {
if (post!= null && post.containsKey(KEY_CURRENT_BLACKLIST) && post.containsKey(KEY_NEW_ENTRY)) {
final String blacklistToUse = post.get(KEY_CURRENT_BLACKLIST, "").trim();
final String entry = post.get(KEY_NEW_ENTRY, "").trim();

3
htroot/api/blacklists/add_entry_p.json
Unescape View File

@ -0,0 +1,3 @@
{
"status":"#(status)#error::ok#(/status)#"
}

4
htroot/api/blacklists/delete_entry_p.java
Unescape View File

@ -13,14 +13,14 @@ public class delete_entry_p {
private static final String RESULT_FAILURE = "0";
private static final String RESULT_SUCCESS = "1";
private static final String XML_ITEM_STATUS = "status";
private static final String KEY_NEW_ENTRY = "entry";
private static final String KEY_NEW_ENTRY = "item";
private static final String KEY_CURRENT_BLACKLIST = "list";
public static serverObjects respond(final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
final serverObjects prop = new serverObjects();
if (post.containsKey(KEY_CURRENT_BLACKLIST) && post.containsKey(KEY_NEW_ENTRY)) {
if (post!= null && post.containsKey(KEY_CURRENT_BLACKLIST) && post.containsKey(KEY_NEW_ENTRY)) {
final String blacklistToUse = post.get(KEY_CURRENT_BLACKLIST, "").trim();
final String entry = post.get(KEY_NEW_ENTRY, "").trim();

3
htroot/api/blacklists/delete_entry_p.json
Unescape View File

@ -0,0 +1,3 @@
{
"status":"#(status)#error::ok#(/status)#"
}

27
htroot/api/blacklists/get_list_p.java
Unescape View File

@ -13,6 +13,8 @@ import net.yacy.server.serverSwitch;
public class get_list_p {
private static final BlacklistType[] BLACKLIST_TYPE_VALUES = BlacklistType.values();
private static final String KEY_CURRENT_BLACKLIST = "list";
private static final String ITEMS = "items";
private static final String POSTFIX_ITEM = "_item";
@ -23,9 +25,14 @@ public class get_list_p {
private static final String PREFIX_TYPES = "types_";
private static final String POSTFIX_VALUE = "_value";
private static final String POSTFIX_NAME = "_name";
private static final String POSTFIX_COMMA = "_comma";
private static final String TYPES_EXT = ".BlackLists";
private static final String BLACK_LISTS_SHARED = "BlackLists.Shared";
private static final int lastTypeIndex = BLACKLIST_TYPE_VALUES.length - 1;
private static final String EMPTY_STRING = "";
private static final String COMMA_STRING = ",";
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
final serverObjects prop = new serverObjects();
@ -34,7 +41,6 @@ public class get_list_p {
final String blackListName = (post == null) ? "" : post.get(KEY_CURRENT_BLACKLIST, "");
int count;
if (dirlist != null) {
for (final String element : dirlist) {
if (element.equals(blackListName)) {
@ -44,10 +50,17 @@ public class get_list_p {
prop.put(SHARED, ListManager.listSetContains(BLACK_LISTS_SHARED, element));
int j = 0;
for (final BlacklistType type : BlacklistType.values()) {
for (final BlacklistType type : BLACKLIST_TYPE_VALUES) {
prop.putXML(PREFIX_TYPES + j + POSTFIX_NAME, type.toString());
prop.put(PREFIX_TYPES + j + POSTFIX_VALUE,
ListManager.listSetContains(type + TYPES_EXT, element));
if (j < lastTypeIndex) {
prop.put(PREFIX_TYPES + j + POSTFIX_COMMA, COMMA_STRING);
} else {
prop.put(PREFIX_TYPES + j + POSTFIX_COMMA, EMPTY_STRING);
}
j++;
}
prop.put(TYPES, BlacklistType.values().length);
@ -56,12 +69,20 @@ public class get_list_p {
final Collection<String> list = FileUtils.getListArray(new File(ListManager.listsPath, element));
count = 0;
int count = 0;
final int lastItemCount = list.size() - 1;
for (final String entry : list){
if (entry.isEmpty()) continue;
if (entry.charAt(0) == '#') continue;
prop.putXML(PREFIX_ITEMS + count + POSTFIX_ITEM, entry);
if (count < lastItemCount) {
prop.put(PREFIX_ITEMS + count + POSTFIX_COMMA, COMMA_STRING);
} else {
prop.put(PREFIX_ITEMS + count + POSTFIX_COMMA, EMPTY_STRING);
}
count++;
}
prop.put(ITEMS, count);

9
htroot/api/blacklists/get_list_p.json
Unescape View File

@ -0,0 +1,9 @@
{
"name": "#[name]#",
"shared": "#[shared]#",
"types": [#{types}#{"#[name]#": "#[value]#"}#[comma]# #{/types}#]
"items":
[
#{items}#"#[item]#"#[comma]#
#{/items}#]
}

35
htroot/api/blacklists/get_metadata_p.java
Unescape View File

@ -12,43 +12,64 @@ import net.yacy.server.serverSwitch;
public class get_metadata_p {
private static final BlacklistType[] BLACKLIST_TYPE_VALUES = BlacklistType.values();
private static final String LISTS = "lists";
private static final String POSTFIX_SHARED = "_shared";
private static final String POSTFIX_TYPES = "_types";
private static final String POSTFIX_VALUE = "_value";
private static final String POSTFIX_NAME = "_name";
private static final String POSTFIX_COMMA = "_comma";
private static final String TYPES_EXT = ".BlackLists";
private static final String INFIX_TYPES = "_types_";
private static final String PREFIX_LISTS = "lists_";
private static final String BLACK_LISTS_SHARED = "BlackLists.Shared";
private static final int lastTypeIndex = BLACKLIST_TYPE_VALUES.length - 1;
private static final String EMPTY_STRING = "";
private static final String COMMA_STRING = ",";
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
final serverObjects prop = new serverObjects();
final Collection<String> dirlist = FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER);
int blacklistCount=0;
final int lastBlacklistCount = dirlist.size() - 1;
int blacklistCount = 0;
if (dirlist != null) {
for (final String element : dirlist) {
prop.putXML(PREFIX_LISTS + blacklistCount + POSTFIX_NAME, element);
prop.put(PREFIX_LISTS + blacklistCount + POSTFIX_SHARED, ListManager.listSetContains(BLACK_LISTS_SHARED, element));
int j = 0;
for (final BlacklistType type : BlacklistType.values()) {
for (final BlacklistType type : BLACKLIST_TYPE_VALUES) {
prop.putXML(PREFIX_LISTS + blacklistCount + INFIX_TYPES + j + POSTFIX_NAME, type.toString());
prop.put(PREFIX_LISTS + blacklistCount + INFIX_TYPES + j + POSTFIX_VALUE,
ListManager.listSetContains(type + TYPES_EXT, element));
if (j < lastTypeIndex) {
prop.put(PREFIX_LISTS + blacklistCount + INFIX_TYPES + j + POSTFIX_COMMA, COMMA_STRING);
} else {
prop.put(PREFIX_LISTS + blacklistCount + INFIX_TYPES + j + POSTFIX_COMMA, EMPTY_STRING);
}
j++;
}
prop.put(PREFIX_LISTS + blacklistCount + POSTFIX_TYPES, BlacklistType.values().length);
prop.put(PREFIX_LISTS + blacklistCount + POSTFIX_TYPES, BLACKLIST_TYPE_VALUES.length);
if (blacklistCount < lastBlacklistCount) {
prop.put(PREFIX_LISTS + blacklistCount + POSTFIX_COMMA, COMMA_STRING);
} else {
prop.put(PREFIX_LISTS + blacklistCount + POSTFIX_COMMA, EMPTY_STRING);
}
blacklistCount++;
}
}
prop.put(LISTS, blacklistCount);
return prop;
}

8
htroot/api/blacklists/get_metadata_p.json
Unescape View File

@ -0,0 +1,8 @@
[
#{lists}#{
"name": "#[name]#",
"shared": "#[shared]#",
"types": [#{types}#{"#[name]#": "#[value]#"}#[comma]# #{/types}#]
}#[comma]#
#{/lists}#
]

2
htroot/index.html
Unescape View File

@ -53,7 +53,7 @@
<h2 class="yacy"><a href="#[promoteSearchPageGreeting.homepage]#" class="yacylogo"><img src="#[promoteSearchPageGreeting.largeImage]#" alt="yacy" style="margin: auto;"/></a></h2>
<h2 class="yacy">#[promoteSearchPageGreeting]#</h2>
<form class="search" action="yacysearch.html" method="get" id="searchform" accept-charset="UTF-8">
<form class="search form-inline" action="yacysearch.html" method="get" id="searchform" accept-charset="UTF-8">
<fieldset class="maininput">
<div class="input-group">
<input name="query" id="search" type="text" size="40" maxlength="80" value="#[former]#" onFocus="this.select()" class="form-control searchinput typeahead" />

1
source/net/yacy/cora/federate/solr/connector/SolrServerConnector.java
Unescape View File

@ -91,7 +91,6 @@ public abstract class SolrServerConnector extends AbstractSolrConnector implemen
@Override
public void optimize(int maxSegments) {
if (this.server == null) return;
if (getSegmentCount() <= maxSegments) return;
synchronized (this.server) {
try {
//this.server.optimize(true, true, maxSegments);

3
source/net/yacy/crawler/HostBalancer.java
Unescape View File

@ -211,7 +211,8 @@ public class HostBalancer implements Balancer {
if (queue == null) {
queue = new HostQueue(this.hostsPath, entry.url().getHost(), entry.url().getPort(), this.queues.size() > 100, this.exceed134217727);
this.queues.put(hosthash, queue);
robots.ensureExist(entry.url(), profile.getAgent(), true); // concurrently load all robots.txt
// profile might be null when continue crawls after YaCy restart
robots.ensureExist(entry.url(), profile == null ? ClientIdentification.yacyInternetCrawlerAgent : profile.getAgent(), true); // concurrently load all robots.txt
}
return queue.push(entry, profile, robots);
}

38
source/net/yacy/document/importer/OAIPMHImporter.java
Unescape View File

@ -22,13 +22,8 @@
package net.yacy.document.importer;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.text.ParseException;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import net.yacy.cora.date.GenericFormatter;
@ -181,43 +176,12 @@ public class OAIPMHImporter extends Thread implements Importer, Comparable<OAIPM
return 0;
}
/**
* get a map for already loaded oai-pmh servers and their latest access date
* @param surrogatesIn
* @param surrogatesOut
* @return a map where the key is the hostID of the servers and the value is the last access date
*/
@SuppressWarnings("unchecked")
public static Map<String, Date> getLoadedOAIServer(File surrogatesIn, File surrogatesOut) {
Map<String, Date> map = getLoadedOAIServer(surrogatesOut);
map.putAll((Map<? extends String, ? extends Date>) getLoadedOAIServer(surrogatesIn).entrySet());
return map;
}
private static Map<String, Date> getLoadedOAIServer(File surrogates) {
HashMap<String, Date> map = new HashMap<String, Date>();
//oaipmh_opus.bsz-bw.de_20091102113118728.xml
for (String s: surrogates.list()) {
if (s.startsWith(filenamePrefix) && s.endsWith(".xml") && s.charAt(s.length() - 22) == filenameSeparationChar) {
try {
Date fd = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(s.substring(s.length() - 21, s.length() - 4));
String hostID = s.substring(7, s.length() - 22);
Date md = map.get(hostID);
if (md == null || fd.after(md)) map.put(hostID, fd);
} catch (final ParseException e) {
ConcurrentLog.logException(e);
}
}
}
return map;
}
public static final char hostReplacementChar = '_';
public static final char filenameSeparationChar = '.';
public static final String filenamePrefix = "oaipmh";
/**
* compute a host id that is also used in the getLoadedOAIServer method for the map key
* compute a host id
* @param source
* @return a string that is a key for the given host
*/

7
source/net/yacy/kelondro/table/Table.java
Unescape View File

@ -474,12 +474,17 @@ public class Table implements Index, Iterable<Row.Entry> {
@Override
public void close() {
if (this.file != null) this.file.close();
String tablefile = null;
if (this.file != null) {
tablefile = this.file.filename().toString();
this.file.close();
}
this.file = null;
if (this.table != null) this.table.close();
this.table = null;
if (this.index != null) this.index.close();
this.index = null;
if (tablefile != null) tableTracker.remove(tablefile);
}
@Override

38
source/net/yacy/search/Switchboard.java
Unescape View File

@ -37,17 +37,14 @@
package net.yacy.search;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Reader;
import java.net.MalformedURLException;
import java.security.NoSuchAlgorithmException;
@ -78,7 +75,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@ -243,7 +239,7 @@ public final class Switchboard extends serverSwitch {
public File networkRoot;
public File queuesRoot;
public File surrogatesInPath;
public File surrogatesOutPath;
//public File surrogatesOutPath;
public Segment index;
public LoaderDispatcher loader;
public CrawlSwitchboard crawler;
@ -675,13 +671,13 @@ public final class Switchboard extends serverSwitch {
SwitchboardConstants.SURROGATES_IN_PATH_DEFAULT);
this.log.info("surrogates.in Path = " + this.surrogatesInPath.getAbsolutePath());
this.surrogatesInPath.mkdirs();
this.surrogatesOutPath =
/* this.surrogatesOutPath =
getDataPath(
SwitchboardConstants.SURROGATES_OUT_PATH,
SwitchboardConstants.SURROGATES_OUT_PATH_DEFAULT);
this.log.info("surrogates.out Path = " + this.surrogatesOutPath.getAbsolutePath());
this.surrogatesOutPath.mkdirs();
*/
// copy opensearch heuristic config (if not exist)
final File osdConfig = new File(getDataPath(), "DATA/SETTINGS/heuristicopensearch.conf");
if (!osdConfig.exists()) {
@ -1833,8 +1829,6 @@ public final class Switchboard extends serverSwitch {
if ( !infile.exists() || !infile.canWrite() || !infile.canRead() ) {
return false;
}
final File outfile = new File(this.surrogatesOutPath, s);
//if (outfile.exists()) return false;
boolean moved = false;
if ( s.endsWith("xml.zip") ) {
// open the zip file with all the xml files in it
@ -1858,7 +1852,7 @@ public final class Switchboard extends serverSwitch {
} catch (final IOException e ) {
ConcurrentLog.logException(e);
} finally {
moved = infile.renameTo(outfile);
moved = infile.delete();
if (zis != null) try {zis.close();} catch (final IOException e) {}
}
return moved;
@ -1874,29 +1868,7 @@ public final class Switchboard extends serverSwitch {
ConcurrentLog.logException(e);
} finally {
if (!shallTerminate()) {
moved = infile.renameTo(outfile);
if ( moved ) {
// check if this file is already compressed, if not, compress now
if ( !outfile.getName().endsWith(".gz") ) {
final String gzname = outfile.getName() + ".gz";
final File gzfile = new File(outfile.getParentFile(), gzname);
try {
final OutputStream os =
new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(gzfile)));
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(outfile));
FileUtils.copy(bis, os);
os.close();
bis.close();
if ( gzfile.exists() ) {
FileUtils.deletedelete(outfile);
}
} catch (final FileNotFoundException e ) {
ConcurrentLog.logException(e);
} catch (final IOException e ) {
ConcurrentLog.logException(e);
}
}
}
moved = infile.delete();
}
if (is != null) try {is.close();} catch (IOException e) {}
}

4
source/net/yacy/search/SwitchboardConstants.java
Unescape View File

@ -387,8 +387,8 @@ public final class SwitchboardConstants {
public static final String SURROGATES_IN_PATH = "surrogates.in";
public static final String SURROGATES_IN_PATH_DEFAULT = "DATA/SURROGATES/in";
public static final String SURROGATES_OUT_PATH = "surrogates.out";
public static final String SURROGATES_OUT_PATH_DEFAULT = "DATA/SURROGATES/out";
//public static final String SURROGATES_OUT_PATH = "surrogates.out";
//public static final String SURROGATES_OUT_PATH_DEFAULT = "DATA/SURROGATES/out";
public static final String DICTIONARY_SOURCE_PATH = "dictionaries";
public static final String DICTIONARY_SOURCE_PATH_DEFAULT = "DATA/DICTIONARIES";

Write Preview
Loading…
Cancel
Save