boxtec
/
yacy_search_server
mirror of https://github.com/yacy/yacy_search_server
4
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

Browse Source
pull/1/head
orbiter 11 years ago
parent 343d2ef49a dd8ea0cdd6
commit dcf46ce8f6
9 changed files with 20 additions and 57 deletions
Show Stats Download Patch File Download Diff File

2
build.xml
Unescape View File

@ -357,6 +357,8 @@
<include name="startYACY_debug.bat"/>
<include name="stopYACY.bat"/>
<include name="getWin32MaxHeap.bat"/>
<include name="installYaCyWindowsService.bat" />
<include name="uninstallYaCyWindowsService.bat" />
</fileset>
</copy>

11
htroot/Blacklist_p.html
Unescape View File

@ -26,11 +26,14 @@
You may also provide your blacklist to other peers by sharing them; in return you may
collect blacklist entries from other peers.
</p>
<h3>Active list: #(disabled)#<em>#[currentBlacklist]#</em>::<span class="error">No blacklist selected</span>#(/disabled)#</h3>
<p>Used Blacklist engine: <span class="settingsValue">#[blacklistEngine]#</span></p>
<h3>Active list: #(disabled)#
#{blackLists}#
&nbsp;[&nbsp;<em>#[name]#</em>&nbsp;]&nbsp;
#{/blackLists}#
::<span class="error">No blacklist selected</span>#(/disabled)#</h3>
<!-- blacklist selection -->
<fieldset class="selectList">
<legend>Select list:</legend>
<legend>Select list to edit:</legend>
#(disabled)#
<form id="selectListName" action="Blacklist_p.html" method="post" enctype="multipart/form-data"><fieldset>
<select name="selectedListName" size="1" onchange="document.getElementById('selectListName').submit();">
@ -63,7 +66,7 @@
<br /><br />
#(disabled)#
<fieldset class="editList">
<legend>Edit this list</legend>
<legend>Edit list <b><em>#[currentBlacklist]#</em></b></legend>
<!-- Blacklist configuration -->
#(edit)#

43
htroot/Blacklist_p.java
Unescape View File

@ -31,11 +31,9 @@
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Arrays;
import java.util.List;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.ConcurrentLog;
@ -64,53 +62,12 @@ public class Blacklist_p {
String blacklistToUse = null;
final serverObjects prop = new serverObjects();
prop.putHTML("blacklistEngine", Blacklist.getEngineInfo());
// do all post operations
if (post != null) {
final String action = post.get("action", "");
if(post.containsKey("testList")) {
prop.put("testlist", "1");
String urlstring = post.get("testurl", "");
if (!urlstring.startsWith("http://") &&
!urlstring.startsWith("https://") &&
!urlstring.startsWith("ftp://") &&
!urlstring.startsWith("smb://") &&
!urlstring.startsWith("file://")) {
urlstring = "http://"+urlstring;
}
DigestURL testurl;
try {
testurl = new DigestURL(urlstring);
} catch (final MalformedURLException e) {
testurl = null;
}
if(testurl != null) {
prop.putHTML("testlist_url",testurl.toString());
if (Switchboard.urlBlacklist.isListed(BlacklistType.CRAWLER, testurl)) {
prop.put("testlist_listedincrawler", "1");
}
if (Switchboard.urlBlacklist.isListed(BlacklistType.DHT, testurl)) {
prop.put("testlist_listedindht", "1");
}
if (Switchboard.urlBlacklist.isListed(BlacklistType.NEWS, testurl)) {
prop.put("testlist_listedinnews", "1");
}
if (Switchboard.urlBlacklist.isListed(BlacklistType.PROXY, testurl)) {
prop.put("testlist_listedinproxy", "1");
}
if (Switchboard.urlBlacklist.isListed(BlacklistType.SEARCH, testurl)) {
prop.put("testlist_listedinsearch", "1");
}
if (Switchboard.urlBlacklist.isListed(BlacklistType.SURFTIPS, testurl)) {
prop.put("testlist_listedinsurftips", "1");
}
} else {
prop.put("testlist_url","not valid");
}
}
if (post.containsKey("selectList")) {
blacklistToUse = post.get("selectedListName");
if (blacklistToUse != null && blacklistToUse.isEmpty()) {

4
htroot/IndexControlRWIs_p.html
Unescape View File

@ -255,8 +255,8 @@
</select>
</dt>
<dd>
<input type="submit" value="Add selected URLs to blacklist" name="blacklisturls class="submitready" style="width:240px;"/><br />
<input type="submit" value="Add selected domains to blacklist" name="blacklistdomains" class="submitready" style="width:240px;"/>
<input type="submit" value="Add selected URLs to blacklist" name="blacklisturls" class="submitready" style="width:240px;"/><br />
<input type="submit" value="Add selected domains to blacklist" name="blacklistdomains" class="submitready" style="width:240px; margin-top: 5px;"/>
</dd>
</dl>
</fieldset>

3
htroot/IndexControlRWIs_p.java
Unescape View File

@ -58,6 +58,7 @@ import net.yacy.kelondro.util.FileUtils;
import net.yacy.peers.DHTSelection;
import net.yacy.peers.Protocol;
import net.yacy.peers.Seed;
import net.yacy.repository.Blacklist;
import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@ -524,7 +525,7 @@ public class IndexControlRWIs_p {
prop.put("genUrlList_urlList", i);
prop.putHTML("genUrlList_keyString", keystring);
prop.put("genUrlList_count", i);
putBlacklists(prop, FileUtils.getDirListing(ListManager.listsPath));
putBlacklists(prop, FileUtils.getDirListing(ListManager.listsPath, Blacklist.BLACKLIST_FILENAME_FILTER));
}
}

6
locales/de.lng
Unescape View File

@ -82,9 +82,9 @@ This function provides an URL filter to the proxy; any blacklisted URL is blocke
from being loaded. You can define several blacklists and activate them separately.==Sie können mehrere Blacklists anlegen und getrennt aktivieren.
You may also provide your blacklist to other peers by sharing them; in return you may==Sie können ebenfalls Ihre Blacklist einem anderen Peer zum Download anbieten.
collect blacklist entries from other peers.==Im Gegenzug können Sie sich selber von anderen Peers Blacklists runterladen.
Active list:==Aktivierte Liste:
Active list:==Aktive Liste(n):
No blacklist selected==Keine Blacklist gewählt
Select list:==Liste auswählen:
Select list to edit:==Liste auswählen:
not shared::shared==nicht freigegeben::freigegeben
"select"=="wählen"
Create new list:==Neue Liste anlegen:
@ -93,7 +93,7 @@ Settings for this list==Einstellungen dieser Liste
"Save"=="Speichern"
Share/don't share this list==Liste freigeben/nicht freigeben
Delete this list==Liste löschen
Edit this list==Diese Liste bearbeiten
Edit list==Bearbeite Liste
These are the domain name/path patterns in==Dies sind die Domainnamen/-pfade in
Blacklist Pattern==Blacklisteintrag
Edit selected pattern\(s\)==Bearbeite gewählten Eintrag

4
source/net/yacy/peers/Protocol.java
Unescape View File

@ -759,7 +759,7 @@ public final class Protocol {
if ( urlEntry.hash().length != 12 ) {
continue; // bad url hash
}
if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry) ) {
if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry.url()) ) {
if ( Network.log.isInfo() ) {
Network.log.info("remote search: filtered blacklisted url " + urlEntry.url() + " from peer " + target.getName());
}
@ -1122,7 +1122,7 @@ public final class Protocol {
}
URIMetadataNode urlEntry = new URIMetadataNode(doc);
if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry) ) {
if ( blacklist.isListed(BlacklistType.SEARCH, urlEntry.url()) ) {
if ( Network.log.isInfo() ) {
if (localsearch) {
Network.log.info("local search (solr): filtered blacklisted url " + urlEntry.url());

2
source/net/yacy/search/query/QueryGoal.java
Unescape View File

@ -345,7 +345,7 @@ public class QueryGoal {
// add filter to prevent that results come from failed urls
q.append(CollectionSchema.httpstatus_i.getSolrFieldName()).append(":200").append(" AND (");
q.append(CollectionSchema.images_urlstub_sxt.getSolrFieldName()).append(":[* TO *] OR ");
q.append(CollectionSchema.url_file_ext_s.getSolrFieldName()).append(":(jpg OR png OR gif) OR");
q.append(CollectionSchema.url_file_ext_s.getSolrFieldName()).append(":(jpg OR png OR gif) OR ");
q.append(CollectionSchema.content_type.getSolrFieldName()).append(":(image/*))");
// parse special requests

2
source/net/yacy/search/query/SearchEvent.java
Unescape View File

@ -1059,7 +1059,7 @@ public final class SearchEvent {
}
// Check for blacklist
if (Switchboard.urlBlacklist.isListed(BlacklistType.SEARCH, page)) {
if (Switchboard.urlBlacklist.isListed(BlacklistType.SEARCH, page.url())) {
if (log.isFine()) log.fine("dropped RWI: url is blacklisted in url blacklist");
if (page.word().local()) this.local_rwi_available.decrementAndGet(); else this.remote_rwi_available.decrementAndGet();
continue;

Write Preview
Loading…
Cancel
Save